Skip to content

Commit a08cf5b

Browse files
ChrisHadChrisHad
authored
Merge pull request #14 from SKA-ScienceDataProcessor/master
Merge main repo
2 parents 869a817 + ecb5c8c commit a08cf5b

7 files changed

Lines changed: 101 additions & 70 deletions

‎Jenkinsfile‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,23 +96,51 @@ pipeline {
9696
post {
9797
always {
9898
echo 'FINISHED'
99+
slackSend baseUrl: 'https://sdp-execution-engine.slack.com/services/hooks/jenkins-ci/',
100+
channel: '#jenkins',
101+
color: 'good',
102+
message: "Pipeline ${currentBuild.fullDisplayName} ${env.JOB_NAME} ${env.BUILD_NUMBER} completed with Status: ${env.BUILD_STATUS} (<${env.BUILD_URL}|Open>)",
103+
tokenCredentialId: 'a06474f9-0c86-4dc7-a477-42d7d1a1cc71'
99104
}
100105
failure {
101106
mail to: 'mf582@mrao.cam.ac.uk, pw410@cam.ac.uk, realtimcornwell@gmail.com',
102-
subject: "Failed Jenkins Pipeline: ${currentBuild.fullDisplayName}",
103-
body: "Something is wrong with ${env.BUILD_URL}"
107+
subject: "Failed Jenkins Pipeline: ${currentBuild.fullDisplayName} Status:${env.BUILD_STATUS} ",
108+
body: "Something is wrong with ${env.BUILD_URL} Status: ${env.BUILD_STATUS} "
109+
104110
}
105111
fixed {
106112
mail to: 'mf582@mrao.cam.ac.uk, pw410@cam.ac.uk, realtimcornwell@gmail.com',
107-
subject: "Jenkins Pipeline is back to normal: ${currentBuild.fullDisplayName}",
113+
subject: "Jenkins Pipeline is back to normal: ${currentBuild.fullDisplayName} Status:${env.BUILD_STATUS} ",
108114
body: "See ${env.BUILD_URL}"
115+
}
116+
success {
117+
sshPublisher alwaysPublishFromMaster: true,
118+
publishers: [sshPublisherDesc(configName: 'vm12',
119+
transfers: [sshTransfer(excludes: '',
120+
execCommand: '', execTimeout: 120000,
121+
flatten: false,
122+
makeEmptyDirs: false,
123+
noDefaultExcludes: false,
124+
patternSeparator: '[, ]+',
125+
remoteDirectory: 'algorithm-reference-library',
126+
remoteDirectorySDF: false,
127+
removePrefix: '',
128+
sourceFiles: 'docs/build/**'),
129+
sshTransfer(excludes: '',
130+
execCommand: '', execTimeout: 120000,
131+
flatten: false,
132+
makeEmptyDirs: false,
133+
noDefaultExcludes: false,
134+
patternSeparator: '[, ]+',
135+
remoteDirectory: 'algorithm-reference-library',
136+
remoteDirectorySDF: false,
137+
removePrefix: '',
138+
sourceFiles: 'coverage/**')],
139+
usePromotionTimestamp: false,
140+
useWorkspaceInPromotion: false,
141+
verbose: false)]
142+
109143
}
110-
// We could send slack notifications but pluggin needs to be installed and configured at server
111-
// success {
112-
// slackSend channel: '#ops-room',
113-
// color: 'good',
114-
// message: "The pipeline ${currentBuild.fullDisplayName} completed successfully."
115-
// }
116144
}
117145
}
118146

‎tests/workflows/test_imaging_arlexecute.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from astropy import units as u
1212
from astropy.coordinates import SkyCoord
1313

14-
from . import ARLExecuteTestCase
14+
from tests.workflows import ARLExecuteTestCase
1515
from data_models.polarisation import PolarisationFrame
1616
from wrappers.arlexecute.execution_support.arlexecute import arlexecute
1717
from wrappers.arlexecute.image.operations import export_image_to_fits, smooth_image

‎tests/workflows/test_imaging_deconvolve_arlexecute.py‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
from astropy.coordinates import SkyCoord
1313

1414
from data_models.polarisation import PolarisationFrame
15-
16-
from . import ARLExecuteTestCase
17-
from workflows.arlexecute.imaging.imaging_arlexecute import invert_list_arlexecute_workflow, deconvolve_list_arlexecute_workflow, \
15+
from tests.workflows import ARLExecuteTestCase
16+
from workflows.arlexecute.imaging.imaging_arlexecute import invert_list_arlexecute_workflow, \
17+
deconvolve_list_arlexecute_workflow, \
1818
residual_list_arlexecute_workflow, restore_list_arlexecute_workflow
1919
from wrappers.arlexecute.execution_support.arlexecute import arlexecute
2020
from wrappers.arlexecute.image.operations import export_image_to_fits, smooth_image
@@ -96,7 +96,8 @@ def actualSetUp(self, add_errors=False, freqwin=7, block=False, dospectral=True,
9696

9797
# Calculate the model convolved with a Gaussian.
9898

99-
model = arlexecute.compute(self.model_imagelist[0], sync=True)
99+
self.model_imagelist = arlexecute.compute(self.model_imagelist, sync=True)
100+
model = self.model_imagelist[0]
100101

101102
self.cmodel = smooth_image(model)
102103
export_image_to_fits(model, '%s/test_imaging_arlexecute_deconvolved_model.fits' % self.dir)
@@ -105,6 +106,8 @@ def actualSetUp(self, add_errors=False, freqwin=7, block=False, dospectral=True,
105106
if add_errors and block:
106107
self.vis_list = [arlexecute.execute(insert_unittest_errors)(self.vis_list[i])
107108
for i, _ in enumerate(self.frequency)]
109+
110+
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
108111

109112
def test_time_setup(self):
110113
self.actualSetUp()
@@ -117,7 +120,8 @@ def test_deconvolve_spectral(self):
117120
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list, self.model_imagelist,
118121
context='2d',
119122
dopsf=True, normalize=True)
120-
deconvolved, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000,
123+
deconvolved, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist,
124+
niter=1000,
121125
fractional_threshold=0.1, scales=[0, 3, 10],
122126
threshold=0.1, gain=0.7)
123127
deconvolved = arlexecute.compute(deconvolved, sync=True)
@@ -131,17 +135,18 @@ def test_deconvolve_and_restore_cube_mmclean(self):
131135
dopsf=False, normalize=True)
132136
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list, self.model_imagelist, context='2d',
133137
dopsf=True, normalize=True)
134-
dec_imagelist, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000,
138+
dec_imagelist, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist,
139+
niter=1000,
135140
fractional_threshold=0.01, scales=[0, 3, 10],
136141
algorithm='mmclean', nmoments=3, nchan=self.freqwin,
137142
threshold=0.1, gain=0.7)
138143
residual_imagelist = residual_list_arlexecute_workflow(self.vis_list, model_imagelist=dec_imagelist,
139144
context='wstack', vis_slices=51)
140-
restored = restore_list_arlexecute_workflow(model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist,
141-
residual_imagelist=residual_imagelist,
142-
empty=self.model_imagelist)[0]
145+
restored_list = restore_list_arlexecute_workflow(model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist,
146+
residual_imagelist=residual_imagelist,
147+
empty=self.model_imagelist)
143148

144-
restored = arlexecute.compute(restored, sync=True)
149+
restored = arlexecute.compute(restored_list, sync=True)[0]
145150

146151
export_image_to_fits(restored, '%s/test_imaging_%s_mmclean_restored.fits' % (self.dir, arlexecute.type()))
147152

@@ -151,17 +156,19 @@ def test_deconvolve_and_restore_cube_mmclean_facets(self):
151156
context='2d', dopsf=False, normalize=True)
152157
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list, self.model_imagelist,
153158
context='2d', dopsf=True, normalize=True)
154-
dec_imagelist, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000,
159+
dec_imagelist, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist,
160+
niter=1000,
155161
fractional_threshold=0.1, scales=[0, 3, 10],
156162
algorithm='mmclean', nmoments=3, nchan=self.freqwin,
157163
threshold=0.01, gain=0.7, deconvolve_facets=8,
158164
deconvolve_overlap=8, deconvolve_taper='tukey')
159-
residual_imagelist = residual_list_arlexecute_workflow(self.vis_list, model_imagelist=dec_imagelist, context='2d')
160-
restored = restore_list_arlexecute_workflow(model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist,
161-
residual_imagelist=residual_imagelist,
162-
empty=self.model_imagelist)[0]
165+
residual_imagelist = residual_list_arlexecute_workflow(self.vis_list, model_imagelist=dec_imagelist,
166+
context='2d')
167+
restored_list = restore_list_arlexecute_workflow(model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist,
168+
residual_imagelist=residual_imagelist,
169+
empty=self.model_imagelist)
163170

164-
restored = arlexecute.compute(restored, sync=True)
171+
restored = arlexecute.compute(restored_list, sync=True)[0]
165172

166173
export_image_to_fits(restored, '%s/test_imaging_%s_overlap_mmclean_restored.fits'
167174
% (self.dir, arlexecute.type()))

‎tests/workflows/test_modelpartition_arlexecute.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from data_models.memory_data_models import SkyModel
1313
from data_models.polarisation import PolarisationFrame
1414

15-
from . import ARLExecuteTestCase
15+
from tests.workflows import ARLExecuteTestCase
1616
from wrappers.arlexecute.calibration.operations import apply_gaintable, create_gaintable_from_blockvisibility
1717
from wrappers.arlexecute.calibration.calibration import solve_gaintable
1818
from wrappers.arlexecute.image.operations import export_image_to_fits, qa_image

‎tests/workflows/test_pipelines_arlexecute.py‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
from data_models.polarisation import PolarisationFrame
1515

16-
from . import ARLExecuteTestCase
16+
from tests.workflows import ARLExecuteTestCase
1717
from wrappers.arlexecute.calibration.calibration_control import create_calibration_controls
1818
from wrappers.arlexecute.execution_support.arlexecute import arlexecute
1919
from workflows.arlexecute.pipelines.pipeline_arlexecute import ical_list_arlexecute_workflow, continuum_imaging_list_arlexecute_workflow
@@ -102,8 +102,7 @@ def actualSetUp(self, add_errors=False, freqwin=5, block=False, dospectral=True,
102102
for freqwin, _ in enumerate(self.frequency)]
103103

104104
# Calculate the model convolved with a Gaussian.
105-
self.model_imagelist = arlexecute.compute(self.model_imagelist, sync=True)
106-
model = self.model_imagelist[0]
105+
model = arlexecute.compute(self.model_imagelist[0], sync=True)
107106
self.cmodel = smooth_image(model)
108107
export_image_to_fits(model, '%s/test_imaging_delayed_model.fits' % self.dir)
109108
export_image_to_fits(self.cmodel, '%s/test_imaging_delayed_cmodel.fits' % self.dir)
@@ -113,12 +112,7 @@ def actualSetUp(self, add_errors=False, freqwin=5, block=False, dospectral=True,
113112
arlexecute.execute(insert_unittest_errors)(self.vis_list[i], amp_errors=amp_errors,
114113
phase_errors=phase_errors)
115114
for i, _ in enumerate(self.frequency)]
116-
117-
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
118-
119-
self.vis_list = arlexecute.scatter(self.vis_list)
120-
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
121-
115+
122116
def test_time_setup(self):
123117
self.actualSetUp()
124118

‎tests/workflows/test_simulation_arlexecute.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from astropy import units as u
1010
from astropy.coordinates import SkyCoord
1111

12-
from . import ARLExecuteTestCase
12+
from tests.workflows import ARLExecuteTestCase
1313
from data_models.memory_data_models import BlockVisibility
1414
from wrappers.arlexecute.execution_support.arlexecute import arlexecute
1515

@@ -32,7 +32,7 @@ def setUp(self):
3232
def test_create_simulate_vis_list(self):
3333
vis_list = simulate_list_arlexecute_workflow(frequency=self.frequency, channel_bandwidth=self.channel_bandwidth)
3434
assert len(vis_list) == len(self.frequency)
35-
vt = arlexecute.compute(vis_list[0])
35+
vt = arlexecute.compute(vis_list[0], sync=True)
3636
assert isinstance(vt, BlockVisibility)
3737
assert vt.nvis > 0
3838

‎workflows/arlexecute/imaging/imaging_arlexecute.py‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414

1515
import numpy
1616

17-
from processing_library.image.operations import copy_image, create_empty_image_like
1817
from data_models.memory_data_models import Image
1918
from data_models.parameters import get_parameter
19+
from processing_library.image.operations import copy_image, create_empty_image_like
2020
from workflows.shared.imaging.imaging_shared import imaging_context
2121
from workflows.shared.imaging.imaging_shared import sum_invert_results, remove_sumwt, sum_predict_results, \
2222
threshold_list
@@ -192,10 +192,14 @@ def restore_list_arlexecute_workflow(model_imagelist, psf_imagelist, residual_im
192192
:param kwargs: Parameters for functions in components
193193
:return:
194194
"""
195-
return [arlexecute.execute(restore_cube)(model_imagelist[i], psf_imagelist[i][0],
196-
residual_imagelist[i][0], **kwargs)
197-
for i, _ in enumerate(model_imagelist)]
198-
195+
psf_list = arlexecute.execute(remove_sumwt)(psf_imagelist)
196+
residual_list = arlexecute.execute(remove_sumwt)(residual_imagelist)
197+
print('model_imagelist', model_imagelist)
198+
print('psf_imagelist', psf_imagelist)
199+
print('residual_imagelist', residual_imagelist)
200+
return [arlexecute.execute(restore_cube)(model_imagelist[i], psf_list[i],
201+
residual_list[i], **kwargs)
202+
for i, _ in enumerate(residual_imagelist)]
199203

200204
def deconvolve_list_arlexecute_workflow(dirty_list, psf_list, model_imagelist, prefix='', **kwargs):
201205
"""Create a graph for deconvolution, adding to the model
@@ -207,48 +211,46 @@ def deconvolve_list_arlexecute_workflow(dirty_list, psf_list, model_imagelist, p
207211
:return: (graph for the deconvolution, graph for the flat)
208212
"""
209213
nchan = len(dirty_list)
210-
214+
nmoments = get_parameter(kwargs, "nmoments", 0)
215+
211216
def deconvolve(dirty, psf, model, facet, gthreshold):
212-
import time
213-
starttime = time.time()
214217
if prefix == '':
215218
lprefix = "facet %d" % facet
216219
else:
217220
lprefix = "%s, facet %d" % (prefix, facet)
218221

219-
nmoments = get_parameter(kwargs, "nmoments", 0)
220-
221222
if nmoments > 0:
222223
moment0 = calculate_image_frequency_moments(dirty)
223224
this_peak = numpy.max(numpy.abs(moment0.data[0, ...])) / dirty.data.shape[0]
224225
else:
225226
this_peak = numpy.max(numpy.abs(dirty.data[0, ...]))
226227

227228
if this_peak > 1.1 * gthreshold:
228-
log.info(
229-
"deconvolve_list_arlexecute_workflow %s: cleaning - peak %.6f > 1.1 * threshold %.6f" % (lprefix, this_peak,
230-
gthreshold))
229+
# log.info(
230+
# "deconvolve_list_arlexecute_workflow %s: cleaning - peak %.6f > 1.1 * threshold %.6f" % (
231+
# lprefix, this_peak,
232+
# gthreshold))
231233
kwargs['threshold'] = gthreshold
232234
result, _ = deconvolve_cube(dirty, psf, prefix=lprefix, **kwargs)
233235

234236
if result.data.shape[0] == model.data.shape[0]:
235237
result.data += model.data
236-
else:
237-
log.warning(
238-
"deconvolve_list_arlexecute_workflow %s: Initial model %s and clean result %s do not have the same shape" %
239-
(lprefix, str(model.data.shape[0]), str(result.data.shape[0])))
240-
238+
# else:
239+
# log.warning(
240+
# "deconvolve_list_arlexecute_workflow %s: Initial model %s and clean result %s do not have the same shape" %
241+
# (lprefix, str(model.data.shape[0]), str(result.data.shape[0])))
242+
#
241243
flux = numpy.sum(result.data[0, 0, ...])
242-
log.info('### %s, %.6f, %.6f, True, %.3f # cycle, facet, peak, cleaned flux, clean, time?'
243-
% (lprefix, this_peak, flux, time.time() - starttime))
244-
244+
# log.info('### %s, %.6f, %.6f, True, # cycle, facet, peak, cleaned flux, clean'
245+
# % (lprefix, this_peak, flux[0]))
246+
#
245247
return result
246248
else:
247-
log.info("deconvolve_list_arlexecute_workflow %s: Not cleaning - peak %.6f <= 1.1 * threshold %.6f" % (
248-
lprefix, this_peak,
249-
gthreshold))
250-
log.info('### %s, %.6f, %.6f, False, %.3f # cycle, facet, peak, cleaned flux, clean, time?'
251-
% (lprefix, this_peak, 0.0, time.time() - starttime))
249+
# log.info("deconvolve_list_arlexecute_workflow %s: Not cleaning - peak %.6f <= 1.1 * threshold %.6f" % (
250+
# lprefix, this_peak,
251+
# gthreshold))
252+
# log.info('### %s, %.6f, %.6f, False, %.3f # cycle, facet, peak, cleaned flux, clean'
253+
# % (lprefix, this_peak, 0.0))
252254

253255
return copy_image(model)
254256

@@ -264,21 +266,21 @@ def deconvolve(dirty, psf, model, facet, gthreshold):
264266

265267
# Scatter the separate channel images into deconvolve facets and then gather channels for each facet.
266268
# This avoids constructing the entire spectral cube.
267-
# dirty_list = arlexecute.execute(remove_sumwt, nout=nchan)(dirty_list)
269+
dirty_list_trimmed = arlexecute.execute(remove_sumwt, nout=nchan)(dirty_list)
268270
scattered_channels_facets_dirty_list = \
269-
[arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(d[0], facets=deconvolve_facets,
271+
[arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(d, facets=deconvolve_facets,
270272
overlap=deconvolve_overlap,
271273
taper=deconvolve_taper)
272-
for d in dirty_list]
274+
for d in dirty_list_trimmed]
273275

274276
# Now we do a transpose and gather
275277
scattered_facets_list = [
276278
arlexecute.execute(image_gather_channels, nout=1)([scattered_channels_facets_dirty_list[chan][facet]
277279
for chan in range(nchan)])
278280
for facet in range(deconvolve_number_facets)]
279281

280-
psf_list = arlexecute.execute(remove_sumwt, nout=nchan)(psf_list)
281-
psf_list = arlexecute.execute(image_gather_channels, nout=1)(psf_list)
282+
psf_list_trimmed = arlexecute.execute(remove_sumwt, nout=nchan)(psf_list)
283+
psf_list_trimmed = arlexecute.execute(image_gather_channels, nout=1)(psf_list_trimmed)
282284

283285
scattered_model_imagelist = \
284286
arlexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(model_imagelist,
@@ -299,7 +301,7 @@ def deconvolve(dirty, psf, model, facet, gthreshold):
299301

300302
facet_list = numpy.arange(deconvolve_number_facets).astype('int')
301303
scattered_results_list = [
302-
arlexecute.execute(deconvolve, nout=1)(d, psf_list, m, facet, global_threshold)
304+
arlexecute.execute(deconvolve, nout=1)(d, psf_list_trimmed, m, facet, global_threshold)
303305
for d, m, facet in zip(scattered_facets_list, scattered_model_imagelist, facet_list)]
304306

305307
# Gather the results back into one image, correcting for overlaps as necessary. The taper function is is used to

0 commit comments

Comments
 (0)