Skip to content

Commit 65265c5

Browse files
timcornwelltimcornwell
committed
MPC pipeline tests work
1 parent e64e693 commit 65265c5

11 files changed

Lines changed: 493 additions & 269 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,5 @@ dask-work-space
5151

5252
*~
5353
.DS_Store
54-
**/.DS_Store
54+
**/.DS_Store
55+
*.dirlock

‎processing_components/calibration/calibration_control.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,8 @@ def solve_calibrate_function(vis, model_vis, calibration_context='T', controls=N
190190

191191
assert isinstance(avis, BlockVisibility), avis
192192

193+
assert amvis.__repr__() != avis.__repr__(), "Vis and model vis are the same object: convert problem"
194+
193195
# Always return a gain table, even if null
194196
for c in calibration_context:
195197
gaintables[c] = \

‎processing_components/image/deconvolution.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,13 @@ def deconvolve_cube(dirty: Image, psf: Image, prefix='', **kwargs) -> (Image, Im
8686
window = numpy.zeros_like(dirty.data)
8787
window[..., (qy + 1):3 * qy, (qx + 1):3 * qx] = 1.0
8888
log.info('deconvolve_cube %s: Cleaning inner quarter of each sky plane' % prefix)
89+
elif window_shape == 'no_edge':
90+
edge = 16
91+
nx = dirty.shape[3]
92+
ny = dirty.shape[2]
93+
window = numpy.zeros_like(dirty.data)
94+
window[..., (edge + 1):(ny - edge), (edge + 1):(nx - edge)] = 1.0
95+
log.info('deconvolve_cube %s: Cleaning omitting %d-pixel edge of each sky plane' % (prefix, edge))
8996
else:
9097
window = None
9198

‎processing_components/skycomponent/operations.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -487,14 +487,19 @@ def image_voronoi_iter(im: Image, components: Skycomponent) -> collections.Itera
487487
:param im: Image
488488
:param components: Components to define Voronoi decomposition
489489
"""
490-
vor, vertex_array = voronoi_decomposition(im, components)
491-
492-
nregions = numpy.max(vertex_array) + 1
493-
for region in range(nregions):
494-
mask = numpy.zeros(im.data.shape)
495-
mask[(vertex_array == region)[numpy.newaxis, numpy.newaxis,...]] = 1.0
490+
if len(components) == 1:
491+
mask = numpy.ones(im.data.shape)
496492
yield create_image_from_array(mask, wcs=im.wcs,
497493
polarisation_frame=im.polarisation_frame)
494+
else:
495+
vor, vertex_array = voronoi_decomposition(im, components)
496+
497+
nregions = numpy.max(vertex_array) + 1
498+
for region in range(nregions):
499+
mask = numpy.zeros(im.data.shape)
500+
mask[(vertex_array == region)[numpy.newaxis, numpy.newaxis,...]] = 1.0
501+
yield create_image_from_array(mask, wcs=im.wcs,
502+
polarisation_frame=im.polarisation_frame)
498503

499504
def partition_skycomponent_neighbours(comps, targets):
500505
""" Partition sky components by nearest target source

‎processing_components/visibility/coalesce.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ def convert_blockvisibility_to_visibility(vis: BlockVisibility) -> Visibility:
108108
return converted_vis
109109

110110

111-
def decoalesce_visibility(vis: Visibility, overwrite=False, **kwargs) -> BlockVisibility:
111+
def decoalesce_visibility(vis: Visibility, **kwargs) -> BlockVisibility:
112112
""" Decoalesce the visibilities to the original values (opposite of coalesce_visibility)
113113
114114
This relies upon the block vis and the index being part of the vis. Needs the index generated by coalesce_visibility
@@ -121,12 +121,8 @@ def decoalesce_visibility(vis: Visibility, overwrite=False, **kwargs) -> BlockVi
121121
assert isinstance(vis.blockvis, BlockVisibility), "No blockvisibility in vis %r" % vis
122122
assert vis.cindex is not None, "No reverse index in Visibility %r" % vis
123123

124-
if overwrite:
125-
log.debug('decoalesce_visibility: Created new Visibility for decoalesced data_models')
126-
decomp_vis = copy_visibility(vis.blockvis)
127-
else:
128-
log.debug('decoalesce_visibility: Filled decoalesced data into template')
129-
decomp_vis = vis.blockvis
124+
log.debug('decoalesce_visibility: Created new Visibility for decoalesced data_models')
125+
decomp_vis = copy_visibility(vis.blockvis)
130126

131127
vshape = decomp_vis.data['vis'].shape
132128

‎tests/processing_components/test_image_deconvolution.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,16 +99,23 @@ def test_deconvolve_msclean_1scale(self):
9999
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
100100
export_image_to_fits(self.cmodel, "%s/test_deconvolve_msclean_1scale-clean.fits" % (self.dir))
101101
assert numpy.max(self.residual.data) < 1.2
102-
102+
103+
def test_deconvolve_hogbom_no_edge(self):
104+
self.comp, self.residual = deconvolve_cube(self.dirty, self.psf, window='no_edge', niter=10000,
105+
gain=0.1, algorithm='hogbom', threshold=0.01)
106+
export_image_to_fits(self.residual, "%s/test_deconvolve_hogbom_noedge-residual.fits" % (self.dir))
107+
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
108+
export_image_to_fits(self.cmodel, "%s/test_deconvolve_hogbom_no_edge-clean.fits" % (self.dir))
109+
assert numpy.max(self.residual.data) < 1.2
110+
103111
def test_deconvolve_hogbom_inner_quarter(self):
104-
105112
self.comp, self.residual = deconvolve_cube(self.dirty, self.psf, window='quarter', niter=10000,
106113
gain=0.1, algorithm='hogbom', threshold=0.01)
107114
export_image_to_fits(self.residual, "%s/test_deconvolve_hogbom_innerquarter-residual.fits" % (self.dir))
108115
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
109116
export_image_to_fits(self.cmodel, "%s/test_deconvolve_hogbom_innerquarter-clean.fits" % (self.dir))
110117
assert numpy.max(self.residual.data) < 1.2
111-
118+
112119
def test_deconvolve_msclean_inner_quarter(self):
113120

114121
self.comp, self.residual = deconvolve_cube(self.dirty, self.psf, window='quarter', niter=1000, gain=0.7,

‎tests/processing_components/test_skymodel_mpc.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from data_models.memory_data_models import SkyModel, Image
1313
from data_models.polarisation import PolarisationFrame
1414
from processing_components.imaging.primary_beams import create_low_test_beam
15-
from operations import expand_skymodel_by_skycomponents
15+
from processing_components.skymodel.operations import expand_skymodel_by_skycomponents
1616
from processing_components.simulation.testing_support import create_named_configuration, \
1717
create_low_test_skycomponents_from_gleam
1818
from processing_components.simulation.testing_support import create_test_image

‎tests/processing_components/test_visibility_coalesce.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def test_coalesce_decoalesce_zero(self):
4242
assert numpy.min(cvis.frequency) > 0.0
4343
dvis = decoalesce_visibility(cvis)
4444
assert dvis.nvis == self.blockvis.nvis
45-
dvis = decoalesce_visibility(cvis, overwrite=True)
45+
dvis = decoalesce_visibility(cvis)
4646
assert dvis.nvis == self.blockvis.nvis
4747

4848
def test_convert_weight(self):
@@ -52,7 +52,7 @@ def test_convert_weight(self):
5252
cvis = weight_visibility(cvis, model)
5353
assert numpy.mean(cvis.data['imaging_weight']) < 1.0
5454
assert numpy.std(cvis.data['imaging_weight']) > 0.0
55-
dvis = decoalesce_visibility(cvis, overwrite=True)
55+
dvis = decoalesce_visibility(cvis)
5656
assert numpy.mean(dvis.data['imaging_weight']) < 1.0
5757
assert numpy.std(dvis.data['imaging_weight']) > 0.0
5858
assert dvis.nvis == self.blockvis.nvis
@@ -63,7 +63,7 @@ def test_convert_decoalesce_zero(self):
6363
assert numpy.min(cvis.frequency) > 0.0
6464
dvis = decoalesce_visibility(cvis)
6565
assert dvis.nvis == self.blockvis.nvis
66-
dvis = decoalesce_visibility(cvis, overwrite=True)
66+
dvis = decoalesce_visibility(cvis)
6767
assert dvis.nvis == self.blockvis.nvis
6868

6969
def test_coalesce_decoalesce(self):
@@ -72,7 +72,7 @@ def test_coalesce_decoalesce(self):
7272
assert numpy.min(cvis.frequency) > 0.0
7373
dvis = decoalesce_visibility(cvis)
7474
assert dvis.nvis == self.blockvis.nvis
75-
dvis = decoalesce_visibility(cvis, overwrite=True)
75+
dvis = decoalesce_visibility(cvis)
7676
assert dvis.nvis == self.blockvis.nvis
7777

7878
def test_coalesce_decoalesce_frequency(self):
@@ -81,7 +81,7 @@ def test_coalesce_decoalesce_frequency(self):
8181
assert numpy.min(cvis.frequency) > 0.0
8282
dvis = decoalesce_visibility(cvis)
8383
assert dvis.nvis == self.blockvis.nvis
84-
dvis = decoalesce_visibility(cvis, overwrite=True)
84+
dvis = decoalesce_visibility(cvis)
8585
assert dvis.nvis == self.blockvis.nvis
8686

8787
def test_coalesce_decoalesce_time(self):
@@ -90,7 +90,7 @@ def test_coalesce_decoalesce_time(self):
9090
assert numpy.min(cvis.frequency) > 0.0
9191
dvis = decoalesce_visibility(cvis)
9292
assert dvis.nvis == self.blockvis.nvis
93-
dvis = decoalesce_visibility(cvis, overwrite=True)
93+
dvis = decoalesce_visibility(cvis)
9494
assert dvis.nvis == self.blockvis.nvis
9595

9696
def test_coalesce_decoalesce_singletime(self):

‎tests/workflows/test_mpc_serial.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from data_models.memory_data_models import Image, SkyModel
1313
from data_models.memory_data_models import Skycomponent
1414
from data_models.polarisation import PolarisationFrame
15-
from operations import expand_skymodel_by_skycomponents
15+
from wrappers.serial.skymodel.operations import expand_skymodel_by_skycomponents
1616
from workflows.serial.skymodel.skymodel_serial import predict_skymodel_list_serial_workflow, \
1717
invert_skymodel_list_serial_workflow, extract_datamodels_skymodel_list_serial_workflow
1818
from workflows.shared.imaging.imaging_shared import sum_predict_results

0 commit comments

Comments
 (0)