Skip to content

Commit c386e75

Browse files
timcornwelltimcornwell
authored
Merge pull request SKA-ScienceDataProcessor#46 from SKA-ScienceDataProcessor/feature_mpc
Feature mpc merge into master
2 parents dee6078 + 6f7087d commit c386e75

86 files changed

Lines changed: 5290 additions & 967 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,21 @@
1+
**January 23, 2019** [Tim] Work on MPC. Documented in SDP memo 97.
2+
Branch feature_mpc merged into master. The old modelpartition code has
3+
been removed.
4+
1. SkyModel now has image, mask, and gaintable.
5+
2. Gaintable now has phasecentre, as needed for non-isoplanatic
6+
processing.
7+
3. Improved speed of scalar calibration.
8+
4. Improved HDF5 for SkyModel. Old ones not compatible.
9+
5. Functions for working with skycomponents e.g. filtering,
10+
neighbour filtering.
11+
6. Voronoi partitioning.
12+
7. Predict and invert workflows for skymodel.
13+
8. Ensure imaging_weights retained in BlockVisibility.
14+
9. Ionospheric phase screen simulation using
15+
ARatmospy: create_gaintable_from_screen. Gaintable phases can be
16+
back-propagated to screen: grid_gaintable_to_screen.
17+
10. Pipeline for MPCCAL: mpccal_skymodel_list_arlexecute_workflow.
18+
119
**November 26, 2018** [Tim] Many changes in run up to demo:
220
1. Parallel weighting, imaging weight added to BlockVisibility
321
2. GLEAM lookup optimised, can now generate a skymodel and predict from it.

‎data/models/create_test_mpc_screen.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:4eeba4aa3e99c83c0654b2f722a1a8de33967e17e301994945d0a97a5dd91610
3+
size 1935

‎data/models/test_mpc_screen.fits‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:922771b6ff466dd2db09ed576d5c228f61c97c72e8c6291648a1d5ca5e677dbe
3+
size 96004800

‎data_models/data_model_helpers.py‎

Lines changed: 151 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,8 @@ def convert_gaintable_to_hdf(gt: GainTable, f):
275275
f.attrs['ARL_data_model'] = 'GainTable'
276276
f.attrs['frequency'] = gt.frequency
277277
f.attrs['receptor_frame'] = gt.receptor_frame.type
278+
f.attrs['phasecentre_coords'] = gt.phasecentre.to_string()
279+
f.attrs['phasecentre_frame'] = gt.phasecentre.frame.name
278280
f['data'] = gt.data
279281
return f
280282

@@ -289,7 +291,10 @@ def convert_hdf_to_gaintable(f):
289291
receptor_frame = ReceptorFrame(f.attrs['receptor_frame'])
290292
frequency = numpy.array(f.attrs['frequency'])
291293
data = numpy.array(f['data'])
292-
gt = GainTable(data=data, receptor_frame=receptor_frame, frequency=frequency)
294+
s = f.attrs['phasecentre_coords'].split()
295+
ss = [float(s[0]), float(s[1])] * u.deg
296+
phasecentre = SkyCoord(ra=ss[0], dec=ss[1], frame=f.attrs['phasecentre_frame'])
297+
gt = GainTable(data=data, receptor_frame=receptor_frame, frequency=frequency, phasecentre=phasecentre)
293298
return gt
294299

295300

@@ -348,7 +353,7 @@ def convert_skycomponent_to_hdf(sc: Skycomponent, f):
348353

349354

350355
def convert_hdf_to_skycomponent(f):
351-
""" Convert HDF root to a GainTable
356+
""" Convert HDF root to a SkyComponent
352357
353358
:param f:
354359
:return:
@@ -409,12 +414,13 @@ def convert_image_to_hdf(im: Image, f):
409414
:param f: HDF root
410415
:return:
411416
"""
412-
assert isinstance(im, Image), im
417+
if isinstance(im, Image):
413418

414-
f.attrs['ARL_data_model'] = 'Image'
415-
f['data'] = im.data
416-
f.attrs['wcs'] = numpy.string_(im.wcs.to_header_string())
417-
f.attrs['polarisation_frame'] = im.polarisation_frame.type
419+
f.attrs['ARL_data_model'] = 'Image'
420+
f['data'] = im.data
421+
f.attrs['wcs'] = numpy.string_(im.wcs.to_header_string())
422+
f.attrs['polarisation_frame'] = im.polarisation_frame.type
423+
418424
return f
419425

420426

@@ -424,13 +430,15 @@ def convert_hdf_to_image(f):
424430
:param f:
425431
:return:
426432
"""
427-
assert f.attrs['ARL_data_model'] == "Image", "Not an Image"
428-
data = numpy.array(f['data'])
429-
polarisation_frame = PolarisationFrame(f.attrs['polarisation_frame'])
430-
wcs = WCS(f.attrs['wcs'])
431-
im = create_image_from_array(data, wcs=wcs,
433+
if 'ARL_data_model' in f.attrs.keys() and f.attrs['ARL_data_model'] == "Image":
434+
data = numpy.array(f['data'])
435+
polarisation_frame = PolarisationFrame(f.attrs['polarisation_frame'])
436+
wcs = WCS(f.attrs['wcs'])
437+
im = create_image_from_array(data, wcs=wcs,
432438
polarisation_frame=polarisation_frame)
433-
return im
439+
return im
440+
else:
441+
return None
434442

435443

436444
def export_image_to_hdf5(im, filename):
@@ -477,20 +485,38 @@ def export_skymodel_to_hdf5(sm, filename):
477485
:return:
478486
"""
479487

480-
assert isinstance(sm, SkyModel)
488+
if not isinstance(sm, collections.Iterable):
489+
sm = [sm]
481490

482491
with h5py.File(filename, 'w') as f:
483-
f.attrs['number_skycomponents'] = len(sm.components)
484-
for i, sc in enumerate(sm.components):
485-
cf = f.create_group('skycomponent%d' % i)
486-
convert_skycomponent_to_hdf(sc, cf)
487-
f.attrs['number_images'] = len(sm.images)
488-
for i, im in enumerate(sm.images):
489-
cf = f.create_group('image%d' % i)
490-
convert_image_to_hdf(im, cf)
491-
492+
f.attrs['number_data_models'] = len(sm)
493+
for i, s in enumerate(sm):
494+
assert isinstance(s, SkyModel)
495+
sf = f.create_group('SkyModel%d' % i)
496+
convert_skymodel_to_hdf(s, sf)
492497
f.flush()
493498
f.close()
499+
500+
def convert_skymodel_to_hdf(sm, f):
501+
"""
502+
503+
:param sm:
504+
:param f:
505+
:return:
506+
"""
507+
f.attrs['ARL_data_model'] = 'SkyModel'
508+
f.attrs['fixed'] = sm.fixed
509+
f.attrs['number_skycomponents'] = len(sm.components)
510+
for i, sc in enumerate(sm.components):
511+
cf = f.create_group('skycomponent%d' % i)
512+
convert_skycomponent_to_hdf(sm.components[i], cf)
513+
cf = f.create_group('image')
514+
convert_image_to_hdf(sm.image, cf)
515+
cf = f.create_group('mask')
516+
convert_image_to_hdf(sm.mask, cf)
517+
cf = f.create_group('gaintable')
518+
convert_gaintable_to_hdf(sm.gaintable, cf)
519+
return f
494520

495521

496522
def import_skymodel_from_hdf5(filename):
@@ -501,83 +527,46 @@ def import_skymodel_from_hdf5(filename):
501527
"""
502528

503529
with h5py.File(filename, 'r') as f:
504-
ncomponents = f.attrs['number_skycomponents']
505-
components = [convert_hdf_to_skycomponent(f['skycomponent%d' % i])
506-
for i in range(ncomponents)]
507-
508-
nimages = f.attrs['number_images']
509-
images = [convert_hdf_to_image(f['image%d' % i]) for i in range(nimages)]
510-
511-
return SkyModel(components=components, images=images)
512-
513-
514-
def memory_data_model_to_buffer(model, jbuff, dm):
515-
""" Copy a memory data model to a buffer data model
530+
nsmlist = f.attrs['number_data_models']
531+
smlist = [convert_hdf_to_skymodel(f['SkyModel%d' % i]) for i in range(nsmlist)]
532+
if nsmlist == 1:
533+
return smlist[0]
534+
else:
535+
return smlist
516536

517-
The file type is derived from the file extension. All are hdf only with the exception of Imaghe which can also be
518-
fits.
519-
520-
:param model: Memory data model to be sent to buffer
521-
:param jbuff: JSON describing buffer
522-
:param dm: JSON describing data model
537+
def convert_hdf_to_skymodel(f):
523538
"""
524-
name = jbuff["directory"] + dm["name"]
539+
540+
:param f:
541+
:return:
542+
"""
543+
assert f.attrs['ARL_data_model'] == "SkyModel", f.attrs['ARL_data_model']
525544

526-
import os
527-
_, file_extension = os.path.splitext(dm["name"])
545+
fixed = f.attrs['fixed']
528546

529-
if dm["data_model"] == "BlockVisibility":
530-
return export_blockvisibility_to_hdf5(model, name)
531-
elif dm["data_model"] == "Image":
532-
if file_extension == ".fits":
533-
return export_image_to_fits(model, name)
534-
else:
535-
return export_image_to_hdf5(model, name)
536-
elif dm["data_model"] == "GridData":
537-
return export_griddata_to_hdf5(model, name)
538-
elif dm["data_model"] == "ConvolutionFunction":
539-
return export_convolutionfunction_to_hdf5(model, name)
540-
elif dm["data_model"] == "SkyModel":
541-
return export_skymodel_to_hdf5(model, name)
542-
elif dm["data_model"] == "GainTable":
543-
return export_gaintable_to_hdf5(model, name)
547+
ncomponents = f.attrs['number_skycomponents']
548+
components = list()
549+
for i in range(ncomponents):
550+
cf = f[('skycomponent%d' % i)]
551+
components.append(convert_hdf_to_skycomponent(cf))
552+
if 'image' in f.keys():
553+
cf = f['image']
554+
image = convert_hdf_to_image(cf)
544555
else:
545-
raise ValueError("Data model %s not supported" % dm["data_model"])
546-
556+
image = None
557+
if 'mask' in f.keys():
558+
cf = f['mask']
559+
mask = convert_hdf_to_image(cf)
560+
else:
561+
mask = None
562+
if 'gaintable' in f.keys():
563+
cf = f['gaintable']
564+
gaintable = convert_hdf_to_gaintable(cf)
565+
else:
566+
gaintable = None
547567

548-
def buffer_data_model_to_memory(jbuff, dm):
549-
"""Copy a buffer data model into memory data model
550568

551-
The file type is derived from the file extension. All are hdf only with the exception of Imaghe which can also be
552-
fits.
553-
554-
:param jbuff: JSON describing buffer
555-
:param dm: JSON describing data model
556-
:return: data model
557-
"""
558-
import os
559-
name = os.path.join(jbuff["directory"], dm["name"])
560-
561-
import os
562-
_, file_extension = os.path.splitext(dm["name"])
563-
564-
if dm["data_model"] == "BlockVisibility":
565-
return import_blockvisibility_from_hdf5(name)
566-
elif dm["data_model"] == "Image":
567-
if file_extension == ".fits":
568-
return import_image_from_fits(name)
569-
else:
570-
return import_image_from_hdf5(name)
571-
elif dm["data_model"] == "SkyModel":
572-
return import_skymodel_from_hdf5(name)
573-
elif dm["data_model"] == "GainTable":
574-
return import_gaintable_from_hdf5(name)
575-
elif dm["data_model"] == "GridData":
576-
return import_griddata_from_hdf5(name)
577-
elif dm["data_model"] == "ConvolutionFunction":
578-
return import_convolutionfunction_from_hdf5(name)
579-
else:
580-
raise ValueError("Data model %s not supported" % dm["data_model"])
569+
return SkyModel(image=image, components=components, gaintable=gaintable, mask=mask, fixed=fixed)
581570

582571

583572
def convert_griddata_to_hdf(gd: GridData, f):
@@ -718,3 +707,72 @@ def import_convolutionfunction_from_hdf5(filename):
718707
return cflist
719708

720709

710+
def memory_data_model_to_buffer(model, jbuff, dm):
711+
""" Copy a memory data model to a buffer data model
712+
713+
The file type is derived from the file extension. All are hdf only with the exception of Imaghe which can also be
714+
fits.
715+
716+
:param model: Memory data model to be sent to buffer
717+
:param jbuff: JSON describing buffer
718+
:param dm: JSON describing data model
719+
"""
720+
name = jbuff["directory"] + dm["name"]
721+
722+
import os
723+
_, file_extension = os.path.splitext(dm["name"])
724+
725+
if dm["data_model"] == "BlockVisibility":
726+
return export_blockvisibility_to_hdf5(model, name)
727+
elif dm["data_model"] == "Image":
728+
if file_extension == ".fits":
729+
return export_image_to_fits(model, name)
730+
else:
731+
return export_image_to_hdf5(model, name)
732+
elif dm["data_model"] == "GridData":
733+
return export_griddata_to_hdf5(model, name)
734+
elif dm["data_model"] == "ConvolutionFunction":
735+
return export_convolutionfunction_to_hdf5(model, name)
736+
elif dm["data_model"] == "SkyModel":
737+
return export_skymodel_to_hdf5(model, name)
738+
elif dm["data_model"] == "GainTable":
739+
return export_gaintable_to_hdf5(model, name)
740+
else:
741+
raise ValueError("Data model %s not supported" % dm["data_model"])
742+
743+
744+
def buffer_data_model_to_memory(jbuff, dm):
745+
"""Copy a buffer data model into memory data model
746+
747+
The file type is derived from the file extension. All are hdf only with the exception of Imaghe which can also be
748+
fits.
749+
750+
:param jbuff: JSON describing buffer
751+
:param dm: JSON describing data model
752+
:return: data model
753+
"""
754+
import os
755+
name = os.path.join(jbuff["directory"], dm["name"])
756+
757+
import os
758+
_, file_extension = os.path.splitext(dm["name"])
759+
760+
if dm["data_model"] == "BlockVisibility":
761+
return import_blockvisibility_from_hdf5(name)
762+
elif dm["data_model"] == "Image":
763+
if file_extension == ".fits":
764+
return import_image_from_fits(name)
765+
else:
766+
return import_image_from_hdf5(name)
767+
elif dm["data_model"] == "SkyModel":
768+
return import_skymodel_from_hdf5(name)
769+
elif dm["data_model"] == "GainTable":
770+
return import_gaintable_from_hdf5(name)
771+
elif dm["data_model"] == "GridData":
772+
return import_griddata_from_hdf5(name)
773+
elif dm["data_model"] == "ConvolutionFunction":
774+
return import_convolutionfunction_from_hdf5(name)
775+
else:
776+
raise ValueError("Data model %s not supported" % dm["data_model"])
777+
778+

0 commit comments

Comments
 (0)