Skip to content

Commit d449dcb

Browse files
Jorge Fernandez HernandezJorge Fernandez Hernandez
Jorge Fernandez Hernandez
authored and
Jorge Fernandez Hernandez
committed
GAIASA-3522 method Gaia.load_data returns tuple
1 parent 6e2a2f6 commit d449dcb

4 files changed

Lines changed: 84 additions & 85 deletions

File tree

‎CHANGES.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ gaia
8080

8181
- The values that the ``data_structure parameter`` can accept have been changed from RAW to DATAMODEL_GAIA, and from
8282
INDIVIDUAL to DATAMODEL_STANDARD. [#3629]
83+
- The method ``load_data`` returns the path of the downloaded DataLink archive. [#3673]
8384

8485
esa.esasky
8586
^^^^^^^^^^

‎astroquery/gaia/core.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import shutil
1515
import zipfile
1616
from collections.abc import Iterable
17+
import warnings
1718

1819
from astropy import units
1920
from astropy import units as u
@@ -24,6 +25,7 @@
2425
from astropy.table import Table
2526
from astropy.units import Quantity
2627
from astropy.utils.decorators import deprecated_renamed_argument
28+
from astropy.utils.exceptions import AstropyUserWarning
2729
from requests import HTTPError
2830

2931
from astroquery import log
@@ -200,15 +202,6 @@ def load_data(self, ids, *, data_release=None, data_structure='DATAMODEL_STANDAR
200202
'EPOCH_SPECTRUM_XP_CROWDING', 'MEAN_SPECTRUM_XP', 'EPOCH_SPECTRUM_XP', 'CROWDED_FIELD_IMAGE',
201203
'EPOCH_ASTROMETRY_BRIGHT', 'MEAN_SPECTRUM_XP_GRAVLENS', 'EPOCH_FLAGS_NSS', 'EPOCH_PARAMETERS_RVS_SINGLE',
202204
'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI', 'RESIDUAL_IMAGE'].
203-
204-
Notes
205-
-----
206-
- ``CROWDED_FIELD_IMAGE`` supports only the ``'fits'`` format. The principal image is not included in the
207-
returned dictionary. To retrieve both the image and the associated tables, inspect each individual fits
208-
file.
209-
210-
- ``RESIDUAL_IMAGE`` also supports only the ``'fits'`` format. Since the FITS files contain images only, the
211-
returned table is empty. Inspect each individual file to access their contents.
212205
linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
213206
By default, all the identifiers are considered as source_id.
214207
@@ -224,7 +217,7 @@ def load_data(self, ids, *, data_release=None, data_structure='DATAMODEL_STANDAR
224217
By default, this value will be set to False .If set to True, the DataLink item tags are not validated.
225218
format : str, optional, default 'votable'
226219
Loading format. Supported values are 'csv', 'ecsv','votable_plain', 'json' and 'fits'
227-
dump_to_file: boolean, optional, default False.
220+
dump_to_file : boolean, optional, default False.
228221
If True, a ZIP archive named "datalink_output_<time_stamp>.zip" is created with all the DataLink
229222
files is made in the current working directory. The <time_stamp> format follows the ISO 8601 standard:
230223
"YYYYMMDD_HHMMSS.mmmmmm".
@@ -233,6 +226,15 @@ def load_data(self, ids, *, data_release=None, data_structure='DATAMODEL_STANDAR
233226
verbose : bool, optional, default 'False'
234227
Flag to display information about the process
235228
229+
Notes
230+
-----
231+
232+
- ``CROWDED_FIELD_IMAGE`` supports only the ``'fits'`` format. The principal image is not included in the
233+
returned dictionary. To retrieve both the image and the associated tables, inspect each individual fits
234+
file.
235+
- ``RESIDUAL_IMAGE`` also supports only the ``'fits'`` format. Since the FITS files contain images only, the
236+
returned table is empty. Inspect each individual file to access their contents.
237+
236238
Returns
237239
-------
238240
tuple
@@ -246,6 +248,13 @@ def load_data(self, ids, *, data_release=None, data_structure='DATAMODEL_STANDAR
246248
otherwise ``None``.
247249
"""
248250

251+
warnings.warn(
252+
"The return value of Gaia.load_data() has changed. The method now "
253+
"returns a tuple containing the DataLink products and the path to the output file.",
254+
AstropyUserWarning,
255+
stacklevel=2,
256+
)
257+
249258
output_file_specified = False
250259

251260
now = datetime.datetime.now(datetime.timezone.utc)

‎astroquery/gaia/tests/test_gaiatap.py‎

Lines changed: 62 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -881,6 +881,7 @@ def test_cone_search_and_changing_MAIN_GAIA_TABLE(mock_querier_async):
881881
assert "name_from_class" in job.parameters["query"]
882882

883883

884+
@pytest.mark.filterwarnings("ignore:")
884885
@pytest.mark.parametrize("overwrite_output_file", [True])
885886
def test_datalink_querier_load_data_vot_exception(mock_datalink_querier, overwrite_output_file):
886887
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
@@ -924,14 +925,16 @@ def test_datalink_querier_load_data_vot_exception(mock_datalink_querier, overwri
924925
assert not os.path.exists(file_final)
925926

926927

928+
@pytest.mark.filterwarnings("ignore:")
927929
def test_datalink_querier_load_data_vot(mock_datalink_querier):
928930
result_dict, file_path = mock_datalink_querier.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
929-
data_structure='DATAMODEL_STANDARD',
930-
retrieval_type="ALL",
931-
linking_parameter='SOURCE_ID', valid_data=False,
932-
avoid_datatype_check=False,
933-
format="votable", dump_to_file=True, overwrite_output_file=True,
934-
verbose=False)
931+
data_structure='DATAMODEL_STANDARD',
932+
retrieval_type="ALL",
933+
linking_parameter='SOURCE_ID', valid_data=False,
934+
avoid_datatype_check=False,
935+
format="votable", dump_to_file=True,
936+
overwrite_output_file=True,
937+
verbose=False)
935938

936939
direc = os.getcwd()
937940
files = os.listdir(direc)
@@ -940,7 +943,7 @@ def test_datalink_querier_load_data_vot(mock_datalink_querier):
940943
Path(direc, f).is_file() and f.endswith(".zip") and f.startswith('datalink_output')]
941944

942945
assert len(files) == 1
943-
assert files[0] == file_path
946+
assert os.path.join(direc, files[0]) == file_path
944947

945948
datalink_output = files[0]
946949

@@ -965,25 +968,32 @@ def test_datalink_querier_load_data_vot(mock_datalink_querier):
965968

966969
# check the returned output file path
967970

971+
972+
@pytest.mark.filterwarnings("ignore:")
973+
def test_datalink_querier_load_data_vot_no_dump_to_file(mock_datalink_querier):
974+
968975
result_dict, file_path = mock_datalink_querier.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
969-
data_structure='DATAMODEL_STANDARD',
970-
retrieval_type="ALL",
971-
linking_parameter='SOURCE_ID', valid_data=False,
972-
avoid_datatype_check=False,
973-
format="votable", dump_to_file=False, overwrite_output_file=True,
974-
verbose=False)
976+
data_structure='DATAMODEL_STANDARD',
977+
retrieval_type="ALL",
978+
linking_parameter='SOURCE_ID', valid_data=False,
979+
avoid_datatype_check=False,
980+
format="votable", dump_to_file=False,
981+
overwrite_output_file=True,
982+
verbose=False)
975983

976984
assert file_path is None
977985

978986

987+
@pytest.mark.filterwarnings("ignore:")
979988
def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):
980989
result_dict, file_path = mock_datalink_querier_ecsv.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
981-
data_structure='DATAMODEL_STANDARD',
982-
retrieval_type="ALL",
983-
linking_parameter='SOURCE_ID', valid_data=False,
984-
avoid_datatype_check=False,
985-
format="ecsv", dump_to_file=True, overwrite_output_file=True,
986-
verbose=False)
990+
data_structure='DATAMODEL_STANDARD',
991+
retrieval_type="ALL",
992+
linking_parameter='SOURCE_ID', valid_data=False,
993+
avoid_datatype_check=False,
994+
format="ecsv", dump_to_file=True,
995+
overwrite_output_file=True,
996+
verbose=False)
987997

988998
direc = os.getcwd()
989999
files = os.listdir(direc)
@@ -992,7 +1002,7 @@ def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):
9921002
Path(direc, f).is_file() and f.endswith(".zip") and f.startswith('datalink_output')]
9931003

9941004
assert len(files) == 1
995-
assert files[0] == file_path
1005+
assert os.path.join(direc, files[0]) == file_path
9961006

9971007
datalink_output = files[0]
9981008

@@ -1019,14 +1029,16 @@ def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):
10191029
assert not os.path.exists(datalink_output)
10201030

10211031

1032+
@pytest.mark.filterwarnings("ignore:")
10221033
def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):
10231034
result_dict, file_path = mock_datalink_querier_csv.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
1024-
data_structure='DATAMODEL_STANDARD',
1025-
retrieval_type="ALL",
1026-
linking_parameter='SOURCE_ID', valid_data=False,
1027-
avoid_datatype_check=False,
1028-
format="csv", dump_to_file=True, overwrite_output_file=True,
1029-
verbose=False)
1035+
data_structure='DATAMODEL_STANDARD',
1036+
retrieval_type="ALL",
1037+
linking_parameter='SOURCE_ID', valid_data=False,
1038+
avoid_datatype_check=False,
1039+
format="csv", dump_to_file=True,
1040+
overwrite_output_file=True,
1041+
verbose=False)
10301042

10311043
direc = os.getcwd()
10321044
files = os.listdir(direc)
@@ -1035,7 +1047,7 @@ def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):
10351047
Path(direc, f).is_file() and f.endswith(".zip") and f.startswith('datalink_output')]
10361048

10371049
assert len(files) == 1
1038-
assert files[0] == file_path
1050+
assert os.path.join(direc, files[0]) == file_path
10391051

10401052
datalink_output = files[0]
10411053

@@ -1065,12 +1077,13 @@ def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):
10651077
@pytest.mark.filterwarnings("ignore:")
10661078
def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
10671079
result_dict, file_path = mock_datalink_querier_fits.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
1068-
data_structure='DATAMODEL_STANDARD',
1069-
retrieval_type="ALL",
1070-
linking_parameter='SOURCE_ID', valid_data=False,
1071-
avoid_datatype_check=False,
1072-
format="fits", dump_to_file=True, overwrite_output_file=True,
1073-
verbose=False)
1080+
data_structure='DATAMODEL_STANDARD',
1081+
retrieval_type="ALL",
1082+
linking_parameter='SOURCE_ID', valid_data=False,
1083+
avoid_datatype_check=False,
1084+
format="fits", dump_to_file=True,
1085+
overwrite_output_file=True,
1086+
verbose=False)
10741087

10751088
direc = os.getcwd()
10761089
files = os.listdir(direc)
@@ -1079,7 +1092,7 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
10791092
Path(direc, f).is_file() and f.endswith(".zip") and f.startswith('datalink_output')]
10801093

10811094
assert len(files) == 1
1082-
assert files[0] == file_path
1095+
assert os.path.join(direc, files[0]) == file_path
10831096

10841097
datalink_output = files[0]
10851098

@@ -1106,6 +1119,7 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
11061119
assert not os.path.exists(datalink_output)
11071120

11081121

1122+
@pytest.mark.filterwarnings("ignore:")
11091123
def test_load_data_vot(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
11101124
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
11111125

@@ -1175,18 +1189,13 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
11751189

11761190
monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
11771191

1178-
GAIA_QUERIER.load_data(
1179-
valid_data=True,
1180-
ids="1,2,3,4",
1181-
format='fits',
1182-
retrieval_type="epoch_photometry",
1183-
verbose=True,
1184-
dump_to_file=True,
1185-
overwrite_output_file=True)
1192+
GAIA_QUERIER.load_data(valid_data=True, ids="1,2,3,4", format='fits', retrieval_type="epoch_photometry",
1193+
verbose=True, dump_to_file=True, overwrite_output_file=True)
11861194

11871195
path.unlink()
11881196

11891197

1198+
@pytest.mark.filterwarnings("ignore:")
11901199
def test_load_data_csv(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
11911200
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
11921201

@@ -1213,18 +1222,13 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
12131222

12141223
monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
12151224

1216-
GAIA_QUERIER.load_data(
1217-
valid_data=True,
1218-
ids="1,2,3,4",
1219-
format='csv',
1220-
retrieval_type="epoch_photometry",
1221-
verbose=True,
1222-
dump_to_file=True,
1223-
overwrite_output_file=True)
1225+
GAIA_QUERIER.load_data(valid_data=True, ids="1,2,3,4", format='csv', retrieval_type="epoch_photometry",
1226+
verbose=True, dump_to_file=True, overwrite_output_file=True)
12241227

12251228
path.unlink()
12261229

12271230

1231+
@pytest.mark.filterwarnings("ignore:")
12281232
def test_load_data_ecsv(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
12291233
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
12301234

@@ -1251,18 +1255,13 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
12511255

12521256
monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
12531257

1254-
GAIA_QUERIER.load_data(
1255-
valid_data=True,
1256-
ids="1,2,3,4",
1257-
format='ecsv',
1258-
retrieval_type="epoch_photometry",
1259-
verbose=True,
1260-
dump_to_file=True,
1261-
overwrite_output_file=True)
1258+
GAIA_QUERIER.load_data(valid_data=True, ids="1,2,3,4", format='ecsv', retrieval_type="epoch_photometry",
1259+
verbose=True, dump_to_file=True, overwrite_output_file=True)
12621260

12631261
path.unlink()
12641262

12651263

1264+
@pytest.mark.filterwarnings("ignore:")
12661265
def test_load_data_linking_parameter(monkeypatch, tmp_path, patch_datetime_now):
12671266
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
12681267

@@ -1289,18 +1288,13 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
12891288

12901289
monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
12911290

1292-
GAIA_QUERIER.load_data(
1293-
ids="1,2,3,4",
1294-
retrieval_type="epoch_photometry",
1295-
linking_parameter="SOURCE_ID",
1296-
valid_data=True,
1297-
verbose=True,
1298-
dump_to_file=True,
1299-
overwrite_output_file=True)
1291+
GAIA_QUERIER.load_data(ids="1,2,3,4", retrieval_type="epoch_photometry", linking_parameter="SOURCE_ID",
1292+
valid_data=True, verbose=True, dump_to_file=True, overwrite_output_file=True)
13001293

13011294
path.unlink()
13021295

13031296

1297+
@pytest.mark.filterwarnings("ignore:")
13041298
@pytest.mark.parametrize("linking_param", ['TRANSIT_ID', 'IMAGE_ID'])
13051299
def test_load_data_linking_parameter_with_values(monkeypatch, tmp_path, linking_param, patch_datetime_now):
13061300
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
@@ -1338,14 +1332,8 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
13381332

13391333
monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
13401334

1341-
GAIA_QUERIER.load_data(
1342-
ids="1,2,3,4",
1343-
retrieval_type="epoch_photometry",
1344-
linking_parameter=linking_param,
1345-
valid_data=True,
1346-
verbose=True,
1347-
dump_to_file=True,
1348-
overwrite_output_file=True)
1335+
GAIA_QUERIER.load_data(ids="1,2,3,4", retrieval_type="epoch_photometry", linking_parameter=linking_param,
1336+
valid_data=True, verbose=True, dump_to_file=True, overwrite_output_file=True)
13491337

13501338
path.unlink()
13511339

‎docs/gaia/gaia.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -914,7 +914,8 @@ The following example shows how to retrieve the DataLink products associated wit
914914
>>> datalink, file_path = Gaia.load_data(ids=[2263166706630078848, 2263178457660566784, 2268372099615724288],
915915
... data_release=data_release, retrieval_type=retrieval_type, data_structure=data_structure)
916916

917-
The DataLink products are stored inside a Python Dictionary. Each of its elements (keys) contains a one-element list that can be extracted as follows:
917+
The variable ``file_path`` contains the absolute path to the downloaded ZIP file when ``dump_to_file=True``.
918+
The variable ``datalink`` is a Python dictionary containing the DataLink products. Each key contains a one-element list, which can be extracted as follows:
918919

919920
.. code-block:: python
920921

0 commit comments

Comments
 (0)