Skip to content

Commit

Permalink
Merge pull request #232 from asfadmin/feature-dataset-search
Browse files Browse the repository at this point in the history 
`Dataset` search keyword
  • Loading branch information
@SpicyGarlicAlbacoreRoll
SpicyGarlicAlbacoreRoll authored Nov 8, 2023
2 parents 75d4bfd + 12a3172 commit 70b6238
Show file tree
Hide file tree
Showing 13 changed files with 409 additions and 15 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-->
------
## [v6.6.4](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.6.4)
### Added
- Adds new `dataset` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page.

### Changed
- Changes `CMR_FORMAT_EXT` constant from `umm_json_v1_4` to `umm_json`, umm returned from CMR will now be in latest umm format by default

### Fixed
- ERS-1, ERS-2, JERS-1, and RADARSAT-1 now assign `FRAME_NUMBER` to the `frameNumber` properties field

Expand Down
11 changes: 6 additions & 5 deletions asf_search/ASFSearchOptions/validator_map.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,13 @@ def validate(key, value):
'instrument': parse_string,
'collections': parse_string_list,
'temporalBaselineDays': parse_string_list,

'dataset': parse_string_list,
'absoluteBurstID': parse_int_list,
'relativeBurstID': parse_int_list,
'fullBurstID': parse_string_list,

# Config parameters Parser
'session': parse_session,
'host': parse_string,
'provider': parse_string,
'absoluteBurstID': parse_int_list,
'relativeBurstID': parse_int_list,
'fullBurstID': parse_string_list
'provider': parse_string
}
3 changes: 2 additions & 1 deletion asf_search/CMR/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from .MissionList import get_campaigns
from .subquery import build_subqueries
from .translate import translate_product, translate_opts, get_additional_fields
from .field_map import field_map
from .field_map import field_map
from .datasets import dataset_collections
327 changes: 327 additions & 0 deletions asf_search/CMR/datasets.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,327 @@
dataset_collections = {
"SENTINEL-1": {
"SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"],
"SENTINEL-1B_SLC": ["C1327985661-ASF", "C1216244348-ASF", "C1234413263-ASFDEV"],
"SENTINEL-1A_DP_GRD_HIGH": [
"C1214470533-ASF",
"C1212201032-ASF",
"C1234413229-ASFDEV",
],
"SENTINEL-1A_DP_META_GRD_HIGH": [
"C1214470576-ASF",
"C1212209226-ASF",
"C1234413232-ASFDEV",
],
"SENTINEL-1B_DP_GRD_HIGH": [
"C1327985645-ASF",
"C1216244589-ASF",
"C1234413247-ASFDEV",
],
"SENTINEL-1A_META_SLC": [
"C1214470496-ASF",
"C1208117434-ASF",
"C1234413236-ASFDEV",
],
"SENTINEL-1A_META_RAW": [
"C1214470532-ASF",
"C1208115009-ASF",
"C1234413235-ASFDEV",
],
"SENTINEL-1A_OCN": ["C1214472977-ASF", "C1212212560-ASF", "C1234413237-ASFDEV"],
"SENTINEL-1A_DP_META_GRD_MEDIUM": [
"C1214472336-ASF",
"C1212212493-ASF",
"C1234413233-ASFDEV",
],
"SENTINEL-1A_META_OCN": [
"C1266376001-ASF",
"C1215704763-ASF",
"C1234413234-ASFDEV",
],
"SENTINEL-1A_SP_META_GRD_HIGH": [
"C1214470732-ASF",
"C1212158326-ASF",
"C1234413243-ASFDEV",
],
"SENTINEL-1B_DP_GRD_MEDIUM": [
"C1327985660-ASF",
"C1216244594-ASF",
"C1234413248-ASFDEV",
],
"SENTINEL-1B_DP_META_GRD_HIGH": [
"C1327985741-ASF",
"C1216244601-ASF",
"C1234413250-ASFDEV",
],
"SENTINEL-1B_DP_META_GRD_MEDIUM": [
"C1327985578-ASF",
"C1216244591-ASF",
"C1234413251-ASFDEV",
],
"SENTINEL-1B_META_RAW": [
"C1327985650-ASF",
"C1216244595-ASF",
"C1234413253-ASFDEV",
],
"SENTINEL-1B_META_SLC": [
"C1327985617-ASF",
"C1216244585-ASF",
"C1234413254-ASFDEV",
],
"SENTINEL-1B_OCN": ["C1327985579-ASF", "C1216244593-ASF", "C1234413255-ASFDEV"],
"SENTINEL-1B_SP_META_GRD_HIGH": [
"C1327985619-ASF",
"C1216244587-ASF",
"C1234413261-ASFDEV",
],
"SENTINEL-1A_SP_GRD_MEDIUM": [
"C1214472994-ASF",
"C1212158318-ASF",
"C1234413241-ASFDEV",
],
"SENTINEL-1A_SP_META_GRD_MEDIUM": [
"C1214473170-ASF",
"C1212233976-ASF",
"C1234413244-ASFDEV",
],
"SENTINEL-1B_META_OCN": [
"C1327985646-ASF",
"C1216244590-ASF",
"C1234413252-ASFDEV",
],
"SENTINEL-1B_SP_GRD_MEDIUM": [
"C1327985740-ASF",
"C1216244600-ASF",
"C1234413259-ASFDEV",
],
"SENTINEL-1B_SP_META_GRD_MEDIUM": [
"C1327985739-ASF",
"C1216244598-ASF",
"C1234413262-ASFDEV",
],
"SENTINEL-1A_RAW": ["C1214470561-ASF", "C1205264459-ASF", "C1234413238-ASFDEV"],
"SENTINEL-1A_DP_GRD_MEDIUM": [
"C1214471521-ASF",
"C1212209035-ASF",
"C1234413230-ASFDEV",
],
"SENTINEL-1A_SP_GRD_HIGH": [
"C1214470682-ASF",
"C1212158327-ASF",
"C1234413240-ASFDEV",
],
"SENTINEL-1B_RAW": ["C1327985647-ASF", "C1216244592-ASF", "C1234413256-ASFDEV"],
"SENTINEL-1A_DP_GRD_FULL": [
"C1214471197-ASF",
"C1212200781-ASF",
"C1234413228-ASFDEV",
],
"SENTINEL-1A_DP_META_GRD_FULL": [
"C1214471960-ASF",
"C1212209075-ASF",
"C1234413231-ASFDEV",
],
"SENTINEL-1A_SP_GRD_FULL": ["C1214472978-ASF", "C1234413239-ASFDEV"],
"SENTINEL-1A_SP_META_GRD_FULL": ["C1214473165-ASF", "C1234413242-ASFDEV"],
"SENTINEL-1B_DP_GRD_FULL": [
"C1327985697-ASF",
"C1216244597-ASF",
"C1234413246-ASFDEV",
],
"SENTINEL-1B_DP_META_GRD_FULL": [
"C1327985651-ASF",
"C1216244596-ASF",
"C1234413249-ASFDEV",
],
"SENTINEL-1B_SP_GRD_FULL": [
"C1327985644-ASF",
"C1216244588-ASF",
"C1234413257-ASFDEV",
],
"SENTINEL-1B_SP_GRD_HIGH": [
"C1327985571-ASF",
"C1216244586-ASF",
"C1234413258-ASFDEV",
],
"SENTINEL-1B_SP_META_GRD_FULL": [
"C1327985674-ASF",
"C1216244599-ASF",
"C1234413260-ASFDEV",
],
"S1_Bursts": ["C1244552887-ASFDEV"],
"SENTINEL-1_BURSTS_DEV10": ["C1257175154-ASFDEV"],
"Sentinel-1_Burst_Map": ["C1244598379-ASFDEV"],
"Various Browse Images": ["C1240784657-ASFDEV"],
},
"OPERA-S1": {
"OPERA_L2_CSLC-S1_V1": ["C2777443834-ASF", "C1259976861-ASF"],
"OPERA_L2_RTC-S1_V1": ["C2777436413-ASF", "C1259974840-ASF"],
"OPERA_L2_CSLC-S1-STATIC_PROVISIONAL_V0": ["C1258354200-ASF"],
"OPERA_L2_CSLC-S1-STATIC_V1": ["C1259982010-ASF"],
"OPERA_L2_CSLC-S1_PROVISIONAL_V0": ["C1257995185-ASF"],
"OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"],
"OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF"],
"OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"],
"OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF"],
"OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF"],
"OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF"],
"OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF"],
},
"SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]},
"ALOS PALSAR": {
"ALOS_PSR_RTC_HIGH": ["C1206487504-ASF", "C1207181535-ASF"],
"ALOS_PSR_L1.5": ["C1206485940-ASF", "C1205261223-ASF"],
"ALOS_PSR_RTC_LOW": ["C1206487217-ASF", "C1208013295-ASF"],
"ALOS_PSR_KMZ": ["C1206156901-ASF", "C1207019609-ASF"],
"ALOS_PSR_L1.0": ["C1206485320-ASF"],
"ALOS_PSR_L1.1": ["C1206485527-ASF", "C1207710476-ASF", "C1239611505-ASFDEV"],
"ALOS_PSR_L2.2": ["C2011599335-ASF", "C1239927797-ASF", "C1238733834-ASFDEV"],
"ALOS_PALSAR_INSAR_METADATA": ["C1229740239-ASF"],
},
"ALOS AVNIR-2": {
"ALOS_AVNIR_OBS_ORI": [
"C1808440897-ASF",
"C1233629671-ASF",
"C1234413224-ASFDEV",
],
"ALOS_AVNIR_OBS_ORI_BROWSE": ["C1234712303-ASF"],
},
"SIR-C": {
"STS-59_BROWSE_GRD": [
"C1661710578-ASF",
"C1226557819-ASF",
"C1234413264-ASFDEV",
],
"STS-59_BROWSE_SLC": [
"C1661710581-ASF",
"C1226557809-ASF",
"C1234413265-ASFDEV",
],
"STS-59_GRD": ["C1661710583-ASF", "C1226557808-ASF", "C1234413266-ASFDEV"],
"STS-59_META_GRD": ["C1661710586-ASF", "C1226557810-ASF", "C1234413267-ASFDEV"],
"STS-59_META_SLC": ["C1661710588-ASF", "C1226557811-ASF", "C1234413268-ASFDEV"],
"STS-59_SLC": ["C1661710590-ASF", "C1226557812-ASF", "C1234413269-ASFDEV"],
"STS-68_BROWSE_GRD": [
"C1661710593-ASF",
"C1226557813-ASF",
"C1234413270-ASFDEV",
],
"STS-68_BROWSE_SLC": [
"C1661710596-ASF",
"C1226557814-ASF",
"C1234413271-ASFDEV",
],
"STS-68_GRD": ["C1661710597-ASF", "C1226557815-ASF", "C1234413272-ASFDEV"],
"STS-68_META_GRD": ["C1661710600-ASF", "C1226557816-ASF", "C1234413273-ASFDEV"],
"STS-68_META_SLC": ["C1661710603-ASF", "C1226557817-ASF", "C1234413274-ASFDEV"],
"STS-68_SLC": ["C1661710604-ASF", "C1226557818-ASF", "C1234413275-ASFDEV"],
},
"ARIA S1 GUNW": {
"SENTINEL-1_INTERFEROGRAMS": ["C1595422627-ASF", "C1225776654-ASF"],
"SENTINEL-1_INTERFEROGRAMS_AMPLITUDE": ["C1596065640-ASF", "C1225776655-ASF"],
"SENTINEL-1_INTERFEROGRAMS_COHERENCE": ["C1596065639-ASF", "C1225776657-ASF"],
"SENTINEL-1_INTERFEROGRAMS_CONNECTED_COMPONENTS": [
"C1596065641-ASF",
"C1225776658-ASF",
],
"SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [
"C1595765183-ASF",
"C1225776659-ASF",
],
},
"SMAP": {
"SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"],
"SPL1A_RO_QA_003": ["C1243124139-ASF", "C1216074923-ASF"],
"SPL1A_001": ["C1214473171-ASF", "C1212243761-ASF"],
"SPL1A_002": ["C1243149604-ASF", "C1213091807-ASF"],
"SPL1A_METADATA_001": ["C1214473426-ASF", "C1212243437-ASF"],
"SPL1A_METADATA_002": ["C1243119801-ASF", "C1213096699-ASF"],
"SPL1A_QA_001": ["C1214473839-ASF", "C1212249653-ASF"],
"SPL1A_QA_002": ["C1243133204-ASF", "C1213101573-ASF"],
"SPL1A_RO_001": ["C1243197402-ASF"],
"SPL1A_RO_002": ["C1243215430-ASF", "C1213136240-ASF"],
"SPL1A_RO_003": ["C1243124754-ASF", "C1216074755-ASF"],
"SPL1A_RO_METADATA_001": ["C1243141638-ASF", "C1213136752-ASF"],
"SPL1A_RO_METADATA_002": ["C1243162394-ASF", "C1213136799-ASF"],
"SPL1A_RO_QA_001": ["C1243168733-ASF", "C1213136709-ASF"],
"SPL1A_RO_QA_002": ["C1243168866-ASF", "C1213136844-ASF"],
"SPL1B_SO_LoRes_001": ["C1214473308-ASF", "C1212249811-ASF"],
"SPL1B_SO_LoRes_002": ["C1243253631-ASF", "C1213125007-ASF"],
"SPL1B_SO_LoRes_003": ["C1243133445-ASF", "C1216074919-ASF"],
"SPL1B_SO_LoRes_METADATA_001": ["C1214473550-ASF", "C1212196951-ASF"],
"SPL1B_SO_LoRes_METADATA_002": ["C1243197502-ASF", "C1213115690-ASF"],
"SPL1B_SO_LoRes_METADATA_003": ["C1243126328-ASF", "C1216074758-ASF"],
"SPL1B_SO_LoRes_QA_001": ["C1214474243-ASF", "C1212243666-ASF"],
"SPL1B_SO_LoRes_QA_002": ["C1243216659-ASF", "C1213115896-ASF"],
"SPL1B_SO_LoRes_QA_003": ["C1243129847-ASF", "C1216074761-ASF"],
"SPL1C_S0_HiRes_001": ["C1214473367-ASF", "C1212250364-ASF"],
"SPL1C_S0_HiRes_002": ["C1243268956-ASF", "C1213134622-ASF"],
"SPL1C_S0_HiRes_003": ["C1243144528-ASF", "C1216074770-ASF"],
"SPL1C_S0_HiRes_METADATA_001": ["C1214473624-ASF", "C1212246173-ASF"],
"SPL1C_S0_HiRes_METADATA_002": ["C1243228612-ASF", "C1213125156-ASF"],
"SPL1C_S0_HiRes_METADATA_003": ["C1243136142-ASF", "C1216074764-ASF"],
"SPL1C_S0_HiRes_QA_001": ["C1214474435-ASF", "C1212249773-ASF"],
"SPL1C_S0_HiRes_QA_002": ["C1243255360-ASF", "C1213134486-ASF"],
"SPL1C_S0_HiRes_QA_003": ["C1243140611-ASF", "C1233101609-ASF"],
"SPL1A_003": ["C1216074922-ASF"],
"SPL1A_METADATA_003": ["C1216074750-ASF"],
"SPL1A_QA_003": ["C1216074751-ASF"],
},
"UAVSAR": {
"UAVSAR_POL_META": ["C1214353986-ASF", "C1210487703-ASF"],
"UAVSAR_INSAR_META": ["C1214336717-ASF", "C1212030772-ASF"],
"UAVSAR_INSAR_INT": ["C1214336045-ASF", "C1212001698-ASF"],
"UAVSAR_INSAR_AMP": ["C1214335430-ASF", "C1206116665-ASF"],
"UAVSAR_INSAR_AMP_GRD": ["C1214335471-ASF", "C1206132445-ASF"],
"UAVSAR_INSAR_DEM": ["C1214335903-ASF", "C1211962154-ASF"],
"UAVSAR_INSAR_INT_GRD": ["C1214336154-ASF", "C1212005594-ASF"],
"UAVSAR_INSAR_KMZ": ["C1214336554-ASF", "C1212019993-ASF"],
"UAVSAR_POL_DEM": ["C1214353593-ASF", "C1207638502-ASF"],
"UAVSAR_POL_INC": ["C1214353754-ASF", "C1210025872-ASF"],
"UAVSAR_POL_KMZ": ["C1214353859-ASF", "C1210485039-ASF"],
"UAVSAR_POL_ML_CMPLX_GRD": ["C1214337770-ASF", "C1207188317-ASF"],
"UAVSAR_POL_ML_CMPLX_GRD_3X3": ["C1214354144-ASF", "C1210546638-ASF"],
"UAVSAR_POL_ML_CMPLX_GRD_5X5": ["C1214354235-ASF", "C1206122195-ASF"],
"UAVSAR_POL_ML_CMPLX_SLANT": ["C1214343609-ASF", "C1209970710-ASF"],
"UAVSAR_POL_PAULI": ["C1214354031-ASF", "C1207038647-ASF"],
"UAVSAR_POL_SLOPE": ["C1214408428-ASF", "C1210599503-ASF"],
"UAVSAR_POL_STOKES": ["C1214419355-ASF", "C1210599673-ASF"],
},
"RADARSAT-1": {
"RSAT-1_L0": ["C1206897141-ASF"],
"RSAT-1_L1": ["C1206936391-ASF", "C1205181982-ASF"],
"RSAT-1_POLAR_YEAR_ANTARCTICA_L1": ["C1215670813-ASF"],
"RSAT-1_POLAR_YEAR_GREENLAND_L0": ["C1215709884-ASF"],
"RSAT-1_POLAR_YEAR_GREENLAND_L1": ["C1215709880-ASF"],
"RSAT-1_POLAR_YEAR_KAMCHATKA_L1": ["C1215714443-ASF"],
"RSAT-1_POLAR_YEAR_SEA_ICE_MIN_MAX_L1": ["C1215775284-ASF"],
"RSAT-1_POLAR_YEAR_TOOLIK_L1": ["C1215614037-ASF"],
},
"ERS": {
"ERS-1_L0": ["C1210197768-ASF", "C1205261222-ASF"],
"ERS-1_L1": ["C1211627521-ASF", "C1205302527-ASF"],
"ERS-2_L0": ["C1208794942-ASF", "C1207143701-ASF"],
"ERS-2_L1": ["C1209373626-ASF", "C1207144966-ASF"],
},
"JERS-1": {
"JERS-1_L0": ["C1208662092-ASF", "C1207175327-ASF"],
"JERS-1_L1": ["C1207933168-ASF", "C1207177736-ASF"],
},
"AIRSAR": {
"AIRSAR_POL_3FP": ["C1213921661-ASF", "C1205256880-ASF"],
"AIRSAR_INT_JPG": ["C1213921626-ASF", "C1000000306-ASF"],
"AIRSAR_POL_SYN_3FP": ["C1213928843-ASF", "C1208713702-ASF"],
"AIRSAR_TOP_C-DEM_STOKES": ["C1213927035-ASF", "C1208707768-ASF"],
"AIRSAR_TOP_DEM": ["C179001730-ASF", "C1208655639-ASF"],
"AIRSAR_TOP_DEM_C": ["C1213925022-ASF", "C1208680681-ASF"],
"AIRSAR_TOP_DEM_L": ["C1213926419-ASF", "C1208691361-ASF"],
"AIRSAR_TOP_DEM_P": ["C1213926777-ASF", "C1208703384-ASF"],
"AIRSAR_TOP_L-STOKES": ["C1213927939-ASF"],
"AIRSAR_TOP_P-STOKES": ["C1213928209-ASF"],
"AIRSAR_INT": ["C1208652494-ASF"],
},
"SEASAT": {
"SEASAT_SAR_L1_TIFF": ["C1206500826-ASF", "C1206752770-ASF"],
"SEASAT_SAR_L1_HDF5": ["C1206500991-ASF", "C1206144699-ASF"],
},
}
2 changes: 1 addition & 1 deletion asf_search/CMR/subquery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]:
if params.get('product_list') is not None:
params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE)

list_param_names = ['platform', 'season', 'collections'] # these parameters will dodge the subquery system
list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system
skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL

params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ])
Expand Down
Loading

0 comments on commit 70b6238

Please sign in to comment.