From 42b98d2968c5e0ede09d1eb88e12e0f8f2cdeb66 Mon Sep 17 00:00:00 2001 From: Philip Yoon Date: Thu, 22 Aug 2024 08:37:31 -0700 Subject: [PATCH] #961: For DISP-S1 historical download works differently than other modes when downloading for k granules --- data_subscriber/asf_cslc_download.py | 33 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/data_subscriber/asf_cslc_download.py b/data_subscriber/asf_cslc_download.py index 5c4495e7..8974b789 100644 --- a/data_subscriber/asf_cslc_download.py +++ b/data_subscriber/asf_cslc_download.py @@ -288,19 +288,30 @@ def get_downloads(self, args, es_conn): all_downloads = [] - # Download CSLC granules - downloads = es_conn.get_download_granule_revision(batch_ids[-1]) - logger.info(f"Got {len(downloads)=} cslc granules downloads for batch_id={batch_ids[-1]}") - assert len(downloads) > 0, f"No downloads found for batch_id={batch_ids[-1]}!" - all_downloads.extend(downloads) - - # Download K-CSLC granules - for batch_id in batch_ids[:-1]: - downloads = k_es_conn.get_download_granule_revision(batch_id) - logger.info(f"Got {len(downloads)=} k cslc downloads for {batch_id=}") - assert len(downloads) > 0, f"No downloads found for batch_id={batch_id}!" + # Historical mode stores all granules in normal cslc_catalog + if "proc_mode" in args and args.proc_mode == "historical": + logger.info("Downloading cslc files for historical mode") + for batch_id in batch_ids: + downloads = es_conn.get_download_granule_revision(batch_id) + logger.info(f"Got {len(downloads)=} cslc downloads for {batch_id=}") + assert len(downloads) > 0, f"No downloads found for batch_id={batch_id}!" + all_downloads.extend(downloads) + + # Forward and reprocessing modes store all granules in k_cslc_catalog + else: + logger.info("Downloading cslc files for forward/reprocessing mode") + downloads = es_conn.get_download_granule_revision(batch_ids[-1]) + logger.info(f"Got {len(downloads)=} cslc granules downloads for batch_id={batch_ids[-1]}") + assert len(downloads) > 0, f"No downloads found for batch_id={batch_ids[-1]}!" all_downloads.extend(downloads) + # Download K-CSLC granules + for batch_id in batch_ids[:-1]: + downloads = k_es_conn.get_download_granule_revision(batch_id) + logger.info(f"Got {len(downloads)=} k cslc downloads for {batch_id=}") + assert len(downloads) > 0, f"No downloads found for batch_id={batch_id}!" + all_downloads.extend(downloads) + return all_downloads def query_cslc_static_files_for_cslc_batch(self, cslc_files, args, token, job_id, settings):