diff --git a/cads_adaptors/tools/url_tools.py b/cads_adaptors/tools/url_tools.py index 77e4a25e..8b034d7c 100644 --- a/cads_adaptors/tools/url_tools.py +++ b/cads_adaptors/tools/url_tools.py @@ -37,7 +37,9 @@ def requests_to_urls( yield {"url": url, "req": req} -def try_download(urls: List[str], context: Context, **kwargs) -> List[str]: +def try_download( + urls: List[str], context: Context, server_suggested_filename=False, **kwargs +) -> List[str]: # Ensure that URLs are unique to prevent downloading the same file multiple times urls = sorted(set(urls)) @@ -48,6 +50,9 @@ def try_download(urls: List[str], context: Context, **kwargs) -> List[str]: kwargs = {"timeout": 3, "maximum_retries": 1, "retry_after": 1, **kwargs} for url in urls: path = urllib.parse.urlparse(url).path.lstrip("/") + if server_suggested_filename: + path = os.path.join(os.path.dirname(path), multiurl.Downloader(url).title()) + dir = os.path.dirname(path) if dir: os.makedirs(dir, exist_ok=True) diff --git a/tests/test_20_url_tools.py b/tests/test_20_url_tools.py index 4753b396..74beeba2 100644 --- a/tests/test_20_url_tools.py +++ b/tests/test_20_url_tools.py @@ -38,6 +38,27 @@ def test_downloaders(tmp_path, monkeypatch, urls, expected_nfiles): assert len(paths) == expected_nfiles +def test_download_with_server_suggested_filename(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) # try_download generates files in the working dir + urls = ["https://gerb.oma.be/c3s/data/ceres-ebaf/tcdr/v4.2/toa_lw_all_mon/2000/07"] + paths_false = url_tools.try_download( + urls, context=url_tools.Context(), server_suggested_filename=False + ) + assert len(paths_false) == 1 + assert os.path.basename(paths_false[0]) == "07" + + paths_true = url_tools.try_download( + urls, context=url_tools.Context(), server_suggested_filename=True + ) + assert len(paths_true) == 1 + assert ( + os.path.basename(paths_true[0]) + == "data_312a_Lot1_ceres-ebaf_tcdr_v4.2_toa_lw_all_mon_2000_07.nc" + ) + + assert os.path.dirname(paths_false[0]) == os.path.dirname(paths_true[0]) + + @pytest.mark.parametrize( "anon", (