tenacity.RetryError: RetryError[<Future at 0x7f0f60dc0d30 state=finished raised HTTPError>]
url = 'https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc'
local_path = 'PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc'
headers = {'Authorization': 'Bearer eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxn...dabNmCJplKiv8UZ_V3Zm05f1pUhqu2W9dM4eOOh74bzTdWX_4ehyL0BTmA62ucCaqr4SwFVd69MJoSyfH74bYAIJeEGCujbTOXcMreeOQGDwKQODHhxBg'}

    @retry(retry=retry_if_exception_type(requests.RequestException), wait=wait_exponential(min=3, max=60), stop=stop_after_attempt(10))
    def download_file(url, local_path, headers=None):
        try:
            with requests.get(url, stream=True, headers=headers) as response:
                response.raise_for_status()  # Check if the request was successful
                with open(local_path, 'wb') as file:
                    for chunk in response.iter_content(chunk_size=8192):
                        if chunk:  # Filter out keep-alive new chunks
                            file.write(chunk)
                logging.info(f"File downloaded successfully: {local_path}")
        except requests.RequestException as e:
            logging.error(f"Failed to download the file. Exception: {e}")
>           raise e  # Re-raise the exception to trigger the retry

tests/verify_collection.py:199: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/verify_collection.py:191: in download_file
    response.raise_for_status()  # Check if the request was successful
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Response [403]>

    def raise_for_status(self):
        """Raises :class:`HTTPError`, if one occurred."""
    
        http_error_msg = ""
        if isinstance(self.reason, bytes):
            # We attempt to decode utf-8 first because some servers
            # choose to localize their reason strings. If the string
            # isn't utf-8, we fall back to iso-8859-1 for all other
            # encodings. (See PR #3538)
            try:
                reason = self.reason.decode("utf-8")
            except UnicodeDecodeError:
                reason = self.reason.decode("iso-8859-1")
        else:
            reason = self.reason
    
        if 400 <= self.status_code < 500:
            http_error_msg = (
                f"{self.status_code} Client Error: {reason} for url: {self.url}"
            )
    
        elif 500 <= self.status_code < 600:
            http_error_msg = (
                f"{self.status_code} Server Error: {reason} for url: {self.url}"
            )
    
        if http_error_msg:
>           raise HTTPError(http_error_msg, response=self)
E           requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc

../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/requests/models.py:1024: HTTPError

The above exception was the direct cause of the following exception:

collection_concept_id = 'C1265136919-OB_CLOUD'
harmony_env = <Environment.UAT: 3>
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...IdabNmCJplKiv8UZ_V3Zm05f1pUhqu2W9dM4eOOh74bzTdWX_4ehyL0BTmA62ucCaqr4SwFVd69MJoSyfH74bYAIJeEGCujbTOXcMreeOQGDwKQODHhxBg'

    @pytest.mark.timeout(600)
    def test_concatenate(collection_concept_id, harmony_env, bearer_token):
    
        max_results = 2
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        collection = harmony.Collection(id=collection_concept_id)
        latest_granule_ids = get_latest_granules(collection_concept_id, max_results, harmony_env, bearer_token)
    
        if latest_granule_ids is None:
            if harmony_env == harmony.config.Environment.UAT:
                pytest.skip(f"No granules found for UAT collection {collection_concept_id}")
            raise Exception('Bad Request', 'Error: No matching granules found.')
    
        request = harmony.Request(
            collection=collection,
            concatenate=True,
            max_results=max_results,
            granule_id=latest_granule_ids,
            skip_preview=True,
            format="application/x-netcdf4",
        )
    
        request.is_valid()
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(request))
    
        try:
            job1_id = harmony_client.submit(request)
        except Exception as ex:
            if str(ex) == "('Bad Request', 'Error: No matching granules found.')":
                if harmony_env == harmony.config.Environment.UAT:
                    pytest.skip(f"No granules found for UAT collection {collection_concept_id}")
            raise ex
    
        logging.info(f'\n{job1_id}')
    
        logging.info(harmony_client.status(job1_id))
    
        logging.info('\nWaiting for the job to finish')
    
        results = harmony_client.result_json(job1_id)
    
        logging.info('\nDownloading results:')
    
        futures = harmony_client.download_all(job1_id)
        file_names = [f.result() for f in futures]
        logging.info('\nDone downloading.')
    
        filename = file_names[0]
    
        # Handle time dimension and variables dropping
        merge_dataset = netCDF4.Dataset(filename, 'r')
    
        headers = {
            "Authorization": f"Bearer {bearer_token}"
        }
    
        original_files = merge_dataset.variables['subset_files']
        history_json = json.loads(merge_dataset.history_json)
        assert len(original_files) == max_results
    
        for url in history_json[0].get("derived_from"):
            local_file_name = os.path.basename(url)
>           download_file(url, local_file_name, headers)

tests/verify_collection.py:293: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/tenacity/__init__.py:330: in wrapped_f
    return self(f, *args, **kw)
../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/tenacity/__init__.py:467: in __call__
    do = self.iter(retry_state=retry_state)
../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/tenacity/__init__.py:368: in iter
    result = action(retry_state)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

rs = <RetryCallState 139704026274240: attempt #10; slept for 246.0; last result: failed (HTTPError 403 Client Error: Forbid...: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc)>

    def exc_check(rs: "RetryCallState") -> None:
        fut = t.cast(Future, rs.outcome)
        retry_exc = self.retry_error_cls(fut)
        if self.reraise:
            raise retry_exc.reraise()
>       raise retry_exc from fut.exception()
E       tenacity.RetryError: RetryError[<Future at 0x7f0f60dc0d30 state=finished raised HTTPError>]

../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/tenacity/__init__.py:411: RetryError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:254 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1265136919-OB_CLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&granuleId=G1268937619-OB_CLOUD&granuleId=G1268937620-OB_CLOUD&format=application%2Fx-netcdf4&maxResults=2&concatenate=true&skipPreview=true
INFO     root:verify_collection.py:264 
20e746af-941b-45e7-8ee8-e53ad0a0fd07
INFO     root:verify_collection.py:266 {'status': 'running', 'message': 'The job is being processed', 'progress': 0, 'created_at': datetime.datetime(2024, 10, 18, 8, 30, 19, 924000, tzinfo=tzlocal()), 'updated_at': datetime.datetime(2024, 10, 18, 8, 30, 19, 924000, tzinfo=tzlocal()), 'created_at_local': '2024-10-18T08:30:19+00:00', 'updated_at_local': '2024-10-18T08:30:19+00:00', 'request': 'https://harmony.uat.earthdata.nasa.gov/C1265136919-OB_CLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&granuleId=G1268937619-OB_CLOUD%2CG1268937620-OB_CLOUD&format=application%2Fx-netcdf4&maxResults=2&concatenate=true&skipPreview=true', 'num_input_granules': 2, 'data_expiration': datetime.datetime(2024, 11, 17, 8, 30, 19, 924000, tzinfo=tzlocal()), 'data_expiration_local': '2024-11-17T08:30:19+00:00'}
INFO     root:verify_collection.py:268 
Waiting for the job to finish
INFO     root:verify_collection.py:272 
Downloading results:
INFO     root:verify_collection.py:276 
Done downloading.
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc
ERROR    root:verify_collection.py:198 Failed to download the file. Exception: 403 Client Error: Forbidden for url: https://obdaac-tea.uat.earthdatacloud.nasa.gov/ob-cumulus-uat-public/PACE_OCI.20240805T081250.L2.OC_BGC.V2_0.NRT.nc