🐛 CDK: fix deletion of request cache files (#30925)

Co-authored-by: Eugene Kulak <kulak.eugene@gmail.com>

airbyte-cdk/python/airbyte_cdk/entrypoint.py

@@ -23,6 +23,7 @@
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
 from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
+from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from requests import PreparedRequest, Response, Session
 
@@ -94,7 +95,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
         source_spec: ConnectorSpecification = self.source.spec(self.logger)
         try:
             with tempfile.TemporaryDirectory() as temp_dir:
-                os.environ["TMPDIR"] = temp_dir  # set this as default temp directory (used by requests_cache to store *.sqlite files)
+                os.environ[ENV_REQUEST_CACHE_PATH] = temp_dir  # set this as default directory for request_cache to store *.sqlite files
                 if cmd == "spec":
                     message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
                     yield from [

airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py

@@ -4,8 +4,10 @@
 
 
 import logging
+import os
 import urllib
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
 from urllib.parse import urljoin
 
@@ -15,9 +17,10 @@
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
 from airbyte_cdk.sources.streams.core import Stream, StreamData
 from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+from airbyte_cdk.sources.utils.types import JsonType
+from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from requests.auth import AuthBase
 
-from ...utils.types import JsonType
 from .auth.core import HttpAuthenticator, NoAuth
 from .exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException
 from .rate_limiting import default_backoff_handler, user_defined_backoff_handler
@@ -62,8 +65,8 @@ def use_cache(self) -> bool:
         return False
 
     def request_cache(self) -> requests.Session:
-        backend = requests_cache.SQLiteCache(use_temp=True)
-        return requests_cache.CachedSession(self.cache_filename, backend=backend)
+        cache_dir = Path(os.getenv(ENV_REQUEST_CACHE_PATH))
+        return requests_cache.CachedSession(str(cache_dir / self.cache_filename), backend="sqlite")
 
     def clear_cache(self) -> None:
         """

airbyte-cdk/python/airbyte_cdk/utils/constants.py

@@ -0,0 +1,5 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH"