feat: disabling S3 syncs on demand.

Signed-off-by: Ubuntu <drugilsberg@gmail.com>
GT4SD · Jul 24, 2023 · bd955fd · bd955fd
1 parent 4e28e17
commit bd955fd
Show file tree

Hide file tree

Showing 10 changed files with 8 additions and 36 deletions.
diff --git a/src/gt4sd/algorithms/conditional_generation/regression_transformer/implementation.py b/src/gt4sd/algorithms/conditional_generation/regression_transformer/implementation.py
@@ -401,7 +401,6 @@ def safely_determine_task(self, x: str) -> str:
             self.tokenizer.mask_token
             in x.split(self.tokenizer.expression_separator)[-1]
         ):
-
             return "generation"
 
         return "regression"

diff --git a/src/gt4sd/algorithms/core.py b/src/gt4sd/algorithms/core.py
@@ -759,17 +759,21 @@ def ensure_artifacts_for_version(cls, algorithm_version: str) -> str:
             cls.get_application_prefix(),
             algorithm_version,
         )
+        local_path = ""
         try:
-            local_path = sync_algorithm_with_s3(prefix)
+            if not gt4sd_configuration_instance.gt4sd_skip_s3_sync_in_inference:
+                local_path = sync_algorithm_with_s3(prefix)
         except (KeyError, S3SyncError) as error:
             logger.info(
                 f"searching S3 raised {error.__class__.__name__}, using local cache only."
             )
             logger.debug(error)
-            local_path = get_cached_algorithm_path(prefix)
+        finally:
+            if not local_path:
+                local_path = get_cached_algorithm_path(prefix)
             if not os.path.isdir(local_path):
                 raise OSError(
-                    f"artifacts directory {local_path} does not exist locally, and syncing with s3 failed: {error}"
+                    f"artifacts directory {local_path} does not exist locally"
                 )
 
         return local_path

diff --git a/src/gt4sd/cli/argument_parser.py b/src/gt4sd/cli/argument_parser.py
@@ -148,7 +148,6 @@ def _add_dataclass_arguments(self, dtype: DataClassType) -> None:
                 else:
                     kwargs["required"] = True
             elif field.type is bool or field.type == Optional[bool]:
-
                 if field.default is True:
                     parser.add_argument(
                         f"--no_{field.name}",

diff --git a/src/gt4sd/cli/load_arguments_from_dataclass.py b/src/gt4sd/cli/load_arguments_from_dataclass.py
@@ -76,28 +76,23 @@ def extract_fields_from_class(
 
     # assign default values
     for field in fields(dataclass):
-
         if not isinstance(field.default, _MISSING_TYPE):
-
             if field.default is None:
                 field.default = "none"
 
             arg_fields[field.name]["default"] = field.default
 
     # convert type to str
     for field_name in arg_fields:
-
         field_type = find_type(arg_fields[field_name]["type"])
 
         if field_type:
-
             arg_fields[field_name]["type"] = field_type
 
         elif (
             hasattr(arg_fields[field_name]["type"], "__origin__")
             and arg_fields[field_name]["type"].__origin__ is Union
         ):
-
             types = [
                 find_type(type) for type in arg_fields[field_name]["type"].__args__
             ]

diff --git a/src/gt4sd/configuration.py b/src/gt4sd/configuration.py
@@ -51,6 +51,7 @@ class GT4SDConfiguration(BaseSettings):
     gt4sd_max_runtime: int = 86400
     gt4sd_create_unverified_ssl_context: bool = False
     gt4sd_disable_cudnn: bool = False
+    gt4sd_skip_s3_sync_in_inference: bool = False
 
     gt4sd_s3_host: str = "s3.par01.cloud-object-storage.appdomain.cloud"
     gt4sd_s3_access_key: str = "6e9891531d724da89997575a65f4592e"
@@ -200,7 +201,6 @@ def sync_algorithm_with_s3(
 def get_cached_algorithm_path(
     prefix: Optional[str] = None, module: str = "algorithms"
 ) -> str:
-
     if module not in gt4sd_artifact_management_configuration.gt4sd_s3_modules:
         raise ValueError(
             f"Unknown cache module: {module}. Supported modules: "

diff --git a/src/gt4sd/training_pipelines/__init__.py b/src/gt4sd/training_pipelines/__init__.py
@@ -77,7 +77,6 @@ def training_pipeline_name_to_metadata(name: str) -> Dict[str, Any]:
 
     metadata: Dict[str, Any] = {"training_pipeline": name, "parameters": {}}
     if name in TRAINING_PIPELINE_ARGUMENTS_MAPPING:
-
         for training_argument_class in TRAINING_PIPELINE_ARGUMENTS_MAPPING[name]:
             field_types = extract_fields_from_class(training_argument_class)
             metadata["parameters"].update(field_types)

diff --git a/src/gt4sd/training_pipelines/regression_transformer/implementation.py b/src/gt4sd/training_pipelines/regression_transformer/implementation.py
@@ -78,7 +78,6 @@ def train(  # type: ignore
             dataset_args: dataset arguments passed to the configuration.
         """
         try:
-
             params = {**training_args, **dataset_args, **model_args}
             # Setup logging
             logging.basicConfig(

diff --git a/src/gt4sd/training_pipelines/regression_transformer/utils.py b/src/gt4sd/training_pipelines/regression_transformer/utils.py
@@ -164,7 +164,6 @@ def prepare_datasets_from_files(
     for i, (data, path) in enumerate(
         zip([train_data, test_data], [train_path, test_path])
     ):
-
         if not path.endswith(".csv"):
             raise TypeError(f"Please provide a csv file not {path}.")
 

diff --git a/src/gt4sd/training_pipelines/tests/test_argument_parser.py b/src/gt4sd/training_pipelines/tests/test_argument_parser.py
@@ -31,7 +31,6 @@
 
 @dataclass
 class TestArguments:
-
     int_arg: int = field(default=0)
 
     float_arg: float = field(default=0.0)
@@ -50,7 +49,6 @@ class TestArguments:
 
 
 def test_int_default():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -60,7 +58,6 @@ def test_int_default():
 
 
 def test_float_default():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -70,7 +67,6 @@ def test_float_default():
 
 
 def test_str_default():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -80,7 +76,6 @@ def test_str_default():
 
 
 def test_bool_default():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -90,7 +85,6 @@ def test_bool_default():
 
 
 def test_int_assigned():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--int_arg", "1"])
@@ -100,7 +94,6 @@ def test_int_assigned():
 
 
 def test_float_assigned():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--float_arg", "1.0"])
@@ -110,7 +103,6 @@ def test_float_assigned():
 
 
 def test_str_assigned():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--str_arg", "my_test"])
@@ -120,7 +112,6 @@ def test_str_assigned():
 
 
 def test_bool_assigned():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--bool_arg", "False"])
@@ -130,7 +121,6 @@ def test_bool_assigned():
 
 
 def test_bool_int_assigned():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--bool_arg", "0"])
@@ -140,7 +130,6 @@ def test_bool_int_assigned():
 
 
 def test_int_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -149,7 +138,6 @@ def test_int_none():
 
 
 def test_float_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -158,7 +146,6 @@ def test_float_none():
 
 
 def test_str_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -167,7 +154,6 @@ def test_str_none():
 
 
 def test_bool_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses([])
@@ -176,7 +162,6 @@ def test_bool_none():
 
 
 def test_int_str_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--int_none_arg", ""])
@@ -185,7 +170,6 @@ def test_int_str_none():
 
 
 def test_float_str_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--float_none_arg", ""])
@@ -194,7 +178,6 @@ def test_float_str_none():
 
 
 def test_str_str_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--str_none_arg", ""])
@@ -203,7 +186,6 @@ def test_str_str_none():
 
 
 def test_bool_str_none():
-
     parser = ArgumentParser((TestArguments))  # type: ignore
 
     args = parser.parse_args_into_dataclasses(["--bool_none_arg", ""])

diff --git a/src/gt4sd/training_pipelines/tests/test_training_regression_transformer.py b/src/gt4sd/training_pipelines/tests/test_training_regression_transformer.py
@@ -98,7 +98,6 @@
 def combine_defaults_and_user_args(
     config: Dict[str, Dict[str, Any]]
 ) -> Dict[str, Dict[str, Any]]:
-
     arguments = TRAINING_PIPELINE_ARGUMENTS_MAPPING["regression-transformer-trainer"]
     """
     We need `conflict_handler='resolve'` because the RT relies on the TrainingArguments
@@ -145,7 +144,6 @@ def combine_defaults_and_user_args(
 
 
 def test_train():
-
     pipeline = TRAINING_PIPELINE_MAPPING.get("regression-transformer-trainer")
     assert pipeline is not None
 
@@ -161,7 +159,6 @@ def test_train():
         importlib_resources.files("gt4sd")
         / "training_pipelines/tests/regression_transformer_raw.csv"
     ) as raw_path:
-
         # Test finetuning the QED model
         config["model_args"]["model_path"] = mol_path
         config["dataset_args"]["train_data_path"] = str(raw_path)
@@ -200,7 +197,6 @@ def test_train():
         importlib_resources.files("gt4sd")
         / "training_pipelines/tests/regression_transformer_copolymer_raw.csv"
     ) as raw_path:
-
         # Test finetuning the polymer model
         polymer_model = RegressionTransformerMolecules(
             algorithm_version="block_copolymer"