From 3bc6f002181ff2c4d49c597bfeec2fa5f09fe363 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 16 Nov 2023 14:04:41 -0600 Subject: [PATCH 01/68] Adding logger.propagate to test resolving of duplicate log output --- integration_test/workflow_tests/custom_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_test/workflow_tests/custom_logger.py b/integration_test/workflow_tests/custom_logger.py index 49fa9a0e0..17aa9de66 100644 --- a/integration_test/workflow_tests/custom_logger.py +++ b/integration_test/workflow_tests/custom_logger.py @@ -14,4 +14,5 @@ def set_logger(group_name): syslog = logging.StreamHandler() logger.addHandler(syslog) logger_adapter = CustomLoggerAdapter(logger, {'my_context': group_name}) + logger.propagate = False return logger_adapter From b5841296f7915b3506cafda56e81157017f23625 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Wed, 22 Nov 2023 08:54:12 -0600 Subject: [PATCH 02/68] added instructions for running integration tests locally --- integration_test/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 integration_test/README.md diff --git a/integration_test/README.md b/integration_test/README.md new file mode 100644 index 000000000..caaa84bb6 --- /dev/null +++ b/integration_test/README.md @@ -0,0 +1,33 @@ + +### Running integration tests locally + +The steps to run ORCA integration tests locally are shown below: + +1. [Deploy ORCA to AWS](https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-with-cumulus). +2. Connect to the NASA VPN. +3. Set the following environment variables: + 1. `orca_API_DEPLOYMENT_INVOKE_URL` Output from the ORCA TF module. ex: `https://0000000000.execute-api.us-west-2.amazonaws.com` + 2. `orca_RECOVERY_STEP_FUNCTION_ARN` ARN of the recovery step function. ex: `arn:aws:states:us-west-2:000000000000:stateMachine:PREFIX-OrcaRecoveryWorkflow` + 3. `orca_COPY_TO_ARCHIVE_STEP_FUNCTION_ARN` ARN of the copy_to_archive step function. ex: `arn:aws:states:us-west-2:000000000000:stateMachine:PREFIX-OrcaCopyToArchiveWorkflow` + 4. `orca_RECOVERY_BUCKET_NAME` S3 bucket name where the recovered files will be archived. ex: `test-orca-primary` + 5. `orca_BUCKETS`The list of ORCA buckets used. ex: + ```json + '{"protected": {"name": "'$PREFIX'-protected", "type": "protected"}, "internal": {"name": "'$PREFIX'-internal", "type": "internal"}, "private": {"name": "'$PREFIX'-private", "type": "private"}, "public": {"name": "'$PREFIX'-public", "type": "public"}, "orca_default": {"name": "'$PREFIX'-orca-primary", "type": "orca"}, "provider": {"name": "orca-sandbox-s3-provider", "type": "provider"}}' + ``` + +4. + Get your Cumulus EC2 instance ID using the following AWS CLI command using your ``. + ```shell + aws ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:Name,Values={PREFIX}-CumulusECSCluster --query "Reservations[*].Instances[*].InstanceId" --output text + ``` + Then run the following bash command, + replacing `i-00000000000000000` with your `PREFIX-CumulusECSCluster` ec2 instance ID, + and `0000000000.execute-api.us-west-2.amazonaws.com` with your API Gateway identifier: + + ```shell + aws ssm start-session --target i-00000000000000000 --document-name AWS-StartPortForwardingSessionToRemoteHost --parameters '{"host":["0000000000.execute-api.us-west-2.amazonaws.com"],"portNumber":["443"], "localPortNumber":["8000"]}' + ``` +5. In the root folder `workflow_tests`, run the following command: + ```shell + bin/run_tests.sh + ``` \ No newline at end of file From 49830cf90219cea21d61842113e3baa84f5c4055 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Wed, 29 Nov 2023 10:25:50 -0600 Subject: [PATCH 03/68] removed hardcoded values from tests --- CHANGELOG.md | 2 + .../test_extract_file_paths_for_granule.py | 128 +++++++++--------- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a7151d0d..21ac48831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and includes an additional section for migration notes. ### Changed +- *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. + ### Deprecated ### Removed diff --git a/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py b/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py index 12168b593..5fba32a8a 100644 --- a/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py +++ b/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py @@ -27,8 +27,16 @@ class TestExtractFilePaths(unittest.TestCase): """ def setUp(self): - # todo: Remove hardcoded and imported values. self.task_input_event = create_task_event() + self.granuleId = uuid.uuid4().__str__() + self.collectionId = uuid.uuid4().__str__() + self.input_bucket = "cumulus-test-sandbox-protected-2" + self.protected_bucket = "sndbx-cumulus-protected" + self.internal_bucket = "sndbx-cumulus-internal" + self.private_bucket = "sndbx-cumulus-private" + self.public_bucket = "sndbx-cumulus-public" + self.sampleFileName1 = "L0A_HR_RAW_product_0010-of-0420" + self.sampleFileName2 = "L0A_HR_RAW_product_0001-of-0019" @patch("extract_filepaths_for_granule.task") @patch("extract_filepaths_for_granule.set_optional_event_property") @@ -43,41 +51,41 @@ def test_handler_happy_path( extract_filepaths_for_granule.CONFIG_FILE_BUCKETS_KEY: [ { "regex": ".*.h5$", - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.h5", + "sampleFileName": self.sampleFileName1 + ".h5", "bucket": "protected", }, { "regex": ".*.cmr.xml$", - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.iso.xml", + "sampleFileName": self.sampleFileName1 + ".iso.xml", "bucket": "protected", }, { "regex": ".*.h5.mp$", - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.h5.mp", + "sampleFileName": self.sampleFileName2 + ".h5.mp", "bucket": "public", }, { "regex": ".*.cmr.json$", - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.cmr.json", + "sampleFileName": self.sampleFileName2 + ".cmr.json", "bucket": "public", }, ], "buckets": { - "protected": {"name": "sndbx-cumulus-protected", "type": "protected"}, - "internal": {"name": "sndbx-cumulus-internal", "type": "internal"}, - "private": {"name": "sndbx-cumulus-private", "type": "private"}, - "public": {"name": "sndbx-cumulus-public", "type": "public"}, + "protected": {"name": self.protected_bucket, "type": "protected"}, + "internal": {"name": self.internal_bucket, "type": "internal"}, + "private": {"name": self.private_bucket, "type": "private"}, + "public": {"name": self.public_bucket, "type": "public"}, }, } mock_task.return_value = { "granules": [ { - "collectionId": uuid.uuid4().__str__(), - "granuleId": "L0A_HR_RAW_product_0003-of-0420", + "collectionId": self.collectionId, + "granuleId": self.granuleId, "keys": [ - "L0A_HR_RAW_product_0003-of-0420.h5", - "L0A_HR_RAW_product_0003-of-0420.cmr.json", + self.granuleId + ".h5", + self.granuleId + ".cmr.json", ], } ] @@ -101,27 +109,27 @@ def test_handler_raises_error_bad_config(self, _: MagicMock, mock_task: MagicMoc handler_input_event["config"] = { extract_filepaths_for_granule.CONFIG_FILE_BUCKETS_KEY: [ { - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.h5", + "sampleFileName": self.sampleFileName1 + ".h5", "bucket": "protected", }, { - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.iso.xml", + "sampleFileName": self.sampleFileName1 + ".iso.xml", "bucket": "protected", }, { - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.h5.mp", + "sampleFileName": self.sampleFileName2 + ".h5.mp", "bucket": "public", }, { - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.cmr.json", + "sampleFileName": self.sampleFileName2 + ".cmr.json", "bucket": "public", }, ], "buckets": { - "protected": {"name": "sndbx-cumulus-protected", "type": "protected"}, - "internal": {"name": "sndbx-cumulus-internal", "type": "internal"}, - "private": {"name": "sndbx-cumulus-private", "type": "private"}, - "public": {"name": "sndbx-cumulus-public", "type": "public"}, + "protected": {"name": self.protected_bucket, "type": "protected"}, + "internal": {"name": self.internal_bucket, "type": "internal"}, + "private": {"name": self.private_bucket, "type": "private"}, + "public": {"name": self.public_bucket, "type": "public"}, }, } context = Mock() @@ -143,15 +151,15 @@ def test_handler_raises_error_bad_input(self, _: MagicMock, mock_task: MagicMock "input": { "granules": [ { - "collectionId": uuid.uuid4().__str__(), + "collectionId": self.collectionId, "status": "completed", "files": [ { "checksumType": "md5", - "bucket": "podaac-ngap-dev-cumulus-test-input", + "bucket": self.internal_bucket, "type": "data", - "fileName": "L0A_HR_RAW_product_0003-of-0420.cmr.json", - "key": "L0A_HR_RAW_product_0003-of-0420.cmr.json", + "fileName": self.sampleFileName1 + ".cmr.json", + "key": self.sampleFileName1 + ".cmr.json", "size": 2154070040, } ], @@ -183,37 +191,35 @@ def test_handler_raises_error_bad_output(self, _: MagicMock, mock_task: MagicMoc extract_filepaths_for_granule.CONFIG_FILE_BUCKETS_KEY: [ { "regex": ".*.h5$", - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.h5", + "sampleFileName": self.sampleFileName1 + ".h5", "bucket": "protected", }, { "regex": ".*.cmr.xml$", - "sampleFileName": "L0A_HR_RAW_product_0010-of-0420.iso.xml", + "sampleFileName": self.sampleFileName1 + ".iso.xml", "bucket": "protected", }, { "regex": ".*.h5.mp$", - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.h5.mp", + "sampleFileName": self.sampleFileName2 + ".h5.mp", "bucket": "public", }, { "regex": ".*.cmr.json$", - "sampleFileName": "L0A_HR_RAW_product_0001-of-0019.cmr.json", + "sampleFileName": self.sampleFileName2 + ".cmr.json", "bucket": "public", }, ], "buckets": { - "protected": {"name": "sndbx-cumulus-protected", "type": "protected"}, - "internal": {"name": "sndbx-cumulus-internal", "type": "internal"}, - "private": {"name": "sndbx-cumulus-private", "type": "private"}, - "public": {"name": "sndbx-cumulus-public", "type": "public"}, + "protected": {"name": self.protected_bucket, "type": "protected"}, + "internal": {"name": self.internal_bucket, "type": "internal"}, + "private": {"name": self.private_bucket, "type": "private"}, + "public": {"name": self.public_bucket, "type": "public"}, }, } mock_task.return_value = { - "granules": [ - {"collectionId": uuid.uuid4().__str__(), "keys": ["key1", "key2"]} - ] + "granules": [{"collectionId": self.collectionId, "keys": ["key1", "key2"]}] } context = Mock() with self.assertRaises(Exception) as ex: @@ -238,19 +244,19 @@ def test_task(self, mock_debug: MagicMock): extract_filepaths_for_granule.OUTPUT_KEY_KEY: self.task_input_event[ "input" ]["granules"][0]["files"][0]["key"], - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-protected", + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.protected_bucket, } exp_key2 = { extract_filepaths_for_granule.OUTPUT_KEY_KEY: self.task_input_event[ "input" ]["granules"][0]["files"][1]["key"], - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-public", + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.public_bucket, } exp_key3 = { extract_filepaths_for_granule.OUTPUT_KEY_KEY: self.task_input_event[ "input" ]["granules"][0]["files"][2]["key"], - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-public", + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.public_bucket, } exp_gran = { "collectionId": self.task_input_event["input"]["granules"][0][ @@ -271,16 +277,15 @@ def test_task_one_file(self, mock_debug: MagicMock): """ Test with one valid file in input. """ - collection_id = uuid.uuid4().__str__() self.task_input_event["input"]["granules"] = [ { - "collectionId": collection_id, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": self.collectionId, + "granuleId": self.granuleId, "files": [ { "key": "MOD09GQ___006/MOD/MOD09GQ.A0219114." "N5aUCG.006.0656338553321.cmr.xml", - "bucket": "cumulus-test-sandbox-protected-2", + "bucket": self.input_bucket, "fileName": "MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr.xml", } ], @@ -293,11 +298,11 @@ def test_task_one_file(self, mock_debug: MagicMock): { extract_filepaths_for_granule.OUTPUT_KEY_KEY: "MOD09GQ___006/MOD/MOD09GQ." # noqa: E501 "A0219114.N5aUCG.006.0656338553321.cmr.xml", - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-protected", # noqa: E501 + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.protected_bucket, # noqa: E501 } ], - "collectionId": collection_id, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": self.collectionId, + "granuleId": self.granuleId, } ] } @@ -314,12 +319,12 @@ def test_task_no_matching_regex_raises_error(self, mock_debug: MagicMock): """ self.task_input_event["input"]["granules"] = [ { - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "granuleId": self.granuleId, "files": [ { "key": "MOD09GQ___006/MOD/MOD09GQ.A0219114." "N5aUCG.006.0656338553321.cmr.blah", - "bucket": "cumulus-test-sandbox-protected-2", + "bucket": self.input_bucket, "fileName": "MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr.blah", } ], @@ -345,15 +350,14 @@ def test_exclude_file_type(self): "extract_filepaths_for_granule/test/unit_tests/testevents/task_event.json" includes "excludedFileExtensions": [".cmr"] """ - collection_id = uuid.uuid4().__str__() self.task_input_event["input"]["granules"] = [ { - "collectionId": collection_id, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": self.collectionId, + "granuleId": self.granuleId, "files": [ { "key": "MOD09GQ___006/MOD/MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr", - "bucket": "cumulus-test-sandbox-protected-2", + "bucket": self.input_bucket, "fileName": "MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr", } ], @@ -363,8 +367,8 @@ def test_exclude_file_type(self): "granules": [ { "keys": [], # this will be empty since the filetype is .cmr - "collectionId": collection_id, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": self.collectionId, + "granuleId": self.granuleId, } ] } @@ -385,23 +389,23 @@ def test_task_two_granules(self, mock_debug: MagicMock): self.task_input_event["input"]["granules"] = [ { "collectionId": collection_id0, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "granuleId": self.granuleId, "files": [ { "fileName": "MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr.xml", "key": "MOD/MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr.xml", - "bucket": "cumulus-test-sandbox-protected-2", + "bucket": self.input_bucket, } ], }, { "collectionId": collection_id1, - "granuleId": "MOD09GQ.A0219115.N5aUCG.006.0656338553321", + "granuleId": self.granuleId, "files": [ { "fileName": "MOD09GQ.A0219115.N5aUCG.006.0656338553321.cmr.xml", "key": "MOD/MOD09GQ.A0219115.N5aUCG.006.0656338553321.cmr.xml", - "bucket": "cumulus-test-sandbox-protected-2", + "bucket": self.input_bucket, } ], }, @@ -413,21 +417,21 @@ def test_task_two_granules(self, mock_debug: MagicMock): "keys": [ { extract_filepaths_for_granule.OUTPUT_KEY_KEY: "MOD/MOD09GQ.A0219114.N5aUCG.006.0656338553321.cmr.xml", # noqa: E501 - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-protected", # noqa: E501 + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.protected_bucket, # noqa: E501 } ], "collectionId": collection_id0, - "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "granuleId": self.granuleId, }, { "keys": [ { extract_filepaths_for_granule.OUTPUT_KEY_KEY: "MOD/MOD09GQ.A0219115.N5aUCG.006.0656338553321.cmr.xml", # noqa: E501 - extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: "sndbx-cumulus-protected", # noqa: E501 + extract_filepaths_for_granule.OUTPUT_DESTINATION_BUCKET_KEY: self.protected_bucket, # noqa: E501 } ], "collectionId": collection_id1, - "granuleId": "MOD09GQ.A0219115.N5aUCG.006.0656338553321", + "granuleId": self.granuleId, }, ] } From 5ec2dd176e273a7bc25891b98b705ad45c788686 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:30:32 -0600 Subject: [PATCH 04/68] testing request_method --- .../orca_shared/recovery/test/unit_tests/test_shared_recovery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index b4e89b03d..512e73112 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -94,6 +94,7 @@ def test_post_entry_to_fifo_queue_no_errors(self): } ] ) + print(request_method.value) @patch.dict( os.environ, From 76767d05fd6308e8a505aa022d7a3b1b75d1b394 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 30 Nov 2023 15:09:33 -0600 Subject: [PATCH 05/68] Adding request_method unit test --- .../recovery/test/unit_tests/test_shared_recovery.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index 512e73112..12714763c 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -94,7 +94,12 @@ def test_post_entry_to_fifo_queue_no_errors(self): } ] ) - print(request_method.value) + # Verifies that request_method is set properly (new_job or update_file) + self.assertTrue( + request_method.value == "new_job" + or request_method.value == "update_file", + f"Incorrect Request Method: {request_method.value}", + ) @patch.dict( os.environ, From 15deed657a465f51d350cf480b24dd2f65c9c024 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 30 Nov 2023 15:20:48 -0600 Subject: [PATCH 06/68] updating CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a7151d0d..596fe34b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and includes an additional section for migration notes. - *Security* - Vulnerabilities fixes and changes. ## [Unreleased] +- *ORCA-366* Adding unit test for request_method ### Migration Notes From 51688c64c758c9e25b75a6ad76ab9130b0fde57e Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Tue, 5 Dec 2023 10:07:38 -0600 Subject: [PATCH 07/68] Testing fail cases --- shared_libraries/orca_shared/recovery/shared_recovery.py | 2 ++ .../recovery/test/unit_tests/test_shared_recovery.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/shared_libraries/orca_shared/recovery/shared_recovery.py b/shared_libraries/orca_shared/recovery/shared_recovery.py index 31f5cbf30..37f977ef3 100644 --- a/shared_libraries/orca_shared/recovery/shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/shared_recovery.py @@ -27,6 +27,8 @@ class RequestMethod(Enum): NEW_JOB = "new_job" UPDATE_FILE = "update_file" + DELETE_FILE = "delete_file" + GET_FILE = "get_file" class OrcaStatus(Enum): diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index 12714763c..17f478ec7 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -39,6 +39,8 @@ def setUp(self): self.request_methods = [ shared_recovery.RequestMethod.NEW_JOB, shared_recovery.RequestMethod.UPDATE_FILE, + shared_recovery.RequestMethod.DELETE_FILE, + shared_recovery.RequestMethod.GET_FILE, ] self.statuses = [ shared_recovery.OrcaStatus.PENDING, From 6479ff39ddbaf5912e9b1724e3491e59ba1f28d0 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Tue, 5 Dec 2023 11:02:35 -0600 Subject: [PATCH 08/68] Reverting fail testing changes --- shared_libraries/orca_shared/recovery/shared_recovery.py | 2 -- .../recovery/test/unit_tests/test_shared_recovery.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/shared_libraries/orca_shared/recovery/shared_recovery.py b/shared_libraries/orca_shared/recovery/shared_recovery.py index 37f977ef3..31f5cbf30 100644 --- a/shared_libraries/orca_shared/recovery/shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/shared_recovery.py @@ -27,8 +27,6 @@ class RequestMethod(Enum): NEW_JOB = "new_job" UPDATE_FILE = "update_file" - DELETE_FILE = "delete_file" - GET_FILE = "get_file" class OrcaStatus(Enum): diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index 17f478ec7..12714763c 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -39,8 +39,6 @@ def setUp(self): self.request_methods = [ shared_recovery.RequestMethod.NEW_JOB, shared_recovery.RequestMethod.UPDATE_FILE, - shared_recovery.RequestMethod.DELETE_FILE, - shared_recovery.RequestMethod.GET_FILE, ] self.statuses = [ shared_recovery.OrcaStatus.PENDING, From 24da4e7170f1a773889bf08fb3cad196a24f0bc4 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Tue, 5 Dec 2023 11:37:12 -0600 Subject: [PATCH 09/68] Updating comment --- .../recovery/test/unit_tests/test_shared_recovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index 12714763c..907b2291f 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -94,7 +94,7 @@ def test_post_entry_to_fifo_queue_no_errors(self): } ] ) - # Verifies that request_method is set properly (new_job or update_file) + # Verifies that request_method is set properly to new_job or update_file, if not set to either value test will fail self.assertTrue( request_method.value == "new_job" or request_method.value == "update_file", From babe582143e6786405c2a16052822be9339d0fec Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Wed, 6 Dec 2023 18:24:22 -0600 Subject: [PATCH 10/68] Testing request_method test --- shared_libraries/orca_shared/recovery/shared_recovery.py | 5 +++++ .../recovery/test/unit_tests/test_shared_recovery.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/shared_libraries/orca_shared/recovery/shared_recovery.py b/shared_libraries/orca_shared/recovery/shared_recovery.py index 31f5cbf30..35d8e3c94 100644 --- a/shared_libraries/orca_shared/recovery/shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/shared_recovery.py @@ -268,3 +268,8 @@ def post_entry_to_standard_queue( raise Exception( f"Calculated MD5 of {md5_body} does not match SQS MD5 of {sqs_md5}" ) + + +def get_request_method(request_method: RequestMethod): + if request_method != "new_job": + raise Exception(f"{request_method}") diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index 907b2291f..cee856502 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -94,7 +94,7 @@ def test_post_entry_to_fifo_queue_no_errors(self): } ] ) - # Verifies that request_method is set properly to new_job or update_file, if not set to either value test will fail + # Verifies that request_method is set properly to new_job or update_file self.assertTrue( request_method.value == "new_job" or request_method.value == "update_file", @@ -287,3 +287,9 @@ def test_update_status_for_file_error_message_empty_raises_error_message(self): uuid.uuid4().__str__(), ) self.assertEqual("Error message is required.", str(cm.exception)) + + def test_request_method_raises_error(self): + for request_method in self.request_methods: + with self.assertRaises(ValueError) as cm: + shared_recovery.get_request_method(request_method.value) + self.assertEqual(str(cm.exception), "new_job") From 2e1eb9a337776c56d5f820706897f4b17af35e09 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Wed, 6 Dec 2023 18:33:03 -0600 Subject: [PATCH 11/68] Updating test function --- shared_libraries/orca_shared/recovery/shared_recovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_libraries/orca_shared/recovery/shared_recovery.py b/shared_libraries/orca_shared/recovery/shared_recovery.py index 35d8e3c94..de67f2da1 100644 --- a/shared_libraries/orca_shared/recovery/shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/shared_recovery.py @@ -271,5 +271,5 @@ def post_entry_to_standard_queue( def get_request_method(request_method: RequestMethod): - if request_method != "new_job": + if request_method != "new_job" or request_method != "update_file": raise Exception(f"{request_method}") From ea8e953313cff19f4c09618c550716359752b323 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 7 Dec 2023 08:06:49 -0600 Subject: [PATCH 12/68] randomized large file --- .../test_multiple_granules_happy_path.py | 530 +++++++++--------- 1 file changed, 274 insertions(+), 256 deletions(-) diff --git a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py index 870f6330b..2f3393f02 100644 --- a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py +++ b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py @@ -16,10 +16,11 @@ class TestMultipleGranulesHappyPath(TestCase): - def test_multiple_granules_happy_path(self): self.maxDiff = None - use_large_file = True # This should not be checked in with any value other than `True` + use_large_file = ( + True # This should not be checked in with any value other than `True` + ) """ - If multiple granules are provided, should store them in DB as well as in recovery bucket. - Files stored in GLACIER are also ingest-able. @@ -47,22 +48,36 @@ def test_multiple_granules_happy_path(self): key_name_1 = "test/" + uuid.uuid4().__str__() + "/" + name_1 file_1_hash = uuid.uuid4().__str__() file_1_hash_type = uuid.uuid4().__str__() + large_file_key_path = ( + "PODAAC/SWOT/ancillary_data_input_forcing_ECCO_V4r4.tar.gz" + ) # Upload the randomized file to source bucket - boto3_session.client('s3').upload_file( + boto3_session.client("s3").upload_file( "file1.hdf", cumulus_bucket_name, key_name_1 ) execution_id = uuid.uuid4().__str__() if use_large_file: - # file for large file test - # Since this file is 191GB, it should already exist in the source bucket. - name_2 = "ancillary_data_input_forcing_ECCO_V4r4.tar.gz" - key_name_2 = "PODAAC/SWOT/" + name_2 + large_file_name = uuid.uuid4().__str__() + ".tar.gz" + key_name_2 = "test/" + uuid.uuid4().__str__() + "/" + large_file_name + try: + # copy the large file with name randomized to a different directory + boto3_session.client("s3").copy( + { + "Bucket": cumulus_bucket_name, + "Key": large_file_key_path, + }, + cumulus_bucket_name, + key_name_2, + ) + except Exception as ex: + logger.error(ex) + raise else: name_2 = uuid.uuid4().__str__() + ".hdf" # refers to file1.hdf key_name_2 = "test/" + uuid.uuid4().__str__() + "/" + name_2 # Upload the randomized file to source bucket - boto3_session.client('s3').upload_file( + boto3_session.client("s3").upload_file( "file1.hdf", cumulus_bucket_name, key_name_2 ) copy_to_archive_input = { @@ -78,69 +93,59 @@ def test_multiple_granules_happy_path(self): "checksum": file_1_hash, "checksumType": file_1_hash_type, } - ] + ], }, { "granuleId": granule_id_2, "createdAt": createdAt_time, "files": [ - { - "bucket": cumulus_bucket_name, - "key": key_name_2 - } - ] - } + {"bucket": cumulus_bucket_name, "key": key_name_2} + ], + }, ] }, "meta": { - "provider": { - "id": provider_id, - "name": provider_name - }, + "provider": {"id": provider_id, "name": provider_name}, "collection": { - "meta": { - "orca": { - "excludedFileExtensions": excluded_filetype - } - }, + "meta": {"orca": {"excludedFileExtensions": excluded_filetype}}, "name": collection_name, - "version": collection_version - } + "version": collection_version, + }, }, - "cumulus_meta": { - "execution_name": execution_id - } + "cumulus_meta": {"execution_name": execution_id}, } expected_output = { - "granules": [ - { - "granuleId": granule_id_1, - "createdAt": createdAt_time, - "files": [ - { - "bucket": cumulus_bucket_name, - "key": key_name_1, - "checksum": file_1_hash, - "checksumType": file_1_hash_type - } - ] - }, - { - "granuleId": granule_id_2, - "createdAt": createdAt_time, - "files": [ - { - "bucket": cumulus_bucket_name, - "key": key_name_2, - } - ] - } - ], - "copied_to_orca": [ - "s3://" + cumulus_bucket_name + "/" + key_name_1, - "s3://" + cumulus_bucket_name + "/" + key_name_2 - ] + "payload": { + "granules": [ + { + "granuleId": granule_id_1, + "createdAt": createdAt_time, + "files": [ + { + "bucket": cumulus_bucket_name, + "key": key_name_1, + "checksum": file_1_hash, + "checksumType": file_1_hash_type, + } + ], + }, + { + "granuleId": granule_id_2, + "createdAt": createdAt_time, + "files": [ + { + "bucket": cumulus_bucket_name, + "key": key_name_2, + } + ], + }, + ], + "copied_to_orca": [ + "s3://" + cumulus_bucket_name + "/" + key_name_1, + "s3://" + cumulus_bucket_name + "/" + key_name_2, + ], + } } execution_info = boto3_session.client("stepfunctions").start_execution( @@ -148,21 +153,20 @@ def test_multiple_granules_happy_path(self): input=json.dumps(copy_to_archive_input, indent=4), ) - step_function_results = helpers.get_state_machine_execution_results(boto3_session, - execution_info[ - "executionArn"] - ) + step_function_results = helpers.get_state_machine_execution_results( + boto3_session, execution_info["executionArn"] + ) self.assertEqual( "SUCCEEDED", step_function_results["status"], "Error occurred while starting step function.", ) - self.assertEqual( - expected_output, - json.loads(step_function_results["output"]), - "Expected ingest step function output not returned.", - ) + # self.assertEqual( + # expected_output, + # json.loads(step_function_results["output"]), + # "Expected ingest step function output not returned.", + # ) s3_versions = [] # verify that the objects exist in recovery bucket try: @@ -175,215 +179,229 @@ def test_multiple_granules_happy_path(self): # use boto3.Session(profile_name="yourAWSConfigureProfileName").client(... # to use a differently configured aws access key head_object_output = boto3_session.client("s3").head_object( - Bucket=recovery_bucket_name, Key=key) + Bucket=recovery_bucket_name, Key=key + ) self.assertEqual( 200, head_object_output["ResponseMetadata"]["HTTPStatusCode"], f"Error searching for object {key} in the {recovery_bucket_name}", ) - self.assertEqual( - "GLACIER", - head_object_output["StorageClass"] - ) + self.assertEqual("GLACIER", head_object_output["StorageClass"]) s3_versions.append(head_object_output.get("VersionId", "null")) except Exception as ex: logger.error(ex) raise - - # Let the catalog update - time.sleep(30) - # noinspection PyArgumentList - catalog_input = { - "pageIndex": 0, - "collectionId": [collection_id], - "granuleId": [ - granule_id_1, - granule_id_2, - ], - "endTimestamp": int((time.time() + 5) * 1000), - } - catalog_output = helpers.post_to_api( - my_session, - helpers.api_url + "/catalog/reconcile/", - data=json.dumps( - catalog_input - ), - headers={"Host": helpers.aws_api_name}, - ) - self.assertEqual( - 200, catalog_output.status_code, f"Error occurred while contacting API: " - f"{catalog_output.content}" - ) - expected_catalog_output_granules = [{ - "providerId": provider_id, - "collectionId": collection_id, - "id": granule_id_1, - "createdAt": createdAt_time, - "executionId": execution_id, - "files": [ - { - "name": name_1, - "cumulusArchiveLocation": cumulus_bucket_name, - "orcaArchiveLocation": recovery_bucket_name, - "keyPath": key_name_1, - "sizeBytes": 6, - "hash": file_1_hash, "hashType": file_1_hash_type, - "storageClass": "GLACIER", - "version": s3_versions[0], - } - ], - "ingestDate": mock.ANY, - "lastUpdate": mock.ANY - }, - { - "providerId": provider_id, - "collectionId": collection_id, - "id": granule_id_2, - "createdAt": createdAt_time, - "executionId": execution_id, - "files": [ - { - "name": name_2, - "cumulusArchiveLocation": cumulus_bucket_name, - "orcaArchiveLocation": recovery_bucket_name, - "keyPath": key_name_2, - "sizeBytes": 205640819682 if use_large_file else 6, - "hash": None, "hashType": None, - "storageClass": "GLACIER", - "version": s3_versions[1], - } - ], - "ingestDate": mock.ANY, - "lastUpdate": mock.ANY - }] - expected_catalog_output = { - "anotherPage": False, - "granules": mock.ANY - } - catalog_output_json = catalog_output.json() - self.assertEqual( - expected_catalog_output, - catalog_output_json, - "Expected API output not returned.", - ) - # Make sure all given granules are present without checking order. - self.assertCountEqual( - expected_catalog_output_granules, - catalog_output_json["granules"], - f"Expected API output granules not returned. Request: {catalog_input}" - ) + # # Let the catalog update + # time.sleep(30) + # # noinspection PyArgumentList + # catalog_input = { + # "pageIndex": 0, + # "collectionId": [collection_id], + # "granuleId": [ + # granule_id_1, + # granule_id_2, + # ], + # "endTimestamp": int((time.time() + 5) * 1000), + # } + # catalog_output = helpers.post_to_api( + # my_session, + # helpers.api_url + "/catalog/reconcile/", + # data=json.dumps(catalog_input), + # headers={"Host": helpers.aws_api_name}, + # ) + # self.assertEqual( + # 200, + # catalog_output.status_code, + # f"Error occurred while contacting API: " f"{catalog_output.content}", + # ) + # expected_catalog_output_granules = [ + # { + # "providerId": provider_id, + # "collectionId": collection_id, + # "id": granule_id_1, + # "createdAt": createdAt_time, + # "executionId": execution_id, + # "files": [ + # { + # "name": name_1, + # "cumulusArchiveLocation": cumulus_bucket_name, + # "orcaArchiveLocation": recovery_bucket_name, + # "keyPath": key_name_1, + # "sizeBytes": 6, + # "hash": file_1_hash, + # "hashType": file_1_hash_type, + # "storageClass": "GLACIER", + # "version": s3_versions[0], + # } + # ], + # "ingestDate": mock.ANY, + # "lastUpdate": mock.ANY, + # }, + # { + # "providerId": provider_id, + # "collectionId": collection_id, + # "id": granule_id_2, + # "createdAt": createdAt_time, + # "executionId": execution_id, + # "files": [ + # { + # "name": name_2, + # "cumulusArchiveLocation": cumulus_bucket_name, + # "orcaArchiveLocation": recovery_bucket_name, + # "keyPath": key_name_2, + # "sizeBytes": 205640819682 if use_large_file else 6, + # "hash": None, + # "hashType": None, + # "storageClass": "GLACIER", + # "version": s3_versions[1], + # } + # ], + # "ingestDate": mock.ANY, + # "lastUpdate": mock.ANY, + # }, + # ] + # expected_catalog_output = {"anotherPage": False, "granules": mock.ANY} + # catalog_output_json = catalog_output.json() + # self.assertEqual( + # expected_catalog_output, + # catalog_output_json, + # "Expected API output not returned.", + # ) + # # Make sure all given granules are present without checking order. + # self.assertCountEqual( + # expected_catalog_output_granules, + # catalog_output_json["granules"], + # f"Expected API output granules not returned. Request: {catalog_input}", + # ) # recovery check - self.partial_test_recovery_happy_path( - boto3_session, - collection_id, granule_id_1, name_1, key_name_1, recovery_bucket_name) + # self.partial_test_recovery_happy_path( + # boto3_session, + # collection_id, + # granule_id_1, + # name_1, + # key_name_1, + # recovery_bucket_name, + # ) except Exception as ex: logger.error(ex) raise - def partial_test_recovery_happy_path( - self, boto3_session, collection_id: str, granule_id: str, file_name: str, - file_key: str, orca_bucket_name: str - ): - target_bucket = helpers.buckets["private"]["name"] - # noinspection PyTypeChecker - recovery_request_record = helpers.RecoveryRequestRecord([ - helpers.RecoveryRequestGranule(collection_id, granule_id, [ - helpers.RecoveryRequestFile(file_name, file_key, orca_bucket_name, target_bucket) - ]) - ], None) - step_function_results = self.initiate_recovery( - boto3_session=boto3_session, - recovery_request_record=recovery_request_record, - file_bucket_maps=[ - { - "regex": ".*", - "bucket": "private" - }] - ) - self.assertEqual( - "SUCCEEDED", - step_function_results["status"], - "Error occurred while starting step function.", - ) - expected_output = { - "granules": [ - { - "collectionId": collection_id, - "granuleId": granule_id, - "keys": [ - { - "key": file_key, - "destBucket": target_bucket - } - ], - "recoverFiles": [ - { - "success": True, - "filename": file_name, - "keyPath": file_key, - "restoreDestination": target_bucket, - "s3MultipartChunksizeMb": None, - "statusId": 1, - "requestTime": mock.ANY, - "lastUpdate": mock.ANY - } - ] - } - ], - "asyncOperationId": mock.ANY - } - actual_output = json.loads(step_function_results["output"]) - self.assertEqual( - expected_output, - actual_output, - "Expected recovery step function output not returned.", - ) + # def partial_test_recovery_happy_path( + # self, + # boto3_session, + # collection_id: str, + # granule_id: str, + # file_name: str, + # file_key: str, + # orca_bucket_name: str, + # ): + # target_bucket = helpers.buckets["private"]["name"] + # # noinspection PyTypeChecker + # recovery_request_record = helpers.RecoveryRequestRecord( + # [ + # helpers.RecoveryRequestGranule( + # collection_id, + # granule_id, + # [ + # helpers.RecoveryRequestFile( + # file_name, file_key, orca_bucket_name, target_bucket + # ) + # ], + # ) + # ], + # None, + # ) + # step_function_results = self.initiate_recovery( + # boto3_session=boto3_session, + # recovery_request_record=recovery_request_record, + # file_bucket_maps=[{"regex": ".*", "bucket": "private"}], + # ) + # self.assertEqual( + # "SUCCEEDED", + # step_function_results["status"], + # "Error occurred while starting step function.", + # ) + # expected_output = { + # "granules": [ + # { + # "collectionId": collection_id, + # "granuleId": granule_id, + # "keys": [{"key": file_key, "destBucket": target_bucket}], + # "recoverFiles": [ + # { + # "success": True, + # "filename": file_name, + # "keyPath": file_key, + # "restoreDestination": target_bucket, + # "s3MultipartChunksizeMb": None, + # "statusId": 1, + # "requestTime": mock.ANY, + # "lastUpdate": mock.ANY, + # } + # ], + # } + # ], + # "asyncOperationId": mock.ANY, + # } + # actual_output = json.loads(step_function_results["output"]) + # self.assertEqual( + # expected_output, + # actual_output, + # "Expected recovery step function output not returned.", + # ) - recovery_request_record.async_operation_id = actual_output["asyncOperationId"] + # recovery_request_record.async_operation_id = actual_output["asyncOperationId"] - helpers.create_recovery_request_record(recovery_request_record_happy_path_filename, - recovery_request_record) + # helpers.create_recovery_request_record( + # recovery_request_record_happy_path_filename, recovery_request_record + # ) - @staticmethod - def initiate_recovery( - boto3_session, - recovery_request_record: helpers.RecoveryRequestRecord, file_bucket_maps: List[Dict], - ): - recovery_step_function_input = { - "payload": { - "granules": [{ - "collectionId": granule.collection_id, - "granuleId": granule.granule_id, - "files": [{ - "fileName": file.file_name, - "key": file.file_key, - "bucket": file.orca_bucket_name - } for file in granule.files], - } for granule in recovery_request_record.granules], - }, - "config": { - "buckets": helpers.buckets, - "fileBucketMaps": file_bucket_maps, - "excludedFileExtensions": [], - "asyncOperationId": None, - "s3MultipartChunksizeMb": None, - "defaultBucketOverride": None, - "defaultRecoveryTypeOverride": None - } - } + # @staticmethod + # def initiate_recovery( + # boto3_session, + # recovery_request_record: helpers.RecoveryRequestRecord, + # file_bucket_maps: List[Dict], + # ): + # recovery_step_function_input = { + # "payload": { + # "granules": [ + # { + # "collectionId": granule.collection_id, + # "granuleId": granule.granule_id, + # "files": [ + # { + # "fileName": file.file_name, + # "key": file.file_key, + # "bucket": file.orca_bucket_name, + # } + # for file in granule.files + # ], + # } + # for granule in recovery_request_record.granules + # ], + # }, + # "config": { + # "buckets": helpers.buckets, + # "fileBucketMaps": file_bucket_maps, + # "excludedFileExtensions": [], + # "asyncOperationId": None, + # "s3MultipartChunksizeMb": None, + # "defaultBucketOverride": None, + # "defaultRecoveryTypeOverride": None, + # }, + # } - logger.info(f"Sending event to recovery step function") - logger.info(recovery_step_function_input) + # logger.info(f"Sending event to recovery step function") + # logger.info(recovery_step_function_input) - execution_info = boto3_session.client("stepfunctions").start_execution( - stateMachineArn=helpers.orca_recovery_step_function_arn, - input=json.dumps(recovery_step_function_input, indent=4), - ) + # execution_info = boto3_session.client("stepfunctions").start_execution( + # stateMachineArn=helpers.orca_recovery_step_function_arn, + # input=json.dumps(recovery_step_function_input, indent=4), + # ) - step_function_results = helpers.get_state_machine_execution_results(boto3_session, - execution_info[ - "executionArn"] - ) + # step_function_results = helpers.get_state_machine_execution_results( + # boto3_session, execution_info["executionArn"] + # ) - return step_function_results + # return step_function_results From c2ce4e632d72a6708342e0f296dca2a4257e9ab3 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 7 Dec 2023 11:15:15 -0600 Subject: [PATCH 13/68] Removing added tests that aren't needed --- shared_libraries/orca_shared/recovery/shared_recovery.py | 5 ----- .../recovery/test/unit_tests/test_shared_recovery.py | 6 ------ 2 files changed, 11 deletions(-) diff --git a/shared_libraries/orca_shared/recovery/shared_recovery.py b/shared_libraries/orca_shared/recovery/shared_recovery.py index de67f2da1..31f5cbf30 100644 --- a/shared_libraries/orca_shared/recovery/shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/shared_recovery.py @@ -268,8 +268,3 @@ def post_entry_to_standard_queue( raise Exception( f"Calculated MD5 of {md5_body} does not match SQS MD5 of {sqs_md5}" ) - - -def get_request_method(request_method: RequestMethod): - if request_method != "new_job" or request_method != "update_file": - raise Exception(f"{request_method}") diff --git a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py index cee856502..b60428c72 100644 --- a/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py +++ b/shared_libraries/orca_shared/recovery/test/unit_tests/test_shared_recovery.py @@ -287,9 +287,3 @@ def test_update_status_for_file_error_message_empty_raises_error_message(self): uuid.uuid4().__str__(), ) self.assertEqual("Error message is required.", str(cm.exception)) - - def test_request_method_raises_error(self): - for request_method in self.request_methods: - with self.assertRaises(ValueError) as cm: - shared_recovery.get_request_method(request_method.value) - self.assertEqual(str(cm.exception), "new_job") From 306e8160271d9ccb891ba6a1a89c6496d019ec19 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 7 Dec 2023 14:04:05 -0600 Subject: [PATCH 14/68] uncommenting code --- .../test_multiple_granules_happy_path.py | 404 +++++++++--------- 1 file changed, 202 insertions(+), 202 deletions(-) diff --git a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py index 2f3393f02..50f299c12 100644 --- a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py +++ b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py @@ -191,217 +191,217 @@ def test_multiple_granules_happy_path(self): except Exception as ex: logger.error(ex) raise - # # Let the catalog update - # time.sleep(30) - # # noinspection PyArgumentList - # catalog_input = { - # "pageIndex": 0, - # "collectionId": [collection_id], - # "granuleId": [ - # granule_id_1, - # granule_id_2, - # ], - # "endTimestamp": int((time.time() + 5) * 1000), - # } - # catalog_output = helpers.post_to_api( - # my_session, - # helpers.api_url + "/catalog/reconcile/", - # data=json.dumps(catalog_input), - # headers={"Host": helpers.aws_api_name}, - # ) - # self.assertEqual( - # 200, - # catalog_output.status_code, - # f"Error occurred while contacting API: " f"{catalog_output.content}", - # ) - # expected_catalog_output_granules = [ - # { - # "providerId": provider_id, - # "collectionId": collection_id, - # "id": granule_id_1, - # "createdAt": createdAt_time, - # "executionId": execution_id, - # "files": [ - # { - # "name": name_1, - # "cumulusArchiveLocation": cumulus_bucket_name, - # "orcaArchiveLocation": recovery_bucket_name, - # "keyPath": key_name_1, - # "sizeBytes": 6, - # "hash": file_1_hash, - # "hashType": file_1_hash_type, - # "storageClass": "GLACIER", - # "version": s3_versions[0], - # } - # ], - # "ingestDate": mock.ANY, - # "lastUpdate": mock.ANY, - # }, - # { - # "providerId": provider_id, - # "collectionId": collection_id, - # "id": granule_id_2, - # "createdAt": createdAt_time, - # "executionId": execution_id, - # "files": [ - # { - # "name": name_2, - # "cumulusArchiveLocation": cumulus_bucket_name, - # "orcaArchiveLocation": recovery_bucket_name, - # "keyPath": key_name_2, - # "sizeBytes": 205640819682 if use_large_file else 6, - # "hash": None, - # "hashType": None, - # "storageClass": "GLACIER", - # "version": s3_versions[1], - # } - # ], - # "ingestDate": mock.ANY, - # "lastUpdate": mock.ANY, - # }, - # ] - # expected_catalog_output = {"anotherPage": False, "granules": mock.ANY} - # catalog_output_json = catalog_output.json() - # self.assertEqual( - # expected_catalog_output, - # catalog_output_json, - # "Expected API output not returned.", - # ) - # # Make sure all given granules are present without checking order. - # self.assertCountEqual( - # expected_catalog_output_granules, - # catalog_output_json["granules"], - # f"Expected API output granules not returned. Request: {catalog_input}", - # ) + # Let the catalog update + time.sleep(30) + # noinspection PyArgumentList + catalog_input = { + "pageIndex": 0, + "collectionId": [collection_id], + "granuleId": [ + granule_id_1, + granule_id_2, + ], + "endTimestamp": int((time.time() + 5) * 1000), + } + catalog_output = helpers.post_to_api( + my_session, + helpers.api_url + "/catalog/reconcile/", + data=json.dumps(catalog_input), + headers={"Host": helpers.aws_api_name}, + ) + self.assertEqual( + 200, + catalog_output.status_code, + f"Error occurred while contacting API: " f"{catalog_output.content}", + ) + expected_catalog_output_granules = [ + { + "providerId": provider_id, + "collectionId": collection_id, + "id": granule_id_1, + "createdAt": createdAt_time, + "executionId": execution_id, + "files": [ + { + "name": name_1, + "cumulusArchiveLocation": cumulus_bucket_name, + "orcaArchiveLocation": recovery_bucket_name, + "keyPath": key_name_1, + "sizeBytes": 6, + "hash": file_1_hash, + "hashType": file_1_hash_type, + "storageClass": "GLACIER", + "version": s3_versions[0], + } + ], + "ingestDate": mock.ANY, + "lastUpdate": mock.ANY, + }, + { + "providerId": provider_id, + "collectionId": collection_id, + "id": granule_id_2, + "createdAt": createdAt_time, + "executionId": execution_id, + "files": [ + { + "name": name_2, + "cumulusArchiveLocation": cumulus_bucket_name, + "orcaArchiveLocation": recovery_bucket_name, + "keyPath": key_name_2, + "sizeBytes": 205640819682 if use_large_file else 6, + "hash": None, + "hashType": None, + "storageClass": "GLACIER", + "version": s3_versions[1], + } + ], + "ingestDate": mock.ANY, + "lastUpdate": mock.ANY, + }, + ] + expected_catalog_output = {"anotherPage": False, "granules": mock.ANY} + catalog_output_json = catalog_output.json() + self.assertEqual( + expected_catalog_output, + catalog_output_json, + "Expected API output not returned.", + ) + # Make sure all given granules are present without checking order. + self.assertCountEqual( + expected_catalog_output_granules, + catalog_output_json["granules"], + f"Expected API output granules not returned. Request: {catalog_input}", + ) - # recovery check - # self.partial_test_recovery_happy_path( - # boto3_session, - # collection_id, - # granule_id_1, - # name_1, - # key_name_1, - # recovery_bucket_name, - # ) + recovery check + self.partial_test_recovery_happy_path( + boto3_session, + collection_id, + granule_id_1, + name_1, + key_name_1, + recovery_bucket_name, + ) except Exception as ex: logger.error(ex) raise - # def partial_test_recovery_happy_path( - # self, - # boto3_session, - # collection_id: str, - # granule_id: str, - # file_name: str, - # file_key: str, - # orca_bucket_name: str, - # ): - # target_bucket = helpers.buckets["private"]["name"] - # # noinspection PyTypeChecker - # recovery_request_record = helpers.RecoveryRequestRecord( - # [ - # helpers.RecoveryRequestGranule( - # collection_id, - # granule_id, - # [ - # helpers.RecoveryRequestFile( - # file_name, file_key, orca_bucket_name, target_bucket - # ) - # ], - # ) - # ], - # None, - # ) - # step_function_results = self.initiate_recovery( - # boto3_session=boto3_session, - # recovery_request_record=recovery_request_record, - # file_bucket_maps=[{"regex": ".*", "bucket": "private"}], - # ) - # self.assertEqual( - # "SUCCEEDED", - # step_function_results["status"], - # "Error occurred while starting step function.", - # ) - # expected_output = { - # "granules": [ - # { - # "collectionId": collection_id, - # "granuleId": granule_id, - # "keys": [{"key": file_key, "destBucket": target_bucket}], - # "recoverFiles": [ - # { - # "success": True, - # "filename": file_name, - # "keyPath": file_key, - # "restoreDestination": target_bucket, - # "s3MultipartChunksizeMb": None, - # "statusId": 1, - # "requestTime": mock.ANY, - # "lastUpdate": mock.ANY, - # } - # ], - # } - # ], - # "asyncOperationId": mock.ANY, - # } - # actual_output = json.loads(step_function_results["output"]) - # self.assertEqual( - # expected_output, - # actual_output, - # "Expected recovery step function output not returned.", - # ) + def partial_test_recovery_happy_path( + self, + boto3_session, + collection_id: str, + granule_id: str, + file_name: str, + file_key: str, + orca_bucket_name: str, + ): + target_bucket = helpers.buckets["private"]["name"] + # noinspection PyTypeChecker + recovery_request_record = helpers.RecoveryRequestRecord( + [ + helpers.RecoveryRequestGranule( + collection_id, + granule_id, + [ + helpers.RecoveryRequestFile( + file_name, file_key, orca_bucket_name, target_bucket + ) + ], + ) + ], + None, + ) + step_function_results = self.initiate_recovery( + boto3_session=boto3_session, + recovery_request_record=recovery_request_record, + file_bucket_maps=[{"regex": ".*", "bucket": "private"}], + ) + self.assertEqual( + "SUCCEEDED", + step_function_results["status"], + "Error occurred while starting step function.", + ) + expected_output = { + "granules": [ + { + "collectionId": collection_id, + "granuleId": granule_id, + "keys": [{"key": file_key, "destBucket": target_bucket}], + "recoverFiles": [ + { + "success": True, + "filename": file_name, + "keyPath": file_key, + "restoreDestination": target_bucket, + "s3MultipartChunksizeMb": None, + "statusId": 1, + "requestTime": mock.ANY, + "lastUpdate": mock.ANY, + } + ], + } + ], + "asyncOperationId": mock.ANY, + } + actual_output = json.loads(step_function_results["output"]) + self.assertEqual( + expected_output, + actual_output, + "Expected recovery step function output not returned.", + ) - # recovery_request_record.async_operation_id = actual_output["asyncOperationId"] + recovery_request_record.async_operation_id = actual_output["asyncOperationId"] - # helpers.create_recovery_request_record( - # recovery_request_record_happy_path_filename, recovery_request_record - # ) + helpers.create_recovery_request_record( + recovery_request_record_happy_path_filename, recovery_request_record + ) - # @staticmethod - # def initiate_recovery( - # boto3_session, - # recovery_request_record: helpers.RecoveryRequestRecord, - # file_bucket_maps: List[Dict], - # ): - # recovery_step_function_input = { - # "payload": { - # "granules": [ - # { - # "collectionId": granule.collection_id, - # "granuleId": granule.granule_id, - # "files": [ - # { - # "fileName": file.file_name, - # "key": file.file_key, - # "bucket": file.orca_bucket_name, - # } - # for file in granule.files - # ], - # } - # for granule in recovery_request_record.granules - # ], - # }, - # "config": { - # "buckets": helpers.buckets, - # "fileBucketMaps": file_bucket_maps, - # "excludedFileExtensions": [], - # "asyncOperationId": None, - # "s3MultipartChunksizeMb": None, - # "defaultBucketOverride": None, - # "defaultRecoveryTypeOverride": None, - # }, - # } + @staticmethod + def initiate_recovery( + boto3_session, + recovery_request_record: helpers.RecoveryRequestRecord, + file_bucket_maps: List[Dict], + ): + recovery_step_function_input = { + "payload": { + "granules": [ + { + "collectionId": granule.collection_id, + "granuleId": granule.granule_id, + "files": [ + { + "fileName": file.file_name, + "key": file.file_key, + "bucket": file.orca_bucket_name, + } + for file in granule.files + ], + } + for granule in recovery_request_record.granules + ], + }, + "config": { + "buckets": helpers.buckets, + "fileBucketMaps": file_bucket_maps, + "excludedFileExtensions": [], + "asyncOperationId": None, + "s3MultipartChunksizeMb": None, + "defaultBucketOverride": None, + "defaultRecoveryTypeOverride": None, + }, + } - # logger.info(f"Sending event to recovery step function") - # logger.info(recovery_step_function_input) + logger.info(f"Sending event to recovery step function") + logger.info(recovery_step_function_input) - # execution_info = boto3_session.client("stepfunctions").start_execution( - # stateMachineArn=helpers.orca_recovery_step_function_arn, - # input=json.dumps(recovery_step_function_input, indent=4), - # ) + execution_info = boto3_session.client("stepfunctions").start_execution( + stateMachineArn=helpers.orca_recovery_step_function_arn, + input=json.dumps(recovery_step_function_input, indent=4), + ) - # step_function_results = helpers.get_state_machine_execution_results( - # boto3_session, execution_info["executionArn"] - # ) + step_function_results = helpers.get_state_machine_execution_results( + boto3_session, execution_info["executionArn"] + ) - # return step_function_results + return step_function_results From e5231ba015ab0346b2419ea99e45c73ea5d606b2 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 7 Dec 2023 14:07:38 -0600 Subject: [PATCH 15/68] updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a7151d0d..30a00fb42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ and includes an additional section for migration notes. ### Fixed - *ORCA-731* Updated boto3 library used for unit tests to version 1.28.76 from version 1.18.40 to fix unit test warnings. +- *ORCA-722* Fixed multiple granules happy path integration tests by randomizing large file name to avoid duplicate data being ingested. ### Security From 0bfd1eb044275f8f71b6e17e867dccbf1cc4d570 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 8 Dec 2023 08:51:06 -0600 Subject: [PATCH 16/68] removed # --- .../test_multiple_granules_happy_path.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py index 50f299c12..09eeceb31 100644 --- a/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py +++ b/integration_test/workflow_tests/test_packages/ingest/test_multiple_granules_happy_path.py @@ -162,11 +162,11 @@ def test_multiple_granules_happy_path(self): step_function_results["status"], "Error occurred while starting step function.", ) - # self.assertEqual( - # expected_output, - # json.loads(step_function_results["output"]), - # "Expected ingest step function output not returned.", - # ) + self.assertEqual( + expected_output, + json.loads(step_function_results["output"]), + "Expected ingest step function output not returned.", + ) s3_versions = [] # verify that the objects exist in recovery bucket try: @@ -274,15 +274,15 @@ def test_multiple_granules_happy_path(self): f"Expected API output granules not returned. Request: {catalog_input}", ) - recovery check - self.partial_test_recovery_happy_path( - boto3_session, - collection_id, - granule_id_1, - name_1, - key_name_1, - recovery_bucket_name, - ) + # recovery check + self.partial_test_recovery_happy_path( + boto3_session, + collection_id, + granule_id_1, + name_1, + key_name_1, + recovery_bucket_name, + ) except Exception as ex: logger.error(ex) raise From d09474d1733c7932bbb6b3b45ae382ab406fb6bb Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 11 Dec 2023 10:48:38 -0600 Subject: [PATCH 17/68] Merge branch 'develop' into feature/ORCA-710 From a95754c079aae2817909f3beb93ae1cac871c63b Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 11 Dec 2023 10:49:52 -0600 Subject: [PATCH 18/68] Adding section to reduce handlers to elimate duplicate logs --- integration_test/workflow_tests/custom_logger.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_test/workflow_tests/custom_logger.py b/integration_test/workflow_tests/custom_logger.py index 17aa9de66..5ca65c5b0 100644 --- a/integration_test/workflow_tests/custom_logger.py +++ b/integration_test/workflow_tests/custom_logger.py @@ -11,8 +11,8 @@ def process(self, msg, kwargs): @staticmethod def set_logger(group_name): logger = logging.getLogger(__name__) - syslog = logging.StreamHandler() - logger.addHandler(syslog) - logger_adapter = CustomLoggerAdapter(logger, {'my_context': group_name}) - logger.propagate = False + logger_adapter = CustomLoggerAdapter(logger, {"my_context": group_name}) + if not logger.handlers: + syslog = logging.StreamHandler() + logger.addHandler(syslog) return logger_adapter From b927410ad99be27d51bfd347a46d1b1f31f08dff Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 11 Dec 2023 10:52:38 -0600 Subject: [PATCH 19/68] Update changelog.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b778d8c1..0524ae0ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,17 @@ and includes an additional section for migration notes. - *Security* - Vulnerabilities fixes and changes. ## [Unreleased] -- *ORCA-366* Adding unit test for request_method ### Migration Notes ### Added +- *ORCA-366* Adding unit test for request_method + ### Changed - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. +- *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` ### Deprecated From c7dc27ff21768e3fb78d36659297ac361fd3947e Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Wed, 13 Dec 2023 16:05:41 -0600 Subject: [PATCH 20/68] Updating documentation to replace restore with copy to ORCA --- website/docs/operator/restore-to-orca.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/docs/operator/restore-to-orca.mdx b/website/docs/operator/restore-to-orca.mdx index 83c9083c0..138061323 100644 --- a/website/docs/operator/restore-to-orca.mdx +++ b/website/docs/operator/restore-to-orca.mdx @@ -19,10 +19,10 @@ ORCA. The sections below go into further detail on how to accomplish the high level steps above. -## Submit Granules to be Restored in ORCA +## Submit Granules to be Copied to ORCA The following steps will guide the operator through submitting a bulk operation -to restore data in ORCA. +to copy data to ORCA. 1. On the **Granules** tab of the Cumulus Dashboard, click on the **Granule Actions** button. The button is highlighted in figure 1 below. @@ -82,7 +82,7 @@ to restore data in ORCA. Monitoring the Bulk Granule Operations Job and *OrcaCopyToArchiveWorkflow* workflow can be done through the *Operations* and *Executions* tabs. The Job ID -obtained from step 5 in the [Submit Granules to be Restored in ORCA](#submit-granules-to-be-restored-in-orca) +obtained from step 5 in the [Submit Granules to be Copied to ORCA](#submit-granules-to-be-copied-to-orca) section above will be needed for monitoring. The sections below go into more details. From 7d49a608fcb5845ae970a78f551d10d9d92ce036 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Wed, 13 Dec 2023 16:12:44 -0600 Subject: [PATCH 21/68] Updating CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34852d37b..c791d781f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` +- *ORCA-784* Changed documentation to replace restore with copy based on task's naming. ### Deprecated From 30002149b4f51d8b9b0b5b85ee7f44e042691362 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 14 Dec 2023 15:02:53 -0600 Subject: [PATCH 22/68] Changed file name and edit sidebars.js due to name change --- CHANGELOG.md | 2 +- .../docs/operator/{restore-to-orca.mdx => reingest-to-orca.mdx} | 2 +- website/sidebars.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename website/docs/operator/{restore-to-orca.mdx => reingest-to-orca.mdx} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c791d781f..f66be7721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` -- *ORCA-784* Changed documentation to replace restore with copy based on task's naming. +- *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. ### Deprecated diff --git a/website/docs/operator/restore-to-orca.mdx b/website/docs/operator/reingest-to-orca.mdx similarity index 99% rename from website/docs/operator/restore-to-orca.mdx rename to website/docs/operator/reingest-to-orca.mdx index 138061323..c1fed538b 100644 --- a/website/docs/operator/restore-to-orca.mdx +++ b/website/docs/operator/reingest-to-orca.mdx @@ -1,5 +1,5 @@ --- -id: restore-to-orca +id: reingest-to-orca title: Re-Ingesting Data to ORCA description: Provides documentation for Operators to restore data back to ORCA. --- diff --git a/website/sidebars.js b/website/sidebars.js index 77c74754f..df64a5972 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -89,7 +89,7 @@ module.exports = { ], "Operations": [ 'operator/data-recovery', - 'operator/restore-to-orca', + 'operator/reingest-to-orca', ], "Configuration": [ 'operator/collection-configuration', From 21a7898c6afa376281849c1eb2e4b9cb3cc4850b Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 14 Dec 2023 15:26:03 -0600 Subject: [PATCH 23/68] Replacing restore with reingest in description --- website/docs/operator/reingest-to-orca.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/operator/reingest-to-orca.mdx b/website/docs/operator/reingest-to-orca.mdx index c1fed538b..e5512b164 100644 --- a/website/docs/operator/reingest-to-orca.mdx +++ b/website/docs/operator/reingest-to-orca.mdx @@ -1,7 +1,7 @@ --- id: reingest-to-orca title: Re-Ingesting Data to ORCA -description: Provides documentation for Operators to restore data back to ORCA. +description: Provides documentation for Operators to reingest data back to ORCA. --- import useBaseUrl from '@docusaurus/useBaseUrl'; From 03a326c79eb1ac3a3211da63830a029049b1a1bb Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Fri, 15 Dec 2023 17:09:15 -0600 Subject: [PATCH 24/68] Updating input/output in data-recovery.md --- CHANGELOG.md | 1 + website/docs/operator/data-recovery.md | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f66be7721..4268e9ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. +- *ORCA-724* Updated documentation to update inputs and outputs in `website/docs/operator/data-recovery.md`. ### Deprecated diff --git a/website/docs/operator/data-recovery.md b/website/docs/operator/data-recovery.md index ef8a2c80c..a9ec45753 100644 --- a/website/docs/operator/data-recovery.md +++ b/website/docs/operator/data-recovery.md @@ -56,13 +56,12 @@ The following is an input event example that an operator might set up while runn "payload": { "granules": [ { + "collectionId": "1234", "granuleId": "integrationGranuleId", - "version": "integrationGranuleVersion", "files": [ { "fileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", "key": "MOD09GQ/006/MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", - "bucket": "test-orca-primary" } ] } @@ -111,10 +110,6 @@ The following is an input event example that an operator might set up while runn ] } }, - "cumulus_meta": { - "system_bucket": "test-internal", - "asyncOperationId": "1234" - } } ``` @@ -125,13 +120,13 @@ The following is the corresponding output that the workflow will return if succe { "granules": [ { + "collectionId": "integrationCollectionId", "granuleId": "integrationGranuleId", "version": "integrationGranuleVersion", "files": [ { "fileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", "key": "MOD09GQ/006/MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", - "bucket": "test-orca-primary" } ], "keys": [ From 18993b7331b270b609c134a923e172ea50666d98 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 18 Dec 2023 15:12:33 -0600 Subject: [PATCH 25/68] Updating changlog and docs with CMA info and collectionId --- CHANGELOG.md | 2 +- website/docs/operator/data-recovery.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4268e9ccf..b4dfd9013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. -- *ORCA-724* Updated documentation to update inputs and outputs in `website/docs/operator/data-recovery.md`. +- *ORCA-724* Updated ORCA recovery documentation to include recovery workflow process and relevant inputs and outputs in `website/docs/operator/data-recovery.md`. ### Deprecated diff --git a/website/docs/operator/data-recovery.md b/website/docs/operator/data-recovery.md index a9ec45753..3b2d8902b 100644 --- a/website/docs/operator/data-recovery.md +++ b/website/docs/operator/data-recovery.md @@ -7,6 +7,10 @@ description: Provides documentation for Operators to recover missing data. import MyImage from '@site/docs/templates/pan-zoom-image.mdx'; import useBaseUrl from '@docusaurus/useBaseUrl'; +:::important +As of ORCA v8.1 the Cumulus Message Adapter is no longer used. Users will need to deploy the recovery adapter before the recovery can be ran. Reference [Deployment with Cumulus](https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-with-cumulus#modify-the-recovery-workflow) +::: + ## Recovery via Cumulus Dashboard Recovery processes are kicked off manually by an operator through the Cumulus Dashboard. @@ -58,10 +62,12 @@ The following is an input event example that an operator might set up while runn { "collectionId": "1234", "granuleId": "integrationGranuleId", + "version": "integrationGranuleVersion", "files": [ { "fileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", "key": "MOD09GQ/006/MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", + "bucket": "test-orca-primary" } ] } @@ -110,6 +116,10 @@ The following is an input event example that an operator might set up while runn ] } }, + "cumulus_meta": { + "system_bucket": "test-internal", + "asyncOperationId": "1234" + } } ``` @@ -127,6 +137,7 @@ The following is the corresponding output that the workflow will return if succe { "fileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", "key": "MOD09GQ/006/MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", + "bucket": "test-orca-primary" } ], "keys": [ From e58db50f6281040a1e6871a94f985a0e26718d44 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Tue, 19 Dec 2023 08:45:09 -0600 Subject: [PATCH 26/68] used python's search function instead of match in regex --- CHANGELOG.md | 1 + .../extract_filepaths_for_granule.py | 5 +++-- .../test/unit_tests/test_extract_file_paths_for_granule.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f66be7721..c395276d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. +- *ORCA-789* Updated `extract_filepaths_for_granule` to more flexibly match file-regex values to keys. ### Deprecated diff --git a/tasks/extract_filepaths_for_granule/extract_filepaths_for_granule.py b/tasks/extract_filepaths_for_granule/extract_filepaths_for_granule.py index b18e7f7fb..7fc370b05 100644 --- a/tasks/extract_filepaths_for_granule/extract_filepaths_for_granule.py +++ b/tasks/extract_filepaths_for_granule/extract_filepaths_for_granule.py @@ -8,9 +8,10 @@ import re from typing import Any, Dict, List -import fastjsonschema as fastjsonschema from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities.typing import LambdaContext + +import fastjsonschema as fastjsonschema from fastjsonschema import JsonSchemaException # Set AWS powertools logger @@ -110,7 +111,7 @@ def task(task_input: Dict[str, Any], config: Dict[str, Any]): else: matching_regex = next( filter( - lambda key: re.compile(key).match(file_name), regex_buckets + lambda key: re.compile(key).search(file_name), regex_buckets ), None, ) diff --git a/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py b/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py index 5fba32a8a..15cf22adf 100644 --- a/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py +++ b/tasks/extract_filepaths_for_granule/test/unit_tests/test_extract_file_paths_for_granule.py @@ -9,11 +9,11 @@ from test.helpers import create_handler_event, create_task_event from unittest.mock import MagicMock, Mock, patch +import extract_filepaths_for_granule + # noinspection PyPackageRequirements import fastjsonschema as fastjsonschema -import extract_filepaths_for_granule - # Generating schema validators can take time, so do it once and reuse. with open("schemas/input.json", "r") as raw_schema: _INPUT_VALIDATE = fastjsonschema.compile(json.loads(raw_schema.read())) From 80731381d680d3e97891ddc34f4b45d98959250a Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 28 Dec 2023 11:28:46 -0600 Subject: [PATCH 27/68] fixed nodejs installation step in docker image --- base_images/Dockerfile.bamboo | 9 +++++---- base_images/Dockerfile.local | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/base_images/Dockerfile.bamboo b/base_images/Dockerfile.bamboo index 1a38f8773..66f41fd9e 100644 --- a/base_images/Dockerfile.bamboo +++ b/base_images/Dockerfile.bamboo @@ -7,16 +7,17 @@ ENV NODE_VERSION="20.x" ENV TERRAFORM_VERSION "1.5.5" ENV PYTHON_VERSION "3.9.17" -# Add NodeJS and Yarn repos & update package index +# Add repos & update package index RUN \ -curl -sL https://rpm.nodesource.com/setup_${NODE_VERSION} | bash - && \ curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" && \ dnf update -y -# install pip, Python 3.9, CLI utilities, session manager and other tools +# Install node, pip, Python 3.9, CLI utilities, session manager and other tools RUN \ -dnf install -y jq gcc git make openssl openssl-devel wget zip unzip bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel uuid-devel libuuid-devel gdbm-devel xz-devel tar nodejs yarn awscli session-manager-plugin.rpm procps parallel && \ +dnf install https://rpm.nodesource.com/pub_${NODE_VERSION}/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y && \ +dnf install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 && \ +dnf install -y jq gcc git make openssl openssl-devel wget zip unzip bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel uuid-devel libuuid-devel gdbm-devel xz-devel tar yarn awscli session-manager-plugin.rpm procps parallel && \ dnf groupinstall -y "Development Tools" && \ # Install Python 3.9 wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ diff --git a/base_images/Dockerfile.local b/base_images/Dockerfile.local index ca16c361b..ba6b31bec 100644 --- a/base_images/Dockerfile.local +++ b/base_images/Dockerfile.local @@ -9,14 +9,15 @@ ENV PYTHON_VERSION "3.9.17" # Add NodeJS and Yarn repos & update package index RUN \ -curl -sL https://rpm.nodesource.com/setup_${NODE_VERSION} | bash - && \ curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" && \ dnf update -y -# install pip, Python 3.9, CLI utilities, session manager and other tools +# Install node, pip, Python 3.9, CLI utilities, session manager and other tools RUN \ -dnf install -y jq gcc git make openssl openssl-devel wget zip unzip bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel uuid-devel libuuid-devel gdbm-devel xz-devel tar nodejs yarn awscli session-manager-plugin.rpm procps parallel && \ +dnf install https://rpm.nodesource.com/pub_${NODE_VERSION}/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y && \ +dnf install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 && \ +dnf install -y jq gcc git make openssl openssl-devel wget zip unzip bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel uuid-devel libuuid-devel gdbm-devel xz-devel tar yarn awscli session-manager-plugin.rpm procps parallel && \ dnf groupinstall -y "Development Tools" && \ # Install Python 3.9 wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ From 69c43313aeee5ca1b5cac0bea572831b0ac48405 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 29 Dec 2023 07:49:10 -0600 Subject: [PATCH 28/68] minor fix --- base_images/Dockerfile.bamboo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_images/Dockerfile.bamboo b/base_images/Dockerfile.bamboo index 66f41fd9e..8a909e19d 100644 --- a/base_images/Dockerfile.bamboo +++ b/base_images/Dockerfile.bamboo @@ -7,7 +7,7 @@ ENV NODE_VERSION="20.x" ENV TERRAFORM_VERSION "1.5.5" ENV PYTHON_VERSION "3.9.17" -# Add repos & update package index +# Add YARN repos & update package index RUN \ curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" && \ From 3d657cc0630bad2bc86d4045e3a093a7685df6e9 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Mon, 1 Jan 2024 08:13:01 -0600 Subject: [PATCH 29/68] updated changelog --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be15a5658..de3fdcbe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,10 @@ and includes an additional section for migration notes. ### Added -- *ORCA-366* Adding unit test for request_method +- *ORCA-366* Added unit test for shared libraries. ### Changed -- *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. -- *ORCA-710* Removing duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. - *ORCA-724* Updated ORCA recovery documentation to include recovery workflow process and relevant inputs and outputs in `website/docs/operator/data-recovery.md`. - *ORCA-789* Updated `extract_filepaths_for_granule` to more flexibly match file-regex values to keys. @@ -33,8 +31,13 @@ and includes an additional section for migration notes. ### Removed +- *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. +- *ORCA-710* Removed duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` + ### Fixed +- *ORCA-811* Fixed `cumulus_orca` docker image by updating nodejs installation process. + ### Security ## [9.0.1] 2023-11-16 From 62c75c05a77da676cf247d3e2b49bc2a30077a15 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 4 Jan 2024 16:04:25 -0600 Subject: [PATCH 30/68] Updated API gateway path --- CHANGELOG.md | 1 + modules/api-gateway/main.tf | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de3fdcbe0..2cd7d021c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and includes an additional section for migration notes. - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. - *ORCA-724* Updated ORCA recovery documentation to include recovery workflow process and relevant inputs and outputs in `website/docs/operator/data-recovery.md`. - *ORCA-789* Updated `extract_filepaths_for_granule` to more flexibly match file-regex values to keys. +- *ORCA-787* Updated `modules/api-gateway/main.tf` to remove extra `orca` in path to avoid confusion. ### Deprecated diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index 387ef8410..fab4be19f 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -233,17 +233,10 @@ resource "aws_lambda_permission" "request_status_for_job_api_permission" { source_arn = "${aws_api_gateway_rest_api.orca_api.execution_arn}/*/${aws_api_gateway_method.request_status_for_job_api_method.http_method}${aws_api_gateway_resource.request_status_for_job_api_resource_recovery_jobs.path}" } -## API resource for pathing orca/ -resource "aws_api_gateway_resource" "orca_api_resource" { - path_part = "orca" - parent_id = aws_api_gateway_rest_api.orca_api.root_resource_id - rest_api_id = aws_api_gateway_rest_api.orca_api.id -} - ## API resource for pathing orca/datamanagement resource "aws_api_gateway_resource" "orca_datamanagement_api_resource" { path_part = "datamanagement" - parent_id = aws_api_gateway_resource.orca_api_resource.id + parent_id = aws_api_gateway_rest_api.orca_api.root_resource_id rest_api_id = aws_api_gateway_rest_api.orca_api.id } From 6038c4f127124804c0adb380072f14195bfea4b5 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Fri, 5 Jan 2024 16:13:13 -0600 Subject: [PATCH 31/68] Changed stage name --- modules/api-gateway/main.tf | 11 +++++++++-- modules/api-gateway/variables.tf | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index fab4be19f..3582b7463 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -233,10 +233,17 @@ resource "aws_lambda_permission" "request_status_for_job_api_permission" { source_arn = "${aws_api_gateway_rest_api.orca_api.execution_arn}/*/${aws_api_gateway_method.request_status_for_job_api_method.http_method}${aws_api_gateway_resource.request_status_for_job_api_resource_recovery_jobs.path}" } +## API resource for pathing orca/ +resource "aws_api_gateway_resource" "orca_api_resource" { + path_part = "orca" + parent_id = aws_api_gateway_rest_api.orca_api.root_resource_id + rest_api_id = aws_api_gateway_rest_api.orca_api.id +} + ## API resource for pathing orca/datamanagement resource "aws_api_gateway_resource" "orca_datamanagement_api_resource" { path_part = "datamanagement" - parent_id = aws_api_gateway_rest_api.orca_api.root_resource_id + parent_id = aws_api_gateway_resource.orca_api_resource.id rest_api_id = aws_api_gateway_rest_api.orca_api.id } @@ -530,4 +537,4 @@ resource "aws_api_gateway_deployment" "orca_api_deployment" { aws_api_gateway_integration.internal_reconcile_report_phantom_api_integration, aws_api_gateway_integration.internal_reconcile_report_mismatch_api_integration ] -} +} \ No newline at end of file diff --git a/modules/api-gateway/variables.tf b/modules/api-gateway/variables.tf index f15bbc6e7..b88e6314a 100644 --- a/modules/api-gateway/variables.tf +++ b/modules/api-gateway/variables.tf @@ -54,7 +54,7 @@ variable "orca_catalog_reporting_invoke_arn" { variable "api_gateway_stage_name" { type = string description = "stage name for the ORCA cumulus reconciliation api gateway" - default = "orca" + default = "api" } variable "vpc_endpoint_id" { From 4e0cea9901aef8ead1387b56c0220964524a83df Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 8 Jan 2024 10:58:06 -0600 Subject: [PATCH 32/68] Adding create_before_destroy for deployment --- modules/api-gateway/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index 3582b7463..e0820069d 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -537,4 +537,7 @@ resource "aws_api_gateway_deployment" "orca_api_deployment" { aws_api_gateway_integration.internal_reconcile_report_phantom_api_integration, aws_api_gateway_integration.internal_reconcile_report_mismatch_api_integration ] + lifecycle { + create_before_destroy = true + } } \ No newline at end of file From 53dd27b41f299406c147bbecc1b9a314879f5531 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Mon, 8 Jan 2024 13:53:32 -0600 Subject: [PATCH 33/68] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd7d021c..171769097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ and includes an additional section for migration notes. - *ORCA-784* Changed documentation to replace restore with copy based on task's naming as well as changed file name from `website/docs/operator/restore-to-orca.mdx` to `website/docs/operator/reingest-to-orca.mdx`. - *ORCA-724* Updated ORCA recovery documentation to include recovery workflow process and relevant inputs and outputs in `website/docs/operator/data-recovery.md`. - *ORCA-789* Updated `extract_filepaths_for_granule` to more flexibly match file-regex values to keys. -- *ORCA-787* Updated `modules/api-gateway/main.tf` to remove extra `orca` in path to avoid confusion. +- *ORCA-787* Modified `modules/api-gateway/main.tf` api gateway stage name to remove the extra orca from the data management URL path ### Deprecated From f21cd063de8fcce3ae70b417efb7f627c10061cc Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Tue, 9 Jan 2024 10:20:37 -0600 Subject: [PATCH 34/68] fixed extract_filepath schemas collectionId --- tasks/extract_filepaths_for_granule/README.md | 2 ++ tasks/extract_filepaths_for_granule/schemas/input.json | 2 +- tasks/extract_filepaths_for_granule/schemas/output.json | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tasks/extract_filepaths_for_granule/README.md b/tasks/extract_filepaths_for_granule/README.md index 2f87050e4..e3f4dbbc2 100644 --- a/tasks/extract_filepaths_for_granule/README.md +++ b/tasks/extract_filepaths_for_granule/README.md @@ -90,6 +90,7 @@ The lambda handler event expects a dictionary having a list of granules as input "granules":[ { "granuleId":"MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": "MOD14A1___061", "dataType":"MOD09GQ_test-jk2-IngestGranuleSuccess-1558420117156", "version":"006", "files":[ @@ -158,6 +159,7 @@ The output of lambda handler returns a dictionary having a list of granules that "granules":[ { "granuleId":"MOD09GQ.A0219114.N5aUCG.006.0656338553321", + "collectionId": "MOD14A1___061", "keys":[ { "key": "MOD09GQ___006/2017/MOD/MOD09GQ.A0219114.N5aUCG.006.0656338553321.h5", diff --git a/tasks/extract_filepaths_for_granule/schemas/input.json b/tasks/extract_filepaths_for_granule/schemas/input.json index 6422f309c..2a0330c30 100644 --- a/tasks/extract_filepaths_for_granule/schemas/input.json +++ b/tasks/extract_filepaths_for_granule/schemas/input.json @@ -14,7 +14,7 @@ "description": "The user specified recovery S3 bucket to store recovered files", "type": ["string", "null"] }, - "collectionId": { + "collectionId": { "description": "The id of the collection containing the granule.", "type": "string" }, diff --git a/tasks/extract_filepaths_for_granule/schemas/output.json b/tasks/extract_filepaths_for_granule/schemas/output.json index 2cf35d43c..887b6deb3 100644 --- a/tasks/extract_filepaths_for_granule/schemas/output.json +++ b/tasks/extract_filepaths_for_granule/schemas/output.json @@ -11,8 +11,8 @@ "type": "object", "properties": { "collectionId": { - "description": "The id of the collection containing the granule.", - "type": "string" + "description": "The id of the collection containing the granule.", + "type": "string" }, "granuleId": { "description": "The ID of the granule.", From 76fc50ce20bd03b004b0f457926fe9566cc0cdd4 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich Date: Thu, 11 Jan 2024 16:16:37 -0600 Subject: [PATCH 35/68] resolved security groups error when upgrading to v9, removed unneeded dependency, and update changelog --- CHANGELOG.md | 1 + modules/orca/main.tf | 2 +- modules/security_groups/main.tf | 4 ++-- modules/security_groups/output.tf | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 171769097..879781c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and includes an additional section for migration notes. - *ORCA-724* Updated ORCA recovery documentation to include recovery workflow process and relevant inputs and outputs in `website/docs/operator/data-recovery.md`. - *ORCA-789* Updated `extract_filepaths_for_granule` to more flexibly match file-regex values to keys. - *ORCA-787* Modified `modules/api-gateway/main.tf` api gateway stage name to remove the extra orca from the data management URL path +- *ORCA-805* Changed `modules/security_groups/main.tf` security group resource name from `vpc_postgres_ingress_all_egress` to `vpc-postgres-ingress-all-egress` to resolve errors when upgrading from ORCA v8 to v9. Also removed graphql_1 dependency `module.orca_lambdas` since this module does not depend on the lambda module in `modules/orca/main.tf` ### Deprecated diff --git a/modules/orca/main.tf b/modules/orca/main.tf index 6d88bcab2..3ad24c912 100644 --- a/modules/orca/main.tf +++ b/modules/orca/main.tf @@ -242,7 +242,7 @@ module "orca_ecs" { ## ============================= module "orca_graphql_1" { source = "../graphql_1" - depends_on = [module.orca_lambdas, module.orca_ecs, module.orca_graphql_0, module.orca_secretsmanager] ## secretsmanager sets up db connection secrets. + depends_on = [module.orca_ecs, module.orca_graphql_0, module.orca_secretsmanager] ## secretsmanager sets up db connection secrets. ## -------------------------- ## Cumulus Variables ## -------------------------- diff --git a/modules/security_groups/main.tf b/modules/security_groups/main.tf index cc5611afa..fdb4f980c 100644 --- a/modules/security_groups/main.tf +++ b/modules/security_groups/main.tf @@ -2,7 +2,7 @@ ## vpc_postgres_ingress_all_egress - PostgreSQL Security Group ## ============================================================================== -resource "aws_security_group" "vpc_postgres_ingress_all_egress" { +resource "aws_security_group" "vpc-postgres-ingress-all-egress" { ## OPTIONAL description = "ORCA security group to allow PostgreSQL access." name = "${var.prefix}-vpc-ingress-all-egress" @@ -38,6 +38,6 @@ resource "aws_security_group_rule" "rds_allow_lambda_access" { to_port = 5432 protocol = "TCP" description = "Allows ${var.prefix} Orca lambda access." - source_security_group_id = aws_security_group.vpc_postgres_ingress_all_egress.id + source_security_group_id = aws_security_group.vpc-postgres-ingress-all-egress.id security_group_id = var.rds_security_group_id } \ No newline at end of file diff --git a/modules/security_groups/output.tf b/modules/security_groups/output.tf index c3d6aafb5..914606ff1 100644 --- a/modules/security_groups/output.tf +++ b/modules/security_groups/output.tf @@ -1,4 +1,4 @@ output "vpc_postgres_ingress_all_egress_id" { - value = aws_security_group.vpc_postgres_ingress_all_egress.id + value = aws_security_group.vpc-postgres-ingress-all-egress.id description = "PostgreSQL security group id" } \ No newline at end of file From 98bcf7527ec945e539a5e269f068464ba5dad1c5 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 12 Jan 2024 07:43:12 -0600 Subject: [PATCH 36/68] updated CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 171769097..087773770 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ and includes an additional section for migration notes. ### Fixed - *ORCA-811* Fixed `cumulus_orca` docker image by updating nodejs installation process. +- *ORCA-802* Fixed `extract_file_for_granule` documentation and schemas to include `collectionId` in input. + ### Security From db7d0db302afe4a05f23d164a6214d407fd295e5 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Sun, 14 Jan 2024 21:48:43 -0600 Subject: [PATCH 37/68] updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087773770..d9671dc13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ and includes an additional section for migration notes. - *ORCA-361* Removed hardcoded test values from `extract_file_paths_for_granule` unit tests. - *ORCA-710* Removed duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` +- *ORCA-815* Removed steps for creating buckets using NGAP form in ORCA archive bucket documentation. + ### Fixed From 9885dc1f2268af187317e9c75b41ddbb365e409f Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Mon, 15 Jan 2024 09:49:55 -0600 Subject: [PATCH 38/68] updated website doc to remove NGAP steps --- .../creating-orca-archive-bucket.md | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md index b802acad7..b6c86b236 100644 --- a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md +++ b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md @@ -81,55 +81,6 @@ Error: Error putting S3 notification configuration: AccessDenied: Access Denied ::: -### Via NGAP form - -If your accounts are both within EDC, you may skip to [the primary method](#via-aws-gui). -Otherwise, create a NASD ticket for cross account bucket access. -This is a turn key request to NGAP. The link to create a ticket is available -[here](https://bugs.earthdata.nasa.gov/servicedesk/customer/portal/7/create/85). - -The sections below provide information on the data needed -for each of the fields and where to look for information. - -#### Project Name - -This is the name of the Disaster Recover OU where the ORCA archive bucket resides. -The project name can be found in your [Cloud Tamer](http://cloud.earthdata.nasa.gov/) -account and is usually in the format of \[project name\]-app-\[application name\]-\[environment\]-\[4 digit number\]. -For example, an ORCA disaster recovery OU project name may look like the following -orca-app-dr-sandbox-1234. - -#### Account Type: - -This is the OU environment the bucket resides in. Typical values for this field -are Sandbox, SIT, UAT, and Production. - -#### Business Justification: - -This is the business justification for the cross account bucket access. Below is -an example of a justification. - -> The ORCA Cumulus application in the Cumulus Sandbox OU needs to read/write to -> the ORCA DR account S3 buckets in order to create an operational archive copy of -> ORCA data and recover data back to the primary Cumulus data holdings in case -> of a failure. Note that only `GLACIER` and `DEEP_ARCHIVE` storage types are allowed for objects written to the bucket. This cross account access will allow the Cumulus application to -> seamlessly perform these functions and provide operators with the capability to -> test and verify disaster recovery scenarios. - -#### Bucket Names(s): - -This is the name of the ORCA archive bucket created in the Disaster Recover OU. -Below is an example name of an ORCA archive bucket and ORCA report bucket. - -> PREFIX-orca-archive -> PREFIX-orca-reports - -#### Policy: - -The policy section is the JSON policy requested for the ORCA archive bucket in -the Disaster Recovery OU. -See [the section below](#via-aws-gui) for policy document examples. - ### Via AWS CloudFormation Template The AWS Cloudformation template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets_cloudformation/dr-buckets.yaml). Make sure you have AWS CLI installed before deploying this template. From 2e8ab8e3dbe2a1d130650aeb0757e3108626ceba Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Wed, 17 Jan 2024 07:54:32 -0600 Subject: [PATCH 39/68] updated package lock --- website/package-lock.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index f03f76890..fc39ee57c 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -20624,7 +20624,7 @@ "parse-entities": "^2.0.0", "repeat-string": "^1.5.4", "state-toggle": "^1.0.0", - "trim": "0.0.1", + "trim": "0.0.3", "trim-trailing-lines": "^1.0.0", "unherit": "^1.0.4", "unist-util-remove-position": "^2.0.0", @@ -20633,8 +20633,8 @@ }, "dependencies": { "trim": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.1.tgz" + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.3.tgz" } } }, @@ -21594,7 +21594,7 @@ }, "trim": { "version": "0.0.3", - "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.3.tgz" + "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.3.tgz", }, "trim-trailing-lines": { "version": "1.1.4", @@ -22460,4 +22460,4 @@ "integrity": "sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==" } } -} +} \ No newline at end of file From 34793f0a00a47ca52043d19b852ebd12e6b93c5b Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Mon, 22 Jan 2024 07:54:47 -0600 Subject: [PATCH 40/68] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9671dc13..013632a42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and includes an additional section for migration notes. - *ORCA-811* Fixed `cumulus_orca` docker image by updating nodejs installation process. - *ORCA-802* Fixed `extract_file_for_granule` documentation and schemas to include `collectionId` in input. +- *ORCA-785* Fixed checksum integrity issue in ORCA documentation bamboo pipeline. ### Security From 0881ea95d9c5c3a8627f834a0a5bb53614b28128 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Mon, 22 Jan 2024 10:11:59 -0600 Subject: [PATCH 41/68] fixed parsing error --- website/package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/package-lock.json b/website/package-lock.json index fc39ee57c..9f3256a20 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -21594,7 +21594,7 @@ }, "trim": { "version": "0.0.3", - "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.3.tgz", + "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.3.tgz" }, "trim-trailing-lines": { "version": "1.1.4", From b925077a7a0cb661138a5909660781b6dae46abd Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:34:39 -0600 Subject: [PATCH 42/68] Added API Gateway Stage resource --- modules/api-gateway/main.tf | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index e0820069d..a29795c24 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -527,7 +527,7 @@ resource "aws_lambda_permission" "internal_reconcile_report_mismatch_api_permiss #deployment for the API resource "aws_api_gateway_deployment" "orca_api_deployment" { rest_api_id = aws_api_gateway_rest_api.orca_api.id - stage_name = var.api_gateway_stage_name + depends_on = [ aws_api_gateway_integration.orca_catalog_reporting_api_integration, aws_api_gateway_integration.request_status_for_job_api_integration, @@ -537,7 +537,11 @@ resource "aws_api_gateway_deployment" "orca_api_deployment" { aws_api_gateway_integration.internal_reconcile_report_phantom_api_integration, aws_api_gateway_integration.internal_reconcile_report_mismatch_api_integration ] - lifecycle { - create_before_destroy = true - } -} \ No newline at end of file +} + +# Stage for API Gateway +resource "aws_api_gateway_stage" "orca_api_stage" { + deployment_id = aws_api_gateway_deployment.orca_api_deployment.id + rest_api_id = aws_api_gateway_rest_api.orca_api.id + stage_name = var.api_gateway_stage_name +} From 2b1f0052ee8a890a2fb5e07366e80dbc22debcad Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:35:29 -0600 Subject: [PATCH 43/68] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9671dc13..297260346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and includes an additional section for migration notes. ### Added - *ORCA-366* Added unit test for shared libraries. +- *ORCA-769* Added API Gateway Stage resource to `modules/api-gateway/main.tf` ### Changed From 1a9b61222a4a7f893d73692a9dd031155cc9b40e Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Mon, 22 Jan 2024 11:35:39 -0600 Subject: [PATCH 44/68] removed sha integrity checksum --- website/package-lock.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 9f3256a20..bc5a42601 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -4150,7 +4150,6 @@ "node_modules/axios": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.0.tgz", - "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", "dependencies": { "follow-redirects": "^1.14.7" } @@ -16138,7 +16137,6 @@ "axios": { "version": "0.25.0", "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", - "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", "requires": { "follow-redirects": "^1.14.7" } From 10df5a31ccd7654259056435ce80ce4687507076 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:37:59 -0600 Subject: [PATCH 45/68] Readding in lifecycle rule on deployment --- modules/api-gateway/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index a29795c24..b0d0bcbc0 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -537,6 +537,10 @@ resource "aws_api_gateway_deployment" "orca_api_deployment" { aws_api_gateway_integration.internal_reconcile_report_phantom_api_integration, aws_api_gateway_integration.internal_reconcile_report_mismatch_api_integration ] + + lifecycle { + create_before_destroy = true + } } # Stage for API Gateway From 5a97ff0dfddbc2a0f55947e9f2f3375d37fe3a24 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:44:11 -0600 Subject: [PATCH 46/68] Reverting resource name --- modules/security_groups/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/security_groups/main.tf b/modules/security_groups/main.tf index fdb4f980c..5ce78c283 100644 --- a/modules/security_groups/main.tf +++ b/modules/security_groups/main.tf @@ -2,7 +2,7 @@ ## vpc_postgres_ingress_all_egress - PostgreSQL Security Group ## ============================================================================== -resource "aws_security_group" "vpc-postgres-ingress-all-egress" { +resource "aws_security_group" "vpc_postgres_ingress_all_egress" { ## OPTIONAL description = "ORCA security group to allow PostgreSQL access." name = "${var.prefix}-vpc-ingress-all-egress" @@ -40,4 +40,4 @@ resource "aws_security_group_rule" "rds_allow_lambda_access" { description = "Allows ${var.prefix} Orca lambda access." source_security_group_id = aws_security_group.vpc-postgres-ingress-all-egress.id security_group_id = var.rds_security_group_id -} \ No newline at end of file +} From bbf849a179b75fed5632beaf92f2921e346bcd1e Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:45:30 -0600 Subject: [PATCH 47/68] Adding Lambda deletion script --- bin/delete_lambda.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 bin/delete_lambda.py diff --git a/bin/delete_lambda.py b/bin/delete_lambda.py new file mode 100644 index 000000000..8a727a418 --- /dev/null +++ b/bin/delete_lambda.py @@ -0,0 +1,18 @@ +import subprocess +import json + +# Gets user input for the prefix of lambdas to delete +prefix = input("Enter Prefix: ") + +# Gets ORCA lambda functions with given prefix +get_functions = f"aws lambda list-functions --query 'Functions[] | [?contains(FunctionName, `{prefix}`) == `true`]'" +completed_process = subprocess.run(get_functions, shell=True, capture_output=True) +output = completed_process.stdout +convert = json.loads(output.decode("utf-8").replace("'","'")) + +# Deletes ORCA lambda functions with the given prefix +for sub in convert: + print("Deleting " + sub['FunctionName']) + lambda_output = sub['FunctionName'] + delete_function = f"aws lambda delete-function --function-name {lambda_output}" + subprocess.run(delete_function, shell=True, capture_output=True) From 44ee79672964f1e4dc9901671f0e279277227f06 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Mon, 22 Jan 2024 12:00:21 -0600 Subject: [PATCH 48/68] Updated migration documentation for v8 to v9 steps --- .../developer/deployment-guide/deployment-upgrading-orca.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md index 09f968ec1..6a3f77c79 100644 --- a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md +++ b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md @@ -75,7 +75,11 @@ To update your ORCA version: 1. Find the desired release on the [ORCA Releases page](https://github.com/nasa/cumulus-orca/releases). 2. Update the `source` in your Terraform deployment files for ORCA by -replacing `vx.x.x` with the desired version of ORCA. +replacing `vx.x.x` with the desired version of ORCA. If upgrading from `v8.x.x` to `v9.x.x` follow the below steps: + 1. Run the Lambda deletion script found in `bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. + 2. Navigate to the AWS console and search for the Cumulus RDS security group. + 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. + 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minuts and then try again. 3. Run `terraform init` to get the latest copies of your updated modules. From 201a02dfb86887820c3b72ad92ab017f9da73e3a Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:45:05 -0600 Subject: [PATCH 49/68] Fix spelling error --- .../developer/deployment-guide/deployment-upgrading-orca.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md index 6a3f77c79..df0789ce0 100644 --- a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md +++ b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md @@ -79,7 +79,7 @@ replacing `vx.x.x` with the desired version of ORCA. If upgrading from `v8.x.x` 1. Run the Lambda deletion script found in `bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. 2. Navigate to the AWS console and search for the Cumulus RDS security group. 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. - 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minuts and then try again. + 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minutes and then try again. 3. Run `terraform init` to get the latest copies of your updated modules. From 7c56348197cd9d041a7aa725b522d2880574ca6c Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Tue, 23 Jan 2024 15:15:06 -0600 Subject: [PATCH 50/68] upgraded bandit and moto libraries --- .../requirements-dev.txt | 2 +- graphql/requirements-dev.txt | 2 +- .../workflow_tests/requirements.txt | 2 +- shared_libraries/requirements.txt | 4 +-- tasks/copy_from_archive/requirements-dev.txt | 4 +-- tasks/copy_to_archive/requirements-dev.txt | 4 +-- .../requirements-dev.txt | 2 +- tasks/db_deploy/requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 34 ++++++++++++------- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 4 +-- tasks/post_to_catalog/requirements-dev.txt | 2 +- tasks/post_to_database/requirements-dev.txt | 2 +- .../requirements-dev.txt | 4 +-- .../request_from_archive/requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- .../requirements-dev.txt | 2 +- 25 files changed, 51 insertions(+), 41 deletions(-) diff --git a/ecs_tasks/internal_reconcile_report_generate/requirements-dev.txt b/ecs_tasks/internal_reconcile_report_generate/requirements-dev.txt index 815315432..9255e1bca 100644 --- a/ecs_tasks/internal_reconcile_report_generate/requirements-dev.txt +++ b/ecs_tasks/internal_reconcile_report_generate/requirements-dev.txt @@ -8,7 +8,7 @@ requests~=2.28.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/graphql/requirements-dev.txt b/graphql/requirements-dev.txt index ef40ff289..0d5831fb5 100644 --- a/graphql/requirements-dev.txt +++ b/graphql/requirements-dev.txt @@ -9,7 +9,7 @@ boto3~=1.18.65 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/integration_test/workflow_tests/requirements.txt b/integration_test/workflow_tests/requirements.txt index b12576c67..24a7427b8 100644 --- a/integration_test/workflow_tests/requirements.txt +++ b/integration_test/workflow_tests/requirements.txt @@ -6,7 +6,7 @@ dataclasses-json ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 \ No newline at end of file diff --git a/shared_libraries/requirements.txt b/shared_libraries/requirements.txt index eea80fee9..abb208d92 100644 --- a/shared_libraries/requirements.txt +++ b/shared_libraries/requirements.txt @@ -5,7 +5,7 @@ pydoc-markdown>=4.0.0,<5.0.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 @@ -22,7 +22,7 @@ coverage==7.2.7 psycopg2-binary==2.9.2 ## Libraries used by recovery package -moto[sqs]==4.2.2 +moto[sqs]==4.2.13 ## Libraries used by reconciliation package # None diff --git a/tasks/copy_from_archive/requirements-dev.txt b/tasks/copy_from_archive/requirements-dev.txt index 324c3bc2e..acb9fc1b8 100644 --- a/tasks/copy_from_archive/requirements-dev.txt +++ b/tasks/copy_from_archive/requirements-dev.txt @@ -1,14 +1,14 @@ boto3==1.18.40 pytest==7.4.0 coverage==7.2.7 -moto[sqs,s3]==4.2.2 +moto[sqs,s3]==4.2.13 fastjsonschema==2.15.0 aws_lambda_powertools==1.31.0 ../../shared_libraries[recovery] ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/copy_to_archive/requirements-dev.txt b/tasks/copy_to_archive/requirements-dev.txt index 750df6172..fa703f7dd 100644 --- a/tasks/copy_to_archive/requirements-dev.txt +++ b/tasks/copy_to_archive/requirements-dev.txt @@ -3,11 +3,11 @@ pytest==7.4.0 aws_lambda_powertools==1.31.0 coverage==7.2.7 fastjsonschema~=2.15.1 -moto[sqs]==4.2.2 +moto[sqs]==4.2.13 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/copy_to_archive_adapter/requirements-dev.txt b/tasks/copy_to_archive_adapter/requirements-dev.txt index 1976caaa4..a0b823c1a 100644 --- a/tasks/copy_to_archive_adapter/requirements-dev.txt +++ b/tasks/copy_to_archive_adapter/requirements-dev.txt @@ -7,7 +7,7 @@ fastjsonschema==2.15.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/db_deploy/requirements-dev.txt b/tasks/db_deploy/requirements-dev.txt index aceba94f1..68e3cd728 100644 --- a/tasks/db_deploy/requirements-dev.txt +++ b/tasks/db_deploy/requirements-dev.txt @@ -5,7 +5,7 @@ pydoc-markdown==4.5.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/delete_old_reconcile_jobs/requirements-dev.txt b/tasks/delete_old_reconcile_jobs/requirements-dev.txt index 23e0f362a..962824731 100644 --- a/tasks/delete_old_reconcile_jobs/requirements-dev.txt +++ b/tasks/delete_old_reconcile_jobs/requirements-dev.txt @@ -12,7 +12,7 @@ boto3==1.18.40 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/extract_filepaths_for_granule/requirements-dev.txt b/tasks/extract_filepaths_for_granule/requirements-dev.txt index 4e8084343..87e6f9164 100644 --- a/tasks/extract_filepaths_for_granule/requirements-dev.txt +++ b/tasks/extract_filepaths_for_granule/requirements-dev.txt @@ -5,7 +5,7 @@ fastjsonschema==2.15.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/get_current_archive_list/requirements-dev.txt b/tasks/get_current_archive_list/requirements-dev.txt index 46bc4d3f7..68e3cd728 100644 --- a/tasks/get_current_archive_list/requirements-dev.txt +++ b/tasks/get_current_archive_list/requirements-dev.txt @@ -1,19 +1,29 @@ -## Libraries needed for testing -## Libraries needed for testing -coverage==7.2.7 -pytest==7.4.0 +## Library used for API documentation +## --------------------------------------------------------------------------- +pydoc-markdown==4.5.0 -## Application libraries needed for testing -psycopg2-binary==2.8.6 - -## Application libraries -aws_lambda_powertools==1.31.0 -fastjsonschema~=2.15.1 -../../shared_libraries[database, reconciliation] ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 + + +## Standard unit test libraries +## --------------------------------------------------------------------------- +pytest==7.4.0 +coverage==7.2.7 + + +## Libraries needed for unit tests +## --------------------------------------------------------------------------- +psycopg2-binary==2.8.6 + + +## Libraries needed by application +## --------------------------------------------------------------------------- +aws_lambda_powertools==1.31.0 +SQLAlchemy~=2.0.5 +../../shared_libraries[database, reconciliation] diff --git a/tasks/internal_reconcile_report_job/requirements-dev.txt b/tasks/internal_reconcile_report_job/requirements-dev.txt index ba8c6fb34..7fa44e926 100644 --- a/tasks/internal_reconcile_report_job/requirements-dev.txt +++ b/tasks/internal_reconcile_report_job/requirements-dev.txt @@ -12,7 +12,7 @@ fastjsonschema~=2.15.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/internal_reconcile_report_mismatch/requirements-dev.txt b/tasks/internal_reconcile_report_mismatch/requirements-dev.txt index 026f1e5c1..d1168f2b0 100644 --- a/tasks/internal_reconcile_report_mismatch/requirements-dev.txt +++ b/tasks/internal_reconcile_report_mismatch/requirements-dev.txt @@ -12,7 +12,7 @@ fastjsonschema~=2.15.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 \ No newline at end of file diff --git a/tasks/internal_reconcile_report_orphan/requirements-dev.txt b/tasks/internal_reconcile_report_orphan/requirements-dev.txt index ba8c6fb34..7fa44e926 100644 --- a/tasks/internal_reconcile_report_orphan/requirements-dev.txt +++ b/tasks/internal_reconcile_report_orphan/requirements-dev.txt @@ -12,7 +12,7 @@ fastjsonschema~=2.15.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/internal_reconcile_report_phantom/requirements-dev.txt b/tasks/internal_reconcile_report_phantom/requirements-dev.txt index ba8c6fb34..7fa44e926 100644 --- a/tasks/internal_reconcile_report_phantom/requirements-dev.txt +++ b/tasks/internal_reconcile_report_phantom/requirements-dev.txt @@ -12,7 +12,7 @@ fastjsonschema~=2.15.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/orca_catalog_reporting/requirements-dev.txt b/tasks/orca_catalog_reporting/requirements-dev.txt index f044900d5..92d06edd6 100644 --- a/tasks/orca_catalog_reporting/requirements-dev.txt +++ b/tasks/orca_catalog_reporting/requirements-dev.txt @@ -7,7 +7,7 @@ coverage==7.2.7 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/orca_recovery_adapter/requirements-dev.txt b/tasks/orca_recovery_adapter/requirements-dev.txt index 1976caaa4..a0b823c1a 100644 --- a/tasks/orca_recovery_adapter/requirements-dev.txt +++ b/tasks/orca_recovery_adapter/requirements-dev.txt @@ -7,7 +7,7 @@ fastjsonschema==2.15.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/perform_orca_reconcile/requirements-dev.txt b/tasks/perform_orca_reconcile/requirements-dev.txt index e57a86b72..73646a2a3 100644 --- a/tasks/perform_orca_reconcile/requirements-dev.txt +++ b/tasks/perform_orca_reconcile/requirements-dev.txt @@ -13,7 +13,7 @@ boto3==1.18.40 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/post_copy_request_to_queue/requirements-dev.txt b/tasks/post_copy_request_to_queue/requirements-dev.txt index 312125dc7..dd6d107e1 100644 --- a/tasks/post_copy_request_to_queue/requirements-dev.txt +++ b/tasks/post_copy_request_to_queue/requirements-dev.txt @@ -4,7 +4,7 @@ pytest==7.4.0 ## Libraries needed for testing the application boto3==1.18.40 -moto[sqs]==4.2.2 +moto[sqs]==4.2.13 psycopg2-binary==2.8.6 ## Application libraries @@ -14,7 +14,7 @@ SQLAlchemy~=2.0.5 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/post_to_catalog/requirements-dev.txt b/tasks/post_to_catalog/requirements-dev.txt index 87b2ef15a..cff989d59 100644 --- a/tasks/post_to_catalog/requirements-dev.txt +++ b/tasks/post_to_catalog/requirements-dev.txt @@ -13,7 +13,7 @@ SQLAlchemy~=2.0.5 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 \ No newline at end of file diff --git a/tasks/post_to_database/requirements-dev.txt b/tasks/post_to_database/requirements-dev.txt index 34b1c810a..d7ac6c75e 100644 --- a/tasks/post_to_database/requirements-dev.txt +++ b/tasks/post_to_database/requirements-dev.txt @@ -12,7 +12,7 @@ fastjsonschema~=2.15.1 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/post_to_queue_and_trigger_step_function/requirements-dev.txt b/tasks/post_to_queue_and_trigger_step_function/requirements-dev.txt index 195f9c270..f0505878c 100644 --- a/tasks/post_to_queue_and_trigger_step_function/requirements-dev.txt +++ b/tasks/post_to_queue_and_trigger_step_function/requirements-dev.txt @@ -1,7 +1,7 @@ ## Libraries needed for testing coverage==7.2.7 pytest==7.4.0 -moto[sqs]==4.2.2 +moto[sqs]==4.2.13 ## Application libraries needed for testing psycopg2-binary==2.8.6 @@ -13,7 +13,7 @@ boto3==1.18.40 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/request_from_archive/requirements-dev.txt b/tasks/request_from_archive/requirements-dev.txt index 7db8fe5e7..4c2ea9e40 100644 --- a/tasks/request_from_archive/requirements-dev.txt +++ b/tasks/request_from_archive/requirements-dev.txt @@ -13,7 +13,7 @@ aws_lambda_powertools==1.31.0 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/request_status_for_granule/requirements-dev.txt b/tasks/request_status_for_granule/requirements-dev.txt index ed318da66..946dd5786 100644 --- a/tasks/request_status_for_granule/requirements-dev.txt +++ b/tasks/request_status_for_granule/requirements-dev.txt @@ -8,7 +8,7 @@ psycopg2-binary==2.8.6 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 diff --git a/tasks/request_status_for_job/requirements-dev.txt b/tasks/request_status_for_job/requirements-dev.txt index 3381d351d..262ea139f 100644 --- a/tasks/request_status_for_job/requirements-dev.txt +++ b/tasks/request_status_for_job/requirements-dev.txt @@ -8,7 +8,7 @@ psycopg2-binary==2.8.6 ## Additional validation libraries ## --------------------------------------------------------------------------- -bandit==1.7.5 +bandit==1.7.6 flake8==6.1.0 black==22.3.0 isort==5.12.0 From ac45104eda9e6ede9bd1afa8a839517a462bb1b4 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:28:56 -0600 Subject: [PATCH 51/68] Updated running the delete_lambda script wording --- .../developer/deployment-guide/deployment-upgrading-orca.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md index df0789ce0..15d1a561e 100644 --- a/website/docs/developer/deployment-guide/deployment-upgrading-orca.md +++ b/website/docs/developer/deployment-guide/deployment-upgrading-orca.md @@ -76,7 +76,7 @@ To update your ORCA version: 2. Update the `source` in your Terraform deployment files for ORCA by replacing `vx.x.x` with the desired version of ORCA. If upgrading from `v8.x.x` to `v9.x.x` follow the below steps: - 1. Run the Lambda deletion script found in `bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. + 1. Run the Lambda deletion script found in `python3 bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. 2. Navigate to the AWS console and search for the Cumulus RDS security group. 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minutes and then try again. From 41c6eec1308530a6022068b984a9e1e83bf4580f Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:51:50 -0600 Subject: [PATCH 52/68] Update CHANGELOG.md with v8 to v9 steps --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 879781c9e..42a7a1687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and includes an additional section for migration notes. ### Migration Notes +- For users upgrading from ORCA v8.x.x to v9.x.x follow the below steps before deploying: + 1. Run the Lambda deletion script found in `python3 bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. + 2. Navigate to the AWS console and search for the Cumulus RDS security group. + 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. + 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minutes and then try again. + ### Added - *ORCA-366* Added unit test for shared libraries. From 22c58cbe853cc291dcd48874ef2da695d78c2e56 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Wed, 24 Jan 2024 14:35:02 -0600 Subject: [PATCH 53/68] Adding lambda deletion steps for v8 to v9 upgrade to v9 migration notes --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a7a1687..c308ffc1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,11 @@ and includes an additional section for migration notes. ### Migration Notes - Update terraform to the latest 1.5 version +- For users upgrading from ORCA v8.x.x to v9.x.x follow the below steps before deploying: + 1. Run the Lambda deletion script found in `python3 bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. + 2. Navigate to the AWS console and search for the Cumulus RDS security group. + 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. + 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minutes and then try again. ### Security From b7c9734445cc381afeb13e8e88eddd5683b1ab2c Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:03:49 -0600 Subject: [PATCH 54/68] Removing lifecycle rule on api gateway deployment resource to mitigate errors --- modules/api-gateway/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/api-gateway/main.tf b/modules/api-gateway/main.tf index b0d0bcbc0..a29795c24 100644 --- a/modules/api-gateway/main.tf +++ b/modules/api-gateway/main.tf @@ -537,10 +537,6 @@ resource "aws_api_gateway_deployment" "orca_api_deployment" { aws_api_gateway_integration.internal_reconcile_report_phantom_api_integration, aws_api_gateway_integration.internal_reconcile_report_mismatch_api_integration ] - - lifecycle { - create_before_destroy = true - } } # Stage for API Gateway From e6a28545568f2f3bf7af066f802aaa5d89082f8e Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 25 Jan 2024 08:12:08 -0600 Subject: [PATCH 55/68] minor fix --- .../requirements-dev.txt | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tasks/get_current_archive_list/requirements-dev.txt b/tasks/get_current_archive_list/requirements-dev.txt index 68e3cd728..616ca2800 100644 --- a/tasks/get_current_archive_list/requirements-dev.txt +++ b/tasks/get_current_archive_list/requirements-dev.txt @@ -1,29 +1,19 @@ -## Library used for API documentation -## --------------------------------------------------------------------------- -pydoc-markdown==4.5.0 +## Libraries needed for testing +## Libraries needed for testing +coverage==7.2.7 +pytest==7.4.0 +## Application libraries needed for testing +psycopg2-binary==2.8.6 + +## Application libraries +aws_lambda_powertools==1.31.0 +fastjsonschema~=2.15.1 +../../shared_libraries[database, reconciliation] ## Additional validation libraries ## --------------------------------------------------------------------------- bandit==1.7.6 flake8==6.1.0 black==22.3.0 -isort==5.12.0 - - -## Standard unit test libraries -## --------------------------------------------------------------------------- -pytest==7.4.0 -coverage==7.2.7 - - -## Libraries needed for unit tests -## --------------------------------------------------------------------------- -psycopg2-binary==2.8.6 - - -## Libraries needed by application -## --------------------------------------------------------------------------- -aws_lambda_powertools==1.31.0 -SQLAlchemy~=2.0.5 -../../shared_libraries[database, reconciliation] +isort==5.12.0 \ No newline at end of file From a65477c2b77e68bea709a671fcbc4f290bac77ff Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 25 Jan 2024 08:18:08 -0600 Subject: [PATCH 56/68] updated changelog for release --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013632a42..d88a2cae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,24 @@ and includes an additional section for migration notes. ### Added +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + +## [9.0.2] 2024-01-26 + +### Migration Notes + +If you are migrating from ORCA v8.x.x to this version, see the migration notes under v9.0.0. + +### Added + - *ORCA-366* Added unit test for shared libraries. ### Changed From 6b9a05b43c3498bd018970b865d662ae2322454e Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 25 Jan 2024 09:30:59 -0600 Subject: [PATCH 57/68] updated changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013632a42..2ba123dae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,13 +36,12 @@ and includes an additional section for migration notes. - *ORCA-710* Removed duplicate logging messages in `integration_test/workflow_tests/custom_logger.py` - *ORCA-815* Removed steps for creating buckets using NGAP form in ORCA archive bucket documentation. - ### Fixed - *ORCA-811* Fixed `cumulus_orca` docker image by updating nodejs installation process. - *ORCA-802* Fixed `extract_file_for_granule` documentation and schemas to include `collectionId` in input. - *ORCA-785* Fixed checksum integrity issue in ORCA documentation bamboo pipeline. - +- *ORCA-820* Updated bandit and moto libraries to fix some snyk vulnerabilities. ### Security From 55d5a629141af9d152f5f0b3659bb54a817d3008 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Thu, 25 Jan 2024 13:53:09 -0600 Subject: [PATCH 58/68] Fixed changelog --- CHANGELOG.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d3fcedbd..b8831aac9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,6 @@ and includes an additional section for migration notes. ### Migration Notes -- For users upgrading from ORCA v8.x.x to v9.x.x follow the below steps before deploying: - 1. Run the Lambda deletion script found in `python3 bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. - 2. Navigate to the AWS console and search for the Cumulus RDS security group. - 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. - 4. Search for `PREFIX-vpc-ingress-all-egress` and delete the security group **NOTE:** Due to the Lambdas using ENIs, when deleting the securty groups it may say they are still associated with a Lambda that was deleted by the script. AWS may need a few minutes to refresh to fully disassociate the ENIs completely, if this error appears wait a few minutes and then try again. - ### Added ### Changed @@ -99,7 +93,7 @@ If you are migrating from ORCA v8.x.x to this version, see the migration notes u ### Migration Notes - Update terraform to the latest 1.5 version -- For users upgrading from ORCA v8.x.x to v9.x.x follow the below steps before deploying: +- For users upgrading from ORCA v8.x.x to v9.x.x, follow the below steps before deploying: 1. Run the Lambda deletion script found in `python3 bin/delete_lambda.py` this will delete all of the ORCA Lambdas with a provided prefix. Or delete them manually in the AWS console. 2. Navigate to the AWS console and search for the Cumulus RDS security group. 3. Remove the inbound rule with the source of `PREFIX-vpc-ingress-all-egress` in Cumulus RDS security group. From 9dc815ba55c0dd1b88143dfa7f1e829fb50aa003 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:21:31 -0600 Subject: [PATCH 59/68] Create dr_buckets.tf --- modules/dr_buckets/dr_buckets.tf | 275 +++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 modules/dr_buckets/dr_buckets.tf diff --git a/modules/dr_buckets/dr_buckets.tf b/modules/dr_buckets/dr_buckets.tf new file mode 100644 index 000000000..8fa1282e4 --- /dev/null +++ b/modules/dr_buckets/dr_buckets.tf @@ -0,0 +1,275 @@ +# ORCA DR S3 Variables +variable "prefix" { + type = string + description = "Prefix used to prepend to all object names and tags." +} + +variable "tags" { + type = map(string) + description = "Tags to be applied to resources that support tags." + default = {} +} + +variable "cumulus_account_id" { + type = number + description = "Account ID of the Cumulus Account" +} + +variable "aws_profile" { + type = string + description = "AWS CLI profile to use for installation and monitoring of AWS objects." +} + +variable "aws_region" { + type = string + description = "AWS Region to create resources in." + default = "us-west-2" +} + +# Provider Config +provider "aws" { + region = var.aws_region + profile = var.aws_profile + + ignore_tags { + key_prefixes = ["gsfc-ngap"] + } +} + +# Get DR Account ID +data "aws_caller_identity" "current" {} + +# ORCA Archive Bucket +resource "aws_s3_bucket" "orca_archive_bucket" { + bucket = "${var.prefix}-orca-archive" + tags = var.tags +} + +# ORCA Archive Bucket Versioning +resource "aws_s3_bucket_versioning" "orca_archive_versioning" { + bucket = aws_s3_bucket.orca_archive_bucket.id + versioning_configuration { + status = "Enabled" + } +} + +# ORCA Archive Policy +resource "aws_s3_bucket_policy" "orca_archive_cross_account_access" { + bucket = aws_s3_bucket.orca_archive_bucket.id + policy = data.aws_iam_policy_document.orca_archive_cross_account_access_policy.json +} + +data "aws_iam_policy_document" "orca_archive_cross_account_access_policy" { + statement { + sid = "CrossAccPolicyDoc" + principals { + type = "AWS" + identifiers = [var.cumulus_account_id] + } + + actions = [ + "s3:GetObject*", + "s3:RestoreObject", + "s3:GetBucket*", + "s3:ListBucket", + "s3:PutBucketNotification", + "s3:GetInventoryConfiguration", + "s3:PutInventoryConfiguration", + "s3:ListBucketVersions" + ] + + resources = [ + aws_s3_bucket.orca_archive_bucket.arn, + "${aws_s3_bucket.orca_archive_bucket.arn}/*", + ] + } + statement { + sid = "Cross Account Write Access" + principals { + type = "AWS" + identifiers = [var.cumulus_account_id] + } + + actions = [ + "s3:PutObject*" + ] + + resources = [ + "${aws_s3_bucket.orca_archive_bucket.arn}/*" + ] + + condition { + test = "StringNotEquals" + variable = "s3:x-amz-storage-class" + values = [ + "GLACIER", + "DEEP_ARCHIVE" + ] + } + condition { + test = "StringNotEquals" + variable = "s3:x-amz-acl" + values = [ + "bucket-owner-full-control" + ] + } + } +} + +# ORCA Archive Worm Bucket +resource "aws_s3_bucket" "orca_archive_worm_bucket" { + bucket = "${var.prefix}-orca-archive-worm" + tags = var.tags +} + +# ORCA Archive Worm Bucket Versioning +resource "aws_s3_bucket_versioning" "orca_archive_worm_versioning" { + bucket = aws_s3_bucket.orca_archive_worm_bucket.id + + versioning_configuration { + status = "Enabled" + } +} + +# ORCA Archive Worm Bucket Object Lock +resource "aws_s3_bucket_object_lock_configuration" "orca_achive_worm_object_lock" { + bucket = aws_s3_bucket.orca_archive_worm_bucket.id + depends_on = [aws_s3_bucket_versioning.orca_archive_worm_versioning] + + rule { + default_retention { + mode = "GOVERNANCE" + days = 5 + } + } +} + +# ORCA Archive Worm Bucket Policy +resource "aws_s3_bucket_policy" "orca_archive_worm_cross_account_access" { + bucket = aws_s3_bucket.orca_archive_worm_bucket.id + policy = data.aws_iam_policy_document.orca_archive_worm_cross_account_access_policy.json +} + +data "aws_iam_policy_document" "orca_archive_worm_cross_account_access_policy" { + statement { + sid = "Cross Account Access" + principals { + type = "AWS" + identifiers = [var.cumulus_account_id] + } + + actions = [ + "s3:PutObject" + ] + + resources = [ + "${aws_s3_bucket.orca_archive_worm_bucket.arn}/*" + ] + condition { + test = "StringEquals" + variable = "s3:x-amz-storage-class" + values = [ + "GLACIER", + "DEEP_ARCHIVE" + ] + } + } +} + +# ORCA Reports Bucket +resource "aws_s3_bucket" "orca_reports_bucket" { + bucket = "${var.prefix}-orca-reports" + tags = var.tags +} + +# ORCA Reports Bucket Lifecycle Rule +resource "aws_s3_bucket_lifecycle_configuration" "orca_reports_bucket_lifecycle" { + bucket = aws_s3_bucket.orca_reports_bucket.id + + rule { + id = "30-day-lifecycle-reports" + expiration { + days = 30 + } + status = "Enabled" + } +} + +# ORCA Reports Bucket Policy +resource "aws_s3_bucket_policy" "orca_reports_cross_account_access" { + bucket = aws_s3_bucket.orca_reports_bucket.id + policy = data.aws_iam_policy_document.orca_reports_cross_account_access_policy.json +} + +data "aws_iam_policy_document" "orca_reports_cross_account_access_policy" { + statement { + sid = "Reports Cross Account Access" + principals { + type = "AWS" + identifiers = [var.cumulus_account_id] + } + + actions = [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:ListBucket", + "s3:PutObject", + "s3:PutObjectAcl", + "s3:PutBucketNotification" + ] + + resources = [ + aws_s3_bucket.orca_reports_bucket.arn, + "${aws_s3_bucket.orca_reports_bucket.arn}/*", + ] + } + statement { + sid = "Inventory ORCA Reports" + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + + actions = [ + "s3:PutObject*" + ] + + resources = [ + "${aws_s3_bucket.orca_reports_bucket.arn}/*" + ] + + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = [ + "bucket-owner-full-control" + ] + } + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [ + "${data.aws_caller_identity.current.account_id}" + ] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_s3_bucket.orca_reports_bucket.arn] + } + } + +} + +# DR Bucket Outputs +output "orca_achive_bucket_arn" { + value = aws_s3_bucket.orca_archive_bucket.arn +} + +output "orca_reports_bucket_arn" { + value = aws_s3_bucket.orca_reports_bucket.arn +} + +output "orca_achive_worm_bucket_arn" { + value = aws_s3_bucket.orca_archive_worm_bucket.arn +} From 68360df4c055d21bcd4214dba398805a70f87bf4 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:09:44 -0600 Subject: [PATCH 60/68] Update creating-orca-archive-bucket.md with Terraform template steps --- .../creating-orca-archive-bucket.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md index b6c86b236..19b788339 100644 --- a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md +++ b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md @@ -93,6 +93,25 @@ aws cloudformation deploy --stack-name -orca-bucket-stack --template-fil ``` This will create archive and reports buckets with the necessary bucket policies giving the Cumulus Account permission to write data to the archive bucket. +### Via Terraform + +The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure have AWS CLI installed and a DR profile for the DR account configured before deploying this template. More information on how to configure an AWS CLI profile can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#cli-configure-options-list:~:text=as%20programmatically%20useful.-,%2D%2Dprofile%20%3Cstring%3E,-Specifies%20the%20named). + +From your terminal, first run `terraform init` followed by `terraform apply`. When running the apply Terraform will ask for the following inputs: +1. `aws_profile` - This is the profile configured for the DR AWS account. +2. `cumulus_account_id` - This is the account ID of the Cumulus AWS account. +3. `prefix` - This is the prefix to use for the bucket names. + +Tags are an optional variable that can be set if you wish to have the DR buckets tagged. + +Optionally you can provide Terraform the required inputs through the terminal with the following: +``` +terraform apply \ +-var=aws_profile="" \ +-var=cumulus_account_id="" \ +-var=prefix="PREFIX" +``` + ### Via AWS GUI For each of the buckets listed below @@ -286,4 +305,4 @@ Replace `` with your `system-bucket` name. Replace `` with your prefix. -Replace `` with your Cumulus OU account number. \ No newline at end of file +Replace `` with your Cumulus OU account number. From ef19d8a4633919cc89366fb6f88a15c9908b169e Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:10:57 -0600 Subject: [PATCH 61/68] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b995107..296616666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and includes an additional section for migration notes. - *ORCA-366* Added unit test for shared libraries. - *ORCA-769* Added API Gateway Stage resource to `modules/api-gateway/main.tf` +- *ORCA-369* Added DR S3 bucket template to `modules/dr_buckets/dr_buckets.tf` and updated S3 deployment documentation with steps. ### Changed From b2cd70456f841a97365a5826a24af7f84b5683f1 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 26 Jan 2024 07:59:49 -0600 Subject: [PATCH 62/68] fixed versioning doc format --- .../docs/developer/development-guide/code/versioning-releases.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/docs/developer/development-guide/code/versioning-releases.md b/website/docs/developer/development-guide/code/versioning-releases.md index e044e545b..0652e8827 100644 --- a/website/docs/developer/development-guide/code/versioning-releases.md +++ b/website/docs/developer/development-guide/code/versioning-releases.md @@ -1,4 +1,3 @@ - --- id: versioning-releases title:  ORCA Versioning and Releases From 446245c21e0f45a5905bb85cb107fa2413f8a9e2 Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 26 Jan 2024 08:07:36 -0600 Subject: [PATCH 63/68] again fixed versioning doc format --- .../developer/development-guide/code/versioning-releases.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/developer/development-guide/code/versioning-releases.md b/website/docs/developer/development-guide/code/versioning-releases.md index 0652e8827..7076ff595 100644 --- a/website/docs/developer/development-guide/code/versioning-releases.md +++ b/website/docs/developer/development-guide/code/versioning-releases.md @@ -1,6 +1,6 @@ --- id: versioning-releases -title:  ORCA Versioning and Releases +title: ORCA Versioning and Releases description: Provides information to developers on semantic versioning and the release process. --- From c4b1584b69e9a197d94d93585619436f5bc1a20d Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Fri, 26 Jan 2024 09:34:52 -0600 Subject: [PATCH 64/68] Grammar changes --- .../deployment-guide/creating-orca-archive-bucket.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md index 19b788339..ab12016e0 100644 --- a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md +++ b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md @@ -95,9 +95,9 @@ This will create archive and reports buckets with the necessary bucket policies ### Via Terraform -The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure have AWS CLI installed and a DR profile for the DR account configured before deploying this template. More information on how to configure an AWS CLI profile can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#cli-configure-options-list:~:text=as%20programmatically%20useful.-,%2D%2Dprofile%20%3Cstring%3E,-Specifies%20the%20named). +The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure you have AWS CLI installed and a DR profile for the DR account configured before deploying this template. More information on how to configure an AWS CLI profile can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#cli-configure-options-list:~:text=as%20programmatically%20useful.-,%2D%2Dprofile%20%3Cstring%3E,-Specifies%20the%20named). -From your terminal, first run `terraform init` followed by `terraform apply`. When running the apply Terraform will ask for the following inputs: +From your terminal, first run `terraform init` followed by `terraform apply`. When running the apply, Terraform will ask for the following inputs: 1. `aws_profile` - This is the profile configured for the DR AWS account. 2. `cumulus_account_id` - This is the account ID of the Cumulus AWS account. 3. `prefix` - This is the prefix to use for the bucket names. From 0ffc6027edd5312f9ca6a1849087bea575055bbc Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:07:46 -0600 Subject: [PATCH 65/68] Update dr_buckets.tf --- modules/dr_buckets/dr_buckets.tf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/dr_buckets/dr_buckets.tf b/modules/dr_buckets/dr_buckets.tf index 8fa1282e4..1733edf55 100644 --- a/modules/dr_buckets/dr_buckets.tf +++ b/modules/dr_buckets/dr_buckets.tf @@ -1,4 +1,3 @@ -# ORCA DR S3 Variables variable "prefix" { type = string description = "Prefix used to prepend to all object names and tags." @@ -20,15 +19,18 @@ variable "aws_profile" { description = "AWS CLI profile to use for installation and monitoring of AWS objects." } -variable "aws_region" { - type = string - description = "AWS Region to create resources in." - default = "us-west-2" +## Terraform Configuration +terraform { + required_version = ">= 1.5" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0.0" + } + } } -# Provider Config provider "aws" { - region = var.aws_region profile = var.aws_profile ignore_tags { From fa578bedef86f8a1669da163d89037c7ae1b2b65 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:33:31 -0600 Subject: [PATCH 66/68] Update dr_buckets.tf --- modules/dr_buckets/dr_buckets.tf | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/modules/dr_buckets/dr_buckets.tf b/modules/dr_buckets/dr_buckets.tf index 1733edf55..c5b45cb7b 100644 --- a/modules/dr_buckets/dr_buckets.tf +++ b/modules/dr_buckets/dr_buckets.tf @@ -14,11 +14,6 @@ variable "cumulus_account_id" { description = "Account ID of the Cumulus Account" } -variable "aws_profile" { - type = string - description = "AWS CLI profile to use for installation and monitoring of AWS objects." -} - ## Terraform Configuration terraform { required_version = ">= 1.5" @@ -30,14 +25,6 @@ terraform { } } -provider "aws" { - profile = var.aws_profile - - ignore_tags { - key_prefixes = ["gsfc-ngap"] - } -} - # Get DR Account ID data "aws_caller_identity" "current" {} From 23cdbf7f722cfcbe4bf1f3c9dadd59b72c840a36 Mon Sep 17 00:00:00 2001 From: Zachary Roehrich <131377950+zroehrich@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:35:18 -0600 Subject: [PATCH 67/68] Update creating-orca-archive-bucket.md to remove references to removed variable --- .../deployment-guide/creating-orca-archive-bucket.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md index ab12016e0..49f6fccec 100644 --- a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md +++ b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md @@ -98,16 +98,14 @@ This will create archive and reports buckets with the necessary bucket policies The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure you have AWS CLI installed and a DR profile for the DR account configured before deploying this template. More information on how to configure an AWS CLI profile can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#cli-configure-options-list:~:text=as%20programmatically%20useful.-,%2D%2Dprofile%20%3Cstring%3E,-Specifies%20the%20named). From your terminal, first run `terraform init` followed by `terraform apply`. When running the apply, Terraform will ask for the following inputs: -1. `aws_profile` - This is the profile configured for the DR AWS account. -2. `cumulus_account_id` - This is the account ID of the Cumulus AWS account. -3. `prefix` - This is the prefix to use for the bucket names. +1. `cumulus_account_id` - This is the account ID of the Cumulus AWS account. +2. `prefix` - This is the prefix to use for the bucket names. Tags are an optional variable that can be set if you wish to have the DR buckets tagged. Optionally you can provide Terraform the required inputs through the terminal with the following: ``` terraform apply \ --var=aws_profile="" \ -var=cumulus_account_id="" \ -var=prefix="PREFIX" ``` From 6fb56c84dadcd0ac14298055217372c60c01fa0d Mon Sep 17 00:00:00 2001 From: Rizbi Hassan Date: Fri, 26 Jan 2024 10:44:17 -0600 Subject: [PATCH 68/68] minor fix in orca deploy docs --- .../developer/deployment-guide/creating-orca-archive-bucket.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md index 49f6fccec..957039e86 100644 --- a/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md +++ b/website/docs/developer/deployment-guide/creating-orca-archive-bucket.md @@ -95,7 +95,7 @@ This will create archive and reports buckets with the necessary bucket policies ### Via Terraform -The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure you have AWS CLI installed and a DR profile for the DR account configured before deploying this template. More information on how to configure an AWS CLI profile can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#cli-configure-options-list:~:text=as%20programmatically%20useful.-,%2D%2Dprofile%20%3Cstring%3E,-Specifies%20the%20named). +The Terraform template for creating the ORCA DR buckets can be found [here](https://github.com/nasa/cumulus-orca/blob/master/modules/dr_buckets/dr_buckets.tf). Make sure you have AWS CLI installed and AWS configured to deploy to your DR account. From your terminal, first run `terraform init` followed by `terraform apply`. When running the apply, Terraform will ask for the following inputs: 1. `cumulus_account_id` - This is the account ID of the Cumulus AWS account.