diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6298285d..0952449f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -97,7 +97,7 @@ def load_backup_class(fs_type): root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) if root not in sys.path: sys.path.append(root) - from s3.test_utils.s3_backup import S3TestBackupDir + from ..test_utils.s3_backup import S3TestBackupDir fs_backup_class = S3TestBackupDir diff --git a/tests/s3_auth_test.py b/tests/s3_auth_test.py new file mode 100644 index 00000000..e02fcefb --- /dev/null +++ b/tests/s3_auth_test.py @@ -0,0 +1,29 @@ +from tests.test_utils.base_test import S3BaseTest + + +class AuthorizationTest(S3BaseTest): + """ + Check connect to S3 via pre_start_checks() function + calling pg_probackup init --s3 + + test that s3 keys allow to connect to all types of storages + """ + + def test_s3_authorisation(self): + console_output = self.pb.init(options=["--log-level-console=VERBOSE"]) + + self.assertNotIn(': 403', console_output) # Because we can have just '403' substring in timestamp + self.assertMessage(console_output, contains='S3_pre_start_check successful') + self.assertMessage(console_output, contains='HTTP response: 200') + self.assertIn( + f"INFO: Backup catalog '{self.backup_dir}' successfully initialized", + console_output) + + def test_log_level_file_requires_log_directory(self): + console_output = self.pb.init(options=["--log-level-file=VERBOSE"], + skip_log_directory=True, + expect_error=True) + + self.assertMessage(console_output, + contains='ERROR: Cannot save S3 logs to a file. You must specify --log-directory option when' + ' running backup with --log-level-file option enabled') diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_utils/base_test.py b/tests/test_utils/base_test.py new file mode 100644 index 00000000..8b58c38f --- /dev/null +++ b/tests/test_utils/base_test.py @@ -0,0 +1,29 @@ +import os +import sys +import unittest + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) +if root not in sys.path: + sys.path.append(root) + +from tests.helpers.ptrack_helpers import ProbackupTest +from . import config_provider + + +class S3BaseTest(ProbackupTest, unittest.TestCase): + + additional_options = [] + + def setUp(self) -> None: + # If env vars weren't set, think that it is a local run and set up variables + if not os.environ.get('PG_PROBACKUP_S3_HOST'): + minio_config = config_provider.read_config()['MINIO'] + os.environ.setdefault('PG_PROBACKUP_S3_ACCESS_KEY', minio_config['access_key']) + os.environ.setdefault('PG_PROBACKUP_S3_SECRET_ACCESS_KEY', minio_config['secret_key']) + os.environ.setdefault('PG_PROBACKUP_S3_HOST', minio_config['local_host']) + os.environ.setdefault('PG_PROBACKUP_S3_PORT', minio_config['api_port']) + os.environ.setdefault('PG_PROBACKUP_S3_BUCKET_NAME', minio_config['bucket_name']) + os.environ.setdefault('PG_PROBACKUP_S3_REGION', minio_config['region']) + # There will be a connector setting + if not os.environ.get("PROBACKUP_S3_TYPE_FULL_TEST"): + os.environ.setdefault('PROBACKUP_S3_TYPE_FULL_TEST', 'minio') diff --git a/tests/test_utils/config_provider.py b/tests/test_utils/config_provider.py new file mode 100644 index 00000000..9d701564 --- /dev/null +++ b/tests/test_utils/config_provider.py @@ -0,0 +1,8 @@ +import configparser +import os.path + + +def read_config(): + configs = configparser.ConfigParser() + configs.read(os.path.join(os.path.dirname(__file__), '../properties.ini')) + return configs diff --git a/tests/test_utils/s3_backup.py b/tests/test_utils/s3_backup.py new file mode 100644 index 00000000..29cd8679 --- /dev/null +++ b/tests/test_utils/s3_backup.py @@ -0,0 +1,118 @@ +import os +import io +import sys + +import minio +from minio import Minio +from minio.deleteobjects import DeleteObject +import urllib3 + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) +if root not in sys.path: + sys.path.append(root) + +# Should fail if either of env vars does not exist +host = os.environ['PG_PROBACKUP_S3_HOST'] +port = os.environ['PG_PROBACKUP_S3_PORT'] +access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY'] +secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY'] +bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME'] +path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX") +https = os.environ.get("PG_PROBACKUP_S3_HTTPS") + +s3_type = os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST') + +status_forcelist = [413, # RequestBodyTooLarge + 429, # TooManyRequests + 500, # InternalError + 503, # ServerBusy + ] + + +class S3TestBackupDir: + is_file_based = False + + def __init__(self, *, rel_path, backup): + path = "pg_probackup" + if path_suffix: + path += "_" + path_suffix + self.path = f"{path}/{rel_path}/{backup}" + secure: bool = False + if https in ['ON', 'HTTPS']: + secure = True + self.conn = Minio(host + ":" + port, secure=secure, access_key=access, + secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5, + backoff_factor=1, + status_forcelist=status_forcelist))) + if not self.conn.bucket_exists(bucket): + raise Exception(f"Test bucket {bucket} does not exist.") + self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}') + return + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True) + return [os.path.basename(os.path.dirname(x.object_name)) + for x in candidates if x.object_name.endswith('backup.control')] + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive) + return [obj.object_name.replace(full_path_dir, '', 1) + for obj in object_list + if not obj.is_dir] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + bytes = self.conn.get_object(bucket, full_path).read() + if not text: + return bytes + return bytes.decode('utf-8') + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + if text: + data = data.encode('utf-8') + self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data)) + + def cleanup(self): + self.remove_dir('') + + def remove_file(self, sub_path): + full_path = os.path.join(self.path, sub_path) + self.conn.remove_object(bucket, full_path) + + def remove_dir(self, sub_path): + if sub_path: + full_path = os.path.join(self.path, sub_path) + else: + full_path = self.path + objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True) + delobjs = (DeleteObject(o.object_name) for o in objs) + errs = list(self.conn.remove_objects(bucket, delobjs)) + if errs: + strerrs = "; ".join(str(err) for err in errs) + raise Exception("There were errors: {0}".format(strerrs)) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + try: + self.conn.stat_object(bucket, full_path) + return True + except minio.error.S3Error as s3err: + if s3err.code == 'NoSuchKey': + return False + raise s3err + except Exception as err: + raise err + + def __str__(self): + return '/' + self.path + + def __repr__(self): + return "S3TestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path