diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce2910d..ef26ff0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,6 +38,9 @@ "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", "python.pythonPath": "/usr/local/bin/python", "restructuredtext.confPath": "${workspaceFolder}/docs/source", + // Scrolling the editor is a nice idea but it doesn't work, always out of sync and impossible to manage + "restructuredtext.preview.scrollEditorWithPreview": false, + "restructuredtext.preview.scrollPreviewWithEditor": false, "terminal.integrated.defaultProfile.linux": "zsh" }, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84b16e0..f2d4019 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,6 +61,8 @@ repos: hooks: - id: check-commit-message-is-conventional stages: [commit-msg] + args: + - --maximum-body-line-length=2000 - repo: https://github.com/thclark/pre-commit-sphinx rev: 0.0.1 diff --git a/README.md b/README.md index 5e318c6..4ed8272 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPI version](https://badge.fury.io/py/django_gcp.svg)](https://badge.fury.io/py/django_gcp) -[![codecov](https://codecov.io/gh/octue/django_gcp/branch/master/graph/badge.svg)](https://codecov.io/gh/octue/django_gcp) +[![codecov](https://codecov.io/gh/octue/django-gcp/branch/main/graph/badge.svg?token=H2QLSCF3DU)](https://codecov.io/gh/octue/django-gcp) [![Documentation](https://readthedocs.org/projects/django_gcp/badge/?version=latest)](https://django_gcp.readthedocs.io/en/latest/?badge=latest) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) @@ -20,9 +20,7 @@ If so, get in touch for a chat. We're doing fun things with Google Cloud. Way fu ## All the :heart: -This app is based heavily on [django-storages](), [django-cloud-tasks]() and uses on the [django-rabid-armadillo]() template. Big love. - -- Template django app, with: +This app is based heavily on [django-storages](https://django-storages.readthedocs.io/en/latest/), [django-google-cloud-tasks](https://github.com/flamingo-run/django-cloud-tasks) and uses the [django-rabid-armadillo](https://github.com/thclark/django-rabid-armadillo) template. Big love. ## Contributing diff --git a/django_gcp/storage/base.py b/django_gcp/storage/base.py deleted file mode 100644 index f919b78..0000000 --- a/django_gcp/storage/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from django.core.exceptions import ImproperlyConfigured -from django.core.files.storage import Storage - - -class BaseStorage(Storage): - def __init__(self, **settings): - default_settings = self.get_default_settings() - - for name, value in default_settings.items(): - if not hasattr(self, name): - setattr(self, name, value) - - for name, value in settings.items(): - if name not in default_settings: - raise ImproperlyConfigured( - "Invalid setting '{}' for {}".format( - name, - self.__class__.__name__, - ) - ) - setattr(self, name, value) - - def get_default_settings(self): - return {} diff --git a/django_gcp/storage/gcloud.py b/django_gcp/storage/gcloud.py index 329cd40..0a5f83b 100644 --- a/django_gcp/storage/gcloud.py +++ b/django_gcp/storage/gcloud.py @@ -2,27 +2,19 @@ # pylint: disable=arguments-renamed import mimetypes -import warnings -from datetime import timedelta from tempfile import SpooledTemporaryFile -from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation +from django.core.exceptions import SuspiciousOperation from django.core.files.base import File +from django.core.files.storage import Storage from django.utils import timezone from django.utils.deconstruct import deconstructible +from google.cloud.exceptions import NotFound +from google.cloud.storage import Blob, Client +from google.cloud.storage.blob import _quote -from .base import BaseStorage from .compress import CompressedFileMixin, CompressStorageMixin -from .utils import check_location, clean_name, get_available_overwrite_name, safe_join, setting, to_bytes - - -try: - from google.cloud.exceptions import NotFound - from google.cloud.storage import Blob, Client - from google.cloud.storage.blob import _quote -except ImportError as e: - raise ImproperlyConfigured( - "Could not load Google Cloud Storage bindings.\n" "See https://github.com/GoogleCloudPlatform/gcloud-python" - ) from e +from .settings import StorageSettings +from .utils import clean_name, get_available_overwrite_name, safe_join, setting, to_bytes CONTENT_ENCODING = "content_encoding" @@ -32,31 +24,32 @@ class GoogleCloudFile(CompressedFileMixin, File): """A django File object representing a GCP storage object""" - def __init__(self, name, mode, storage): + def __init__(self, name, mode, storage): # pylint: disable=super-init-not-called self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and "w" in mode: - self.blob = Blob(self.name, storage.bucket, chunk_size=storage.blob_chunk_size) + self.blob = Blob(self.name, storage.bucket, chunk_size=storage.settings.blob_chunk_size) self._file = None self._is_dirty = False - @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( - max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR") + max_size=self._storage.settings.max_memory_size, + suffix=".GSStorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR"), ) if "r" in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) - if self._storage.gzip and self.blob.content_encoding == "gzip": + if self._storage.settings.gzip and self.blob.content_encoding == "gzip": self._file = self._decompress_file(mode=self._mode, file=self._file) return self._file @@ -91,75 +84,55 @@ def close(self): self.file, rewind=True, content_type=self.mime_type, - predefined_acl=blob_params.get("acl", self._storage.default_acl), + predefined_acl=blob_params.get("acl", self._storage.settings.default_acl), ) self._file.close() self._file = None @deconstructible -class GoogleCloudStorage(CompressStorageMixin, BaseStorage): - """Storage class allowing django to use GCS as an object store""" +class GoogleCloudStorage(CompressStorageMixin, Storage): + """Storage class allowing django to use GCS as an object store - def __init__(self, **settings): - super().__init__(**settings) + Instantiates as the `media` store by default. Pass `media`, `static` or any of the keys in""" - check_location(self) + def __init__(self, store_key="media", **overrides): + super().__init__() + self.settings = StorageSettings(store_key, **overrides) self._bucket = None self._client = None def get_accessed_time(self, *args, **kwargs): - """Not implemented yet""" - raise NotImplementedError("Need to override the abstract base class for this method") + """Get the last accessed time of the file + + This value is ALWAYS None because we cannot know the last accessed time on a cloud file. + + This method is here for API compatibility with django's Storage class. + """ + return None def path(self, *args, **kwargs): - """Not implemented yet""" - raise NotImplementedError("Need to override the abstract base class for this method") - - def get_default_settings(self): - return { - "project_id": setting("GS_PROJECT_ID"), - "credentials": setting("GS_CREDENTIALS"), - "bucket_name": setting("GS_BUCKET_NAME"), - "custom_endpoint": setting("GS_CUSTOM_ENDPOINT", None), - "location": setting("GS_LOCATION", ""), - "default_acl": setting("GS_DEFAULT_ACL"), - "querystring_auth": setting("GS_QUERYSTRING_AUTH", True), - "expiration": setting("GS_EXPIRATION", timedelta(seconds=86400)), - "gzip": setting("GS_IS_GZIPPED", False), - "gzip_content_types": setting( - "GZIP_CONTENT_TYPES", - ( - "text/css", - "text/javascript", - "application/javascript", - "application/x-javascript", - "image/svg+xml", - ), - ), - "file_overwrite": setting("GS_FILE_OVERWRITE", True), - "cache_control": setting("GS_CACHE_CONTROL"), - "object_parameters": setting("GS_OBJECT_PARAMETERS", {}), - # The max amount of memory a returned file can take up before being - # rolled over into a temporary file on disk. Default is 0: Do not - # roll over. - "max_memory_size": setting("GS_MAX_MEMORY_SIZE", 0), - "blob_chunk_size": setting("GS_BLOB_CHUNK_SIZE"), - } + """Get the local path of the file + + This value is ALWAYS None because the path is not necessarily distinct for an object + not on the local filesystem. + + This method is here for API compatibility with django's Storage class. + """ @property def client(self): """The google-storage client for this store""" if self._client is None: - self._client = Client(project=self.project_id, credentials=self.credentials) + self._client = Client(project=self.settings.project_id, credentials=self.settings.credentials) return self._client @property def bucket(self): """The google-storage bucket object for this store""" if self._bucket is None: - self._bucket = self.client.bucket(self.bucket_name) + self._bucket = self.client.bucket(self.settings.bucket_name) return self._bucket def _normalize_name(self, name): @@ -170,7 +143,7 @@ def _normalize_name(self, name): to is not outside the directory specified by the LOCATION setting. """ try: - return safe_join(self.location, name) + return safe_join(self.settings.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name) @@ -190,10 +163,14 @@ def _save(self, name, content): upload_params = {} blob_params = self.get_object_parameters(name) - upload_params["predefined_acl"] = blob_params.pop("acl", self.default_acl) + upload_params["predefined_acl"] = blob_params.pop("acl", self.settings.default_acl) upload_params[CONTENT_TYPE] = blob_params.pop(CONTENT_TYPE, file_object.mime_type) - if self.gzip and upload_params[CONTENT_TYPE] in self.gzip_content_types and CONTENT_ENCODING not in blob_params: + if ( + self.settings.gzip + and upload_params[CONTENT_TYPE] in self.settings.gzip_content_types + and CONTENT_ENCODING not in blob_params + ): content = self._compress_content(content) blob_params[CONTENT_ENCODING] = "gzip" @@ -206,20 +183,9 @@ def _save(self, name, content): def get_object_parameters(self, name): """Override this to return a dictionary of overwritable blob-property to value. - Returns GS_OBJECT_PARAMETRS by default. See the docs for all possible options. + Returns GS_OBJECT_PARAMETERS by default. See the docs for all possible options. """ - object_parameters = self.object_parameters.copy() - - if "cache_control" not in object_parameters and self.cache_control: - warnings.warn( - "The GS_CACHE_CONTROL setting is deprecated. Use GS_OBJECT_PARAMETERS to set any " - "writable blob property or override GoogleCloudStorage.get_object_parameters to " - "vary the parameters per object.", - DeprecationWarning, - ) - object_parameters["cache_control"] = self.cache_control - - return object_parameters + return self.settings.object_parameters.copy() def delete(self, name): name = self._normalize_name(clean_name(name)) @@ -306,26 +272,60 @@ def url(self, name): name = self._normalize_name(clean_name(name)) blob = self.bucket.blob(name) blob_params = self.get_object_parameters(name) - no_signed_url = blob_params.get("acl", self.default_acl) == "publicRead" or not self.querystring_auth + no_signed_url = ( + blob_params.get("acl", self.settings.default_acl) == "publicRead" or not self.settings.querystring_auth + ) - if not self.custom_endpoint and no_signed_url: + if not self.settings.custom_endpoint and no_signed_url: return blob.public_url elif no_signed_url: return "{storage_base_url}/{quoted_name}".format( - storage_base_url=self.custom_endpoint, + storage_base_url=self.settings.custom_endpoint, quoted_name=_quote(name, safe=b"/~"), ) - elif not self.custom_endpoint: - return blob.generate_signed_url(expiration=self.expiration, version="v4") + elif not self.settings.custom_endpoint: + return blob.generate_signed_url(expiration=self.settings.expiration, version="v4") else: return blob.generate_signed_url( - bucket_bound_hostname=self.custom_endpoint, - expiration=self.expiration, + bucket_bound_hostname=self.settings.custom_endpoint, + expiration=self.settings.expiration, version="v4", ) def get_available_name(self, name, max_length=None): name = clean_name(name) - if self.file_overwrite: + if self.settings.file_overwrite: return get_available_overwrite_name(name, max_length) return super().get_available_name(name, max_length) + + +class GoogleCloudMediaStorage(GoogleCloudStorage): + """Storage whose bucket name is taken from the GCP_STORAGE_MEDIA_NAME setting + + This actually behaves exactly as a default instantitation of the base + ``GoogleCloudStorage`` class, but is there to make configuration more + explicit for first-timers. + + """ + + def __init__(self, **overrides): + if overrides.pop("store_key", "media") != "media": + raise ValueError("You cannot instantiate GoogleCloudMediaStorage with a store_key other than 'media'") + super().__init__(store_key="media", **overrides) + + +class GoogleCloudStaticStorage(GoogleCloudStorage): + """Storage defined with an appended bucket name (called "-static") + + We define that static files are stored in a different bucket than the (private) media files, which: + 1. gives us less risk of accidentally setting private files as public + 2. allows us easier visual inspection in the console of what's private and what's public static + 3. allows us to set blanket public ACLs on the static bucket + 4. makes it easier to clean up private files with no entry in the DB + + """ + + def __init__(self, **overrides): + if overrides.pop("store_key", "static") != "static": + raise ValueError("You cannot instantiate GoogleCloudStaticStorage with a store_key other than 'static'") + super().__init__(store_key="static", **overrides) diff --git a/django_gcp/storage/settings.py b/django_gcp/storage/settings.py new file mode 100644 index 0000000..fc59e2e --- /dev/null +++ b/django_gcp/storage/settings.py @@ -0,0 +1,129 @@ +from datetime import timedelta +from django.conf import settings as django_settings +from django.core.exceptions import ImproperlyConfigured +from django.core.signals import setting_changed + + +DEFAULT_GZIP_CONTENT_TYPES = ( + "text/css", + "text/javascript", + "application/javascript", + "application/x-javascript", + "image/svg+xml", +) + +DEFAULT_OBJECT_PARAMETERS = {} + +DEFAULT_GCP_SETTINGS = { + "project_id": None, + "credentials": None, +} + +DEFAULT_GCP_STORAGE_SETTINGS = { + "bucket_name": None, + "custom_endpoint": None, + "location": "", + "default_acl": None, + "querystring_auth": True, + "expiration": timedelta(seconds=86400), + "gzip": False, + "gzip_content_types": DEFAULT_GZIP_CONTENT_TYPES, + "file_overwrite": True, + "object_parameters": DEFAULT_OBJECT_PARAMETERS, + "max_memory_size": 0, + "blob_chunk_size": None, +} + + +class StorageSettings: + """Combine GCP_ and GCP_STORAGE_ settings for a given store + + Settings are determined and cached. Initial determination is done lazily (on first setting access + rather than on initialisation). This allows this class to be initialised prior to + django's apps being ready. + + """ + + def __init__(self, store_key, **overrides): + setting_changed.connect(self._handle_settings_changed, dispatch_uid="ic" + str(id(self))) + self._store_key = store_key + self._cache = None + self._overrides = overrides + + def __getattr__(self, setting_key): + """Get a setting value""" + + if self._cache is None: + self._update_settings() + + try: + return self._cache[setting_key] + except KeyError as e: + raise AttributeError(f"No setting '{setting_key}' available in StorageSettings") from e + + @property + def _stores_settings(self): + """Get a complete dict of all stores defined in settings.py (media + static + extras)""" + all_stores = { + "media": getattr(django_settings, "GCP_STORAGE_MEDIA", None), + "static": getattr(django_settings, "GCP_STORAGE_STATIC", None), + **getattr(django_settings, "GCP_STORAGE_EXTRA_STORES", {}), + } + + return dict((k, v) for k, v in all_stores.items() if v is not None) + + @property + def _store_settings(self): + """Dict of store settings defined in settings.py for the current store key""" + try: + return self._stores_settings[self._store_key] + except KeyError as e: + raise ImproperlyConfigured( + f"Mismatch: specified store key '{self._store_key}' does not match 'media', 'static', or any store key defined in GCP_STORAGE_EXTRA_STORES" + ) from e + + def _handle_settings_changed(self, **kwargs): + self._update_settings() + + def _update_settings(self): + """Fetch settings from django configuration, merge with defaults and cache + + Re-run on receiving ``setting_changed`` signal from django. + """ + + # Start with the default settings + to_cache = { + **DEFAULT_GCP_SETTINGS, + **DEFAULT_GCP_STORAGE_SETTINGS, + } + + # Add GCP_ settings from settings.py (common root level settings, used by the storage module) + for setting_key in DEFAULT_GCP_SETTINGS.keys(): # pylint: disable=consider-iterating-dictionary + try: + to_cache[setting_key] = getattr(django_settings, f"GCP_{setting_key.upper()}") + except AttributeError: + # Not defined in django settings, don't add it to the dict + pass + + # Add GCP_STORAGE_ settings (storage-specific settings for the current store) + for setting_key in DEFAULT_GCP_STORAGE_SETTINGS.keys(): # pylint: disable=consider-iterating-dictionary + try: + to_cache[setting_key] = self._store_settings[setting_key] + except KeyError: + pass + + # Add overrides and place in cache + self._cache = { + **to_cache, + **self._overrides, + } + + self.check() + + def check(self): + """Check the settings on this object""" + if self.location.startswith("/"): + correct = self.location.lstrip("/\\") + raise ImproperlyConfigured( + f"'location' option in GCP_STORAGE_ cannot begin with a leading slash. Found '{self.location}'. Use '{correct}' instead." + ) diff --git a/django_gcp/storage/utils.py b/django_gcp/storage/utils.py index 0911542..8420b54 100644 --- a/django_gcp/storage/utils.py +++ b/django_gcp/storage/utils.py @@ -1,7 +1,7 @@ import os import posixpath from django.conf import settings -from django.core.exceptions import ImproperlyConfigured, SuspiciousFileOperation +from django.core.exceptions import SuspiciousFileOperation from django.utils.encoding import force_bytes @@ -82,18 +82,6 @@ def safe_join(base, *paths): return final_path.lstrip("/") -def check_location(storage): - if storage.location.startswith("/"): - correct = storage.location.lstrip("/") - raise ImproperlyConfigured( - "{}.location cannot begin with a leading slash. Found '{}'. Use '{}' instead.".format( - storage.__class__.__name__, - storage.location, - correct, - ) - ) - - def get_available_overwrite_name(name, max_length): if max_length is None or len(name) <= max_length: return name diff --git a/docs/source/images/buckets.png b/docs/source/images/buckets.png new file mode 100644 index 0000000..5a51d0b Binary files /dev/null and b/docs/source/images/buckets.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index e0ec52d..d2345b1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -48,6 +48,7 @@ Contents self installation authentication + storage license version_history diff --git a/docs/source/storage.rst b/docs/source/storage.rst new file mode 100644 index 0000000..8cfc909 --- /dev/null +++ b/docs/source/storage.rst @@ -0,0 +1,355 @@ +.. _storage: + +Storage +==================== + +This module provides a Django File API for `Google Cloud Storage `_. + +Installation and Authentication +------------------------------- +First, follow the instructions to :ref:`install `, :ref:`authenticate ` and :ref:`set your project `. + +Create bucket(s) +---------------- +This library doesn't create buckets for you: infrastructure operations should be kept separate and dealt with using +tools built for the purpose, like terraform or Deployment Manager. + +If you're setting up for the first time and don't want to get into that kind of infrastructure-as-code stuff, then +manually create two buckets in your project: + - One with **object-level** permissions for **media** files. + - One with **uniform, public** permissions for **static** files. + +.. TIP:: + Having two buckets like this means it's easier to configure which files are public and which aren't. + Plus, you can serve your static files much more efficiently - `publicly shared files are cached in google's + cloud CDN `_, + so they're lightning quick for users to download, and egress costs you amost nothing. + +.. TIP:: + To make it easy and consistent to set up (and remember which is which!), we always use kebab case for our bucket names in the form: + + .. code-block:: + + --- + + The buckets for a staging environment in one of our apps look like this: + + .. figure:: images/buckets.png + :align: center + :figclass: align-center + :alt: Buckets configuration + +Setup Media and Static Storage +------------------------------ + +The most common types of storage are for media and static files. +We derived a custom storage type for each, making it easier to name them. + +In your ``settings.py`` file, do: + +.. code-block:: python + + # Set the default storage (for media files) + DEFAULT_FILE_STORAGE = "django_gcp.storage.GoogleCloudMediaStorage" + GCP_STORAGE_MEDIA = { + "bucket_name": "app-assets-environment-media" # Or whatever name you chose + } + + # Set the static file storage + # This allows `manage.py collectstatic` to automatically upload your static files + STATICFILES_STORAGE = "django_gcp.storage.GoogleCloudStaticStorage" + GCP_STORAGE_STATIC = { + "bucket_name": "app-assets-environment-static" # or whatever name you chose + } + + # Point the urls to the store locations + # You could customise the base URLs later with your own cdn, eg https://static.you.com + # But that's only if you feel like being ultra fancy + MEDIA_URL = f"https://storage.googleapis.com/{GCP_STORAGE_MEDIA_NAME}/" + MEDIA_ROOT = "/media/" + STATIC_URL = f"https://storage.googleapis.com/{GCP_STORAGE_STATIC_NAME}/" + STATIC_ROOT = "/static/" + + +Usage +----- + +.. tabs:: + + .. group-tab:: Extra Stores + + Any number of extra stores can be added, each corresponding to a different bucket in GCS. + + You'll need to give each one a "storage key" to identify it. In your ``settings.py``, include extra stores as: + + .. code-block:: python + + GCP_STORAGE_EXTRA_STORES = { + "my_fun_store_key": { + "bucket_name": "all-the-fun-datafiles" + }, + "my_sad_store_key": { + "bucket_name": "all-the-sad-datafiles" + } + } + + + .. group-tab:: Default Storage + + Once you're done, default_storage will be your Google Cloud Media Storage: + + .. code-block:: python + + >>> from django.core.files.storage import default_storage + >>> print(default_storage.__class__) + + + This way, if you define a new FileField, it will use that storage bucket: + + .. code-block:: python + + >>> from django.db import models + >>> class MyModel(models.Model): + ... my_file_field = models.FileField(upload_to='pdfs') + ... my_image_field = models.ImageField(upload_to='photos') + ... + >>> obj1 = MyModel() + >>> print(resume.pdf.storage) + + + .. group-tab:: File Access + + Standard file access options are available, and work as expected + + .. code-block:: python + + >>> default_storage.exists('storage_test') + False + >>> file = default_storage.open('storage_test', 'w') + >>> file.write('storage contents') + >>> file.close() + + >>> default_storage.exists('storage_test') + True + >>> file = default_storage.open('storage_test', 'r') + >>> file.read() + 'storage contents' + >>> file.close() + + >>> default_storage.delete('storage_test') + >>> default_storage.exists('storage_test') + False + + .. group-tab:: Models and FileFields + + An object without a file has limited functionality + + .. code-block:: python + + >>> obj1 = MyModel() + >>> obj1.my_file_field + + >>> obj1.my_file_field.size + Traceback (most recent call last): + ... + ValueError: The 'my_file_field' attribute has no file associated with it. + + Saving a file enables full functionality + + .. code-block:: python + + >>> obj1.my_file_field.save('django_test.txt', ContentFile('content')) + >>> obj1.my_file_field + + >>> obj1.my_file_field.size + 7 + >>> obj1.my_file_field.read() + 'content' + + Files can be read in a little at a time, if necessary + + .. code-block:: python + + >>> obj1.my_file_field.open() + >>> obj1.my_file_field.read(3) + 'con' + >>> obj1.my_file_field.read() + 'tent' + >>> '-'.join(obj1.my_file_field.chunks(chunk_size=2)) + 'co-nt-en-t' + + Save another file with the same name + + .. code-block:: python + + >>> obj2 = MyModel() + >>> obj2.my_file_field.save('django_test.txt', ContentFile('more content')) + >>> obj2.my_file_field + + >>> obj2.my_file_field.size + 12 + + Push the objects into the cache to make sure they pickle properly + + .. code-block:: python + + >>> cache.set('obj1', obj1) + >>> cache.set('obj2', obj2) + >>> cache.get('obj2').my_file_field + + + +Storage Setting Options +----------------------- + +Each store can be set up with different options, passed within the dict given to ``GCP_STORAGE_MEDIA``, ``GCP_STORAGE_STATIC`` or within the dicts given to ``GCP_STORAGE_EXTRA_STORES``. + +For example, to set the media storage up so that files go to a different location than the root of the bucket, you'd use: + +.. code-block:: python + + GCP_STORAGE_MEDIA = { + "bucket_name": "app-assets-environment-media" + "location": "not/the/bucket/root/", + # ... and whatever other options you want + } + +The full range of options (and their defaults, which apply to all stores) is as follows: + +gzip +^^^^ +Type: ``boolean`` + +Default: ``False`` + +Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES`` + +gzip_content_types +^^^^^^^^^^^^^^^^^^ +Type: ``tuple`` + +Default: (``text/css``, ``text/javascript``, ``application/javascript``, ``application/x-javascript``, ``image/svg+xml``) + +Content types which will be gzipped when ``GCP_STORAGE_IS_GZIPPED`` is ``True`` + +default_acl +^^^^^^^^^^^ +Type: ``string or None`` + +Default: ``None`` + +ACL used when creating a new blob, from the +`list of predefined ACLs `_. +(A "JSON API" ACL is preferred but an "XML API/gsutil" ACL will be +translated.) + +For most cases, the blob will need to be set to the ``publicRead`` ACL in order for the file to be viewed. +If ``GCP_STORAGE_DEFAULT_ACL`` is not set, the blob will have the default permissions set by the bucket. + +``publicRead`` files will return a public, non-expiring url. All other files return +a signed (expiring) url. + +ACL Options are: ``projectPrivate``, ``bucketOwnerRead``, ``bucketOwnerFullControl``, ``private``, ``authenticatedRead``, ``publicRead``, ``publicReadWrite`` + +.. note:: + GCP_STORAGE_DEFAULT_ACL must be set to 'publicRead' to return a public url. Even if you set + the bucket to public or set the file permissions directly in GCS to public. + +.. note:: + When using this setting, make sure you have ``fine-grained`` access control enabled on your bucket, + as opposed to ``Uniform`` access control, or else, file uploads will return with HTTP 400. If you + already have a bucket with ``Uniform`` access control set to public read, please keep + ``GCP_STORAGE_DEFAULT_ACL`` to ``None`` and set ``GCP_STORAGE_QUERYSTRING_AUTH`` to ``False``. + +querystring_auth +^^^^^^^^^^^^^^^^ +Type: ``boolean`` +Default: ``True`` + +If set to ``False`` it forces the url not to be signed. This setting is useful if you need to have a +bucket configured with ``Uniform`` access control configured with public read. In that case you should +force the flag ``GCP_STORAGE_QUERYSTRING_AUTH = False`` and ``GCP_STORAGE_DEFAULT_ACL = None`` + +file_overwrite +^^^^^^^^^^^^^^ +Type: ``boolean`` +Default: ``True`` + +By default files with the same name will overwrite each other. Set this to ``False`` to have extra characters appended. + +max_memory_size +^^^^^^^^^^^^^^^ +Type: ``integer`` +Default: ``0`` (do not roll over) + +The maximum amount of memory a returned file can take up (in bytes) before being +rolled over into a temporary file on disk. Default is 0: Do not roll over. + +blob_chunk_size +^^^^^^^^^^^^^^^ +Type: ``integer`` or ``None`` +Default ``None`` + +The size of blob chunks that are sent via resumable upload. If this is not set then the generated request +must fit in memory. Recommended if you are going to be uploading large files. + +.. note:: + + This must be a multiple of 256K (1024 * 256) + +object_parameters +^^^^^^^^^^^^^^^^^ +Type: ``dict`` +Default: ``{}`` + +Dictionary of key-value pairs mapping from blob property name to value. + +Use this to set parameters on **all** objects. To set these on a per-object +basis, subclass the backend and override ``GoogleCloudStorage.get_object_parameters``. + +The valid property names are :: + + acl + cache_control + content_disposition + content_encoding + content_language + content_type + metadata + storage_class + +If not set, the ``content_type`` property will be guessed. + +If set, ``acl`` overrides :ref:`GCP_STORAGE_DEFAULT_ACL `. + +.. warning:: + + Do not set ``name``. This is set automatically based on the filename. + +custom_endpoint +^^^^^^^^^^^^^^^ +Type: ``string`` or ``None`` +Default: ``None`` + +Sets a `custom endpoint `_, +that will be used instead of ``https://storage.googleapis.com`` when generating URLs for files. + +location +^^^^^^^^ +Type: ``string`` +Default: ``""`` + +Subdirectory in which the files will be stored. +Defaults to the root of the bucket. + +expiration +^^^^^^^^^^ +Type: ``datetime.timedelta`` ``datetime.datetime``, ``integer`` (seconds since epoch) +Default: ``timedelta(seconds=86400)`` + +The time that a generated URL is valid before expiration. The default is 1 day. +Public files will return a url that does not expire. Files will be signed by +the credentials provided during :ref:`authentication `. + +The ``GCP_STORAGE_EXPIRATION`` value is handled by the underlying `Google library `_. +It supports `timedelta`, `datetime`, or `integer` seconds since epoch time. diff --git a/poetry.lock b/poetry.lock index 02fb805..e39527a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -239,6 +239,17 @@ sqlparse = ">=0.2.2" argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-app-settings" +version = "0.7.1" +description = "Application settings helper for Django apps." +category = "main" +optional = false +python-versions = "~=3.5" + +[package.dependencies] +Django = "*" + [[package]] name = "docutils" version = "0.16" @@ -969,7 +980,7 @@ test = ["coverage", "pytest", "pycodestyle", "pylint"] [[package]] name = "twisted" -version = "21.7.0" +version = "22.1.0" description = "An asynchronous networking framework written in Python" category = "dev" optional = false @@ -984,24 +995,24 @@ idna = {version = ">=2.4", optional = true, markers = "extra == \"tls\""} incremental = ">=21.3.0" pyopenssl = {version = ">=16.0.0", optional = true, markers = "extra == \"tls\""} service-identity = {version = ">=18.1.0", optional = true, markers = "extra == \"tls\""} -twisted-iocpsupport = {version = ">=1.0.0,<1.1.0", markers = "platform_system == \"Windows\""} +twisted-iocpsupport = {version = ">=1.0.2,<2", markers = "platform_system == \"Windows\""} typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] contextvars = ["contextvars (>=2.4,<3)"] -dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=3.3,<4.0)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.2.2,<21.3.0)"] -dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=3.3,<4.0)", "pydoctor (>=21.2.2,<21.3.0)"] +dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] +dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.910)", "mypy-zope (==0.3.2)", "types-setuptools", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=3.3,<4.0)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.2.2,<21.3.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] -test = ["cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)"] +test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0,<2.0)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] [[package]] name = "twisted-iocpsupport" @@ -1091,7 +1102,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3ebf2a165ccfb93a347b1ff670eb701229b082af3fe0ada7290549b0884cbd39" +content-hash = "954904771f08a4241776b35067b0b3796f3a8eacc9a57b226af042ccf1f4c1f2" [metadata.files] alabaster = [ @@ -1284,6 +1295,9 @@ django = [ {file = "Django-3.2.11-py3-none-any.whl", hash = "sha256:0a0a37f0b93aef30c4bf3a839c187e1175bcdeb7e177341da0cb7b8194416891"}, {file = "Django-3.2.11.tar.gz", hash = "sha256:69c94abe5d6b1b088bf475e09b7b74403f943e34da107e798465d2045da27e75"}, ] +django-app-settings = [ + {file = "django-app-settings-0.7.1.tar.gz", hash = "sha256:75c9e951e4fd857f2e1abf46be459891a8fd8e86625adec243b56765d2d9e68e"}, +] docutils = [ {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, @@ -1699,8 +1713,8 @@ tox-poetry = [ {file = "tox_poetry-0.4.1-py2.py3-none-any.whl", hash = "sha256:11d9cd4e51d4cd9484b3ba63f2650ab4cfb4096e5f0682ecf561ddfc3c8e8c92"}, ] twisted = [ - {file = "Twisted-21.7.0-py3-none-any.whl", hash = "sha256:13c1d1d2421ae556d91e81e66cf0d4f4e4e1e4a36a0486933bee4305c6a4fb9b"}, - {file = "Twisted-21.7.0.tar.gz", hash = "sha256:2cd652542463277378b0d349f47c62f20d9306e57d1247baabd6d1d38a109006"}, + {file = "Twisted-22.1.0-py3-none-any.whl", hash = "sha256:ccd638110d9ccfdc003042aa3e1b6d6af272eaca45d36e083359560549e3e848"}, + {file = "Twisted-22.1.0.tar.gz", hash = "sha256:b7971ec9805b0f80e1dcb1a3721d7bfad636d5f909de687430ce373979d67b61"}, ] twisted-iocpsupport = [ {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"}, diff --git a/pyproject.toml b/pyproject.toml index e7b9b41..9541d8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-gcp" -version = "0.1.1" +version = "0.2.0" description = "Utilities to run Django on Google Cloud Platform" authors = ["Tom Clark"] license = "MIT" @@ -25,6 +25,7 @@ Django = "^3.0" python = "^3.8" google-cloud-storage = "^2.1.0" google-auth = "^2.6.0" +django-app-settings = "^0.7.1" [tool.poetry.dev-dependencies] diff --git a/setup.cfg b/setup.cfg index a04b960..27df18f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,12 +5,12 @@ long_description_content_type = text/markdown; charset=UTF-8 [flake8] max-line-length = 120 exclude = .devcontainer,.tox,.git,*/migrations/*,*/static/*,docs,node_modules,build,.pytest_cache -ignore = F405, E501 +ignore = F405, E501, W503 [isort] line_length=120 default_section = THIRDPARTY -known_third_party =django*,drf*,environ,google*,graphene,grapple,guardian,requests,rest* +known_third_party =django*,drf*,environ,google,google*,graphene,grapple,guardian,requests,rest* known_first_party = django_gcp,app,settings,test,tests sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER no_lines_before = THIRDPARTY, LOCALFOLDER diff --git a/tests/settings.py b/tests/settings.py index 9f857b2..5d173b7 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -67,6 +67,12 @@ def get_db_conf(): ROOT_URLCONF = "tests.urls" +DEFAULT_FILE_STORAGE = "django_gcp.storage.GoogleCloudMediaStorage" +GCP_STORAGE_MEDIA = {"bucket_name": "test-media"} + +STATICFILES_STORAGE = "django_gcp.storage.GoogleCloudStaticStorage" +GCP_STORAGE_STATIC = {"bucket_name": "test-static"} + STATIC_URL = "static_test/" MEDIA_URL = "media_test/" diff --git a/tests/test_storage.py b/tests/test_storage.py index 35607b4..8daa20e 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -32,10 +32,6 @@ def seek(self, pos, whence=0): class GCloudTestCase(TestCase): def setUp(self): - self.bucket_name = "test_bucket" - self.filename = "test_file.txt" - - self.storage = gcloud.GoogleCloudStorage(bucket_name=self.bucket_name) self.client_patcher = mock.patch("django_gcp.storage.gcloud.Client") self.client_patcher.start() @@ -45,6 +41,12 @@ def tearDown(self): class GCloudStorageTests(GCloudTestCase): + def setUp(self): + super().setUp() + self.bucket_name = "test-media" + self.filename = "test_file.txt" + self.storage = gcloud.GoogleCloudStorage(store_key="media", bucket_name=self.bucket_name) + def test_open_read(self): """ Test opening a file and reading from it @@ -89,24 +91,27 @@ def test_open_write(self, MockBlob): """ Test opening a file and writing to it """ - data = "This is some test write data." + with override_settings(GCP_STORAGE_MEDIA={"bucket_name": self.bucket_name, "default_acl": "projectPrivate"}): + data = "This is some test write data." - # Simulate the file not existing before the write - self.storage._bucket = mock.MagicMock() - self.storage._bucket.get_blob.return_value = None - self.storage.default_acl = "projectPrivate" + # Simulate the file not existing before the write + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None - f = self.storage.open(self.filename, "wb") - MockBlob.assert_called_with(self.filename, self.storage._bucket, chunk_size=None) + f = self.storage.open(self.filename, "wb") + MockBlob.assert_called_with(self.filename, self.storage._bucket, chunk_size=None) - f.write(data) - tmpfile = f._file - # File data is not actually written until close(), so do that. - f.close() + f.write(data) + tmpfile = f._file + # File data is not actually written until close(), so do that. + f.close() - MockBlob().upload_from_file.assert_called_with( - tmpfile, rewind=True, content_type=mimetypes.guess_type(self.filename)[0], predefined_acl="projectPrivate" - ) + MockBlob().upload_from_file.assert_called_with( + tmpfile, + rewind=True, + content_type=mimetypes.guess_type(self.filename)[0], + predefined_acl="projectPrivate", + ) def test_save(self): data = "This is some test content." @@ -140,21 +145,16 @@ def test_save_with_default_acl(self): filename = "ủⓝï℅ⅆℇ.txt" content = ContentFile(data) - # ACL Options - # 'projectPrivate', 'bucketOwnerRead', 'bucketOwnerFullControl', - # 'private', 'authenticatedRead', 'publicRead', 'publicReadWrite' - self.storage.default_acl = "publicRead" - - self.storage.save(filename, content) - - self.storage._client.bucket.assert_called_with(self.bucket_name) - self.storage._bucket.get_blob().upload_from_file.assert_called_with( - content, - rewind=True, - size=len(data), - content_type=mimetypes.guess_type(filename)[0], - predefined_acl="publicRead", - ) + with override_settings(GCP_STORAGE_MEDIA={"bucket_name": self.bucket_name, "default_acl": "publicRead"}): + self.storage.save(filename, content) + self.storage._client.bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob().upload_from_file.assert_called_with( + content, + rewind=True, + size=len(data), + content_type=mimetypes.guess_type(filename)[0], + predefined_acl="publicRead", + ) def test_delete(self): self.storage.delete(self.filename) @@ -324,16 +324,17 @@ def test_modified_time_no_file(self): def test_url_public_object(self): url = f"https://example.com/mah-bukkit/{self.filename}" - self.storage.default_acl = "publicRead" - self.storage._bucket = mock.MagicMock() - blob = mock.MagicMock() - blob.public_url = url - blob.generate_signed_url = "not called" - self.storage._bucket.blob.return_value = blob + with override_settings(GCP_STORAGE_MEDIA={"bucket_name": self.bucket_name, "default_acl": "publicRead"}): - self.assertEqual(self.storage.url(self.filename), url) - self.storage._bucket.blob.assert_called_with(self.filename) + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.public_url = url + blob.generate_signed_url = "not called" + self.storage._bucket.blob.return_value = blob + + self.assertEqual(self.storage.url(self.filename), url) + self.storage._bucket.blob.assert_called_with(self.filename) def test_url_not_public_file(self): secret_filename = "secret_file.txt" @@ -350,52 +351,72 @@ def test_url_not_public_file(self): blob.generate_signed_url.assert_called_with(expiration=timedelta(seconds=86400), version="v4") def test_url_not_public_file_with_custom_expires(self): - secret_filename = "secret_file.txt" - self.storage._bucket = mock.MagicMock() - blob = mock.MagicMock() - generate_signed_url = mock.MagicMock(return_value="http://signed_url") - blob.generate_signed_url = generate_signed_url - self.storage._bucket.blob.return_value = blob - - self.storage.expiration = timedelta(seconds=3600) - url = self.storage.url(secret_filename) - self.storage._bucket.blob.assert_called_with(secret_filename) - self.assertEqual(url, "http://signed_url") - blob.generate_signed_url.assert_called_with(expiration=timedelta(seconds=3600), version="v4") + expiration = timedelta(seconds=3600) + with override_settings(GCP_STORAGE_MEDIA={"bucket_name": self.bucket_name, "expiration": expiration}): + secret_filename = "secret_file.txt" + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + generate_signed_url = mock.MagicMock(return_value="http://signed_url") + blob.generate_signed_url = generate_signed_url + self.storage._bucket.blob.return_value = blob + url = self.storage.url(secret_filename) + self.storage._bucket.blob.assert_called_with(secret_filename) + self.assertEqual(url, "http://signed_url") + blob.generate_signed_url.assert_called_with(expiration=expiration, version="v4") def test_custom_endpoint(self): - self.storage.custom_endpoint = "https://example.com" - - self.storage.default_acl = "publicRead" - url = "{}/{}".format(self.storage.custom_endpoint, self.filename) - self.assertEqual(self.storage.url(self.filename), url) - bucket_name = "hyacinth" - self.storage.default_acl = "projectPrivate" - self.storage._bucket = mock.MagicMock() - blob = mock.MagicMock() - generate_signed_url = mock.MagicMock() - blob.bucket = mock.MagicMock() - type(blob.bucket).name = mock.PropertyMock(return_value=bucket_name) - blob.generate_signed_url = generate_signed_url - self.storage._bucket.blob.return_value = blob - self.storage.url(self.filename) - blob.generate_signed_url.assert_called_with( - bucket_bound_hostname=self.storage.custom_endpoint, - expiration=timedelta(seconds=86400), - version="v4", - ) + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "custom_endpoint": "https://example.com", + "default_acl": "publicRead", + } + ): + url = f"{self.storage.settings.custom_endpoint}/{self.filename}" + self.assertEqual(self.storage.url(self.filename), url) + + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "custom_endpoint": "https://example.com", + "default_acl": "projectPrivate", + } + ): + bucket_name = "hyacinth" + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + generate_signed_url = mock.MagicMock() + blob.bucket = mock.MagicMock() + type(blob.bucket).name = mock.PropertyMock(return_value=bucket_name) + blob.generate_signed_url = generate_signed_url + self.storage._bucket.blob.return_value = blob + self.storage.url(self.filename) + blob.generate_signed_url.assert_called_with( + bucket_bound_hostname=self.storage.settings.custom_endpoint, + expiration=timedelta(seconds=86400), + version="v4", + ) def test_get_available_name(self): - self.storage.file_overwrite = True - self.assertEqual(self.storage.get_available_name(self.filename), self.filename) - - self.storage._bucket = mock.MagicMock() - self.storage._bucket.get_blob.return_value = None - self.storage.file_overwrite = False - self.assertEqual(self.storage.get_available_name(self.filename), self.filename) - self.storage._bucket.get_blob.assert_called_with(self.filename) + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "file_overwrite": True, + } + ): + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "file_overwrite": False, + } + ): + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + self.storage._bucket.get_blob.assert_called_with(self.filename) def test_get_available_name_unicode(self): filename = "ủⓝï℅ⅆℇ.txt" @@ -406,13 +427,17 @@ def test_cache_control(self): filename = "cache_control_file.txt" content = ContentFile(data) cache_control = "public, max-age=604800" - - self.storage.cache_control = cache_control - self.storage.save(filename, content) - - bucket = self.storage.client.bucket(self.bucket_name) - blob = bucket.get_blob(filename) - self.assertEqual(blob.cache_control, cache_control) + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "file_overwrite": True, + "object_parameters": {"cache_control": cache_control}, + } + ): + self.storage.save(filename, content) + bucket = self.storage.client.bucket(self.bucket_name) + blob = bucket.get_blob(filename) + self.assertEqual(blob.cache_control, cache_control) def test_storage_save_gzipped(self): """ @@ -438,53 +463,63 @@ def test_storage_save_gzip(self): """ Test saving a file with gzip enabled. """ - self.storage.gzip = True - name = "test_storage_save.css" - content = ContentFile("I should be gzip'd") - self.storage.save(name, content) - self.storage._client.bucket.assert_called_with(self.bucket_name) - obj = self.storage._bucket.get_blob() - self.assertEqual(obj.content_encoding, "gzip") - obj.upload_from_file.assert_called_with( - mock.ANY, - rewind=True, - size=None, - predefined_acl=None, - content_type="text/css", - ) - args, _ = obj.upload_from_file.call_args - content = args[0] - zfile = gzip.GzipFile(mode="rb", fileobj=content) - self.assertEqual(zfile.read(), b"I should be gzip'd") + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "gzip": True, + } + ): + + name = "test_storage_save.css" + content = ContentFile("I should be gzip'd") + self.storage.save(name, content) + self.storage._client.bucket.assert_called_with(self.bucket_name) + obj = self.storage._bucket.get_blob() + self.assertEqual(obj.content_encoding, "gzip") + obj.upload_from_file.assert_called_with( + mock.ANY, + rewind=True, + size=None, + predefined_acl=None, + content_type="text/css", + ) + args, _ = obj.upload_from_file.call_args + content = args[0] + zfile = gzip.GzipFile(mode="rb", fileobj=content) + self.assertEqual(zfile.read(), b"I should be gzip'd") def test_storage_save_gzip_twice(self): """ Test saving the same file content twice with gzip enabled. """ - # Given - self.storage.gzip = True - name = "test_storage_save.css" - content = ContentFile("I should be gzip'd") - - # When - self.storage.save(name, content) - self.storage.save("test_storage_save_2.css", content) - - # Then - self.storage._client.bucket.assert_called_with(self.bucket_name) - obj = self.storage._bucket.get_blob() - self.assertEqual(obj.content_encoding, "gzip") - obj.upload_from_file.assert_called_with( - mock.ANY, - rewind=True, - size=None, - predefined_acl=None, - content_type="text/css", - ) - args, _ = obj.upload_from_file.call_args - content = args[0] - zfile = gzip.GzipFile(mode="rb", fileobj=content) - self.assertEqual(zfile.read(), b"I should be gzip'd") + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "gzip": True, + } + ): + name = "test_storage_save.css" + content = ContentFile("I should be gzip'd") + + # When + self.storage.save(name, content) + self.storage.save("test_storage_save_2.css", content) + + # Then + self.storage._client.bucket.assert_called_with(self.bucket_name) + obj = self.storage._bucket.get_blob() + self.assertEqual(obj.content_encoding, "gzip") + obj.upload_from_file.assert_called_with( + mock.ANY, + rewind=True, + size=None, + predefined_acl=None, + content_type="text/css", + ) + args, _ = obj.upload_from_file.call_args + content = args[0] + zfile = gzip.GzipFile(mode="rb", fileobj=content) + self.assertEqual(zfile.read(), b"I should be gzip'd") def test_compress_content_len(self): """ @@ -496,20 +531,53 @@ def test_compress_content_len(self): self.assertTrue(len(content.read()) > 0) def test_location_leading_slash(self): - msg = "GoogleCloudStorage.location cannot begin with a leading slash. Found '/'. Use '' instead." + msg = "'location' option in GCP_STORAGE_ cannot begin with a leading slash. Found '/'. Use '' instead." with self.assertRaises(ImproperlyConfigured, msg=msg): - gcloud.GoogleCloudStorage(location="/") + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": self.bucket_name, + "location": "/", + } + ): + self.storage.settings.check() + +class GCloudStorageClassTests(GCloudTestCase): def test_override_settings(self): - with override_settings(GS_LOCATION="foo1"): + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": "test_media", + "location": "foo1", + } + ): storage = gcloud.GoogleCloudStorage() - self.assertEqual(storage.location, "foo1") - with override_settings(GS_LOCATION="foo2"): + self.assertEqual(storage.settings.location, "foo1") + with override_settings( + GCP_STORAGE_MEDIA={ + "bucket_name": "test_media", + "location": "foo2", + } + ): storage = gcloud.GoogleCloudStorage() - self.assertEqual(storage.location, "foo2") + self.assertEqual(storage.settings.location, "foo2") def test_override_init_argument(self): storage = gcloud.GoogleCloudStorage(location="foo1") - self.assertEqual(storage.location, "foo1") + self.assertEqual(storage.settings.location, "foo1") storage = gcloud.GoogleCloudStorage(location="foo2") - self.assertEqual(storage.location, "foo2") + self.assertEqual(storage.settings.location, "foo2") + + def test_media_storage_instantiation(self): + storage = gcloud.GoogleCloudMediaStorage() + self.assertEqual(storage.settings.bucket_name, "test-media") + + def test_static_storage_instantiation(self): + storage = gcloud.GoogleCloudStaticStorage() + self.assertEqual(storage.settings.bucket_name, "test-static") + + def test_instiantiation_with_store_key_raises_exception(self): + with self.assertRaises(ValueError): + gcloud.GoogleCloudMediaStorage(store_key="not-media") + + with self.assertRaises(ValueError): + gcloud.GoogleCloudStaticStorage(store_key="not-static")