diff --git a/.gitignore b/.gitignore index ba74660..a40e226 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,14 @@ docs/_build/ # PyBuilder target/ + +# Virtual environtment +venv +.env +.venv + +# Editor stuff +.vscode + +# OS stuff +.DS_Store diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7d48925 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: prepare-test +prepare-test: + pip install pyvows coverage tornado_pyvows + +.PHONY: test +test: + @echo "Restart MongoDB" + @docker-compose down + @docker-compose up -d + @echo "Run Vows" + @pyvows -c -l tc_mongodb diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..81eb92e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.3' +services: + mongo: + image: mvertes/alpine-mongo:4.0.5-0 + container_name: thumbor_mongo + ports: + - 27017:27017 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..14aba17 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pymongo==3.10.1 +thumbor>=5.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 1c3c78a..58b7eeb 100644 --- a/setup.py +++ b/setup.py @@ -14,17 +14,18 @@ def read(fname): setup( name="tc_mongodb", - version="5.1.0", + version="5.3.0", author="Thumbor Community", description=("Thumbor thumbor storage adapters"), license="MIT", keywords="thumbor mongodb mongo", url="https://github.com/thumbor-community/mongodb", - packages=[ + packages=find_packages(include=[ 'tc_mongodb', + 'tc_mongodb.mongodb', 'tc_mongodb.storages', 'tc_mongodb.result_storages' - ], + ]), long_description=long_description, classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -32,7 +33,7 @@ def read(fname): 'Programming Language :: Python :: 2.7', ], install_requires=[ - 'thumbor>=5.0.0', - 'pymongo' + 'thumbor>=6.5.1,<7.0.0', + 'pymongo==3.10.1' ] ) diff --git a/tc_mongodb/__init__.py b/tc_mongodb/__init__.py index f124b15..e69de29 100644 --- a/tc_mongodb/__init__.py +++ b/tc_mongodb/__init__.py @@ -1,5 +0,0 @@ - -Config.define('MONGO_STORAGE_SERVER_HOST', 'localhost', 'MongoDB storage server host', 'MongoDB Storage') -Config.define('MONGO_STORAGE_SERVER_PORT', 27017, 'MongoDB storage server port', 'MongoDB Storage') -Config.define('MONGO_STORAGE_SERVER_DB', 'thumbor', 'MongoDB storage server database name', 'MongoDB Storage') -Config.define('MONGO_STORAGE_SERVER_COLLECTION', 'images', 'MongoDB storage image collection', 'MongoDB Storage') diff --git a/tc_mongodb/mongodb/__init__.py b/tc_mongodb/mongodb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tc_mongodb/mongodb/connector_result_storage.py b/tc_mongodb/mongodb/connector_result_storage.py new file mode 100644 index 0000000..68432fb --- /dev/null +++ b/tc_mongodb/mongodb/connector_result_storage.py @@ -0,0 +1,54 @@ +from pymongo import ASCENDING, DESCENDING, MongoClient + + +class Singleton(type): + """ + Define an Instance operation that lets clients access its unique + instance. + """ + + def __init__(cls, name, bases, attrs, **kwargs): + super(Singleton, cls).__init__(name, bases, attrs) + cls._instance = None + + def __call__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instance + + +class MongoConnector(object): + __metaclass__ = Singleton + + def __init__(self, + uri=None, + host=None, + port=None, + db_name=None, + coll_name=None): + self.uri = uri + self.host = host + self.port = port + self.db_name = db_name + self.coll_name = coll_name + self.db_conn, self.coll_conn = self.create_connection() + self.ensure_index() + + def create_connection(self): + if self.uri: + connection = MongoClient(self.uri) + else: + connection = MongoClient(self.host, self.port) + + db_conn = connection[self.db_name] + coll_conn = db_conn[self.coll_name] + + return db_conn, coll_conn + + def ensure_index(self): + index_name = 'key_1_created_at_-1' + if index_name not in self.coll_conn.index_information(): + self.coll_conn.create_index( + [('key', ASCENDING), ('created_at', DESCENDING)], + name=index_name + ) diff --git a/tc_mongodb/mongodb/connector_storage.py b/tc_mongodb/mongodb/connector_storage.py new file mode 100644 index 0000000..a8d0deb --- /dev/null +++ b/tc_mongodb/mongodb/connector_storage.py @@ -0,0 +1,54 @@ +from pymongo import ASCENDING, DESCENDING, MongoClient + + +class Singleton(type): + """ + Define an Instance operation that lets clients access its unique + instance. + """ + + def __init__(cls, name, bases, attrs, **kwargs): + super(Singleton, cls).__init__(name, bases, attrs) + cls._instance = None + + def __call__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instance + + +class MongoConnector(object): + __metaclass__ = Singleton + + def __init__(self, + uri=None, + host=None, + port=None, + db_name=None, + coll_name=None): + self.uri = uri + self.host = host + self.port = port + self.db_name = db_name + self.coll_name = coll_name + self.db_conn, self.coll_conn = self.create_connection() + self.ensure_index() + + def create_connection(self): + if self.uri: + connection = MongoClient(self.uri) + else: + connection = MongoClient(self.host, self.port) + + db_conn = connection[self.db_name] + coll_conn = db_conn[self.coll_name] + + return db_conn, coll_conn + + def ensure_index(self): + index_name = 'path_1_created_at_-1' + if index_name not in self.coll_conn.index_information(): + self.coll_conn.create_index( + [('path', ASCENDING), ('created_at', DESCENDING)], + name=index_name + ) diff --git a/tc_mongodb/result_storages/mongo_result_storage.py b/tc_mongodb/result_storages/mongo_result_storage.py new file mode 100644 index 0000000..c5e3752 --- /dev/null +++ b/tc_mongodb/result_storages/mongo_result_storage.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2015 Thumbor-Community + +import time +from datetime import datetime, timedelta +import pytz + +import gridfs +from pymongo.errors import PyMongoError +from tornado.concurrent import return_future +from thumbor.engines import BaseEngine +from thumbor.result_storages import BaseStorage, ResultStorageResult +from thumbor.utils import logger +from tc_mongodb.utils import OnException +from tc_mongodb.mongodb.connector_result_storage import MongoConnector + + +class Storage(BaseStorage): + + '''start_time is used to calculate the last modified value when an item + has no expiration date. + ''' + start_time = None + + def __init__(self, context): + BaseStorage.__init__(self, context) + self.database, self.storage = self.__conn__() + + if not Storage.start_time: + Storage.start_time = time.time() + super(Storage, self).__init__(context) + + def __conn__(self): + '''Return the MongoDB database and collection object. + :returns: MongoDB DB and Collection + :rtype: pymongo.database.Database, pymongo.database.Collection + ''' + + mongo_conn = MongoConnector( + uri=self.context.config.MONGO_RESULT_STORAGE_URI, + host=self.context.config.MONGO_RESULT_STORAGE_SERVER_HOST, + port=self.context.config.MONGO_RESULT_STORAGE_SERVER_PORT, + db_name=self.context.config.MONGO_RESULT_STORAGE_SERVER_DB, + coll_name= + self.context.config.MONGO_RESULT_STORAGE_SERVER_COLLECTION + ) + + database = mongo_conn.db_conn + storage = mongo_conn.coll_conn + + return database, storage + + def on_mongodb_error(self, fname, exc_type, exc_value): + '''Callback executed when there is a redis error. + :param string fname: Function name that was being called. + :param type exc_type: Exception type + :param Exception exc_value: The current exception + :returns: Default value or raise the current exception + ''' + + logger.error("[MONGODB_RESULT_STORAGE] %s,%s" % exc_type, exc_value) + if fname == '_exists': + return False + return None + + def is_auto_webp(self): + ''' + TODO This should be moved into the base storage class. + It is shared with file_result_storage + :return: If the file is a webp + :rettype: boolean + ''' + + return self.context.config.AUTO_WEBP \ + and self.context.request.accepts_webp + + def get_key_from_request(self): + '''Return a key for the current request url. + :return: The storage key for the current url + :rettype: string + ''' + + path = "result:%s" % self.context.request.url + + if self.is_auto_webp(): + path += '/webp' + + return path + + def get_max_age(self): + '''Return the TTL of the current request. + :returns: The TTL value for the current request. + :rtype: int + ''' + + default_ttl = self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS + + return default_ttl + + def is_expired(self, key): + """ + Tells whether key has expired + :param string key: Path to check + :return: Whether it is expired or not + :rtype: bool + """ + if key: + expire = self.get_max_age + + if expire is None or expire == 0: + return False + + image = next(self.storage.find({ + 'key': key, + 'created_at': { + '$gte': datetime.utcnow() - timedelta( + seconds=self.get_max_age() + ) + }, + }, { + 'created_at': True, '_id': False + }).limit(1), None) + + if image: + age = int( + (datetime.utcnow() - image['created_at']).total_seconds() + ) + timediff = datetime.utcnow() - timedelta(seconds=age) + return timediff.seconds > expire + else: + return True + + @OnException(on_mongodb_error, PyMongoError) + def put(self, bytes): + '''Save to mongodb + :param bytes: Bytes to write to the storage. + :return: MongoDB _id for the current url + :rettype: string + ''' + + doc = { + 'key': self.get_key_from_request(), + 'created_at': datetime.utcnow() + } + + if self.context.config.get("MONGO_STORE_METADATA", False): + doc['metadata'] = dict(self.context.headers) + else: + doc['metadata'] = {} + + file_doc = dict(doc) + + file_storage = gridfs.GridFS(self.database) + file_data = file_storage.put(bytes, **doc) + + file_doc['file_id'] = file_data + self.storage.insert_one(file_doc) + + @return_future + def get(self, callback): + '''Get the item from MongoDB.''' + + key = self.get_key_from_request() + callback(self._get(key)) + + @OnException(on_mongodb_error, PyMongoError) + def _get(self, key): + stored = next(self.storage.find({ + 'key': key, + 'created_at': { + '$gte': datetime.utcnow() - timedelta( + seconds=self.get_max_age() + ) + }, + }, { + 'file_id': True, + 'created_at': True, + 'metadata': True + }).limit(1), None) + + if not stored: + return None + + file_storage = gridfs.GridFS(self.database) + + contents = file_storage.get(stored['file_id']).read() + + metadata = stored['metadata'] + metadata['LastModified'] = stored['created_at'].replace( + tzinfo=pytz.utc + ) + metadata['ContentLength'] = len(contents) + metadata['ContentType'] = BaseEngine.get_mimetype(contents) + result = ResultStorageResult( + buffer=contents, + metadata=metadata, + successful=True + ) + return result + + @OnException(on_mongodb_error, PyMongoError) + def last_updated(self): + '''Return the last_updated time of the current request item + :return: A DateTime object + :rettype: datetetime.datetime + ''' + + key = self.get_key_from_request() + max_age = self.get_max_age() + + if max_age == 0: + return datetime.fromtimestamp(Storage.start_time) + + image = next(self.storage.find({ + 'key': key, + 'created_at': { + '$gte': datetime.utcnow() - timedelta( + seconds=self.get_max_age() + ) + }, + }, { + 'created_at': True, '_id': False + }).limit(1), None) + + if image: + age = int( + (datetime.utcnow() - image['created_at']).total_seconds() + ) + ttl = max_age - age + + if max_age <= 0: + return datetime.fromtimestamp(Storage.start_time) + + if ttl >= 0: + return datetime.utcnow() - timedelta( + seconds=( + max_age - ttl + ) + ) + + # Should never reach here. It means the storage put failed or the item + # somehow does not exists anymore + return datetime.utcnow() diff --git a/tc_mongodb/storages/mongo_storage.py b/tc_mongodb/storages/mongo_storage.py index ec7be03..f4ca3fc 100644 --- a/tc_mongodb/storages/mongo_storage.py +++ b/tc_mongodb/storages/mongo_storage.py @@ -5,118 +5,176 @@ # Copyright (c) 2011 globo.com timehome@corp.globo.com from datetime import datetime, timedelta -from cStringIO import StringIO - -from pymongo import MongoClient import gridfs - -from thumbor.storages import BaseStorage +from pymongo.errors import PyMongoError from tornado.concurrent import return_future +from thumbor.storages import BaseStorage +from thumbor.utils import logger +from tc_mongodb.utils import OnException +from tc_mongodb.mongodb.connector_storage import MongoConnector class Storage(BaseStorage): + def __init__(self, context): + '''Initialize the MongoStorage + + :param thumbor.context.Context shared_client: Current context + ''' + BaseStorage.__init__(self, context) + self.database, self.storage = self.__conn__() + super(Storage, self).__init__(context) + def __conn__(self): - connection = MongoClient( - self.context.config.MONGO_STORAGE_SERVER_HOST, - self.context.config.MONGO_STORAGE_SERVER_PORT + '''Return the MongoDB database and collection object. + :returns: MongoDB DB and Collection + :rtype: pymongo.database.Database, pymongo.database.Collection + ''' + + mongo_conn = MongoConnector( + uri=self.context.config.MONGO_STORAGE_URI, + host=self.context.config.MONGO_STORAGE_SERVER_HOST, + port=self.context.config.MONGO_STORAGE_SERVER_PORT, + db_name=self.context.config.MONGO_STORAGE_SERVER_DB, + coll_name= + self.context.config.MONGO_STORAGE_SERVER_COLLECTION ) - db = connection[self.context.config.MONGO_STORAGE_SERVER_DB] - storage = db[self.context.config.MONGO_STORAGE_SERVER_COLLECTION] + database = mongo_conn.db_conn + storage = mongo_conn.coll_conn - return connection, db, storage + return database, storage - def put(self, path, bytes): - connection, db, storage = self.__conn__() + def on_mongodb_error(self, fname, exc_type, exc_value): + '''Callback executed when there is a redis error. + :param string fname: Function name that was being called. + :param type exc_type: Exception type + :param Exception exc_value: The current exception + :returns: Default value or raise the current exception + ''' + if self.context.config.MONGODB_STORAGE_IGNORE_ERRORS: + logger.error("[MONGODB_STORAGE] %s,%s" % exc_type, exc_value) + if fname == '_exists': + return False + return None + else: + raise exc_value + + def get_max_age(self): + '''Return the TTL of the current request. + :returns: The TTL value for the current request. + :rtype: int + ''' + + return self.context.config.STORAGE_EXPIRATION_SECONDS + + @OnException(on_mongodb_error, PyMongoError) + def put(self, path, bytes): doc = { 'path': path, - 'created_at': datetime.now() + 'created_at': datetime.utcnow() } doc_with_crypto = dict(doc) if self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: if not self.context.server.security_key: - raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified") + raise RuntimeError( + "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True \ + if no SECURITY_KEY specified") doc_with_crypto['crypto'] = self.context.server.security_key - fs = gridfs.GridFS(db) - file_data = fs.put(StringIO(bytes), **doc) + file_storage = gridfs.GridFS(self.database) + file_data = file_storage.put(bytes, **doc) doc_with_crypto['file_id'] = file_data - storage.insert(doc_with_crypto) - return path + self.storage.insert_one(doc_with_crypto) + @OnException(on_mongodb_error, PyMongoError) def put_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: - return - - connection, db, storage = self.__conn__() + return None if not self.context.server.security_key: - raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified") + raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be \ + True if no SECURITY_KEY specified") - crypto = storage.find_one({'path': path}) - - crypto['crypto'] = self.context.server.security_key - storage.update({'path': path}, crypto) - return path + self.storage.update_one( + {'path': path}, + {'$set': {'crypto': self.context.server.security_key}} + ) + @OnException(on_mongodb_error, PyMongoError) def put_detector_data(self, path, data): - connection, db, storage = self.__conn__() - - storage.update({'path': path}, {"$set": {"detector_data": data}}) - return path + self.storage.update({'path': path}, {"$set": {"detector_data": data}}) @return_future def get_crypto(self, path, callback): - connection, db, storage = self.__conn__() + callback(self._get_crypto(path)) - crypto = storage.find_one({'path': path}) - callback(crypto.get('crypto') if crypto else None) + @OnException(on_mongodb_error, PyMongoError) + def _get_crypto(self, path): + crypto = self.storage.find_one({'path': path}) + return crypto.get('crypto') if crypto else None @return_future def get_detector_data(self, path, callback): - connection, db, storage = self.__conn__() + callback(self._get_detector_data(path)) - doc = storage.find_one({'path': path}) - callback(doc.get('detector_data') if doc else None) + @OnException(on_mongodb_error, PyMongoError) + def _get_detector_data(self, path): + doc = next(self.storage.find({ + 'path': path, + 'detector_data': {'$ne': None}, + }, { + 'detector_data': True, + }).limit(1), None) + + return doc.get('detector_data') if doc else None @return_future def get(self, path, callback): - connection, db, storage = self.__conn__() - - stored = storage.find_one({'path': path}) - - if not stored or self.__is_expired(stored): - callback(None) - return + callback(self._get(path)) + + @OnException(on_mongodb_error, PyMongoError) + def _get(self, path): + now = datetime.utcnow() + stored = next( + self.storage.find({ + 'path': path, + 'created_at': { + '$gte': now - timedelta(seconds=self.get_max_age())}, + }, {'file_id': True}).limit(1), None + ) - fs = gridfs.GridFS(db) + if not stored: + return None - contents = fs.get(stored['file_id']).read() + file_storage = gridfs.GridFS(self.database) - callback(str(contents)) + contents = file_storage.get(stored['file_id']).read() + return contents @return_future def exists(self, path, callback): - connection, db, storage = self.__conn__() - - stored = storage.find_one({'path': path}) + callback(self._exists(path)) - if not stored or self.__is_expired(stored): - callback(False) - else: - callback(True) + @OnException(on_mongodb_error, PyMongoError) + def _exists(self, path): + return self.storage.find({ + 'path': path, + 'created_at': { + '$gte': + datetime.utcnow() - timedelta(seconds=self.get_max_age()) + }, + }).limit(1).count() >= 1 + @OnException(on_mongodb_error, PyMongoError) def remove(self, path): - if not self.exists(path): - return - - connection, db, storage = self.__conn__() - storage.remove({'path': path}) + self.storage.delete_many({'path': path}) - def __is_expired(self, stored): - timediff = datetime.now() - stored.get('created_at') - return timediff > timedelta(seconds=self.context.config.STORAGE_EXPIRATION_SECONDS) + file_storage = gridfs.GridFS(self.database) + file_datas = file_storage.find({'path': path}) + if file_datas: + for file_data in file_datas: + file_storage.delete(file_data._id) diff --git a/tc_mongodb/utils.py b/tc_mongodb/utils.py new file mode 100644 index 0000000..e11dd8e --- /dev/null +++ b/tc_mongodb/utils.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + + +class OnException(object): # NOQA + + def __init__(self, callback, exception_class=Exception): + self.callback = callback + self.exception_class = exception_class + + def __call__(self, fn): + def wrapper(*args, **kwargs): + self_instance = args[0] if args else None + try: + return fn(*args, **kwargs) + except self.exception_class as exc_value: + if self.callback: + # Execute the callback and let it handle the exception + if self_instance: + return self.callback( + self_instance, + fn.__name__, + self.exception_class, + exc_value + ) + else: + return self.callback( + fn.__name__, + self.exception_class, + exc_value + ) + else: + raise + + return wrapper diff --git a/vows/__init__.py b/vows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vows/fixtures/__init__.py b/vows/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vows/fixtures/image.png b/vows/fixtures/image.png new file mode 100644 index 0000000..1470407 Binary files /dev/null and b/vows/fixtures/image.png differ diff --git a/vows/fixtures/storage_fixtures.py b/vows/fixtures/storage_fixtures.py new file mode 100644 index 0000000..d396179 --- /dev/null +++ b/vows/fixtures/storage_fixtures.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + +from thumbor.context import ServerParameters +from os.path import join, abspath, dirname + +SAME_IMAGE_URL = 's.glbimg.com/some_other/image_%d.png' +IMAGE_URL = 's.glbimg.com/some/image_%d.png' +IMAGE_PATH = join(abspath(dirname(__file__)), 'image.png') + +with open(IMAGE_PATH, 'r') as img: + IMAGE_BYTES = img.read() + + +def get_server(key=None): + server_params = ServerParameters( + 8888, 'localhost', 'thumbor.conf', None, 'info', None + ) + server_params.security_key = key + return server_params diff --git a/vows/mongodb_storage_vows.py b/vows/mongodb_storage_vows.py new file mode 100644 index 0000000..6b08962 --- /dev/null +++ b/vows/mongodb_storage_vows.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/globocom/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + + +from tc_mongodb.storages.mongo_storage import Storage as MongoStorage +from thumbor.context import Context +from thumbor.config import Config +from pymongo import MongoClient +from pymongo.errors import PyMongoError +from pyvows import Vows, expect +from fixtures.storage_fixtures import IMAGE_URL, IMAGE_BYTES, get_server + + +class MongoDBContext(Vows.Context): + def setup(self): + self.connection = MongoClient( + 'localhost', + 27017 + ) + self.database = self.connection['thumbor'] + self.storage = self.database['images'] + +@Vows.batch +class MongoStorageVows(MongoDBContext): + class CanStoreImage(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + storage.put(IMAGE_URL % 9999, IMAGE_BYTES) + return storage.exists(IMAGE_URL % 9999) + + def should_be_in_catalog(self, topic): + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + class KnowsImageDoesNotExist(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + return storage.exists(IMAGE_URL % 10000) + + def should_not_exist(self, topic): + expect(topic.exception()).not_to_be_an_error() + expect(topic.result()).to_be_false() + + class CanRemoveImage(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + storage.put(IMAGE_URL % 10001, IMAGE_BYTES) + storage.remove(IMAGE_URL % 10001) + return storage._get(IMAGE_URL % 10001) + + def should_not_be_in_catalog(self, topic): + expect(topic).not_to_be_an_error() + expect(topic).to_be_null() + + class CanReRemoveImage(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + return storage.remove(IMAGE_URL % 10001) + + def should_not_be_in_catalog(self, topic): + expect(topic).not_to_be_an_error() + expect(topic).to_be_null() + + class CanGetImage(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + + storage.put(IMAGE_URL % 2, IMAGE_BYTES) + return storage.get(IMAGE_URL % 2) + + def should_not_be_null(self, topic): + expect(topic.result()).not_to_be_null() + expect(topic.exception()).not_to_be_an_error() + + def should_have_proper_bytes(self, topic): + expect(topic.result()).to_equal(IMAGE_BYTES) + + class HandleErrors(Vows.Context): + class CanRaiseErrors(Vows.Context): + @Vows.capture_error + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27018, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600, + MONGODB_STORAGE_IGNORE_ERRORS=False + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + + return storage.exists(IMAGE_URL % 2) + + def should_throw_an_exception(self, topic): + expect( + topic + ).to_be_an_error_like(PyMongoError) + + class IgnoreErrors(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27018, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600, + MONGODB_STORAGE_IGNORE_ERRORS=True + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + + return storage + + def should_return_false(self, storage): + result = storage.exists(IMAGE_URL % 2) + expect(result.result()).to_equal(False) + expect(result.exception()).not_to_be_an_error() + + def should_return_none(self, storage): + result = storage.get(IMAGE_URL % 2) + expect(result.result()).to_equal(None) + expect(result.exception()).not_to_be_an_error() + + class CryptoVows(Vows.Context): + class RaisesIfInvalidConfig(Vows.Context): + @Vows.capture_error + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600, + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True + ) + storage = MongoStorage(Context( + config=config, server=get_server('') + )) + storage.put(IMAGE_URL % 3, IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % 3) + + def should_be_an_error(self, topic): + expect(topic).to_be_an_error_like(RuntimeError) + expect(topic).to_have_an_error_message_of( + "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True \ + if no SECURITY_KEY specified" + ) + + class GettingCryptoForANewImageReturnsNone(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600, + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + return storage.get_crypto(IMAGE_URL % 9999) + + def should_be_null(self, topic): + expect(topic.result()).to_be_null() + + class DoesNotStoreIfConfigSaysNotTo(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + storage.put(IMAGE_URL % 5, IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % 5) + return storage.get_crypto(IMAGE_URL % 5) + + def should_be_null(self, topic): + expect(topic.result()).to_be_null() + + class CanStoreCrypto(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600, + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + storage.put(IMAGE_URL % 6, IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % 6) + return storage.get_crypto(IMAGE_URL % 6) + + def should_not_be_null(self, topic): + expect(topic.result()).not_to_be_null() + expect(topic.exception()).not_to_be_an_error() + + def should_have_proper_key(self, topic): + expect(topic.result()).to_equal('ACME-SEC') + + class DetectorVows(Vows.Context): + class CanStoreDetectorData(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + storage.put(IMAGE_URL % 7, IMAGE_BYTES) + storage.put_detector_data(IMAGE_URL % 7, 'some-data') + return storage.get_detector_data(IMAGE_URL % 7) + + def should_not_be_null(self, topic): + expect(topic.result()).not_to_be_null() + expect(topic.exception()).not_to_be_an_error() + + def should_equal_some_data(self, topic): + expect(topic.result()).to_equal('some-data') + + class ReturnsNoneIfNoDetectorData(Vows.Context): + def topic(self): + config = Config( + MONGO_STORAGE_URI="", + MONGO_STORAGE_SERVER_HOST='localhost', + MONGO_STORAGE_SERVER_PORT=27017, + MONGO_STORAGE_SERVER_DB='thumbor', + MONGO_STORAGE_SERVER_COLLECTION='images', + STORAGE_EXPIRATION_SECONDS=3600 + ) + storage = MongoStorage(Context( + config=config, server=get_server('ACME-SEC') + )) + return storage.get_detector_data(IMAGE_URL % 10000) + + def should_not_be_null(self, topic): + expect(topic.result()).to_be_null() +