From 9a30af706e365a2a194062e4d38e48bc0c4869e6 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 11 Sep 2021 18:34:58 +0300 Subject: [PATCH 01/22] Bump version --- CHANGELOG.rst | 5 +++++ environ/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b3f5576..cf3b0003 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.8.0`_ - 00-Unreleased-2021 +------------------------------ + + `v0.7.0`_ - 11-September-2021 ------------------------------ Added @@ -219,6 +223,7 @@ Added - Initial release. +.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...develop .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 diff --git a/environ/__init__.py b/environ/__init__.py index bf8689a0..e37b6e65 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.7.0' +__version__ = '0.8.0' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' From 57cdc98b6d43db3ec908f40b4fcfc1980b97d572 Mon Sep 17 00:00:00 2001 From: John Bergvall Date: Wed, 15 Sep 2021 00:52:10 +0200 Subject: [PATCH 02/22] Keep newline/tab escapes in quoted strings (fix certificate serializing) (#296) Limit backslash escape removal for quoted newlines Keep escape backslash for newline/tab characters. Parsing still differs from shell syntax but keeps string structure intact. Example certificate data VAR="---BEGIN---\r\n---END---" now becomes "---BEGIN---\r\n---END---" instead of "---BEGIN---rn---END---" Co-authored-by: Serghei Iakovlev --- CHANGELOG.rst | 3 +++ docs/tips.rst | 49 +++++++++++++++++++++++++++++++++++++++++----- environ/environ.py | 12 ++++++++++-- tests/fixtures.py | 2 ++ tests/test_env.py | 4 ++++ tests/test_env.txt | 2 ++ 6 files changed, 65 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf3b0003..c067f280 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ and this project adheres to `Semantic Versioning >> print env.str('MULTILINE_TEXT', multiline=True) - Hello - World + # settings.py file contents + import environ + + + env = environ.Env() + + print(env.str('UNQUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('UNQUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('QUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('QUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('ESCAPED_CERT', multiline=True)) + # ---BEGIN---\ + # ---END--- + print(env.str('ESCAPED_CERT', multiline=False)) + # ---BEGIN---\\n---END--- Proxy value =========== diff --git a/environ/environ.py b/environ/environ.py index d312c296..12d0ca1f 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -181,7 +181,7 @@ def str(self, var, default=NOTSET, multiline=False): """ value = self.get_value(var, cast=str, default=default) if multiline: - return value.replace('\\n', '\n') + return re.sub(r'(\\r)?\\n', r'\n', value) return value def unicode(self, var, default=NOTSET): @@ -770,6 +770,13 @@ def read_env(cls, env_file=None, **overrides): logger.debug('Read environment variables from: {}'.format(env_file)) + def _keep_escaped_format_characters(match): + """Keep escaped newline/tabs in quoted strings""" + escaped_char = match.group(1) + if escaped_char in 'rnt': + return '\\' + escaped_char + return escaped_char + for line in content.splitlines(): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: @@ -779,7 +786,8 @@ def read_env(cls, env_file=None, **overrides): val = m2.group(1) m3 = re.match(r'\A"(.*)"\Z', val) if m3: - val = re.sub(r'\\(.)', r'\1', m3.group(1)) + val = re.sub(r'\\(.)', _keep_escaped_format_characters, + m3.group(1)) cls.ENVIRON.setdefault(key, str(val)) # set defaults diff --git a/tests/fixtures.py b/tests/fixtures.py index f685f5cb..29b4dc9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -31,6 +31,8 @@ class FakeEnv: def generate_data(cls): return dict(STR_VAR='bar', MULTILINE_STR_VAR='foo\\nbar', + MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---', + MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---', INT_VAR='42', FLOAT_VAR='33.3', FLOAT_COMMA_VAR='33,3', diff --git a/tests/test_env.py b/tests/test_env.py index efb09fbc..afa6470b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -57,6 +57,10 @@ def test_contains(self): ('STR_VAR', 'bar', False), ('MULTILINE_STR_VAR', 'foo\\nbar', False), ('MULTILINE_STR_VAR', 'foo\nbar', True), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\\r\\n---END---', False), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\n---END---', True), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\\n---END---', False), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\n---END---', True), ], ) def test_str(self, var, val, multiline): diff --git a/tests/test_env.txt b/tests/test_env.txt index dfbd61ef..befeed76 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -33,6 +33,8 @@ INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar MULTILINE_STR_VAR=foo\nbar +MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---" +MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END--- INT_LIST=42,33 CYRILLIC_VAR=фуубар INT_TUPLE=(42,33) From d3e25b9cf8e17cee65771ceaf5686d1157a775b2 Mon Sep 17 00:00:00 2001 From: Eduardo Lima Date: Mon, 4 Jan 2021 16:33:44 -0300 Subject: [PATCH 03/22] Log invalid lines --- environ/environ.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index 12d0ca1f..83642ff5 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -789,6 +789,8 @@ def _keep_escaped_format_characters(match): val = re.sub(r'\\(.)', _keep_escaped_format_characters, m3.group(1)) cls.ENVIRON.setdefault(key, str(val)) + else: + logger.warn('Invalid line: %s', line) # set defaults for key, value in overrides.items(): From e99045386d8744f2428c5f34dda602ee2a3967f0 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 02:17:12 +0300 Subject: [PATCH 04/22] Update change log --- CHANGELOG.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c067f280..28e97cbb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,15 @@ and this project adheres to `Semantic Versioning `_. + Fixed +++++ -- Keep newline/tb escaped in quoted strings +- Keep newline/tab escapes in quoted strings + `#296 `_. `v0.7.0`_ - 11-September-2021 From e2c1412987c4250f9ee4a08f82a7bc91dc14de78 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 02:17:52 +0300 Subject: [PATCH 05/22] Update change log --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 28e97cbb..07cc1dbe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ Added Fixed +++++ - Keep newline/tab escapes in quoted strings - `#296 `_. + `#296 `_. `v0.7.0`_ - 11-September-2021 From cbbd6d6d454352e4ff712947502466a84dd8faef Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 11:00:24 +0300 Subject: [PATCH 06/22] Update change log --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 07cc1dbe..b64dc00c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Added +++++ - Log invalid lines when parse .env file `#283 `_. +- Added docker-style file variable support + `#189 `_. Fixed +++++ From 790106fbe4348c1a19e980f543a373e29a3e8319 Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 28 Sep 2021 03:38:36 +1300 Subject: [PATCH 07/22] Option to override existing variables with read_env (#329) * Option to override existing variables with read_env Fixes #249 * Improve Python 3.5 support * Add another test to bump up coverage since my simplified code dropped the percentage :P --- docs/tips.rst | 20 ++++++++++++++++++-- environ/environ.py | 25 +++++++++++++++++-------- tests/test_env.py | 33 ++++++++++++++++++++++----------- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/docs/tips.rst b/docs/tips.rst index ab3161dd..cff0d862 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -195,10 +195,13 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +Reading env files +================= + .. _multiple-env-files-label: Multiple env files -================== +------------------ There is an ability point to the .env file location using an environment variable. This feature may be convenient in a production systems with a @@ -227,7 +230,7 @@ while ``./manage.py runserver`` uses ``.env``. Using Path objects when reading env -=================================== +----------------------------------- It is possible to use of ``pathlib.Path`` objects when reading environment file from the filesystem: @@ -249,3 +252,16 @@ It is possible to use of ``pathlib.Path`` objects when reading environment file env.read_env(os.path.join(BASE_DIR, '.env')) env.read_env(pathlib.Path(str(BASE_DIR)).joinpath('.env')) env.read_env(pathlib.Path(str(BASE_DIR)) / '.env') + + +Overwriting existing environment values from env files +------------------------------------------------------ + +If you want variables set within your env files to take higher precidence than +an existing set environment variable, use the ``overwrite=True`` argument of +``read_env``. For example: + +.. code-block:: python + + env = environ.Env() + env.read_env(BASE_DIR('.env'), overwrite=True) diff --git a/environ/environ.py b/environ/environ.py index 83642ff5..a38324f4 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -31,10 +31,10 @@ try: from os import PathLike +except ImportError: # Python 3.5 support + from pathlib import PurePath as PathLike - Openable = (str, PathLike) -except ImportError: - Openable = (str,) +Openable = (str, PathLike) logger = logging.getLogger(__name__) @@ -732,13 +732,16 @@ def search_url_config(cls, url, engine=None): return config @classmethod - def read_env(cls, env_file=None, **overrides): + def read_env(cls, env_file=None, overwrite=False, **overrides): """Read a .env file into os.environ. If not given a path to a dotenv path, does filthy magic stack backtracking to find the dotenv in the same directory as the file that called read_env. + By default, won't overwrite any existing environment variables. You can + enable this behaviour by setting ``overwrite=True``. + Refs: - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 @@ -757,7 +760,8 @@ def read_env(cls, env_file=None, **overrides): try: if isinstance(env_file, Openable): - with open(env_file) as f: + # Python 3.5 support (wrap path with str). + with open(str(env_file)) as f: content = f.read() else: with env_file as f: @@ -788,13 +792,18 @@ def _keep_escaped_format_characters(match): if m3: val = re.sub(r'\\(.)', _keep_escaped_format_characters, m3.group(1)) - cls.ENVIRON.setdefault(key, str(val)) + overrides[key] = str(val) else: logger.warn('Invalid line: %s', line) - # set defaults + if overwrite: + def set_environ(key, value): + cls.ENVIRON[key] = value + else: + set_environ = cls.ENVIRON.setdefault + for key, value in overrides.items(): - cls.ENVIRON.setdefault(key, value) + set_environ(key, value) class FileAwareEnv(Env): diff --git a/tests/test_env.py b/tests/test_env.py index afa6470b..acd8dc40 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -152,6 +152,7 @@ def test_dict_value(self): [ ('a=1', dict, {'a': '1'}), ('a=1', dict(value=int), {'a': 1}), + ('a=1', dict(value=float), {'a': 1.0}), ('a=1,2,3', dict(value=[str]), {'a': ['1', '2', '3']}), ('a=1,2,3', dict(value=[int]), {'a': [1, 2, 3]}), ('a=1;b=1.1,2.2;c=3', dict(value=int, cast=dict(b=[float])), @@ -163,6 +164,7 @@ def test_dict_value(self): ids=[ 'dict', 'dict_int', + 'dict_float', 'dict_str_list', 'dict_int_list', 'dict_int_cast', @@ -307,34 +309,43 @@ def setup_method(self, method): PATH_VAR=Path(__file__, is_file=True).__root__ ) - def test_read_env_path_like(self): + def create_temp_env_file(self, name): import pathlib import tempfile - path_like = (pathlib.Path(tempfile.gettempdir()) / 'test_pathlib.env') + env_file_path = (pathlib.Path(tempfile.gettempdir()) / name) try: - path_like.unlink() + env_file_path.unlink() except FileNotFoundError: pass - assert not path_like.exists() + assert not env_file_path.exists() + return env_file_path + + def test_read_env_path_like(self): + env_file_path = self.create_temp_env_file('test_pathlib.env') env_key = 'SECRET' env_val = 'enigma' env_str = env_key + '=' + env_val # open() doesn't take path-like on Python < 3.6 - try: - with open(path_like, 'w', encoding='utf-8') as f: - f.write(env_str + '\n') - except TypeError: - return + with open(str(env_file_path), 'w', encoding='utf-8') as f: + f.write(env_str + '\n') - assert path_like.exists() - self.env.read_env(path_like) + self.env.read_env(env_file_path) assert env_key in self.env.ENVIRON assert self.env.ENVIRON[env_key] == env_val + @pytest.mark.parametrize("overwrite", [True, False]) + def test_existing_overwrite(self, overwrite): + env_file_path = self.create_temp_env_file('test_existing.env') + with open(str(env_file_path), 'w') as f: + f.write("EXISTING=b") + self.env.ENVIRON['EXISTING'] = "a" + self.env.read_env(env_file_path, overwrite=overwrite) + assert self.env.ENVIRON["EXISTING"] == ("b" if overwrite else "a") + class TestSubClass(TestEnv): def setup_method(self, method): From fb6a325e9b746919961b79fa82dc9597d158495a Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:01:55 +0300 Subject: [PATCH 08/22] Refactor a bit read_env to simplify overriding env vars --- environ/environ.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index a38324f4..8e9cef37 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -739,12 +739,22 @@ def read_env(cls, env_file=None, overwrite=False, **overrides): backtracking to find the dotenv in the same directory as the file that called read_env. - By default, won't overwrite any existing environment variables. You can - enable this behaviour by setting ``overwrite=True``. + Existing environment variables take precedent and are NOT overwritten + by the file content. ``overwrite=True`` will force an overwrite of + existing environment variables. Refs: - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 + + :param env_file: The path to the `.env` file your application should + use. If a path is not provided, `read_env` will attempt to import + the Django settings module from the Django project root. + :param overwrite: ``overwrite=True`` will force an overwrite of + existing environment variables. + :param **overrides: Any additional keyword arguments provided directly + to read_env will be added to the environment. If the key matches an + existing environment variable, the value will be overridden. """ if env_file is None: frame = sys._getframe() @@ -796,14 +806,19 @@ def _keep_escaped_format_characters(match): else: logger.warn('Invalid line: %s', line) - if overwrite: - def set_environ(key, value): - cls.ENVIRON[key] = value - else: - set_environ = cls.ENVIRON.setdefault + def set_environ(envval): + """Return lambda to set environ. + + Use setdefault unless overwrite is specified. + """ + if overwrite: + return lambda k,v: envval.update({k: str(v)}) + return lambda k,v: envval.setdefault(k,str(v)) + + setenv = set_environ(cls.ENVIRON) for key, value in overrides.items(): - set_environ(key, value) + setenv(key, value) class FileAwareEnv(Env): From 5e337533d585637554ae42e11acc214701e84430 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:02:21 +0300 Subject: [PATCH 09/22] Replace deprecated logger.warn by logger.warning --- environ/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 8e9cef37..42fbffc8 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -804,7 +804,7 @@ def _keep_escaped_format_characters(match): m3.group(1)) overrides[key] = str(val) else: - logger.warn('Invalid line: %s', line) + logger.warning('Invalid line: %s', line) def set_environ(envval): """Return lambda to set environ. From a8718082754246e32d0611655a4f6f487a34ddb7 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:02:36 +0300 Subject: [PATCH 10/22] Add missed copyright notice --- tests/test_fileaware.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_fileaware.py b/tests/test_fileaware.py index 11e9d8a1..b4733bc2 100644 --- a/tests/test_fileaware.py +++ b/tests/test_fileaware.py @@ -1,3 +1,11 @@ +# This file is part of the django-environ. +# +# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2013-2021, Daniele Faraglia +# +# For the full copyright and license information, please view +# the LICENSE.txt file that was distributed with this source code. + import os import tempfile from contextlib import contextmanager From cc0138b673616a62fd87d5d64242712c82ae7a4a Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:04:58 +0300 Subject: [PATCH 11/22] Update change log --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b64dc00c..47d9673f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ Added `#283 `_. - Added docker-style file variable support `#189 `_. +- Added option to override existing variables with ``read_env`` + `#103 `_, + `#249 `_. + Fixed +++++ From 444c3ca8289676b78d4db84d295c7b773d6ccc88 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:07:14 +0300 Subject: [PATCH 12/22] Fix code style issues --- environ/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 42fbffc8..b678096d 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -812,8 +812,8 @@ def set_environ(envval): Use setdefault unless overwrite is specified. """ if overwrite: - return lambda k,v: envval.update({k: str(v)}) - return lambda k,v: envval.setdefault(k,str(v)) + return lambda k, v: envval.update({k: str(v)}) + return lambda k, v: envval.setdefault(k, str(v)) setenv = set_environ(cls.ENVIRON) From a6d90fb570f59fff65fa5c0a9c72c29e6f39df6b Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Tue, 5 Oct 2021 22:50:47 +0330 Subject: [PATCH 13/22] Add support for empty var with None default value Signed-off-by: Mehdy Khoshnoody --- environ/environ.py | 2 ++ tests/test_env.py | 1 + tests/test_env.txt | 1 + 3 files changed, 4 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index b678096d..7cb20404 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -375,6 +375,8 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): not isinstance(default, NoValue): cast = type(default) + value = None if default is None and value == '' else value + if value != default or (parse_default and value): value = self.parse_value(value, cast) diff --git a/tests/test_env.py b/tests/test_env.py index acd8dc40..55147fbc 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -86,6 +86,7 @@ def test_int(self): def test_int_with_none_default(self): assert self.env('NOT_PRESENT_VAR', cast=int, default=None) is None + assert self.env('EMPTY_INT_VAR', cast=int, default=None) is None @pytest.mark.parametrize( 'value,variable', diff --git a/tests/test_env.txt b/tests/test_env.txt index befeed76..a179a48e 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -29,6 +29,7 @@ FLOAT_STRANGE_VAR2=123.420.333,3 FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR EMPTY_LIST= +EMPTY_INT_VAR= INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar From 26fb010ab301da30158e847c2de159cb73c22dd2 Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Wed, 6 Oct 2021 13:59:01 +0330 Subject: [PATCH 14/22] Add update to CHANGELOG.rst Signed-off-by: Mehdy Khoshnoody --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47d9673f..c46d3ffb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ Added - Added option to override existing variables with ``read_env`` `#103 `_, `#249 `_. +- Added support for empty var with None default value + `#209 `_. Fixed From af4c68db08e2ca28801ead5ab70fcae1181d832e Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Wed, 6 Oct 2021 17:02:40 +0330 Subject: [PATCH 15/22] Handle escaped dollar sign in values Signed-off-by: Mehdy Khoshnoody --- CHANGELOG.rst | 2 ++ environ/environ.py | 5 +++++ tests/fixtures.py | 1 + tests/test_env.py | 8 ++++++++ tests/test_env.txt | 1 + 5 files changed, 17 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47d9673f..b41318e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,8 @@ Fixed +++++ - Keep newline/tab escapes in quoted strings `#296 `_. +- Handle escaped dolloar sign in values + `#271 `_. `v0.7.0`_ - 11-September-2021 diff --git a/environ/environ.py b/environ/environ.py index b678096d..b177784e 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -160,6 +160,7 @@ class Env: def __init__(self, **scheme): self.smart_cast = True + self.escape_proxy = False self.scheme = scheme def __call__(self, var, cast=None, default=NOTSET, parse_default=False): @@ -365,10 +366,14 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' + escape = rb'\$' if isinstance(value, bytes) else r'\$' if hasattr(value, 'startswith') and value.startswith(prefix): value = value.lstrip(prefix) value = self.get_value(value, cast=cast, default=default) + if self.escape_proxy and hasattr(value, 'replace'): + value = value.replace(escape, prefix) + # Smart casting if self.smart_cast: if cast is None and default is not None and \ diff --git a/tests/fixtures.py b/tests/fixtures.py index 29b4dc9f..782123df 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -53,6 +53,7 @@ def generate_data(cls): BOOL_FALSE_STRING_LIKE_BOOL='False', BOOL_FALSE_BOOL=False, PROXIED_VAR='$STR_VAR', + ESCAPED_VAR=r'\$baz', INT_LIST='42,33', INT_TUPLE='(42,33)', STR_LIST_WITH_SPACES=' foo, bar', diff --git a/tests/test_env.py b/tests/test_env.py index acd8dc40..e66cf99d 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -126,6 +126,14 @@ def test_bool_true(self, value, variable): def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' + def test_escaped_dollar_sign(self): + self.env.escape_proxy = True + assert self.env('ESCAPED_VAR') == '$baz' + + def test_escaped_dollar_sign_disabled(self): + self.env.escape_proxy = False + assert self.env('ESCAPED_VAR') == r'\$baz' + def test_int_list(self): assert_type_and_value(list, [42, 33], self.env('INT_LIST', cast=[int])) assert_type_and_value(list, [42, 33], self.env.list('INT_LIST', int)) diff --git a/tests/test_env.txt b/tests/test_env.txt index befeed76..fa593d27 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -28,6 +28,7 @@ FLOAT_STRANGE_VAR1=123,420,333.3 FLOAT_STRANGE_VAR2=123.420.333,3 FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR +ESCAPED_VAR=\$baz EMPTY_LIST= INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar From 1b6c4ba7e1fc1548a5b9692c923a6523a11ef4af Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Thu, 7 Oct 2021 17:12:49 +0330 Subject: [PATCH 16/22] Add docs for escape proxy feature Signed-off-by: Mehdy Khoshnoody --- docs/tips.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/tips.rst b/docs/tips.rst index cff0d862..dbb762fb 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -195,6 +195,23 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +Escape Proxy +============ + +If you're having trouble with values starting with dollar sign ($) without the intention of proxying the value to +another, You should enbale the ``escape_proxy`` and prepend a backslash to it. + +.. code-block:: python + + import environ + + env = environ.Env() + env.escape_proxy = True + + # ESCAPED_VAR=\$baz + env.str('ESCAPED_VAR') # $baz + + Reading env files ================= From f2163279c915b3ff678592bff15373bc13d4881e Mon Sep 17 00:00:00 2001 From: Mirco Grillo Date: Fri, 8 Oct 2021 14:42:40 +0200 Subject: [PATCH 17/22] Fix incorrect parsing when using CloudSQL db url Add test for covering it --- environ/environ.py | 6 +++++- tests/fixtures.py | 2 ++ tests/test_env.py | 3 +++ tests/test_env.txt | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 1ebbb8d7..676d04b4 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -157,6 +157,7 @@ class Env: "xapian": "haystack.backends.xapian_backend.XapianEngine", "simple": "haystack.backends.simple_backend.SimpleEngine", } + CLOUDSQL = 'cloudsql' def __init__(self, **scheme): self.smart_cast = True @@ -502,7 +503,10 @@ def db_url_config(cls, url, engine=None): 'PORT': _cast_int(url.port) or '', }) - if url.scheme in cls.POSTGRES_FAMILY and path.startswith('/'): + if ( + url.scheme in cls.POSTGRES_FAMILY and path.startswith('/') + or cls.CLOUDSQL in path and path.startswith('/') + ): config['HOST'], config['NAME'] = path.rsplit('/', 1) if url.scheme == 'oracle' and path == '': diff --git a/tests/fixtures.py b/tests/fixtures.py index 782123df..25213ac7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -13,6 +13,7 @@ class FakeEnv: URL = 'http://www.google.com/' POSTGRES = 'postgres://uf07k1:wegauwhg@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722' MYSQL = 'mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true' + MYSQL_CLOUDSQL_URL = 'mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase' MYSQLGIS = 'mysqlgis://user:password@127.0.0.1/some_database' SQLITE = 'sqlite:////full/path/to/your/database/file.sqlite' ORACLE_TNS = 'oracle://user:password@sid/' @@ -67,6 +68,7 @@ def generate_data(cls): DATABASE_ORACLE_TNS_URL=cls.ORACLE_TNS, DATABASE_REDSHIFT_URL=cls.REDSHIFT, DATABASE_CUSTOM_BACKEND_URL=cls.CUSTOM_BACKEND, + DATABASE_MYSQL_CLOUDSQL_URL=cls.MYSQL_CLOUDSQL_URL, CACHE_URL=cls.MEMCACHE, CACHE_REDIS=cls.REDIS, EMAIL_URL=cls.EMAIL, diff --git a/tests/test_env.py b/tests/test_env.py index 81aa6545..0be9a60a 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -218,6 +218,8 @@ def test_url_encoded_parts(self): '/full/path/to/your/database/file.sqlite', '', '', '', ''), ('DATABASE_CUSTOM_BACKEND_URL', 'custom.backend', 'database', 'example.com', 'user', 'password', 5430), + ('DATABASE_MYSQL_CLOUDSQL_URL', 'django.db.backends.mysql', 'mydatabase', + '/cloudsql/arvore-codelab:us-central1:mysqlinstance', 'djuser', 'hidden-password', ''), ], ids=[ 'postgres', @@ -228,6 +230,7 @@ def test_url_encoded_parts(self): 'redshift', 'sqlite', 'custom', + 'cloudsql', ], ) def test_db_url_value(self, var, engine, name, host, user, passwd, port): diff --git a/tests/test_env.txt b/tests/test_env.txt index 60ea78c3..c6363ed3 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -1,5 +1,6 @@ DICT_VAR=foo=bar,test=on DATABASE_MYSQL_URL=mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true +DATABASE_MYSQL_CLOUDSQL_URL=mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase DATABASE_MYSQL_GIS_URL=mysqlgis://user:password@127.0.0.1/some_database CACHE_URL=memcache://127.0.0.1:11211 CACHE_REDIS=rediscache://127.0.0.1:6379/1?client_class=django_redis.client.DefaultClient&password=secret From f5ab1f13defd536fe9a80c7e4a777254b12f5f7d Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 9 Oct 2021 10:44:21 +0300 Subject: [PATCH 18/22] Update change log --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50109b18..9e11ac4e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,8 +24,10 @@ Fixed +++++ - Keep newline/tab escapes in quoted strings `#296 `_. -- Handle escaped dolloar sign in values +- Handle escaped dollar sign in values `#271 `_. +- Fixed incorrect parsing of ``DATABASES_URL`` for Google Cloud MySQL + `#294 `_. `v0.7.0`_ - 11-September-2021 From 132e3861ed640610e62f159bca46ecff15892fe1 Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 12 Oct 2021 13:30:16 +1300 Subject: [PATCH 19/22] Add pymemcache cache backend for Django 3.2+ --- docs/types.rst | 6 ++++-- environ/compat.py | 20 ++++++++++++++++---- environ/environ.py | 5 +++-- tests/test_cache.py | 22 ++++++++++++++++++++-- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/types.rst b/docs/types.rst index 1e80e582..292ed64f 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -34,8 +34,10 @@ Supported types * Dummy: ``dummycache://`` * File: ``filecache://`` * Memory: ``locmemcache://`` - * Memcached: ``memcache://`` - * Python memory: ``pymemcache://`` + * Memcached: + * ``memcache://`` (uses ``python-memcached`` backend, deprecated in Django 3.2) + * ``pymemcache://`` (uses ``pymemcache`` backend if Django >=3.2 and package is installed, otherwise will use ``pylibmc`` backend to keep config backwards compatibility) + * ``pylibmc://`` * Redis: ``rediscache://``, ``redis://``, or ``rediss://`` * ``search_url`` diff --git a/environ/compat.py b/environ/compat.py index 0f9eb9a5..9f50f68f 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -8,15 +8,15 @@ """This module handles import compatibility issues.""" -import pkgutil +from pkgutil import find_loader -if pkgutil.find_loader('simplejson'): +if find_loader('simplejson'): import simplejson as json else: import json -if pkgutil.find_loader('django'): +if find_loader('django'): from django import VERSION as DJANGO_VERSION from django.core.exceptions import ImproperlyConfigured else: @@ -33,7 +33,19 @@ class ImproperlyConfigured(Exception): DJANGO_POSTGRES = 'django.db.backends.postgresql' # back compatibility with redis_cache package -if pkgutil.find_loader('redis_cache'): +if find_loader('redis_cache'): REDIS_DRIVER = 'redis_cache.RedisCache' else: REDIS_DRIVER = 'django_redis.cache.RedisCache' + + +# back compatibility for pymemcache +def choose_pymemcache_driver(): + if (DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2)) or not find_loader('pymemcache'): + # The original backend choice for the 'pymemcache' scheme is unfortunately + # 'pylibmc'. + return 'django.core.cache.backends.memcached.PyLibMCCache' + return 'django.core.cache.backends.memcached.PyMemcacheCache' + + +PYMEMCACHE_DRIVER = choose_pymemcache_driver() \ No newline at end of file diff --git a/environ/environ.py b/environ/environ.py index 676d04b4..8b08dc5a 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -26,7 +26,7 @@ urlunparse, ) -from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, REDIS_DRIVER +from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, PYMEMCACHE_DRIVER, REDIS_DRIVER from .fileaware_mapping import FileAwareMapping try: @@ -116,7 +116,8 @@ class Env: 'filecache': 'django.core.cache.backends.filebased.FileBasedCache', 'locmemcache': 'django.core.cache.backends.locmem.LocMemCache', 'memcache': 'django.core.cache.backends.memcached.MemcachedCache', - 'pymemcache': 'django.core.cache.backends.memcached.PyLibMCCache', + 'pymemcache': PYMEMCACHE_DRIVER, + 'pylibmc': 'django.core.cache.backends.memcached.PyLibMCCache', 'rediscache': REDIS_DRIVER, 'redis': REDIS_DRIVER, 'rediss': REDIS_DRIVER, diff --git a/tests/test_cache.py b/tests/test_cache.py index 42c72bbc..53f42110 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -6,10 +6,13 @@ # For the full copyright and license information, please view # the LICENSE.txt file that was distributed with this source code. +from unittest import mock + import pytest +import environ.compat from environ import Env -from environ.compat import REDIS_DRIVER, ImproperlyConfigured +from environ.compat import PYMEMCACHE_DRIVER, REDIS_DRIVER, ImproperlyConfigured def test_base_options_parsing(): @@ -63,7 +66,7 @@ def test_base_options_parsing(): 'django.core.cache.backends.memcached.MemcachedCache', '127.0.0.1:11211'), ('pymemcache://127.0.0.1:11211', - 'django.core.cache.backends.memcached.PyLibMCCache', + PYMEMCACHE_DRIVER, '127.0.0.1:11211'), ], ids=[ @@ -90,6 +93,21 @@ def test_cache_parsing(url, backend, location): assert url['LOCATION'] == location +@pytest.mark.parametrize('django_version', ((3, 2), (3, 1), None)) +@pytest.mark.parametrize('pymemcache_installed', (True, False)) +def test_pymemcache_compat(django_version, pymemcache_installed): + old = 'django.core.cache.backends.memcached.PyLibMCCache' + new = 'django.core.cache.backends.memcached.PyMemcacheCache' + with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): + with mock.patch('environ.compat.find_loader') as mock_find_loader: + mock_find_loader.return_value = pymemcache_installed + driver = environ.compat.choose_pymemcache_driver() + if django_version and django_version < (3, 2): + assert driver == old + else: + assert driver == new if pymemcache_installed else old + + def test_redis_parsing(): url = ('rediscache://127.0.0.1:6379/1?client_class=' 'django_redis.client.DefaultClient&password=secret') From 1c62ff53c91bf610589ba2a8e7d79a889898fc8f Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 12 Oct 2021 13:39:01 +1300 Subject: [PATCH 20/22] Lint fixes --- environ/compat.py | 9 +++++---- environ/environ.py | 8 +++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/environ/compat.py b/environ/compat.py index 9f50f68f..8c259f85 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -41,11 +41,12 @@ class ImproperlyConfigured(Exception): # back compatibility for pymemcache def choose_pymemcache_driver(): - if (DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2)) or not find_loader('pymemcache'): - # The original backend choice for the 'pymemcache' scheme is unfortunately - # 'pylibmc'. + old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2) + if old_django or not find_loader('pymemcache'): + # The original backend choice for the 'pymemcache' scheme is + # unfortunately 'pylibmc'. return 'django.core.cache.backends.memcached.PyLibMCCache' return 'django.core.cache.backends.memcached.PyMemcacheCache' -PYMEMCACHE_DRIVER = choose_pymemcache_driver() \ No newline at end of file +PYMEMCACHE_DRIVER = choose_pymemcache_driver() diff --git a/environ/environ.py b/environ/environ.py index 8b08dc5a..505577fc 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -26,7 +26,13 @@ urlunparse, ) -from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, PYMEMCACHE_DRIVER, REDIS_DRIVER +from .compat import ( + DJANGO_POSTGRES, + ImproperlyConfigured, + json, + PYMEMCACHE_DRIVER, + REDIS_DRIVER, +) from .fileaware_mapping import FileAwareMapping try: From 2aa4335972286b136eabb286400c168dd9d9d485 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Thu, 14 Oct 2021 01:38:41 +0300 Subject: [PATCH 21/22] Update change log --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e11ac4e..6f615e45 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,8 @@ Added `#249 `_. - Added support for empty var with None default value `#209 `_. +- Added ``pymemcache`` cache backend for Django 3.2+ + `#335 `_. Fixed From 813d1038f0ba6b8e2734490fc5d4200d33b2bf30 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 17 Oct 2021 15:22:21 +0300 Subject: [PATCH 22/22] Update change log --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6f615e45..0637b00b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.8.0`_ - 00-Unreleased-2021 +`v0.8.0`_ - 17-October-2021 ------------------------------ Added +++++ @@ -246,7 +246,7 @@ Added - Initial release. -.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...develop +.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0