From 6b06a539295bee66fb89c406b0c0b424382ca6e6 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 11:46:45 -0400 Subject: [PATCH 01/16] ARXIVNG-2462 cleaning up Dockerfiles for performance, size --- .dockerignore | 137 +++++++++++++++++++++++++++++++++++++++ Dockerfile | 29 +++++---- fourohfour/Dockerfile | 16 ++--- fourohfour/entrypoint.sh | 4 -- fourohfour/uwsgi.ini | 10 +-- fourohfour/wsgi.py | 9 ++- 6 files changed, 170 insertions(+), 35 deletions(-) create mode 100644 .dockerignore delete mode 100755 fourohfour/entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7384e274 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,137 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +requirements.txt +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# sphinx +docs/source/_build + +# mypy +.mypy_cache/ + +.coverage +.DS_Store +zero.db + +# Editor files & misc +*~ +*.idea +*.vscode +default.nix +*.db +*#* +*.#* + +*.pyc +build/ +dist/ +*.egg-info/ +*.DS_Store +.sass-cache/ + +*tests/* +.* +README.md +update-docs.sh +LICENSE +.pylintrc +.coverage +.coveragerc +.gitignore +docs/* +four \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2e461eee..7809d657 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,17 @@ FROM centos:centos7 -# +WORKDIR /opt/arxiv + # Below we use && chaining and an embedded script in a single RUN # command to keep image size and layer count to a minimum, while # the embedded script will make 'docker build' fail fast # if a package is missing. # -RUN yum -y update && yum -y install epel-release \ -&& yum -y install https://centos7.iuscommunity.org/ius-release.rpm \ -&& yum -y update --security \ -&& echo $'#!/bin/bash\n\ +RUN yum -y install epel-release \ + && yum -y install https://centos7.iuscommunity.org/ius-release.rpm \ + && yum -y update \ + && echo $'#!/bin/bash\n\ PKGS_TO_INSTALL=$(cat <<-END\n\ ca-certificates\n\ gcc\n\ @@ -21,6 +22,7 @@ PKGS_TO_INSTALL=$(cat <<-END\n\ python36u-devel\n\ which\n\ wget\n\ + mariadb-devel\n\ END\n\ )\n\ for pkg in ${PKGS_TO_INSTALL}; do\n\ @@ -31,16 +33,17 @@ for pkg in ${PKGS_TO_INSTALL}; do\n\ }\n\ done\n\ yum -y install ${PKGS_TO_INSTALL}\n' >> /tmp/safe_yum.sh \ -&& /bin/bash /tmp/safe_yum.sh \ -&& yum clean all + && /bin/bash /tmp/safe_yum.sh \ + && yum clean all \ + && rm /tmp/safe_yum.sh RUN wget https://bootstrap.pypa.io/get-pip.py \ - && python3.6 get-pip.py + && python3.6 get-pip.py \ + && pip install -U pip pipenv uwsgi \ + && rm -rf ~/.cache/pip -# -# This is needed by click: http://click.pocoo.org/5/python3/ -# -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + APPLICATION_ROOT="/" CMD /bin/bash diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index de179097..216eb6a7 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -1,21 +1,13 @@ # arxiv/fourohfour -FROM arxiv/base:0.14.2 +FROM arxiv/base:ARXIVNG-2462 -WORKDIR /opt/arxiv - -RUN yum install -y which mysql mysql-devel - -ENV LOGLEVEL 10 -ENV LC_ALL en_US.utf-8 -ENV LANG en_US.utf-8 - -RUN pip install -U pip pipenv uwsgi -RUN pipenv install arxiv-base arxiv-auth +RUN pipenv install arxiv-base arxiv-auth \ + && rm -rf ~/.cache/pip EXPOSE 8000 -ADD wsgi.py uwsgi.ini entrypoint.sh /opt/arxiv/ +COPY wsgi.py uwsgi.ini /opt/arxiv/ ENTRYPOINT ["pipenv", "run"] CMD ["uwsgi", "--ini", "/opt/arxiv/uwsgi.ini"] diff --git a/fourohfour/entrypoint.sh b/fourohfour/entrypoint.sh deleted file mode 100755 index b213a702..00000000 --- a/fourohfour/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -/usr/bin/uwsgi -H $(pipenv --venv) "$@" diff --git a/fourohfour/uwsgi.ini b/fourohfour/uwsgi.ini index 1950600c..4f8bc3b7 100644 --- a/fourohfour/uwsgi.ini +++ b/fourohfour/uwsgi.ini @@ -6,8 +6,10 @@ callable = application master = true harakiri = 3000 manage-script-name = true -processes = 1 -queue = 0 -threads = 1 -mount = /=wsgi.py +processes = 8 +vacuum = true +single-interpreter = true +mount = $(APPLICATION_ROOT)=wsgi.py logformat = "%(addr) %(addr) - %(user_id)|%(session_id) [%(rtime)] [%(uagent)] \"%(method) %(uri) %(proto)\" %(status) %(size) %(micros) %(ttfb)" +buffer-size = 65535 +wsgi-disable-file-wrapper = true diff --git a/fourohfour/wsgi.py b/fourohfour/wsgi.py index 7c0084ee..b7224625 100644 --- a/fourohfour/wsgi.py +++ b/fourohfour/wsgi.py @@ -21,9 +21,14 @@ def healthz(): return app +__flask_app__ = create_web_app() + + def application(environ, start_response): """WSGI application factory.""" for key, value in environ.items(): + if key == 'SERVER_NAME': # This will only confuse Flask. + continue os.environ[key] = str(value) - app = create_web_app() - return app(environ, start_response) + __flask_app__.config[key] = value + return __flask_app__(environ, start_response) From dce0639f7eeaf3c091474f3d603d1d69455b780c Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 12:15:27 -0400 Subject: [PATCH 02/16] ARXIVNG-2462 parameterized base version in fourohfour image, added labels --- Dockerfile | 10 ++++++++++ fourohfour/Dockerfile | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7809d657..1176a4e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,4 +46,14 @@ ENV LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ APPLICATION_ROOT="/" +LABEL version="0.15.9" \ + maintainer="arXiv " \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="arXiv Base" \ + org.label-schema.description="Base image for arXiv NG applications" \ + org.label-schema.url="https://arxiv.github.io/arxiv-base" \ + org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ + org.label-schema.vendor="arXiv.org" \ + org.label-schema.version="0.15.9" + CMD /bin/bash diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index 216eb6a7..49139cbf 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -1,6 +1,8 @@ # arxiv/fourohfour -FROM arxiv/base:ARXIVNG-2462 +ARG BASE_VERSION=latest + +FROM arxiv/base:${BASE_VERSION} RUN pipenv install arxiv-base arxiv-auth \ && rm -rf ~/.cache/pip @@ -9,5 +11,15 @@ EXPOSE 8000 COPY wsgi.py uwsgi.ini /opt/arxiv/ +LABEL version="0.15.9" \ + maintainer="arXiv " \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="arXiv FourOhFour" \ + org.label-schema.description="Default error backend for arXiv k8s clusters" \ + org.label-schema.url="https://arxiv.github.io/arxiv-base" \ + org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ + org.label-schema.vendor="arXiv.org" \ + org.label-schema.version="0.15.9" + ENTRYPOINT ["pipenv", "run"] CMD ["uwsgi", "--ini", "/opt/arxiv/uwsgi.ini"] From 3701e20d9ed4db4ac02b8e2a46fe17299875468e Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 12:17:48 -0400 Subject: [PATCH 03/16] ARXIVNG-2462 removed workdir from base image --- Dockerfile | 2 -- fourohfour/Dockerfile | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1176a4e5..ad62ee71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,6 @@ FROM centos:centos7 -WORKDIR /opt/arxiv - # Below we use && chaining and an embedded script in a single RUN # command to keep image size and layer count to a minimum, while # the embedded script will make 'docker build' fail fast diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index 49139cbf..49999c34 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -9,6 +9,8 @@ RUN pipenv install arxiv-base arxiv-auth \ EXPOSE 8000 +WORKDIR /opt/arxiv + COPY wsgi.py uwsgi.ini /opt/arxiv/ LABEL version="0.15.9" \ From 87eff3201e17e856a04a321fa25d223eb0128c9f Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 12:34:01 -0400 Subject: [PATCH 04/16] ARXIVNG-2462 moved labels --- Dockerfile | 19 +++++++++---------- fourohfour/Dockerfile | 17 ++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index ad62ee71..b349f9f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,14 @@ FROM centos:centos7 +LABEL maintainer="arXiv " \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="arXiv Base" \ + org.label-schema.description="Base image for arXiv NG applications" \ + org.label-schema.url="https://arxiv.github.io/arxiv-base" \ + org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ + org.label-schema.vendor="arXiv.org" + # Below we use && chaining and an embedded script in a single RUN # command to keep image size and layer count to a minimum, while # the embedded script will make 'docker build' fail fast @@ -9,7 +17,6 @@ FROM centos:centos7 # RUN yum -y install epel-release \ && yum -y install https://centos7.iuscommunity.org/ius-release.rpm \ - && yum -y update \ && echo $'#!/bin/bash\n\ PKGS_TO_INSTALL=$(cat <<-END\n\ ca-certificates\n\ @@ -44,14 +51,6 @@ ENV LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ APPLICATION_ROOT="/" -LABEL version="0.15.9" \ - maintainer="arXiv " \ - org.label-schema.schema-version="1.0" \ - org.label-schema.name="arXiv Base" \ - org.label-schema.description="Base image for arXiv NG applications" \ - org.label-schema.url="https://arxiv.github.io/arxiv-base" \ - org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ - org.label-schema.vendor="arXiv.org" \ - org.label-schema.version="0.15.9" +LABEL version="0.15.9" CMD /bin/bash diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index 49999c34..dd14cda9 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -4,24 +4,19 @@ ARG BASE_VERSION=latest FROM arxiv/base:${BASE_VERSION} +LABEL org.label-schema.name="arXiv FourOhFour" \ + org.label-schema.description="Default error backend for arXiv k8s clusters" + +WORKDIR /opt/arxiv + RUN pipenv install arxiv-base arxiv-auth \ && rm -rf ~/.cache/pip EXPOSE 8000 -WORKDIR /opt/arxiv - COPY wsgi.py uwsgi.ini /opt/arxiv/ -LABEL version="0.15.9" \ - maintainer="arXiv " \ - org.label-schema.schema-version="1.0" \ - org.label-schema.name="arXiv FourOhFour" \ - org.label-schema.description="Default error backend for arXiv k8s clusters" \ - org.label-schema.url="https://arxiv.github.io/arxiv-base" \ - org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ - org.label-schema.vendor="arXiv.org" \ - org.label-schema.version="0.15.9" +LABEL org.label-schema.version="0.15.9" ENTRYPOINT ["pipenv", "run"] CMD ["uwsgi", "--ini", "/opt/arxiv/uwsgi.ini"] From 2b753db7d9fab06b9de0a8a9b893ca8d2935e880 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 13:01:30 -0400 Subject: [PATCH 05/16] ARXIVNG-2186 csrf form can use either request.session or request.auth for authn context --- arxiv/forms/csrf.py | 9 +++-- arxiv/forms/tests.py | 80 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/arxiv/forms/csrf.py b/arxiv/forms/csrf.py index 3266f9ce..b205c090 100644 --- a/arxiv/forms/csrf.py +++ b/arxiv/forms/csrf.py @@ -151,15 +151,18 @@ def csrf_secret(self) -> str: @property def csrf_context(self) -> Dict[str, str]: """Session information used to generate a CSRF token.""" - if not request or not request.session: + if not request or (not request.session and not request.auth): raise RuntimeError('Missing active user session') + # Per ARXIVNG-1944 in arxiv-auth v0.4.1 the session will be called + # request.auth by default. + session = request.auth or request.session + # Sessions provided by arxiv.auth should have a nonce that was # generated when the session was created. Legacy sessions, however, # do not support this. So we'll fall back to using the session ID # instead. - nonce = getattr(request.session, 'nonce', - request.session.session_id) + nonce = getattr(session, 'nonce', session.session_id) return { 'ip_address': request.remote_addr, 'nonce': nonce diff --git a/arxiv/forms/tests.py b/arxiv/forms/tests.py index 7870023e..2fbb56c1 100644 --- a/arxiv/forms/tests.py +++ b/arxiv/forms/tests.py @@ -15,6 +15,7 @@ class TestCSRFForm(TestCase): def test_invalid_token(self, mock_get_config, mock_request): """An invalid CSRF token is passed in the form data.""" mock_request.remote_addr = '10.10.10.10' + mock_request.auth = None mock_request.session = mock.MagicMock(nonce='foononce123') mock_get_config.return_value = {'CSRF_SECRET': 'foosecret'} @@ -35,6 +36,7 @@ def test_valid_token(self, mock_get_config, mock_request): nonce = 'foononce123' secret = 'foosecret' mock_request.remote_addr = ip_address + mock_request.auth = None mock_request.session = mock.MagicMock(nonce=nonce) mock_get_config.return_value = {'CSRF_SECRET': secret} @@ -66,6 +68,7 @@ def test_expired_token(self, mock_get_config, mock_request): csrf_token = csrf.SessionCSRF._join(digest, expires) mock_request.remote_addr = ip_address + mock_request.auth = None mock_request.session = mock.MagicMock(nonce=nonce) mock_get_config.return_value = {'CSRF_SECRET': secret} @@ -78,3 +81,80 @@ class ProtectedForm(csrf.CSRFForm): 'csrf_token': csrf_token}) form = ProtectedForm(data) self.assertFalse(form.validate(), "The form is not valid") + + + +class TestCSRFFormWithNewSessionRef(TestCase): + """Test using the ``request.auth`` session ref in accounts v0.4.1.""" + + @mock.patch(f'{csrf.__name__}.request') + @mock.patch(f'{csrf.__name__}.get_application_config') + def test_invalid_token(self, mock_get_config, mock_request): + """An invalid CSRF token is passed in the form data.""" + mock_request.remote_addr = '10.10.10.10' + mock_request.session = None + mock_request.auth = mock.MagicMock(nonce='foononce123') + mock_get_config.return_value = {'CSRF_SECRET': 'foosecret'} + + class ProtectedForm(csrf.CSRFForm): + """A CSRF-protected form.""" + + something_sensitive = StringField('Something sensitive') + + data = MultiDict({'something_sensitive': 'foo', 'csrf_token': 'nope'}) + form = ProtectedForm(data) + self.assertFalse(form.validate(), "The form is not valid") + + @mock.patch(f'{csrf.__name__}.request') + @mock.patch(f'{csrf.__name__}.get_application_config') + def test_valid_token(self, mock_get_config, mock_request): + """A valid CSRF token is passed in the form data.""" + ip_address = '10.10.10.10' + nonce = 'foononce123' + secret = 'foosecret' + mock_request.remote_addr = ip_address + mock_request.session = None + mock_request.auth = mock.MagicMock(nonce=nonce) + mock_get_config.return_value = {'CSRF_SECRET': secret} + + class ProtectedForm(csrf.CSRFForm): + """A CSRF-protected form.""" + + something_sensitive = StringField('Something sensitive') + + form = ProtectedForm() + + expires = csrf.SessionCSRF._new_expiry(form.meta.csrf_timeout) + digest = csrf.SessionCSRF._hash(secret, nonce, ip_address, expires) + csrf_token = csrf.SessionCSRF._join(digest, expires) + + data = MultiDict({'something_sensitive': 'foo', + 'csrf_token': csrf_token}) + form = ProtectedForm(data) + self.assertTrue(form.validate(), "The form is valid") + + @mock.patch(f'{csrf.__name__}.request') + @mock.patch(f'{csrf.__name__}.get_application_config') + def test_expired_token(self, mock_get_config, mock_request): + """A valid but expired CSRF token is passed in the form data.""" + ip_address = '10.10.10.10' + nonce = 'foononce123' + secret = 'foosecret' + expires = datetime.now().isoformat() + digest = csrf.SessionCSRF._hash(secret, nonce, ip_address, expires) + csrf_token = csrf.SessionCSRF._join(digest, expires) + + mock_request.remote_addr = ip_address + mock_request.session = None + mock_request.auth = mock.MagicMock(nonce=nonce) + mock_get_config.return_value = {'CSRF_SECRET': secret} + + class ProtectedForm(csrf.CSRFForm): + """A CSRF-protected form.""" + + something_sensitive = StringField('Something sensitive') + + data = MultiDict({'something_sensitive': 'foo', + 'csrf_token': csrf_token}) + form = ProtectedForm(data) + self.assertFalse(form.validate(), "The form is not valid") From 67927e799ced2119a20bc08bc6af4739e2558903 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 14:58:48 -0400 Subject: [PATCH 06/16] ARXIVNG-2462 moved env line up, switched to OCI labels --- Dockerfile | 22 ++++++++++++---------- fourohfour/Dockerfile | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index b349f9f2..e8fa72cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,18 @@ FROM centos:centos7 LABEL maintainer="arXiv " \ - org.label-schema.schema-version="1.0" \ - org.label-schema.name="arXiv Base" \ - org.label-schema.description="Base image for arXiv NG applications" \ - org.label-schema.url="https://arxiv.github.io/arxiv-base" \ - org.label-schema.vcs-url="https://github.com/arxiv/arxiv-base" \ - org.label-schema.vendor="arXiv.org" + org.opencontainers.image.authors="arXiv IT Team " \ + org.opencontainers.image.version="1.0" \ + org.opencontainers.image.title="arXiv Base" \ + org.opencontainers.image.description="Base image for arXiv NG applications" \ + org.opencontainers.image.url="https://arxiv.github.io/arxiv-base" \ + org.opencontainers.image.source="https://github.com/arxiv/arxiv-base" \ + org.opencontainers.image.vendor="arXiv.org" \ + org.opencontainers.image.licenses="MIT" + +ENV LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + APPLICATION_ROOT="/" # Below we use && chaining and an embedded script in a single RUN # command to keep image size and layer count to a minimum, while @@ -47,10 +53,6 @@ RUN wget https://bootstrap.pypa.io/get-pip.py \ && pip install -U pip pipenv uwsgi \ && rm -rf ~/.cache/pip -ENV LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - APPLICATION_ROOT="/" - LABEL version="0.15.9" CMD /bin/bash diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index dd14cda9..b3bf233d 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -4,8 +4,8 @@ ARG BASE_VERSION=latest FROM arxiv/base:${BASE_VERSION} -LABEL org.label-schema.name="arXiv FourOhFour" \ - org.label-schema.description="Default error backend for arXiv k8s clusters" +LABEL org.opencontainers.image.title="arXiv FourOhFour" \ + org.opencontainers.image.description="Default error backend for arXiv k8s clusters" WORKDIR /opt/arxiv From 3d21f7ae824812ad52a68c9f029becc707075528 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 15:15:45 -0400 Subject: [PATCH 07/16] Removed unused dockerfile-with-git --- Dockerfile-with-git | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Dockerfile-with-git diff --git a/Dockerfile-with-git b/Dockerfile-with-git deleted file mode 100644 index d56f75d5..00000000 --- a/Dockerfile-with-git +++ /dev/null @@ -1,14 +0,0 @@ -FROM arxiv/base:0.14.3 - -WORKDIR /opt/arxiv/ - -# Install MySQL. -RUN yum install -y which mysql mysql-devel -RUN yum install autoconf libcurl-devel expat-devel gcc gettext-devel kernel-headers openssl-devel perl-devel zlib-devel make -y -RUN curl -O -L https://github.com/git/git/archive/v2.20.1.tar.gz -RUN tar -zxvf v2.20.1.tar.gz -RUN ls -la /opt/arxiv - -WORKDIR /opt/arxiv/git-2.20.1 -RUN ls -la /opt/arxiv/git-2.20.1 -RUN make configure && ./configure --prefix=/usr && make all && make install From f491c19dabfdc1a07502da488096e7c325f103dc Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 15:27:54 -0400 Subject: [PATCH 08/16] pr #139 moved LABEL for version up above dependencies --- Dockerfile | 4 ++-- fourohfour/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8fa72cd..954c63e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ LABEL maintainer="arXiv " \ org.opencontainers.image.vendor="arXiv.org" \ org.opencontainers.image.licenses="MIT" +LABEL version="0.15.9" + ENV LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ APPLICATION_ROOT="/" @@ -53,6 +55,4 @@ RUN wget https://bootstrap.pypa.io/get-pip.py \ && pip install -U pip pipenv uwsgi \ && rm -rf ~/.cache/pip -LABEL version="0.15.9" - CMD /bin/bash diff --git a/fourohfour/Dockerfile b/fourohfour/Dockerfile index b3bf233d..3c317877 100644 --- a/fourohfour/Dockerfile +++ b/fourohfour/Dockerfile @@ -9,6 +9,8 @@ LABEL org.opencontainers.image.title="arXiv FourOhFour" \ WORKDIR /opt/arxiv +LABEL org.label-schema.version="0.15.9" + RUN pipenv install arxiv-base arxiv-auth \ && rm -rf ~/.cache/pip @@ -16,7 +18,5 @@ EXPOSE 8000 COPY wsgi.py uwsgi.ini /opt/arxiv/ -LABEL org.label-schema.version="0.15.9" - ENTRYPOINT ["pipenv", "run"] CMD ["uwsgi", "--ini", "/opt/arxiv/uwsgi.ini"] From d1640d8d3a95256fe30393ba731e8d2265686ed2 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Mon, 15 Jul 2019 17:10:35 -0400 Subject: [PATCH 09/16] ARXIVNG-2585 implement nginx error mirroring --- arxiv/base/exceptions.py | 78 +++------------------ arxiv/base/routes.py | 54 ++++---------- arxiv/base/templates/base/400.html | 15 ---- arxiv/base/templates/base/401.html | 15 ---- arxiv/base/templates/base/403.html | 15 ---- arxiv/base/templates/base/404.html | 15 ---- arxiv/base/templates/base/405.html | 15 ---- arxiv/base/templates/base/413.html | 15 ---- arxiv/base/templates/base/500.html | 15 ---- arxiv/base/templates/base/error_macros.html | 23 ++++++ arxiv/base/templates/base/exception.html | 14 ++-- fourohfour/Pipfile | 2 + fourohfour/Pipfile.lock | 76 ++++++++++++++++++++ fourohfour/tests.py | 42 ++++++----- fourohfour/wsgi.py | 68 +++++++++++++++--- 15 files changed, 209 insertions(+), 253 deletions(-) delete mode 100644 arxiv/base/templates/base/400.html delete mode 100644 arxiv/base/templates/base/401.html delete mode 100644 arxiv/base/templates/base/403.html delete mode 100644 arxiv/base/templates/base/404.html delete mode 100644 arxiv/base/templates/base/405.html delete mode 100644 arxiv/base/templates/base/413.html delete mode 100644 arxiv/base/templates/base/500.html create mode 100644 arxiv/base/templates/base/error_macros.html create mode 100644 fourohfour/Pipfile.lock diff --git a/arxiv/base/exceptions.py b/arxiv/base/exceptions.py index 7f7ebf0f..49ca87ce 100644 --- a/arxiv/base/exceptions.py +++ b/arxiv/base/exceptions.py @@ -19,8 +19,8 @@ from werkzeug.exceptions import NotFound, Forbidden, Unauthorized, \ MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError, \ - HTTPException -from flask import render_template, make_response, Response + HTTPException, default_exceptions +from flask import render_template, make_response, Response, request from http import HTTPStatus as status @@ -53,71 +53,15 @@ def get_handlers() -> List[Tuple[type, Callable]]: return _handlers -@handler(NotFound) -def handle_not_found(error: NotFound) -> Response: - """Render the base 404 error page.""" - rendered = render_template("base/404.html", error=error, - pagetitle="404 Not Found") - response: Response = make_response(rendered) - response.status_code = status.NOT_FOUND +def handle_exception(error: HTTPException) -> Response: + """Render a generic error handler.""" + rendered = render_template("base/exception.html", error=error, + pagetitle=f"{error.code} {error.name }") + response: Response = make_response(rendered, error.code) return response -@handler(Forbidden) -def handle_forbidden(error: Forbidden) -> Response: - """Render the base 403 error page.""" - rendered = render_template("base/403.html", error=error, - pagetitle="403 Forbidden") - response: Response = make_response(rendered) - response.status_code = status.FORBIDDEN - return response - - -@handler(Unauthorized) -def handle_unauthorized(error: Unauthorized) -> Response: - """Render the base 401 error page.""" - rendered = render_template("base/401.html", error=error, - pagetitle="401 Unauthorized") - response: Response = make_response(rendered) - response.status_code = status.UNAUTHORIZED - return response - - -@handler(MethodNotAllowed) -def handle_method_not_allowed(error: MethodNotAllowed) -> Response: - """Render the base 405 error page.""" - rendered = render_template("base/405.html", error=error, - pagetitle="405 Method Not Allowed") - response: Response = make_response(rendered) - response.status_code = status.METHOD_NOT_ALLOWED - return response - - -@handler(RequestEntityTooLarge) -def handle_request_entity_too_large(error: RequestEntityTooLarge) -> Response: - """Render the base 413 error page.""" - rendered = render_template("base/413.html", error=error, - pagetitle="413 Request Entity Too Large") - response: Response = make_response(rendered) - response.status_code = status.REQUEST_ENTITY_TOO_LARGE - return response - - -@handler(BadRequest) -def handle_bad_request(error: BadRequest) -> Response: - """Render the base 400 error page.""" - rendered = render_template("base/400.html", error=error, - pagetitle="400 Bad Request") - response: Response = make_response(rendered) - response.status_code = status.BAD_REQUEST - return response - - -@handler(InternalServerError) -def handle_internal_server_error(error: InternalServerError) -> Response: - """Render the base 500 error page.""" - rendered = render_template("base/500.html", error=error, - pagetitle="500 Internal Server Error") - response: Response = make_response(rendered) - response.status_code = status.INTERNAL_SERVER_ERROR - return response +# Generate handlers programmatically from the built-in Werkzeug HTTP +# exceptions. +for code, exception in default_exceptions.items(): + handler(exception)(handle_exception) \ No newline at end of file diff --git a/arxiv/base/routes.py b/arxiv/base/routes.py index ecd9fe22..95300b7c 100644 --- a/arxiv/base/routes.py +++ b/arxiv/base/routes.py @@ -5,14 +5,15 @@ it is not attached by :class:`arxiv.base.Base`. """ -from typing import Any, Tuple, Callable, Dict +from typing import Any, Tuple, Callable, Dict, Type from datetime import datetime from flask import Blueprint, render_template, current_app, make_response, \ Response, flash, url_for from http import HTTPStatus as status from arxiv.base.exceptions import NotFound, Forbidden, Unauthorized, \ - MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError + MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError, \ + default_exceptions from . import alerts @@ -75,43 +76,16 @@ def test_macros() -> Response: return response -@blueprint.route('/404', methods=['GET']) -def test_404() -> Response: - """Test the 404 error page.""" - raise NotFound() +def make_route(exception: Type) -> Callable[[], Response]: + """Create a route that generates a Werkzeug HTTP exception.""" + def _route() -> Response: + raise exception() + return _route -@blueprint.route('/403', methods=['GET']) -def test_403() -> Response: - """Test the 403 error page.""" - raise Forbidden() - - -@blueprint.route('/401', methods=['GET']) -def test_401() -> Response: - """Test the 401 error page.""" - raise Unauthorized() - - -@blueprint.route('/405', methods=['GET']) -def test_405() -> Response: - """Test the 405 error page.""" - raise MethodNotAllowed() - - -@blueprint.route('/413', methods=['GET']) -def test_413() -> Response: - """Test the 413 error page.""" - raise RequestEntityTooLarge() - - -@blueprint.route('/400', methods=['GET']) -def test_400() -> Response: - """Test the 400 error page.""" - raise BadRequest() - - -@blueprint.route('/500', methods=['GET']) -def test_500() -> Response: - """Test the 500 error page.""" - raise InternalServerError() +# Programatically generate exception-generating routes to test custom error +# pages. +for code, exception in default_exceptions.items(): + register = blueprint.route(f'/{code}', methods=['GET'], + endpoint=f'test_{code}') + register(make_route(exception)) diff --git a/arxiv/base/templates/base/400.html b/arxiv/base/templates/base/400.html deleted file mode 100644 index 09efe8cd..00000000 --- a/arxiv/base/templates/base/400.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Bad Request{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 0 -
-{% endblock %} diff --git a/arxiv/base/templates/base/401.html b/arxiv/base/templates/base/401.html deleted file mode 100644 index 98f14b31..00000000 --- a/arxiv/base/templates/base/401.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Unauthorized{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 1 -
-{% endblock %} diff --git a/arxiv/base/templates/base/403.html b/arxiv/base/templates/base/403.html deleted file mode 100644 index 88ae8b47..00000000 --- a/arxiv/base/templates/base/403.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Not Authorized{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 3 -
-{% endblock %} diff --git a/arxiv/base/templates/base/404.html b/arxiv/base/templates/base/404.html deleted file mode 100644 index 2c5c77bf..00000000 --- a/arxiv/base/templates/base/404.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Page Not Found{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 4 -
-{% endblock %} diff --git a/arxiv/base/templates/base/405.html b/arxiv/base/templates/base/405.html deleted file mode 100644 index 0f9b134f..00000000 --- a/arxiv/base/templates/base/405.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Method Not Allowed{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 5 -
-{% endblock %} diff --git a/arxiv/base/templates/base/413.html b/arxiv/base/templates/base/413.html deleted file mode 100644 index ffe95a4c..00000000 --- a/arxiv/base/templates/base/413.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Request Entity Too Large{% endblock %} - -{% block error_illustration %} -
- 4 -
-
- 1 -
-
- 3 -
-{% endblock %} diff --git a/arxiv/base/templates/base/500.html b/arxiv/base/templates/base/500.html deleted file mode 100644 index 1af06f72..00000000 --- a/arxiv/base/templates/base/500.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base/exception.html" %} - -{% block error_title %}Internal Server Error{% endblock %} - -{% block error_illustration %} -
- 5 -
-
- 0 -
-
- 0 -
-{% endblock %} diff --git a/arxiv/base/templates/base/error_macros.html b/arxiv/base/templates/base/error_macros.html new file mode 100644 index 00000000..0e0c70f0 --- /dev/null +++ b/arxiv/base/templates/base/error_macros.html @@ -0,0 +1,23 @@ + +{% macro render_error_code_digit(digit, idx, code) -%} + {% if digit == '0' %} + 0 + {% else %} + {{ digit }} + {% endif %} +{%- endmacro %} + +{% macro render_error_code(code) %} + {% for digit in code|string %} +
+ {{ render_error_code_digit(digit, loop.index, code) }} +
+ {% endfor %} +{% endmacro %} \ No newline at end of file diff --git a/arxiv/base/templates/base/exception.html b/arxiv/base/templates/base/exception.html index d47db798..70c1334d 100644 --- a/arxiv/base/templates/base/exception.html +++ b/arxiv/base/templates/base/exception.html @@ -1,21 +1,15 @@ {% extends "base/base.html" %} +{% import "base/error_macros.html" as error_macros %} + {% block content %}

- {% block error_title %}Page Not Found{% endblock %} + {% block error_title %}{{ error.name }}{% endblock %}

{% block error_illustration %} -
- 4 -
-
- 0 -
-
- 4 -
+ {{ error_macros.render_error_code(error.code) }} {% endblock %}

diff --git a/fourohfour/Pipfile b/fourohfour/Pipfile index 7a9e19a0..8c427d30 100644 --- a/fourohfour/Pipfile +++ b/fourohfour/Pipfile @@ -4,8 +4,10 @@ verify_ssl = true name = "pypi" [packages] +arxiv-base = {path = "./.."} [dev-packages] +nose2 = "*" [requires] python_version = "3.6" diff --git a/fourohfour/Pipfile.lock b/fourohfour/Pipfile.lock new file mode 100644 index 00000000..8ad95cd2 --- /dev/null +++ b/fourohfour/Pipfile.lock @@ -0,0 +1,76 @@ +{ + "_meta": { + "hash": { + "sha256": "8d2e0d8dcf32221d0cb8c91398b93190c40b59c848d26d8e3517e5fad7c0028c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "arxiv-base": { + "path": "./.." + } + }, + "develop": { + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "version": "==4.5.3" + }, + "nose2": { + "hashes": [ + "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999", + "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + } + } +} diff --git a/fourohfour/tests.py b/fourohfour/tests.py index fb0de887..7ef140c9 100644 --- a/fourohfour/tests.py +++ b/fourohfour/tests.py @@ -6,6 +6,8 @@ from wsgi import create_web_app from http import HTTPStatus as status +from werkzeug.exceptions import default_exceptions + class TestFourOhFour(TestCase): """Four oh four abounds.""" @@ -23,14 +25,6 @@ def test_returns_404(self): self.assertEqual(response.status_code, status.NOT_FOUND, "The root endpoint returns 400") - def test_returns_404_on_post(self): - """The fourohfour app returns 404 when you POST.""" - with self.app.app_context(): - response = self.client.post('/') - - self.assertEqual(response.status_code, status.NOT_FOUND, - "The root endpoint returns 400") - def test_returns_404_on_head(self): """The fourohfour app returns 404 when you HEAD.""" with self.app.app_context(): @@ -39,20 +33,6 @@ def test_returns_404_on_head(self): self.assertEqual(response.status_code, status.NOT_FOUND, "The root endpoint returns 400") - def test_it_really_returns_404(self): - """The fourohfour app returns 404 no matter what, really.""" - def random_path(): - path = ''.join(random.choices(string.ascii_uppercase + '/', k=20)) - return '/' + path - - for i in range(50): - with self.app.app_context(): - response = self.client.get(random_path()) - - self.assertEqual(response.status_code, - status.NOT_FOUND, - "404 all day long") - class TestHealthCheck(TestCase): """Test the health check endpoint.""" @@ -69,3 +49,21 @@ def test_returns_200(self): self.assertEqual(response.status_code, status.OK, "The health check endpoint returns 200") + + +class TestNGINXErrorHandling(TestCase): + """Propagates errors generated by NGINX.""" + + def setUp(self): + """We have an app and a client.""" + self.app = create_web_app() + self.client = self.app.test_client() + + def test_echos_error(self): + """Response status code reflects ``X-Code`` header.""" + with self.app.app_context(): + for code in default_exceptions.keys(): + response = self.client.get('/', headers={'X-Code': code, + 'X-Request-ID': '1'}) + self.assertEqual(response.status_code, code, + 'Response status code matches request header') diff --git a/fourohfour/wsgi.py b/fourohfour/wsgi.py index 7c0084ee..d0b487b3 100644 --- a/fourohfour/wsgi.py +++ b/fourohfour/wsgi.py @@ -1,11 +1,52 @@ -"""Web Server Gateway Interface entry-point.""" +""" +Web Server Gateway Interface entry-point.""" import os +from http import HTTPStatus as status from typing import Mapping -from flask import Flask -from arxiv.base import Base -from http import HTTPStatus as status +from flask import Flask, Response, make_response, request, jsonify +from werkzeug.exceptions import default_exceptions, NotFound, HTTPException + +from arxiv.base import Base, exceptions + + +def make_error_response() -> None: + """Raise an :class:`.HTTPException` based on the status in ``X-Code``.""" + data = {'request_id': request.headers['X-Request-ID']} + code = int(request.headers['X-Code']) + exception = default_exceptions[code] + raise exception(data) + + +def content_aware_exception_handler(error: HTTPException) -> Response: + """ + Error handler with support for the ``X-Format`` header. + + Looks to ``X-Format`` for the content type originally requested by the + client (e.g. via ``Accept`` request header). + + Falls back to the base exception handler. + """ + if request.headers.get('X-Format') == 'application/json': + data = {'error': error.code, 'name': error.name, + 'detail': error.description} + return make_response(jsonify(data), error.code) + return exceptions.handle_exception(error) + + +def echo() -> None: + """Propagate an exception from NGINX.""" + try: + make_error_response() + except KeyError: # Fall back to a 404 if error info is not available. + raise NotFound('Nope') + + +def healthz() -> Response: + """Health check endpoint.""" + response: Response = make_response("i'm still here", status.OK) + return response def create_web_app() -> Flask: @@ -13,17 +54,26 @@ def create_web_app() -> Flask: app = Flask("fourohfour") Base(app) - @app.route('/healthz') - def healthz(): - """Health check endpoint.""" - return "i'm still here", status.OK, {} + # Override the default error handlers to provide content negotation. + for _, error in default_exceptions.items(): + app.errorhandler(error)(content_aware_exception_handler) + app.route('/healthz')(healthz) + app.route('/')(echo) return app +__flask_app__ = create_web_app() +app = __flask_app__ + + def application(environ, start_response): """WSGI application factory.""" for key, value in environ.items(): + if key == 'SERVER_NAME': + continue os.environ[key] = str(value) - app = create_web_app() + if key in __flask_app__.config: + __flask_app__.config[key] = value + return app(environ, start_response) From 215ceaf0c941ee65241574a0238ce309d095a4c5 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Tue, 16 Jul 2019 05:28:34 -0400 Subject: [PATCH 10/16] minor --- .travis.yml | 1 + fourohfour/wsgi.py | 95 +++++++++++++++++++++++++--------------------- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad010a82..4398066d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ script: - pip install pipenv - pipenv install --dev - pipenv install ./ +- pipenv run nose2 -s fourohfour - pipenv run nose2 -s arxiv --with-coverage - "./tests/lintstats.sh" - pipenv run python tests/run_app_tests.py diff --git a/fourohfour/wsgi.py b/fourohfour/wsgi.py index d0b487b3..51ec48cc 100644 --- a/fourohfour/wsgi.py +++ b/fourohfour/wsgi.py @@ -1,5 +1,12 @@ """ -Web Server Gateway Interface entry-point.""" +Default backend for NGINX HTTP errors. + +NGINX will delegate the generation of a response in the case that it generates +an HTTP error. This allows us to return customized (arXiv-branded) error pages +even if the error orginated with NGINX rather than an upstream application. For +details on the API, see +`https://kubernetes.github.io/ingress-nginx/user-guide/custom-errors/`_. +""" import os from http import HTTPStatus as status @@ -11,6 +18,49 @@ from arxiv.base import Base, exceptions +def application(environ, start_response): + """ + Main WSGI application. + + Updates the Flask app config before using it to handle the request. + """ + for key, value in environ.items(): + if key == 'SERVER_NAME': + continue + os.environ[key] = str(value) + if key in __flask_app__.config: + __flask_app__.config[key] = value + return __flask_app__(environ, start_response) + + +def create_web_app() -> Flask: + """Create a new :class:`.Flask` app and define the API.""" + app = Flask("fourohfour") + Base(app) + + # Override the default error handlers to provide content negotation. + for _, error in default_exceptions.items(): + app.errorhandler(error)(content_aware_exception_handler) + + app.route('/healthz')(healthz) + app.route('/')(echo) + return app + + +def healthz() -> Response: + """Health check endpoint.""" + response: Response = make_response("i'm still here", status.OK) + return response + + +def echo() -> None: + """Propagate an exception from NGINX.""" + try: + make_error_response() + except KeyError: # Fall back to a 404 if error info is not available. + raise NotFound('Nope') + + def make_error_response() -> None: """Raise an :class:`.HTTPException` based on the status in ``X-Code``.""" data = {'request_id': request.headers['X-Request-ID']} @@ -35,45 +85,4 @@ def content_aware_exception_handler(error: HTTPException) -> Response: return exceptions.handle_exception(error) -def echo() -> None: - """Propagate an exception from NGINX.""" - try: - make_error_response() - except KeyError: # Fall back to a 404 if error info is not available. - raise NotFound('Nope') - - -def healthz() -> Response: - """Health check endpoint.""" - response: Response = make_response("i'm still here", status.OK) - return response - - -def create_web_app() -> Flask: - """Create a :class:`.Flask` app.""" - app = Flask("fourohfour") - Base(app) - - # Override the default error handlers to provide content negotation. - for _, error in default_exceptions.items(): - app.errorhandler(error)(content_aware_exception_handler) - - app.route('/healthz')(healthz) - app.route('/')(echo) - return app - - -__flask_app__ = create_web_app() -app = __flask_app__ - - -def application(environ, start_response): - """WSGI application factory.""" - for key, value in environ.items(): - if key == 'SERVER_NAME': - continue - os.environ[key] = str(value) - if key in __flask_app__.config: - __flask_app__.config[key] = value - - return app(environ, start_response) +__flask_app__ = create_web_app() \ No newline at end of file From ac3ebf1bfef4a2aa2bc0bbb2dfc003b7ad4b13b4 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Tue, 16 Jul 2019 08:25:36 -0400 Subject: [PATCH 11/16] upgraded mypy, fixed type errors --- Pipfile | 2 +- Pipfile.lock | 288 +++++++++++++++---------------- arxiv/base/exceptions.py | 15 +- arxiv/base/factory.py | 2 +- arxiv/base/routes.py | 9 +- arxiv/base/urls/links.py | 5 +- arxiv/integration/api/service.py | 2 +- 7 files changed, 161 insertions(+), 162 deletions(-) diff --git a/Pipfile b/Pipfile index 7aa890d4..c88a7ac2 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,7 @@ typing-extensions = "*" [dev-packages] pydocstyle = "*" -mypy = "==0.670" +mypy = "==0.720" pylint = "<2" "nose2" = "*" hypothesis = "*" diff --git a/Pipfile.lock b/Pipfile.lock index d18978a9..eac997b1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f7d5bb5e5f397c4601346adfb115126d3e50d6bb653d06d3ee6061497a4971d5" + "sha256": "467d65c728d345f8663b2533f29161627f432ecc382e518a41299c2a55c62600" }, "pipfile-spec": 6, "requires": {}, @@ -46,10 +46,10 @@ }, "botocore": { "hashes": [ - "sha256:0d95794f6b1239c75e2c5f966221bcd4b68020fddb5676f757531eedbb612ed8", - "sha256:3213cf48cf2ceee10fc3b93221f2cd1c38521cca7584f547d5c086213cc60f35" + "sha256:140ab03867d912b9cf44421861e6191681cbc065e36ccb51ece865b0ee30b5f3", + "sha256:f9c54673beb91ea21c718f1c1fef64862851c561a3810a18b39d3fdbd62ce32c" ], - "version": "==1.12.137" + "version": "==1.12.188" }, "click": { "hashes": [ @@ -75,11 +75,11 @@ }, "flask": { "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" ], "index": "pypi", - "version": "==1.0.2" + "version": "==1.1.1" }, "flask-s3": { "hashes": [ @@ -160,9 +160,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:5403d37f4d55ff4572b5b5676890589f367a9569529c6f728c11046c4ea4272b" + "sha256:50cffebc87ca91b9d4be2dcc2e479272bcb466b5a0487b6c271f7ddea6917e14" ], - "version": "==0.15.1" + "version": "==0.15.3" }, "python-dateutil": { "hashes": [ @@ -190,10 +190,10 @@ }, "s3transfer": { "hashes": [ - "sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e", - "sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021" + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" ], - "version": "==0.2.0" + "version": "==0.2.1" }, "six": { "hashes": [ @@ -204,20 +204,20 @@ }, "typing-extensions": { "hashes": [ - "sha256:07b2c978670896022a43c4b915df8958bec4a6b84add7f2c87b2b728bda3ba64", - "sha256:f3f0e67e1d42de47b5c67c32c9b26641642e9170fe7e292991793705cd5fef7c", - "sha256:fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71" + "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", + "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", + "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" ], "index": "pypi", - "version": "==3.7.2" + "version": "==3.7.4" }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], "markers": "python_version >= '3.4'", - "version": "==1.24.2" + "version": "==1.25.3" }, "uwsgi": { "hashes": [ @@ -235,10 +235,10 @@ }, "werkzeug": { "hashes": [ - "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", - "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" + "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", + "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" ], - "version": "==0.15.2" + "version": "==0.15.4" }, "wtforms": { "hashes": [ @@ -274,17 +274,17 @@ }, "babel": { "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", + "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" ], - "version": "==2.6.0" + "version": "==2.7.0" }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "chardet": { "hashes": [ @@ -295,37 +295,35 @@ }, "coverage": { "hashes": [ - "sha256:029c69deaeeeae1b15bc6c59f0ffa28aa8473721c614a23f2c2976dec245cd12", - "sha256:02abbbebc6e9d5abe13cd28b5e963dedb6ffb51c146c916d17b18f141acd9947", - "sha256:1bbfe5b82a3921d285e999c6d256c1e16b31c554c29da62d326f86c173d30337", - "sha256:210c02f923df33a8d0e461c86fdcbbb17228ff4f6d92609fc06370a98d283c2d", - "sha256:2d0807ba935f540d20b49d5bf1c0237b90ce81e133402feda906e540003f2f7a", - "sha256:35d7a013874a7c927ce997350d314144ffc5465faf787bb4e46e6c4f381ef562", - "sha256:3636f9d0dcb01aed4180ef2e57a4e34bb4cac3ecd203c2a23db8526d86ab2fb4", - "sha256:42f4be770af2455a75e4640f033a82c62f3fb0d7a074123266e143269d7010ef", - "sha256:48440b25ba6cda72d4c638f3a9efa827b5b87b489c96ab5f4ff597d976413156", - "sha256:4dac8dfd1acf6a3ac657475dfdc66c621f291b1b7422a939cc33c13ac5356473", - "sha256:4e8474771c69c2991d5eab65764289a7dd450bbea050bc0ebb42b678d8222b42", - "sha256:551f10ddfeff56a1325e5a34eff304c5892aa981fd810babb98bfee77ee2fb17", - "sha256:5b104982f1809c1577912519eb249f17d9d7e66304ad026666cb60a5ef73309c", - "sha256:5c62aef73dfc87bfcca32cee149a1a7a602bc74bac72223236b0023543511c88", - "sha256:633151f8d1ad9467b9f7e90854a7f46ed8f2919e8bc7d98d737833e8938fc081", - "sha256:772207b9e2d5bf3f9d283b88915723e4e92d9a62c83f44ec92b9bd0cd685541b", - "sha256:7d5e02f647cd727afc2659ec14d4d1cc0508c47e6cfb07aea33d7aa9ca94d288", - "sha256:a9798a4111abb0f94584000ba2a2c74841f2cfe5f9254709756367aabbae0541", - "sha256:b38ea741ab9e35bfa7015c93c93bbd6a1623428f97a67083fc8ebd366238b91f", - "sha256:b6a5478c904236543c0347db8a05fac6fc0bd574c870e7970faa88e1d9890044", - "sha256:c6248bfc1de36a3844685a2e10ba17c18119ba6252547f921062a323fb31bff1", - "sha256:c705ab445936457359b1424ef25ccc0098b0491b26064677c39f1d14a539f056", - "sha256:d95a363d663ceee647291131dbd213af258df24f41350246842481ec3709bd33", - "sha256:e27265eb80cdc5dab55a40ef6f890e04ecc618649ad3da5265f128b141f93f78", - "sha256:ebc276c9cb5d917bd2ae959f84ffc279acafa9c9b50b0fa436ebb70bbe2166ea", - "sha256:f4d229866d030863d0fe3bf297d6d11e6133ca15bbb41ed2534a8b9a3d6bd061", - "sha256:f95675bd88b51474d4fe5165f3266f419ce754ffadfb97f10323931fa9ac95e5", - "sha256:f95bc54fb6d61b9f9ff09c4ae8ff6a3f5edc937cda3ca36fc937302a7c152bf1", - "sha256:fd0f6be53de40683584e5331c341e65a679dbe5ec489a0697cec7c2ef1a48cda" - ], - "version": "==5.0a4" + "sha256:0402b1822d513d0231589494bceddb067d20581f5083598c451b56c684b0e5d6", + "sha256:0644e28e8aea9d9d563607ee8b7071b07dd57a4a3de11f8684cd33c51c0d1b93", + "sha256:0874a283686803884ec0665018881130604956dbaa344f2539c46d82cbe29eda", + "sha256:0988c3837df4bc371189bb3425d5232cf150055452034c232dda9cbe04f9c38e", + "sha256:20bc3205b3100956bb72293fabb97f0ed972c81fed10b3251c90c70dcb0599ab", + "sha256:2cc9142a3367e74eb6b19d58c53ebb1dfd7336b91cdcc91a6a2888bf8c7af984", + "sha256:3ae9a0a59b058ce0761c3bd2c2d66ecb2ee2b8ac592620184370577f7a546fb3", + "sha256:3b2e30b835df58cb973f478d09f3d82e90c98c8e5059acc245a8e4607e023801", + "sha256:401e9b04894eb1498c639c6623ee78a646990ce5f095248e2440968aafd6e90e", + "sha256:41ec5812d5decdaa72708be3018e7443e90def4b5a71294236a4df192cf9eab9", + "sha256:475769b638a055e75b3d3219e054fe2a023c0b077ff15bff6c95aba7e93e6cac", + "sha256:61424f4e2e82c4129a4ba71e10ebacb32a9ecd6f80de2cd05bdead6ba75ed736", + "sha256:811969904d4dd0bee7d958898be8d9d75cef672d9b7e7db819dfeac3d20d2d0c", + "sha256:86224bb99abfd672bf2f9fcecad5e8d7a3fa94f7f71513f2210460a0350307cd", + "sha256:9a238a20a3af00665f8381f7e53e9c606f9bb652d2423f6b822f6cb790d887e8", + "sha256:a23b3fbc14d4e6182ecebfd22f3729beef0636d151d94764a1c28330d185e4e5", + "sha256:ac162b4ebe51b7a2b7f5e462c4402802633eb81e77c94f8a7c1ed8a556e72c75", + "sha256:b6187378726c84365bf297b5dcdae8789b6a5823b200bea23797777e5a63be09", + "sha256:bcd5723d905ed4a825f17410a53535f880b6d7548ae3d89078db7b1ceefcd853", + "sha256:c48a4f9c5fb385269bb7fbaf9c1326a94863b65ec7f5c96b2ea56b252f01ad08", + "sha256:cd40199d6f1c29c85b170d25589be9a97edff8ee7e62be180a2a137823896030", + "sha256:d1bc331a7d069485ac1d8c25a0ea1f6aab6cb2a87146fb652222481c1bddc9ff", + "sha256:d7e0cdc249aa0f94aa2e531b03999ddaf03a10b4fa090a894712d4c8066abd89", + "sha256:e9ee8fcd8e067fcc5d7276d46e07e863102b70a52545ef4254df1ff0893ce75f", + "sha256:eb313c23d983b7810504f42104e8dcd1c7ccdda8fbaab82aab92ab79fea19345", + "sha256:f9cfd478654b509941b85ed70f870f5e3c74678f566bec12fd26545e5340ba47", + "sha256:fae1fa144034d021a52cb9ea200eb8dedf91869c6df8202ad5d149b41ed91cc8" + ], + "version": "==5.0a5" }, "docutils": { "hashes": [ @@ -337,12 +335,11 @@ }, "hypothesis": { "hashes": [ - "sha256:4e1378aec40b109f2212f8416a0ef27635b40fdc22a7a8116cd5527776ce1f9e", - "sha256:63d33b2394410ab09a05a19354d826e2aa7814288b0800eb5e89857181def40a", - "sha256:905395b9da7fe04e5ce32b41eace83b613586a104db4b4b0a2552db980a40dd2" + "sha256:4da34bfc2bc8bfbda7fccbc38a1ca8e906c1a31b273805d1df435b1393859947", + "sha256:8564107158c6853a990c769d5155180a51c6db49c32b5eec6f6960671fde3280" ], "index": "pypi", - "version": "==4.18.0" + "version": "==4.28.2" }, "idna": { "hashes": [ @@ -360,10 +357,10 @@ }, "isort": { "hashes": [ - "sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43", - "sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a" + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" ], - "version": "==4.3.17" + "version": "==4.3.21" }, "jinja2": { "hashes": [ @@ -374,37 +371,26 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", - "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", - "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", - "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", - "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", - "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", - "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", - "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", - "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", - "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", - "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", - "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", - "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", - "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", - "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", - "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", - "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", - "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", - "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", - "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", - "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", - "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", - "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", - "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", - "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", - "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", - "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", - "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", - "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" - ], - "version": "==1.3.1" + "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", + "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", + "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", + "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", + "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", + "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", + "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", + "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", + "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", + "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", + "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", + "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", + "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", + "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", + "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", + "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", + "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", + "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" + ], + "version": "==1.4.1" }, "markupsafe": { "hashes": [ @@ -448,11 +434,20 @@ }, "mypy": { "hashes": [ - "sha256:308c274eb8482fbf16006f549137ddc0d69e5a589465e37b99c4564414363ca7", - "sha256:e80fd6af34614a0e898a57f14296d0dacb584648f0339c2e000ddbf0f4cc2f8d" + "sha256:0107bff4f46a289f0e4081d59b77cef1c48ea43da5a0dbf0005d54748b26df2a", + "sha256:07957f5471b3bb768c61f08690c96d8a09be0912185a27a68700f3ede99184e4", + "sha256:10af62f87b6921eac50271e667cc234162a194e742d8e02fc4ddc121e129a5b0", + "sha256:11fd60d2f69f0cefbe53ce551acf5b1cec1a89e7ce2d47b4e95a84eefb2899ae", + "sha256:15e43d3b1546813669bd1a6ec7e6a11d2888db938e0607f7b5eef6b976671339", + "sha256:352c24ba054a89bb9a35dd064ee95ab9b12903b56c72a8d3863d882e2632dc76", + "sha256:437020a39417e85e22ea8edcb709612903a9924209e10b3ec6d8c9f05b79f498", + "sha256:49925f9da7cee47eebf3420d7c0e00ec662ec6abb2780eb0a16260a7ba25f9c4", + "sha256:6724fcd5777aa6cebfa7e644c526888c9d639bd22edd26b2a8038c674a7c34bd", + "sha256:7a17613f7ea374ab64f39f03257f22b5755335b73251d0d253687a69029701ba", + "sha256:cdc1151ced496ca1496272da7fc356580e95f2682be1d32377c22ddebdf73c91" ], "index": "pypi", - "version": "==0.670" + "version": "==0.720" }, "mypy-extensions": { "hashes": [ @@ -478,19 +473,17 @@ }, "pydocstyle": { "hashes": [ - "sha256:2258f9b0df68b97bf3a6c29003edc5238ff8879f1efb6f1999988d934e432bd8", - "sha256:5741c85e408f9e0ddf873611085e819b809fca90b619f5fd7f34bd4959da3dd4", - "sha256:ed79d4ec5e92655eccc21eb0c6cf512e69512b4a97d215ace46d17e4990f2039" + "sha256:58c421dd605eec0bce65df8b8e5371bb7ae421582cdf0ba8d9435ac5b0ffc36a" ], "index": "pypi", - "version": "==3.0.0" + "version": "==4.0.0" }, "pygments": { "hashes": [ - "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", - "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" ], - "version": "==2.3.1" + "version": "==2.4.2" }, "pylint": { "hashes": [ @@ -517,10 +510,10 @@ }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "version": "==2.21.0" + "version": "==2.22.0" }, "six": { "hashes": [ @@ -531,18 +524,17 @@ }, "snowballstemmer": { "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + "sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9" ], - "version": "==1.2.1" + "version": "==1.9.0" }, "sphinx": { "hashes": [ - "sha256:423280646fb37944dd3c85c58fb92a20d745793a9f6c511f59da82fa97cd404b", - "sha256:de930f42600a4fef993587633984cc5027dedba2464bcf00ddace26b40f8d9ce" + "sha256:22538e1bbe62b407cf5a8aabe1bb15848aa66bb79559f42f5202bbce6b757a69", + "sha256:f9a79e746b87921cabc3baa375199c6076d1270cee53915dbd24fdbeaaacc427" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.2" }, "sphinx-autodoc-typehints": { "hashes": [ @@ -596,50 +588,54 @@ }, "sphinxcontrib-websupport": { "hashes": [ - "sha256:4044751a075b6560f155c96f9fec6bc5198cd5307e5db9f77c7b1c5247ac9a09", - "sha256:c1b918b1b41cde045cdb9755941086b4ce4ebbfd7bff41d10ffb6d325779cbf9" + "sha256:1501befb0fdf1d1c29a800fdbf4ef5dc5369377300ddbdd16d2cd40e54c6eefc", + "sha256:e02f717baf02d0b6c3dd62cf81232ffca4c9d5c331e03766982e3ff9f1d2bc3f" ], "index": "pypi", - "version": "==1.1.1.dev20190321" + "version": "==1.1.2" }, "typed-ast": { "hashes": [ - "sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200", - "sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0", - "sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c", - "sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99", - "sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7", - "sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1", - "sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d", - "sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8", - "sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de", - "sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682", - "sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db", - "sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8", - "sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7", - "sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f", - "sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15", - "sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae", - "sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3", - "sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e", - "sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a", - "sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7" - ], - "version": "==1.3.4" + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "version": "==1.4.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", + "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", + "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + ], + "index": "pypi", + "version": "==3.7.4" }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], "markers": "python_version >= '3.4'", - "version": "==1.24.2" + "version": "==1.25.3" }, "wrapt": { "hashes": [ - "sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533" + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], - "version": "==1.11.1" + "version": "==1.11.2" } } } diff --git a/arxiv/base/exceptions.py b/arxiv/base/exceptions.py index 49ca87ce..51a07a4f 100644 --- a/arxiv/base/exceptions.py +++ b/arxiv/base/exceptions.py @@ -15,7 +15,7 @@ markup. """ -from typing import Callable, List, Tuple +from typing import Callable, List, Tuple, Type, Dict from werkzeug.exceptions import NotFound, Forbidden, Unauthorized, \ MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError, \ @@ -24,23 +24,24 @@ from http import HTTPStatus as status -_handlers = [] +_handlers: List[Tuple[Type[HTTPException], + Callable[[HTTPException], Response]]] = [] class ConfigurationError(InternalServerError): """Raised when a configuration parameter is missing or invalid.""" -def handler(exception: type) -> Callable: +def handler(exception_type: Type[HTTPException]) -> Callable: """Generate a decorator to register a handler for an exception.""" def deco(func: Callable) -> Callable: """Register a function as an exception handler.""" - _handlers.append((exception, func)) + _handlers.append((exception_type, func)) return func return deco -def get_handlers() -> List[Tuple[type, Callable]]: +def get_handlers() -> List[Tuple[Type[HTTPException], Callable]]: """ Get a list of registered exception handlers. @@ -63,5 +64,5 @@ def handle_exception(error: HTTPException) -> Response: # Generate handlers programmatically from the built-in Werkzeug HTTP # exceptions. -for code, exception in default_exceptions.items(): - handler(exception)(handle_exception) \ No newline at end of file +for code, exception_type in default_exceptions.items(): + handler(exception_type)(handle_exception) \ No newline at end of file diff --git a/arxiv/base/factory.py b/arxiv/base/factory.py index ea4c3bf0..4f2057a9 100644 --- a/arxiv/base/factory.py +++ b/arxiv/base/factory.py @@ -16,7 +16,7 @@ def create_web_app() -> Flask: """Initialize and configure the base application.""" app = Flask('base_test') # .config is an instance of a dict subclass with some methods. - app.config.from_object(config) # type: ignore + app.config.from_object(config) Base(app) # Gives us access to the base UI templates and resources. app.register_blueprint(routes.blueprint) diff --git a/arxiv/base/routes.py b/arxiv/base/routes.py index 95300b7c..e917c83a 100644 --- a/arxiv/base/routes.py +++ b/arxiv/base/routes.py @@ -13,7 +13,7 @@ from http import HTTPStatus as status from arxiv.base.exceptions import NotFound, Forbidden, Unauthorized, \ MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError, \ - default_exceptions + default_exceptions, HTTPException from . import alerts @@ -72,11 +72,12 @@ def test_macros() -> Response: 'doi': '10.1000/182', 'pagetitle': 'Home' } - response: Response = render_template("base/testmacros.html", **context) + response: Response = make_response(render_template("base/testmacros.html", + **context)) return response -def make_route(exception: Type) -> Callable[[], Response]: +def make_exception_route(exception: Type[HTTPException]) -> Callable[[], Response]: """Create a route that generates a Werkzeug HTTP exception.""" def _route() -> Response: raise exception() @@ -88,4 +89,4 @@ def _route() -> Response: for code, exception in default_exceptions.items(): register = blueprint.route(f'/{code}', methods=['GET'], endpoint=f'test_{code}') - register(make_route(exception)) + register(make_exception_route(exception)) diff --git a/arxiv/base/urls/links.py b/arxiv/base/urls/links.py index 36ec6c8c..707f966e 100644 --- a/arxiv/base/urls/links.py +++ b/arxiv/base/urls/links.py @@ -33,7 +33,8 @@ Updated 20 March, 2019: refactored to provide independent bleach attribute callbacks for each kind of link. """ -from typing import List, Pattern, Tuple, Callable, Dict, Union +from typing import List, Pattern, Tuple, Callable, Dict, Union, Text, Any, \ + MutableMapping import re from functools import reduce from urllib.parse import quote, urlparse @@ -46,7 +47,7 @@ from . import clickthrough -Attrs = Dict[Union[str, Tuple[None, str]], str] +Attrs = MutableMapping[Any, Text] Callback = Callable[[Attrs, bool], Attrs] Callable_Linker = Callable[[str], str] diff --git a/arxiv/integration/api/service.py b/arxiv/integration/api/service.py index 800b5cf5..b0985082 100644 --- a/arxiv/integration/api/service.py +++ b/arxiv/integration/api/service.py @@ -246,7 +246,7 @@ def get_session(cls, app: Optional[Flask] = None) -> 'HTTPIntegration': app = current_app name = cls.Meta.service_name.upper() try: - params = app.config.get_namespace(f'{name}_') # type: ignore + params = app.config.get_namespace(f'{name}_') endpoint = params.pop('endpoint') verify = params.pop('verify') except KeyError as e: From bb77b957c6ac4cd2e54addb39f5529fe14a2e920 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Tue, 16 Jul 2019 10:22:49 -0400 Subject: [PATCH 12/16] added sqlite --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 954c63e0..165a0d60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ PKGS_TO_INSTALL=$(cat <<-END\n\ which\n\ wget\n\ mariadb-devel\n\ + sqlite\n\ END\n\ )\n\ for pkg in ${PKGS_TO_INSTALL}; do\n\ From 4e471073188a53ca77d80b10a6441d324c52bd72 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Tue, 16 Jul 2019 10:41:48 -0400 Subject: [PATCH 13/16] pr #139 sort yum packages alphabetically --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 165a0d60..b33e6681 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,12 +31,12 @@ PKGS_TO_INSTALL=$(cat <<-END\n\ gcc\n\ gcc-c++ \n\ git\n\ + mariadb-devel\n\ python36u\n\ python36u-devel\n\ - which\n\ - wget\n\ - mariadb-devel\n\ sqlite\n\ + wget\n\ + which\n\ END\n\ )\n\ for pkg in ${PKGS_TO_INSTALL}; do\n\ From 921cf547e6e62a581c114f8340ccc5ebc9058bbc Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Tue, 23 Jul 2019 12:24:51 -0400 Subject: [PATCH 14/16] minor --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b33e6681..0a8b5a23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,9 @@ done\n\ yum -y install ${PKGS_TO_INSTALL}\n' >> /tmp/safe_yum.sh \ && /bin/bash /tmp/safe_yum.sh \ && yum clean all \ - && rm /tmp/safe_yum.sh + && rm /tmp/safe_yum.sh \ + && rm -rf /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* RUN wget https://bootstrap.pypa.io/get-pip.py \ && python3.6 get-pip.py \ From 5fd7c46c275b59c45e5158dbed329c79e515104e Mon Sep 17 00:00:00 2001 From: Brian Caruso Date: Tue, 30 Jul 2019 15:12:11 -0400 Subject: [PATCH 15/16] Only disable greek textism, not letter ones (#145) * In tex2utf(), split greek and letter TeXisms The fix for ARXIVNG-1612 disabled a set of texism letters and greek symbols when only the greek symbols should have been disabled. ARXIVOPS-805 --- arxiv/base/filters.py | 10 +++++----- arxiv/util/tests/test_tex2utf.py | 31 +++++++++++++------------------ arxiv/util/tex2utf.py | 18 ++++++++++++++---- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/arxiv/base/filters.py b/arxiv/base/filters.py index ab689cb9..34bf4365 100644 --- a/arxiv/base/filters.py +++ b/arxiv/base/filters.py @@ -43,12 +43,12 @@ def abstract_lf_to_br(text: JinjaFilterInput) -> Markup: def f_tex2utf(text: JinjaFilterInput, - letters: bool = True) -> Markup: + greek: bool = True) -> Markup: """Return output of tex2utf function as escaped Markup.""" if isinstance(text, Markup): - return escape(tex2utf(text.unescape(), letters=letters)) + return escape(tex2utf(text.unescape(), greek=greek)) else: - return Markup(escape(tex2utf(text, letters=letters))) + return Markup(escape(tex2utf(text, greek=greek))) def embed_content(path: str) -> Markup: @@ -98,8 +98,8 @@ def register_filters(app: Flask) -> None: """ app.template_filter('abstract_lf_to_br')(abstract_lf_to_br) app.template_filter('urlize')(urlizer()) - app.template_filter('tex2utf')(partial(f_tex2utf, letters=True)) - app.template_filter('tex2utf_no_symbols')(partial(f_tex2utf, letters=False)) + app.template_filter('tex2utf')(partial(f_tex2utf, greek=True)) + app.template_filter('tex2utf_no_symbols')(partial(f_tex2utf, greek=False)) app.template_filter('canonical_url')(canonical_url) app.template_filter('clickthrough_url')(clickthrough_url) app.template_filter('get_category_display')(get_category_display) diff --git a/arxiv/util/tests/test_tex2utf.py b/arxiv/util/tests/test_tex2utf.py index cf09eb20..619ddf09 100644 --- a/arxiv/util/tests/test_tex2utf.py +++ b/arxiv/util/tests/test_tex2utf.py @@ -11,7 +11,7 @@ def test_tex2utf(self): utf_out = tex2utf(test_str) self.assertEqual(utf_out, test_str) - utf_out = tex2utf(test_str, letters=False) + utf_out = tex2utf(test_str, greek=False) self.assertEqual(utf_out, test_str) test_str = "\\'e" @@ -24,12 +24,12 @@ def test_tex2utf(self): 0xc9)) test_str = "\\'E" - utf_out = tex2utf(test_str, letters=True) + utf_out = tex2utf(test_str, greek=True) self.assertEqual(utf_out, chr( 0xc9)) test_str = "\\'E" - utf_out = tex2utf(test_str, letters=False) + utf_out = tex2utf(test_str, greek=False) self.assertEqual(utf_out, chr(0xc9)) test_str = "\\'E" @@ -186,18 +186,13 @@ def test_ARXIVDEV2322fixes(self): utf_out = tex2utf(test_str) self.assertEqual(utf_out, 'ARXIVDEV-2322 ' + chr(0x110) + ' fix') - # (tex, err) = arXiv:: Filters: : Tex2UTF: : UTF2tex("Test Test_String. \x{03bb}" ) - # self.assertEqual(tex, - # "Test Test_String. {\\lambda}", - # "arXiv::Filters::Tex2UTF::UTF2tex()") - # - # self.assertEqual(arXiv:: Filters: : Tex2UTF: : escapeUTF8("\x{03bb}"), - # "λ", - # 'arXiv::Filters::Tex2UTF::escapeUTF8("\x{03bb}")') - # - # latin1 = "\x{91}\x{92}\x{93}\x{94}\x{96}\x{97}\x{98}\x{A0}\x{A6}\x{B1}\x{B2}\x{B3}\x{B5}\x{BC}\x{BD}\x{BE}" - # latin1expected = "`'\"\"----~ |{\\pm}^2^3{\\mu}1/41/23/4" - # is (arXiv: : Filters: : Tex2UTF: : latin2tex("Test Test_String. \x{03bb} latin1" ), - # "Test Test_String. {\\lambda} latin1expected", - # "arXiv::Filters::Tex2UTF::latin2tex(`'\"\"----~ |{\\pm}^2^3{\\mu}1/41/23/4)") - # done_testing + def test_ARXIVOPS805(self): + """{\aa} wasn't being converted correctly in /abs abstract field to å""" + self.assertEqual(tex2utf("ARXIVOPS-805 \\aa fix"), + 'ARXIVOPS-805 å fix') + + self.assertEqual(tex2utf("ARXIVOPS-805 \\aa fix", greek=False), + 'ARXIVOPS-805 å fix') + + self.assertEqual(tex2utf("ARXIVOPS-805 \\aa fix", greek=True), + 'ARXIVOPS-805 å fix') diff --git a/arxiv/util/tex2utf.py b/arxiv/util/tex2utf.py index 6aba2a5e..286f9d64 100644 --- a/arxiv/util/tex2utf.py +++ b/arxiv/util/tex2utf.py @@ -74,6 +74,9 @@ 'dh': 0x00f0, 'dj': 0x0111, 'eth': 0x00f0, 'i': 0x0131, 'l': 0x0142, 'ng': 0x014b, 'o': 0x00f8, 'ss': 0x00df, 'th': 0x00fe, + } + +textgreek = { # Greek (upper) 'Gamma': 0x0393, 'Delta': 0x0394, 'Theta': 0x0398, 'Lambda': 0x039b, 'Xi': 0x039E, 'Pi': 0x03a0, @@ -101,6 +104,7 @@ def _p_to_match(tex_to_chr: Dict[str, int]) -> Pattern: textlet_pattern = _p_to_match(textlet) +textgreek_pattern = _p_to_match(textgreek) textsym = { 'P': 0x00b6, 'S': 0x00a7, 'copyright': 0x00a9, @@ -119,6 +123,10 @@ def _textsym_sub(match: Match) -> str: return chr(textsym[match.group(2)]) +def _textgreek_sub(match: Match) -> str: + return chr(textgreek[match.group(2)]) + + def texch2UTF(acc: str) -> str: """Convert single character TeX accents to UTF-8. @@ -134,12 +142,12 @@ def texch2UTF(acc: str) -> str: return re.sub(r'[^\w]+', '', acc, flags=re.IGNORECASE) -def tex2utf(tex: str, letters: bool = True) -> str: +def tex2utf(tex: str, greek: bool = True) -> str: r"""Convert some TeX accents and greek symbols to UTF-8 characters. :param tex: Text to filter. - :param letters: If False, do not convert greek letters or + :param greek: If False, do not convert greek letters or ligatures. Greek symbols can cause problems. Ex. \phi is not suppose to look like φ. φ looks like \varphi. See ARXIVNG-1612 @@ -150,8 +158,10 @@ def tex2utf(tex: str, letters: bool = True) -> str: utf = re.sub(r"/(\\['`\^\"\~\=\.uvH])\{\\([ij])\}", r"\g<1>\{\g<2>\}", tex) # Now work on the Tex sequences, first those with letters only match - if letters: - utf = textlet_pattern.sub(_textlet_sub, utf) + utf = textlet_pattern.sub(_textlet_sub, utf) + + if greek: + utf = textgreek_pattern.sub(_textgreek_sub, utf) utf = textsym_pattern.sub(_textsym_sub, utf) From 78048cbb7857429b0de3b130c51c97bba8f4a8d2 Mon Sep 17 00:00:00 2001 From: erickpeirson Date: Fri, 2 Aug 2019 11:11:34 -0400 Subject: [PATCH 16/16] issue #146 made retry configurable in base HTTPIntegration --- .gitignore | 1 + arxiv/integration/api/service.py | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4e99f3f3..fcbcd019 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,4 @@ dist/ *.egg-info/ *.DS_Store .sass-cache/ +.pytest_cache/ diff --git a/arxiv/integration/api/service.py b/arxiv/integration/api/service.py index b0985082..43ec93a4 100644 --- a/arxiv/integration/api/service.py +++ b/arxiv/integration/api/service.py @@ -110,6 +110,15 @@ class Meta: service_name = "base" + default_retry_config = Retry( + total=10, + read=10, + connect=10, + status=10, + backoff_factor=0.5 + ) + """Default retry behavior for HTTP request.""" + def __init__(self, endpoint: str, verify: bool = True, headers: dict = {}, **extra: Any) -> None: """ @@ -129,13 +138,7 @@ def __init__(self, endpoint: str, verify: bool = True, self._extra = extra self._session = requests.Session() self._verify = verify - self._retry = Retry( - total=10, - read=10, - connect=10, - status=10, - backoff_factor=0.5 - ) + self._retry = self.get_retry_config() self._adapter = requests.adapters.HTTPAdapter(max_retries=self._retry) self._session.mount(f'{urlparse(endpoint).scheme}://', self._adapter) if not endpoint.endswith('/'): @@ -143,6 +146,17 @@ def __init__(self, endpoint: str, verify: bool = True, self._endpoint = endpoint self._session.headers.update(headers) + def get_retry_config(self) -> Retry: + """ + Defines the retry behavior for HTTP requests. + + This is defined as an instance method so that it can be defined in + relation to an app config or other contextual information. + + See :class:`urllib3.util.retry.Retry`. + """ + return self.default_retry_config + def _path(self, path: str, query: dict = {}) -> str: """Generate a full path for a request.""" o = urlparse(self._endpoint)