bug-1912360: shade elasticsearch libs to make way for new versions (#…

…6687)

.devcontainer/Dockerfile

@@ -31,10 +31,20 @@ RUN /tmp/set_up_stackwalker.sh && \
 COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
 RUN cd /webapp-frontend-deps/ && npm install
 
+# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the
+# new location (aka shaded), so that they can be installed at the same time as newer versions
+COPY --chown=app:app legacy-es-requirements.txt /app/
+RUN pip install -U 'pip==24.0' && \
+    pip install --no-cache-dir --no-deps -r legacy-es-requirements.txt && \
+    cd /usr/local/lib/python3.11/site-packages/ && \
+    mv elasticsearch elasticsearch_1_9_0 && \
+    mv elasticsearch_dsl elasticsearch_dsl_0_0_11 && \
+    cd elasticsearch_dsl_0_0_11/ && \
+    sed 's/^from elasticsearch/from elasticsearch_1_9_0/' -i connections.py search.py serializer.py
+
 # Install Python dependencies
 COPY requirements.txt /app/
-RUN pip install -U 'pip>=20' && \
-    pip install --no-cache-dir --no-deps -r requirements.txt && \
+RUN pip install --no-cache-dir --no-deps -r requirements.txt && \
     pip install --no-cache-dir ipython && \
     pip check --disable-pip-version-check
 

Makefile

@@ -146,6 +146,8 @@ testshell: my.env .docker-build  ## | Open a shell in the test environment.
 .PHONY: rebuildreqs
 rebuildreqs: .env .docker-build  ## | Rebuild requirements.txt file after requirements.in changes.
 	${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras
+	${DC} run --rm --no-deps app shell pip-compile --generate-hashes \
+		--unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in
 
 .PHONY: updatereqs
 updatereqs: .env .docker-build  ## | Update deps in requirements.txt file.

bin/es_cli.py

@@ -13,8 +13,8 @@
 import json
 
 import click
-from elasticsearch_dsl import Search
-from elasticsearch.client import IndicesClient
+from elasticsearch_dsl_0_0_11 import Search
+from elasticsearch_1_9_0.client import IndicesClient
 
 from socorro import settings
 from socorro.external.es.base import generate_list_of_indexes

bin/verify_reqs.sh

@@ -14,3 +14,7 @@ set -euo pipefail
 cp requirements.txt requirements.txt.orig
 pip-compile --quiet --generate-hashes --strip-extras
 diff requirements.txt requirements.txt.orig
+
+cp legacy-es-requirements.txt legacy-es-requirements.txt.orig
+pip-compile --quiet --generate-hashes --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in
+diff legacy-es-requirements.txt legacy-es-requirements.txt.orig

docker/Dockerfile

@@ -30,9 +30,15 @@ RUN /tmp/set_up_stackwalker.sh && \
 COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
 RUN cd /webapp-frontend-deps/ && npm install
 
+# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the
+# new location (aka shaded), so that they can be installed at the same time as newer versions
+COPY --chown=app:app legacy-es-requirements.txt /app/
+COPY docker/set_up_legacy_es_requirements.sh /tmp/set_up_legacy_es_requirements.sh
+RUN /tmp/set_up_legacy_es_requirements.sh && \
+    rm /tmp/set_up_legacy_es_requirements.sh
+
 COPY --chown=app:app requirements.txt /app/
-RUN pip install -U 'pip==24.0' && \
-    pip install --no-cache-dir --no-deps -r requirements.txt && \
+RUN pip install --no-cache-dir --no-deps -r requirements.txt && \
     pip check --disable-pip-version-check
 
 ENV PYTHONUNBUFFERED=1 \

docker/set_up_legacy_es_requirements.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Usage: docker/set_up_legacy_es_requirements.sh
+#
+# Installs legacy elasticsearch requirements to locations shaded by version.
+#
+# These dependencies are installed separately, relocated, and patched to reference the
+# new location (aka shaded), so that they can be installed at the same time as newer versions
+
+# install shaded packages without dependencies
+pip install --no-cache-dir --no-deps -r legacy-es-requirements.txt
+
+# move packages to shaded locations
+cd /usr/local/lib/python3.11/site-packages/
+mv elasticsearch elasticsearch_1_9_0
+mv elasticsearch_dsl elasticsearch_dsl_0_0_11
+
+# patch shaded elasticsearch-dsl to use shaded elasticearch
+cd elasticsearch_dsl_0_0_11/
+sed 's/^from elasticsearch/from elasticsearch_1_9_0/' -i connections.py search.py serializer.py

legacy-es-requirements.in

@@ -0,0 +1,4 @@
+# NOTE(willkg): Need to keep elasticsearch and elasticsearch-dsl at these versions
+# because they work with the cluster we're using
+elasticsearch==1.9.0
+elasticsearch-dsl==0.0.11

legacy-es-requirements.txt

@@ -0,0 +1,23 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --generate-hashes --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in
+#
+elasticsearch==1.9.0 \
+    --hash=sha256:cbee23c29f9947b1b78be5c0f741c1f8b504d3c53163b1355ea08f69cbd0fc0c \
+    --hash=sha256:eb0d4b5b20a4ccf9e49bd20498b6215d5e960d8a63a0eae44cd4431e3f303a24
+    # via
+    #   -r legacy-es-requirements.in
+    #   elasticsearch-dsl
+elasticsearch-dsl==0.0.11 \
+    --hash=sha256:59a76c4142478a1952bba6f9a9ca4fc7b029afb619e8ffcf0d135ce37ea692da \
+    --hash=sha256:663fb62ad39200c7d903e973aa0aa693578613264d83796455cbf4cd172bd878
+    # via -r legacy-es-requirements.in
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes and the requirement is not
+# satisfied by a package already installed. Consider using the --allow-unsafe flag.
+# python-dateutil
+# six
+# urllib3

requirements.in

@@ -66,5 +66,13 @@ django==4.2.15
 
 # NOTE(willkg): Need to keep elasticsearch and elasticsearch-dsl at these versions
 # because they work with the cluster we're using
-elasticsearch==1.9.0
-elasticsearch-dsl==0.0.11
+# via docker/set_up_legacy_es_requirements.sh and legacy-es-requirements.txt
+# NOTE(relud): these dependencies are installed separately via legacy-es-requirements.txt and
+# docker/set_up_legacy_es_requirements.sh, relocated, and patched to reference the new location (aka
+# shaded), so that they can be installed at the same time as newer versions
+# elasticsearch==1.9.0
+# elasticsearch-dsl==0.0.11
+# NOTE(relud): specify transitive dependencies that are not shaded
+python-dateutil # via elasticsearch-dsl==0.0.11
+six # via elasticsearch-dsl==0.0.11
+urllib3>=1.8, <2.0 # via elasticsearch==1.9.0

requirements.txt

@@ -298,16 +298,6 @@ docutils==0.20.1 \
     # via
     #   sphinx
     #   sphinx-rtd-theme
-elasticsearch==1.9.0 \
-    --hash=sha256:cbee23c29f9947b1b78be5c0f741c1f8b504d3c53163b1355ea08f69cbd0fc0c \
-    --hash=sha256:eb0d4b5b20a4ccf9e49bd20498b6215d5e960d8a63a0eae44cd4431e3f303a24
-    # via
-    #   -r requirements.in
-    #   elasticsearch-dsl
-elasticsearch-dsl==0.0.11 \
-    --hash=sha256:59a76c4142478a1952bba6f9a9ca4fc7b029afb619e8ffcf0d135ce37ea692da \
-    --hash=sha256:663fb62ad39200c7d903e973aa0aa693578613264d83796455cbf4cd172bd878
-    # via -r requirements.in
 enforce-typing==1.0.0.post1 \
     --hash=sha256:90347a61d08e7f7578d9714b4f0fd8abd9b6bc48c8ac8d46d7f290d413afabb7 \
     --hash=sha256:d3184dfdbfd7f9520c884986561751a6106c57cdd65d730470645d2d40c47e18
@@ -1002,7 +992,7 @@ python-dateutil==2.8.2 \
     --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
     --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
     # via
-    #   elasticsearch-dsl
+    #   -r requirements.in
     #   freezegun
 python-decouple==3.8 \
     --hash=sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f \
@@ -1224,7 +1214,7 @@ six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
-    #   elasticsearch-dsl
+    #   -r requirements.in
     #   isodate
     #   oauth2client
     #   python-dateutil
@@ -1292,7 +1282,7 @@ urllib3==1.26.19 \
     --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \
     --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429
     # via
-    #   elasticsearch
+    #   -r requirements.in
     #   requests
     #   sentry-sdk
 urlwait==1.0 \

socorro/external/es/connection_context.py

@@ -4,7 +4,7 @@
 
 import contextlib
 
-import elasticsearch
+import elasticsearch_1_9_0 as elasticsearch
 
 
 class ConnectionContext:

socorro/external/es/crashstorage.py

@@ -8,9 +8,9 @@
 import re
 import time
 
-import elasticsearch
-from elasticsearch.exceptions import NotFoundError
-from elasticsearch_dsl import Search
+import elasticsearch_1_9_0 as elasticsearch
+from elasticsearch_1_9_0.exceptions import NotFoundError
+from elasticsearch_dsl_0_0_11 import Search
 import glom
 import markus
 

socorro/external/es/query.py

@@ -3,7 +3,7 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import datetime
-import elasticsearch
+import elasticsearch_1_9_0 as elasticsearch
 import json
 import re
 

socorro/external/es/supersearch.py

@@ -7,8 +7,8 @@
 import datetime
 import re
 
-from elasticsearch.exceptions import NotFoundError, RequestError
-from elasticsearch_dsl import A, F, Q, Search
+from elasticsearch_1_9_0.exceptions import NotFoundError, RequestError
+from elasticsearch_dsl_0_0_11 import A, F, Q, Search
 
 from socorro.external.es.base import generate_list_of_indexes
 from socorro.external.es.super_search_fields import get_search_key

socorro/tests/conftest.py

@@ -13,7 +13,7 @@
 import uuid
 import sys
 
-from elasticsearch_dsl import Search
+from elasticsearch_dsl_0_0_11 import Search
 from google.api_core.exceptions import AlreadyExists, NotFound
 from google.auth.credentials import AnonymousCredentials
 from google.cloud import storage

webapp/crashstats/supersearch/libsupersearch.py

@@ -5,8 +5,8 @@
 from dataclasses import dataclass
 import datetime
 
-import elasticsearch
-from elasticsearch_dsl import Search
+import elasticsearch_1_9_0 as elasticsearch
+from elasticsearch_dsl_0_0_11 import Search
 
 from socorro import settings as socorro_settings
 from socorro.external.es.super_search_fields import FIELDS