ci: merge main to release

.gitignore

@@ -17,6 +17,7 @@ datatracker.sublime-workspace
 /docker/docker-compose.extend-custom.yml
 /env
 /ghostdriver.log
+/geckodriver.log
 /htmlcov
 /ietf/static/dist-neue
 /latest-coverage.json

dev/deploy-to-container/settings_local.py

@@ -71,5 +71,11 @@
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 
+# No real secrets here, these are public testing values _only_
+APP_API_TOKENS = {
+  "ietf.api.views.ingest_email_test": ["ingestion-test-token"]
+}
+
+
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
     app:
         build:

docker/docker-compose.celery.yml

@@ -1,7 +1,3 @@
-version: '2.4'
-# Use version 2.4 for mem_limit setting. Version 3+ uses deploy.resources.limits.memory
-# instead, but that only works for swarm with docker-compose 1.25.1.
-
 services:
   mq:
     image: rabbitmq:3-alpine

docker/docker-compose.extend.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
     app:
         ports:

ietf/api/tests.py

@@ -1022,7 +1022,9 @@ def test_role_holder_addresses(self):
             sorted(e.address for e in emails),
         )
 
-    @override_settings(APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token"})
+    @override_settings(
+        APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token", "ietf.api.views.ingest_email_test": "test-token"}
+    )
     @mock.patch("ietf.api.views.iana_ingest_review_email")
     @mock.patch("ietf.api.views.ipr_ingest_response_email")
     @mock.patch("ietf.api.views.nomcom_ingest_feedback_email")
@@ -1032,29 +1034,47 @@ def test_ingest_email(
         mocks = {mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest}
         empty_outbox()        
         url = urlreverse("ietf.api.views.ingest_email")
+        test_mode_url = urlreverse("ietf.api.views.ingest_email_test")
 
         # test various bad calls
         r = self.client.get(url)
         self.assertEqual(r.status_code, 403)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.get(test_mode_url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(url)
         self.assertEqual(r.status_code, 403)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(test_mode_url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(r.status_code, 405)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.get(test_mode_url, headers={"X-Api-Key": "test-token"})
+        self.assertEqual(r.status_code, 405)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(r.status_code, 415)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(test_mode_url, headers={"X-Api-Key": "test-token"})
+        self.assertEqual(r.status_code, 415)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url, content_type="application/json", headers={"X-Api-Key": "valid-token"}
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url, content_type="application/json", headers={"X-Api-Key": "test-token"}
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url,
@@ -1064,6 +1084,14 @@ def test_ingest_email(
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            "this is not JSON!",
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url,
@@ -1073,6 +1101,14 @@ def test_ingest_email(
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            {"json": "yes", "valid_schema": False},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         # bad destination
         message_b64 = base64.b64encode(b"This is a message").decode()
@@ -1086,6 +1122,16 @@ def test_ingest_email(
         self.assertEqual(r.headers["Content-Type"], "application/json")
         self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "not-a-destination", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+        self.assertFalse(any(m.called for m in mocks))
 
         # test that valid requests call handlers appropriately
         r = self.client.post(
@@ -1102,6 +1148,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
         mock_iana_ingest.reset_mock()
 
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "iana-review", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_iana_ingest.reset_mock()
+
         r = self.client.post(
             url,
             {"dest": "ipr-response", "message": message_b64},
@@ -1116,6 +1175,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest})))
         mock_ipr_ingest.reset_mock()
 
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "ipr-response", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_ipr_ingest.reset_mock()
+
         # bad nomcom-feedback dest
         for bad_nomcom_dest in [
             "nomcom-feedback",  # no suffix
@@ -1133,6 +1205,16 @@ def test_ingest_email(
             self.assertEqual(r.headers["Content-Type"], "application/json")
             self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
             self.assertFalse(any(m.called for m in mocks))
+            r = self.client.post(
+                test_mode_url,
+                {"dest": bad_nomcom_dest, "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "test-token"},
+            )
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(r.headers["Content-Type"], "application/json")
+            self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+            self.assertFalse(any(m.called for m in mocks))
 
         # good nomcom-feedback dest
         random_year = randrange(100000)
@@ -1150,6 +1232,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest})))
         mock_nomcom_ingest.reset_mock()
 
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": f"nomcom-feedback-{random_year}", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_nomcom_ingest.reset_mock()
+
         # test that exceptions lead to email being sent - assumes that iana-review handling is representative
         mock_iana_ingest.side_effect = EmailIngestionError("Error: don't send email")
         r = self.client.post(

ietf/api/urls.py

@@ -27,6 +27,8 @@
     url(r'^doc/draft-aliases/$', api_views.draft_aliases),
     # email ingestor
     url(r'email/$', api_views.ingest_email),
+    # email ingestor
+    url(r'email/test/$', api_views.ingest_email_test),
     # GDPR: export of personal information for the logged-in person
     url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()),
     # Email alias information for groups

ietf/api/views.py

@@ -614,14 +614,16 @@ def as_emailmessage(self) -> Optional[EmailMessage]:
         return msg
 
 
-@requires_api_token
-@csrf_exempt
-def ingest_email(request):
-    """Ingest incoming email
+def ingest_email_handler(request, test_mode=False):
+    """Ingest incoming email - handler
 
     Returns a 4xx or 5xx status code if the HTTP request was invalid or something went
     wrong while processing it. If the request was valid, returns a 200. This may or may
     not indicate that the message was accepted.
+
+    If test_mode is true, actual processing of a valid message will be skipped. In this
+    mode, a valid request with a valid destination will be treated as accepted. The
+    "bad_dest" error may still be returned.
     """
 
     def _http_err(code, text):
@@ -657,15 +659,18 @@ def _api_response(result):
     try:
         if dest == "iana-review":
             valid_dest = True
-            iana_ingest_review_email(message)
+            if not test_mode:
+                iana_ingest_review_email(message)
         elif dest == "ipr-response":
             valid_dest = True
-            ipr_ingest_response_email(message)
+            if not test_mode:
+                ipr_ingest_response_email(message)
         elif dest.startswith("nomcom-feedback-"):
             maybe_year = dest[len("nomcom-feedback-"):]
             if maybe_year.isdecimal():
                 valid_dest = True
-                nomcom_ingest_feedback_email(message, int(maybe_year))
+                if not test_mode:
+                    nomcom_ingest_feedback_email(message, int(maybe_year))
     except EmailIngestionError as err:
         error_email = err.as_emailmessage()
         if error_email is not None:
@@ -677,3 +682,25 @@ def _api_response(result):
         return _api_response("bad_dest")
 
     return _api_response("ok")
+
+
+@requires_api_token
+@csrf_exempt
+def ingest_email(request):
+    """Ingest incoming email
+
+    Hands off to ingest_email_handler() with test_mode=False. This allows @requires_api_token to
+    give the test endpoint a distinct token from the real one.
+    """
+    return ingest_email_handler(request, test_mode=False)
+
+
+@requires_api_token
+@csrf_exempt
+def ingest_email_test(request):
+    """Ingest incoming email test endpoint
+
+    Hands off to ingest_email_handler() with test_mode=True. This allows @requires_api_token to
+    give the test endpoint a distinct token from the real one.
+    """
+    return ingest_email_handler(request, test_mode=True)

ietf/community/tests.py

@@ -108,10 +108,8 @@ def email_or_name_set(self, person):
         return [e for e in Email.objects.filter(person=person)] + \
                [a for a in Alias.objects.filter(person=person)]
 
-    def test_view_list(self):
-        person = self.complex_person(user__username='plain')
+    def do_view_list_test(self, person):
         draft = WgDraftFactory()
-
         # without list
         for id in self.email_or_name_set(person):
             url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id })
@@ -134,6 +132,15 @@ def test_view_list(self):
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertContains(r, draft.name)
 
+    def test_view_list(self):
+        person = self.complex_person(user__username='plain')
+        self.do_view_list_test(person)
+
+    def test_view_list_without_active_email(self):
+        person = self.complex_person(user__username='plain')
+        person.email_set.update(active=False)
+        self.do_view_list_test(person)
+
     def test_manage_personal_list(self):
         person = self.complex_person(user__username='plain')
         ad = Person.objects.get(user__username='ad')

ietf/community/views.py

@@ -68,6 +68,7 @@ def view_list(request, email_or_name=None):
         'meta': meta,
         'can_manage_list': can_manage_community_list(request.user, clist),
         'subscribed': subscribed,
+        "email_or_name": email_or_name,
     })
 
 @login_required

ietf/doc/utils.py

@@ -1046,6 +1046,8 @@ def build_file_urls(doc: Union[Document, DocHistory]):
 
         file_urls = []
         for t in found_types:
+            if t == "ps": # Postscript might have been submitted but should not be displayed in the list of URLs
+                continue
             label = "plain text" if t == "txt" else t
             file_urls.append((label, base + doc.name + "." + t))
 

ietf/doc/views_search.py

@@ -485,6 +485,29 @@ def _state_to_doc_type(state):
         )
         ad.buckets = copy.deepcopy(bucket_template)
 
+        # https://github.com/ietf-tools/datatracker/issues/4577
+        docs_via_group_ad = Document.objects.exclude(
+            group__acronym="none"
+        ).filter(
+            group__role__name="ad",
+            group__role__person=ad
+        ).filter(
+            states__type="draft-stream-ietf",
+            states__slug__in=["wg-doc","wg-lc","waiting-for-implementation","chair-w","writeupw"]
+        )
+
+        doc_for_ad = Document.objects.filter(ad=ad)
+
+        ad.pre_pubreq = (docs_via_group_ad | doc_for_ad).filter(
+            type="draft"
+        ).filter(
+            states__type="draft",
+            states__slug="active"
+        ).filter(
+            states__type="draft-iesg",
+            states__slug="idexists"
+        ).distinct().count()
+
         for doc in Document.objects.exclude(type_id="rfc").filter(ad=ad):
             dt = doc_type(doc)
             state = doc_state(doc)