test: assert crash-level fatal in integration tests (#856)

.github/workflows/ci.yml

@@ -121,7 +121,7 @@ jobs:
           submodules: recursive
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: 'pip'
 
       - name: Installing Linux Dependencies

tests/assertions.py

@@ -1,9 +1,12 @@
-import datetime
 import email
 import gzip
 import platform
 import re
 import sys
+from dataclasses import dataclass
+from datetime import datetime
+
+import msgpack
 
 from .conditions import is_android
 
@@ -161,7 +164,7 @@ def assert_minidump(envelope):
     assert minidump.payload.bytes.startswith(b"MDMP")
 
 
-def assert_timestamp(ts, now=datetime.datetime.utcnow()):
+def assert_timestamp(ts, now=datetime.utcnow()):
     assert ts[:11] == now.isoformat()[:11]
 
 
@@ -176,6 +179,14 @@ def assert_event(envelope):
     assert_timestamp(event["timestamp"])
 
 
+def assert_breakpad_crash(envelope):
+    event = envelope.get_event()
+    expected = {
+        "level": "fatal",
+    }
+    assert_matches(event, expected)
+
+
 def assert_exception(envelope):
     event = envelope.get_event()
     exception = {
@@ -186,7 +197,7 @@ def assert_exception(envelope):
     assert_timestamp(event["timestamp"])
 
 
-def assert_crash(envelope):
+def assert_inproc_crash(envelope):
     event = envelope.get_event()
     assert_matches(event, {"level": "fatal"})
     # depending on the unwinder, we currently don’t get any stack frames from
@@ -213,16 +224,62 @@ def assert_no_before_send(envelope):
     assert ("adapted_by", "before_send") not in event.items()
 
 
+@dataclass(frozen=True)
+class CrashpadAttachments:
+    event: dict
+    breadcrumb1: list
+    breadcrumb2: list
+
+
+def _unpack_breadcrumbs(payload):
+    unpacker = msgpack.Unpacker()
+    unpacker.feed(payload)
+    return [unpacked for unpacked in unpacker]
+
+
+def _load_crashpad_attachments(msg):
+    event = {}
+    breadcrumb1 = []
+    breadcrumb2 = []
+    for part in msg.walk():
+        match part.get_filename():
+            case "__sentry-event":
+                event = msgpack.unpackb(part.get_payload(decode=True))
+            case "__sentry-breadcrumb1":
+                breadcrumb1 = _unpack_breadcrumbs(part.get_payload(decode=True))
+            case "__sentry-breadcrumb2":
+                breadcrumb2 = _unpack_breadcrumbs(part.get_payload(decode=True))
+
+    return CrashpadAttachments(event, breadcrumb1, breadcrumb2)
+
+
+def is_valid_timestamp(timestamp):
+    try:
+        datetime.fromisoformat(timestamp)
+        return True
+    except ValueError:
+        return False
+
+
+def _validate_breadcrumb_seq(seq, breadcrumb_func):
+    for i in seq:
+        breadcrumb = breadcrumb_func(i)
+        assert breadcrumb["message"] == str(i)
+        assert is_valid_timestamp(breadcrumb["timestamp"])
+
+
 def assert_crashpad_upload(req):
     multipart = gzip.decompress(req.get_data())
     msg = email.message_from_bytes(bytes(str(req.headers), encoding="utf8") + multipart)
-    files = [part.get_filename() for part in msg.walk()]
+    attachments = _load_crashpad_attachments(msg)
+
+    if len(attachments.breadcrumb1) > 3:
+        _validate_breadcrumb_seq(range(97), lambda i: attachments.breadcrumb1[3 + i])
+        _validate_breadcrumb_seq(
+            range(97, 101), lambda i: attachments.breadcrumb2[i - 97]
+        )
 
-    # TODO:
-    # Actually assert that we get a correct event/breadcrumbs payload
-    assert "__sentry-breadcrumb1" in files
-    assert "__sentry-breadcrumb2" in files
-    assert "__sentry-event" in files
+    assert attachments.event["level"] == "fatal"
 
     assert any(
         b'name="upload_file_minidump"' in part.as_bytes()

tests/requirements.txt

@@ -1,3 +1,4 @@
 black==23.3.0
-pytest==7.2.2
-pytest-httpserver==1.0.6
+pytest==7.4.0
+pytest-httpserver==1.0.8
+msgpack==1.0.5

tests/test_integration_http.py

@@ -1,22 +1,25 @@
-import pytest
+import itertools
+import json
 import os
 import time
-import itertools
 import uuid
-import json
+
+import pytest
+
 from . import make_dsn, run, Envelope
-from .conditions import has_http, has_breakpad, has_files
 from .assertions import (
     assert_attachment,
     assert_meta,
     assert_breadcrumb,
     assert_stacktrace,
     assert_event,
     assert_exception,
-    assert_crash,
+    assert_inproc_crash,
     assert_session,
     assert_minidump,
+    assert_breakpad_crash,
 )
+from .conditions import has_http, has_breakpad, has_files
 
 pytestmark = pytest.mark.skipif(not has_http, reason="tests need http")
 
@@ -233,7 +236,7 @@ def test_inproc_crash_http(cmake, httpserver):
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
 
-    assert_crash(envelope)
+    assert_inproc_crash(envelope)
 
 
 def test_inproc_reinstall(cmake, httpserver):
@@ -318,6 +321,7 @@ def test_breakpad_crash_http(cmake, httpserver):
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
 
+    assert_breakpad_crash(envelope)
     assert_minidump(envelope)
 
 

tests/test_integration_stdout.py

@@ -12,12 +12,12 @@
     assert_breadcrumb,
     assert_stacktrace,
     assert_event,
-    assert_crash,
+    assert_inproc_crash,
     assert_minidump,
-    assert_timestamp,
     assert_before_send,
     assert_no_before_send,
     assert_crash_timestamp,
+    assert_breakpad_crash,
 )
 from .conditions import has_breakpad, has_files
 
@@ -92,9 +92,9 @@ def test_multi_process(cmake):
 
     # while the processes are running, we expect two runs
     runs = [
-        run
-        for run in os.listdir(os.path.join(cwd, ".sentry-native"))
-        if run.endswith(".run")
+        db_run
+        for db_run in os.listdir(os.path.join(cwd, ".sentry-native"))
+        if db_run.endswith(".run")
     ]
     assert len(runs) == 2
 
@@ -108,9 +108,9 @@ def test_multi_process(cmake):
     subprocess.run([cmd], cwd=cwd)
 
     runs = [
-        run
-        for run in os.listdir(os.path.join(cwd, ".sentry-native"))
-        if run.endswith(".run") or run.endswith(".lock")
+        db_run
+        for db_run in os.listdir(os.path.join(cwd, ".sentry-native"))
+        if db_run.endswith(".run") or db_run.endswith(".lock")
     ]
     assert len(runs) == 0
 
@@ -136,7 +136,7 @@ def test_inproc_crash_stdout(cmake):
     assert_meta(envelope, integration="inproc")
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
-    assert_crash(envelope)
+    assert_inproc_crash(envelope)
 
 
 def test_inproc_crash_stdout_before_send(cmake):
@@ -148,7 +148,7 @@ def test_inproc_crash_stdout_before_send(cmake):
     assert_meta(envelope, integration="inproc")
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
-    assert_crash(envelope)
+    assert_inproc_crash(envelope)
     assert_before_send(envelope)
 
 
@@ -175,7 +175,7 @@ def test_inproc_crash_stdout_before_send_and_on_crash(cmake):
     assert_meta(envelope, integration="inproc")
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
-    assert_crash(envelope)
+    assert_inproc_crash(envelope)
 
 
 @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend")
@@ -189,6 +189,7 @@ def test_breakpad_crash_stdout(cmake):
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
     assert_minidump(envelope)
+    assert_breakpad_crash(envelope)
 
 
 @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend")
@@ -203,6 +204,7 @@ def test_breakpad_crash_stdout_before_send(cmake):
     assert_attachment(envelope)
     assert_minidump(envelope)
     assert_before_send(envelope)
+    assert_breakpad_crash(envelope)
 
 
 @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend")
@@ -230,3 +232,4 @@ def test_breakpad_crash_stdout_before_send_and_on_crash(cmake):
     assert_meta(envelope, integration="breakpad")
     assert_breadcrumb(envelope)
     assert_attachment(envelope)
+    assert_breakpad_crash(envelope)