From 1a7184ba4b7df84a0acdd0f3bf5d136abb4d16ed Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 27 Jun 2023 13:12:33 +0200 Subject: [PATCH] test: assert crash-level fatal in integration tests (#856) --- .github/workflows/ci.yml | 2 +- tests/assertions.py | 75 ++++++++++++++++++++++++++++---- tests/requirements.txt | 5 ++- tests/test_integration_http.py | 16 ++++--- tests/test_integration_stdout.py | 25 ++++++----- 5 files changed, 94 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5d03e81a..13c748df8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: submodules: recursive - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' cache: 'pip' - name: Installing Linux Dependencies diff --git a/tests/assertions.py b/tests/assertions.py index 6622c788f..71a7ba48f 100644 --- a/tests/assertions.py +++ b/tests/assertions.py @@ -1,9 +1,12 @@ -import datetime import email import gzip import platform import re import sys +from dataclasses import dataclass +from datetime import datetime + +import msgpack from .conditions import is_android @@ -161,7 +164,7 @@ def assert_minidump(envelope): assert minidump.payload.bytes.startswith(b"MDMP") -def assert_timestamp(ts, now=datetime.datetime.utcnow()): +def assert_timestamp(ts, now=datetime.utcnow()): assert ts[:11] == now.isoformat()[:11] @@ -176,6 +179,14 @@ def assert_event(envelope): assert_timestamp(event["timestamp"]) +def assert_breakpad_crash(envelope): + event = envelope.get_event() + expected = { + "level": "fatal", + } + assert_matches(event, expected) + + def assert_exception(envelope): event = envelope.get_event() exception = { @@ -186,7 +197,7 @@ def assert_exception(envelope): assert_timestamp(event["timestamp"]) -def assert_crash(envelope): +def assert_inproc_crash(envelope): event = envelope.get_event() assert_matches(event, {"level": "fatal"}) # depending on the unwinder, we currently don’t get any stack frames from @@ -213,16 +224,62 @@ def assert_no_before_send(envelope): assert ("adapted_by", "before_send") not in event.items() +@dataclass(frozen=True) +class CrashpadAttachments: + event: dict + breadcrumb1: list + breadcrumb2: list + + +def _unpack_breadcrumbs(payload): + unpacker = msgpack.Unpacker() + unpacker.feed(payload) + return [unpacked for unpacked in unpacker] + + +def _load_crashpad_attachments(msg): + event = {} + breadcrumb1 = [] + breadcrumb2 = [] + for part in msg.walk(): + match part.get_filename(): + case "__sentry-event": + event = msgpack.unpackb(part.get_payload(decode=True)) + case "__sentry-breadcrumb1": + breadcrumb1 = _unpack_breadcrumbs(part.get_payload(decode=True)) + case "__sentry-breadcrumb2": + breadcrumb2 = _unpack_breadcrumbs(part.get_payload(decode=True)) + + return CrashpadAttachments(event, breadcrumb1, breadcrumb2) + + +def is_valid_timestamp(timestamp): + try: + datetime.fromisoformat(timestamp) + return True + except ValueError: + return False + + +def _validate_breadcrumb_seq(seq, breadcrumb_func): + for i in seq: + breadcrumb = breadcrumb_func(i) + assert breadcrumb["message"] == str(i) + assert is_valid_timestamp(breadcrumb["timestamp"]) + + def assert_crashpad_upload(req): multipart = gzip.decompress(req.get_data()) msg = email.message_from_bytes(bytes(str(req.headers), encoding="utf8") + multipart) - files = [part.get_filename() for part in msg.walk()] + attachments = _load_crashpad_attachments(msg) + + if len(attachments.breadcrumb1) > 3: + _validate_breadcrumb_seq(range(97), lambda i: attachments.breadcrumb1[3 + i]) + _validate_breadcrumb_seq( + range(97, 101), lambda i: attachments.breadcrumb2[i - 97] + ) - # TODO: - # Actually assert that we get a correct event/breadcrumbs payload - assert "__sentry-breadcrumb1" in files - assert "__sentry-breadcrumb2" in files - assert "__sentry-event" in files + assert attachments.event["level"] == "fatal" assert any( b'name="upload_file_minidump"' in part.as_bytes() diff --git a/tests/requirements.txt b/tests/requirements.txt index 4d181eb9d..c90370ba0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ black==23.3.0 -pytest==7.2.2 -pytest-httpserver==1.0.6 +pytest==7.4.0 +pytest-httpserver==1.0.8 +msgpack==1.0.5 diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index c304f8441..c17439d75 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -1,11 +1,12 @@ -import pytest +import itertools +import json import os import time -import itertools import uuid -import json + +import pytest + from . import make_dsn, run, Envelope -from .conditions import has_http, has_breakpad, has_files from .assertions import ( assert_attachment, assert_meta, @@ -13,10 +14,12 @@ assert_stacktrace, assert_event, assert_exception, - assert_crash, + assert_inproc_crash, assert_session, assert_minidump, + assert_breakpad_crash, ) +from .conditions import has_http, has_breakpad, has_files pytestmark = pytest.mark.skipif(not has_http, reason="tests need http") @@ -233,7 +236,7 @@ def test_inproc_crash_http(cmake, httpserver): assert_breadcrumb(envelope) assert_attachment(envelope) - assert_crash(envelope) + assert_inproc_crash(envelope) def test_inproc_reinstall(cmake, httpserver): @@ -318,6 +321,7 @@ def test_breakpad_crash_http(cmake, httpserver): assert_breadcrumb(envelope) assert_attachment(envelope) + assert_breakpad_crash(envelope) assert_minidump(envelope) diff --git a/tests/test_integration_stdout.py b/tests/test_integration_stdout.py index ecb2c6f95..a6c71e5b0 100644 --- a/tests/test_integration_stdout.py +++ b/tests/test_integration_stdout.py @@ -12,12 +12,12 @@ assert_breadcrumb, assert_stacktrace, assert_event, - assert_crash, + assert_inproc_crash, assert_minidump, - assert_timestamp, assert_before_send, assert_no_before_send, assert_crash_timestamp, + assert_breakpad_crash, ) from .conditions import has_breakpad, has_files @@ -92,9 +92,9 @@ def test_multi_process(cmake): # while the processes are running, we expect two runs runs = [ - run - for run in os.listdir(os.path.join(cwd, ".sentry-native")) - if run.endswith(".run") + db_run + for db_run in os.listdir(os.path.join(cwd, ".sentry-native")) + if db_run.endswith(".run") ] assert len(runs) == 2 @@ -108,9 +108,9 @@ def test_multi_process(cmake): subprocess.run([cmd], cwd=cwd) runs = [ - run - for run in os.listdir(os.path.join(cwd, ".sentry-native")) - if run.endswith(".run") or run.endswith(".lock") + db_run + for db_run in os.listdir(os.path.join(cwd, ".sentry-native")) + if db_run.endswith(".run") or db_run.endswith(".lock") ] assert len(runs) == 0 @@ -136,7 +136,7 @@ def test_inproc_crash_stdout(cmake): assert_meta(envelope, integration="inproc") assert_breadcrumb(envelope) assert_attachment(envelope) - assert_crash(envelope) + assert_inproc_crash(envelope) def test_inproc_crash_stdout_before_send(cmake): @@ -148,7 +148,7 @@ def test_inproc_crash_stdout_before_send(cmake): assert_meta(envelope, integration="inproc") assert_breadcrumb(envelope) assert_attachment(envelope) - assert_crash(envelope) + assert_inproc_crash(envelope) assert_before_send(envelope) @@ -175,7 +175,7 @@ def test_inproc_crash_stdout_before_send_and_on_crash(cmake): assert_meta(envelope, integration="inproc") assert_breadcrumb(envelope) assert_attachment(envelope) - assert_crash(envelope) + assert_inproc_crash(envelope) @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend") @@ -189,6 +189,7 @@ def test_breakpad_crash_stdout(cmake): assert_breadcrumb(envelope) assert_attachment(envelope) assert_minidump(envelope) + assert_breakpad_crash(envelope) @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend") @@ -203,6 +204,7 @@ def test_breakpad_crash_stdout_before_send(cmake): assert_attachment(envelope) assert_minidump(envelope) assert_before_send(envelope) + assert_breakpad_crash(envelope) @pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend") @@ -230,3 +232,4 @@ def test_breakpad_crash_stdout_before_send_and_on_crash(cmake): assert_meta(envelope, integration="breakpad") assert_breadcrumb(envelope) assert_attachment(envelope) + assert_breakpad_crash(envelope)