:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): skip appsignal slow-event buckets outside tracing retention (#126474)
Tom Owers0a0c346d5ffa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.pyindex cde541a610bc..3d27e68e5d17 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py@@ -1105,6 +1105,30 @@ def _walk_aggregate_windows(resumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))+def _fetch_tracing_range_or_empty_if_aged_out(+ session: requests.Session,+ api_token: str,+ path: str,+ logger: FilteringBoundLogger,+ body: dict[str, Any],+) -> list[Any]:+ """POST a tracing V2 endpoint, treating a `from`/`to` range outside retention as empty.++ AppSignal returns 404 or 422 for a range older than the account's tracing retention, the+ same response `/deploys/stats` returns for a revision whose data has aged out (see+ `_fetch_deploy_stats`). Any other error status still raises: a broad, persistent 404/422+ would instead mean a genuinely malformed request, and `validate_credentials` already+ confirms the token and app ID before a sync starts.+ """+ try:+ return _fetch_v2(session, api_token, path, logger, body=body) or []+ except requests.HTTPError as error:+ if error.response is None or error.response.status_code not in (404, 422):+ raise+ logger.warning(f"AppSignal: {path} range {body.get('from')}-{body.get('to')} is outside tracing retention")+ return []++def _fetch_slow_events(session: requests.Session,api_token: str,@@ -1112,20 +1136,17 @@ def _fetch_slow_events(window: TimeWindow,logger: FilteringBoundLogger,) -> list[dict[str, Any]]:- events = (- _fetch_v2(- session,- api_token,- "/tracing/slow_events",- logger,- body={- "site_id": app_id,- "from": _to_iso(window.since),- "to": _to_iso(window.before),- "limit": SLOW_EVENTS_LIMIT,- },- )- or []+ events = _fetch_tracing_range_or_empty_if_aged_out(+ session,+ api_token,+ "/tracing/slow_events",+ logger,+ body={+ "site_id": app_id,+ "from": _to_iso(window.since),+ "to": _to_iso(window.before),+ "limit": SLOW_EVENTS_LIMIT,+ },)return [event for event in events if isinstance(event, dict) and event.get("digest")]@@ -1346,20 +1367,17 @@ def _get_slow_event_action_rows(for event in _fetch_slow_events(session, api_token, app_id, window, logger):digests_fetched += 1digest = event["digest"]- actions = (- _fetch_v2(- session,- api_token,- "/tracing/slow_events/actions",- logger,- body={- "site_id": app_id,- "from": _to_iso(window.since),- "to": _to_iso(window.before),- "digest": digest,- },- )- or []+ actions = _fetch_tracing_range_or_empty_if_aged_out(+ session,+ api_token,+ "/tracing/slow_events/actions",+ logger,+ body={+ "site_id": app_id,+ "from": _to_iso(window.since),+ "to": _to_iso(window.before),+ "digest": digest,+ },)rows.extend({**action, "digest": digest, "timestamp": bucket}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/tests/test_appsignal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/tests/test_appsignal.pyindex 4372499a1de3..edf969189f04 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/tests/test_appsignal.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/tests/test_appsignal.py@@ -568,6 +568,37 @@ class TestServiceEdgeRows:class TestSlowEventRows:+ @pytest.mark.parametrize("status_code", [404, 422])+ @mock.patch(f"{MODULE}.SLOW_EVENT_BUCKET_SECONDS", 3600)+ @mock.patch(f"{MODULE}.SLOW_EVENT_INITIAL_LOOKBACK_SECONDS", 3600)+ def test_a_bucket_older_than_retention_is_skipped_not_fatal(self, status_code: int) -> None:+ # A bucket outside the account's tracing retention gets the same aged-out-range response+ # as /deploys/stats (see TestDeployStatsRows), not a hard failure for the whole sync.+ result = _driver().run("slow_events", _v2_script(post={"/tracing/slow_events": status_code}))++ assert result.raised is None+ assert result.rows == []++ @mock.patch(f"{MODULE}.SLOW_EVENT_BUCKET_SECONDS", 3600)+ @mock.patch(f"{MODULE}.SLOW_EVENT_INITIAL_LOOKBACK_SECONDS", 2 * 3600)+ def test_a_later_in_retention_bucket_still_yields_rows_after_a_skipped_one(self) -> None:+ # The skip must not stop the walk: an in-retention bucket after a skipped one still+ # contributes its rows, rather than the whole sync going quietly empty.+ calls: list[dict[str, Any]] = []++ def respond(body: dict[str, Any]) -> Any:+ calls.append(body)+ return 422 if len(calls) == 1 else [{"digest": "d1", "impact": 9}]++ # Pinned to a bucket boundary so the lookback produces exactly two windows.+ now = datetime.fromtimestamp(1_000_000 - 1_000_000 % 3600, UTC)+ with time_machine.travel(now, tick=False):+ result = _driver().run("slow_events", _v2_script(post={"/tracing/slow_events": respond}))++ assert result.raised is None+ assert [row["digest"] for row in result.rows] == ["d1"]+ assert len(calls) == 2+@mock.patch(f"{MODULE}.MAX_SLOW_EVENT_DIGESTS_PER_SYNC", 1)@mock.patch(f"{MODULE}.SLOW_EVENT_BUCKET_SECONDS", 3600)@mock.patch(f"{MODULE}.SLOW_EVENT_INITIAL_LOOKBACK_SECONDS", 2 * 3600)