:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse_sources): add hookdeck ignored events, executions and queue metrics (#122048)
Tom Owers71d241511d55
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 89626e0a00c4..8259af668f71 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6024,10 +6024,10 @@ Today (10): `attempts`, `bookmarks`, `connections`, `destinations`, `events`, `iDiffed against: <https://hookdeck.com/docs/api/inspect.md>-- [ ] `Request ignored events (GET /requests/{id}/ignored_events)` — explain filtered or deduplicated deliveries that never appear in the events table (high)-- [ ] `Queue depth metrics (GET /metrics/queue-depth)` — track historical delivery backlogs and identify overloaded destinations (high)-- [ ] `Events pending timeseries (GET /metrics/events-pending-timeseries)` — measure how pending deliveries accumulate and recover over time (medium)-- [ ] `Transformation executions (GET /transformations/{id}/executions)` — analyze transformation failures, execution times, and their impact on event delivery (medium)+- [x] `Request ignored events (GET /requests/{id}/ignored_events)` — explain filtered or deduplicated deliveries that never appear in the events table (high)+- [x] `Queue depth metrics (GET /metrics/queue-depth)` — track historical delivery backlogs and identify overloaded destinations (high)+- [x] `Events pending timeseries (GET /metrics/events-pending-timeseries)` — measure how pending deliveries accumulate and recover over time (medium)+- [x] `Transformation executions (GET /transformations/{id}/executions)` — analyze transformation failures, execution times, and their impact on event delivery (medium)- [ ] `Request, event, and attempt metrics (GET /metrics/requests; GET /metrics/events; GET /metrics/attempts)` — compare vendor aggregates for throughput, delivery outcomes, and response times across reporting windows (medium)- [ ] `Bulk operation history (GET /bulk/events/retry; GET /bulk/events/cancel; GET /bulk/ignored-events/retry; GET /bulk/requests/retry; GET /bulk/requests/replay)` — audit retry, replay, and cancellation operations with their progress and affected delivery counts (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.pyindex 0aab4d76db0d..dfd66ee446a0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py@@ -188,4 +188,53 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "Date the bookmark was created.",},},+ "request_ignored_events": {+ "description": "A delivery Hookdeck skipped for an inbound request: one row per request and connection whose rules kept the request from becoming an event.",+ "docs_url": f"{_API_DOCS}/inspect#getrequestignoredevents",+ "columns": {+ "id": "ID of the ignored event, built from the request ID and the connection ID.",+ "team_id": "ID of the Hookdeck project.",+ "webhook_id": "ID of the associated connection (webhook).",+ "cause": "Why no event was created: DISABLED, FILTERED, TRANSFORMATION_FAILED or CLI_DISCONNECTED.",+ "request_id": "ID of the request that was ignored.",+ "meta": "Detail for the cause: the request parts a filter rejected, or the transformation that failed.",+ "created_at": "Date the ignored event was created.",+ },+ },+ "transformation_executions": {+ "description": "One run of a transformation against an event, with its log level and console output.",+ "docs_url": f"{_API_DOCS}/configure#gettransformationexecutions",+ "columns": {+ "id": "ID of the execution.",+ "transformation_id": "ID of the transformation that ran.",+ "team_id": "ID of the Hookdeck project.",+ "webhook_id": "ID of the connection the execution was run for.",+ "original_event_data_id": "ID of the event data before the transformation.",+ "transformed_event_data_id": "ID of the event data the transformation produced.",+ "log_level": "Highest log level the execution emitted.",+ "logs": "Console lines the transformation wrote, each with a type and a message.",+ "issue_id": "ID of the issue raised for a failed execution.",+ "updated_at": "Date the execution was last updated.",+ "created_at": "Date the transformation ran.",+ },+ },+ "queue_depth_metrics": {+ "description": "Hourly delivery backlog per destination, from the queue depth metrics endpoint.",+ "docs_url": f"{_API_DOCS}/inspect#queryqueuedepthmetrics",+ "columns": {+ "time_bucket": "Start of the hourly bucket.",+ "destination_id": "ID of the destination.",+ "max_depth": "Largest number of pending events sampled in the bucket.",+ "max_age": "Longest time an event sat pending in the bucket, in minutes.",+ },+ },+ "events_pending_timeseries": {+ "description": "Hourly count of events pending delivery per destination.",+ "docs_url": f"{_API_DOCS}/inspect#queryeventspendingtimeseriesmetrics",+ "columns": {+ "time_bucket": "Start of the hourly bucket.",+ "destination_id": "ID of the destination.",+ "count": "Number of events pending delivery in the bucket.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.pyindex 522b35dd8014..bf7430647c1b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py@@ -1,33 +1,46 @@import dataclasses-from collections.abc import Iterable, Iterator-from datetime import UTC, date, datetime+from collections.abc import Callable, Iterable, Iterator+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom requests import Response+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponseCursorPaginator,+ SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.hookdeck.settings import (HOOKDECK_API_HOST,HOOKDECK_ENDPOINTS,+ METRICS_BACKFILL_DAYS,+ METRICS_GRANULARITY,PAGE_SIZE,HookdeckEndpointConfig,+ HookdeckMetricsConfig,)[email protected](frozen=False)class HookdeckResumeConfig:# Opaque `pagination.next` cursor pointing at the page after the last one yielded.- next_cursor: str+ next_cursor: Optional[str] = None+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished+ # and where the current one stopped.+ fanout_state: Optional[dict[str, Any]] = NoneREDACTED = "***redacted***"@@ -167,6 +180,70 @@ class HookdeckCursorPaginator(JSONResponseCursorPaginator):self._has_next_page = False+def _skip_requests_without_ignored_events(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:+ # Most requests ignore nothing, so only fan out where Hookdeck counted an ignored event. A null+ # count is unknown, not zero, and is still read.+ return [] if row.get("ignored_count") == 0 else row+++_PARENT_DATA_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any] | list[dict[str, Any]]]] = {+ "request_ignored_events": _skip_requests_without_ignored_events,+}+++def build_metrics_params(+ metrics: HookdeckMetricsConfig,+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Optional[Any],+ now: datetime,+) -> dict[str, Any]:+ start = parse_datetime_value(db_incremental_field_last_value) if should_use_incremental_field else None+ if start is None:+ start = now - timedelta(days=METRICS_BACKFILL_DAYS)+ # Align to the bucket boundary so a re-read bucket covers the same hour it did before.+ start = start.replace(minute=0, second=0, microsecond=0)++ # The metrics endpoints take structured values in bracket notation; `[]` keeps a single+ # dimension an array.+ return {+ "date_range[start]": _format_datetime(start),+ "date_range[end]": _format_datetime(now),+ "granularity": METRICS_GRANULARITY,+ "measures[]": list(metrics.measures),+ "dimensions[]": list(metrics.dimensions),+ }+++class HookdeckMetricsTruncatedError(Exception):+ pass+++class HookdeckMetricsPaginator(SinglePagePaginator):+ """Fails a metrics query that hit Hookdeck's row limit.++ A truncated response still carries data, and accepting it would let the watermark move past the+ buckets Hookdeck left out.+ """++ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:+ super().update_state(response, data)+ body = response.json()+ metadata = body.get("metadata") if isinstance(body, dict) else None+ if isinstance(metadata, dict) and metadata.get("truncated"):+ raise HookdeckMetricsTruncatedError(+ f"Hookdeck truncated the metrics response at {metadata.get('row_limit')} rows, so the sync stopped "+ "rather than store incomplete buckets."+ )+++def _flatten_metric_point(point: dict[str, Any]) -> dict[str, Any]:+ return {+ "time_bucket": point.get("time_bucket"),+ **(point.get("dimensions") or {}),+ **(point.get("metrics") or {}),+ }++def validate_credentials(api_key: str, api_version: str) -> tuple[bool, int | None]:return validate_via_probe(# capture=False: the probe hits `/sources`, whose rows carry verification secrets the@@ -190,19 +267,44 @@ def hookdeck_source() -> SourceResponse:config = HOOKDECK_ENDPOINTS[endpoint]+ client_config: ClientConfig = {+ "base_url": base_url(api_version),+ # The API key rides the framework's bearer auth so it is redacted from logs and errors;+ # only the non-secret Accept header is set here.+ "headers": {"Accept": "application/json"},+ "auth": {"type": "bearer", "token": api_key},+ # capture=False: raw responses reach the sampler before `_redact_pages` runs, and the+ # name-based scrubbers don't recognise Hookdeck's secret containers (`auth_method`,+ # `verification`, `env`) or raw inbound request payloads, so keep every response body+ # out of shared HTTP sample storage. Requests are still metered and logged (key redacted).+ "session": make_tracked_session(capture=False, redact_values=(api_key,)),+ }++ if config.metrics is not None:+ return _metrics_source(+ config,+ config.metrics,+ client_config,+ team_id,+ job_id,+ should_use_incremental_field,+ db_incremental_field_last_value,+ )+ if config.fanout is not None:+ return _fanout_source(+ config,+ config.fanout,+ client_config,+ team_id,+ job_id,+ resumable_source_manager,+ should_use_incremental_field,+ db_incremental_field_last_value,+ incremental_field,+ )+rest_config: RESTAPIConfig = {- "client": {- "base_url": base_url(api_version),- # The API key rides the framework's bearer auth so it is redacted from logs and errors;- # only the non-secret Accept header is set here.- "headers": {"Accept": "application/json"},- "auth": {"type": "bearer", "token": api_key},- # capture=False: raw responses reach the sampler before `_redact_pages` runs, and the- # name-based scrubbers don't recognise Hookdeck's secret containers (`auth_method`,- # `verification`, `env`) or raw inbound request payloads, so keep every response body- # out of shared HTTP sample storage. Requests are still metered and logged (key redacted).- "session": make_tracked_session(capture=False, redact_values=(api_key,)),- },+ "client": client_config,"resource_defaults": {},"resources": [{@@ -225,7 +327,7 @@ def hookdeck_source(initial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()- if resume is not None:+ if resume is not None and resume.next_cursor:initial_paginator_state = {"cursor": resume.next_cursor}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:@@ -258,3 +360,103 @@ def hookdeck_source(# Every request pins `dir=asc` on the same column the watermark tracks.sort_mode="asc",)+++def _fanout_source(+ config: HookdeckEndpointConfig,+ fanout: DependentEndpointConfig,+ client_config: ClientConfig,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[HookdeckResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Optional[Any],+ incremental_field: Optional[str],+) -> SourceResponse:+ parent_params = build_params(HOOKDECK_ENDPOINTS[fanout.parent_name], False, None, None)+ child_params = build_params(+ config, should_use_incremental_field, db_incremental_field_last_value, incremental_field+ )++ resume = (+ resumable_source_manager.load_state()+ if config.checkpoint_fanout and resumable_source_manager.can_resume()+ else None+ )++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(HookdeckResumeConfig(fanout_state=state))++ resource = build_dependent_resource(+ endpoint_configs=HOOKDECK_ENDPOINTS,+ child_endpoint=config.name,+ fanout=dataclasses.replace(fanout, parent_params=parent_params),+ client_config=client_config,+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ should_use_incremental_field=should_use_incremental_field,+ incremental_field=incremental_field,+ # The date filter is already in the static params above, so the framework's own cursor+ # injection stays off; the child still merges on its primary key.+ incremental_config_factory=lambda _field: None,+ parent_endpoint_extra={"data_selector": "models", "paginator": HookdeckCursorPaginator()},+ child_endpoint_extra={"data_selector": "models", "paginator": HookdeckCursorPaginator()},+ child_params_extra=child_params,+ parent_data_map=_PARENT_DATA_MAPS.get(config.name),+ resume_hook=save_fanout_checkpoint if config.checkpoint_fanout else None,+ initial_paginator_state=resume.fanout_state if resume else None,+ )++ return SourceResponse(+ name=config.name,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ # Rows arrive in order per parent only, not as one ordered series.+ sort_mode=None,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ )+++def _metrics_source(+ config: HookdeckEndpointConfig,+ metrics: HookdeckMetricsConfig,+ client_config: ClientConfig,+ team_id: int,+ job_id: str,+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Optional[Any],+) -> SourceResponse:+ rest_config: RESTAPIConfig = {+ "client": client_config,+ "resource_defaults": {},+ "resources": [+ {+ "name": config.name,+ "endpoint": {+ "path": config.path,+ "params": build_metrics_params(+ metrics, should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC)+ ),+ "data_selector": "data",+ "paginator": HookdeckMetricsPaginator(),+ },+ }+ ],+ }+ resource = rest_api_resource(rest_config, team_id, job_id, None).add_map(_flatten_metric_point)++ return SourceResponse(+ name=config.name,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ # Hookdeck documents no order for the data points of one query.+ sort_mode=None,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.pyindex 9047367ce692..5560425d5cb1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py@@ -1,6 +1,9 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType# Hookdeck pins the API version as the first path segment. Omitting it tracks the latest version,@@ -16,6 +19,18 @@ PAGE_SIZE = 250# value they had when first imported._RESTATED_RECORD_LOOKBACK_SECONDS = 24 * 60 * 60+# A fan-out run reads each parent's children at a different moment but stores one watermark for+# the whole table, so the next run re-reads a trailing day to pick up children that landed under a+# parent after it was read.+_FANOUT_LOOKBACK_SECONDS = 24 * 60 * 60++# Metrics are bucketed per hour, and the latest bucket is still filling when it is read.+METRICS_GRANULARITY = "1h"+_METRICS_LOOKBACK_SECONDS = 2 * 60 * 60+# How far back a full refresh or a first incremental sync reads. Hookdeck documents no retention+# window for metrics, so the backfill stays bounded rather than asking for all history.+METRICS_BACKFILL_DAYS = 30+def _datetime_field(name: str) -> IncrementalField:return {@@ -26,7 +41,15 @@ def _datetime_field(name: str) -> IncrementalField:}-@dataclass+@dataclass(frozen=True)+class HookdeckMetricsConfig:+ measures: tuple[str, ...]+ # Each dimension becomes a column and part of the primary key, so only list dimensions that are+ # never null on a row.+ dimensions: tuple[str, ...]+++@dataclass(frozen=False)class HookdeckEndpointConfig:name: strpath: str@@ -47,6 +70,17 @@ class HookdeckEndpointConfig:# When set, those nested fields are masked before rows are written to the warehouse so a reader# of the table can't recover credentials they don't have the Hookdeck key for.contains_credentials: bool = False+ page_size: int = PAGE_SIZE+ # Child endpoints are read once per row of `fanout.parent_name`.+ fanout: Optional[DependentEndpointConfig] = None+ checkpoint_fanout: bool = True+ # Aggregated time series from `/metrics/*`, read as one bucketed query over a date range+ # instead of a paginated list.+ metrics: Optional[HookdeckMetricsConfig] = None++ @property+ def default_incremental_field(self) -> Optional[str]:+ return self.incremental_fields[0]["field"] if self.incremental_fields else NoneHOOKDECK_ENDPOINTS: dict[str, HookdeckEndpointConfig] = {@@ -118,6 +152,59 @@ HOOKDECK_ENDPOINTS: dict[str, HookdeckEndpointConfig] = {path="/bookmarks",description="Saved event payloads kept for replaying against a connection",),+ "request_ignored_events": HookdeckEndpointConfig(+ name="request_ignored_events",+ path="/requests/{request_id}/ignored_events",+ # Ignored event ids embed the request id, but the parent id is kept in the key so rows from+ # different requests can never collide.+ primary_keys=["request_id", "id"],+ # Full refresh only: the child has no date filter, and retrying an old request re-runs its+ # event logic and can add ignored events, so no parent creation window is safe either.+ description="Deliveries Hookdeck skipped for a request, one row per connection, with the cause (filtered, disabled, transformation failed or CLI disconnected)",+ fanout=DependentEndpointConfig(+ parent_name="requests",+ resolve_param="request_id",+ resolve_field="id",+ include_from_parent=[],+ ),+ # The fan-out checkpoint lists every completed parent, and this parent is every request in+ # the retention window, so checkpointing would cost more than restarting the run.+ checkpoint_fanout=False,+ ),+ "transformation_executions": HookdeckEndpointConfig(+ name="transformation_executions",+ path="/transformations/{transformation_id}/executions",+ primary_keys=["transformation_id", "id"],+ incremental_fields=[_datetime_field("created_at")],+ default_incremental_lookback_seconds=_FANOUT_LOOKBACK_SECONDS,+ description="Runs of each transformation against an event, with the log level, console output and any issue raised",+ fanout=DependentEndpointConfig(+ parent_name="transformations",+ resolve_param="transformation_id",+ resolve_field="id",+ include_from_parent=[],+ ),+ ),+ "queue_depth_metrics": HookdeckEndpointConfig(+ name="queue_depth_metrics",+ path="/metrics/queue-depth",+ primary_keys=["time_bucket", "destination_id"],+ incremental_fields=[_datetime_field("time_bucket")],+ partition_key="time_bucket",+ default_incremental_lookback_seconds=_METRICS_LOOKBACK_SECONDS,+ description="Hourly delivery backlog per destination: the most events pending and the longest an event waited",+ metrics=HookdeckMetricsConfig(measures=("max_depth", "max_age"), dimensions=("destination_id",)),+ ),+ "events_pending_timeseries": HookdeckEndpointConfig(+ name="events_pending_timeseries",+ path="/metrics/events-pending-timeseries",+ primary_keys=["time_bucket", "destination_id"],+ incremental_fields=[_datetime_field("time_bucket")],+ partition_key="time_bucket",+ default_incremental_lookback_seconds=_METRICS_LOOKBACK_SECONDS,+ description="Hourly count of events pending delivery per destination",+ metrics=HookdeckMetricsConfig(measures=("count",), dimensions=("destination_id",)),+ ),}ENDPOINTS = tuple(HOOKDECK_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.pyindex abc4535b9d3b..2e0f232927ee 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py@@ -16,10 +16,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_)from products.warehouse_sources.backend.temporal.data_imports.sources.hookdeck.hookdeck import (REDACTED,+ HookdeckMetricsTruncatedError,HookdeckResumeConfig,_format_datetime,_redact_secrets,base_url,+ build_metrics_params,build_params,validate_credentials,)@@ -34,8 +36,13 @@ SUPPORTED_VERSIONS = ["2025-07-01", "2026-09-01"]INCREMENTAL_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.incremental_fields)FULL_REFRESH_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if not c.incremental_fields)+PAGINATED_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.metrics is None)+TOP_LEVEL_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.metrics is None and c.fanout is None)ENDPOINT_FIELD_PAIRS = sorted(- (name, entry["field"]) for name, c in HOOKDECK_ENDPOINTS.items() for entry in c.incremental_fields+ (name, entry["field"])+ for name, c in HOOKDECK_ENDPOINTS.items()+ if c.metrics is None+ for entry in c.incremental_fields)@@ -81,7 +88,7 @@ class TestHookdeckTransport:assert result.requests[0].origin == "https://api.hookdeck.com"assert result.requests[0].path == f"/{api_version}/destinations"- @pytest.mark.parametrize("endpoint", sorted(HOOKDECK_ENDPOINTS))+ @pytest.mark.parametrize("endpoint", PAGINATED_ENDPOINTS)def test_every_request_pins_ascending_order(self, endpoint: str) -> None:# Hookdeck defaults to `dir=desc`; sort_mode="asc" would corrupt the watermark without this.params = build_params(@@ -319,7 +326,7 @@ class TestHookdeckSampleCapture:class TestHookdeckSourceResponse:- @pytest.mark.parametrize("endpoint", sorted(HOOKDECK_ENDPOINTS))+ @pytest.mark.parametrize("endpoint", TOP_LEVEL_ENDPOINTS)def test_source_response_shape(self, endpoint: str) -> None:result = _driver().run(endpoint, [_response([], None)])response = result.response@@ -333,3 +340,176 @@ class TestHookdeckSourceResponse:assert response.partition_mode == "datetime"assert response.partition_keys == ["created_at"]assert response.partition_format == "month"++ @pytest.mark.parametrize(+ "endpoint, script",+ [+ ("request_ignored_events", [_response([], None)]),+ ("transformation_executions", [_response([], None)]),+ ("queue_depth_metrics", [ScriptedResponse(status=200, json={"data": [], "metadata": {}})]),+ ("events_pending_timeseries", [ScriptedResponse(status=200, json={"data": [], "metadata": {}})]),+ ],+ )+ def test_unordered_reads_promise_no_sort(self, endpoint: str, script: list[ScriptedResponse]) -> None:+ # Fan-out rows are ordered per parent and metric points are unordered, so "asc" would lose rows.+ result = _driver().run(endpoint, script)++ assert result.raised is None+ assert result.response is not None+ assert result.response.sort_mode is None+++class TestHookdeckFanout:+ def test_ignored_events_fan_out_only_over_requests_that_ignored_something(self) -> None:+ result = _driver().run(+ "request_ignored_events",+ [+ _response(+ [{"id": "req_1", "ignored_count": 2}, {"id": "req_2", "ignored_count": 0}, {"id": "req_3"}],+ None,+ ),+ _response([{"id": "req_1:web_a", "request_id": "req_1"}], "cur_2"),+ _response([{"id": "req_1:web_b", "request_id": "req_1"}], None),+ _response([{"id": "req_3:web_a", "request_id": "req_3"}], None),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["req_1:web_a", "req_1:web_b", "req_3:web_a"]+ assert [request.path for request in result.requests] == [+ f"/{API_VERSION}/requests",+ f"/{API_VERSION}/requests/req_1/ignored_events",+ f"/{API_VERSION}/requests/req_1/ignored_events",+ f"/{API_VERSION}/requests/req_3/ignored_events",+ ]+ # A retried old request can gain ignored events, so the parent is never windowed by date.+ assert result.params("created_at[gte]") == [None, None, None, None]+ assert result.requests[2].param("next") == "cur_2"+ # One completed path per request would make every checkpoint grow with the request history.+ assert result.saved_states == []++ def test_transformation_executions_filter_each_child_not_the_parent(self) -> None:+ result = _driver().run(+ "transformation_executions",+ [+ _response([{"id": "trs_1", "env": {"KEY": "secret"}}, {"id": "trs_2"}], None),+ _response([{"id": "exe_1", "transformation_id": "trs_1"}], None),+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),+ ],+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC),+ incremental_field="created_at",+ )++ assert result.raised is None+ assert result.rows == [+ {"id": "exe_1", "transformation_id": "trs_1"},+ {"id": "exe_2", "transformation_id": "trs_2"},+ ]+ assert result.params("created_at[gte]") == [None, "2026-03-04T02:58:14.000Z", "2026-03-04T02:58:14.000Z"]+ assert result.params("dir") == ["asc", "asc", "asc"]++ def test_resume_skips_parents_already_completed(self) -> None:+ first = _driver().run(+ "transformation_executions",+ [+ _response([{"id": "trs_1"}, {"id": "trs_2"}], None),+ _response([{"id": "exe_1", "transformation_id": "trs_1"}], None),+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),+ ],+ )+ assert first.raised is None+ checkpoint = first.saved_states[0]+ assert checkpoint.next_cursor is None+ assert checkpoint.fanout_state is not None++ resumed = _driver().run(+ "transformation_executions",+ [+ _response([{"id": "trs_1"}, {"id": "trs_2"}], None),+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),+ ],+ resume_state=checkpoint,+ )++ assert resumed.raised is None+ assert [row["id"] for row in resumed.rows] == ["exe_2"]+ assert [request.path for request in resumed.requests] == [+ f"/{API_VERSION}/transformations",+ f"/{API_VERSION}/transformations/trs_2/executions",+ ]+++class TestHookdeckMetrics:+ NOW = datetime(2026, 3, 10, 12, 34, 56, 789000, tzinfo=UTC)++ @pytest.mark.parametrize(+ "should_use_incremental_field, last_value, expected_start",+ [+ # The watermark is floored to its hourly bucket so the bucket is re-read whole.+ (True, datetime(2026, 3, 9, 7, 45, 12, tzinfo=UTC), "2026-03-09T07:00:00.000Z"),+ (True, "2026-03-09T07:00:00.000Z", "2026-03-09T07:00:00.000Z"),+ (True, None, "2026-02-08T12:00:00.000Z"),+ (False, datetime(2026, 3, 9, 7, 45, 12, tzinfo=UTC), "2026-02-08T12:00:00.000Z"),+ ],+ )+ def test_date_range_starts_at_the_watermark_or_the_backfill_window(+ self, should_use_incremental_field: bool, last_value: object, expected_start: str+ ) -> None:+ metrics = HOOKDECK_ENDPOINTS["queue_depth_metrics"].metrics+ assert metrics is not None++ params = build_metrics_params(metrics, should_use_incremental_field, last_value, self.NOW)++ assert params["date_range[start]"] == expected_start+ assert params["date_range[end]"] == "2026-03-10T12:34:56.789Z"+ assert params["granularity"] == "1h"++ def test_query_uses_bracket_arrays_and_rows_are_flattened(self) -> None:+ result = _driver().run(+ "queue_depth_metrics",+ [+ ScriptedResponse(+ status=200,+ json={+ "data": [+ {+ "time_bucket": "2026-03-04T02:00:00.000Z",+ "dimensions": {"destination_id": "des_1"},+ "metrics": {"max_depth": 12, "max_age": 3.5},+ }+ ],+ "metadata": {"granularity": "1h", "truncated": False},+ },+ )+ ],+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC),+ incremental_field="time_bucket",+ )++ assert result.raised is None+ assert result.rows == [+ {"time_bucket": "2026-03-04T02:00:00.000Z", "destination_id": "des_1", "max_depth": 12, "max_age": 3.5}+ ]+ request = result.requests[0]+ assert request.path == f"/{API_VERSION}/metrics/queue-depth"+ assert request.query["measures[]"] == ("max_depth", "max_age")+ assert request.query["dimensions[]"] == ("destination_id",)+ assert request.param("date_range[start]") == "2026-03-04T02:00:00.000Z"++ def test_truncated_response_fails_the_sync(self) -> None:+ result = _driver().run(+ "events_pending_timeseries",+ [+ ScriptedResponse(+ status=200,+ json={+ "data": [{"time_bucket": "2026-03-04T02:00:00.000Z", "dimensions": {}, "metrics": {}}],+ "metadata": {"row_limit": 1000000, "truncated": True},+ },+ )+ ],+ )++ assert isinstance(result.raised, HookdeckMetricsTruncatedError)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.pyindex 0e2f8663e1de..95668f5debfd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py@@ -68,7 +68,10 @@ class TestHookdeckSource:assert schema.supports_incremental is True# Hookdeck restates rows as retries land, so a re-read window is merged, never appended.assert schema.supports_append is False- assert schema.default_incremental_lookback_seconds == 24 * 60 * 60+ assert (+ schema.default_incremental_lookback_seconds == HOOKDECK_ENDPOINTS[name].default_incremental_lookback_seconds+ )+ assert schema.default_incremental_lookback_secondsassert {f["field"] for f in schema.incremental_fields} == {f["field"] for f in HOOKDECK_ENDPOINTS[name].incremental_fields}