Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): add hookdeck ignored events, executions and queue metrics (#122048)

Tom Owers71d241511d55
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 89626e0a00c4..8259af668f71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6024,10 +6024,10 @@ Today (10): `attempts`, `bookmarks`, `connections`, `destinations`, `events`, `i
Diffed against: <https://hookdeck.com/docs/api/inspect.md>
-- [ ] `Request ignored events (GET /requests/{id}/ignored_events)` — explain filtered or deduplicated deliveries that never appear in the events table (high)
-- [ ] `Queue depth metrics (GET /metrics/queue-depth)` — track historical delivery backlogs and identify overloaded destinations (high)
-- [ ] `Events pending timeseries (GET /metrics/events-pending-timeseries)` — measure how pending deliveries accumulate and recover over time (medium)
-- [ ] `Transformation executions (GET /transformations/{id}/executions)` — analyze transformation failures, execution times, and their impact on event delivery (medium)
+- [x] `Request ignored events (GET /requests/{id}/ignored_events)` — explain filtered or deduplicated deliveries that never appear in the events table (high)
+- [x] `Queue depth metrics (GET /metrics/queue-depth)` — track historical delivery backlogs and identify overloaded destinations (high)
+- [x] `Events pending timeseries (GET /metrics/events-pending-timeseries)` — measure how pending deliveries accumulate and recover over time (medium)
+- [x] `Transformation executions (GET /transformations/{id}/executions)` — analyze transformation failures, execution times, and their impact on event delivery (medium)
- [ ] `Request, event, and attempt metrics (GET /metrics/requests; GET /metrics/events; GET /metrics/attempts)` — compare vendor aggregates for throughput, delivery outcomes, and response times across reporting windows (medium)
- [ ] `Bulk operation history (GET /bulk/events/retry; GET /bulk/events/cancel; GET /bulk/ignored-events/retry; GET /bulk/requests/retry; GET /bulk/requests/replay)` — audit retry, replay, and cancellation operations with their progress and affected delivery counts (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py
index 0aab4d76db0d..dfd66ee446a0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/canonical_descriptions.py
@@ -188,4 +188,53 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "Date the bookmark was created.",
},
},
+ "request_ignored_events": {
+ "description": "A delivery Hookdeck skipped for an inbound request: one row per request and connection whose rules kept the request from becoming an event.",
+ "docs_url": f"{_API_DOCS}/inspect#getrequestignoredevents",
+ "columns": {
+ "id": "ID of the ignored event, built from the request ID and the connection ID.",
+ "team_id": "ID of the Hookdeck project.",
+ "webhook_id": "ID of the associated connection (webhook).",
+ "cause": "Why no event was created: DISABLED, FILTERED, TRANSFORMATION_FAILED or CLI_DISCONNECTED.",
+ "request_id": "ID of the request that was ignored.",
+ "meta": "Detail for the cause: the request parts a filter rejected, or the transformation that failed.",
+ "created_at": "Date the ignored event was created.",
+ },
+ },
+ "transformation_executions": {
+ "description": "One run of a transformation against an event, with its log level and console output.",
+ "docs_url": f"{_API_DOCS}/configure#gettransformationexecutions",
+ "columns": {
+ "id": "ID of the execution.",
+ "transformation_id": "ID of the transformation that ran.",
+ "team_id": "ID of the Hookdeck project.",
+ "webhook_id": "ID of the connection the execution was run for.",
+ "original_event_data_id": "ID of the event data before the transformation.",
+ "transformed_event_data_id": "ID of the event data the transformation produced.",
+ "log_level": "Highest log level the execution emitted.",
+ "logs": "Console lines the transformation wrote, each with a type and a message.",
+ "issue_id": "ID of the issue raised for a failed execution.",
+ "updated_at": "Date the execution was last updated.",
+ "created_at": "Date the transformation ran.",
+ },
+ },
+ "queue_depth_metrics": {
+ "description": "Hourly delivery backlog per destination, from the queue depth metrics endpoint.",
+ "docs_url": f"{_API_DOCS}/inspect#queryqueuedepthmetrics",
+ "columns": {
+ "time_bucket": "Start of the hourly bucket.",
+ "destination_id": "ID of the destination.",
+ "max_depth": "Largest number of pending events sampled in the bucket.",
+ "max_age": "Longest time an event sat pending in the bucket, in minutes.",
+ },
+ },
+ "events_pending_timeseries": {
+ "description": "Hourly count of events pending delivery per destination.",
+ "docs_url": f"{_API_DOCS}/inspect#queryeventspendingtimeseriesmetrics",
+ "columns": {
+ "time_bucket": "Start of the hourly bucket.",
+ "destination_id": "ID of the destination.",
+ "count": "Number of events pending delivery in the bucket.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py
index 522b35dd8014..bf7430647c1b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/hookdeck.py
@@ -1,33 +1,46 @@
import dataclasses
-from collections.abc import Iterable, Iterator
-from datetime import UTC, date, datetime
+from collections.abc import Callable, Iterable, Iterator
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from requests import Response
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
+ SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.hookdeck.settings import (
HOOKDECK_API_HOST,
HOOKDECK_ENDPOINTS,
+ METRICS_BACKFILL_DAYS,
+ METRICS_GRANULARITY,
PAGE_SIZE,
HookdeckEndpointConfig,
+ HookdeckMetricsConfig,
)
[email protected](frozen=False)
class HookdeckResumeConfig:
# Opaque `pagination.next` cursor pointing at the page after the last one yielded.
- next_cursor: str
+ next_cursor: Optional[str] = None
+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished
+ # and where the current one stopped.
+ fanout_state: Optional[dict[str, Any]] = None
REDACTED = "***redacted***"
@@ -167,6 +180,70 @@ class HookdeckCursorPaginator(JSONResponseCursorPaginator):
self._has_next_page = False
+def _skip_requests_without_ignored_events(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
+ # Most requests ignore nothing, so only fan out where Hookdeck counted an ignored event. A null
+ # count is unknown, not zero, and is still read.
+ return [] if row.get("ignored_count") == 0 else row
+
+
+_PARENT_DATA_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any] | list[dict[str, Any]]]] = {
+ "request_ignored_events": _skip_requests_without_ignored_events,
+}
+
+
+def build_metrics_params(
+ metrics: HookdeckMetricsConfig,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ now: datetime,
+) -> dict[str, Any]:
+ start = parse_datetime_value(db_incremental_field_last_value) if should_use_incremental_field else None
+ if start is None:
+ start = now - timedelta(days=METRICS_BACKFILL_DAYS)
+ # Align to the bucket boundary so a re-read bucket covers the same hour it did before.
+ start = start.replace(minute=0, second=0, microsecond=0)
+
+ # The metrics endpoints take structured values in bracket notation; `[]` keeps a single
+ # dimension an array.
+ return {
+ "date_range[start]": _format_datetime(start),
+ "date_range[end]": _format_datetime(now),
+ "granularity": METRICS_GRANULARITY,
+ "measures[]": list(metrics.measures),
+ "dimensions[]": list(metrics.dimensions),
+ }
+
+
+class HookdeckMetricsTruncatedError(Exception):
+ pass
+
+
+class HookdeckMetricsPaginator(SinglePagePaginator):
+ """Fails a metrics query that hit Hookdeck's row limit.
+
+ A truncated response still carries data, and accepting it would let the watermark move past the
+ buckets Hookdeck left out.
+ """
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ super().update_state(response, data)
+ body = response.json()
+ metadata = body.get("metadata") if isinstance(body, dict) else None
+ if isinstance(metadata, dict) and metadata.get("truncated"):
+ raise HookdeckMetricsTruncatedError(
+ f"Hookdeck truncated the metrics response at {metadata.get('row_limit')} rows, so the sync stopped "
+ "rather than store incomplete buckets."
+ )
+
+
+def _flatten_metric_point(point: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "time_bucket": point.get("time_bucket"),
+ **(point.get("dimensions") or {}),
+ **(point.get("metrics") or {}),
+ }
+
+
def validate_credentials(api_key: str, api_version: str) -> tuple[bool, int | None]:
return validate_via_probe(
# capture=False: the probe hits `/sources`, whose rows carry verification secrets the
@@ -190,19 +267,44 @@ def hookdeck_source(
) -> SourceResponse:
config = HOOKDECK_ENDPOINTS[endpoint]
+ client_config: ClientConfig = {
+ "base_url": base_url(api_version),
+ # The API key rides the framework's bearer auth so it is redacted from logs and errors;
+ # only the non-secret Accept header is set here.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "bearer", "token": api_key},
+ # capture=False: raw responses reach the sampler before `_redact_pages` runs, and the
+ # name-based scrubbers don't recognise Hookdeck's secret containers (`auth_method`,
+ # `verification`, `env`) or raw inbound request payloads, so keep every response body
+ # out of shared HTTP sample storage. Requests are still metered and logged (key redacted).
+ "session": make_tracked_session(capture=False, redact_values=(api_key,)),
+ }
+
+ if config.metrics is not None:
+ return _metrics_source(
+ config,
+ config.metrics,
+ client_config,
+ team_id,
+ job_id,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ )
+ if config.fanout is not None:
+ return _fanout_source(
+ config,
+ config.fanout,
+ client_config,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ incremental_field,
+ )
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": base_url(api_version),
- # The API key rides the framework's bearer auth so it is redacted from logs and errors;
- # only the non-secret Accept header is set here.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "bearer", "token": api_key},
- # capture=False: raw responses reach the sampler before `_redact_pages` runs, and the
- # name-based scrubbers don't recognise Hookdeck's secret containers (`auth_method`,
- # `verification`, `env`) or raw inbound request payloads, so keep every response body
- # out of shared HTTP sample storage. Requests are still metered and logged (key redacted).
- "session": make_tracked_session(capture=False, redact_values=(api_key,)),
- },
+ "client": client_config,
"resource_defaults": {},
"resources": [
{
@@ -225,7 +327,7 @@ def hookdeck_source(
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
- if resume is not None:
+ if resume is not None and resume.next_cursor:
initial_paginator_state = {"cursor": resume.next_cursor}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
@@ -258,3 +360,103 @@ def hookdeck_source(
# Every request pins `dir=asc` on the same column the watermark tracks.
sort_mode="asc",
)
+
+
+def _fanout_source(
+ config: HookdeckEndpointConfig,
+ fanout: DependentEndpointConfig,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[HookdeckResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ incremental_field: Optional[str],
+) -> SourceResponse:
+ parent_params = build_params(HOOKDECK_ENDPOINTS[fanout.parent_name], False, None, None)
+ child_params = build_params(
+ config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
+ )
+
+ resume = (
+ resumable_source_manager.load_state()
+ if config.checkpoint_fanout and resumable_source_manager.can_resume()
+ else None
+ )
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(HookdeckResumeConfig(fanout_state=state))
+
+ resource = build_dependent_resource(
+ endpoint_configs=HOOKDECK_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=dataclasses.replace(fanout, parent_params=parent_params),
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ should_use_incremental_field=should_use_incremental_field,
+ incremental_field=incremental_field,
+ # The date filter is already in the static params above, so the framework's own cursor
+ # injection stays off; the child still merges on its primary key.
+ incremental_config_factory=lambda _field: None,
+ parent_endpoint_extra={"data_selector": "models", "paginator": HookdeckCursorPaginator()},
+ child_endpoint_extra={"data_selector": "models", "paginator": HookdeckCursorPaginator()},
+ child_params_extra=child_params,
+ parent_data_map=_PARENT_DATA_MAPS.get(config.name),
+ resume_hook=save_fanout_checkpoint if config.checkpoint_fanout else None,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ )
+
+ return SourceResponse(
+ name=config.name,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ # Rows arrive in order per parent only, not as one ordered series.
+ sort_mode=None,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ )
+
+
+def _metrics_source(
+ config: HookdeckEndpointConfig,
+ metrics: HookdeckMetricsConfig,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+) -> SourceResponse:
+ rest_config: RESTAPIConfig = {
+ "client": client_config,
+ "resource_defaults": {},
+ "resources": [
+ {
+ "name": config.name,
+ "endpoint": {
+ "path": config.path,
+ "params": build_metrics_params(
+ metrics, should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC)
+ ),
+ "data_selector": "data",
+ "paginator": HookdeckMetricsPaginator(),
+ },
+ }
+ ],
+ }
+ resource = rest_api_resource(rest_config, team_id, job_id, None).add_map(_flatten_metric_point)
+
+ return SourceResponse(
+ name=config.name,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ # Hookdeck documents no order for the data points of one query.
+ sort_mode=None,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py
index 9047367ce692..5560425d5cb1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/settings.py
@@ -1,6 +1,9 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# Hookdeck pins the API version as the first path segment. Omitting it tracks the latest version,
@@ -16,6 +19,18 @@ PAGE_SIZE = 250
# value they had when first imported.
_RESTATED_RECORD_LOOKBACK_SECONDS = 24 * 60 * 60
+# A fan-out run reads each parent's children at a different moment but stores one watermark for
+# the whole table, so the next run re-reads a trailing day to pick up children that landed under a
+# parent after it was read.
+_FANOUT_LOOKBACK_SECONDS = 24 * 60 * 60
+
+# Metrics are bucketed per hour, and the latest bucket is still filling when it is read.
+METRICS_GRANULARITY = "1h"
+_METRICS_LOOKBACK_SECONDS = 2 * 60 * 60
+# How far back a full refresh or a first incremental sync reads. Hookdeck documents no retention
+# window for metrics, so the backfill stays bounded rather than asking for all history.
+METRICS_BACKFILL_DAYS = 30
+
def _datetime_field(name: str) -> IncrementalField:
return {
@@ -26,7 +41,15 @@ def _datetime_field(name: str) -> IncrementalField:
}
-@dataclass
+@dataclass(frozen=True)
+class HookdeckMetricsConfig:
+ measures: tuple[str, ...]
+ # Each dimension becomes a column and part of the primary key, so only list dimensions that are
+ # never null on a row.
+ dimensions: tuple[str, ...]
+
+
+@dataclass(frozen=False)
class HookdeckEndpointConfig:
name: str
path: str
@@ -47,6 +70,17 @@ class HookdeckEndpointConfig:
# When set, those nested fields are masked before rows are written to the warehouse so a reader
# of the table can't recover credentials they don't have the Hookdeck key for.
contains_credentials: bool = False
+ page_size: int = PAGE_SIZE
+ # Child endpoints are read once per row of `fanout.parent_name`.
+ fanout: Optional[DependentEndpointConfig] = None
+ checkpoint_fanout: bool = True
+ # Aggregated time series from `/metrics/*`, read as one bucketed query over a date range
+ # instead of a paginated list.
+ metrics: Optional[HookdeckMetricsConfig] = None
+
+ @property
+ def default_incremental_field(self) -> Optional[str]:
+ return self.incremental_fields[0]["field"] if self.incremental_fields else None
HOOKDECK_ENDPOINTS: dict[str, HookdeckEndpointConfig] = {
@@ -118,6 +152,59 @@ HOOKDECK_ENDPOINTS: dict[str, HookdeckEndpointConfig] = {
path="/bookmarks",
description="Saved event payloads kept for replaying against a connection",
),
+ "request_ignored_events": HookdeckEndpointConfig(
+ name="request_ignored_events",
+ path="/requests/{request_id}/ignored_events",
+ # Ignored event ids embed the request id, but the parent id is kept in the key so rows from
+ # different requests can never collide.
+ primary_keys=["request_id", "id"],
+ # Full refresh only: the child has no date filter, and retrying an old request re-runs its
+ # event logic and can add ignored events, so no parent creation window is safe either.
+ description="Deliveries Hookdeck skipped for a request, one row per connection, with the cause (filtered, disabled, transformation failed or CLI disconnected)",
+ fanout=DependentEndpointConfig(
+ parent_name="requests",
+ resolve_param="request_id",
+ resolve_field="id",
+ include_from_parent=[],
+ ),
+ # The fan-out checkpoint lists every completed parent, and this parent is every request in
+ # the retention window, so checkpointing would cost more than restarting the run.
+ checkpoint_fanout=False,
+ ),
+ "transformation_executions": HookdeckEndpointConfig(
+ name="transformation_executions",
+ path="/transformations/{transformation_id}/executions",
+ primary_keys=["transformation_id", "id"],
+ incremental_fields=[_datetime_field("created_at")],
+ default_incremental_lookback_seconds=_FANOUT_LOOKBACK_SECONDS,
+ description="Runs of each transformation against an event, with the log level, console output and any issue raised",
+ fanout=DependentEndpointConfig(
+ parent_name="transformations",
+ resolve_param="transformation_id",
+ resolve_field="id",
+ include_from_parent=[],
+ ),
+ ),
+ "queue_depth_metrics": HookdeckEndpointConfig(
+ name="queue_depth_metrics",
+ path="/metrics/queue-depth",
+ primary_keys=["time_bucket", "destination_id"],
+ incremental_fields=[_datetime_field("time_bucket")],
+ partition_key="time_bucket",
+ default_incremental_lookback_seconds=_METRICS_LOOKBACK_SECONDS,
+ description="Hourly delivery backlog per destination: the most events pending and the longest an event waited",
+ metrics=HookdeckMetricsConfig(measures=("max_depth", "max_age"), dimensions=("destination_id",)),
+ ),
+ "events_pending_timeseries": HookdeckEndpointConfig(
+ name="events_pending_timeseries",
+ path="/metrics/events-pending-timeseries",
+ primary_keys=["time_bucket", "destination_id"],
+ incremental_fields=[_datetime_field("time_bucket")],
+ partition_key="time_bucket",
+ default_incremental_lookback_seconds=_METRICS_LOOKBACK_SECONDS,
+ description="Hourly count of events pending delivery per destination",
+ metrics=HookdeckMetricsConfig(measures=("count",), dimensions=("destination_id",)),
+ ),
}
ENDPOINTS = tuple(HOOKDECK_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py
index abc4535b9d3b..2e0f232927ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck.py
@@ -16,10 +16,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
)
from products.warehouse_sources.backend.temporal.data_imports.sources.hookdeck.hookdeck import (
REDACTED,
+ HookdeckMetricsTruncatedError,
HookdeckResumeConfig,
_format_datetime,
_redact_secrets,
base_url,
+ build_metrics_params,
build_params,
validate_credentials,
)
@@ -34,8 +36,13 @@ SUPPORTED_VERSIONS = ["2025-07-01", "2026-09-01"]
INCREMENTAL_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.incremental_fields)
FULL_REFRESH_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if not c.incremental_fields)
+PAGINATED_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.metrics is None)
+TOP_LEVEL_ENDPOINTS = sorted(name for name, c in HOOKDECK_ENDPOINTS.items() if c.metrics is None and c.fanout is None)
ENDPOINT_FIELD_PAIRS = sorted(
- (name, entry["field"]) for name, c in HOOKDECK_ENDPOINTS.items() for entry in c.incremental_fields
+ (name, entry["field"])
+ for name, c in HOOKDECK_ENDPOINTS.items()
+ if c.metrics is None
+ for entry in c.incremental_fields
)
@@ -81,7 +88,7 @@ class TestHookdeckTransport:
assert result.requests[0].origin == "https://api.hookdeck.com"
assert result.requests[0].path == f"/{api_version}/destinations"
- @pytest.mark.parametrize("endpoint", sorted(HOOKDECK_ENDPOINTS))
+ @pytest.mark.parametrize("endpoint", PAGINATED_ENDPOINTS)
def test_every_request_pins_ascending_order(self, endpoint: str) -> None:
# Hookdeck defaults to `dir=desc`; sort_mode="asc" would corrupt the watermark without this.
params = build_params(
@@ -319,7 +326,7 @@ class TestHookdeckSampleCapture:
class TestHookdeckSourceResponse:
- @pytest.mark.parametrize("endpoint", sorted(HOOKDECK_ENDPOINTS))
+ @pytest.mark.parametrize("endpoint", TOP_LEVEL_ENDPOINTS)
def test_source_response_shape(self, endpoint: str) -> None:
result = _driver().run(endpoint, [_response([], None)])
response = result.response
@@ -333,3 +340,176 @@ class TestHookdeckSourceResponse:
assert response.partition_mode == "datetime"
assert response.partition_keys == ["created_at"]
assert response.partition_format == "month"
+
+ @pytest.mark.parametrize(
+ "endpoint, script",
+ [
+ ("request_ignored_events", [_response([], None)]),
+ ("transformation_executions", [_response([], None)]),
+ ("queue_depth_metrics", [ScriptedResponse(status=200, json={"data": [], "metadata": {}})]),
+ ("events_pending_timeseries", [ScriptedResponse(status=200, json={"data": [], "metadata": {}})]),
+ ],
+ )
+ def test_unordered_reads_promise_no_sort(self, endpoint: str, script: list[ScriptedResponse]) -> None:
+ # Fan-out rows are ordered per parent and metric points are unordered, so "asc" would lose rows.
+ result = _driver().run(endpoint, script)
+
+ assert result.raised is None
+ assert result.response is not None
+ assert result.response.sort_mode is None
+
+
+class TestHookdeckFanout:
+ def test_ignored_events_fan_out_only_over_requests_that_ignored_something(self) -> None:
+ result = _driver().run(
+ "request_ignored_events",
+ [
+ _response(
+ [{"id": "req_1", "ignored_count": 2}, {"id": "req_2", "ignored_count": 0}, {"id": "req_3"}],
+ None,
+ ),
+ _response([{"id": "req_1:web_a", "request_id": "req_1"}], "cur_2"),
+ _response([{"id": "req_1:web_b", "request_id": "req_1"}], None),
+ _response([{"id": "req_3:web_a", "request_id": "req_3"}], None),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["req_1:web_a", "req_1:web_b", "req_3:web_a"]
+ assert [request.path for request in result.requests] == [
+ f"/{API_VERSION}/requests",
+ f"/{API_VERSION}/requests/req_1/ignored_events",
+ f"/{API_VERSION}/requests/req_1/ignored_events",
+ f"/{API_VERSION}/requests/req_3/ignored_events",
+ ]
+ # A retried old request can gain ignored events, so the parent is never windowed by date.
+ assert result.params("created_at[gte]") == [None, None, None, None]
+ assert result.requests[2].param("next") == "cur_2"
+ # One completed path per request would make every checkpoint grow with the request history.
+ assert result.saved_states == []
+
+ def test_transformation_executions_filter_each_child_not_the_parent(self) -> None:
+ result = _driver().run(
+ "transformation_executions",
+ [
+ _response([{"id": "trs_1", "env": {"KEY": "secret"}}, {"id": "trs_2"}], None),
+ _response([{"id": "exe_1", "transformation_id": "trs_1"}], None),
+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),
+ ],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC),
+ incremental_field="created_at",
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"id": "exe_1", "transformation_id": "trs_1"},
+ {"id": "exe_2", "transformation_id": "trs_2"},
+ ]
+ assert result.params("created_at[gte]") == [None, "2026-03-04T02:58:14.000Z", "2026-03-04T02:58:14.000Z"]
+ assert result.params("dir") == ["asc", "asc", "asc"]
+
+ def test_resume_skips_parents_already_completed(self) -> None:
+ first = _driver().run(
+ "transformation_executions",
+ [
+ _response([{"id": "trs_1"}, {"id": "trs_2"}], None),
+ _response([{"id": "exe_1", "transformation_id": "trs_1"}], None),
+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),
+ ],
+ )
+ assert first.raised is None
+ checkpoint = first.saved_states[0]
+ assert checkpoint.next_cursor is None
+ assert checkpoint.fanout_state is not None
+
+ resumed = _driver().run(
+ "transformation_executions",
+ [
+ _response([{"id": "trs_1"}, {"id": "trs_2"}], None),
+ _response([{"id": "exe_2", "transformation_id": "trs_2"}], None),
+ ],
+ resume_state=checkpoint,
+ )
+
+ assert resumed.raised is None
+ assert [row["id"] for row in resumed.rows] == ["exe_2"]
+ assert [request.path for request in resumed.requests] == [
+ f"/{API_VERSION}/transformations",
+ f"/{API_VERSION}/transformations/trs_2/executions",
+ ]
+
+
+class TestHookdeckMetrics:
+ NOW = datetime(2026, 3, 10, 12, 34, 56, 789000, tzinfo=UTC)
+
+ @pytest.mark.parametrize(
+ "should_use_incremental_field, last_value, expected_start",
+ [
+ # The watermark is floored to its hourly bucket so the bucket is re-read whole.
+ (True, datetime(2026, 3, 9, 7, 45, 12, tzinfo=UTC), "2026-03-09T07:00:00.000Z"),
+ (True, "2026-03-09T07:00:00.000Z", "2026-03-09T07:00:00.000Z"),
+ (True, None, "2026-02-08T12:00:00.000Z"),
+ (False, datetime(2026, 3, 9, 7, 45, 12, tzinfo=UTC), "2026-02-08T12:00:00.000Z"),
+ ],
+ )
+ def test_date_range_starts_at_the_watermark_or_the_backfill_window(
+ self, should_use_incremental_field: bool, last_value: object, expected_start: str
+ ) -> None:
+ metrics = HOOKDECK_ENDPOINTS["queue_depth_metrics"].metrics
+ assert metrics is not None
+
+ params = build_metrics_params(metrics, should_use_incremental_field, last_value, self.NOW)
+
+ assert params["date_range[start]"] == expected_start
+ assert params["date_range[end]"] == "2026-03-10T12:34:56.789Z"
+ assert params["granularity"] == "1h"
+
+ def test_query_uses_bracket_arrays_and_rows_are_flattened(self) -> None:
+ result = _driver().run(
+ "queue_depth_metrics",
+ [
+ ScriptedResponse(
+ status=200,
+ json={
+ "data": [
+ {
+ "time_bucket": "2026-03-04T02:00:00.000Z",
+ "dimensions": {"destination_id": "des_1"},
+ "metrics": {"max_depth": 12, "max_age": 3.5},
+ }
+ ],
+ "metadata": {"granularity": "1h", "truncated": False},
+ },
+ )
+ ],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC),
+ incremental_field="time_bucket",
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"time_bucket": "2026-03-04T02:00:00.000Z", "destination_id": "des_1", "max_depth": 12, "max_age": 3.5}
+ ]
+ request = result.requests[0]
+ assert request.path == f"/{API_VERSION}/metrics/queue-depth"
+ assert request.query["measures[]"] == ("max_depth", "max_age")
+ assert request.query["dimensions[]"] == ("destination_id",)
+ assert request.param("date_range[start]") == "2026-03-04T02:00:00.000Z"
+
+ def test_truncated_response_fails_the_sync(self) -> None:
+ result = _driver().run(
+ "events_pending_timeseries",
+ [
+ ScriptedResponse(
+ status=200,
+ json={
+ "data": [{"time_bucket": "2026-03-04T02:00:00.000Z", "dimensions": {}, "metrics": {}}],
+ "metadata": {"row_limit": 1000000, "truncated": True},
+ },
+ )
+ ],
+ )
+
+ assert isinstance(result.raised, HookdeckMetricsTruncatedError)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py
index 0e2f8663e1de..95668f5debfd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hookdeck/tests/test_hookdeck_source.py
@@ -68,7 +68,10 @@ class TestHookdeckSource:
assert schema.supports_incremental is True
# Hookdeck restates rows as retries land, so a re-read window is merged, never appended.
assert schema.supports_append is False
- assert schema.default_incremental_lookback_seconds == 24 * 60 * 60
+ assert (
+ schema.default_incremental_lookback_seconds == HOOKDECK_ENDPOINTS[name].default_incremental_lookback_seconds
+ )
+ assert schema.default_incremental_lookback_seconds
assert {f["field"] for f in schema.incremental_fields} == {
f["field"] for f in HOOKDECK_ENDPOINTS[name].incremental_fields
}