Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add raygun error instance, deployment and metrics tables (#126982)

Tom Owersa299185d166c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 1bb4d1ff58ab..0383874edcdc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10042,10 +10042,10 @@ Today (6): `applications`, `customers`, `deployments`, `error_groups`, `pages`,
Diffed against: <https://api.raygun.com/v3/raygun-openapi-spec.json>
-- [ ] `applications/{id}/error-groups/{id}/instances` — Individual error occurrences with stack trace, environment and user context - the event-level data under the groups we already sync (high)
-- [ ] `metrics/{application}/errors/time-series` — Raygun's headline error-rate metric, pre-aggregated over time (medium)
-- [ ] `applications/{id}/deployments/{id}/error-groups` — Maps error groups to the deployment that introduced them, the core regression-detection join (medium)
-- [ ] `metrics/{application}/pages/time-series and /pages/histogram` — Real user monitoring page-load performance over time and its distribution (medium)
+- [x] `applications/{id}/error-groups/{id}/instances` — Individual error occurrences with stack trace, environment and user context - the event-level data under the groups we already sync (high)
+- [x] `metrics/{application}/errors/time-series` — Raygun's headline error-rate metric, pre-aggregated over time (medium)
+- [x] `applications/{id}/deployments/{id}/error-groups` — Maps error groups to the deployment that introduced them, the core regression-detection join (medium)
+- [x] `metrics/{application}/pages/time-series and /pages/histogram` — Real user monitoring page-load performance over time and its distribution (medium) — time-series only; the histogram is a snapshot over a caller-chosen window with no time dimension, and the time-series percentiles cover the distribution over time
- [ ] `teams` — Lookup resolving team ownership of applications (low)
Note: Spec URL is not linked from the product docs page; found it by reading the Swagger UI bootstrap at https://api.raygun.com/v3/swagger/index.js. Coverage of the top-level collections is good - the gap is one level down, at error instances and the metrics endpoints.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/canonical_descriptions.py
index b96b20099103..7d6d141e7ced 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/canonical_descriptions.py
@@ -99,4 +99,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"lastSeenAt": "Timestamp when the page was most recently seen.",
},
},
+ "error_instances": {
+ "description": "A single occurrence of an error, listed under the error group it belongs to.",
+ "docs_url": "https://api.raygun.com/v3/swagger",
+ "columns": {
+ "identifier": "Identifier of the error instance.",
+ "occurredOn": "Timestamp when the error instance occurred.",
+ "applicationIdentifier": "Identifier of the application the error instance belongs to.",
+ "errorGroupIdentifier": "Identifier of the error group the error instance belongs to.",
+ },
+ },
+ "deployment_error_groups": {
+ "description": "An error group that is new in, or regressed in, a deployment of an application.",
+ "docs_url": "https://api.raygun.com/v3/swagger",
+ "columns": {
+ "identifier": "Unique identifier for the error group.",
+ "applicationIdentifier": "Identifier of the application the error group belongs to.",
+ "deploymentIdentifier": "Identifier of the deployment the error group is linked to.",
+ "deploymentState": "How the error group relates to the deployment: new (first seen in it) or regressed (returned after it was resolved).",
+ "message": "The error message representing the group.",
+ "status": "Current status of the error group (e.g. active, resolved, ignored).",
+ "lastOccurredAt": "Timestamp when the error group most recently occurred.",
+ "createdAt": "Timestamp when the error group was first created.",
+ },
+ },
+ "error_metrics": {
+ "description": "Hourly count of error instances for an application.",
+ "docs_url": "https://api.raygun.com/v3/swagger",
+ "columns": {
+ "applicationIdentifier": "Identifier of the application the metric belongs to.",
+ "metric": "Name of the metric (errorInstances).",
+ "aggregation": "Aggregation applied to the metric within each hour (count).",
+ "time": "Start of the hourly bucket.",
+ "value": "Aggregated metric value for the hour.",
+ },
+ },
+ "page_metrics": {
+ "description": "Hourly real user monitoring page metrics for an application: page views and page performance timings.",
+ "docs_url": "https://api.raygun.com/v3/swagger",
+ "columns": {
+ "applicationIdentifier": "Identifier of the application the metric belongs to.",
+ "metric": "Name of the metric, e.g. pageViews, loadTime or largestContentfulPaint.",
+ "aggregation": "Aggregation applied to the metric within each hour: count, average, median, p95 or p99.",
+ "time": "Start of the hourly bucket.",
+ "value": "Aggregated metric value for the hour. Timings are in milliseconds; cumulativeLayoutShift is a unitless score.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/raygun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/raygun.py
index f73ba16d77a8..06827ffa26b1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/raygun.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/raygun.py
@@ -1,4 +1,6 @@
import dataclasses
+from collections.abc import Iterator
+from datetime import UTC, datetime
from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -9,16 +11,26 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
rest_api_resource,
rest_api_resources,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (
+ create_response_hooks,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
+ SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.raygun.settings import (
+ METRICS_GRANULARITY,
+ METRICS_INITIAL_LOOKBACK,
+ METRICS_WINDOW,
PAGE_SIZE,
RAYGUN_BASE_URL,
RAYGUN_ENDPOINTS,
+ RaygunEndpointConfig,
)
# The `applications` response carries `apiKey` — the ingestion key for that application — which
@@ -27,7 +39,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.raygun.set
_APPLICATIONS_SENSITIVE_FIELDS = frozenset({"apiKey"})
[email protected](frozen=True)
class RaygunResumeConfig:
# Offset (rows to skip) for the next page of a top-level list endpoint.
offset: int = 0
@@ -37,6 +49,14 @@ class RaygunResumeConfig:
# Framework dependent-resource resume state for fan-out endpoints, of the shape
# ``{"completed": [child_path, ...], "current": child_path | None, "child_state": {...} | None}``.
fanout_state: dict[str, Any] | None = None
+ # Two-level fan-out endpoints (application -> parent -> child): the application being read, the
+ # parent page offset, the parent's index on that page, which state filter it is on, and the
+ # offset of the next child page.
+ nested_application_identifier: str | None = None
+ nested_parent_offset: int = 0
+ nested_parent_index: int = 0
+ nested_state_index: int = 0
+ nested_child_offset: int = 0
def _strip_application_api_key(row: dict[str, Any]) -> dict[str, Any]:
@@ -66,6 +86,244 @@ def _client_config(personal_access_token: str) -> ClientConfig:
}
+def _rest_client(personal_access_token: str) -> RESTClient:
+ return RESTClient(
+ base_url=RAYGUN_BASE_URL,
+ headers={"Accept": "application/json"},
+ auth=BearerTokenAuth(personal_access_token),
+ session=_make_session(personal_access_token),
+ )
+
+
+# An application, error group or deployment deleted after it was listed has nothing left to read.
+_SKIP_DELETED_PARENT = create_response_hooks([{"status_code": 404, "action": "ignore"}])
+
+
+def _parse_timestamp(value: Any) -> Optional[datetime]:
+ if value is None:
+ return None
+ if isinstance(value, datetime):
+ parsed = value
+ else:
+ try:
+ parsed = datetime.fromisoformat(str(value))
+ except ValueError:
+ return None
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
+
+
+def _format_timestamp(value: datetime) -> str:
+ return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _floor_to_hour(value: datetime) -> datetime:
+ return value.astimezone(UTC).replace(minute=0, second=0, microsecond=0)
+
+
+def _get_page(client: RESTClient, path: str, params: dict[str, Any]) -> list[dict[str, Any]]:
+ return next(
+ client.paginate(path=path, params=params, paginator=SinglePagePaginator(), hooks=_SKIP_DELETED_PARENT),
+ [],
+ )
+
+
+def _list_application_identifiers(
+ client: RESTClient, resumable_source_manager: ResumableSourceManager[RaygunResumeConfig]
+) -> list[str]:
+ identifiers: list[str] = []
+ offset = 0
+ while True:
+ page = _get_page(client, "/applications", {"orderby": "name", "offset": offset, "count": PAGE_SIZE})
+ identifiers.extend(row["identifier"] for row in page)
+ resumable_source_manager.safe_point()
+ if len(page) < PAGE_SIZE:
+ return identifiers
+ offset += PAGE_SIZE
+
+
+def _nested_child_pages(
+ client: RESTClient,
+ config: RaygunEndpointConfig,
+ path: str,
+ state_filter: Optional[str],
+ offset: int,
+ watermark: Optional[datetime],
+) -> Iterator[tuple[list[dict[str, Any]], Optional[int]]]:
+ """Yield each child page with the offset of the page after it, or None after the last page."""
+ while True:
+ params: dict[str, Any] = {"orderby": config.orderby, "offset": offset, "count": PAGE_SIZE}
+ if state_filter is not None:
+ params["filter"] = f"state eq {state_filter}"
+ rows = _get_page(client, path, params)
+ is_last = len(rows) < PAGE_SIZE
+ if watermark is not None and config.incremental_fields and rows:
+ # Children sort newest first, so a page that reaches below the watermark is the last new one.
+ oldest = _parse_timestamp(rows[-1].get(config.incremental_fields[0]["field"]))
+ is_last = is_last or (oldest is not None and oldest < watermark)
+ offset += PAGE_SIZE
+ yield rows, None if is_last else offset
+ if is_last:
+ return
+
+
+def _nested_fan_out_rows(
+ personal_access_token: str,
+ endpoint: str,
+ resumable_source_manager: ResumableSourceManager[RaygunResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+) -> Iterator[list[dict[str, Any]]]:
+ """Read a child listing for every parent row of every application.
+
+ The rest_source dependent-resource support covers one level of fan-out; these endpoints sit two
+ levels below the application list. Resume state points at the next child page, so a resumed
+ run reads no page twice.
+ """
+ config = RAYGUN_ENDPOINTS[endpoint]
+ parent = config.nested_parent
+ assert parent is not None
+ client = _rest_client(personal_access_token)
+ watermark = _parse_timestamp(db_incremental_field_last_value)
+ state_filters: tuple[Optional[str], ...] = config.state_filters or (None,)
+
+ application_identifiers = _list_application_identifiers(client, resumable_source_manager)
+
+ start_index, parent_offset, parent_index, state_index, child_offset = 0, 0, 0, 0, 0
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.nested_application_identifier in application_identifiers:
+ start_index = application_identifiers.index(resume.nested_application_identifier)
+ parent_offset = resume.nested_parent_offset
+ parent_index = resume.nested_parent_index
+ state_index = resume.nested_state_index
+ child_offset = resume.nested_child_offset
+
+ for application_identifier in application_identifiers[start_index:]:
+ parent_path = parent.path.format(application_identifier=application_identifier)
+ while True:
+ parents = _get_page(
+ client, parent_path, {"orderby": parent.orderby, "offset": parent_offset, "count": PAGE_SIZE}
+ )
+ for index in range(parent_index, len(parents)):
+ parent_row = parents[index]
+ parent_identifier = parent_row["identifier"]
+ if watermark is not None and parent.skip_below_watermark_field is not None:
+ last_activity = _parse_timestamp(parent_row.get(parent.skip_below_watermark_field))
+ if last_activity is not None and last_activity < watermark:
+ # Advance the cursor past it, so a resume after many quiet parents skips them.
+ resumable_source_manager.save_state(
+ RaygunResumeConfig(
+ nested_application_identifier=application_identifier,
+ nested_parent_offset=parent_offset,
+ nested_parent_index=index + 1,
+ )
+ )
+ state_index, child_offset = 0, 0
+ resumable_source_manager.safe_point()
+ continue
+
+ child_path = config.path.format(
+ application_identifier=application_identifier, parent_identifier=parent_identifier
+ )
+ for current_state_index in range(state_index, len(state_filters)):
+ state_filter = state_filters[current_state_index]
+ for rows, next_offset in _nested_child_pages(
+ client, config, child_path, state_filter, child_offset, watermark
+ ):
+ if next_offset is not None:
+ position = (index, current_state_index, next_offset)
+ elif current_state_index + 1 < len(state_filters):
+ position = (index, current_state_index + 1, 0)
+ else:
+ position = (index + 1, 0, 0)
+ resumable_source_manager.save_state(
+ RaygunResumeConfig(
+ nested_application_identifier=application_identifier,
+ nested_parent_offset=parent_offset,
+ nested_parent_index=position[0],
+ nested_state_index=position[1],
+ nested_child_offset=position[2],
+ )
+ )
+ if rows:
+ yield [
+ {
+ **row,
+ "applicationIdentifier": application_identifier,
+ parent.identifier_column: parent_identifier,
+ **({config.state_column: state_filter} if config.state_column else {}),
+ }
+ for row in rows
+ ]
+ else:
+ resumable_source_manager.safe_point()
+ child_offset = 0
+ state_index, child_offset = 0, 0
+ resumable_source_manager.safe_point()
+
+ if len(parents) < PAGE_SIZE:
+ break
+ parent_offset += PAGE_SIZE
+ parent_index = 0
+ parent_offset, parent_index, state_index, child_offset = 0, 0, 0, 0
+
+
+def _metrics_rows(
+ personal_access_token: str,
+ endpoint: str,
+ resumable_source_manager: ResumableSourceManager[RaygunResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+) -> Iterator[list[dict[str, Any]]]:
+ """Read hourly metric series for every application, one row per metric, aggregation and hour.
+
+ Each run re-reads from the watermark (or the initial lookback) to now. That is a few requests
+ per application, so these endpoints keep no resume state.
+ """
+ config = RAYGUN_ENDPOINTS[endpoint]
+ client = _rest_client(personal_access_token)
+ end = _floor_to_hour(datetime.now(UTC))
+ watermark = _parse_timestamp(db_incremental_field_last_value)
+ start = _floor_to_hour(watermark if watermark is not None else end - METRICS_INITIAL_LOOKBACK)
+
+ for application_identifier in _list_application_identifiers(client, resumable_source_manager):
+ path = config.path.format(application_identifier=application_identifier)
+ window_start = start
+ while window_start < end:
+ window_end = min(window_start + METRICS_WINDOW, end)
+ for request in config.metrics_requests:
+ body = {
+ "start": _format_timestamp(window_start),
+ "end": _format_timestamp(window_end),
+ "granularity": METRICS_GRANULARITY,
+ "aggregation": request.aggregation,
+ "metrics": list(request.metrics),
+ }
+ series_list = next(
+ client.paginate(
+ path=path,
+ method="post",
+ json=body,
+ paginator=SinglePagePaginator(),
+ hooks=_SKIP_DELETED_PARENT,
+ ),
+ [],
+ )
+ rows = [
+ {
+ "applicationIdentifier": application_identifier,
+ "metric": series.get("metric"),
+ "aggregation": series.get("aggregation"),
+ "time": point.get("time"),
+ "value": point.get("value"),
+ }
+ for series in series_list
+ for point in series.get("series") or []
+ ]
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+ window_start = window_end
+
+
def validate_token(personal_access_token: str) -> tuple[bool, int | None]:
"""Probe the token against the cheapest scoped endpoint. Returns (is_valid, status_code).
@@ -198,7 +456,16 @@ def raygun_source(
) -> SourceResponse:
config = RAYGUN_ENDPOINTS[endpoint]
- if config.fan_out_over_applications:
+ resource: Any
+ if config.metrics_requests:
+ resource = _metrics_rows(
+ personal_access_token, endpoint, resumable_source_manager, db_incremental_field_last_value
+ )
+ elif config.nested_parent is not None:
+ resource = _nested_fan_out_rows(
+ personal_access_token, endpoint, resumable_source_manager, db_incremental_field_last_value
+ )
+ elif config.fan_out_over_applications:
resource = _fan_out_source(
personal_access_token,
endpoint,
@@ -224,4 +491,6 @@ def raygun_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
+ # Metrics endpoints save no cursor, so a handed-off run would start over rather than resume.
+ supports_resume=not config.metrics_requests,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/settings.py
index a75db294e5eb..318894a47718 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/settings.py
@@ -1,7 +1,8 @@
from dataclasses import dataclass, field
+from datetime import timedelta
from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# Raygun's data API is a single global host (no regional variants) exposing v3 REST/JSON.
RAYGUN_BASE_URL = "https://api.raygun.com/v3"
@@ -10,8 +11,34 @@ RAYGUN_BASE_URL = "https://api.raygun.com/v3"
# responses small while limiting the number of round trips.
PAGE_SIZE = 100
+# The metrics endpoints return at most 4,320 points per request. Hourly buckets over 30-day windows
+# stay well under that, and requests start on an hour boundary so bucket times are stable across syncs.
+METRICS_GRANULARITY = "1h"
+METRICS_WINDOW = timedelta(days=30)
+METRICS_INITIAL_LOOKBACK = timedelta(days=30)
+# Re-read the last few hourly buckets on each incremental sync, because late-arriving data restates them.
+METRICS_INCREMENTAL_LOOKBACK_SECONDS = 3 * 60 * 60
-@dataclass
+
+@dataclass(frozen=True)
+class RaygunNestedParent:
+ # Per-application listing whose rows each own a child listing (two-level fan-out).
+ path: str
+ orderby: str
+ # Column each child row gets with the parent's identifier, since child rows don't carry it.
+ identifier_column: str
+ # Parent field compared against the incremental watermark: a parent older than the watermark
+ # has no new child rows, so its child listing is not requested.
+ skip_below_watermark_field: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class RaygunMetricsRequest:
+ aggregation: str
+ metrics: tuple[str, ...]
+
+
+@dataclass(frozen=True)
class RaygunEndpointConfig:
name: str
# Path relative to RAYGUN_BASE_URL. Fan-out children carry a `{application_identifier}`
@@ -20,8 +47,9 @@ class RaygunEndpointConfig:
primary_keys: list[str]
# `orderby` value passed on every request. Offset pagination over data that is being
# inserted concurrently can skip or duplicate rows, so we always sort by a stable
- # (immutable or create-time) field to keep page boundaries deterministic.
- orderby: str
+ # (immutable or create-time) field to keep page boundaries deterministic. Unset for the
+ # metrics endpoints, which aren't paginated.
+ orderby: Optional[str] = None
# Stable create-time field used for datetime partitioning. Only set when the field is
# guaranteed present on every row (in the API's `required` set) — never `lastSeenAt`/
# `updatedAt`, which move over time and would rewrite partitions each sync.
@@ -30,12 +58,21 @@ class RaygunEndpointConfig:
# endpoint is a single top-level list.
fan_out_over_applications: bool = False
incremental_fields: list[IncrementalField] = field(default_factory=list)
+ # Set for child endpoints under a per-application parent listing; the child `path` then also
+ # carries a `{parent_identifier}` placeholder.
+ nested_parent: Optional[RaygunNestedParent] = None
+ # Values for the child listing's `state eq <value>` filter. Each value is requested separately
+ # and written to `state_column`, because the API returns the state only through the filter.
+ state_filters: tuple[str, ...] = ()
+ state_column: Optional[str] = None
+ # Set for the POST time-series metrics endpoints: one request per entry per time window.
+ metrics_requests: tuple[RaygunMetricsRequest, ...] = ()
-# Raygun exposes no server-side "updated since" filter on any list endpoint (only offset/count
-# pagination plus an `orderby`), so every endpoint syncs as a full refresh — declaring an
-# incremental field here would fetch every page anyway, giving no cost saving. See the source
-# docstring and PR notes.
+# Raygun list endpoints have no server-side "updated since" filter (only offset/count pagination
+# plus an `orderby`), so they sync as a full refresh. The exceptions: the metrics endpoints take a
+# time range, and error instances sort newest first under error groups that report when they last
+# occurred, so an incremental sync stops at the watermark instead of reading all history.
RAYGUN_ENDPOINTS: dict[str, RaygunEndpointConfig] = {
"applications": RaygunEndpointConfig(
name="applications",
@@ -80,8 +117,94 @@ RAYGUN_ENDPOINTS: dict[str, RaygunEndpointConfig] = {
orderby="uri",
fan_out_over_applications=True,
),
+ "error_instances": RaygunEndpointConfig(
+ name="error_instances",
+ path="/applications/{application_identifier}/error-groups/{parent_identifier}/instances",
+ primary_keys=["applicationIdentifier", "errorGroupIdentifier", "identifier"],
+ orderby="occurredOn desc",
+ partition_key="occurredOn",
+ nested_parent=RaygunNestedParent(
+ path="/applications/{application_identifier}/error-groups",
+ orderby="createdAt",
+ identifier_column="errorGroupIdentifier",
+ skip_below_watermark_field="lastOccurredAt",
+ ),
+ incremental_fields=[
+ {
+ "label": "occurredOn",
+ "type": IncrementalFieldType.DateTime,
+ "field": "occurredOn",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ ),
+ "deployment_error_groups": RaygunEndpointConfig(
+ name="deployment_error_groups",
+ path="/applications/{application_identifier}/deployments/{parent_identifier}/error-groups",
+ primary_keys=["applicationIdentifier", "deploymentIdentifier", "identifier"],
+ orderby="createdAt",
+ partition_key="createdAt",
+ nested_parent=RaygunNestedParent(
+ path="/applications/{application_identifier}/deployments",
+ orderby="deployedAt",
+ identifier_column="deploymentIdentifier",
+ ),
+ # `stillOccurring` is left out: it repeats every active error group for every deployment,
+ # which costs the most requests and says nothing about what the deployment changed.
+ state_filters=("new", "regressed"),
+ state_column="deploymentState",
+ ),
+ "error_metrics": RaygunEndpointConfig(
+ name="error_metrics",
+ path="/metrics/{application_identifier}/errors/time-series",
+ primary_keys=["applicationIdentifier", "metric", "aggregation", "time"],
+ partition_key="time",
+ metrics_requests=(RaygunMetricsRequest(aggregation="count", metrics=("errorInstances",)),),
+ incremental_fields=[
+ {
+ "label": "time",
+ "type": IncrementalFieldType.DateTime,
+ "field": "time",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ ),
+ "page_metrics": RaygunEndpointConfig(
+ name="page_metrics",
+ path="/metrics/{application_identifier}/pages/time-series",
+ primary_keys=["applicationIdentifier", "metric", "aggregation", "time"],
+ partition_key="time",
+ metrics_requests=(
+ RaygunMetricsRequest(aggregation="count", metrics=("pageViews",)),
+ *(
+ RaygunMetricsRequest(
+ aggregation=aggregation,
+ metrics=(
+ "loadTime",
+ "firstPaint",
+ "firstContentfulPaint",
+ "largestContentfulPaint",
+ "firstInputDelay",
+ "cumulativeLayoutShift",
+ "interactionToNextPaint",
+ ),
+ )
+ for aggregation in ("average", "median", "p95", "p99")
+ ),
+ ),
+ incremental_fields=[
+ {
+ "label": "time",
+ "type": IncrementalFieldType.DateTime,
+ "field": "time",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ ),
}
+METRICS_ENDPOINTS = tuple(name for name, config in RAYGUN_ENDPOINTS.items() if config.metrics_requests)
+
ENDPOINTS = tuple(RAYGUN_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/source.py
index ed8cf003ce97..2b9992c85563 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/source.py
@@ -30,6 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.raygun.ray
from products.warehouse_sources.backend.temporal.data_imports.sources.raygun.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ METRICS_ENDPOINTS,
+ METRICS_INCREMENTAL_LOOKBACK_SECONDS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -42,6 +44,10 @@ Grant the token these read scopes for the tables you want to sync:
- `customers:read` — Customers
- `rum.sessions:read` — Sessions
- `rum.pages:read` — Pages
+- `cr.errors:read` — Error instances
+- `deployments:read` and `cr.errors:read` — Deployment error groups
+- `metrics:errors` — Error metrics
+- `metrics:pages` — Page metrics
"""
@@ -128,9 +134,15 @@ class RaygunSource(ResumableSource[RaygunSourceConfig, RaygunResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Raygun exposes no server-side "updated since" filter, so every endpoint is full refresh —
- # no endpoint declares incremental fields, so none support incremental/append sync.
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # Incremental endpoints re-read rows at the watermark (metric buckets, the last page of error
+ # instances), so append would duplicate them; they merge on their primary keys instead.
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ merge_only=[name for name, fields in INCREMENTAL_FIELDS.items() if fields],
+ default_incremental_lookback_seconds=dict.fromkeys(METRICS_ENDPOINTS, METRICS_INCREMENTAL_LOOKBACK_SECONDS),
+ )
def validate_credentials(
self,
@@ -170,7 +182,9 @@ class RaygunSource(ResumableSource[RaygunSourceConfig, RaygunResumeConfig]):
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
- db_incremental_field_last_value=None, # every Raygun endpoint is full refresh
+ db_incremental_field_last_value=(
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
+ ),
)
@property
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/tests/test_raygun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/tests/test_raygun.py
index 1c8bf8963b17..2812c87d6802 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/tests/test_raygun.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/raygun/tests/test_raygun.py
@@ -1,5 +1,7 @@
+from datetime import UTC, datetime, timedelta
from typing import Any
+import pytest
from unittest.mock import patch
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
@@ -97,6 +99,132 @@ class TestFanOutPagination:
assert child_requests[0].param("offset") == "6"
+class TestNestedFanOut:
+ @patch(f"{MODULE}.PAGE_SIZE", 2)
+ def test_incremental_skips_quiet_groups_and_stops_paging_at_the_watermark(self) -> None:
+ groups = [
+ {"identifier": "eg-old", "lastOccurredAt": "2026-01-01T00:00:00Z"},
+ {"identifier": "eg-new", "lastOccurredAt": "2026-03-01T00:00:00Z"},
+ ]
+ instances = {
+ "0": [
+ {"identifier": "i-3", "occurredOn": "2026-03-01T00:00:00Z"},
+ {"identifier": "i-2", "occurredOn": "2026-02-20T00:00:00Z"},
+ ],
+ "2": [
+ {"identifier": "i-1", "occurredOn": "2026-02-10T00:00:00Z"},
+ {"identifier": "i-0", "occurredOn": "2026-01-10T00:00:00Z"},
+ ],
+ }
+
+ def script(request: Any) -> ScriptedResponse:
+ offset = request.param("offset") or "0"
+ if request.path.endswith("/applications"):
+ return ScriptedResponse(json=[{"identifier": "app-1"}] if offset == "0" else [])
+ if request.path.endswith("/error-groups"):
+ return ScriptedResponse(json=groups if offset == "0" else [])
+ assert "/error-groups/eg-new/instances" in request.path
+ return ScriptedResponse(json=instances.get(offset, []))
+
+ result = _run(
+ "error_instances",
+ script,
+ incremental_field="occurredOn",
+ db_incremental_field_last_value=datetime(2026, 2, 1, tzinfo=UTC),
+ )
+
+ instance_requests = [r for r in result.requests if r.path.endswith("/instances")]
+ # The group last seen before the watermark is never read, and paging stops on the page that
+ # reaches below the watermark instead of reading the group's whole history.
+ assert [r.param("offset") for r in instance_requests] == ["0", "2"]
+ assert all(r.param("orderby") == "occurredOn desc" for r in instance_requests)
+ assert [row["identifier"] for row in result.rows] == ["i-3", "i-2", "i-1", "i-0"]
+ assert {(row["applicationIdentifier"], row["errorGroupIdentifier"]) for row in result.rows} == {
+ ("app-1", "eg-new")
+ }
+ # Skipping the quiet group still advances the cursor, so a resume does not re-check it.
+ assert result.saved_states[0] == RaygunResumeConfig(
+ nested_application_identifier="app-1", nested_parent_index=1
+ )
+
+ @patch(f"{MODULE}.PAGE_SIZE", 2)
+ def test_resume_continues_at_the_saved_state_filter_and_child_offset(self) -> None:
+ def script(request: Any) -> ScriptedResponse:
+ offset = request.param("offset") or "0"
+ if request.path.endswith("/applications"):
+ return ScriptedResponse(
+ json=[{"identifier": "app-1"}, {"identifier": "app-2"}] if offset == "0" else []
+ )
+ if request.path.endswith("/deployments"):
+ return ScriptedResponse(json=[{"identifier": "d-1"}] if offset == "0" else [])
+ return ScriptedResponse(json=[{"identifier": f"eg-{offset}", "applicationIdentifier": "app-2"}])
+
+ result = _run(
+ "deployment_error_groups",
+ script,
+ resume_state=RaygunResumeConfig(
+ nested_application_identifier="app-2", nested_state_index=1, nested_child_offset=4
+ ),
+ )
+
+ child_requests = [r for r in result.requests if r.path.endswith("/error-groups")]
+ # app-1 and the `new` filter of app-2 were done before the interruption.
+ assert [(r.path, r.param("filter"), r.param("offset")) for r in child_requests] == [
+ ("/v3/applications/app-2/deployments/d-1/error-groups", "state eq regressed", "4")
+ ]
+ assert result.rows == [
+ {
+ "identifier": "eg-4",
+ "applicationIdentifier": "app-2",
+ "deploymentIdentifier": "d-1",
+ "deploymentState": "regressed",
+ }
+ ]
+ assert result.final_state == RaygunResumeConfig(nested_application_identifier="app-2", nested_parent_index=1)
+
+
+class TestMetrics:
+ @pytest.mark.parametrize("incremental", [True, False])
+ def test_request_window_and_row_shape(self, incremental: bool) -> None:
+ def script(request: Any) -> ScriptedResponse:
+ if request.path.endswith("/applications"):
+ return ScriptedResponse(json=[{"identifier": "app-1"}] if request.param("offset") == "0" else [])
+ return ScriptedResponse(
+ json=[
+ {
+ "aggregation": "count",
+ "metric": "errorInstances",
+ "series": [{"time": "2026-10-31T10:00:00Z", "value": 3}],
+ }
+ ]
+ )
+
+ watermark = datetime.now(UTC) - timedelta(days=2, minutes=17)
+ inputs: dict[str, Any] = (
+ {"incremental_field": "time", "db_incremental_field_last_value": watermark} if incremental else {}
+ )
+ result = _run("error_metrics", script, **inputs)
+
+ bodies = [r.json() for r in result.requests if r.method == "POST"]
+ assert len(bodies) == 1
+ start = datetime.fromisoformat(bodies[0]["start"])
+ end = datetime.fromisoformat(bodies[0]["end"])
+ expected_start = watermark if incremental else end - timedelta(days=30)
+ assert start == expected_start.replace(minute=0, second=0, microsecond=0)
+ assert (end.minute, end.second) == (0, 0)
+ assert bodies[0]["granularity"] == "1h"
+ assert bodies[0]["metrics"] == ["errorInstances"]
+ assert result.rows == [
+ {
+ "applicationIdentifier": "app-1",
+ "metric": "errorInstances",
+ "aggregation": "count",
+ "time": "2026-10-31T10:00:00Z",
+ "value": 3,
+ }
+ ]
+
+
class TestRaygunSource:
def test_primary_keys_and_partitioning_per_endpoint(self) -> None:
def response_of(endpoint: str) -> Any: