Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the PagerDuty source (#120290)

Tom Owers262b02a0cd77
diff --git a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
index 263f8c5cfbc6..c832115fe4f9 100644
--- a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
+++ b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
@@ -37,7 +37,6 @@ newsdata/newsdata.py::get_rows
noaa_cdo/noaa_cdo.py::noaa_cdo_source.items
okendo/okendo.py::okendo_source.items
okta/okta.py::okta_source.items
-pagerduty/pagerduty.py::pagerduty_source.items
paymongo/paymongo.py::pull_rows
pipeliner/pipeliner.py::pipeliner_source.items
plunk/plunk.py::plunk_source.items
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 69ba16a1f1d4..778648745598 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9079,10 +9079,10 @@ Today (9): `escalation_policies`, `incidents`, `log_entries`, `priorities`, `sch
Diffed against: <https://raw.githubusercontent.com/PagerDuty/api-schema/main/reference/REST/openapiv3.json>
-- [ ] `/oncalls` — who was on call for which escalation policy and when - PagerDuty's headline operational dataset (high)
-- [ ] `/incidents/{id}/alerts` — alert-level rows under each incident; the actual monitoring events, and where dedup/noise analysis happens (high)
-- [ ] `/teams/{id}/members` — membership lookup joining users to the teams we already sync (high)
-- [ ] `/change_events` — deploy/change events, the standard correlate-incidents-to-deploys join (high)
+- [x] `/oncalls` — who was on call for which escalation policy and when - PagerDuty's headline operational dataset (high)
+- [x] `/incidents/{id}/alerts` — alert-level rows under each incident; the actual monitoring events, and where dedup/noise analysis happens (high)
+- [x] `/teams/{id}/members` — membership lookup joining users to the teams we already sync (high)
+- [x] `/change_events` — deploy/change events, the standard correlate-incidents-to-deploys join (high)
- [ ] `/analytics/raw/incidents (POST)` — precomputed per-incident MTTA/MTTR, engaged seconds, and escalation counts (high)
- [ ] `/incidents/{id}/notes` — responder notes attached to incidents, for postmortem/context analysis (medium)
- [ ] `/schedules/{id}/users` — membership lookup resolving which users belong to each schedule we already sync (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/canonical_descriptions.py
index 26275f8a0003..744c02df916d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/canonical_descriptions.py
@@ -137,4 +137,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
integration_guide_url="URL of the vendor's PagerDuty integration guide.",
),
},
+ "incident_alerts": {
+ "description": "An alert grouped under an incident: one event from a monitoring tool, before de-duplication into the incident.",
+ "docs_url": "https://developer.pagerduty.com/api-reference",
+ "columns": _columns(
+ incident_id="ID of the incident the alert was listed under.",
+ created_at="Time at which the alert was first triggered.",
+ status="Current status of the alert: triggered or resolved.",
+ alert_key="The alert's de-duplication key.",
+ service="The service the alert belongs to.",
+ incident="The incident the alert is grouped under.",
+ suppressed="Whether the alert is suppressed. Suppressed alerts are not created with a parent incident.",
+ severity="Magnitude of the problem as reported by the monitoring tool: info, warning, error, or critical.",
+ integration="The integration that sent the alert.",
+ body="Data that describes the alert, including the details sent by the monitoring tool.",
+ first_trigger_log_entry="The log entry that recorded the first trigger of the alert.",
+ ),
+ },
+ "change_events": {
+ "description": "A change event, such as a deploy or a configuration change, sent to PagerDuty by an integration.",
+ "docs_url": "https://developer.pagerduty.com/api-reference",
+ "columns": _columns(
+ summary="Short text summary of the change.",
+ timestamp="Time at which the emitting tool detected or generated the change.",
+ source="Unique name of the location where the change occurred.",
+ services="Services the change event is associated with.",
+ integration="The integration that received the change event.",
+ routing_key="Integration key of the service integration that received the event.",
+ links="Links included with the change event.",
+ images="Images included with the change event.",
+ custom_details="Additional details about the change, as sent by the emitting tool.",
+ ),
+ },
+ "team_members": {
+ "description": "A membership row that joins a user to a team, with the user's role on that team.",
+ "docs_url": "https://developer.pagerduty.com/api-reference",
+ "columns": {
+ "team_id": "ID of the team.",
+ "user_id": "ID of the user who is a member of the team.",
+ "user": "Reference to the user who is a member of the team.",
+ "role": "The user's role on the team (e.g. manager, responder, observer).",
+ },
+ },
+ "oncalls": {
+ "description": "An on-call entry: a user who is on call for a level of an escalation policy at the time of the sync.",
+ "docs_url": "https://developer.pagerduty.com/api-reference",
+ "columns": {
+ "id": "Identifier built from the escalation policy, escalation level, user, schedule, and start time.",
+ "user_id": "ID of the user who is on call.",
+ "user": "Reference to the user who is on call.",
+ "escalation_policy_id": "ID of the escalation policy the user is on call for.",
+ "escalation_policy": "Reference to the escalation policy the user is on call for.",
+ "escalation_level": "The escalation level of the on-call user within the escalation policy.",
+ "schedule_id": "ID of the schedule that puts the user on call. Empty when the escalation rule targets the user directly.",
+ "schedule": "Reference to the schedule that puts the user on call, if any.",
+ "start": "Start of the on-call period. Empty for a permanent on-call.",
+ "end": "End of the on-call period. Empty when the user does not go off call.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/pagerduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/pagerduty.py
index 88d9027cb670..b1bd4ecd121f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/pagerduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/pagerduty.py
@@ -1,5 +1,5 @@
-import dataclasses
-from collections.abc import Iterator
+import functools
+from collections.abc import Callable, Iterable, Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import urlencode
@@ -8,12 +8,19 @@ from requests import Request, Response
from requests.exceptions import HTTPError
from structlog.types import FilteringBoundLogger
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
- rest_api_resource,
+ rest_api_resources,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ rename_parent_fields,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -39,10 +46,13 @@ MAX_OFFSET = 10_000
# status on that same endpoint is a genuine failure and must still raise.
+@frozen
class PagerDutyResumeConfig:
# Row offset of the next unfetched page — PagerDuty list endpoints paginate with limit/offset.
offset: int = 0
+ # Framework fan-out checkpoint ({"completed": [...], "current": ..., "child_state": ...}) for
+ # endpoints that read one child list per parent row.
+ fanout_state: Optional[dict[str, Any]] = None
def _format_incremental_value(value: Any) -> str:
@@ -138,8 +148,72 @@ def _non_secret_headers() -> dict[str, str]:
}
+def _reference_id(row: dict[str, Any], key: str) -> Optional[str]:
+ reference = row.get(key)
+ return reference.get("id") if isinstance(reference, dict) else None
+
+
+def _shape_oncall(row: dict[str, Any]) -> dict[str, Any]:
+ row["user_id"] = _reference_id(row, "user")
+ row["escalation_policy_id"] = _reference_id(row, "escalation_policy")
+ row["schedule_id"] = _reference_id(row, "schedule")
+ # An on-call entry has no id of its own. A user holds one escalation level of one policy through
+ # one schedule (none for a direct user target) for one shift, so those fields together name it.
+ key_parts = (row["escalation_policy_id"], row.get("escalation_level"), row["user_id"], row["schedule_id"])
+ row["id"] = ":".join("" if part is None else str(part) for part in (*key_parts, row.get("start")))
+ return row
+
+
+def _shape_team_member(row: dict[str, Any]) -> dict[str, Any]:
+ row["user_id"] = _reference_id(row, "user")
+ return row
+
+
+_ROW_SHAPERS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
+ "oncalls": _shape_oncall,
+ "team_members": _shape_team_member,
+}
+
+
+def _row_map(endpoint: str, config: PagerDutyEndpointConfig) -> Optional[Callable[[dict[str, Any]], dict[str, Any]]]:
+ rename_parent_id = (
+ rename_parent_fields(config.parent, {"id": config.parent_id_column})
+ if config.parent and config.parent_id_column
+ else None
+ )
+ shaper = _ROW_SHAPERS.get(endpoint)
+ if rename_parent_id is None and shaper is None:
+ return None
+
+ def row_map(row: dict[str, Any]) -> dict[str, Any]:
+ if rename_parent_id is not None:
+ row = rename_parent_id(row)
+ return shaper(row) if shaper is not None else row
+
+ return row_map
+
+
+def _list_params(config: PagerDutyEndpointConfig, should_use_incremental_field: bool) -> dict[str, Any]:
+ params: dict[str, Any] = {}
+ if config.supports_since:
+ # created_at is immutable, so an ascending sort means new rows append to the end and never
+ # shift pages we've already read. Sent on every sync (not just incremental ones) so full
+ # refreshes paginate over a stable ordering too.
+ params["sort_by"] = "created_at:asc"
+ if should_use_incremental_field:
+ # The framework injects the watermark as `since`; on the first incremental sync the value
+ # is None and the param is dropped, so we only bound the window once we have a cursor.
+ params["since"] = {
+ "type": "incremental",
+ "cursor_path": "created_at",
+ "initial_value": None,
+ "convert": _format_incremental_value_or_none,
+ }
+ return params
+
+
def _skip_when_plan_gated(
- items: Iterator[Any],
+ items: Iterable[Any],
endpoint: str,
plan_gated_feature: str,
plan_gated_status: int,
@@ -163,6 +237,9 @@ def validate_credentials(api_token: str, endpoint: Optional[str] = None) -> tupl
The caller decides how to treat 403 (valid token, missing scope for the probed endpoint).
"""
config = PAGERDUTY_ENDPOINTS.get(endpoint) if endpoint else None
+ if config is not None and config.parent is not None:
+ # A fan-out path carries an `{id}` placeholder. The parent list needs the same access scope.
+ config = PAGERDUTY_ENDPOINTS[config.parent]
path = config.path if config else "/users"
url = f"{PAGERDUTY_BASE_URL}{path}?{urlencode({'limit': 1})}"
@@ -194,21 +271,46 @@ def pagerduty_source(
) -> SourceResponse:
config: PagerDutyEndpointConfig = PAGERDUTY_ENDPOINTS[endpoint]
- params: dict[str, Any] = {}
- if config.supports_since:
- # created_at is immutable, so an ascending sort means new rows append to the end and never
- # shift pages we've already read. Sent on every sync (not just incremental ones) so full
- # refreshes paginate over a stable ordering too.
- params["sort_by"] = "created_at:asc"
- if should_use_incremental_field:
- # The framework injects the watermark as `since`; on the first incremental sync the value
- # is None and the param is dropped, so we only bound the window once we have a cursor.
- params["since"] = {
- "type": "incremental",
- "cursor_path": "created_at",
- "initial_value": None,
- "convert": _format_incremental_value_or_none,
- }
+ endpoint_resource: EndpointResource
+ resources: list[str | EndpointResource]
+ if config.parent is None:
+ endpoint_resource = {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": _list_params(config, should_use_incremental_field),
+ # A missing envelope key yields no rows and stops pagination — the API returning an
+ # unexpected shape isn't treated as fatal here (matches the prior data.get(key, [])).
+ "data_selector": config.envelope_key,
+ },
+ }
+ resources = [endpoint_resource]
+ else:
+ parent_config = PAGERDUTY_ENDPOINTS[config.parent]
+ endpoint_resource = {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": {"id": {"type": "resolve", "resource": config.parent, "field": "id"}},
+ "data_selector": config.envelope_key,
+ # A parent removed between the parent listing and its child fetch has no children left.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "include_from_parent": ["id"],
+ }
+ parent_resource: EndpointResource = {
+ "name": config.parent,
+ "endpoint": {
+ "path": parent_config.path,
+ "params": _list_params(parent_config, should_use_incremental_field=False),
+ "data_selector": parent_config.envelope_key,
+ },
+ }
+ resources = [parent_resource, endpoint_resource]
+
+ row_map = _row_map(endpoint, config)
+ if row_map is not None:
+ endpoint_resource["data_map"] = row_map
rest_config: RESTAPIConfig = {
"client": {
@@ -222,54 +324,62 @@ def pagerduty_source(
},
"paginator": PagerDutyPaginator(limit=PAGE_SIZE, maximum_offset=MAX_OFFSET),
},
- "resources": [
- {
- "name": endpoint,
- "endpoint": {
- "path": config.path,
- "params": params,
- # A missing envelope key yields no rows and stops pagination — the API returning an
- # unexpected shape isn't treated as fatal here (matches the prior data.get(key, [])).
- "data_selector": config.envelope_key,
- },
- }
- ],
+ "resources": resources,
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
- initial_paginator_state = {"offset": resume.offset}
+ initial_paginator_state = resume.fanout_state if config.parent else {"offset": resume.offset}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if not state:
+ return
+ if config.parent is not None:
+ resumable_source_manager.save_state(PagerDutyResumeConfig(fanout_state=state))
# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields the
# last page (merge dedupes) rather than skipping it.
- if state and state.get("offset") is not None:
+ elif state.get("offset") is not None:
resumable_source_manager.save_state(PagerDutyResumeConfig(offset=int(state["offset"])))
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- db_incremental_field_last_value if should_use_incremental_field else None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
+ resource = next(
+ built
+ for built in rest_api_resources(
+ rest_config,
+ team_id,
+ job_id,
+ db_incremental_field_last_value if should_use_incremental_field else None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ if built.name == endpoint
)
- plan_gated_feature = config.plan_gated_feature
- plan_gated_status = config.plan_gated_status
+ gated_resource = resource
+ if config.plan_gated_feature is not None and config.plan_gated_status is not None:
+ # The plan gate hands each page on unchanged and holds no rows. As a `Resource` it keeps the
+ # framework's checkpoints and safe points, which a plain generator wrapper would disable.
+ gated_resource = Resource(
+ functools.partial(
+ _skip_when_plan_gated,
+ resource,
+ endpoint,
+ config.plan_gated_feature,
+ config.plan_gated_status,
+ logger,
+ ),
+ name=endpoint,
+ hints=resource._hints,
+ )
- def items() -> Iterator[Any]:
- if plan_gated_feature is None or plan_gated_status is None:
- yield from resource
- return
- yield from _skip_when_plan_gated(iter(resource), endpoint, plan_gated_feature, plan_gated_status, logger)
+ def items() -> Iterable[Any]:
+ return gated_resource
return SourceResponse(
name=endpoint,
items=items,
- primary_keys=[config.primary_key],
+ primary_keys=config.primary_keys,
# We always request created_at ascending where a sort is available, and full-refresh endpoints
# replace wholesale, so ascending is correct everywhere.
sort_mode="asc",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/settings.py
index 5a6d19b178b2..1f559a6a64c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/settings.py
@@ -10,7 +10,7 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
class PagerDutyEndpointConfig:
path: str # Path under the API base URL, e.g. "/incidents"
envelope_key: str # Key the list of objects is wrapped under in the response body
- primary_key: str = "id"
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
partition_key: Optional[str] = None # Stable datetime field used to partition (never a mutable field)
incremental_fields: list[IncrementalField] = field(default_factory=list)
# True only when the endpoint exposes a server-side `since` filter on created_at AND
@@ -28,6 +28,11 @@ class PagerDutyEndpointConfig:
# still raise instead of being swallowed as an empty table.
plan_gated_feature: Optional[str] = None
plan_gated_status: Optional[int] = None
+ # Fan-out child: `parent` names the endpoint whose rows supply the `{id}` in `path`, and that id
+ # lands on each child row as `parent_id_column`. Child ids are only documented as unique within
+ # their parent, so the parent id belongs in the child's primary key.
+ parent: Optional[str] = None
+ parent_id_column: Optional[str] = None
_CREATED_AT_INCREMENTAL: list[IncrementalField] = [
@@ -53,6 +58,17 @@ PAGERDUTY_ENDPOINTS: dict[str, PagerDutyEndpointConfig] = {
# would be needed for true change capture.
supports_since=True,
),
+ "incident_alerts": PagerDutyEndpointConfig(
+ path="/incidents/{id}/alerts",
+ envelope_key="alerts",
+ primary_keys=["incident_id", "id"],
+ partition_key="created_at",
+ # Full refresh only: the child endpoint has no `since` filter, and an open incident keeps
+ # collecting alerts after it was created, so a window on the parent would miss them. The
+ # fan-out covers the incidents the parent listing returns, which is its default date range.
+ parent="incidents",
+ parent_id_column="incident_id",
+ ),
"log_entries": PagerDutyEndpointConfig(
path="/log_entries",
envelope_key="log_entries",
@@ -61,6 +77,13 @@ PAGERDUTY_ENDPOINTS: dict[str, PagerDutyEndpointConfig] = {
# we can't guarantee an ascending watermark. Ship full refresh rather than risk the
# cursor advancing past unread rows.
),
+ "change_events": PagerDutyEndpointConfig(
+ path="/change_events",
+ envelope_key="change_events",
+ partition_key="timestamp",
+ # `/change_events` accepts `since`/`until` but no sort param, and `timestamp` is set by the
+ # tool that emits the event, so a late-arriving event can predate the watermark. Full refresh.
+ ),
"services": PagerDutyEndpointConfig(
path="/services",
envelope_key="services",
@@ -77,6 +100,15 @@ PAGERDUTY_ENDPOINTS: dict[str, PagerDutyEndpointConfig] = {
plan_gated_feature="teams",
plan_gated_status=402,
),
+ "team_members": PagerDutyEndpointConfig(
+ path="/teams/{id}/members",
+ envelope_key="members",
+ primary_keys=["team_id", "user_id"],
+ plan_gated_feature="teams",
+ plan_gated_status=402,
+ parent="teams",
+ parent_id_column="team_id",
+ ),
"escalation_policies": PagerDutyEndpointConfig(
path="/escalation_policies",
envelope_key="escalation_policies",
@@ -85,6 +117,12 @@ PAGERDUTY_ENDPOINTS: dict[str, PagerDutyEndpointConfig] = {
path="/schedules",
envelope_key="schedules",
),
+ "oncalls": PagerDutyEndpointConfig(
+ path="/oncalls",
+ envelope_key="oncalls",
+ # Without `since`/`until` the API answers the on-call entries at the current time, so each
+ # sync replaces the table with who is on call now.
+ ),
"priorities": PagerDutyEndpointConfig(
path="/priorities",
envelope_key="priorities",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/tests/test_pagerduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/tests/test_pagerduty.py
index 3f25d25c2023..f175cef2f02f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/tests/test_pagerduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/tests/test_pagerduty.py
@@ -10,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pagerduty import (
@@ -75,6 +76,118 @@ class TestResume:
assert result.raised is None
assert result.params("offset") == [str(PAGE_SIZE)]
+ def test_fan_out_skips_parents_completed_before_resume(self) -> None:
+ result = _driver().run(
+ "incident_alerts",
+ route(
+ {
+ "/incidents": [ScriptedResponse(json={"incidents": [{"id": "P1"}, {"id": "P2"}], "more": False})],
+ "/incidents/P2/alerts": [ScriptedResponse(json={"alerts": [{"id": "A2"}], "more": False})],
+ }
+ ),
+ resume_state=PagerDutyResumeConfig(
+ fanout_state={"completed": ["/incidents/P1/alerts"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/incidents", "/incidents/P2/alerts"]
+ assert [row["id"] for row in result.rows] == ["A2"]
+ assert result.saved_states[-1].fanout_state["completed"] == ["/incidents/P1/alerts", "/incidents/P2/alerts"]
+
+
+class TestFanOut:
+ @pytest.mark.parametrize(
+ "endpoint,parent_path,parent_envelope,child_envelope,parent_column,expected_user_id",
+ [
+ ("incident_alerts", "/incidents", "incidents", "alerts", "incident_id", None),
+ ("team_members", "/teams", "teams", "members", "team_id", "U1"),
+ ],
+ )
+ def test_reads_each_parents_children_and_tags_the_parent_id(
+ self,
+ endpoint: str,
+ parent_path: str,
+ parent_envelope: str,
+ child_envelope: str,
+ parent_column: str,
+ expected_user_id: str | None,
+ ) -> None:
+ child_row = {"id": "C1", "user": {"id": "U1", "type": "user_reference"}}
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ parent_path: [
+ ScriptedResponse(json={parent_envelope: [{"id": "P1"}, {"id": "P2"}], "more": False})
+ ],
+ # P1 was removed after the parent listing; its 404 must not fail the sync.
+ f"{parent_path}/P1/{child_envelope}": [ScriptedResponse(status=404, json={"error": {}})],
+ f"{parent_path}/P2/{child_envelope}": [
+ ScriptedResponse(json={child_envelope: [child_row], "more": True}),
+ ScriptedResponse(json={child_envelope: [child_row], "more": False}),
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ parent_path,
+ f"{parent_path}/P1/{child_envelope}",
+ f"{parent_path}/P2/{child_envelope}",
+ f"{parent_path}/P2/{child_envelope}",
+ ]
+ assert result.params("offset") == ["0", "0", "0", str(PAGE_SIZE)]
+ assert [row[parent_column] for row in result.rows] == ["P2", "P2"]
+ assert all(f"_{parent_envelope}_id" not in row for row in result.rows)
+ assert [row.get("user_id") for row in result.rows] == [expected_user_id, expected_user_id]
+
+
+class TestOncalls:
+ @pytest.mark.parametrize(
+ "oncall,expected_id,expected_schedule_id",
+ [
+ (
+ {
+ "escalation_policy": {"id": "EP1"},
+ "escalation_level": 1,
+ "user": {"id": "U1"},
+ "schedule": {"id": "S1"},
+ "start": "2026-01-05T09:00:00Z",
+ "end": "2026-01-12T09:00:00Z",
+ },
+ "EP1:1:U1:S1:2026-01-05T09:00:00Z",
+ "S1",
+ ),
+ (
+ # A direct user target on an escalation rule: no schedule and a permanent on-call.
+ {
+ "escalation_policy": {"id": "EP1"},
+ "escalation_level": 2,
+ "user": {"id": "U2"},
+ "schedule": None,
+ "start": None,
+ "end": None,
+ },
+ "EP1:2:U2::",
+ None,
+ ),
+ ],
+ )
+ def test_rows_get_flat_reference_ids_and_a_derived_id(
+ self, oncall: dict[str, Any], expected_id: str, expected_schedule_id: str | None
+ ) -> None:
+ result = _driver().run("oncalls", [ScriptedResponse(json={"oncalls": [oncall], "more": False})])
+
+ assert result.raised is None
+ assert result.paths == ["/oncalls"]
+ [row] = result.rows
+ assert row["id"] == expected_id
+ assert row["escalation_policy_id"] == "EP1"
+ assert row["user_id"] == oncall["user"]["id"]
+ assert row["schedule_id"] == expected_schedule_id
+
class TestIncremental:
def test_incremental_endpoint_sends_since_and_sort(self) -> None:
@@ -108,6 +221,22 @@ class TestValidateCredentials:
assert status == expected_status
assert {request.path for request in network.requests_log} == {"/users"}
+ @pytest.mark.parametrize(
+ "endpoint,expected_path",
+ [
+ ("change_events", "/change_events"),
+ # A fan-out child path carries an `{id}` placeholder, so the probe reads the parent list.
+ ("incident_alerts", "/incidents"),
+ ("team_members", "/teams"),
+ ],
+ )
+ def test_probe_path(self, endpoint: str, expected_path: str) -> None:
+ with scripted_network(always(ScriptedResponse(status=200))) as network:
+ ok, _status, _error = validate_credentials("tok", endpoint)
+
+ assert ok is True
+ assert [request.path for request in network.requests_log] == [expected_path]
+
def test_transport_failure_returns_zero_status(self) -> None:
with scripted_network(always(ConnectionError("no network"))) as network:
ok, status, error = validate_credentials("tok")
@@ -120,24 +249,38 @@ class TestValidateCredentials:
class TestPlanGatedEndpoints:
@pytest.mark.parametrize(
- "endpoint,status_code,body,feature",
+ "endpoint,status_code,body,feature,expected_path",
[
# The statuses PagerDuty answers on an account whose plan lacks the feature. The 402
# body keeps PagerDuty's documented error code 2014, which the client appends to the
# raised message as "api error: code=2014".
- ("teams", 402, {"error": {"code": 2014, "message": "Required abilities are unavailable"}}, "teams"),
- ("priorities", 404, {"error": {"message": "Not Found"}}, "incident priorities"),
+ (
+ "teams",
+ 402,
+ {"error": {"code": 2014, "message": "Required abilities are unavailable"}},
+ "teams",
+ "/teams",
+ ),
+ ("priorities", 404, {"error": {"message": "Not Found"}}, "incident priorities", "/priorities"),
+ # The fan-out parent request fails first, inside the child resource.
+ (
+ "team_members",
+ 402,
+ {"error": {"code": 2014, "message": "Required abilities are unavailable"}},
+ "teams",
+ "/teams",
+ ),
],
)
def test_plan_gated_endpoint_syncs_no_rows_and_warns(
- self, endpoint: str, status_code: int, body: dict[str, Any], feature: str
+ self, endpoint: str, status_code: int, body: dict[str, Any], feature: str, expected_path: str
) -> None:
logger = MagicMock()
result = _driver().run(endpoint, [ScriptedResponse(status=status_code, json=body)], logger=logger)
assert result.raised is None
assert result.rows == []
- assert result.paths == [f"/{endpoint}"]
+ assert result.paths == [expected_path]
warning = logger.warning.call_args.args[0]
assert f"does not include {feature}" in warning
assert f"the {endpoint} table synced no rows" in warning