Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Squadcast source (#120675)

Tom Owersbd3db38abf3d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a13a6718c9e4..0f2311f0d22f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11605,8 +11605,8 @@ Today (9): `escalation_policies`, `incidents`, `postmortems`, `runbooks`, `sched
Diffed against: <https://developers.incidents.cloud.solarwinds.com/llms.txt>
- [ ] `incidents/{incidentID}/events (Get Incident Events)` — the incident timeline - acknowledge/reassign/resolve state transitions behind MTTA and MTTR (high)
-- [ ] `squads` — lookup plus membership table resolving squad ids on incidents, services and escalation policies (high)
-- [ ] `schedules/{scheduleID}/rotations and .../rotations/{id}/participants` — on-call rotations and who is in them; schedules are synced but are meaningless without rotations (high)
+- [x] `squads` — lookup plus membership table resolving squad ids on incidents, services and escalation policies (high)
+- [x] `schedules/{scheduleID}/rotations and .../rotations/{id}/participants` — on-call rotations and who is in them; schedules are synced but are meaningless without rotations (high)
- [ ] `analytics (org-level and team-level)` — the vendor's headline MTTA/MTTR/incident-volume metrics (high)
- [ ] `incidents/tags` — incident tag values, the primary breakdown dimension for incident reporting (medium)
- [ ] `incidents/{incidentID}/notes` — responder commentary attached to incidents (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/canonical_descriptions.py
index 396d09eb9041..533f55da51bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/canonical_descriptions.py
@@ -125,6 +125,29 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"team_id": _TEAM_ID_COLUMN,
},
},
+ "schedule_rotations": {
+ "description": "A rotation inside an on-call schedule, with its shift pattern and the participants who take turns being on call.",
+ "docs_url": "https://developers.incidents.cloud.solarwinds.com/api-reference/rotation",
+ "columns": {
+ "id": "Identifier of the rotation, unique within its schedule.",
+ "name": "Name of the rotation.",
+ "scheduleID": "ID of the schedule the rotation belongs to.",
+ "color": "Color used to show the rotation in the schedule view.",
+ "startDate": "Date on which the rotation starts.",
+ "period": "Repeat period of the rotation shifts (e.g. daily, weekly, custom).",
+ "customPeriodFrequency": "Number of custom period units in one repeat, when the period is custom.",
+ "customPeriodUnit": "Unit of the custom repeat period, when the period is custom.",
+ "shiftTimeSlots": "Shift time slots of the rotation, each with a start hour, start minute, duration and optional day of week.",
+ "changeParticipantsFrequency": "Number of units after which the on-call participant group changes.",
+ "changeParticipantsUnit": "Unit for the participant change frequency.",
+ "endDate": "Date on which the rotation ends, if set.",
+ "endsAfterIterations": "Number of iterations after which the rotation ends, if set.",
+ "rotationParticipants": "Participant groups that take turns in the rotation; each participant has an ID and a type (e.g. user, squad, team).",
+ "createdAt": "Time at which the rotation was created.",
+ "updatedAt": "Time at which the rotation was last updated.",
+ "team_id": _TEAM_ID_COLUMN,
+ },
+ },
"runbooks": {
"description": "A runbook of steps responders can attach to and follow during an incident.",
"docs_url": "https://developers.incidents.cloud.solarwinds.com/api-reference/runbooks",
@@ -167,6 +190,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"team_id": _TEAM_ID_COLUMN,
},
},
+ "squads": {
+ "description": "A squad: a named group of users inside a team, used as an owner or a responder on incidents, schedules and escalation policies.",
+ "docs_url": "https://developers.incidents.cloud.solarwinds.com/api-reference/squads",
+ "columns": {
+ "id": "Unique identifier for the squad.",
+ "org_id": "ID of the organization the squad belongs to.",
+ "team_id": "ID of the team the squad belongs to.",
+ "name": "Name of the squad.",
+ "members": "Members of the squad, each with a user ID and an optional role.",
+ "createdAt": "Time at which the squad was created.",
+ "createdBy": "ID of the user who created the squad.",
+ "updatedAt": "Time at which the squad was last updated.",
+ "updatedBy": "ID of the user who last updated the squad.",
+ },
+ },
"users": {
"description": "A user in the Squadcast organization.",
"docs_url": "https://developers.incidents.cloud.solarwinds.com/api-reference/users",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/settings.py
index 152fba2de643..ab8ee94f8989 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/settings.py
@@ -1,6 +1,8 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from typing import Literal, Optional
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# How each endpoint is iterated. Squadcast mixes several pagination styles across its API
@@ -10,10 +12,11 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
# - "cursor": `pageSize`/`cursor` params with a `pageInfo.nextCursor` reference (v4 schedules).
# - "incident_export": required `start_time`/`end_time` window, chunked client-side (incidents).
# - "postmortems": required `fromDate`/`toDate`/`limit` window, single request per team.
-PaginationStyle = Literal["none", "offset", "cursor", "incident_export", "postmortems"]
+# - "schedule_rotations": walks each team's v4 schedules and lists the rotations of every schedule.
+PaginationStyle = Literal["none", "offset", "cursor", "incident_export", "postmortems", "schedule_rotations"]
-@dataclass
+@frozen
class SquadcastEndpointConfig:
path: str # Path under the regional API base URL, e.g. "/v3/services"
pagination: PaginationStyle = "none"
@@ -22,7 +25,7 @@ class SquadcastEndpointConfig:
team_param: Optional[str] = None
# Keys to walk in the response body to reach the list of rows.
envelope: tuple[str, ...] = ("data",)
- primary_key: str = "id"
+ primary_keys: tuple[str, ...] = ("id",)
partition_key: Optional[str] = None # Stable datetime field used to partition (never a mutable field)
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Credential-bearing row fields dropped before rows are yielded, so live keys never
@@ -77,6 +80,13 @@ SQUADCAST_ENDPOINTS: dict[str, SquadcastEndpointConfig] = {
pagination="cursor",
team_param="teamID",
),
+ "schedule_rotations": SquadcastEndpointConfig(
+ path="/v4/schedules/{schedule_id}/rotations",
+ pagination="schedule_rotations",
+ team_param="teamID",
+ # Rotation ids are only documented as unique within their schedule.
+ primary_keys=("scheduleID", "id"),
+ ),
"runbooks": SquadcastEndpointConfig(
path="/v3/runbooks",
team_param="owner_id",
@@ -87,6 +97,12 @@ SQUADCAST_ENDPOINTS: dict[str, SquadcastEndpointConfig] = {
team_param="owner_id",
envelope=("data", "slos"),
),
+ "squads": SquadcastEndpointConfig(
+ # The response carries `pageInfo`, but the endpoint documents no page size or cursor
+ # params, so we make one request per team and warn if `pageInfo.hasNext` is set.
+ path="/v4/squads",
+ team_param="owner_id",
+ ),
"users": SquadcastEndpointConfig(
path="/v3/users",
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
index 756b57e7cd9a..0efadce5d4f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
@@ -343,6 +343,62 @@ def _get_postmortem_rows(
yield _prepare_rows(rows, config, team_id)
+def _iter_schedule_rotations(
+ client: SquadcastClient,
+ config: SquadcastEndpointConfig,
+ team_id: str,
+ start_cursor: str | None,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[SquadcastResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ schedules_config = SQUADCAST_ENDPOINTS["schedules"]
+ assert schedules_config.team_param is not None
+ cursor = start_cursor
+ while True:
+ params: dict[str, Any] = {schedules_config.team_param: team_id, "pageSize": PAGE_SIZE}
+ if cursor:
+ params["cursor"] = cursor
+
+ data = client.get(schedules_config.path, params)
+ page_info = data.get("pageInfo", {}) if isinstance(data, dict) else {}
+ next_cursor = page_info.get("nextCursor") if page_info.get("hasNext") else None
+
+ # Stage the cursor of the current schedules page, not the next one: a resume re-reads
+ # this page's rotations, and merge dedupes them on the primary key.
+ resumable_source_manager.save_state(SquadcastResumeConfig(team_id=team_id, cursor=cursor))
+
+ for schedule in _extract_rows(data, schedules_config.envelope):
+ schedule_id = schedule.get("id") if isinstance(schedule, dict) else None
+ if schedule_id is None:
+ continue
+ try:
+ rotations = _extract_rows(client.get(config.path.format(schedule_id=schedule_id)), config.envelope)
+ except requests.HTTPError as exc:
+ # 404: the schedule was deleted after the list request. 403: the role can't read this
+ # schedule. Either way skip only this schedule, so the team's other schedules still sync.
+ if exc.response is not None and exc.response.status_code in (403, 404):
+ logger.warning(
+ f"Squadcast: cannot read rotations of schedule {schedule_id} for team {team_id} "
+ f"(status={exc.response.status_code}), skipping"
+ )
+ rotations = []
+ else:
+ raise
+
+ for rotation in rotations:
+ if isinstance(rotation, dict):
+ rotation.setdefault("scheduleID", schedule_id)
+
+ if rotations:
+ yield _prepare_rows(rotations, config, team_id)
+ else:
+ resumable_source_manager.safe_point()
+
+ if not next_cursor:
+ break
+ cursor = next_cursor
+
+
def _iter_team_rows(
client: SquadcastClient,
config: SquadcastEndpointConfig,
@@ -362,19 +418,22 @@ def _iter_team_rows(
)
elif config.pagination == "postmortems":
yield from _get_postmortem_rows(client, config, team_id, logger, db_incremental_field_last_value)
+ elif config.pagination == "schedule_rotations":
+ yield from _iter_schedule_rotations(client, config, team_id, start_cursor, logger, resumable_source_manager)
else:
assert config.team_param is not None
data = client.get(config.path, {config.team_param: team_id})
rows = _extract_rows(data, config.envelope)
- # See settings.py: escalation policies aren't paginated client-side, so check the
- # reported total for silent truncation.
+ # See settings.py: escalation policies and squads aren't paginated client-side, so check
+ # the reported total or next-page flag for silent truncation.
if isinstance(data, dict):
total_count = data.get("meta", {}).get("total_count")
- if total_count is not None and total_count > len(rows):
+ has_next = bool((data.get("pageInfo") or {}).get("hasNext"))
+ if has_next or (total_count is not None and total_count > len(rows)):
logger.warning(
f"Squadcast: endpoint '{config.path}' for team {team_id} returned {len(rows)} "
- f"of {total_count} rows; results may be truncated"
+ f"of {total_count if total_count is not None else 'more'} rows; results may be truncated"
)
if rows:
@@ -460,7 +519,7 @@ def squadcast_source(
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
),
- primary_keys=[config.primary_key],
+ primary_keys=list(config.primary_keys),
# Incremental endpoints fan out over teams, so rows are not globally ordered by the
# incremental field; "desc" makes the pipeline persist the watermark only after the
# whole job completes instead of checkpointing a possibly-unsafe per-batch maximum.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/tests/test_squadcast.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/tests/test_squadcast.py
index 9d337af23c7c..5b72792214f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/tests/test_squadcast.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/tests/test_squadcast.py
@@ -419,22 +419,137 @@ class TestPostmortems:
assert "truncated" in logger.warning.call_args.args[0]
-class TestEscalationPolicies:
- def test_reported_total_above_returned_rows_is_logged(self) -> None:
- body = {"data": [{"id": "ep_1"}], "meta": {"total_count": 10}}
+class TestUnpaginatedTeamEndpoints:
+ @pytest.mark.parametrize(
+ "endpoint,path,body,expect_warning",
+ [
+ (
+ "escalation_policies",
+ "/v3/escalation-policies",
+ {"data": [{"id": "r1"}], "meta": {"total_count": 10}},
+ True,
+ ),
+ (
+ "escalation_policies",
+ "/v3/escalation-policies",
+ {"data": [{"id": "r1"}], "meta": {"total_count": 1}},
+ False,
+ ),
+ ("squads", "/v4/squads", {"data": [{"id": "r1"}], "pageInfo": {"hasNext": True, "hasPrev": False}}, True),
+ ("squads", "/v4/squads", {"data": [{"id": "r1"}], "pageInfo": {"hasNext": False, "hasPrev": False}}, False),
+ ("squads", "/v4/squads", {"data": [{"id": "r1"}]}, False),
+ ],
+ )
+ def test_possible_truncation_is_logged(
+ self, endpoint: str, path: str, body: dict[str, Any], expect_warning: bool
+ ) -> None:
+ def handler(request: RecordedRequest) -> ScriptedResponse:
+ team_param = SQUADCAST_ENDPOINTS[endpoint].team_param
+ assert team_param is not None
+ assert _params(request) == {team_param: "t1"}
+ return _json(body)
+
logger = MagicMock()
result = _run(
- "escalation_policies",
- _standard_handler(
- teams=[{"id": "t1"}],
- endpoint_responses={"/v3/escalation-policies": lambda request: _json(body)},
- ),
+ endpoint,
+ _standard_handler(teams=[{"id": "t1"}], endpoint_responses={path: handler}),
logger=logger,
)
- assert result.items == [[{"id": "ep_1", "team_id": "t1"}]]
- assert logger.warning.called
+ assert result.items == [[{"id": "r1", "team_id": "t1"}]]
+ assert logger.warning.called is expect_warning
+
+
+def _schedules_handler(pages: dict[str | None, dict[str, Any]]) -> Handler:
+ def handler(request: RecordedRequest) -> ScriptedResponse:
+ params = _params(request)
+ assert params["teamID"] == "t1"
+ assert params["pageSize"] == str(PAGE_SIZE)
+ return _json(pages[params.get("cursor")])
+
+ return handler
+
+
+def _constant(response: ScriptedResponse) -> Handler:
+ return lambda request: response
+
+
+class TestScheduleRotations:
+ def _endpoint_responses(
+ self, pages: dict[str | None, dict[str, Any]], rotations: dict[str, ScriptedResponse]
+ ) -> dict[str, Handler]:
+ responses: dict[str, Handler] = {"/v4/schedules": _schedules_handler(pages)}
+ for schedule_id, response in rotations.items():
+ responses[f"/v4/schedules/{schedule_id}/rotations"] = _constant(response)
+ return responses
+
+ def test_walks_schedule_pages_and_stages_current_page_cursor(self) -> None:
+ pages: dict[str | None, dict[str, Any]] = {
+ None: {"data": [{"id": 1}, {"id": 2}], "pageInfo": {"hasNext": True, "nextCursor": "cur_2"}},
+ "cur_2": {"data": [{"id": 3}], "pageInfo": {"hasNext": False}},
+ }
+ rotations = {
+ "1": _json({"data": [{"id": 10, "scheduleID": 1}, {"id": 11, "scheduleID": 1}]}),
+ "2": _json({"data": None}),
+ "3": _json({"data": [{"id": 10, "scheduleID": 3}]}),
+ }
+
+ result = _run(
+ "schedule_rotations",
+ _standard_handler(teams=[{"id": "t1"}], endpoint_responses=self._endpoint_responses(pages, rotations)),
+ )
+
+ assert result.items == [
+ [{"id": 10, "scheduleID": 1, "team_id": "t1"}, {"id": 11, "scheduleID": 1, "team_id": "t1"}],
+ [{"id": 10, "scheduleID": 3, "team_id": "t1"}],
+ ]
+ # The cursor staged before each page's rows points at that page, so a resume re-reads
+ # the page instead of skipping its remaining schedules.
+ assert [(s.team_id, s.cursor) for s in result.saved_states] == [("t1", None), ("t1", "cur_2")]
+ assert result.response is not None
+ assert result.response.primary_keys == ["scheduleID", "id"]
+
+ def test_resumes_from_saved_schedules_cursor(self) -> None:
+ pages: dict[str | None, dict[str, Any]] = {"cur_2": {"data": [{"id": 3}], "pageInfo": {"hasNext": False}}}
+ rotations = {"3": _json({"data": [{"id": 30, "scheduleID": 3}]})}
+
+ result = _run(
+ "schedule_rotations",
+ _standard_handler(teams=[{"id": "t1"}], endpoint_responses=self._endpoint_responses(pages, rotations)),
+ resume_state=SquadcastResumeConfig(team_id="t1", cursor="cur_2"),
+ )
+
+ assert result.items == [[{"id": 30, "scheduleID": 3, "team_id": "t1"}]]
+
+ @pytest.mark.parametrize(
+ "failing_status,expect_raised",
+ [
+ (404, False),
+ (403, False),
+ (400, True),
+ ],
+ )
+ def test_rotation_fetch_error_for_one_schedule(self, failing_status: int, expect_raised: bool) -> None:
+ pages: dict[str | None, dict[str, Any]] = {
+ None: {"data": [{"id": 1}, {"id": 2}], "pageInfo": {"hasNext": False}},
+ }
+ rotations = {
+ "1": _json({"error": {"code": "x", "message": "boom"}}, status_code=failing_status),
+ "2": _json({"data": [{"id": 20}]}),
+ }
+
+ result = _run(
+ "schedule_rotations",
+ _standard_handler(teams=[{"id": "t1"}], endpoint_responses=self._endpoint_responses(pages, rotations)),
+ )
+
+ if expect_raised:
+ assert isinstance(result.raised, requests.HTTPError)
+ assert result.items == []
+ else:
+ assert result.raised is None
+ assert result.items == [[{"id": 20, "scheduleID": 2, "team_id": "t1"}]]
class TestSquadcastSourceResponse:
@@ -458,7 +573,7 @@ class TestSquadcastSourceResponse:
response = _run(endpoint, always(_json({"data": []}))).response
assert response is not None
assert response.name == endpoint
- assert response.primary_keys == [SQUADCAST_ENDPOINTS[endpoint].primary_key]
+ assert response.primary_keys == list(SQUADCAST_ENDPOINTS[endpoint].primary_keys)
assert callable(response.items)