Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Xmatters source (#124591)

Tom Owers60f8eaa3a5ec
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 606df03d27d9..6521822d5285 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13428,10 +13428,10 @@ Diffed against: <https://help.xmatters.com/xmapi/index.html>
- [x] `incidents (GET /api/xm/1/incidents, /incidents/{id}/timeline-entries)` — incident objects and their timeline; the headline object of xMatters incident management (high)
- [x] `group members (GET /api/xm/1/groups/{id}/members)` — membership lookup joining the people and groups we already sync (high)
- [x] `group shifts (GET /api/xm/1/groups/{id}/shifts)` — on-call rotation definitions; without them groups carry no schedule (high)
-- [ ] `on-call (GET /api/xm/1/on-call)` — who is on call over a timeframe, the vendor's headline query (high)
-- [ ] `services (GET /api/xm/1/services)` — lookup resolving the service referenced by incidents and signals (high)
-- [ ] `event user deliveries (GET /api/xm/1/events/{id}/user-deliveries)` — per-recipient delivery and response outcomes for each event (medium)
-- [ ] `signals (GET /api/xm/1/signals)` — the raw inbound alerts that events are created from (medium)
+- [x] `on-call (GET /api/xm/1/on-call)` — who is on call over a timeframe, the vendor's headline query (high)
+- [x] `services (GET /api/xm/1/services)` — lookup resolving the service referenced by incidents and signals (high)
+- [x] `event user deliveries (GET /api/xm/1/events/{id}/user-deliveries)` — per-recipient delivery and response outcomes for each event (medium)
+- [x] `signals (GET /api/xm/1/signals)` — the raw inbound alerts that events are created from (medium)
- [ ] `shift members (GET /api/xm/1/groups/{id}/shifts/{id}/members)` — who is rostered on each shift, with rotation order (medium)
- [ ] `on-call summary (GET /api/xm/1/on-call-summary)` — aggregated on-call load per person and group (medium)
- [ ] `forms (GET /api/xm/1/forms)` — lookup resolving the form id carried on events and scenarios (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py
index 8b04ff0bcdc4..a5ec84543b6f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py
@@ -185,4 +185,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
rotation="How members rotate through the shift.",
),
},
+ "services": {
+ "description": "A service in the xMatters service catalog, with its type, tier, and owning group.",
+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-services",
+ "columns": _columns(
+ recipientType="Always SERVICE.",
+ description="A description of the service.",
+ serviceType="APPLICATION for an interface between business services, TECHNICAL for any other service.",
+ serviceTier="The importance level of the service: PLATINUM, GOLD, SILVER, BRONZE, or NONE.",
+ ownedBy="Reference to the group that owns the service.",
+ ),
+ },
+ "signals": {
+ "description": "An inbound signal (alert or request) that xMatters received from a trigger.",
+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-signals",
+ "columns": {
+ "id": "Unique identifier (UUID) for the signal.",
+ "received": "Date and time xMatters received the signal; the incremental cursor.",
+ "httpResponse": "The HTTP response xMatters returned for the signal.",
+ "workflow": "Reference to the workflow (plan) the signal was sent to.",
+ "workflowType": "The type of the workflow, for example USER or RESERVED.",
+ "trigger": "Reference to the trigger that received the signal.",
+ "triggerLabel": "User-provided name of the trigger that sent the signal.",
+ "triggerType": "The type of the trigger, for example INBOUND_WEBHOOK.",
+ "authenticatedUser": "The user that authenticated the inbound request.",
+ "suppressedCount": "Number of correlated signals suppressed by flood control settings.",
+ "payloadType": "The content type of the signal payload.",
+ },
+ },
+ "user_deliveries": {
+ "description": "Delivery and response outcome of an event for one notified user, from historical data.",
+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-user-delivery-data",
+ "columns": {
+ "event_id": "Identifier of the event that the user was notified about.",
+ "person_id": "Identifier of the notified user.",
+ "event": "Reference to the event.",
+ "person": "Reference to the notified user.",
+ "deliveryStatus": "Consolidated status across all notifications sent to the user's devices, for example RESPONDED or FAILED.",
+ "notifications": "The notifications sent to the user's devices, with delivery times and responses.",
+ "response": "The response the user gave to the event, if any.",
+ },
+ },
+ "on_call": {
+ "description": "A shift occurrence that is on call now for an on-call group, with the members on duty.",
+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-who-is-on-call",
+ "columns": {
+ "group_id": "Identifier of the group.",
+ "shift_id": "Identifier of the shift.",
+ "group": "Reference to the group.",
+ "shift": "Reference to the shift.",
+ "start": "Start of the shift occurrence (UTC).",
+ "end": "End of the shift occurrence (UTC).",
+ "members": "The members of the shift occurrence, in escalation order.",
+ "occurrenceType": "The type of shift occurrence, for example SHIFT_OCCURRENCE.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py
index e4f37087e461..af4e80a7082c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py
@@ -22,6 +22,8 @@ class XmattersEndpointConfig:
# required for safe incremental sync: the filter bounds the window and the ascending
# sort guarantees a stable watermark. Reference/config endpoints stay full-refresh.
supports_from: bool = False
+ # The `sortBy` value that orders rows by the same timestamp `from` filters on.
+ sort_by: Optional[str] = None
# Set for per-parent endpoints; the parent is another endpoint in this catalog.
fanout: Optional[DependentEndpointConfig] = None
default_incremental_field: Optional[str] = None
@@ -38,6 +40,16 @@ _CREATED_INCREMENTAL: list[IncrementalField] = [
]
+_RECEIVED_INCREMENTAL: list[IncrementalField] = [
+ {
+ "label": "received",
+ "type": IncrementalFieldType.DateTime,
+ "field": "received",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+]
+
+
# Sent explicitly so the audit table does not depend on the API's default set of types.
AUDIT_TYPES = (
"EVENT_ANNOTATED",
@@ -62,6 +74,7 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {
# Note this picks up newly *created* events only — status changes to events created before
# the cursor are not re-fetched (xMatters filters on created time, not an updated cursor).
supports_from=True,
+ sort_by="START_TIME",
),
"people": XmattersEndpointConfig(
name="people",
@@ -95,6 +108,19 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {
name="subscriptions",
path="/subscriptions",
),
+ "services": XmattersEndpointConfig(
+ name="services",
+ path="/services",
+ ),
+ "signals": XmattersEndpointConfig(
+ name="signals",
+ path="/signals",
+ partition_key="received",
+ incremental_fields=_RECEIVED_INCREMENTAL,
+ # `from` keeps signals received at or after the cursor, and RECEIVED is a documented sort.
+ supports_from=True,
+ sort_by="RECEIVED",
+ ),
"incidents": XmattersEndpointConfig(
name="incidents",
path="/incidents",
@@ -149,6 +175,39 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {
child_response_actions=[{"status_code": 404, "action": "ignore"}],
),
),
+ "user_deliveries": XmattersEndpointConfig(
+ name="user_deliveries",
+ # One row per notified person per event. The endpoint reads historical data only, so the
+ # transport adds the required `at` timestamp per sync.
+ path="/events/{event_id}/user-deliveries",
+ primary_keys=["event_id", "person_id"],
+ fanout=DependentEndpointConfig(
+ parent_name="events",
+ resolve_param="event_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "event_id"},
+ parent_params={"sortBy": "START_TIME", "sortOrder": "ASCENDING"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "on_call": XmattersEndpointConfig(
+ name="on_call",
+ # With no timeframe, /on-call returns the shift occurrences that are on call right now.
+ path="/on-call?groups={group_id}",
+ primary_keys=["group_id", "shift_id", "start"],
+ fanout=DependentEndpointConfig(
+ parent_name="groups",
+ resolve_param="group_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "group_id"},
+ parent_params={"groupType": "ON_CALL"},
+ # The default is 3 members per shift; 100 is the documented maximum.
+ child_params={"membersPerShift": 100},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(XMATTERS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py
index db41a5ce0486..aa8a743cca35 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py
@@ -112,7 +112,7 @@ Use HTTP Basic auth with a REST Web Service User (or an API key as the username
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Only `events` carries incremental fields; every endpoint is merge/full-refresh (no append).
+ # Only `events` and `signals` carry incremental fields; every endpoint is merge/full-refresh (no append).
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=ENDPOINTS)
def validate_credentials(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py
index 3355195f2779..7eae77defc8c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py
@@ -1,4 +1,4 @@
-from datetime import UTC, date, datetime
+from datetime import UTC, date, datetime, timedelta, timezone
from typing import Any, Optional
import pytest
@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.xmatters.x
XmattersResumeConfig,
_base_url,
_format_incremental_value,
+ _historical_at,
is_valid_subdomain,
validate_credentials,
)
@@ -49,6 +50,19 @@ class TestFormatIncrementalValue:
assert _format_incremental_value(value) == expected
+class TestHistoricalAt:
+ @pytest.mark.parametrize(
+ "now,expected",
+ [
+ (datetime(2026, 3, 4, 12, 0, 0, tzinfo=UTC), "2026-03-04T11:40:00Z"),
+ (datetime(2026, 3, 4, 0, 10, 30, 999999, tzinfo=UTC), "2026-03-03T23:50:30Z"),
+ (datetime(2026, 3, 4, 14, 0, 0, tzinfo=timezone(timedelta(hours=2))), "2026-03-04T11:40:00Z"),
+ ],
+ )
+ def test_stays_outside_the_synchronization_window_in_utc(self, now: datetime, expected: str) -> None:
+ assert _historical_at(now) == expected
+
+
class TestSubdomainValidation:
# An editor-controlled subdomain like `attacker.example/` would make the worker send its
# requests (and Basic auth header) to an arbitrary host instead of *.xmatters.com (SSRF).
@@ -130,9 +144,16 @@ class TestPagination:
assert len(result.requests) == 1
assert result.saved_states == []
- def test_events_incremental_sends_from_and_sort(self) -> None:
+ @pytest.mark.parametrize(
+ "endpoint,expected_sort_by",
+ [
+ ("events", "START_TIME"),
+ ("signals", "RECEIVED"),
+ ],
+ )
+ def test_incremental_sends_from_and_sort(self, endpoint: str, expected_sort_by: str) -> None:
result = _driver().run(
- "events",
+ endpoint,
[_response([{"id": "1"}])],
should_use_incremental_field=True,
db_incremental_field_last_value=datetime(2026, 1, 1, tzinfo=UTC),
@@ -140,9 +161,24 @@ class TestPagination:
request = result.requests[0]
assert request.param("from") == "2026-01-01T00:00:00+00:00"
- assert request.param("sortBy") == "START_TIME"
+ assert request.param("sortBy") == expected_sort_by
assert request.param("sortOrder") == "ASCENDING"
+ @pytest.mark.parametrize(
+ "endpoint,expected_sort_by",
+ [
+ ("services", None),
+ ("signals", "RECEIVED"),
+ ],
+ )
+ def test_full_refresh_sends_no_from(self, endpoint: str, expected_sort_by: Optional[str]) -> None:
+ result = _driver().run(endpoint, [_response([{"id": "1"}])])
+
+ assert [row["id"] for row in result.rows] == ["1"]
+ request = result.requests[0]
+ assert request.param("from") is None
+ assert request.param("sortBy") == expected_sort_by
+
class TestFanout:
@pytest.mark.parametrize(
@@ -179,6 +215,35 @@ class TestFanout:
[{"id": "s-1", "name": "Weekdays", "created": "2026-02-01T10:00:00.000Z"}],
[{"id": "s-1", "group_id": "p-1"}],
),
+ (
+ "user_deliveries",
+ "/events",
+ "/events/p-1/user-deliveries",
+ {"sortBy": "START_TIME", "sortOrder": "ASCENDING"},
+ {},
+ [
+ {"event": {"id": "p-1"}, "person": {"id": "u-1"}, "deliveryStatus": "RESPONDED"},
+ {"event": {"id": "p-1"}, "person": {"id": "u-2"}, "deliveryStatus": "FAILED"},
+ ],
+ [{"event_id": "p-1", "person_id": "u-1"}, {"event_id": "p-1", "person_id": "u-2"}],
+ ),
+ (
+ "on_call",
+ "/groups",
+ "/on-call",
+ {"groupType": "ON_CALL"},
+ {"groups": "p-1", "membersPerShift": "100"},
+ [
+ {
+ "group": {"id": "p-1"},
+ "shift": {"id": "s-1"},
+ "start": "2026-02-01T08:00:00Z",
+ "end": "2026-02-01T16:00:00Z",
+ "members": {"count": 0, "total": 0, "data": []},
+ }
+ ],
+ [{"group_id": "p-1", "shift_id": "s-1", "start": "2026-02-01T08:00:00Z"}],
+ ),
],
)
def test_fans_out_over_parent_rows(
@@ -208,6 +273,8 @@ class TestFanout:
assert child_request.param(name) == value
if endpoint == "audits":
assert child_request.param("eventId") == "p-1"
+ # user-deliveries only serves historical data, so every request must carry `at`.
+ assert (child_request.param("at") is not None) is (endpoint == "user_deliveries")
def test_fanout_saves_per_parent_progress(self) -> None:
script = route(
@@ -225,6 +292,25 @@ class TestFanout:
assert state is not None
assert "completed" in state
+ @pytest.mark.parametrize("resumed", [False, True])
+ def test_user_deliveries_keeps_one_historical_at_across_resumes(self, resumed: bool) -> None:
+ saved_at = "2026-01-01T00:00:00Z"
+ script = route(
+ {
+ "/events/e-1/user-deliveries": [_response([{"event": {"id": "e-1"}, "person": {"id": "u-1"}}])],
+ "/events": [_response([{"id": "e-1"}])],
+ }
+ )
+ resume_state = XmattersResumeConfig(paginator_state={}, historical_at=saved_at) if resumed else None
+ result = _driver().run("user_deliveries", script, resume_state=resume_state)
+
+ assert result.raised is None
+ child_ats = {r.param("at") for r in result.requests if r.path.endswith("/user-deliveries")}
+ assert len(child_ats) == 1
+ assert (child_ats == {saved_at}) is resumed
+ assert result.saved_states
+ assert {state.historical_at for state in result.saved_states} == child_ats
+
def test_child_404_for_a_deleted_parent_is_skipped(self) -> None:
script = route(
{
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
index fde8b496ab80..67614fb33454 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
@@ -1,7 +1,8 @@
import re
import base64
import dataclasses
-from datetime import UTC, date, datetime
+from collections.abc import Callable
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional, cast
from urllib.parse import urlencode
@@ -39,6 +40,9 @@ BASE_URL_TEMPLATE = "https://{subdomain}.xmatters.com/api/xm/1"
# `@`, etc.) could redirect worker requests to an attacker-controlled host (SSRF).
SUBDOMAIN_REGEX = re.compile(r"^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$")
+# Historical data has a 15-minute synchronization window, so an `at` closer to now can be rejected.
+HISTORICAL_DATA_LAG = timedelta(minutes=20)
+
@dataclasses.dataclass(frozen=True)
class XmattersResumeConfig:
@@ -46,6 +50,8 @@ class XmattersResumeConfig:
# per-parent checkpoint through `paginator_state`.
offset: int = 0
paginator_state: Optional[dict[str, Any]] = None
+ # A resumed child offset only addresses the same rows under the same historical snapshot.
+ historical_at: Optional[str] = None
def is_valid_subdomain(subdomain: str) -> bool:
@@ -124,6 +130,25 @@ def _normalize_group_member(item: dict[str, Any]) -> dict[str, Any]:
return {**item, "member_id": item["member"]["id"]}
+def _normalize_on_call(item: dict[str, Any]) -> dict[str, Any]:
+ return {**item, "shift_id": item["shift"]["id"]}
+
+
+def _normalize_user_delivery(item: dict[str, Any]) -> dict[str, Any]:
+ return {**item, "person_id": item["person"]["id"]}
+
+
+_FANOUT_ROW_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
+ "group_members": _normalize_group_member,
+ "on_call": _normalize_on_call,
+ "user_deliveries": _normalize_user_delivery,
+}
+
+
+def _historical_at(now: datetime) -> str:
+ return (now - HISTORICAL_DATA_LAG).astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
def _build_params(
config: XmattersEndpointConfig,
should_use_incremental_field: bool,
@@ -132,11 +157,10 @@ def _build_params(
params: dict[str, Any] = {}
if config.supports_from:
- # START_TIME is the event's initiation (created) time, which is exactly what `from`
- # filters on — sorting ascending on it means new events append to the end and never
- # shift pages we've already read. Sent on every sync so full refreshes paginate over a
- # stable ordering too.
- params["sortBy"] = "START_TIME"
+ # `sort_by` is the timestamp `from` filters on — sorting ascending on it means new rows
+ # append to the end and never shift pages we've already read. Sent on every sync so full
+ # refreshes paginate over a stable ordering too.
+ params["sortBy"] = config.sort_by
params["sortOrder"] = "ASCENDING"
if should_use_incremental_field and db_incremental_field_last_value:
params["from"] = _format_incremental_value(db_incremental_field_last_value)
@@ -186,14 +210,22 @@ def _fanout_source(
assert config.fanout is not None
initial_paginator_state: Optional[dict[str, Any]] = None
+ historical_at: Optional[str] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
initial_paginator_state = resume.paginator_state
+ historical_at = resume.historical_at
+
+ if config.name == "user_deliveries" and historical_at is None:
+ historical_at = _historical_at(datetime.now(UTC))
+ child_params_extra = {"at": historical_at} if historical_at is not None else None
def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
if state:
- resumable_source_manager.save_state(XmattersResumeConfig(paginator_state=dict(state)))
+ resumable_source_manager.save_state(
+ XmattersResumeConfig(paginator_state=dict(state), historical_at=historical_at)
+ )
resource = cast(
Resource,
@@ -209,12 +241,14 @@ def _fanout_source(
db_incremental_field_last_value=None,
parent_endpoint_extra={"data_selector": "data"},
child_endpoint_extra={"data_selector": "data"},
+ child_params_extra=child_params_extra,
resume_hook=save_fanout_checkpoint,
initial_paginator_state=initial_paginator_state,
),
)
- if config.name == "group_members":
- resource = resource.add_map(_normalize_group_member)
+ row_map = _FANOUT_ROW_MAPS.get(config.name)
+ if row_map is not None:
+ resource = resource.add_map(row_map)
return SourceResponse(
name=config.name,
@@ -294,7 +328,7 @@ def xmatters_source(
name=endpoint,
items=lambda: resource,
primary_keys=config.primary_keys,
- # We request START_TIME ascending where a sort is available, and full-refresh endpoints
+ # We request an ascending `sort_by` where a sort is available, and full-refresh endpoints
# replace wholesale, so ascending is correct everywhere.
sort_mode="asc",
partition_count=1,