:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Xmatters source (#124591)
Tom Owers60f8eaa3a5ec
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 606df03d27d9..6521822d5285 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -13428,10 +13428,10 @@ Diffed against: <https://help.xmatters.com/xmapi/index.html>- [x] `incidents (GET /api/xm/1/incidents, /incidents/{id}/timeline-entries)` — incident objects and their timeline; the headline object of xMatters incident management (high)- [x] `group members (GET /api/xm/1/groups/{id}/members)` — membership lookup joining the people and groups we already sync (high)- [x] `group shifts (GET /api/xm/1/groups/{id}/shifts)` — on-call rotation definitions; without them groups carry no schedule (high)-- [ ] `on-call (GET /api/xm/1/on-call)` — who is on call over a timeframe, the vendor's headline query (high)-- [ ] `services (GET /api/xm/1/services)` — lookup resolving the service referenced by incidents and signals (high)-- [ ] `event user deliveries (GET /api/xm/1/events/{id}/user-deliveries)` — per-recipient delivery and response outcomes for each event (medium)-- [ ] `signals (GET /api/xm/1/signals)` — the raw inbound alerts that events are created from (medium)+- [x] `on-call (GET /api/xm/1/on-call)` — who is on call over a timeframe, the vendor's headline query (high)+- [x] `services (GET /api/xm/1/services)` — lookup resolving the service referenced by incidents and signals (high)+- [x] `event user deliveries (GET /api/xm/1/events/{id}/user-deliveries)` — per-recipient delivery and response outcomes for each event (medium)+- [x] `signals (GET /api/xm/1/signals)` — the raw inbound alerts that events are created from (medium)- [ ] `shift members (GET /api/xm/1/groups/{id}/shifts/{id}/members)` — who is rostered on each shift, with rotation order (medium)- [ ] `on-call summary (GET /api/xm/1/on-call-summary)` — aggregated on-call load per person and group (medium)- [ ] `forms (GET /api/xm/1/forms)` — lookup resolving the form id carried on events and scenarios (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.pyindex 8b04ff0bcdc4..a5ec84543b6f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/canonical_descriptions.py@@ -185,4 +185,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {rotation="How members rotate through the shift.",),},+ "services": {+ "description": "A service in the xMatters service catalog, with its type, tier, and owning group.",+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-services",+ "columns": _columns(+ recipientType="Always SERVICE.",+ description="A description of the service.",+ serviceType="APPLICATION for an interface between business services, TECHNICAL for any other service.",+ serviceTier="The importance level of the service: PLATINUM, GOLD, SILVER, BRONZE, or NONE.",+ ownedBy="Reference to the group that owns the service.",+ ),+ },+ "signals": {+ "description": "An inbound signal (alert or request) that xMatters received from a trigger.",+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-signals",+ "columns": {+ "id": "Unique identifier (UUID) for the signal.",+ "received": "Date and time xMatters received the signal; the incremental cursor.",+ "httpResponse": "The HTTP response xMatters returned for the signal.",+ "workflow": "Reference to the workflow (plan) the signal was sent to.",+ "workflowType": "The type of the workflow, for example USER or RESERVED.",+ "trigger": "Reference to the trigger that received the signal.",+ "triggerLabel": "User-provided name of the trigger that sent the signal.",+ "triggerType": "The type of the trigger, for example INBOUND_WEBHOOK.",+ "authenticatedUser": "The user that authenticated the inbound request.",+ "suppressedCount": "Number of correlated signals suppressed by flood control settings.",+ "payloadType": "The content type of the signal payload.",+ },+ },+ "user_deliveries": {+ "description": "Delivery and response outcome of an event for one notified user, from historical data.",+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-user-delivery-data",+ "columns": {+ "event_id": "Identifier of the event that the user was notified about.",+ "person_id": "Identifier of the notified user.",+ "event": "Reference to the event.",+ "person": "Reference to the notified user.",+ "deliveryStatus": "Consolidated status across all notifications sent to the user's devices, for example RESPONDED or FAILED.",+ "notifications": "The notifications sent to the user's devices, with delivery times and responses.",+ "response": "The response the user gave to the event, if any.",+ },+ },+ "on_call": {+ "description": "A shift occurrence that is on call now for an on-call group, with the members on duty.",+ "docs_url": "https://help.xmatters.com/xmapi/index.html#get-who-is-on-call",+ "columns": {+ "group_id": "Identifier of the group.",+ "shift_id": "Identifier of the shift.",+ "group": "Reference to the group.",+ "shift": "Reference to the shift.",+ "start": "Start of the shift occurrence (UTC).",+ "end": "End of the shift occurrence (UTC).",+ "members": "The members of the shift occurrence, in escalation order.",+ "occurrenceType": "The type of shift occurrence, for example SHIFT_OCCURRENCE.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.pyindex e4f37087e461..af4e80a7082c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/settings.py@@ -22,6 +22,8 @@ class XmattersEndpointConfig:# required for safe incremental sync: the filter bounds the window and the ascending# sort guarantees a stable watermark. Reference/config endpoints stay full-refresh.supports_from: bool = False+ # The `sortBy` value that orders rows by the same timestamp `from` filters on.+ sort_by: Optional[str] = None# Set for per-parent endpoints; the parent is another endpoint in this catalog.fanout: Optional[DependentEndpointConfig] = Nonedefault_incremental_field: Optional[str] = None@@ -38,6 +40,16 @@ _CREATED_INCREMENTAL: list[IncrementalField] = []+_RECEIVED_INCREMENTAL: list[IncrementalField] = [+ {+ "label": "received",+ "type": IncrementalFieldType.DateTime,+ "field": "received",+ "field_type": IncrementalFieldType.DateTime,+ },+]++# Sent explicitly so the audit table does not depend on the API's default set of types.AUDIT_TYPES = ("EVENT_ANNOTATED",@@ -62,6 +74,7 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {# Note this picks up newly *created* events only — status changes to events created before# the cursor are not re-fetched (xMatters filters on created time, not an updated cursor).supports_from=True,+ sort_by="START_TIME",),"people": XmattersEndpointConfig(name="people",@@ -95,6 +108,19 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {name="subscriptions",path="/subscriptions",),+ "services": XmattersEndpointConfig(+ name="services",+ path="/services",+ ),+ "signals": XmattersEndpointConfig(+ name="signals",+ path="/signals",+ partition_key="received",+ incremental_fields=_RECEIVED_INCREMENTAL,+ # `from` keeps signals received at or after the cursor, and RECEIVED is a documented sort.+ supports_from=True,+ sort_by="RECEIVED",+ ),"incidents": XmattersEndpointConfig(name="incidents",path="/incidents",@@ -149,6 +175,39 @@ XMATTERS_ENDPOINTS: dict[str, XmattersEndpointConfig] = {child_response_actions=[{"status_code": 404, "action": "ignore"}],),),+ "user_deliveries": XmattersEndpointConfig(+ name="user_deliveries",+ # One row per notified person per event. The endpoint reads historical data only, so the+ # transport adds the required `at` timestamp per sync.+ path="/events/{event_id}/user-deliveries",+ primary_keys=["event_id", "person_id"],+ fanout=DependentEndpointConfig(+ parent_name="events",+ resolve_param="event_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "event_id"},+ parent_params={"sortBy": "START_TIME", "sortOrder": "ASCENDING"},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "on_call": XmattersEndpointConfig(+ name="on_call",+ # With no timeframe, /on-call returns the shift occurrences that are on call right now.+ path="/on-call?groups={group_id}",+ primary_keys=["group_id", "shift_id", "start"],+ fanout=DependentEndpointConfig(+ parent_name="groups",+ resolve_param="group_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "group_id"},+ parent_params={"groupType": "ON_CALL"},+ # The default is 3 members per shift; 100 is the documented maximum.+ child_params={"membersPerShift": 100},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),}ENDPOINTS = tuple(XMATTERS_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.pyindex db41a5ce0486..aa8a743cca35 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/source.py@@ -112,7 +112,7 @@ Use HTTP Basic auth with a REST Web Service User (or an API key as the usernameforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Only `events` carries incremental fields; every endpoint is merge/full-refresh (no append).+ # Only `events` and `signals` carry incremental fields; every endpoint is merge/full-refresh (no append).return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=ENDPOINTS)def validate_credentials(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.pyindex 3355195f2779..7eae77defc8c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/tests/test_xmatters.py@@ -1,4 +1,4 @@-from datetime import UTC, date, datetime+from datetime import UTC, date, datetime, timedelta, timezonefrom typing import Any, Optionalimport pytest@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.xmatters.xXmattersResumeConfig,_base_url,_format_incremental_value,+ _historical_at,is_valid_subdomain,validate_credentials,)@@ -49,6 +50,19 @@ class TestFormatIncrementalValue:assert _format_incremental_value(value) == expected+class TestHistoricalAt:+ @pytest.mark.parametrize(+ "now,expected",+ [+ (datetime(2026, 3, 4, 12, 0, 0, tzinfo=UTC), "2026-03-04T11:40:00Z"),+ (datetime(2026, 3, 4, 0, 10, 30, 999999, tzinfo=UTC), "2026-03-03T23:50:30Z"),+ (datetime(2026, 3, 4, 14, 0, 0, tzinfo=timezone(timedelta(hours=2))), "2026-03-04T11:40:00Z"),+ ],+ )+ def test_stays_outside_the_synchronization_window_in_utc(self, now: datetime, expected: str) -> None:+ assert _historical_at(now) == expected++class TestSubdomainValidation:# An editor-controlled subdomain like `attacker.example/` would make the worker send its# requests (and Basic auth header) to an arbitrary host instead of *.xmatters.com (SSRF).@@ -130,9 +144,16 @@ class TestPagination:assert len(result.requests) == 1assert result.saved_states == []- def test_events_incremental_sends_from_and_sort(self) -> None:+ @pytest.mark.parametrize(+ "endpoint,expected_sort_by",+ [+ ("events", "START_TIME"),+ ("signals", "RECEIVED"),+ ],+ )+ def test_incremental_sends_from_and_sort(self, endpoint: str, expected_sort_by: str) -> None:result = _driver().run(- "events",+ endpoint,[_response([{"id": "1"}])],should_use_incremental_field=True,db_incremental_field_last_value=datetime(2026, 1, 1, tzinfo=UTC),@@ -140,9 +161,24 @@ class TestPagination:request = result.requests[0]assert request.param("from") == "2026-01-01T00:00:00+00:00"- assert request.param("sortBy") == "START_TIME"+ assert request.param("sortBy") == expected_sort_byassert request.param("sortOrder") == "ASCENDING"+ @pytest.mark.parametrize(+ "endpoint,expected_sort_by",+ [+ ("services", None),+ ("signals", "RECEIVED"),+ ],+ )+ def test_full_refresh_sends_no_from(self, endpoint: str, expected_sort_by: Optional[str]) -> None:+ result = _driver().run(endpoint, [_response([{"id": "1"}])])++ assert [row["id"] for row in result.rows] == ["1"]+ request = result.requests[0]+ assert request.param("from") is None+ assert request.param("sortBy") == expected_sort_by+class TestFanout:@pytest.mark.parametrize(@@ -179,6 +215,35 @@ class TestFanout:[{"id": "s-1", "name": "Weekdays", "created": "2026-02-01T10:00:00.000Z"}],[{"id": "s-1", "group_id": "p-1"}],),+ (+ "user_deliveries",+ "/events",+ "/events/p-1/user-deliveries",+ {"sortBy": "START_TIME", "sortOrder": "ASCENDING"},+ {},+ [+ {"event": {"id": "p-1"}, "person": {"id": "u-1"}, "deliveryStatus": "RESPONDED"},+ {"event": {"id": "p-1"}, "person": {"id": "u-2"}, "deliveryStatus": "FAILED"},+ ],+ [{"event_id": "p-1", "person_id": "u-1"}, {"event_id": "p-1", "person_id": "u-2"}],+ ),+ (+ "on_call",+ "/groups",+ "/on-call",+ {"groupType": "ON_CALL"},+ {"groups": "p-1", "membersPerShift": "100"},+ [+ {+ "group": {"id": "p-1"},+ "shift": {"id": "s-1"},+ "start": "2026-02-01T08:00:00Z",+ "end": "2026-02-01T16:00:00Z",+ "members": {"count": 0, "total": 0, "data": []},+ }+ ],+ [{"group_id": "p-1", "shift_id": "s-1", "start": "2026-02-01T08:00:00Z"}],+ ),],)def test_fans_out_over_parent_rows(@@ -208,6 +273,8 @@ class TestFanout:assert child_request.param(name) == valueif endpoint == "audits":assert child_request.param("eventId") == "p-1"+ # user-deliveries only serves historical data, so every request must carry `at`.+ assert (child_request.param("at") is not None) is (endpoint == "user_deliveries")def test_fanout_saves_per_parent_progress(self) -> None:script = route(@@ -225,6 +292,25 @@ class TestFanout:assert state is not Noneassert "completed" in state+ @pytest.mark.parametrize("resumed", [False, True])+ def test_user_deliveries_keeps_one_historical_at_across_resumes(self, resumed: bool) -> None:+ saved_at = "2026-01-01T00:00:00Z"+ script = route(+ {+ "/events/e-1/user-deliveries": [_response([{"event": {"id": "e-1"}, "person": {"id": "u-1"}}])],+ "/events": [_response([{"id": "e-1"}])],+ }+ )+ resume_state = XmattersResumeConfig(paginator_state={}, historical_at=saved_at) if resumed else None+ result = _driver().run("user_deliveries", script, resume_state=resume_state)++ assert result.raised is None+ child_ats = {r.param("at") for r in result.requests if r.path.endswith("/user-deliveries")}+ assert len(child_ats) == 1+ assert (child_ats == {saved_at}) is resumed+ assert result.saved_states+ assert {state.historical_at for state in result.saved_states} == child_ats+def test_child_404_for_a_deleted_parent_is_skipped(self) -> None:script = route({diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.pyindex fde8b496ab80..67614fb33454 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py@@ -1,7 +1,8 @@import reimport base64import dataclasses-from datetime import UTC, date, datetime+from collections.abc import Callable+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optional, castfrom urllib.parse import urlencode@@ -39,6 +40,9 @@ BASE_URL_TEMPLATE = "https://{subdomain}.xmatters.com/api/xm/1"# `@`, etc.) could redirect worker requests to an attacker-controlled host (SSRF).SUBDOMAIN_REGEX = re.compile(r"^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$")+# Historical data has a 15-minute synchronization window, so an `at` closer to now can be rejected.+HISTORICAL_DATA_LAG = timedelta(minutes=20)+@dataclasses.dataclass(frozen=True)class XmattersResumeConfig:@@ -46,6 +50,8 @@ class XmattersResumeConfig:# per-parent checkpoint through `paginator_state`.offset: int = 0paginator_state: Optional[dict[str, Any]] = None+ # A resumed child offset only addresses the same rows under the same historical snapshot.+ historical_at: Optional[str] = Nonedef is_valid_subdomain(subdomain: str) -> bool:@@ -124,6 +130,25 @@ def _normalize_group_member(item: dict[str, Any]) -> dict[str, Any]:return {**item, "member_id": item["member"]["id"]}+def _normalize_on_call(item: dict[str, Any]) -> dict[str, Any]:+ return {**item, "shift_id": item["shift"]["id"]}+++def _normalize_user_delivery(item: dict[str, Any]) -> dict[str, Any]:+ return {**item, "person_id": item["person"]["id"]}+++_FANOUT_ROW_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {+ "group_members": _normalize_group_member,+ "on_call": _normalize_on_call,+ "user_deliveries": _normalize_user_delivery,+}+++def _historical_at(now: datetime) -> str:+ return (now - HISTORICAL_DATA_LAG).astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")++def _build_params(config: XmattersEndpointConfig,should_use_incremental_field: bool,@@ -132,11 +157,10 @@ def _build_params(params: dict[str, Any] = {}if config.supports_from:- # START_TIME is the event's initiation (created) time, which is exactly what `from`- # filters on — sorting ascending on it means new events append to the end and never- # shift pages we've already read. Sent on every sync so full refreshes paginate over a- # stable ordering too.- params["sortBy"] = "START_TIME"+ # `sort_by` is the timestamp `from` filters on — sorting ascending on it means new rows+ # append to the end and never shift pages we've already read. Sent on every sync so full+ # refreshes paginate over a stable ordering too.+ params["sortBy"] = config.sort_byparams["sortOrder"] = "ASCENDING"if should_use_incremental_field and db_incremental_field_last_value:params["from"] = _format_incremental_value(db_incremental_field_last_value)@@ -186,14 +210,22 @@ def _fanout_source(assert config.fanout is not Noneinitial_paginator_state: Optional[dict[str, Any]] = None+ historical_at: Optional[str] = Noneif resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()if resume is not None:initial_paginator_state = resume.paginator_state+ historical_at = resume.historical_at++ if config.name == "user_deliveries" and historical_at is None:+ historical_at = _historical_at(datetime.now(UTC))+ child_params_extra = {"at": historical_at} if historical_at is not None else Nonedef save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:if state:- resumable_source_manager.save_state(XmattersResumeConfig(paginator_state=dict(state)))+ resumable_source_manager.save_state(+ XmattersResumeConfig(paginator_state=dict(state), historical_at=historical_at)+ )resource = cast(Resource,@@ -209,12 +241,14 @@ def _fanout_source(db_incremental_field_last_value=None,parent_endpoint_extra={"data_selector": "data"},child_endpoint_extra={"data_selector": "data"},+ child_params_extra=child_params_extra,resume_hook=save_fanout_checkpoint,initial_paginator_state=initial_paginator_state,),)- if config.name == "group_members":- resource = resource.add_map(_normalize_group_member)+ row_map = _FANOUT_ROW_MAPS.get(config.name)+ if row_map is not None:+ resource = resource.add_map(row_map)return SourceResponse(name=config.name,@@ -294,7 +328,7 @@ def xmatters_source(name=endpoint,items=lambda: resource,primary_keys=config.primary_keys,- # We request START_TIME ascending where a sort is available, and full-refresh endpoints+ # We request an ascending `sort_by` where a sort is available, and full-refresh endpoints# replace wholesale, so ascending is correct everywhere.sort_mode="asc",partition_count=1,