Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the Axiom source (#124629)

Tom Owers54a34d623952
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 2cc7a831ac9d..606df03d27d9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1372,7 +1372,7 @@ Diffed against: <https://raw.githubusercontent.com/axiomhq/docs/main/content/doc
- [x] `Users (GET /v2/users)` — resolve createdBy on synced monitors to people and analyze ownership across monitoring resources (high). Added as `users`.
- [ ] `Dataset events (Run query, POST /v1/datasets/_apl; edge deployments use POST /v1/query/_apl)` — analyze actual logs, traces, and events instead of only dataset and dashboard metadata (high); skipped, it is an APL query endpoint on the v1 API rather than a list endpoint: each dataset has its own user-defined schema, so it needs per-dataset table discovery, time windowing and regional endpoint routing that a static endpoint catalog cannot express
-- [ ] `Monitor notification history (GET /v2/monitors/{id}/history)` — measure alert frequency, notification outcomes, and changes in monitor behavior over time (medium); skipped, rows carry no identifier and repeat once per notifier and email recipient, so no primary key is unique, and the endpoint requires a `startTime`/`endTime` window with no documented retention bound
+- [x] `Monitor notification history (GET /v2/monitors/{id}/history)` — measure alert frequency, notification outcomes, and changes in monitor behavior over time (medium); skipped, rows carry no identifier and repeat once per notifier and email recipient, so no primary key is unique, and the endpoint requires a `startTime`/`endTime` window with no documented retention bound. Added as `monitor_history`, keyed with a `deliveryIndex` column and read in 7-day windows from each monitor's `createdAt`.
- [x] `Groups (GET /v2/rbac/groups)` — analyze organization membership and connect users to operational teams (medium). Added as `groups`.
Note: The official documentation repository provides the v2, v1, and v1-edge-query OpenAPI files used for this comparison. The v1 specs confirm that dataset event queries are read operations sent by POST; the source currently imports metadata only. Event imports require dataset selection, time windows, pagination, and the correct regional query endpoint; history requires requests for each monitor. Tokens, notifiers, field settings, views, ingestion writes, and other configuration operations are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/axiom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/axiom.py
index af6446633962..e27708f59062 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/axiom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/axiom.py
@@ -1,3 +1,6 @@
+from collections import Counter
+from collections.abc import Iterator
+from datetime import UTC, datetime
from typing import Any
from requests.exceptions import HTTPError
@@ -8,6 +11,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.axiom.sett
AUTH_ERRORS,
BASE_URL,
ENDPOINTS,
+ MONITOR_HISTORY,
+ MONITOR_HISTORY_DEFAULT_LOOKBACK,
+ MONITOR_HISTORY_PRIMARY_KEYS,
+ MONITOR_HISTORY_WINDOW,
PRIMARY_KEYS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
@@ -41,6 +48,9 @@ def validate_credentials(config: AxiomSourceConfig, schema_name: str | None = No
allow_redirects=False,
request_timeout=30,
)
+ if schema_name == MONITOR_HISTORY:
+ # History needs a monitor ID, and the same `monitors|read` permission covers the list.
+ schema_name = "monitors"
endpoints = [schema_for_resource(ENDPOINTS, schema_name)] if schema_name else list(ENDPOINTS.values())
for endpoint in endpoints:
path = endpoint.get("path")
@@ -62,11 +72,114 @@ def validate_credentials(config: AxiomSourceConfig, schema_name: str | None = No
return False, AUTH_ERRORS[403]
+def _client(config: AxiomSourceConfig) -> RESTClient:
+ return RESTClient(
+ base_url=BASE_URL,
+ auth=BearerTokenAuth(token=config.api_token),
+ headers={"x-axiom-org-id": config.org_id} if config.org_id else {},
+ paginator=SinglePagePaginator(),
+ allowed_hosts=[],
+ allow_redirects=False,
+ request_timeout=30,
+ )
+
+
+def _parse_time(value: Any) -> datetime | None:
+ if isinstance(value, str):
+ try:
+ value = datetime.fromisoformat(value)
+ except ValueError:
+ return None
+ if not isinstance(value, datetime):
+ return None
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
+
+
+def _format_time(value: datetime) -> str:
+ return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _history_rows(rows: list[dict[str, Any]], start: datetime, end: datetime) -> list[dict[str, Any]]:
+ # Windows share their bounds, so a row stays only in the window whose half-open range holds it.
+ # All copies of a delivery share a timestamp and so land in the same window, which keeps the
+ # delivery index stable between runs.
+ seen: Counter[tuple[Any, Any, Any]] = Counter()
+ kept = []
+ for row in rows:
+ timestamp = _parse_time(row.get("timestamp"))
+ if timestamp is not None and not start <= timestamp < end:
+ continue
+ key = (row.get("checkId"), row.get("timestamp"), row.get("state"))
+ kept.append({**row, "deliveryIndex": seen[key]})
+ seen[key] += 1
+ return kept
+
+
+def axiom_monitor_history_source(
+ config: AxiomSourceConfig,
+ inputs: SourceInputs,
+ manager: ResumableSourceManager[AxiomResumeConfig],
+) -> SourceResponse:
+ last_value = _parse_time(inputs.db_incremental_field_last_value) if inputs.should_use_incremental_field else None
+
+ def items() -> Iterator[list[dict[str, Any]]]:
+ client = _client(config)
+ end = datetime.now(UTC).replace(microsecond=0)
+ monitors = [monitor for page in client.paginate("monitors", data_selector_required=True) for monitor in page]
+ manager.safe_point()
+ for monitor in monitors:
+ start = _parse_time(monitor.get("createdAt")) or end - MONITOR_HISTORY_DEFAULT_LOOKBACK
+ if last_value is not None:
+ start = max(start, last_value)
+ # Request bounds are whole seconds, so windows must be too or rows between a truncated
+ # bound and the precise one would fall outside every window.
+ start = start.replace(microsecond=0)
+ while start < end:
+ window_end = min(start + MONITOR_HISTORY_WINDOW, end)
+ try:
+ pages = list(
+ client.paginate(
+ f"monitors/{monitor['id']}/history",
+ params={"startTime": _format_time(start), "endTime": _format_time(window_end)},
+ data_selector_required=True,
+ )
+ )
+ except HTTPError as error:
+ # The monitor was deleted after the monitor list was read.
+ if error.response is not None and error.response.status_code == 404:
+ manager.safe_point()
+ break
+ raise
+ rows = _history_rows([row for page in pages for row in page], start, window_end)
+ if rows:
+ yield rows
+ else:
+ manager.safe_point()
+ start = window_end
+
+ return SourceResponse(
+ name=MONITOR_HISTORY,
+ items=items,
+ primary_keys=MONITOR_HISTORY_PRIMARY_KEYS,
+ # Rows are ordered per monitor, not across monitors.
+ sort_mode=None,
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime",
+ partition_format="month",
+ partition_keys=["timestamp"],
+ # No cursor is saved across monitors and windows, so a handoff would replay from the start.
+ supports_resume=False,
+ )
+
+
def axiom_source(
config: AxiomSourceConfig,
inputs: SourceInputs,
manager: ResumableSourceManager[AxiomResumeConfig],
) -> SourceResponse:
+ if inputs.schema_name == MONITOR_HISTORY:
+ return axiom_monitor_history_source(config, inputs, manager)
endpoint = schema_for_resource(ENDPOINTS, inputs.schema_name)
paginated = endpoint["paginator"] != "single_page"
initial_state: dict[str, Any] | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/canonical_descriptions.py
index e261c55abdf3..be7428d00928 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/canonical_descriptions.py
@@ -91,4 +91,15 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"isManaged": "Whether the group is synced from the identity provider.",
},
},
+ "monitor_history": {
+ "description": "Notifications that Axiom monitors sent, one row for each notifier and email recipient.",
+ "docs_url": "https://axiom.co/docs/restapi/endpoints/getMonitorHistory",
+ "columns": {
+ "checkId": "ID of the monitor that sent the notification. Matches id on monitors.",
+ "name": "Notification title: Triggered or Resolved, followed by the monitor name.",
+ "state": "open when the notification reported that the monitor triggered, closed when it reported that the monitor resolved.",
+ "timestamp": "Time when Axiom recorded the delivery, after any retries.",
+ "deliveryIndex": "Position of the row among deliveries with the same monitor, timestamp and state. Added by PostHog to keep rows unique.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/settings.py
index b820b4e71926..43bebf7f1fe2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/settings.py
@@ -1,4 +1,7 @@
+from datetime import timedelta
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
API_VERSION = "v2"
BASE_URL = f"https://api.axiom.co/{API_VERSION}/"
@@ -22,6 +25,26 @@ ENDPOINTS: dict[str, Endpoint] = {
"groups": {"path": "rbac/groups", "paginator": "single_page"},
}
+MONITOR_HISTORY = "monitor_history"
+# The history API returns one identical row per notifier and email recipient, so the key adds the
+# position of each row among the rows that share the other key values.
+MONITOR_HISTORY_PRIMARY_KEYS = ["checkId", "timestamp", "state", "deliveryIndex"]
+# The endpoint has no pagination, so a bounded window keeps each response small.
+MONITOR_HISTORY_WINDOW = timedelta(days=7)
+# Start of the first window when a monitor has no createdAt value.
+MONITOR_HISTORY_DEFAULT_LOOKBACK = timedelta(days=90)
+
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ MONITOR_HISTORY: [
+ IncrementalField(
+ label="timestamp",
+ type=IncrementalFieldType.DateTime,
+ field="timestamp",
+ field_type=IncrementalFieldType.DateTime,
+ )
+ ],
+}
+
AUTH_ERRORS = {
401: "Axiom rejected the token. Check that the token is valid and has not expired.",
403: "Axiom denied access. Check the token permissions and the organization ID for personal access tokens.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/source.py
index 3afb7b9b2f3a..dd33a8ce6c52 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/source.py
@@ -20,6 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.axiom.sett
API_VERSION,
AUTH_ERRORS,
ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ MONITOR_HISTORY,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
@@ -89,7 +91,9 @@ class AxiomSource(ResumableSource[AxiomSourceConfig, AxiomResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, {}, names)
+ return build_endpoint_schemas(
+ [*ENDPOINTS, MONITOR_HISTORY], INCREMENTAL_FIELDS, names, merge_only=[MONITOR_HISTORY]
+ )
def validate_credentials(
self,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
index 0a037f6088dc..cac29565978d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
@@ -2,6 +2,7 @@ from collections.abc import Iterable
from typing import Any, cast
import pytest
+import time_machine
from unittest.mock import MagicMock
import structlog
@@ -138,16 +139,23 @@ def test_credentials_fall_back_to_an_authorized_resource() -> None:
]
-def test_credentials_use_selected_resource() -> None:
+ "schema_name,path,query",
+ [
+ ("dashboards", "/v2/dashboards", {"limit": ("1",)}),
+ ("monitor_history", "/v2/monitors", {}),
+ ],
+)
+def test_credentials_use_selected_resource(schema_name: str, path: str, query: dict[str, tuple[str, ...]]) -> None:
with scripted_network([ScriptedResponse(json=[])]) as network:
- assert AxiomSource().validate_credentials(AxiomSourceConfig(api_token="test-token"), 1, "dashboards") == (
+ assert AxiomSource().validate_credentials(AxiomSourceConfig(api_token="test-token"), 1, schema_name) == (
True,
None,
)
assert len(network.requests_log) == 1
- assert network.requests_log[0].path == "/v2/dashboards"
- assert network.requests_log[0].query == {"limit": ("1",)}
+ assert network.requests_log[0].path == path
+ assert network.requests_log[0].query == query
@pytest.mark.parametrize(
@@ -210,3 +218,176 @@ def test_unexpected_response_is_not_imported_as_a_record(body: dict[str, str] |
with pytest.raises(ValueError, match="Required a list response body"):
AxiomSource().validate_credentials(config, 1)
assert [request.path for request in network.requests_log] == ["/v2/datasets"]
+
+
+def history_query(start: str, end: str) -> dict[str, tuple[str, ...]]:
+ return {"startTime": (start,), "endTime": (end,)}
+
+
+@time_machine.travel("2025-03-01T00:00:00Z", tick=False)
+ "created_at,incremental,last_value,expected_queries",
+ [
+ (
+ "2025-02-17T00:00:00Z",
+ False,
+ None,
+ [
+ history_query("2025-02-17T00:00:00Z", "2025-02-24T00:00:00Z"),
+ history_query("2025-02-24T00:00:00Z", "2025-03-01T00:00:00Z"),
+ ],
+ ),
+ (
+ "2025-02-17T00:00:00Z",
+ False,
+ "2025-02-26T12:00:00Z",
+ [
+ history_query("2025-02-17T00:00:00Z", "2025-02-24T00:00:00Z"),
+ history_query("2025-02-24T00:00:00Z", "2025-03-01T00:00:00Z"),
+ ],
+ ),
+ (
+ "2025-02-17T00:00:00Z",
+ True,
+ "2025-02-26T12:00:00.500Z",
+ [history_query("2025-02-26T12:00:00Z", "2025-03-01T00:00:00Z")],
+ ),
+ (
+ "2025-02-27T00:00:00Z",
+ True,
+ "2025-02-20T00:00:00Z",
+ [history_query("2025-02-27T00:00:00Z", "2025-03-01T00:00:00Z")],
+ ),
+ ("2025-02-17T00:00:00Z", True, "2025-03-02T00:00:00Z", []),
+ (
+ None,
+ False,
+ None,
+ [
+ history_query("2024-12-01T00:00:00Z", "2024-12-08T00:00:00Z"),
+ *[{} for _ in range(11)],
+ history_query("2025-02-23T00:00:00Z", "2025-03-01T00:00:00Z"),
+ ],
+ ),
+ ],
+)
+def test_monitor_history_windows(
+ created_at: str | None,
+ incremental: bool,
+ last_value: str | None,
+ expected_queries: list[dict[str, tuple[str, ...]]],
+) -> None:
+ monitor: dict[str, str] = {"id": "monitor-1", "name": "Example monitor"}
+ if created_at is not None:
+ monitor["createdAt"] = created_at
+ script = [ScriptedResponse(json=[monitor])] + [ScriptedResponse(json=[]) for _ in expected_queries]
+
+ result = SourceDriver(AxiomSource(), AxiomSourceConfig(api_token="test-token")).run(
+ "monitor_history",
+ script,
+ should_use_incremental_field=incremental,
+ incremental_field="timestamp" if incremental else None,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.raised is None
+ assert result.rows == []
+ assert result.paths == ["/v2/monitors"] + ["/v2/monitors/monitor-1/history"] * len(expected_queries)
+ queries = result.queries[1:]
+ assert len(queries) == len(expected_queries)
+ for query, expected in zip(queries, expected_queries):
+ if expected:
+ assert query == expected
+
+
+def history_row(timestamp: str, state: str = "open", name: str = "Triggered: Example monitor") -> dict[str, str]:
+ return {"checkId": "monitor-1", "name": name, "state": state, "timestamp": timestamp}
+
+
+@time_machine.travel("2025-03-01T00:00:00Z", tick=False)
+def test_monitor_history_rows_keep_each_delivery_once() -> None:
+ first_window = [
+ history_row("2025-02-19T10:00:00Z"),
+ history_row("2025-02-19T10:00:00Z"),
+ history_row("2025-02-19T10:00:00Z", state="closed", name="Resolved: Example monitor"),
+ history_row("2025-02-16T23:59:59Z"),
+ history_row("2025-02-24T00:00:00Z"),
+ history_row("not-a-timestamp"),
+ ]
+ second_window = [history_row("2025-02-24T00:00:00Z"), history_row("2025-02-24T00:00:00.250Z")]
+
+ result = SourceDriver(AxiomSource(), AxiomSourceConfig(api_token="test-token")).run(
+ "monitor_history",
+ [
+ ScriptedResponse(json=[{"id": "monitor-1", "createdAt": "2025-02-17T00:00:00Z"}]),
+ ScriptedResponse(json=first_window),
+ ScriptedResponse(json=second_window),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {**history_row("2025-02-19T10:00:00Z"), "deliveryIndex": 0},
+ {**history_row("2025-02-19T10:00:00Z"), "deliveryIndex": 1},
+ {
+ **history_row("2025-02-19T10:00:00Z", state="closed", name="Resolved: Example monitor"),
+ "deliveryIndex": 0,
+ },
+ {**history_row("not-a-timestamp"), "deliveryIndex": 0},
+ {**history_row("2025-02-24T00:00:00Z"), "deliveryIndex": 0},
+ {**history_row("2025-02-24T00:00:00.250Z"), "deliveryIndex": 0},
+ ]
+
+
+@time_machine.travel("2025-03-01T00:00:00Z", tick=False)
+def test_monitor_history_keeps_rows_near_fractional_window_bounds() -> None:
+ row = history_row("2025-02-24T00:00:00.250Z")
+
+ result = SourceDriver(AxiomSource(), AxiomSourceConfig(api_token="test-token")).run(
+ "monitor_history",
+ [
+ ScriptedResponse(json=[{"id": "monitor-1", "createdAt": "2025-02-17T00:00:00.500Z"}]),
+ ScriptedResponse(json=[]),
+ ScriptedResponse(json=[row]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.queries[1:] == [
+ history_query("2025-02-17T00:00:00Z", "2025-02-24T00:00:00Z"),
+ history_query("2025-02-24T00:00:00Z", "2025-03-01T00:00:00Z"),
+ ]
+ assert result.rows == [{**row, "deliveryIndex": 0}]
+
+
+def test_monitor_history_does_not_claim_resumability() -> None:
+ inputs = cast(SourceInputs, MagicMock(schema_name="monitor_history", should_use_incremental_field=False))
+ response = axiom_source(AxiomSourceConfig(api_token="test-token"), inputs, MagicMock())
+ assert response.supports_resume is False
+
+
+@time_machine.travel("2025-03-01T00:00:00Z", tick=False)
[email protected]("status,raises", [(404, False), (403, True)])
+def test_monitor_history_skips_only_deleted_monitors(status: int, raises: bool) -> None:
+ monitors = [
+ {"id": "monitor-1", "createdAt": "2025-02-27T00:00:00Z"},
+ {"id": "monitor-2", "createdAt": "2025-02-27T00:00:00Z"},
+ ]
+ row = {**history_row("2025-02-28T00:00:00Z"), "checkId": "monitor-2"}
+
+ result = SourceDriver(AxiomSource(), AxiomSourceConfig(api_token="test-token")).run(
+ "monitor_history",
+ [
+ ScriptedResponse(json=monitors),
+ ScriptedResponse(status=status, json={"code": status}),
+ ScriptedResponse(json=[row]),
+ ],
+ )
+
+ if raises:
+ assert isinstance(result.raised, HTTPError)
+ assert result.rows == []
+ else:
+ assert result.raised is None
+ assert result.paths == ["/v2/monitors", "/v2/monitors/monitor-1/history", "/v2/monitors/monitor-2/history"]
+ assert result.rows == [{**row, "deliveryIndex": 0}]