Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add monitors, charts and events tables to secoda (#129124)

Tom Owers48bcc63a2739
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index bf12e2e2a207..e9dac686c355 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10686,12 +10686,12 @@ Diffed against: <https://docs.secoda.co/llms.txt>
- [x] `schemas (GET /api/v1/database/schemas)` — lookup resolving the schema id carried on every synced table (high)
- [x] `integrations (GET /api/v1/integration/integrations)` — lookup resolving the integration id present on every resource, i.e. which source system an asset came from (high)
- [x] `lineage (GET /api/v1/lineage/manual/)` — the dependency graph between resources — the core analytical object of a catalog (high)
-- [ ] `monitors and incidents (/api/v1/monitor/monitors, /monitor/incidents)` — data-quality monitors and the incidents they raise, Secoda's headline reliability metric (high)
-- [ ] `monitor measurements (GET /api/v1/monitor/measurements)` — the time series behind each monitor; required for any quality trend analysis (high)
+- [x] `monitors and incidents (/api/v1/monitor/monitors, /monitor/incidents)` — data-quality monitors and the incidents they raise, Secoda's headline reliability metric (high)
+- [x] `monitor measurements (GET /api/v1/monitor/measurements)` — the time series behind each monitor; required for any quality trend analysis (high)
- [ ] `queries (GET /api/v1/query/queries)` — query history driving table popularity and usage rankings (medium)
- [ ] `dashboards (GET /api/v1/dashboard/dashboards)` — BI assets, the downstream consumers in lineage (medium)
-- [ ] `charts (GET /api/v1/dashboard/charts)` — chart-level assets under dashboards, needed for column-to-chart impact analysis (medium)
-- [ ] `events and event properties (/api/v1/event/events, /event/event_properties, /event/category)` — the event taxonomy tracked in the catalog plus its lookup categories (medium)
+- [x] `charts (GET /api/v1/dashboard/charts)` — chart-level assets under dashboards, needed for column-to-chart impact analysis (medium)
+- [x] `events and event properties (/api/v1/event/events, /event/event_properties, /event/category)` — the event taxonomy tracked in the catalog plus its lookup categories (medium)
- [ ] `audit logs and resource logs (/api/v1/activity_log/audit_logs/, /resource_logs/)` — who touched which asset when — governance and adoption reporting (medium)
- [ ] `custom properties (GET /api/v1/resource/all_v2/custom_properties/)` — lookup resolving customer-defined property values attached to tables and columns (medium)
diff --git a/products/warehouse_sources/sources/secoda/canonical_descriptions.py b/products/warehouse_sources/sources/secoda/canonical_descriptions.py
index b4b1f9118a67..ba0c1b849449 100644
--- a/products/warehouse_sources/sources/secoda/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/secoda/canonical_descriptions.py
@@ -151,4 +151,126 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the lineage object was last updated.",
},
},
+ "monitors": {
+ "description": "A data-quality monitor that measures a metric on a Secoda resource on a schedule.",
+ "docs_url": "https://docs.secoda.co/api/reference/monitors",
+ "columns": {
+ "id": "The unique ID of the monitor.",
+ "name": "The full name of the monitor, including the metric type and target resource.",
+ "target": "The ID of the resource that the monitor watches.",
+ "integration": "The ID of the integration associated with the monitored resource.",
+ "metric_type": "The type of metric the monitor measures.",
+ "metric_config": "The configuration of the metric.",
+ "frequency": "How often the monitor runs.",
+ "is_enabled": "Whether the monitor is enabled.",
+ "status": "The status of the monitor.",
+ "condition_auto_sensitivity": "The sensitivity of the automatic threshold condition.",
+ "condition_manual_min": "The manual minimum threshold, if the user set one.",
+ "condition_manual_max": "The manual maximum threshold, if the user set one.",
+ "last_measurement_value": "The most recent measured value, if one is available.",
+ "last_measurement": "The ID of the most recent measurement.",
+ "last_incident": "The ID of the last active incident. Null when the last incident is closed.",
+ "last_attempted_at": "When the monitor last attempted a run.",
+ "last_success_at": "When the monitor last read its data source successfully.",
+ "last_error_at": "When the monitor last failed, if ever.",
+ "last_error_message": "The message of the last failure, if any.",
+ "next_run_at": "When the monitor runs next. The run can occur shortly after this time, never before.",
+ "created_at": "When the monitor was created.",
+ "updated_at": "When the monitor was last updated.",
+ },
+ },
+ "incidents": {
+ "description": "An incident a Secoda monitor raised when a measurement crossed its thresholds.",
+ "docs_url": "https://docs.secoda.co/api/reference/monitors",
+ "columns": {
+ "id": "The unique ID of the incident.",
+ "monitor": "The ID of the monitor that raised the incident.",
+ "target": "The ID of the monitored resource.",
+ "name": "The name of the incident, which shows the metric type of the monitor.",
+ "status": "The status of the incident.",
+ "summary": "A short, automatically generated summary of the incident.",
+ "value": "The measured value when the incident was created.",
+ "actions": "The actions associated with the incident.",
+ "first_seen_at": "When the incident was first observed.",
+ "last_seen_at": "When the incident was most recently observed.",
+ "acknowledged_at": "When the incident was acknowledged. Null until someone acknowledges it.",
+ "resolved_at": "When the incident was resolved. Null while the incident is open.",
+ "last_notified_at": "When the last notification about the incident was sent.",
+ "created_at": "When the incident was created.",
+ "updated_at": "When the incident was last updated.",
+ },
+ },
+ "monitor_measurements": {
+ "description": "One measurement a Secoda monitor took, with the thresholds in force at that time.",
+ "docs_url": "https://docs.secoda.co/api/reference/monitors",
+ "columns": {
+ "id": "The unique ID of the measurement.",
+ "monitor_id": "The ID of the monitor the measurement was fetched for.",
+ "monitor": "The ID of the monitor that took the measurement.",
+ "incident": "The ID of the incident the measurement belongs to. Null when there is no incident.",
+ "incident_metadata": "Additional information about the incident.",
+ "value": "The measured value.",
+ "upper_threshold": "The upper threshold. A value above it triggers an incident.",
+ "lower_threshold": "The lower threshold. A value below it triggers an incident.",
+ "created_at": "When the measurement was created.",
+ "updated_at": "When the measurement was last updated.",
+ },
+ },
+ "charts": {
+ "description": "A chart catalogued in Secoda, usually a child of a BI dashboard.",
+ "docs_url": "https://docs.secoda.co/api/reference/charts",
+ "columns": {
+ "id": "The unique ID of the chart.",
+ "title": "The chart title.",
+ "title_full": "The full title of the chart, including its ancestry.",
+ "description": "The chart's documented description.",
+ "parent": "The ID of the parent resource, usually the dashboard.",
+ "integration": "The ID of the integration the chart was ingested from.",
+ "native_type": "The type of the chart as the integration names it.",
+ "url": "The URL of the chart.",
+ "external_usage": "The usage count of the chart from the source system.",
+ "stale": "Whether the chart no longer exists in the source, as of the last extraction.",
+ "created_at": "When the chart was first catalogued in Secoda.",
+ "updated_at": "When the chart's metadata was last updated in Secoda.",
+ },
+ },
+ "events": {
+ "description": "A tracked product analytics event catalogued in Secoda.",
+ "docs_url": "https://docs.secoda.co/api/reference/events",
+ "columns": {
+ "id": "The unique ID of the event.",
+ "title": "The event name.",
+ "description": "The event's documented description.",
+ "parent": "The ID of the parent resource, such as the event category.",
+ "integration": "The ID of the integration the event was ingested from.",
+ "native_type": "The type of the event as the integration names it.",
+ "created_at": "When the event was first catalogued in Secoda.",
+ "updated_at": "When the event's metadata was last updated in Secoda.",
+ },
+ },
+ "event_properties": {
+ "description": "A property sent with a tracked event, catalogued in Secoda.",
+ "docs_url": "https://docs.secoda.co/api/reference/events",
+ "columns": {
+ "id": "The unique ID of the event property.",
+ "title": "The property name.",
+ "description": "The property's documented description.",
+ "parent": "The ID of the event the property belongs to.",
+ "integration": "The ID of the integration the property was ingested from.",
+ "native_type": "The type of the property as the integration names it.",
+ "created_at": "When the property was first catalogued in Secoda.",
+ "updated_at": "When the property's metadata was last updated in Secoda.",
+ },
+ },
+ "event_categories": {
+ "description": "A category used to group tracked events in Secoda.",
+ "docs_url": "https://docs.secoda.co/api/reference/events",
+ "columns": {
+ "id": "The unique ID of the event category.",
+ "title": "The category name.",
+ "description": "The category's documented description.",
+ "created_at": "When the category was created in Secoda.",
+ "updated_at": "When the category was last updated in Secoda.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/secoda/secoda.py b/products/warehouse_sources/sources/secoda/secoda.py
index c8b86b292cf6..e359292c846d 100644
--- a/products/warehouse_sources/sources/secoda/secoda.py
+++ b/products/warehouse_sources/sources/secoda/secoda.py
@@ -2,16 +2,20 @@ import dataclasses
from typing import Any, Optional
from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ DependentEndpointConfig,
+ Endpoint,
EndpointResource,
JSONResponsePaginator,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
+ build_dependent_resource,
make_tracked_session,
rest_api_resource,
validate_via_probe,
)
-from products.warehouse_sources.sources.secoda.settings import SECODA_ENDPOINTS
+from products.warehouse_sources.sources.secoda.settings import SECODA_ENDPOINTS, SecodaEndpointConfig
# US cloud host. EU (eapi.secoda.co), APAC (aapi.secoda.co) and self-hosted domains are not yet
# selectable — see the region caveat in the source docs.
@@ -24,6 +28,10 @@ DEFAULT_PROBE_PATH = "/api/v1/user"
# though a couple of endpoints expose it top-level under ``next``. Accept either; ``links.next`` wins.
NEXT_URL_PATH = "(links.next) | next"
+# A 200 whose body isn't the expected ``{"results": [...]}`` shape (bare list, dict without
+# ``results``, or ``results`` not a list) is treated as transient and reissued.
+LIST_ENDPOINT_SELECTOR: Endpoint = {"data_selector": "results", "data_selector_malformed_retryable": True}
+
@dataclasses.dataclass(frozen=False)
class SecodaResumeConfig:
@@ -46,34 +54,20 @@ def secoda_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = SECODA_ENDPOINTS[endpoint]
+ client_config = _client_config(api_key)
+
+ if config.fanout is not None:
+ return _fanout_source(config, config.fanout, client_config, team_id, job_id)
resource_config: EndpointResource = {
"name": endpoint,
- "endpoint": {
- "path": config.path,
- "data_selector": "results",
- # A 200 whose body isn't the expected ``{"results": [...]}`` shape (bare list,
- # dict without ``results``, or ``results`` not a list) is treated as transient
- # and reissued — mirrors the old client raising a retryable error on it.
- "data_selector_malformed_retryable": True,
- },
+ "endpoint": {"path": config.path, **LIST_ENDPOINT_SELECTOR},
}
if config.drop_fields:
resource_config["data_map"] = lambda row: _drop_fields(row, config.drop_fields)
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": SECODA_BASE_URL,
- # Auth (Bearer) is supplied via the framework auth config so its value is redacted from
- # logs and raised errors; only the non-secret Accept header is set here.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "bearer", "token": api_key},
- # The bearer key must not follow a server-supplied `links.next` (or persisted resume
- # URL) or a 3xx to another host. `allowed_hosts=[]` pins requests to SECODA_BASE_URL's host.
- "allowed_hosts": [],
- "allow_redirects": False,
- "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),
- },
+ "client": client_config,
"resource_defaults": {},
"resources": [resource_config],
}
@@ -108,6 +102,59 @@ def secoda_source(
)
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": SECODA_BASE_URL,
+ # Auth (Bearer) is supplied via the framework auth config so its value is redacted from
+ # logs and raised errors; only the non-secret Accept header is set here.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "bearer", "token": api_key},
+ # The bearer key must not follow a server-supplied `links.next` (or persisted resume
+ # URL) or a 3xx to another host. `allowed_hosts=[]` pins requests to SECODA_BASE_URL's host.
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),
+ }
+
+
+def _fanout_source(
+ config: SecodaEndpointConfig,
+ fanout: DependentEndpointConfig,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+) -> SourceResponse:
+ # Not resumable: the dependent-resource resume state lists every completed parent, which would
+ # grow with the monitor count and be rewritten after each monitor.
+ resource = build_dependent_resource(
+ endpoint_configs=SECODA_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ page_size_param=None,
+ parent_endpoint_extra=LIST_ENDPOINT_SELECTOR,
+ # The framework reads a path that ends in a placeholder as a single-entity endpoint and
+ # defaults it to one page, so the child names its paginator explicitly.
+ child_endpoint_extra={
+ **LIST_ENDPOINT_SELECTOR,
+ "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),
+ },
+ )
+
+ return SourceResponse(
+ name=config.name,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ supports_resume=False,
+ )
+
+
def validate_credentials(api_key: str) -> tuple[bool, str | None]:
"""Probe a single list endpoint to validate the workspace API key.
diff --git a/products/warehouse_sources/sources/secoda/settings.py b/products/warehouse_sources/sources/secoda/settings.py
index c4ad3271e705..7ee681a662cf 100644
--- a/products/warehouse_sources/sources/secoda/settings.py
+++ b/products/warehouse_sources/sources/secoda/settings.py
@@ -1,6 +1,6 @@
from dataclasses import field
-from products.warehouse_sources.sources.sdk import IncrementalField, frozen
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, frozen
@frozen
@@ -11,6 +11,12 @@ class SecodaEndpointConfig:
primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Top-level keys removed from every row before it reaches the warehouse.
drop_fields: tuple[str, ...] = ()
+ fanout: DependentEndpointConfig | None = None
+ # Read by the shared fan-out helper. Secoda list endpoints take no page-size param, so
+ # page_size is never sent, and every endpoint is full refresh.
+ page_size: int = 0
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: str | None = None
# Secoda v1 REST list endpoints. All are full-refresh only: Secoda's List Resources filter supports
@@ -31,6 +37,29 @@ SECODA_ENDPOINTS: dict[str, SecodaEndpointConfig] = {
name="integrations", path="/api/v1/integration/integrations", drop_fields=("credentials",)
),
"lineage": SecodaEndpointConfig(name="lineage", path="/api/v1/lineage/manual/"),
+ "monitors": SecodaEndpointConfig(name="monitors", path="/api/v1/monitor/monitors"),
+ "incidents": SecodaEndpointConfig(name="incidents", path="/api/v1/monitor/incidents"),
+ # Secoda requires a `monitor` (or `incident_id`) filter on this endpoint, so it fans out over monitors.
+ # The filter sits in the path because the fan-out helper binds parent values only into the path;
+ # monitor ids are UUIDs, so they need no escaping.
+ "monitor_measurements": SecodaEndpointConfig(
+ name="monitor_measurements",
+ path="/api/v1/monitor/measurements?monitor={monitor}",
+ primary_keys=["monitor_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="monitors",
+ resolve_param="monitor",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "monitor_id"},
+ # A monitor deleted between the monitors listing and its measurements fetch must not fail the sync.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "charts": SecodaEndpointConfig(name="charts", path="/api/v1/dashboard/charts"),
+ "events": SecodaEndpointConfig(name="events", path="/api/v1/event/events"),
+ "event_properties": SecodaEndpointConfig(name="event_properties", path="/api/v1/event/event_properties"),
+ "event_categories": SecodaEndpointConfig(name="event_categories", path="/api/v1/event/category"),
}
ENDPOINTS = tuple(SECODA_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/secoda/source.py b/products/warehouse_sources/sources/secoda/source.py
index 0ca37755517f..7d532b4d9001 100644
--- a/products/warehouse_sources/sources/secoda/source.py
+++ b/products/warehouse_sources/sources/secoda/source.py
@@ -45,7 +45,7 @@ class SecodaSource(ResumableSource[SecodaSourceConfig, SecodaResumeConfig]):
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your Secoda API key to pull your data catalog into the PostHog Data warehouse.
-You can create an API key under **Settings → API** in [Secoda](https://app.secoda.co). The key inherits your workspace permissions and grants read access to your databases, schemas, tables, columns, lineage, integrations, collections, users, groups, and tags.
+You can create an API key under **Settings → API** in [Secoda](https://app.secoda.co). The key inherits your workspace permissions and grants read access to your databases, schemas, tables, columns, lineage, integrations, collections, users, groups, tags, monitors, incidents, charts, and events.
""",
iconPath="/static/services/secoda.png",
docsUrl="https://posthog.com/docs/cdp/sources/secoda",
diff --git a/products/warehouse_sources/sources/secoda/tests/test_secoda.py b/products/warehouse_sources/sources/secoda/tests/test_secoda.py
index 004dd51f60cf..11c0f15f194c 100644
--- a/products/warehouse_sources/sources/secoda/tests/test_secoda.py
+++ b/products/warehouse_sources/sources/secoda/tests/test_secoda.py
@@ -9,6 +9,8 @@ from products.warehouse_sources.sources.secoda.settings import ENDPOINTS, SECODA
from products.warehouse_sources.sources.secoda.source import SecodaSource
TABLES_URL = f"{SECODA_BASE_URL}/api/v1/table/tables"
+MONITORS_URL = f"{SECODA_BASE_URL}/api/v1/monitor/monitors"
+MEASUREMENTS_URL = f"{SECODA_BASE_URL}/api/v1/monitor/measurements"
def _driver() -> SourceDriver:
@@ -50,6 +52,40 @@ class TestPagination:
assert result.urls == [saved]
+class TestMonitorMeasurementsFanout:
+ def test_fetches_measurements_per_monitor_across_pages(self) -> None:
+ monitors_page_2 = f"{MONITORS_URL}?page=2"
+ measurements_page_2 = f"{MEASUREMENTS_URL}?monitor=mon-a&page=2"
+
+ result = _driver().run(
+ "monitor_measurements",
+ [
+ _page([{"id": "mon-a"}], links_next=monitors_page_2),
+ _page([{"id": "m1", "monitor": "mon-a", "value": 1.0}], links_next=measurements_page_2),
+ _page([{"id": "m2", "monitor": "mon-a", "value": 2.0}]),
+ _page([{"id": "mon-b"}, {"id": "mon-deleted"}]),
+ _page([{"id": "m3", "monitor": "mon-b", "value": 3.0}]),
+ # A monitor deleted after the monitors listing must not fail the sync.
+ ScriptedResponse(status=404, json={"detail": "Not found."}),
+ ],
+ )
+
+ assert result.rows == [
+ {"id": "m1", "monitor": "mon-a", "value": 1.0, "monitor_id": "mon-a"},
+ {"id": "m2", "monitor": "mon-a", "value": 2.0, "monitor_id": "mon-a"},
+ {"id": "m3", "monitor": "mon-b", "value": 3.0, "monitor_id": "mon-b"},
+ ]
+ assert result.urls == [
+ MONITORS_URL,
+ f"{MEASUREMENTS_URL}?monitor=mon-a",
+ measurements_page_2,
+ monitors_page_2,
+ f"{MEASUREMENTS_URL}?monitor=mon-b",
+ f"{MEASUREMENTS_URL}?monitor=mon-deleted",
+ ]
+ assert result.saved_states == []
+
+
class TestTransientResponsesRetried:
@parameterized.expand(
[
@@ -105,10 +141,6 @@ class TestSecodaSourceResponse:
response = result.response
assert response is not None
assert response.name == endpoint
- assert response.primary_keys == ["id"]
+ assert response.primary_keys == SECODA_ENDPOINTS[endpoint].primary_keys
# No stable creation timestamp is guaranteed across every object, so we don't partition.
assert response.partition_mode is None
-
- def test_every_endpoint_uses_id_primary_key(self) -> None:
- assert all(config.primary_keys == ["id"] for config in SECODA_ENDPOINTS.values())
- assert set(SECODA_ENDPOINTS) == set(ENDPOINTS)