:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add monitors, charts and events tables to secoda (#129124)
Tom Owers48bcc63a2739
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex bf12e2e2a207..e9dac686c355 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10686,12 +10686,12 @@ Diffed against: <https://docs.secoda.co/llms.txt>- [x] `schemas (GET /api/v1/database/schemas)` — lookup resolving the schema id carried on every synced table (high)- [x] `integrations (GET /api/v1/integration/integrations)` — lookup resolving the integration id present on every resource, i.e. which source system an asset came from (high)- [x] `lineage (GET /api/v1/lineage/manual/)` — the dependency graph between resources — the core analytical object of a catalog (high)-- [ ] `monitors and incidents (/api/v1/monitor/monitors, /monitor/incidents)` — data-quality monitors and the incidents they raise, Secoda's headline reliability metric (high)-- [ ] `monitor measurements (GET /api/v1/monitor/measurements)` — the time series behind each monitor; required for any quality trend analysis (high)+- [x] `monitors and incidents (/api/v1/monitor/monitors, /monitor/incidents)` — data-quality monitors and the incidents they raise, Secoda's headline reliability metric (high)+- [x] `monitor measurements (GET /api/v1/monitor/measurements)` — the time series behind each monitor; required for any quality trend analysis (high)- [ ] `queries (GET /api/v1/query/queries)` — query history driving table popularity and usage rankings (medium)- [ ] `dashboards (GET /api/v1/dashboard/dashboards)` — BI assets, the downstream consumers in lineage (medium)-- [ ] `charts (GET /api/v1/dashboard/charts)` — chart-level assets under dashboards, needed for column-to-chart impact analysis (medium)-- [ ] `events and event properties (/api/v1/event/events, /event/event_properties, /event/category)` — the event taxonomy tracked in the catalog plus its lookup categories (medium)+- [x] `charts (GET /api/v1/dashboard/charts)` — chart-level assets under dashboards, needed for column-to-chart impact analysis (medium)+- [x] `events and event properties (/api/v1/event/events, /event/event_properties, /event/category)` — the event taxonomy tracked in the catalog plus its lookup categories (medium)- [ ] `audit logs and resource logs (/api/v1/activity_log/audit_logs/, /resource_logs/)` — who touched which asset when — governance and adoption reporting (medium)- [ ] `custom properties (GET /api/v1/resource/all_v2/custom_properties/)` — lookup resolving customer-defined property values attached to tables and columns (medium)diff --git a/products/warehouse_sources/sources/secoda/canonical_descriptions.py b/products/warehouse_sources/sources/secoda/canonical_descriptions.pyindex b4b1f9118a67..ba0c1b849449 100644--- a/products/warehouse_sources/sources/secoda/canonical_descriptions.py+++ b/products/warehouse_sources/sources/secoda/canonical_descriptions.py@@ -151,4 +151,126 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the lineage object was last updated.",},},+ "monitors": {+ "description": "A data-quality monitor that measures a metric on a Secoda resource on a schedule.",+ "docs_url": "https://docs.secoda.co/api/reference/monitors",+ "columns": {+ "id": "The unique ID of the monitor.",+ "name": "The full name of the monitor, including the metric type and target resource.",+ "target": "The ID of the resource that the monitor watches.",+ "integration": "The ID of the integration associated with the monitored resource.",+ "metric_type": "The type of metric the monitor measures.",+ "metric_config": "The configuration of the metric.",+ "frequency": "How often the monitor runs.",+ "is_enabled": "Whether the monitor is enabled.",+ "status": "The status of the monitor.",+ "condition_auto_sensitivity": "The sensitivity of the automatic threshold condition.",+ "condition_manual_min": "The manual minimum threshold, if the user set one.",+ "condition_manual_max": "The manual maximum threshold, if the user set one.",+ "last_measurement_value": "The most recent measured value, if one is available.",+ "last_measurement": "The ID of the most recent measurement.",+ "last_incident": "The ID of the last active incident. Null when the last incident is closed.",+ "last_attempted_at": "When the monitor last attempted a run.",+ "last_success_at": "When the monitor last read its data source successfully.",+ "last_error_at": "When the monitor last failed, if ever.",+ "last_error_message": "The message of the last failure, if any.",+ "next_run_at": "When the monitor runs next. The run can occur shortly after this time, never before.",+ "created_at": "When the monitor was created.",+ "updated_at": "When the monitor was last updated.",+ },+ },+ "incidents": {+ "description": "An incident a Secoda monitor raised when a measurement crossed its thresholds.",+ "docs_url": "https://docs.secoda.co/api/reference/monitors",+ "columns": {+ "id": "The unique ID of the incident.",+ "monitor": "The ID of the monitor that raised the incident.",+ "target": "The ID of the monitored resource.",+ "name": "The name of the incident, which shows the metric type of the monitor.",+ "status": "The status of the incident.",+ "summary": "A short, automatically generated summary of the incident.",+ "value": "The measured value when the incident was created.",+ "actions": "The actions associated with the incident.",+ "first_seen_at": "When the incident was first observed.",+ "last_seen_at": "When the incident was most recently observed.",+ "acknowledged_at": "When the incident was acknowledged. Null until someone acknowledges it.",+ "resolved_at": "When the incident was resolved. Null while the incident is open.",+ "last_notified_at": "When the last notification about the incident was sent.",+ "created_at": "When the incident was created.",+ "updated_at": "When the incident was last updated.",+ },+ },+ "monitor_measurements": {+ "description": "One measurement a Secoda monitor took, with the thresholds in force at that time.",+ "docs_url": "https://docs.secoda.co/api/reference/monitors",+ "columns": {+ "id": "The unique ID of the measurement.",+ "monitor_id": "The ID of the monitor the measurement was fetched for.",+ "monitor": "The ID of the monitor that took the measurement.",+ "incident": "The ID of the incident the measurement belongs to. Null when there is no incident.",+ "incident_metadata": "Additional information about the incident.",+ "value": "The measured value.",+ "upper_threshold": "The upper threshold. A value above it triggers an incident.",+ "lower_threshold": "The lower threshold. A value below it triggers an incident.",+ "created_at": "When the measurement was created.",+ "updated_at": "When the measurement was last updated.",+ },+ },+ "charts": {+ "description": "A chart catalogued in Secoda, usually a child of a BI dashboard.",+ "docs_url": "https://docs.secoda.co/api/reference/charts",+ "columns": {+ "id": "The unique ID of the chart.",+ "title": "The chart title.",+ "title_full": "The full title of the chart, including its ancestry.",+ "description": "The chart's documented description.",+ "parent": "The ID of the parent resource, usually the dashboard.",+ "integration": "The ID of the integration the chart was ingested from.",+ "native_type": "The type of the chart as the integration names it.",+ "url": "The URL of the chart.",+ "external_usage": "The usage count of the chart from the source system.",+ "stale": "Whether the chart no longer exists in the source, as of the last extraction.",+ "created_at": "When the chart was first catalogued in Secoda.",+ "updated_at": "When the chart's metadata was last updated in Secoda.",+ },+ },+ "events": {+ "description": "A tracked product analytics event catalogued in Secoda.",+ "docs_url": "https://docs.secoda.co/api/reference/events",+ "columns": {+ "id": "The unique ID of the event.",+ "title": "The event name.",+ "description": "The event's documented description.",+ "parent": "The ID of the parent resource, such as the event category.",+ "integration": "The ID of the integration the event was ingested from.",+ "native_type": "The type of the event as the integration names it.",+ "created_at": "When the event was first catalogued in Secoda.",+ "updated_at": "When the event's metadata was last updated in Secoda.",+ },+ },+ "event_properties": {+ "description": "A property sent with a tracked event, catalogued in Secoda.",+ "docs_url": "https://docs.secoda.co/api/reference/events",+ "columns": {+ "id": "The unique ID of the event property.",+ "title": "The property name.",+ "description": "The property's documented description.",+ "parent": "The ID of the event the property belongs to.",+ "integration": "The ID of the integration the property was ingested from.",+ "native_type": "The type of the property as the integration names it.",+ "created_at": "When the property was first catalogued in Secoda.",+ "updated_at": "When the property's metadata was last updated in Secoda.",+ },+ },+ "event_categories": {+ "description": "A category used to group tracked events in Secoda.",+ "docs_url": "https://docs.secoda.co/api/reference/events",+ "columns": {+ "id": "The unique ID of the event category.",+ "title": "The category name.",+ "description": "The category's documented description.",+ "created_at": "When the category was created in Secoda.",+ "updated_at": "When the category was last updated in Secoda.",+ },+ },}diff --git a/products/warehouse_sources/sources/secoda/secoda.py b/products/warehouse_sources/sources/secoda/secoda.pyindex c8b86b292cf6..e359292c846d 100644--- a/products/warehouse_sources/sources/secoda/secoda.py+++ b/products/warehouse_sources/sources/secoda/secoda.py@@ -2,16 +2,20 @@ import dataclassesfrom typing import Any, Optionalfrom products.warehouse_sources.sources.sdk import (+ ClientConfig,+ DependentEndpointConfig,+ Endpoint,EndpointResource,JSONResponsePaginator,RESTAPIConfig,ResumableSourceManager,SourceResponse,+ build_dependent_resource,make_tracked_session,rest_api_resource,validate_via_probe,)-from products.warehouse_sources.sources.secoda.settings import SECODA_ENDPOINTS+from products.warehouse_sources.sources.secoda.settings import SECODA_ENDPOINTS, SecodaEndpointConfig# US cloud host. EU (eapi.secoda.co), APAC (aapi.secoda.co) and self-hosted domains are not yet# selectable — see the region caveat in the source docs.@@ -24,6 +28,10 @@ DEFAULT_PROBE_PATH = "/api/v1/user"# though a couple of endpoints expose it top-level under ``next``. Accept either; ``links.next`` wins.NEXT_URL_PATH = "(links.next) | next"+# A 200 whose body isn't the expected ``{"results": [...]}`` shape (bare list, dict without+# ``results``, or ``results`` not a list) is treated as transient and reissued.+LIST_ENDPOINT_SELECTOR: Endpoint = {"data_selector": "results", "data_selector_malformed_retryable": True}+@dataclasses.dataclass(frozen=False)class SecodaResumeConfig:@@ -46,34 +54,20 @@ def secoda_source(db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:config = SECODA_ENDPOINTS[endpoint]+ client_config = _client_config(api_key)++ if config.fanout is not None:+ return _fanout_source(config, config.fanout, client_config, team_id, job_id)resource_config: EndpointResource = {"name": endpoint,- "endpoint": {- "path": config.path,- "data_selector": "results",- # A 200 whose body isn't the expected ``{"results": [...]}`` shape (bare list,- # dict without ``results``, or ``results`` not a list) is treated as transient- # and reissued — mirrors the old client raising a retryable error on it.- "data_selector_malformed_retryable": True,- },+ "endpoint": {"path": config.path, **LIST_ENDPOINT_SELECTOR},}if config.drop_fields:resource_config["data_map"] = lambda row: _drop_fields(row, config.drop_fields)rest_config: RESTAPIConfig = {- "client": {- "base_url": SECODA_BASE_URL,- # Auth (Bearer) is supplied via the framework auth config so its value is redacted from- # logs and raised errors; only the non-secret Accept header is set here.- "headers": {"Accept": "application/json"},- "auth": {"type": "bearer", "token": api_key},- # The bearer key must not follow a server-supplied `links.next` (or persisted resume- # URL) or a 3xx to another host. `allowed_hosts=[]` pins requests to SECODA_BASE_URL's host.- "allowed_hosts": [],- "allow_redirects": False,- "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),- },+ "client": client_config,"resource_defaults": {},"resources": [resource_config],}@@ -108,6 +102,59 @@ def secoda_source()+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": SECODA_BASE_URL,+ # Auth (Bearer) is supplied via the framework auth config so its value is redacted from+ # logs and raised errors; only the non-secret Accept header is set here.+ "headers": {"Accept": "application/json"},+ "auth": {"type": "bearer", "token": api_key},+ # The bearer key must not follow a server-supplied `links.next` (or persisted resume+ # URL) or a 3xx to another host. `allowed_hosts=[]` pins requests to SECODA_BASE_URL's host.+ "allowed_hosts": [],+ "allow_redirects": False,+ "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),+ }+++def _fanout_source(+ config: SecodaEndpointConfig,+ fanout: DependentEndpointConfig,+ client_config: ClientConfig,+ team_id: int,+ job_id: str,+) -> SourceResponse:+ # Not resumable: the dependent-resource resume state lists every completed parent, which would+ # grow with the monitor count and be rewritten after each monitor.+ resource = build_dependent_resource(+ endpoint_configs=SECODA_ENDPOINTS,+ child_endpoint=config.name,+ fanout=fanout,+ client_config=client_config,+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ page_size_param=None,+ parent_endpoint_extra=LIST_ENDPOINT_SELECTOR,+ # The framework reads a path that ends in a placeholder as a single-entity endpoint and+ # defaults it to one page, so the child names its paginator explicitly.+ child_endpoint_extra={+ **LIST_ENDPOINT_SELECTOR,+ "paginator": JSONResponsePaginator(next_url_path=NEXT_URL_PATH),+ },+ )++ return SourceResponse(+ name=config.name,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ supports_resume=False,+ )++def validate_credentials(api_key: str) -> tuple[bool, str | None]:"""Probe a single list endpoint to validate the workspace API key.diff --git a/products/warehouse_sources/sources/secoda/settings.py b/products/warehouse_sources/sources/secoda/settings.pyindex c4ad3271e705..7ee681a662cf 100644--- a/products/warehouse_sources/sources/secoda/settings.py+++ b/products/warehouse_sources/sources/secoda/settings.py@@ -1,6 +1,6 @@from dataclasses import field-from products.warehouse_sources.sources.sdk import IncrementalField, frozen+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, frozen@frozen@@ -11,6 +11,12 @@ class SecodaEndpointConfig:primary_keys: list[str] = field(default_factory=lambda: ["id"])# Top-level keys removed from every row before it reaches the warehouse.drop_fields: tuple[str, ...] = ()+ fanout: DependentEndpointConfig | None = None+ # Read by the shared fan-out helper. Secoda list endpoints take no page-size param, so+ # page_size is never sent, and every endpoint is full refresh.+ page_size: int = 0+ incremental_fields: list[IncrementalField] = field(default_factory=list)+ default_incremental_field: str | None = None# Secoda v1 REST list endpoints. All are full-refresh only: Secoda's List Resources filter supports@@ -31,6 +37,29 @@ SECODA_ENDPOINTS: dict[str, SecodaEndpointConfig] = {name="integrations", path="/api/v1/integration/integrations", drop_fields=("credentials",)),"lineage": SecodaEndpointConfig(name="lineage", path="/api/v1/lineage/manual/"),+ "monitors": SecodaEndpointConfig(name="monitors", path="/api/v1/monitor/monitors"),+ "incidents": SecodaEndpointConfig(name="incidents", path="/api/v1/monitor/incidents"),+ # Secoda requires a `monitor` (or `incident_id`) filter on this endpoint, so it fans out over monitors.+ # The filter sits in the path because the fan-out helper binds parent values only into the path;+ # monitor ids are UUIDs, so they need no escaping.+ "monitor_measurements": SecodaEndpointConfig(+ name="monitor_measurements",+ path="/api/v1/monitor/measurements?monitor={monitor}",+ primary_keys=["monitor_id", "id"],+ fanout=DependentEndpointConfig(+ parent_name="monitors",+ resolve_param="monitor",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "monitor_id"},+ # A monitor deleted between the monitors listing and its measurements fetch must not fail the sync.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "charts": SecodaEndpointConfig(name="charts", path="/api/v1/dashboard/charts"),+ "events": SecodaEndpointConfig(name="events", path="/api/v1/event/events"),+ "event_properties": SecodaEndpointConfig(name="event_properties", path="/api/v1/event/event_properties"),+ "event_categories": SecodaEndpointConfig(name="event_categories", path="/api/v1/event/category"),}ENDPOINTS = tuple(SECODA_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/secoda/source.py b/products/warehouse_sources/sources/secoda/source.pyindex 0ca37755517f..7d532b4d9001 100644--- a/products/warehouse_sources/sources/secoda/source.py+++ b/products/warehouse_sources/sources/secoda/source.py@@ -45,7 +45,7 @@ class SecodaSource(ResumableSource[SecodaSourceConfig, SecodaResumeConfig]):releaseStatus=ReleaseStatus.BETA,caption="""Enter your Secoda API key to pull your data catalog into the PostHog Data warehouse.-You can create an API key under **Settings → API** in [Secoda](https://app.secoda.co). The key inherits your workspace permissions and grants read access to your databases, schemas, tables, columns, lineage, integrations, collections, users, groups, and tags.+You can create an API key under **Settings → API** in [Secoda](https://app.secoda.co). The key inherits your workspace permissions and grants read access to your databases, schemas, tables, columns, lineage, integrations, collections, users, groups, tags, monitors, incidents, charts, and events.""",iconPath="/static/services/secoda.png",docsUrl="https://posthog.com/docs/cdp/sources/secoda",diff --git a/products/warehouse_sources/sources/secoda/tests/test_secoda.py b/products/warehouse_sources/sources/secoda/tests/test_secoda.pyindex 004dd51f60cf..11c0f15f194c 100644--- a/products/warehouse_sources/sources/secoda/tests/test_secoda.py+++ b/products/warehouse_sources/sources/secoda/tests/test_secoda.py@@ -9,6 +9,8 @@ from products.warehouse_sources.sources.secoda.settings import ENDPOINTS, SECODAfrom products.warehouse_sources.sources.secoda.source import SecodaSourceTABLES_URL = f"{SECODA_BASE_URL}/api/v1/table/tables"+MONITORS_URL = f"{SECODA_BASE_URL}/api/v1/monitor/monitors"+MEASUREMENTS_URL = f"{SECODA_BASE_URL}/api/v1/monitor/measurements"def _driver() -> SourceDriver:@@ -50,6 +52,40 @@ class TestPagination:assert result.urls == [saved]+class TestMonitorMeasurementsFanout:+ def test_fetches_measurements_per_monitor_across_pages(self) -> None:+ monitors_page_2 = f"{MONITORS_URL}?page=2"+ measurements_page_2 = f"{MEASUREMENTS_URL}?monitor=mon-a&page=2"++ result = _driver().run(+ "monitor_measurements",+ [+ _page([{"id": "mon-a"}], links_next=monitors_page_2),+ _page([{"id": "m1", "monitor": "mon-a", "value": 1.0}], links_next=measurements_page_2),+ _page([{"id": "m2", "monitor": "mon-a", "value": 2.0}]),+ _page([{"id": "mon-b"}, {"id": "mon-deleted"}]),+ _page([{"id": "m3", "monitor": "mon-b", "value": 3.0}]),+ # A monitor deleted after the monitors listing must not fail the sync.+ ScriptedResponse(status=404, json={"detail": "Not found."}),+ ],+ )++ assert result.rows == [+ {"id": "m1", "monitor": "mon-a", "value": 1.0, "monitor_id": "mon-a"},+ {"id": "m2", "monitor": "mon-a", "value": 2.0, "monitor_id": "mon-a"},+ {"id": "m3", "monitor": "mon-b", "value": 3.0, "monitor_id": "mon-b"},+ ]+ assert result.urls == [+ MONITORS_URL,+ f"{MEASUREMENTS_URL}?monitor=mon-a",+ measurements_page_2,+ monitors_page_2,+ f"{MEASUREMENTS_URL}?monitor=mon-b",+ f"{MEASUREMENTS_URL}?monitor=mon-deleted",+ ]+ assert result.saved_states == []++class TestTransientResponsesRetried:@parameterized.expand([@@ -105,10 +141,6 @@ class TestSecodaSourceResponse:response = result.responseassert response is not Noneassert response.name == endpoint- assert response.primary_keys == ["id"]+ assert response.primary_keys == SECODA_ENDPOINTS[endpoint].primary_keys# No stable creation timestamp is guaranteed across every object, so we don't partition.assert response.partition_mode is None-- def test_every_endpoint_uses_id_primary_key(self) -> None:- assert all(config.primary_keys == ["id"] for config in SECODA_ENDPOINTS.values())- assert set(SECODA_ENDPOINTS) == set(ENDPOINTS)