:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add four bigeye metric and schema change tables (#118513)
Tom Owers3e2fc01e442c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 37ead4d7ae38..e4507752781a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1583,11 +1583,11 @@ Today (6): `Collections`, `Issues`, `Metrics`, `Sources`, `Tables`, `Workspaces`Diffed against: <https://docs.bigeye.com/llms.txt>-- [ ] `Metric runs (POST /api/v1/metrics/runs)` — measure observed values, failed checks, and data quality trends over time (high)-- [ ] `Metric scorecard (POST /api/v1/metrics/scorecard)` — report monitoring coverage and successful checks over time, which metric configuration cannot provide (high)-- [ ] `Schema changes (POST /api/v1/schema-changes/fetch)` — track schema changes and connect added, removed, or changed fields to data quality incidents (medium)+- [x] `Metric runs (POST /api/v1/metrics/runs)` — measure observed values, failed checks, and data quality trends over time (high)+- [x] `Metric scorecard (POST /api/v1/metrics/scorecard)` — report monitoring coverage and successful checks over time, which metric configuration cannot provide (high)+- [x] `Schema changes (POST /api/v1/schema-changes/fetch)` — track schema changes and connect added, removed, or changed fields to data quality incidents (medium)- [ ] `Deltas (POST /api/v1/deltas/fetch)` — analyze data comparison results to find replication failures and inconsistent datasets (medium)-- [ ] `Metric revisions (GET /api/v1/metrics/{metricId}/revisions)` — track changes to checks and thresholds when investigating changes in alert frequency (medium)+- [x] `Metric revisions (GET /api/v1/metrics/{metricId}/revisions)` — track changes to checks and thresholds when investigating changes in alert frequency (medium)- [ ] `Lineage graph (GET /api/v2/lineage/nodes/{dataNodeId}/graph)` — connect data incidents to affected tables and downstream assets (medium)Note: The official index links supplied OpenAPI definitions for every listed gap. Metrics contains configuration; observed results need separate calls for each metric. Read-only POST collections are included alongside the lineage GET route. Embedded catalog details, issue events, settings, keys, and execution commands are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.pyindex 9627130d9094..43b091911c87 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py@@ -1,4 +1,6 @@import re+from bisect import bisect_left+from collections.abc import Iteratorfrom typing import Any, Optional, castimport requests@@ -8,6 +10,8 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigeye.settings import (BIGEYE_ENDPOINTS,REQUEST_TIMEOUT_SECONDS,+ SCORECARD_DAYS_OF_HISTORY,+ BigeyeEndpointConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe@@ -15,14 +19,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (+ create_response_hooks,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,JSONResponseCursorPaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (Endpoint,EndpointResource,HTTPMethodBasic,+ ResponseAction,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -40,10 +51,17 @@ WORKSPACE_ID_REQUIRED_MESSAGE = "A workspace ID must be supplied"_HOST_RE = re.compile(r"^https?://", re.IGNORECASE)+# A metric deleted between the Metrics listing and its own request returns 404; skip it instead of+# failing the whole fan-out.+_PER_METRIC_RESPONSE_ACTIONS: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]+@frozenclass BigeyeResumeConfig:next_cursor: str+ # Set only by the per-metric endpoints: the metric to resume from. `next_cursor` is then that+ # metric's page cursor, or empty to start the metric from its first page.+ metric_id: Optional[int] = Nonedef normalize_host(host: Optional[str]) -> str:@@ -70,6 +88,19 @@ def _flatten_collection(item: dict[str, Any]) -> dict[str, Any]:return {**item, "id": config.get("id"), "name": config.get("name")}+def _flatten_revision(item: dict[str, Any]) -> dict[str, Any]:+ info = item.get("revisionInfo") or {}+ return {**item, "revision_epoch_seconds": info.get("epochSeconds"), "revision_type": info.get("type")}+++def _explode_scorecard(item: dict[str, Any]) -> list[dict[str, Any]]:+ category = item.get("metricCategory")+ return [+ {**counts, "metric_category": category, "run_at_epoch_seconds": counts.get("runAtMinute")}+ for counts in item.get("metricRunCounts") or []+ ]++def get_resource(name: str, workspace_id: Optional[int]) -> EndpointResource:endpoint_config = BIGEYE_ENDPOINTS[name]@@ -93,6 +124,11 @@ def get_resource(name: str, workspace_id: Optional[int]) -> EndpointResource:endpoint["paginator"] = SinglePagePaginator()if name == "Collections" and workspace_id is not None:endpoint["params"] = {"workspaceId": workspace_id}+ if name == "MetricScorecard":+ scorecard_body: dict[str, Any] = {"daysOfHistory": SCORECARD_DAYS_OF_HISTORY}+ if workspace_id is not None:+ scorecard_body["where"] = {"workspaceId": workspace_id}+ endpoint["json"] = scorecard_bodyresource: EndpointResource = {"name": name,@@ -103,9 +139,95 @@ def get_resource(name: str, workspace_id: Optional[int]) -> EndpointResource:}if name == "Collections":resource["data_map"] = _flatten_collection+ elif name == "MetricScorecard":+ resource["data_map"] = _explode_scorecardreturn resource+def _list_metric_ids(client: RESTClient) -> list[int]:+ metrics_config = BIGEYE_ENDPOINTS["Metrics"]+ metric_ids: set[int] = set()+ for page in client.paginate(+ path=metrics_config.path, paginator=SinglePagePaginator(), data_selector=metrics_config.data_selector+ ):+ metric_ids.update(int(metric["id"]) for metric in page if metric.get("id") is not None)+ # Sorted so a resumed run walks the metrics in the same order as the run that saved the state.+ return sorted(metric_ids)+++def _per_metric_request(endpoint_config: BigeyeEndpointConfig, metric_id: int) -> dict[str, Any]:+ paginator: BasePaginator+ if endpoint_config.paginated:+ paginator = JSONResponseCursorPaginator(+ cursor_path="paginationInfo.nextCursor",+ cursor_param="pageCursor",+ param_location="json",+ )+ return {+ "path": endpoint_config.path,+ "method": endpoint_config.method,+ "json": {"metricIdentifier": {"metricId": metric_id}, "pageSize": PAGE_SIZE},+ "paginator": paginator,+ }+ paginator = SinglePagePaginator()+ return {+ "path": endpoint_config.path.format(metricId=metric_id),+ "method": endpoint_config.method,+ "paginator": paginator,+ }+++def _iter_per_metric(+ client: RESTClient,+ endpoint_config: BigeyeEndpointConfig,+ resumable_source_manager: ResumableSourceManager[BigeyeResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ metric_ids = _list_metric_ids(client)++ start_index = 0+ resume_cursor: Optional[str] = None+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None and resume.metric_id is not None:+ # A saved metric deleted since the checkpoint resumes at the next metric id, not at the start.+ start_index = bisect_left(metric_ids, resume.metric_id)+ if start_index < len(metric_ids) and metric_ids[start_index] == resume.metric_id:+ resume_cursor = resume.next_cursor or None++ hooks = create_response_hooks(_PER_METRIC_RESPONSE_ACTIONS, resource_name=endpoint_config.name)+ row_map = _flatten_revision if endpoint_config.name == "MetricRevisions" else None+ next_page: dict[str, Optional[str]] = {}++ def remember_next_page(state: Optional[dict[str, Any]], has_next_page: bool) -> None:+ cursor = (state or {}).get("cursor") if has_next_page else None+ next_page["cursor"] = str(cursor) if cursor else None++ for index in range(start_index, len(metric_ids)):+ metric_id = metric_ids[index]+ next_metric_id = metric_ids[index + 1] if index + 1 < len(metric_ids) else None+ cursor = resume_cursor if index == start_index else None+ next_page.clear()++ for page in client.paginate(+ **_per_metric_request(endpoint_config, metric_id),+ data_selector=endpoint_config.data_selector,+ hooks=hooks,+ page_state_hook=remember_next_page,+ initial_paginator_state={"cursor": cursor} if cursor else None,+ ):+ next_cursor = next_page.get("cursor")+ if next_cursor:+ resumable_source_manager.save_state(BigeyeResumeConfig(next_cursor=next_cursor, metric_id=metric_id))+ elif next_metric_id is not None:+ resumable_source_manager.save_state(BigeyeResumeConfig(next_cursor="", metric_id=next_metric_id))++ rows = [{**(row_map(row) if row_map else row), "metric_id": metric_id} for row in page]+ if rows:+ yield rows++ # Most metrics return few or no rows, so a long run of requests can pass without a yield.+ resumable_source_manager.safe_point()++def bigeye_source(api_key: str,host: Optional[str],@@ -117,6 +239,23 @@ def bigeye_source() -> SourceResponse:endpoint_config = BIGEYE_ENDPOINTS[endpoint]+ if endpoint_config.per_metric:+ client = RESTClient(+ base_url=_base_url(host),+ auth=APIKeyAuth(api_key=_auth_header_value(api_key), name="Authorization", location="header"),+ # The per-metric requests take no `workspaceId` parameter, so scope them with Bigeye's header.+ headers={"X-Bigeye-Workspace-Id": str(workspace_id)} if workspace_id is not None else None,+ request_timeout=REQUEST_TIMEOUT_SECONDS,+ allow_redirects=False,+ )+ return SourceResponse(+ name=endpoint_config.name,+ items=lambda: _iter_per_metric(client, endpoint_config, resumable_source_manager),+ primary_keys=list(endpoint_config.primary_keys),+ partition_count=1,+ partition_size=1,+ )+config: RESTAPIConfig = {"client": {"base_url": _base_url(host),@@ -158,7 +297,7 @@ def bigeye_source(return SourceResponse(name=resource.name,items=lambda: resource,- primary_keys=[endpoint_config.primary_key],+ primary_keys=list(endpoint_config.primary_keys),column_hints=resource.column_hints,# None of these tables expose a stable datetime field worth partitioning on (see# settings.py) — small metadata catalogs, full refresh only.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.pyindex 72681bd94bbc..1aba198fde0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py@@ -77,4 +77,58 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"metricIds": "Identifiers of the metrics alerting on this issue.",},},+ "SchemaChanges": {+ "description": "A schema change Bigeye detected in a monitored source: a schema, table, or column created, deleted, or retyped.",+ "docs_url": "https://docs.bigeye.com/reference/schemachangeservice_getschemachanges-1",+ "columns": {+ "id": "Unique identifier for the schema change.",+ "changeType": "Kind of change: created, deleted, or type changed.",+ "objectType": "Kind of catalog object that changed (schema, table, or column).",+ "detectedAt": "Unix timestamp (seconds) Bigeye detected the change.",+ "fullyQualifiedName": "Fully qualified name of the changed object.",+ "oldValue": "Value before the change, such as the previous column type.",+ "newValue": "Value after the change, such as the new column type.",+ "sourceId": "Identifier of the source that contains the changed object.",+ "tableId": "Identifier of the table that contains the changed object.",+ "columnId": "Identifier of the changed column, for column changes.",+ },+ },+ "MetricRuns": {+ "description": "One execution of a Bigeye metric: the observed value, the thresholds it was checked against, and the result.",+ "docs_url": "https://docs.bigeye.com/reference/metricservice_getmetricruns-1",+ "columns": {+ "id": "Identifier of the metric run.",+ "metric_id": "Identifier of the metric this run belongs to.",+ "runAtEpochSeconds": "Unix timestamp (seconds) the metric ran.",+ "observedValue": "Value the metric measured on this run.",+ "thresholds": "Thresholds the observed value was checked against.",+ "status": "Result of the run against its thresholds (e.g. ok, upper bound critical).",+ "grainStartEpochSeconds": "Start of the data window the run measured, for metric-time based runs.",+ "grainEndEpochSeconds": "End of the data window the run measured, for metric-time based runs.",+ "rowsScanned": "Number of rows the run scanned.",+ "failureReason": "Why the run failed to complete, if it did.",+ },+ },+ "MetricRevisions": {+ "description": "One revision in a metric's history: the metric configuration after a create, update, or delete.",+ "docs_url": "https://docs.bigeye.com/reference/metricservice_getrevisions-1",+ "columns": {+ "metric_id": "Identifier of the revised metric.",+ "revision_epoch_seconds": "Unix timestamp (seconds) of the revision.",+ "revision_type": "Kind of revision: create, update, or delete.",+ "metric": "Full metric configuration as of this revision, including its thresholds.",+ "revisionInfo": "Revision details, including the user who made the change.",+ "collectionIds": "Identifiers of the collections the metric belonged to at this revision.",+ },+ },+ "MetricScorecard": {+ "description": "Daily metric scorecard counts per metric category, for the last 90 days.",+ "docs_url": "https://docs.bigeye.com/reference/metricservice_getscorecard-1",+ "columns": {+ "metric_category": "Metric category the counts belong to (e.g. freshness and volume, completeness, validity).",+ "run_at_epoch_seconds": "Unix timestamp (seconds) of the day the counts cover.",+ "hadMetricsDeployed": "Whether metrics in this category were deployed on this day.",+ "truePositiveIssuesOpened": "Number of true positive issues opened on this day.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.pyindex 9a08185fef66..6a7af82f6455 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py@@ -23,7 +23,10 @@ class BigeyeEndpointConfig:# True for the `POST .../fetch` endpoints that page via a `pageCursor` in the JSON body.# False for the small GET endpoints that return their full result in one response.paginated: bool- primary_key: str = "id"+ primary_keys: tuple[str, ...] = ("id",)+ # True for endpoints scoped to one metric. They fan out over the Metrics list, and each row+ # carries the parent's id as `metric_id`.+ per_metric: bool = False# Bigeye's v1 API has no documented "updated since" filter on any list endpoint (see@@ -72,7 +75,7 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {paginated=False,# Rows nest their id under `collectionConfiguration.id`; `_flatten_collection` copies it# to the top level before this primary key is applied.- primary_key="id",+ primary_keys=("id",),),"Issues": BigeyeEndpointConfig(name="Issues",@@ -83,8 +86,50 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {data_selector="issue",paginated=True,),+ "SchemaChanges": BigeyeEndpointConfig(+ name="SchemaChanges",+ table_name="schema_changes",+ path="/api/v1/schema-changes/fetch",+ method="post",+ data_selector="schemaChanges",+ paginated=True,+ ),+ "MetricRuns": BigeyeEndpointConfig(+ name="MetricRuns",+ table_name="metric_runs",+ path="/api/v1/metrics/runs",+ method="post",+ data_selector="metricRuns",+ paginated=True,+ per_metric=True,+ primary_keys=("metric_id", "id"),+ ),+ "MetricRevisions": BigeyeEndpointConfig(+ name="MetricRevisions",+ table_name="metric_revisions",+ path="/api/v1/metrics/{metricId}/revisions",+ method="get",+ data_selector="revisions",+ paginated=False,+ per_metric=True,+ # Revisions have no id of their own. `_flatten_revision` lifts these out of `revisionInfo`.+ primary_keys=("metric_id", "revision_epoch_seconds", "revision_type"),+ ),+ "MetricScorecard": BigeyeEndpointConfig(+ name="MetricScorecard",+ table_name="metric_scorecard",+ path="/api/v1/metrics/scorecard",+ method="post",+ data_selector="scorecards",+ paginated=False,+ # `_explode_scorecard` emits one row per metric category per day.+ primary_keys=("metric_category", "run_at_epoch_seconds"),+ ),}+# The scorecard API's documented maximum history window.+SCORECARD_DAYS_OF_HISTORY = 90+ENDPOINTS = tuple(BIGEYE_ENDPOINTS.keys())# No endpoint exposes a server-side "updated since" / "modified after" filter in the publisheddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.pyindex 40061b2c318d..89aea477720d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py@@ -208,6 +208,112 @@ class TestCursorPagination:assert result.requests[0].json() == {"pageSize": 100, "pageCursor": "resumed"}+class TestPerMetricEndpoints:+ def test_metric_runs_walk_each_metric_and_stage_the_next_position(self) -> None:+ result = _driver(workspace_id=7).run(+ "MetricRuns",+ [+ ScriptedResponse(json={"metrics": [{"id": 2}, {"id": 1}]}),+ ScriptedResponse(json={"metricRuns": [{"id": 10}], "paginationInfo": {"nextCursor": "c1"}}),+ ScriptedResponse(json={"metricRuns": [{"id": 11}], "paginationInfo": {"nextCursor": ""}}),+ ScriptedResponse(json={"metricRuns": [{"id": 20}], "paginationInfo": {"nextCursor": ""}}),+ ],+ )++ assert result.raised is None+ assert [(row["metric_id"], row["id"]) for row in result.rows] == [(1, 10), (1, 11), (2, 20)]+ assert result.paths == ["/api/v1/metrics"] + ["/api/v1/metrics/runs"] * 3+ assert all(request.headers.get("x-bigeye-workspace-id") == "7" for request in result.requests)+ assert [request.json() for request in result.requests[1:]] == [+ {"metricIdentifier": {"metricId": 1}, "pageSize": 100},+ {"metricIdentifier": {"metricId": 1}, "pageSize": 100, "pageCursor": "c1"},+ {"metricIdentifier": {"metricId": 2}, "pageSize": 100},+ ]+ assert result.saved_states == [+ BigeyeResumeConfig(next_cursor="c1", metric_id=1),+ BigeyeResumeConfig(next_cursor="", metric_id=2),+ ]++ @pytest.mark.parametrize(+ ("listed_metric_ids", "expected_body"),+ [+ ([1, 2, 3], {"metricIdentifier": {"metricId": 2}, "pageSize": 100, "pageCursor": "r"}),+ # The saved metric was deleted: resume at the next id, without its stale cursor.+ ([1, 3], {"metricIdentifier": {"metricId": 3}, "pageSize": 100}),+ ],+ )+ def test_metric_runs_resume_from_saved_metric(+ self, listed_metric_ids: list[int], expected_body: dict[str, Any]+ ) -> None:+ result = _driver().run(+ "MetricRuns",+ [+ ScriptedResponse(json={"metrics": [{"id": metric_id} for metric_id in listed_metric_ids]}),+ ScriptedResponse(json={"metricRuns": [{"id": 21}], "paginationInfo": {"nextCursor": ""}}),+ ScriptedResponse(json={"metricRuns": [], "paginationInfo": {"nextCursor": ""}}),+ ],+ resume_state=BigeyeResumeConfig(next_cursor="r", metric_id=2),+ )++ assert result.raised is None+ assert result.requests[1].json() == expected_body++ def test_metric_revisions_flatten_revision_info_and_skip_deleted_metrics(self) -> None:+ result = _driver().run(+ "MetricRevisions",+ [+ ScriptedResponse(json={"metrics": [{"id": 1}, {"id": 2}]}),+ ScriptedResponse(status=404, json={"message": "metric not found"}),+ ScriptedResponse(+ json={+ "revisions": [+ {+ "metric": {"id": 2},+ "revisionInfo": {"epochSeconds": "1700000000", "type": "REVISION_TYPE_UPDATE"},+ }+ ]+ }+ ),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/api/v1/metrics", "/api/v1/metrics/1/revisions", "/api/v1/metrics/2/revisions"]+ assert [(row["metric_id"], row["revision_epoch_seconds"], row["revision_type"]) for row in result.rows] == [+ (2, "1700000000", "REVISION_TYPE_UPDATE")+ ]+++class TestMetricScorecard:+ def test_explodes_daily_counts_per_category(self) -> None:+ result = _driver(workspace_id=7).run(+ "MetricScorecard",+ [+ ScriptedResponse(+ json={+ "scorecards": [+ {+ "metricCategory": "METRIC_CATEGORY_COMPLETENESS",+ "metricRunCounts": [+ {"runAtMinute": "1700000000", "truePositiveIssuesOpened": 2},+ {"runAtMinute": "1700086400", "truePositiveIssuesOpened": 0},+ ],+ },+ {"metricCategory": "METRIC_CATEGORY_VALIDITY"},+ ]+ }+ )+ ],+ )++ assert result.raised is None+ assert result.requests[0].json() == {"daysOfHistory": 90, "where": {"workspaceId": 7}}+ assert [(row["metric_category"], row["run_at_epoch_seconds"]) for row in result.rows] == [+ ("METRIC_CATEGORY_COMPLETENESS", "1700000000"),+ ("METRIC_CATEGORY_COMPLETENESS", "1700086400"),+ ]++class TestBigeyeSourceResponse:def test_auth_header_carries_apikey_prefix(self) -> None:# Bigeye's auth header is `apikey <token>`, not the framework's default `Bearer <token>`.