:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 2 missing tables to the Bigeye source (#124652)
Tom Owersd003a1db55bf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c439411dd2a0..456a3c9a3215 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1586,9 +1586,9 @@ Diffed against: <https://docs.bigeye.com/llms.txt>- [x] `Metric runs (POST /api/v1/metrics/runs)` — measure observed values, failed checks, and data quality trends over time (high)- [x] `Metric scorecard (POST /api/v1/metrics/scorecard)` — report monitoring coverage and successful checks over time, which metric configuration cannot provide (high)- [x] `Schema changes (POST /api/v1/schema-changes/fetch)` — track schema changes and connect added, removed, or changed fields to data quality incidents (medium)-- [ ] `Deltas (POST /api/v1/deltas/fetch)` — analyze data comparison results to find replication failures and inconsistent datasets (medium)+- [x] `Deltas (POST /api/v1/deltas/fetch)` — analyze data comparison results to find replication failures and inconsistent datasets (medium)- [x] `Metric revisions (GET /api/v1/metrics/{metricId}/revisions)` — track changes to checks and thresholds when investigating changes in alert frequency (medium)-- [ ] `Lineage graph (GET /api/v2/lineage/nodes/{dataNodeId}/graph)` — connect data incidents to affected tables and downstream assets (medium)+- [x] `Lineage graph (GET /api/v2/lineage/nodes/{dataNodeId}/graph)` — connect data incidents to affected tables and downstream assets (medium)Note: The official index links supplied OpenAPI definitions for every listed gap. Metrics contains configuration; observed results need separate calls for each metric. Read-only POST collections are included alongside the lineage GET route. Embedded catalog details, issue events, settings, keys, and execution commands are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.pyindex 43b091911c87..54be5871240c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py@@ -9,6 +9,7 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigeye.settings import (BIGEYE_ENDPOINTS,+ LINEAGE_GRAPH_DEPTH,REQUEST_TIMEOUT_SECONDS,SCORECARD_DAYS_OF_HISTORY,BigeyeEndpointConfig,@@ -51,9 +52,9 @@ WORKSPACE_ID_REQUIRED_MESSAGE = "A workspace ID must be supplied"_HOST_RE = re.compile(r"^https?://", re.IGNORECASE)-# A metric deleted between the Metrics listing and its own request returns 404; skip it instead of-# failing the whole fan-out.-_PER_METRIC_RESPONSE_ACTIONS: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]+# A metric or data node deleted between its listing and its own request returns 404; skip it instead+# of failing the whole fan-out.+_FAN_OUT_RESPONSE_ACTIONS: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]@frozen@@ -62,6 +63,8 @@ class BigeyeResumeConfig:# Set only by the per-metric endpoints: the metric to resume from. `next_cursor` is then that# metric's page cursor, or empty to start the metric from its first page.metric_id: Optional[int] = None+ # Set only by the per-data-node endpoints: the lineage data node to resume from.+ data_node_id: Optional[int] = Nonedef normalize_host(host: Optional[str]) -> str:@@ -88,6 +91,11 @@ def _flatten_collection(item: dict[str, Any]) -> dict[str, Any]:return {**item, "id": config.get("id"), "name": config.get("name")}+def _flatten_delta(item: dict[str, Any]) -> dict[str, Any]:+ delta = item.get("delta") or {}+ return {**item, "id": delta.get("id"), "name": delta.get("name")}++def _flatten_revision(item: dict[str, Any]) -> dict[str, Any]:info = item.get("revisionInfo") or {}return {**item, "revision_epoch_seconds": info.get("epochSeconds"), "revision_type": info.get("type")}@@ -139,6 +147,8 @@ def get_resource(name: str, workspace_id: Optional[int]) -> EndpointResource:}if name == "Collections":resource["data_map"] = _flatten_collection+ elif name == "Deltas":+ resource["data_map"] = _flatten_deltaelif name == "MetricScorecard":resource["data_map"] = _explode_scorecardreturn resource@@ -155,6 +165,85 @@ def _list_metric_ids(client: RESTClient) -> list[int]:return sorted(metric_ids)+def _list_table_data_node_ids(+ client: RESTClient,+ workspace_id: Optional[int],+ resumable_source_manager: ResumableSourceManager[BigeyeResumeConfig],+) -> list[int]:+ tables_config = BIGEYE_ENDPOINTS["Tables"]+ body: dict[str, Any] = {"pageSize": PAGE_SIZE, "includeDataNodeIds": True, "ignoreFields": True}+ if workspace_id is not None:+ body["workspaceId"] = workspace_id+ data_node_ids: set[int] = set()+ for page in client.paginate(+ path=tables_config.path,+ method=tables_config.method,+ json=body,+ paginator=JSONResponseCursorPaginator(+ cursor_path="paginationInfo.nextCursor",+ cursor_param="pageCursor",+ param_location="json",+ ),+ data_selector=tables_config.data_selector,+ ):+ # Tables without lineage have no data node and so no graph to fetch.+ data_node_ids.update(int(table["dataNodeId"]) for table in page if table.get("dataNodeId"))+ # Nothing is staged or held yet, so a hand-off here only lists the tables again.+ resumable_source_manager.safe_point()+ return sorted(data_node_ids)+++def _lineage_graph_rows(root_data_node_id: int, page: list[dict[str, Any]]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for nodes in page:+ for node_id, node in (nodes or {}).items():+ lineage_node = node.get("lineageNode") or {}+ rows.append(+ {+ **node,+ "id": lineage_node.get("id", int(node_id)),+ "node_name": lineage_node.get("nodeName"),+ "root_data_node_id": root_data_node_id,+ }+ )+ return rows+++def _iter_lineage_graphs(+ client: RESTClient,+ endpoint_config: BigeyeEndpointConfig,+ workspace_id: Optional[int],+ resumable_source_manager: ResumableSourceManager[BigeyeResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ data_node_ids = _list_table_data_node_ids(client, workspace_id, resumable_source_manager)++ start_index = 0+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None and resume.data_node_id is not None:+ start_index = bisect_left(data_node_ids, resume.data_node_id)++ hooks = create_response_hooks(_FAN_OUT_RESPONSE_ACTIONS, resource_name=endpoint_config.name)++ for index in range(start_index, len(data_node_ids)):+ data_node_id = data_node_ids[index]+ for page in client.paginate(+ path=endpoint_config.path.format(dataNodeId=data_node_id),+ params={"depth": LINEAGE_GRAPH_DEPTH, "direction": "ALL"},+ paginator=SinglePagePaginator(),+ data_selector=endpoint_config.data_selector,+ hooks=hooks,+ ):+ rows = _lineage_graph_rows(data_node_id, page)+ if index + 1 < len(data_node_ids):+ resumable_source_manager.save_state(+ BigeyeResumeConfig(next_cursor="", data_node_id=data_node_ids[index + 1])+ )+ if rows:+ yield rows++ resumable_source_manager.safe_point()++def _per_metric_request(endpoint_config: BigeyeEndpointConfig, metric_id: int) -> dict[str, Any]:paginator: BasePaginatorif endpoint_config.paginated:@@ -193,7 +282,7 @@ def _iter_per_metric(if start_index < len(metric_ids) and metric_ids[start_index] == resume.metric_id:resume_cursor = resume.next_cursor or None- hooks = create_response_hooks(_PER_METRIC_RESPONSE_ACTIONS, resource_name=endpoint_config.name)+ hooks = create_response_hooks(_FAN_OUT_RESPONSE_ACTIONS, resource_name=endpoint_config.name)row_map = _flatten_revision if endpoint_config.name == "MetricRevisions" else Nonenext_page: dict[str, Optional[str]] = {}@@ -239,18 +328,23 @@ def bigeye_source() -> SourceResponse:endpoint_config = BIGEYE_ENDPOINTS[endpoint]- if endpoint_config.per_metric:+ if endpoint_config.per_metric or endpoint_config.per_table_data_node:client = RESTClient(base_url=_base_url(host),auth=APIKeyAuth(api_key=_auth_header_value(api_key), name="Authorization", location="header"),- # The per-metric requests take no `workspaceId` parameter, so scope them with Bigeye's header.+ # The fan-out requests take no `workspaceId` parameter, so scope them with Bigeye's header.headers={"X-Bigeye-Workspace-Id": str(workspace_id)} if workspace_id is not None else None,request_timeout=REQUEST_TIMEOUT_SECONDS,allow_redirects=False,)+ items = (+ (lambda: _iter_lineage_graphs(client, endpoint_config, workspace_id, resumable_source_manager))+ if endpoint_config.per_table_data_node+ else (lambda: _iter_per_metric(client, endpoint_config, resumable_source_manager))+ )return SourceResponse(name=endpoint_config.name,- items=lambda: _iter_per_metric(client, endpoint_config, resumable_source_manager),+ items=items,primary_keys=list(endpoint_config.primary_keys),partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.pyindex 1aba198fde0d..db4d42d73ec1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py@@ -131,4 +131,33 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"truePositiveIssuesOpened": "Number of true positive issues opened on this day.",},},+ "Deltas": {+ "description": "A Bigeye Delta: a comparison between a source table and one or more target tables, with the result of its last run.",+ "docs_url": "https://docs.bigeye.com/reference/deltaservice_fetchdeltas-1",+ "columns": {+ "id": "Identifier of the delta, copied from `delta.id`.",+ "name": "Display name of the delta, copied from `delta.name`.",+ "delta": "Delta configuration, including the source table, schedule, and metric counts of the last run.",+ "comparisonTargetInfos": "Per-target comparison results, including row counts, key matches, and failed metrics.",+ "activeIssue": "The open issue for this delta, if one exists.",+ "nextRunEpochSeconds": "Unix timestamp (seconds) of the next scheduled run.",+ "isFavorite": "Whether the delta is marked as a favorite in the Bigeye UI.",+ },+ },+ "LineageGraph": {+ "description": "The lineage nodes directly upstream and downstream of each Bigeye table, one row per table and node.",+ "docs_url": "https://docs.bigeye.com/reference/lineagev2service_getlineagegraphfornode-1",+ "columns": {+ "root_data_node_id": "Data node id of the table whose lineage graph contains this node.",+ "id": "Data node id of this node.",+ "node_name": "Name of this node, copied from `lineageNode.nodeName`.",+ "lineageNode": "Node details, including its type, catalog path, and source.",+ "upstreamEdges": "Edges from the nodes directly upstream of this node.",+ "downstreamEdges": "Edges to the nodes directly downstream of this node.",+ "isUpstream": "Whether this node is upstream of the root table.",+ "isDownstream": "Whether this node is downstream of the root table.",+ "issueCount": "Number of open issues on this node.",+ "metricCount": "Number of metrics on this node.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.pyindex 6a7af82f6455..e64daa8b3296 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py@@ -27,6 +27,9 @@ class BigeyeEndpointConfig:# True for endpoints scoped to one metric. They fan out over the Metrics list, and each row# carries the parent's id as `metric_id`.per_metric: bool = False+ # True for endpoints scoped to one lineage data node. They fan out over the data node ids of the+ # Tables list, and each row carries the parent's node id as `root_data_node_id`.+ per_table_data_node: bool = False# Bigeye's v1 API has no documented "updated since" filter on any list endpoint (see@@ -86,6 +89,16 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {data_selector="issue",paginated=True,),+ "Deltas": BigeyeEndpointConfig(+ name="Deltas",+ table_name="deltas",+ path="/api/v1/deltas/fetch",+ method="post",+ data_selector="deltaInfos",+ paginated=True,+ # Rows nest their id under `delta.id`; `_flatten_delta` copies it to the top level.+ primary_keys=("id",),+ ),"SchemaChanges": BigeyeEndpointConfig(name="SchemaChanges",table_name="schema_changes",@@ -125,8 +138,23 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {# `_explode_scorecard` emits one row per metric category per day.primary_keys=("metric_category", "run_at_epoch_seconds"),),+ "LineageGraph": BigeyeEndpointConfig(+ name="LineageGraph",+ table_name="lineage_graph",+ path="/api/v2/lineage/nodes/{dataNodeId}/graph",+ method="get",+ # A map of node id -> node, not a list. `_lineage_graph_rows` emits one row per node.+ data_selector="nodes",+ paginated=False,+ per_table_data_node=True,+ primary_keys=("root_data_node_id", "id"),+ ),}+# Only the direct neighbours of each table. Every edge still appears in the graph of both of its+# tables, and an unbounded depth repeats the whole connected graph once per table in it.+LINEAGE_GRAPH_DEPTH = 1+# The scorecard API's documented maximum history window.SCORECARD_DAYS_OF_HISTORY = 90diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.pyindex 0af2dee241f4..3f9a049018cc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py@@ -124,11 +124,18 @@ class TestSinglePageEndpoints:assert result.raised is Noneassert [row["id"] for row in result.rows] == [5]- def test_collections_flattens_nested_id(self) -> None:- result = _driver().run(- "Collections",- [ScriptedResponse(json={"collectionInfos": [{"collectionConfiguration": {"id": 9, "name": "Nightly"}}]})],- )+ @pytest.mark.parametrize(+ ("endpoint", "body"),+ [+ ("Collections", {"collectionInfos": [{"collectionConfiguration": {"id": 9, "name": "Nightly"}}]}),+ (+ "Deltas",+ {"deltaInfos": [{"delta": {"id": 9, "name": "Nightly"}}], "paginationInfo": {"nextCursor": ""}},+ ),+ ],+ )+ def test_flattens_nested_id(self, endpoint: str, body: dict[str, Any]) -> None:+ result = _driver().run(endpoint, [ScriptedResponse(json=body)])assert result.raised is Noneassert result.rows[0]["id"] == 9@@ -283,6 +290,106 @@ class TestPerMetricEndpoints:]+def _lineage_graph(*node_ids: int) -> dict[str, Any]:+ return {+ "nodes": {+ str(node_id): {"lineageNode": {"id": node_id, "nodeName": f"node_{node_id}"}, "issueCount": 0}+ for node_id in node_ids+ }+ }+++class TestLineageGraph:+ def test_walks_each_table_data_node_and_stages_the_next_node(self) -> None:+ result = _driver(workspace_id=7).run(+ "LineageGraph",+ [+ ScriptedResponse(+ json={+ # A table with no data node has no lineage and gets no graph request.+ "tables": [{"id": 1, "dataNodeId": 30}, {"id": 2}],+ "paginationInfo": {"nextCursor": "c1"},+ }+ ),+ ScriptedResponse(json={"tables": [{"id": 3, "dataNodeId": 10}], "paginationInfo": {"nextCursor": ""}}),+ ScriptedResponse(json=_lineage_graph(10, 11)),+ ScriptedResponse(json=_lineage_graph(30, 10)),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/api/v1/tables/fetch"] * 2 + [+ "/api/v2/lineage/nodes/10/graph",+ "/api/v2/lineage/nodes/30/graph",+ ]+ assert [request.json() for request in result.requests[:2]] == [+ {"pageSize": 100, "includeDataNodeIds": True, "ignoreFields": True, "workspaceId": 7},+ {"pageSize": 100, "includeDataNodeIds": True, "ignoreFields": True, "workspaceId": 7, "pageCursor": "c1"},+ ]+ assert result.queries[2:] == [{"depth": ("1",), "direction": ("ALL",)}] * 2+ assert all(request.headers.get("x-bigeye-workspace-id") == "7" for request in result.requests)+ assert [(row["root_data_node_id"], row["id"], row["node_name"]) for row in result.rows] == [+ (10, 10, "node_10"),+ (10, 11, "node_11"),+ (30, 30, "node_30"),+ (30, 10, "node_10"),+ ]+ assert result.saved_states == [BigeyeResumeConfig(next_cursor="", data_node_id=30)]++ @pytest.mark.parametrize(+ ("listed_node_ids", "expected_path"),+ [+ ([10, 20, 30], "/api/v2/lineage/nodes/20/graph"),+ # The saved node is gone: resume at the next node id.+ ([10, 30], "/api/v2/lineage/nodes/30/graph"),+ ],+ )+ def test_resumes_from_saved_node(self, listed_node_ids: list[int], expected_path: str) -> None:+ result = _driver().run(+ "LineageGraph",+ [+ ScriptedResponse(+ json={+ "tables": [{"id": node_id, "dataNodeId": node_id} for node_id in listed_node_ids],+ "paginationInfo": {"nextCursor": ""},+ }+ ),+ ScriptedResponse(json=_lineage_graph(1)),+ ScriptedResponse(json=_lineage_graph(2)),+ ],+ resume_state=BigeyeResumeConfig(next_cursor="", data_node_id=20),+ )++ assert result.raised is None+ assert result.paths[1] == expected_path++ @pytest.mark.parametrize(+ "graph_response",+ [+ ScriptedResponse(status=404, json={"message": "data node not found"}),+ # An empty graph omits the `nodes` map.+ ScriptedResponse(json={}),+ ],+ )+ def test_skips_missing_or_empty_graphs(self, graph_response: ScriptedResponse) -> None:+ result = _driver().run(+ "LineageGraph",+ [+ ScriptedResponse(+ json={+ "tables": [{"id": 1, "dataNodeId": 10}, {"id": 2, "dataNodeId": 20}],+ "paginationInfo": {"nextCursor": ""},+ }+ ),+ graph_response,+ ScriptedResponse(json=_lineage_graph(20)),+ ],+ )++ assert result.raised is None+ assert [(row["root_data_node_id"], row["id"]) for row in result.rows] == [(20, 20)]++class TestMetricScorecard:def test_explodes_daily_counts_per_category(self) -> None:result = _driver(workspace_id=7).run(