Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Bigeye source (#124652)

Tom Owersd003a1db55bf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c439411dd2a0..456a3c9a3215 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1586,9 +1586,9 @@ Diffed against: <https://docs.bigeye.com/llms.txt>
- [x] `Metric runs (POST /api/v1/metrics/runs)` — measure observed values, failed checks, and data quality trends over time (high)
- [x] `Metric scorecard (POST /api/v1/metrics/scorecard)` — report monitoring coverage and successful checks over time, which metric configuration cannot provide (high)
- [x] `Schema changes (POST /api/v1/schema-changes/fetch)` — track schema changes and connect added, removed, or changed fields to data quality incidents (medium)
-- [ ] `Deltas (POST /api/v1/deltas/fetch)` — analyze data comparison results to find replication failures and inconsistent datasets (medium)
+- [x] `Deltas (POST /api/v1/deltas/fetch)` — analyze data comparison results to find replication failures and inconsistent datasets (medium)
- [x] `Metric revisions (GET /api/v1/metrics/{metricId}/revisions)` — track changes to checks and thresholds when investigating changes in alert frequency (medium)
-- [ ] `Lineage graph (GET /api/v2/lineage/nodes/{dataNodeId}/graph)` — connect data incidents to affected tables and downstream assets (medium)
+- [x] `Lineage graph (GET /api/v2/lineage/nodes/{dataNodeId}/graph)` — connect data incidents to affected tables and downstream assets (medium)
Note: The official index links supplied OpenAPI definitions for every listed gap. Metrics contains configuration; observed results need separate calls for each metric. Read-only POST collections are included alongside the lineage GET route. Embedded catalog details, issue events, settings, keys, and execution commands are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py
index 43b091911c87..54be5871240c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/bigeye.py
@@ -9,6 +9,7 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.bigeye.settings import (
BIGEYE_ENDPOINTS,
+ LINEAGE_GRAPH_DEPTH,
REQUEST_TIMEOUT_SECONDS,
SCORECARD_DAYS_OF_HISTORY,
BigeyeEndpointConfig,
@@ -51,9 +52,9 @@ WORKSPACE_ID_REQUIRED_MESSAGE = "A workspace ID must be supplied"
_HOST_RE = re.compile(r"^https?://", re.IGNORECASE)
-# A metric deleted between the Metrics listing and its own request returns 404; skip it instead of
-# failing the whole fan-out.
-_PER_METRIC_RESPONSE_ACTIONS: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]
+# A metric or data node deleted between its listing and its own request returns 404; skip it instead
+# of failing the whole fan-out.
+_FAN_OUT_RESPONSE_ACTIONS: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]
@frozen
@@ -62,6 +63,8 @@ class BigeyeResumeConfig:
# Set only by the per-metric endpoints: the metric to resume from. `next_cursor` is then that
# metric's page cursor, or empty to start the metric from its first page.
metric_id: Optional[int] = None
+ # Set only by the per-data-node endpoints: the lineage data node to resume from.
+ data_node_id: Optional[int] = None
def normalize_host(host: Optional[str]) -> str:
@@ -88,6 +91,11 @@ def _flatten_collection(item: dict[str, Any]) -> dict[str, Any]:
return {**item, "id": config.get("id"), "name": config.get("name")}
+def _flatten_delta(item: dict[str, Any]) -> dict[str, Any]:
+ delta = item.get("delta") or {}
+ return {**item, "id": delta.get("id"), "name": delta.get("name")}
+
+
def _flatten_revision(item: dict[str, Any]) -> dict[str, Any]:
info = item.get("revisionInfo") or {}
return {**item, "revision_epoch_seconds": info.get("epochSeconds"), "revision_type": info.get("type")}
@@ -139,6 +147,8 @@ def get_resource(name: str, workspace_id: Optional[int]) -> EndpointResource:
}
if name == "Collections":
resource["data_map"] = _flatten_collection
+ elif name == "Deltas":
+ resource["data_map"] = _flatten_delta
elif name == "MetricScorecard":
resource["data_map"] = _explode_scorecard
return resource
@@ -155,6 +165,85 @@ def _list_metric_ids(client: RESTClient) -> list[int]:
return sorted(metric_ids)
+def _list_table_data_node_ids(
+ client: RESTClient,
+ workspace_id: Optional[int],
+ resumable_source_manager: ResumableSourceManager[BigeyeResumeConfig],
+) -> list[int]:
+ tables_config = BIGEYE_ENDPOINTS["Tables"]
+ body: dict[str, Any] = {"pageSize": PAGE_SIZE, "includeDataNodeIds": True, "ignoreFields": True}
+ if workspace_id is not None:
+ body["workspaceId"] = workspace_id
+ data_node_ids: set[int] = set()
+ for page in client.paginate(
+ path=tables_config.path,
+ method=tables_config.method,
+ json=body,
+ paginator=JSONResponseCursorPaginator(
+ cursor_path="paginationInfo.nextCursor",
+ cursor_param="pageCursor",
+ param_location="json",
+ ),
+ data_selector=tables_config.data_selector,
+ ):
+ # Tables without lineage have no data node and so no graph to fetch.
+ data_node_ids.update(int(table["dataNodeId"]) for table in page if table.get("dataNodeId"))
+ # Nothing is staged or held yet, so a hand-off here only lists the tables again.
+ resumable_source_manager.safe_point()
+ return sorted(data_node_ids)
+
+
+def _lineage_graph_rows(root_data_node_id: int, page: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ for nodes in page:
+ for node_id, node in (nodes or {}).items():
+ lineage_node = node.get("lineageNode") or {}
+ rows.append(
+ {
+ **node,
+ "id": lineage_node.get("id", int(node_id)),
+ "node_name": lineage_node.get("nodeName"),
+ "root_data_node_id": root_data_node_id,
+ }
+ )
+ return rows
+
+
+def _iter_lineage_graphs(
+ client: RESTClient,
+ endpoint_config: BigeyeEndpointConfig,
+ workspace_id: Optional[int],
+ resumable_source_manager: ResumableSourceManager[BigeyeResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ data_node_ids = _list_table_data_node_ids(client, workspace_id, resumable_source_manager)
+
+ start_index = 0
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.data_node_id is not None:
+ start_index = bisect_left(data_node_ids, resume.data_node_id)
+
+ hooks = create_response_hooks(_FAN_OUT_RESPONSE_ACTIONS, resource_name=endpoint_config.name)
+
+ for index in range(start_index, len(data_node_ids)):
+ data_node_id = data_node_ids[index]
+ for page in client.paginate(
+ path=endpoint_config.path.format(dataNodeId=data_node_id),
+ params={"depth": LINEAGE_GRAPH_DEPTH, "direction": "ALL"},
+ paginator=SinglePagePaginator(),
+ data_selector=endpoint_config.data_selector,
+ hooks=hooks,
+ ):
+ rows = _lineage_graph_rows(data_node_id, page)
+ if index + 1 < len(data_node_ids):
+ resumable_source_manager.save_state(
+ BigeyeResumeConfig(next_cursor="", data_node_id=data_node_ids[index + 1])
+ )
+ if rows:
+ yield rows
+
+ resumable_source_manager.safe_point()
+
+
def _per_metric_request(endpoint_config: BigeyeEndpointConfig, metric_id: int) -> dict[str, Any]:
paginator: BasePaginator
if endpoint_config.paginated:
@@ -193,7 +282,7 @@ def _iter_per_metric(
if start_index < len(metric_ids) and metric_ids[start_index] == resume.metric_id:
resume_cursor = resume.next_cursor or None
- hooks = create_response_hooks(_PER_METRIC_RESPONSE_ACTIONS, resource_name=endpoint_config.name)
+ hooks = create_response_hooks(_FAN_OUT_RESPONSE_ACTIONS, resource_name=endpoint_config.name)
row_map = _flatten_revision if endpoint_config.name == "MetricRevisions" else None
next_page: dict[str, Optional[str]] = {}
@@ -239,18 +328,23 @@ def bigeye_source(
) -> SourceResponse:
endpoint_config = BIGEYE_ENDPOINTS[endpoint]
- if endpoint_config.per_metric:
+ if endpoint_config.per_metric or endpoint_config.per_table_data_node:
client = RESTClient(
base_url=_base_url(host),
auth=APIKeyAuth(api_key=_auth_header_value(api_key), name="Authorization", location="header"),
- # The per-metric requests take no `workspaceId` parameter, so scope them with Bigeye's header.
+ # The fan-out requests take no `workspaceId` parameter, so scope them with Bigeye's header.
headers={"X-Bigeye-Workspace-Id": str(workspace_id)} if workspace_id is not None else None,
request_timeout=REQUEST_TIMEOUT_SECONDS,
allow_redirects=False,
)
+ items = (
+ (lambda: _iter_lineage_graphs(client, endpoint_config, workspace_id, resumable_source_manager))
+ if endpoint_config.per_table_data_node
+ else (lambda: _iter_per_metric(client, endpoint_config, resumable_source_manager))
+ )
return SourceResponse(
name=endpoint_config.name,
- items=lambda: _iter_per_metric(client, endpoint_config, resumable_source_manager),
+ items=items,
primary_keys=list(endpoint_config.primary_keys),
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py
index 1aba198fde0d..db4d42d73ec1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/canonical_descriptions.py
@@ -131,4 +131,33 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"truePositiveIssuesOpened": "Number of true positive issues opened on this day.",
},
},
+ "Deltas": {
+ "description": "A Bigeye Delta: a comparison between a source table and one or more target tables, with the result of its last run.",
+ "docs_url": "https://docs.bigeye.com/reference/deltaservice_fetchdeltas-1",
+ "columns": {
+ "id": "Identifier of the delta, copied from `delta.id`.",
+ "name": "Display name of the delta, copied from `delta.name`.",
+ "delta": "Delta configuration, including the source table, schedule, and metric counts of the last run.",
+ "comparisonTargetInfos": "Per-target comparison results, including row counts, key matches, and failed metrics.",
+ "activeIssue": "The open issue for this delta, if one exists.",
+ "nextRunEpochSeconds": "Unix timestamp (seconds) of the next scheduled run.",
+ "isFavorite": "Whether the delta is marked as a favorite in the Bigeye UI.",
+ },
+ },
+ "LineageGraph": {
+ "description": "The lineage nodes directly upstream and downstream of each Bigeye table, one row per table and node.",
+ "docs_url": "https://docs.bigeye.com/reference/lineagev2service_getlineagegraphfornode-1",
+ "columns": {
+ "root_data_node_id": "Data node id of the table whose lineage graph contains this node.",
+ "id": "Data node id of this node.",
+ "node_name": "Name of this node, copied from `lineageNode.nodeName`.",
+ "lineageNode": "Node details, including its type, catalog path, and source.",
+ "upstreamEdges": "Edges from the nodes directly upstream of this node.",
+ "downstreamEdges": "Edges to the nodes directly downstream of this node.",
+ "isUpstream": "Whether this node is upstream of the root table.",
+ "isDownstream": "Whether this node is downstream of the root table.",
+ "issueCount": "Number of open issues on this node.",
+ "metricCount": "Number of metrics on this node.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py
index 6a7af82f6455..e64daa8b3296 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/settings.py
@@ -27,6 +27,9 @@ class BigeyeEndpointConfig:
# True for endpoints scoped to one metric. They fan out over the Metrics list, and each row
# carries the parent's id as `metric_id`.
per_metric: bool = False
+ # True for endpoints scoped to one lineage data node. They fan out over the data node ids of the
+ # Tables list, and each row carries the parent's node id as `root_data_node_id`.
+ per_table_data_node: bool = False
# Bigeye's v1 API has no documented "updated since" filter on any list endpoint (see
@@ -86,6 +89,16 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {
data_selector="issue",
paginated=True,
),
+ "Deltas": BigeyeEndpointConfig(
+ name="Deltas",
+ table_name="deltas",
+ path="/api/v1/deltas/fetch",
+ method="post",
+ data_selector="deltaInfos",
+ paginated=True,
+ # Rows nest their id under `delta.id`; `_flatten_delta` copies it to the top level.
+ primary_keys=("id",),
+ ),
"SchemaChanges": BigeyeEndpointConfig(
name="SchemaChanges",
table_name="schema_changes",
@@ -125,8 +138,23 @@ BIGEYE_ENDPOINTS: dict[str, BigeyeEndpointConfig] = {
# `_explode_scorecard` emits one row per metric category per day.
primary_keys=("metric_category", "run_at_epoch_seconds"),
),
+ "LineageGraph": BigeyeEndpointConfig(
+ name="LineageGraph",
+ table_name="lineage_graph",
+ path="/api/v2/lineage/nodes/{dataNodeId}/graph",
+ method="get",
+ # A map of node id -> node, not a list. `_lineage_graph_rows` emits one row per node.
+ data_selector="nodes",
+ paginated=False,
+ per_table_data_node=True,
+ primary_keys=("root_data_node_id", "id"),
+ ),
}
+# Only the direct neighbours of each table. Every edge still appears in the graph of both of its
+# tables, and an unbounded depth repeats the whole connected graph once per table in it.
+LINEAGE_GRAPH_DEPTH = 1
+
# The scorecard API's documented maximum history window.
SCORECARD_DAYS_OF_HISTORY = 90
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py
index 0af2dee241f4..3f9a049018cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigeye/tests/test_bigeye.py
@@ -124,11 +124,18 @@ class TestSinglePageEndpoints:
assert result.raised is None
assert [row["id"] for row in result.rows] == [5]
- def test_collections_flattens_nested_id(self) -> None:
- result = _driver().run(
- "Collections",
- [ScriptedResponse(json={"collectionInfos": [{"collectionConfiguration": {"id": 9, "name": "Nightly"}}]})],
- )
+ @pytest.mark.parametrize(
+ ("endpoint", "body"),
+ [
+ ("Collections", {"collectionInfos": [{"collectionConfiguration": {"id": 9, "name": "Nightly"}}]}),
+ (
+ "Deltas",
+ {"deltaInfos": [{"delta": {"id": 9, "name": "Nightly"}}], "paginationInfo": {"nextCursor": ""}},
+ ),
+ ],
+ )
+ def test_flattens_nested_id(self, endpoint: str, body: dict[str, Any]) -> None:
+ result = _driver().run(endpoint, [ScriptedResponse(json=body)])
assert result.raised is None
assert result.rows[0]["id"] == 9
@@ -283,6 +290,106 @@ class TestPerMetricEndpoints:
]
+def _lineage_graph(*node_ids: int) -> dict[str, Any]:
+ return {
+ "nodes": {
+ str(node_id): {"lineageNode": {"id": node_id, "nodeName": f"node_{node_id}"}, "issueCount": 0}
+ for node_id in node_ids
+ }
+ }
+
+
+class TestLineageGraph:
+ def test_walks_each_table_data_node_and_stages_the_next_node(self) -> None:
+ result = _driver(workspace_id=7).run(
+ "LineageGraph",
+ [
+ ScriptedResponse(
+ json={
+ # A table with no data node has no lineage and gets no graph request.
+ "tables": [{"id": 1, "dataNodeId": 30}, {"id": 2}],
+ "paginationInfo": {"nextCursor": "c1"},
+ }
+ ),
+ ScriptedResponse(json={"tables": [{"id": 3, "dataNodeId": 10}], "paginationInfo": {"nextCursor": ""}}),
+ ScriptedResponse(json=_lineage_graph(10, 11)),
+ ScriptedResponse(json=_lineage_graph(30, 10)),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/v1/tables/fetch"] * 2 + [
+ "/api/v2/lineage/nodes/10/graph",
+ "/api/v2/lineage/nodes/30/graph",
+ ]
+ assert [request.json() for request in result.requests[:2]] == [
+ {"pageSize": 100, "includeDataNodeIds": True, "ignoreFields": True, "workspaceId": 7},
+ {"pageSize": 100, "includeDataNodeIds": True, "ignoreFields": True, "workspaceId": 7, "pageCursor": "c1"},
+ ]
+ assert result.queries[2:] == [{"depth": ("1",), "direction": ("ALL",)}] * 2
+ assert all(request.headers.get("x-bigeye-workspace-id") == "7" for request in result.requests)
+ assert [(row["root_data_node_id"], row["id"], row["node_name"]) for row in result.rows] == [
+ (10, 10, "node_10"),
+ (10, 11, "node_11"),
+ (30, 30, "node_30"),
+ (30, 10, "node_10"),
+ ]
+ assert result.saved_states == [BigeyeResumeConfig(next_cursor="", data_node_id=30)]
+
+ @pytest.mark.parametrize(
+ ("listed_node_ids", "expected_path"),
+ [
+ ([10, 20, 30], "/api/v2/lineage/nodes/20/graph"),
+ # The saved node is gone: resume at the next node id.
+ ([10, 30], "/api/v2/lineage/nodes/30/graph"),
+ ],
+ )
+ def test_resumes_from_saved_node(self, listed_node_ids: list[int], expected_path: str) -> None:
+ result = _driver().run(
+ "LineageGraph",
+ [
+ ScriptedResponse(
+ json={
+ "tables": [{"id": node_id, "dataNodeId": node_id} for node_id in listed_node_ids],
+ "paginationInfo": {"nextCursor": ""},
+ }
+ ),
+ ScriptedResponse(json=_lineage_graph(1)),
+ ScriptedResponse(json=_lineage_graph(2)),
+ ],
+ resume_state=BigeyeResumeConfig(next_cursor="", data_node_id=20),
+ )
+
+ assert result.raised is None
+ assert result.paths[1] == expected_path
+
+ @pytest.mark.parametrize(
+ "graph_response",
+ [
+ ScriptedResponse(status=404, json={"message": "data node not found"}),
+ # An empty graph omits the `nodes` map.
+ ScriptedResponse(json={}),
+ ],
+ )
+ def test_skips_missing_or_empty_graphs(self, graph_response: ScriptedResponse) -> None:
+ result = _driver().run(
+ "LineageGraph",
+ [
+ ScriptedResponse(
+ json={
+ "tables": [{"id": 1, "dataNodeId": 10}, {"id": 2, "dataNodeId": 20}],
+ "paginationInfo": {"nextCursor": ""},
+ }
+ ),
+ graph_response,
+ ScriptedResponse(json=_lineage_graph(20)),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["root_data_node_id"], row["id"]) for row in result.rows] == [(20, 20)]
+
+
class TestMetricScorecard:
def test_explodes_daily_counts_per_category(self) -> None:
result = _driver(workspace_id=7).run(