Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): filter lineage neighbours by per-object warehouse access (#125069)

Jovan Sakovic9673c38622c2
diff --git a/docs/published/handbook/engineering/data-warehouse.md b/docs/published/handbook/engineering/data-warehouse.md
index 695b544632df..4825333a3f1f 100644
--- a/docs/published/handbook/engineering/data-warehouse.md
+++ b/docs/published/handbook/engineering/data-warehouse.md
@@ -55,6 +55,8 @@ Insights can be saved or updated before running the SQL. Updating a view still r
The data warehouse sidebar lets you rename saved views. The dialog lists references from data model lineage. Renaming does not update SQL references in saved views, materialized views, endpoints, insights, dashboards, notebooks, or external systems.
+Saved-query lineage endpoints return only neighboring saved views and warehouse tables the caller can read. PostHog system tables remain visible, denied nodes do not stop the walk, and dependency counts use the same filtered neighbors.
+
Returning to a notebook can reuse completed query results from the last two minutes in the same browser session. Saved insights still fetch their latest definition; changed queries load their own results. Explicit **Run** and refresh actions fetch results again. This browser cache is bounded and excludes shared notebooks, errors, incomplete responses, and large results; it does not change server cache freshness.
## Choosing data in Business intelligence
diff --git a/products/access_control/backend/facade/subject_access_control.py b/products/access_control/backend/facade/subject_access_control.py
index 10a19b3b8757..0a0309608f42 100644
--- a/products/access_control/backend/facade/subject_access_control.py
+++ b/products/access_control/backend/facade/subject_access_control.py
@@ -223,6 +223,7 @@ class SubjectAccessControl(UserAccessControl):
pool = rows if rows is not None else self.team_access_controls
role_ids = frozenset(str(role_id) for role_id in self._user_role_ids)
self.__dict__["_cached_access_controls"] = [ac for ac in pool if self._applies_to_subject(ac, role_ids)]
+ self.__dict__.pop("_cached_access_controls_by_object", None)
def _is_creator_id(self, created_by_id: Optional[int]) -> bool:
"""The subject created the object, not the requesting user. A role and the default subject
diff --git a/products/access_control/backend/facade/user_access_control.py b/products/access_control/backend/facade/user_access_control.py
index afd7952f86d6..acf5e9028062 100644
--- a/products/access_control/backend/facade/user_access_control.py
+++ b/products/access_control/backend/facade/user_access_control.py
@@ -523,6 +523,7 @@ class UserAccessControl:
# Pop from __dict__ rather than hasattr/delattr
# hasattr on an un-computed cached_property would re-populate the value we're clearing
self.__dict__.pop("_cached_access_controls", None)
+ self.__dict__.pop("_cached_access_controls_by_object", None)
self.__dict__.pop("_dashboard_edit_limits", None)
self.__dict__.pop("_is_project_admin", None)
self.__dict__.pop("blocked_resource_ids_by_scope", None)
@@ -612,6 +613,13 @@ class UserAccessControl:
# team__organization_id), so `_row_matches` never reads that attribute off these rows.
return list(AccessControl.objects.filter(self._filter_options({"team_id": self._team.id})))
+ @cached_property
+ def _cached_access_controls_by_object(self) -> dict[tuple[str, str | None], list[_AccessControl]]:
+ access_controls_by_object: defaultdict[tuple[str, str | None], list[_AccessControl]] = defaultdict(list)
+ for access_control in self._cached_access_controls:
+ access_controls_by_object[(access_control.resource, access_control.resource_id)].append(access_control)
+ return dict(access_controls_by_object)
+
@property
def user(self) -> User:
"""The principal this access control was built for. Callers that run HogQL on a user's behalf
@@ -728,6 +736,12 @@ class UserAccessControl:
# Team-scoped lookups are served from the single bulk preload (filtered in memory);
# org-scoped lookups fall through to a targeted query.
if self._can_serve_from_preload(filters):
+ if set(filters) == {"team_id", "resource", "resource_id"}:
+ # Object and resource lookups are exact matches, so read them from the pool's index
+ # rather than scanning every row.
+ return list(
+ self._cached_access_controls_by_object.get((filters["resource"], filters["resource_id"]), [])
+ )
return [ac for ac in self._cached_access_controls if self._row_matches(ac, filters)]
key = json.dumps(filters, sort_keys=True)
@@ -813,7 +827,8 @@ class UserAccessControl:
return
if all(self._can_serve_from_preload(filters) for filters in filter_groups):
- self._fill_filters_cache(filter_groups, self._cached_access_controls)
+ # Lookups are answered from the pool, so loading it is all the preload has to do.
+ _ = self._cached_access_controls
return
q = Q()
diff --git a/products/access_control/backend/tests/test_user_access_control.py b/products/access_control/backend/tests/test_user_access_control.py
index e3d5695d39e4..0756b72312e7 100644
--- a/products/access_control/backend/tests/test_user_access_control.py
+++ b/products/access_control/backend/tests/test_user_access_control.py
@@ -356,6 +356,18 @@ class TestUserAccessControl(BaseUserAccessControlTest):
self.user_access_control = UserAccessControl(self.user, self.team)
assert self.user_access_control.access_level_for_object(self.team) == "admin"
+ def test_object_lookup_returns_only_the_rows_of_that_object(self):
+ first = self._create_access_control(resource="dashboard", resource_id="first", access_level="none")
+ self._create_access_control(resource="dashboard", resource_id="second", access_level="viewer")
+ resource_wide = self._create_access_control(resource="dashboard", resource_id=None, access_level="editor")
+ self._create_access_control(resource="notebook", resource_id="first", access_level="none")
+ self._clear_uac_caches()
+ access = self.user_access_control
+
+ assert access._get_access_controls(access._access_controls_filters_for_object("dashboard", "first")) == [first]
+ assert access._get_access_controls(access._access_controls_filters_for_resource("dashboard")) == [resource_wide]
+ assert access._get_access_controls(access._access_controls_filters_for_object("dashboard", "third")) == []
+
def test_org_admin_always_has_access(self):
self._create_access_control(resource_id=self.team.id, access_level="none")
assert self.other_user_access_control.check_access_level_for_object(self.team, "member") is False
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/lineage.py b/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
index ab1d63069dd4..42899b16aaf3 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
@@ -1,9 +1,15 @@
"""Walking a saved query's ancestors and descendants through the data modeling DAG."""
+from uuid import UUID
+
from rest_framework import request, serializers
+from posthog.uuidt import UUIDT
+
+from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.data_modeling.backend.facade.api import reachable_node_ids, saved_query_node_ids
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery, Node, NodeType
+from products.warehouse_sources.backend.facade.api import allowed_table_ids
class SavedQueryLineageRequestSerializer(serializers.Serializer):
@@ -48,7 +54,13 @@ def _parse_level(request: request.Request) -> int | None:
return serializer.validated_data.get("level")
-def _related_saved_queries(saved_query: DataWarehouseSavedQuery, *, upstream: bool, max_depth: int | None) -> set[str]:
+def _related_saved_queries(
+ saved_query: DataWarehouseSavedQuery,
+ user_access_control: UserAccessControl,
+ *,
+ upstream: bool,
+ max_depth: int | None,
+) -> set[str]:
"""Walk the data modeling graph from a saved query and identify what it reaches.
The walk starts from every node that stands for the query and unions what they reach.
@@ -64,17 +76,38 @@ def _related_saved_queries(saved_query: DataWarehouseSavedQuery, *, upstream: bo
reached = reachable_node_ids(saved_query.team_id, start_node_ids, upstream=upstream, max_depth=max_depth)
- identifiers: set[str] = set()
+ visible_identifiers: set[str] = set()
+ saved_query_ids: set[str] = set()
+ warehouse_table_ids: set[str] = set()
for node_type, saved_query_id, name, properties in Node.objects.filter(
team=saved_query.team, id__in=reached
).values_list("type", "saved_query_id", "name", "properties"):
properties = properties or {}
if node_type == NodeType.METRIC:
continue
- if saved_query_id is not None:
- identifiers.add(str(saved_query_id))
+ if properties.get("origin") == "posthog":
+ visible_identifiers.add(name)
+ elif saved_query_id is not None:
+ saved_query_ids.add(str(saved_query_id))
elif properties.get("warehouse_table_id"):
- identifiers.add(str(properties["warehouse_table_id"]))
- else:
- identifiers.add(name)
- return identifiers
+ warehouse_table_ids.add(str(properties["warehouse_table_id"]))
+
+ if saved_query_ids:
+ creators = DataWarehouseSavedQuery.objects.filter(
+ id__in=saved_query_ids, team_id=saved_query.team_id
+ ).values_list("id", "created_by_id")
+ levels = user_access_control.bulk_object_access_levels(
+ "warehouse_view", [(str(saved_query_id), created_by_id) for saved_query_id, created_by_id in creators]
+ )
+ visible_identifiers.update(
+ saved_query_id for saved_query_id, level in levels.items() if level is not None and level != "none"
+ )
+
+ if warehouse_table_ids:
+ # A malformed id in node properties must not break the lookup.
+ candidate_ids = [UUID(table_id) for table_id in warehouse_table_ids if UUIDT.is_valid_uuid(table_id)]
+ visible_identifiers.update(
+ str(table_id) for table_id in allowed_table_ids(saved_query.team_id, user_access_control, ids=candidate_ids)
+ )
+
+ return visible_identifiers
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
index ba2ceae3c1d4..c30dbcb0acc3 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
@@ -581,7 +581,12 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
to walk, so 1 gives the immediate parents.
"""
saved_query = self.get_object()
- ancestors = lineage._related_saved_queries(saved_query, upstream=True, max_depth=lineage._parse_level(request))
+ ancestors = lineage._related_saved_queries(
+ saved_query,
+ self.user_access_control,
+ upstream=True,
+ max_depth=lineage._parse_level(request),
+ )
return response.Response({"ancestors": ancestors})
@extend_schema(
@@ -596,7 +601,10 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
"""
saved_query = self.get_object()
descendants = lineage._related_saved_queries(
- saved_query, upstream=False, max_depth=lineage._parse_level(request)
+ saved_query,
+ self.user_access_control,
+ upstream=False,
+ max_depth=lineage._parse_level(request),
)
return response.Response({"descendants": descendants})
@@ -681,8 +689,12 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
saved_query = self.get_object()
return response.Response(
{
- "upstream_count": len(lineage._related_saved_queries(saved_query, upstream=True, max_depth=1)),
- "downstream_count": len(lineage._related_saved_queries(saved_query, upstream=False, max_depth=1)),
+ "upstream_count": len(
+ lineage._related_saved_queries(saved_query, self.user_access_control, upstream=True, max_depth=1)
+ ),
+ "downstream_count": len(
+ lineage._related_saved_queries(saved_query, self.user_access_control, upstream=False, max_depth=1)
+ ),
}
)
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query_access_control.py b/products/data_warehouse/backend/tests/api/test_saved_query_access_control.py
index a81635435464..6e3811007050 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query_access_control.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query_access_control.py
@@ -294,6 +294,127 @@ class TestDataWarehouseSavedQueryAccessControl(WarehouseAccessControlTestMixin):
self.assertEqual(self.client.get(other_url).status_code, status.HTTP_403_FORBIDDEN)
+class TestSavedQueryLineageAccessControl(WarehouseAccessControlTestMixin):
+ resource = "warehouse_objects"
+
+ def _saved_query(self, name: str) -> DataWarehouseSavedQuery:
+ return DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name=name,
+ query={"kind": "HogQLQuery", "query": "select 1"},
+ created_by=self.user,
+ )
+
+ def setUp(self):
+ super().setUp()
+ self.source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_id="lineage_source",
+ connection_id="lineage_connection",
+ status=ExternalDataSource.Status.COMPLETED,
+ source_type=ExternalDataSourceType.STRIPE,
+ prefix="lineage_",
+ )
+ self.table = DataWarehouseTable.objects.create(
+ name="lineage_table", team=self.team, external_data_source=self.source
+ )
+ self.upstream = self._saved_query("lineage_upstream")
+ self.middle = self._saved_query("lineage_middle")
+ self.downstream = self._saved_query("lineage_downstream")
+
+ dag = DAG.objects.create(team=self.team, name="lineage_access_control")
+ events_node = Node.objects.create(
+ team=self.team,
+ dag=dag,
+ name="events",
+ type=NodeType.TABLE,
+ properties={"origin": "posthog"},
+ )
+ table_node = Node.objects.create(
+ team=self.team,
+ dag=dag,
+ name=self.table.name,
+ type=NodeType.TABLE,
+ properties={"origin": "warehouse", "warehouse_table_id": str(self.table.id)},
+ )
+ upstream_node = Node.objects.create(
+ team=self.team, dag=dag, name=self.upstream.name, saved_query=self.upstream, type=NodeType.VIEW
+ )
+ middle_node = Node.objects.create(
+ team=self.team, dag=dag, name=self.middle.name, saved_query=self.middle, type=NodeType.VIEW
+ )
+ downstream_node = Node.objects.create(
+ team=self.team, dag=dag, name=self.downstream.name, saved_query=self.downstream, type=NodeType.VIEW
+ )
+ for source, target in (
+ (events_node, upstream_node),
+ (table_node, upstream_node),
+ (upstream_node, middle_node),
+ (middle_node, downstream_node),
+ ):
+ Edge.objects.create(team=self.team, dag=dag, source=source, target=target)
+
+ def _lineage(self, user: User, saved_query: DataWarehouseSavedQuery, action: str, **data: int) -> set[str]:
+ self.client.force_login(user)
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/warehouse_saved_queries/{saved_query.id}/{action}/",
+ data,
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
+ return set(response.json()[action])
+
+ def _dependencies(self, user: User, saved_query: DataWarehouseSavedQuery) -> dict[str, int]:
+ self.client.force_login(user)
+ response = self.client.get(
+ f"/api/environments/{self.team.pk}/warehouse_saved_queries/{saved_query.id}/dependencies/"
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
+ return response.json()
+
+ def test_lineage_withholds_denied_resources_without_pruning_the_walk(self):
+ self.assertEqual(
+ self._lineage(self.user, self.downstream, "ancestors"),
+ {"events", str(self.table.id), str(self.upstream.id), str(self.middle.id)},
+ )
+ self.assertEqual(
+ self._lineage(self.user, self.upstream, "descendants"),
+ {str(self.middle.id), str(self.downstream.id)},
+ )
+
+ self._create_access_control(self.viewer_user, access_level="editor")
+ self._create_access_control(
+ self.viewer_user,
+ resource="warehouse_view",
+ resource_id=str(self.middle.id),
+ access_level="none",
+ )
+ self._create_access_control(
+ self.viewer_user,
+ resource="external_data_source",
+ resource_id=str(self.source.id),
+ access_level="none",
+ )
+
+ self.assertEqual(
+ self._lineage(self.viewer_user, self.downstream, "ancestors"),
+ {"events", str(self.upstream.id)},
+ )
+ self.assertEqual(
+ self._lineage(self.viewer_user, self.upstream, "descendants"),
+ {str(self.downstream.id)},
+ )
+
+ immediate_ancestors = self._lineage(self.viewer_user, self.upstream, "ancestors", level=1)
+ immediate_descendants = self._lineage(self.viewer_user, self.upstream, "descendants", level=1)
+ dependencies = self._dependencies(self.viewer_user, self.upstream)
+ self.assertEqual(immediate_ancestors, {"events"})
+ self.assertEqual(immediate_descendants, set())
+ self.assertEqual(dependencies["upstream_count"], len(immediate_ancestors))
+ self.assertEqual(dependencies["downstream_count"], len(immediate_descendants))
+
+
@pytest.mark.ee
class TestDataWarehouseSavedQueryFolderAccessControl(WarehouseAccessControlTestMixin):
# Folder resource-level AC goes through warehouse_table via inheritance.