:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): copy access rules when migrating self-managed tables (#129218)
Tom Owersbe5a8bcff30f
diff --git a/products/access_control/backend/facade/api.py b/products/access_control/backend/facade/api.pyindex 3d7c3d8e73e2..f9d5b56697f1 100644--- a/products/access_control/backend/facade/api.py+++ b/products/access_control/backend/facade/api.py@@ -19,6 +19,7 @@ Do NOT:from __future__ import annotationsfrom collections import defaultdict+from collections.abc import Collection, Sequencefrom copy import copyfrom typing import castfrom uuid import UUID@@ -418,6 +419,100 @@ def resource_has_rules(*, team_id: int, resource: APIScopeObject) -> bool:return AccessControl.objects.filter(team_id=team_id, resource=resource, resource_id__isnull=True).exists()+def access_rules(*, team_id: int, resources: Collection[str]) -> list[contracts.AccessRule]:+ """Every rule of the team on these resources, object-level and resource-level, whatever its level.++ For a caller that must predict what resolution decides after it changes which rules reach an+ object. Like `object_ids_with_rules`, it ignores whether the organization enforces rules today,+ because the rules apply again when it does.+ """+ team = Team.objects.only("id", "organization_id").get(id=team_id)+ rows = AccessControl.objects.filter(team_id=team_id, resource__in=list(resources)).values_list(+ "resource",+ "resource_id",+ "access_level",+ "organization_member_id",+ "role_id",+ "organization_member__organization_id",+ "role__organization_id",+ )+ return [+ contracts.AccessRule(+ resource=resource,+ resource_id=resource_id,+ access_level=access_level,+ organization_member_id=member_id,+ role_id=role_id,+ # Resolution reads a member's rules only in the member's own organization, and a role's+ # rules only for a role of that organization (see UserAccessControl._filter_options).+ applies=(member_id is None or role_id is None)+ and (member_id is None or member_organization_id == team.organization_id)+ and (role_id is None or role_organization_id == team.organization_id),+ )+ for (+ resource,+ resource_id,+ access_level,+ member_id,+ role_id,+ member_organization_id,+ role_organization_id,+ ) in rows+ ]+++def replace_object_access_rules(+ *, team_id: int, resource: str, resource_id: str, rules: Sequence[contracts.ObjectAccessRuleInput]+) -> None:+ """Make the rules on one object exactly `rules`, the object's default rule included.++ A rule that is already there stays. The others are created or deleted one at a time, so each+ change goes into the activity log. An empty `rules` removes every rule on the object. The+ caller decides whether the change is allowed.+ """+ team = Team.objects.only("id", "organization_id").get(id=team_id)+ wanted: dict[tuple[UUID | None, UUID | None], str] = {}+ for rule in rules:+ _validate_resource_access_level(resource, rule.access_level)+ key = (rule.organization_member_id, rule.role_id)+ if key in wanted:+ raise InvalidObjectAccessControlError("Provide each subject only once.")+ wanted[key] = rule.access_level+ member_ids = {member_id for member_id, _ in wanted if member_id is not None}+ role_ids = {role_id for _, role_id in wanted if role_id is not None}+ if member_ids != set(+ OrganizationMembership.objects.filter(organization_id=team.organization_id, id__in=member_ids).values_list(+ "id", flat=True+ )+ ):+ raise InvalidObjectAccessControlError("organization_member does not belong to this team's organization.")+ if role_ids != set(+ Role.objects.filter(organization_id=team.organization_id, id__in=role_ids).values_list("id", flat=True)+ ):+ raise InvalidObjectAccessControlError("role does not belong to this team's organization.")++ with transaction.atomic():+ kept: set[tuple[UUID | None, UUID | None]] = set()+ for existing in AccessControl.objects.select_for_update().filter(+ team_id=team_id, resource=resource, resource_id=resource_id+ ):+ key = (existing.organization_member_id, existing.role_id)+ if key not in kept and wanted.get(key) == existing.access_level:+ kept.add(key)+ else:+ existing.delete()+ for (member_id, role_id), access_level in wanted.items():+ if (member_id, role_id) not in kept:+ AccessControl.objects.create(+ team_id=team_id,+ resource=resource,+ resource_id=resource_id,+ access_level=access_level,+ organization_member_id=member_id,+ role_id=role_id,+ )++def role_belongs_to_organization(*, role_id: str | UUID, organization_id: UUID) -> bool:"""Whether the role is a role of that organization.@@ -571,20 +666,23 @@ def _to_object_rule(rule: AccessControl) -> contracts.ObjectAccessControlRule:def _validate_object_access_control_level(input: contracts.SetObjectAccessControlInput) -> APIScopeObject:- resource = input.resource+ return _validate_resource_access_level(input.resource, input.access_level)+++def _validate_resource_access_level(resource: str, access_level: str | None) -> APIScopeObject:if resource not in API_SCOPE_OBJECTS or resource in INTERNAL_API_SCOPE_OBJECTS:raise InvalidObjectAccessControlError(f"{resource} is not an access controlled resource.")scoped_resource = cast(APIScopeObject, resource)- if input.access_level is not None:+ if access_level is not None:levels = ordered_access_levels(scoped_resource)- if input.access_level not in levels:+ if access_level not in levels:raise InvalidObjectAccessControlError(f"Invalid access level for {resource}. Must be one of: {', '.join(levels)}.")# `ordered_access_levels` can include levels the settings UI never offers for a resource# (for example "none" on a project), so the bounds apply here as well as in the API.- level_index = levels.index(input.access_level)+ level_index = levels.index(cast(AccessControlLevel, access_level))if level_index < levels.index(minimum_access_level(scoped_resource)):raise InvalidObjectAccessControlError(f"Access level is below the minimum for {resource}.")if level_index > levels.index(highest_access_level(scoped_resource)):diff --git a/products/access_control/backend/facade/contracts.py b/products/access_control/backend/facade/contracts.pyindex f8a4e53adf72..c755f088a98b 100644--- a/products/access_control/backend/facade/contracts.py+++ b/products/access_control/backend/facade/contracts.py@@ -149,6 +149,35 @@ class ObjectAccessControlRule:role_id: UUID | None+@frozen+class AccessRule:+ """One stored rule, object-level or resource-level, as resolution reads it.++ `applies` is False for a rule that resolution never uses: it names a member or a role of+ another organization, or it names both a member and a role.+ """++ resource: str+ resource_id: str | None+ access_level: str+ organization_member_id: UUID | None+ role_id: UUID | None+ applies: bool+++@frozen+class ObjectAccessRuleInput:+ """One rule to keep on an object. With no member and no role, it is the object's default rule."""++ access_level: str+ organization_member_id: UUID | None = None+ role_id: UUID | None = None++ def __post_init__(self) -> None:+ if self.organization_member_id is not None and self.role_id is not None:+ raise ValueError("Set at most one of organization_member_id or role_id.")++@frozenclass TerraformLock:"""The state of the Terraform lock of one project. has_terraform_account is true when Terraformdiff --git a/products/access_control/backend/tests/test_api.py b/products/access_control/backend/tests/test_api.pyindex c9904bb3a828..5c7a7a1c4d85 100644--- a/products/access_control/backend/tests/test_api.py+++ b/products/access_control/backend/tests/test_api.py@@ -17,6 +17,7 @@ from products.access_control.backend.facade.api import (bulk_set_object_access_controls,every_member_has_resource_access,object_ids_restricted_from_any_member,+ replace_object_access_rules,role_belongs_to_organization,set_object_access_control,valid_role_member_user_ids,@@ -27,7 +28,7 @@ from products.access_control.backend.models.role import Role, RoleMembershipfrom products.dashboards.backend.models.dashboard import Dashboard-class TestSetObjectAccessControl(BaseTest):+class _DashboardRulesTest(BaseTest):def setUp(self) -> None:super().setUp()self.organization.available_product_features = [@@ -40,6 +41,11 @@ class TestSetObjectAccessControl(BaseTest):self.dashboard = Dashboard.objects.create(team=self.team, created_by=self.user)AccessControl.objects.create(team=self.team, resource="dashboard", resource_id=None, access_level="none")+ def _other_user_level(self) -> str | None:+ return UserAccessControl(user=self.other_user, team=self.team).get_user_access_level(self.dashboard)+++class TestSetObjectAccessControl(_DashboardRulesTest):def _grant(self, access_level: str | None = "viewer", **overrides: object) -> contracts.ObjectAccessControlRule | None:@@ -54,9 +60,6 @@ class TestSetObjectAccessControl(BaseTest):team_id=self.team.id, input=contracts.SetObjectAccessControlInput(**input_kwargs))- def _other_user_level(self) -> str | None:- return UserAccessControl(user=self.other_user, team=self.team).get_user_access_level(self.dashboard)-def test_grant_update_and_revoke_are_visible_through_user_access_control(self) -> None:assert self._other_user_level() == "none"@@ -246,6 +249,76 @@ class TestSetObjectAccessControl(BaseTest):assert not AccessControl.objects.filter(team=self.team, resource_id=str(self.dashboard.id)).exists()+class TestReplaceObjectAccessRules(_DashboardRulesTest):+ def _replace(self, *rules: contracts.ObjectAccessRuleInput) -> None:+ replace_object_access_rules(+ team_id=self.team.id, resource="dashboard", resource_id=str(self.dashboard.id), rules=list(rules)+ )++ def _rules(self) -> dict[tuple[object, object], tuple[object, str]]:+ return {+ (rule.organization_member_id, rule.role_id): (rule.id, rule.access_level)+ for rule in AccessControl.objects.filter(team=self.team, resource_id=str(self.dashboard.id))+ }++ def test_the_object_ends_with_exactly_the_given_rules(self) -> None:+ self._replace(+ contracts.ObjectAccessRuleInput(access_level="viewer"),+ contracts.ObjectAccessRuleInput(access_level="none", organization_member_id=self.other_membership.id),+ )+ kept_id = self._rules()[(None, None)][0]+ assert self._other_user_level() == "none"++ self._replace(+ contracts.ObjectAccessRuleInput(access_level="viewer"),+ contracts.ObjectAccessRuleInput(access_level="editor", organization_member_id=self.other_membership.id),+ )++ rules = self._rules()+ assert rules[(None, None)] == (kept_id, "viewer")+ assert rules[(self.other_membership.id, None)][1] == "editor"+ assert len(rules) == 2+ assert self._other_user_level() == "editor"+ logged = ActivityLog.objects.filter(+ scope="AccessControl", detail__name=f"dashboard {self.dashboard.id}"+ ).values_list("activity", flat=True)+ assert sorted(logged) == ["created", "created", "created", "deleted"]++ self._replace()++ assert self._rules() == {}++ @parameterized.expand([("unknown_level",), ("foreign_member",), ("foreign_role",), ("same_subject_twice",)])+ def test_rejects_invalid_rules_before_writing(self, case: str) -> None:+ other_org = Organization.objects.create(name="Other")+ outsider = User.objects.create_and_join(other_org, "[email protected]", "testtest")+ rules = {+ "unknown_level": [contracts.ObjectAccessRuleInput(access_level="owner")],+ "foreign_member": [+ contracts.ObjectAccessRuleInput(+ access_level="viewer",+ organization_member_id=OrganizationMembership.objects.get(user=outsider).id,+ )+ ],+ "foreign_role": [+ contracts.ObjectAccessRuleInput(+ access_level="viewer", role_id=Role.objects.create(name="Outside", organization=other_org).id+ )+ ],+ "same_subject_twice": [+ contracts.ObjectAccessRuleInput(access_level="viewer"),+ contracts.ObjectAccessRuleInput(access_level="none"),+ ],+ }[case]+ self._replace(contracts.ObjectAccessRuleInput(access_level="editor"))+ before = self._rules()++ with pytest.raises(InvalidObjectAccessControlError):+ self._replace(*rules)++ assert self._rules() == before++class TestEveryMemberHasResourceAccess(BaseTest):"""`account` inherits from `customer_analytics`, so every rule here is written on the parent,which is where resolution reads them from."""diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.pyindex fe7193012757..8daffb22829e 100644--- a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py@@ -29,7 +29,8 @@ class Command(BaseCommand):"in a source for its bucket, and nothing syncs: the table keeps reading its own bucket until the user ""turns syncing on, and the first sync then replaces it with the same id and name. A table made from an ""uploaded file gets a file upload source of its own, and its next uploaded file is the first sync. "- "Runs only for teams "+ "Each source gets the access rules that keep every member's access to its tables, or the table stays "+ "self-managed. Runs only for teams ""with the object storage sources feature flag. Without --live-run the command only reports what it ""would do.")diff --git a/products/warehouse_sources/backend/self_managed_access.py b/products/warehouse_sources/backend/self_managed_access.pynew file mode 100644index 000000000000..0859efa2dadf--- /dev/null+++ b/products/warehouse_sources/backend/self_managed_access.py@@ -0,0 +1,412 @@+"""The access rules a migration source needs, so that moving a self-managed table into it changes nobody's access.++A self-managed table has no source, so access to it resolves from the table's own rules and the rules for the+whole warehouse (`warehouse_objects`). Once a source adopts the table, access also falls back to the rules on+that source and then to the rules for all sources (`RESOURCE_FALLBACK_MAP` in the access control facade). The+source also has access of its own: to its settings, its schemas and its syncs, and through them to the files+the table reads.++So each migration source gets rules that make two things true for every member, whatever roles the member+has, under both resolution orders an organization can use:++- the member's access to each table of the source is the same before and after the source adopts it, and+- the member's access to the source is never more than their access to each of its tables.++The rules copy the table's member and role rules, which decide on the table before anything reaches the+source, and add a default rule on the source when one is needed. When no such rules exist, the table can't+move. Tables of one bucket share a source only when they need the same rules (see `SourceAccessPlan.signature`).++The prediction mirrors `UserAccessControl` (`_object_access_level_from_rows` and+`_most_specific_object_access_from_rows`). The tests check it against that code.+"""++import json+import hashlib+import itertools+from collections import defaultdict+from collections.abc import Iterable, Sequence+from typing import Literal+from uuid import UUID++from posthog.dataclasses import frozen++from products.access_control.backend.facade.api import access_rules, replace_object_access_rules+from products.access_control.backend.facade.contracts import AccessRule, ObjectAccessRuleInput+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable++# The `connection_metadata` key of a source the migration created.+MIGRATION_MARKER_KEY = "self_managed_migration"++TABLE_RESOURCE = "warehouse_table"+WAREHOUSE_RESOURCE = "warehouse_objects"+SOURCE_RESOURCE = "external_data_source"++LEVELS: tuple[str, ...] = ("none", "viewer", "editor", "manager")+# The level a table or a source resolves to when no rule decides (`default_access_level`).+_SYSTEM_DEFAULT = "editor"+# Each role profile doubles the subjects to check, so a rule set that names more is refused.+_MAX_ROLE_PROFILES = 10++# The start of the error a sync raises when it must not replace a self-managed table. The import+# classifies it as permanent, so the job fails once and syncing stops until someone fixes the rules.+ADOPTION_ACCESS_ERROR = "PostHog did not replace the self-managed table"++TABLE_RULES_REASON = "the table has its own access rules, which the source PostHog would create can't carry"+PROJECT_RULES_REASON = "the project has access rules for sources or for the warehouse, which the move would change"+ROLES_REASON = "the access rules name too many roles to check"++_Subject = Literal["member", "role", "default"]+# Most specific first, the order most-specific resolution reads one scope's rules in.+_SUBJECTS: tuple[_Subject, ...] = ("member", "role", "default")+_Scope = Literal["table", "warehouse", "sources", "source"]+_SCOPES: tuple[_Scope, ...] = ("table", "warehouse", "sources", "source")+++@frozen+class SourceRule:+ """One rule on a migration source. With no member and no role, it is the source's default rule."""++ access_level: str+ organization_member_id: str | None = None+ role_id: str | None = None++ @property+ def subject(self) -> _Subject:+ if self.organization_member_id is not None:+ return "member"+ if self.role_id is not None:+ return "role"+ return "default"+++@frozen+class SourceAccessPlan:+ """The rules a migration source needs for its tables to keep their access."""++ rules: tuple[SourceRule, ...] = ()++ @property+ def signature(self) -> str:+ """Identifies the rules, so only tables that need the same rules share a source. Empty without rules."""+ if not self.rules:+ return ""+ payload = json.dumps([[r.access_level, r.organization_member_id, r.role_id] for r in self.rules])+ return hashlib.sha256(payload.encode()).hexdigest()[:32]++ def describe(self) -> str:+ parts = []+ for rule in self.rules:+ subject = {+ "member": f"member {rule.organization_member_id}",+ "role": f"role {rule.role_id}",+ "default": "everyone",+ }[rule.subject]+ parts.append(f"{subject}={rule.access_level}")+ return ", ".join(parts)+++@frozen+class _Row:+ level: str+ subject: _Subject+++class AdoptionAccessError(Exception):+ """The access rules changed after the migration, so the first sync must not replace the table."""+++def _rank(level: str) -> int:+ return LEVELS.index(level)+++def _highest(rows: Iterable[_Row]) -> str:+ return max((row.level for row in rows), key=_rank)+++def _decision(rows: Sequence[_Row]) -> _Row | None:+ """The deciding row of one scope under most-specific resolution: the member's, then the roles', then the default."""+ for subject in _SUBJECTS:+ group = [row for row in rows if row.subject == subject]+ if group:+ return _Row(level=_highest(group), subject=subject)+ return None+++def _legacy_level(+ object_rows: Sequence[_Row],+ parent_rows: Sequence[_Row] | None,+ resource_rows: Sequence[_Row],+ parent_resource_rows: Sequence[_Row],+) -> str:+ """`UserAccessControl._object_access_level_from_rows`, for a member who is no admin and not the creator.++ `parent_rows` is None for an object without a fallback parent: a self-managed table, or a source.+ """+ explicit = [row for row in object_rows if row.subject != "default"]+ if explicit:+ return _highest(explicit)+ if object_rows and _highest(object_rows) == "none":+ return "none"+ if parent_rows:+ return _highest(parent_rows)+ if resource_rows:+ return _highest(resource_rows)+ if parent_rows is not None and parent_resource_rows:+ return _highest(parent_resource_rows)+ if object_rows:+ return _highest(object_rows)+ return _SYSTEM_DEFAULT+++def _most_specific_level(+ object_rows: Sequence[_Row],+ parent_rows: Sequence[_Row] | None,+ resource_rows: Sequence[_Row],+ parent_resource_rows: Sequence[_Row],+) -> str:+ """`UserAccessControl._most_specific_object_access_from_rows`, for a member who is no admin and not the creator."""++ def resource_level() -> str:+ if resource_rows:+ return _decided_level(resource_rows)+ if parent_rows is not None and parent_resource_rows:+ return _decided_level(parent_resource_rows)+ return _SYSTEM_DEFAULT++ for rows in (object_rows, parent_rows or ()):+ row = _decision(rows)+ if row is not None:+ # A default grant on the object or its parent can't widen "none" for the resource.+ if row.subject == "default" and row.level != "none" and resource_level() == "none":+ return "none"+ return row.level+ return resource_level()+++def _decided_level(rows: Sequence[_Row]) -> str:+ row = _decision(rows)+ assert row is not None+ return row.level+++class TeamAccessRules:+ """The rules of one team that decide access to its warehouse tables and its sources.++ Rules that resolution never applies (a member or a role of another organization) are left out.+ Several rows for one subject on one object act as their highest level, so they count as one.+ """++ def __init__(self, team_id: int) -> None:+ self.team_id = team_id+ rows = [+ rule+ for rule in access_rules(team_id=team_id, resources=(TABLE_RESOURCE, WAREHOUSE_RESOURCE, SOURCE_RESOURCE))+ if rule.applies+ ]+ self._warehouse = _subject_rules(r for r in rows if r.resource == WAREHOUSE_RESOURCE and r.resource_id is None)+ self._sources = _subject_rules(r for r in rows if r.resource == SOURCE_RESOURCE and r.resource_id is None)+ by_object: dict[tuple[str, str], list[AccessRule]] = defaultdict(list)+ for rule in rows:+ if rule.resource_id is not None and rule.resource in (TABLE_RESOURCE, SOURCE_RESOURCE):+ by_object[(rule.resource, rule.resource_id)].append(rule)+ self._objects = {key: _subject_rules(rules) for key, rules in by_object.items()}+ self._plans: dict[tuple[SourceRule, ...], SourceAccessPlan | str] = {}++ def source_rules(self, source_id: str) -> tuple[SourceRule, ...]:+ """The rules on a source as they are now, in the order a plan lists them."""+ return self._objects.get((SOURCE_RESOURCE, source_id), ())++ def plan_for(self, table_id: str) -> SourceAccessPlan | str:+ """The rules a source needs for the table to keep its access, or the reason no rules can."""+ table_rules = self._objects.get((TABLE_RESOURCE, table_id), ())+ if table_rules not in self._plans:+ self._plans[table_rules] = self._plan(table_rules)+ return self._plans[table_rules]++ def _plan(self, table_rules: tuple[SourceRule, ...]) -> SourceAccessPlan | str:+ unknown = next(+ (+ rule.access_level+ for rule in (*table_rules, *self._warehouse, *self._sources)+ if rule.access_level not in LEVELS+ ),+ None,+ )+ if unknown is not None:+ return f"an access rule has the level '{unknown}', which PostHog doesn't know"+ found = self._search(table_rules)+ if found is not None:+ return found+ # When the same table without rules of its own could move, its own rules are the cause.+ if table_rules and isinstance(self._search(()), SourceAccessPlan):+ return TABLE_RULES_REASON+ return PROJECT_RULES_REASON++ def _search(self, table_rules: tuple[SourceRule, ...]) -> SourceAccessPlan | str | None:+ """The first source rules that keep every member's access, or None when no rules can."""+ # A member or role rule on the table decides before anything reaches the source. So a copy of+ # it on the source changes nothing on the table, and gives that subject equal access to both.+ copies = tuple(rule for rule in table_rules if rule.subject != "default")+ table_default = next((rule.access_level for rule in table_rules if rule.subject == "default"), None)+ # The table's own default comes first, then no default, so the source follows the table where it can.+ for source_default in dict.fromkeys([table_default, None, *LEVELS]):+ rules = _sorted([*copies, *([SourceRule(access_level=source_default)] if source_default else [])])+ verdict = self._keeps_access(table_rules, rules)+ if verdict is True:+ return SourceAccessPlan(rules=rules)+ if isinstance(verdict, str):+ return verdict+ return None++ def _keeps_access(self, table_rules: tuple[SourceRule, ...], source_rules: tuple[SourceRule, ...]) -> bool | str:+ """Whether every member keeps their access to the table, and gets no more to the source than to the table.++ Checked for each kind of member (the rules that name them) with each combination of roles,+ because a member's roles can change at any time, and under both resolution orders, because an+ organization can change its order. Admins and the table's creator get the highest level before+ any rule is read, so the move can't change their access.+ """+ scopes: dict[_Scope, tuple[SourceRule, ...]] = {+ "table": table_rules,+ "warehouse": self._warehouse,+ "sources": self._sources,+ "source": source_rules,+ }+ defaults = {+ scope: [_Row(level=rule.access_level, subject="default") for rule in rules if rule.subject == "default"]+ for scope, rules in scopes.items()+ }+ role_profiles = [profile for profile in _profiles(scopes, "role") if profile != _EMPTY_PROFILE]+ if len(role_profiles) > _MAX_ROLE_PROFILES:+ return ROLES_REASON+ for member_profile in _profiles(scopes, "member"):+ for count in range(len(role_profiles) + 1):+ for roles in itertools.combinations(role_profiles, count):+ rows = {+ scope: [+ *defaults[scope],+ *(+ [_Row(level=member_level, subject="member")]+ if (member_level := member_profile[i])+ else []+ ),+ *(_Row(level=level, subject="role") for role in roles if (level := role[i]) is not None),+ ]+ for i, scope in enumerate(_SCOPES)+ }+ for resolve in (_legacy_level, _most_specific_level):+ before = resolve(rows["table"], None, rows["warehouse"], rows["sources"])+ after = resolve(rows["table"], rows["source"], rows["warehouse"], rows["sources"])+ source = resolve(rows["source"], None, rows["sources"], ())+ if after != before or _rank(source) > _rank(before):+ return False+ return True+++_Profile = tuple[str | None, ...]+_EMPTY_PROFILE: _Profile = (None, None, None, None)+++def _profiles(scopes: dict[_Scope, tuple[SourceRule, ...]], subject: Literal["member", "role"]) -> list[_Profile]:+ """The level each subject of one kind has in each scope, once per distinct set of levels.++ Subjects with the same levels resolve the same, so each set is checked once. The empty set stands+ for a member that no rule names, or for having no role.+ """+ levels: dict[str, dict[_Scope, str]] = defaultdict(dict)+ for scope, rules in scopes.items():+ for rule in rules:+ if rule.subject == subject:+ subject_id = rule.organization_member_id if subject == "member" else rule.role_id+ assert subject_id is not None+ levels[subject_id][scope] = rule.access_level+ profiles = {tuple(by_scope.get(scope) for scope in _SCOPES) for by_scope in levels.values()}+ profiles.add(_EMPTY_PROFILE)+ return sorted(profiles, key=repr)+++def _subject_rules(rules: Iterable[AccessRule]) -> tuple[SourceRule, ...]:+ highest: dict[tuple[str | None, str | None], str] = {}+ for rule in rules:+ key = (+ str(rule.organization_member_id) if rule.organization_member_id else None,+ str(rule.role_id) if rule.role_id else None,+ )+ current, level = highest.get(key), rule.access_level+ # An unknown level is kept, so the plan refuses the table instead of guessing what it means.+ if current is None or level not in LEVELS or (current in LEVELS and _rank(level) > _rank(current)):+ highest[key] = level+ return _sorted(+ SourceRule(access_level=level, organization_member_id=member_id, role_id=role_id)+ for (member_id, role_id), level in highest.items()+ )+++def _sorted(rules: Iterable[SourceRule]) -> tuple[SourceRule, ...]:+ return tuple(+ sorted(+ rules,+ key=lambda r: (_SUBJECTS.index(r.subject), r.organization_member_id or "", r.role_id or "", r.access_level),+ )+ )+++def write_source_rules(team_id: int, source_id: str, plan: SourceAccessPlan) -> None:+ replace_object_access_rules(+ team_id=team_id,+ resource=SOURCE_RESOURCE,+ resource_id=source_id,+ rules=[+ ObjectAccessRuleInput(+ access_level=rule.access_level,+ organization_member_id=UUID(rule.organization_member_id) if rule.organization_member_id else None,+ role_id=UUID(rule.role_id) if rule.role_id else None,+ )+ for rule in plan.rules+ ],+ )+++def remove_source_rules(team_id: int, source_id: str) -> None:+ replace_object_access_rules(team_id=team_id, resource=SOURCE_RESOURCE, resource_id=source_id, rules=[])+++def adoption_access_refusal(+ table: DataWarehouseTable, source: ExternalDataSource, rules: TeamAccessRules | None = None+) -> str | None:+ """Why the source must not adopt the table now, or None when the move keeps everyone's access.++ The rules may have changed since the migration linked the table: on the table, on the source, or+ for the whole warehouse or all sources. The source's rules must still be the ones the table needs.+ """+ rules = rules or TeamAccessRules(table.team_id)+ planned = rules.plan_for(str(table.id))+ if isinstance(planned, str):+ return f"the access rules changed after the migration, and now {planned}"+ if rules.source_rules(str(source.pk)) != planned.rules:+ return "the access rules of the table or of its source changed after the migration"+ return None+++def adoption_access_error(table: DataWarehouseTable, source: ExternalDataSource) -> str | None:+ """The error to show when a migration source must not adopt the table, or None when it may.++ Sources the migration didn't make are not checked.+ """+ if MIGRATION_MARKER_KEY not in (source.connection_metadata or {}):+ return None+ refusal = adoption_access_refusal(table, source)+ if refusal is None:+ return None+ return (+ f"{ADOPTION_ACCESS_ERROR} '{table.name}', because {refusal}. The table still reads its own files. "+ "To continue, an organization admin must make the access rules of this source the same as the "+ "rules of the table, then turn syncing on again. PostHog support can help with this."+ )+++def assert_adoption_keeps_access(table: DataWarehouseTable, source: ExternalDataSource) -> None:+ error = adoption_access_error(table, source)+ if error is not None:+ raise AdoptionAccessError(error)diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.pyindex effa281d2d1e..e5b6de97eef8 100644--- a/products/warehouse_sources/backend/self_managed_migration.py+++ b/products/warehouse_sources/backend/self_managed_migration.py@@ -15,6 +15,11 @@ upload folder, and nothing is copied. The next upload for the table points the sand starts a normal billable sync, which adopts the table. The old file is deleted after that sync,when nothing else reads it (see `file_upload_sources.replace_file_upload`).+A table in a source also resolves access through the source's rules, so each source gets the access+rules that keep every member's access to its tables, and tables share a source only when they need the+same rules. A table whose access no such rules can keep stays self-managed. The first sync checks the+rules again before it replaces the table (see `self_managed_access`).+The state of each table comes from the rows themselves: a migration source carries a marker in its`connection_metadata`, and its schemas name their tables. Running the migration again thereforecontinues where it stopped.@@ -43,7 +48,6 @@ from botocore.exceptions import BotoCoreError, ClientErrorfrom posthog.dataclasses import frozenfrom posthog.models.team import Team-from products.access_control.backend.facade.api import object_ids_with_rules, resource_has_rulesfrom products.data_warehouse.backend.facade.api import delete_external_data_schedulefrom products.warehouse_sources.backend.facade.contracts import UploadReadSettingsfrom products.warehouse_sources.backend.file_upload_sources import (@@ -77,6 +81,14 @@ from products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (source_feature_flag_missing,)+from products.warehouse_sources.backend.self_managed_access import (+ MIGRATION_MARKER_KEY,+ SourceAccessPlan,+ TeamAccessRules,+ adoption_access_refusal,+ remove_source_rules,+ write_source_rules,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (UnsupportedCsvColumnError,)@@ -115,8 +127,6 @@ from products.warehouse_sources.backend.types import ExternalDataJobStatuslogger = structlog.get_logger(__name__)-# The `connection_metadata` key of a source the migration created.-MIGRATION_MARKER_KEY = "self_managed_migration"DEFAULT_ABANDON_REASON = "Left self-managed by an operator."_DELETED_REASON = "its migration schema or source was deleted"@@ -149,6 +159,8 @@ class TableMigrationStatus:source_id: str | None = Noneschema_id: str | None = Nonenested_text_column_types: dict[str, str] | None = None+ # The access rules the table's source gets, for a table not linked yet.+ planned_access_rules: str | None = None@frozen@@ -199,6 +211,8 @@ class _TablePlan:# The bucket a shared source reads, which names its table prefix when one is needed.bucket: str = ""credential_fingerprint: str = dataclasses.field(default="", repr=False)+ # The rules the source gets. Only tables that need the same rules share a source.+ access: SourceAccessPlan = SourceAccessPlan()@frozen@@ -313,13 +327,23 @@ def _source_fingerprints(sources: Sequence[ExternalDataSource]) -> dict[uuid.UUIreturn fingerprints-def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str) -> None:+def _access_signature(source: ExternalDataSource) -> str:+ """The access signature in a source's marker. A source made before markers held one has no rules."""+ signature = _marker(source).get("access_signature")+ return signature if isinstance(signature, str) else ""+++def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str, access_signature: str) -> None:"""Hold the group's lock until the transaction ends.Every table of one source shares the lock, so two runs can't create two sources for the group,- and a merge can't move schemas while a run adds to the group.+ and a merge can't move schemas while a run adds to the group. A group without access rules keeps+ the lock name it had before groups had access signatures."""- name = "\n".join((MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint))+ parts = [MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint]+ if access_signature:+ parts.append(access_signature)+ name = "\n".join(parts)lock_id = int.from_bytes(hashlib.blake2b(name.encode(), digest_size=8).digest(), "big", signed=True)with connection.cursor() as cursor:cursor.execute("SELECT pg_advisory_xact_lock(%s)", [lock_id])@@ -334,8 +358,7 @@ class _Planner:self._upload_source = file_upload_source()self._flag_missing: dict[tuple[int, str], bool] = {}self._regions: dict[tuple[str, str], str | None] = {}- self._tables_with_rules: dict[int, set[str]] = {}- self._teams_with_source_rules: dict[int, bool] = {}+ self._access: dict[int, TeamAccessRules] = {}def plan(self, table: DataWarehouseTable) -> _TablePlan | str:"""The plan for a table, or the reason the table can't be migrated."""@@ -343,18 +366,14 @@ class _Planner:hosted = table.hosted_upload if credential is None else Noneif credential is None and hosted is None:return "the table has no credential"- if self._has_access_rules(table):- # The new source would get default access, and a member the table's rules shut out- # could read the same files through a schema of that source.- return "the table has its own access rules, which the source PostHog would create can't carry"- if self._has_source_rules(table):- # A table in a source falls back to the rules for sources, which never reached a- # self-managed table, so moving it could take access away from members who have it.- # Rules for the whole warehouse don't reach the new source, so a member they shut out- # could read the table's files through it.- return "the project has access rules for sources or for the warehouse, which the move would change"+ # A table in a source falls back to the source's rules and to the rules for all sources, and a+ # member who can use the source can read the table's files through it. So the source gets rules+ # that keep each member's access, or the table stays self-managed.+ access = self.team_access(table.team_id).plan_for(str(table.id))+ if isinstance(access, str):+ return accessif hosted is not None:- return self._plan_upload(table, hosted)+ return self._plan_upload(table, hosted, access)assert credential is not Nonefingerprint = _credential_fingerprint(credential)try:@@ -383,9 +402,17 @@ class _Planner:description=f"Self-managed tables in {location.bucket}",bucket=location.bucket,credential_fingerprint=fingerprint,+ access=access,)- def _plan_upload(self, table: DataWarehouseTable, hosted: HostedUpload) -> _TablePlan | str:+ def team_access(self, team_id: int) -> TeamAccessRules:+ if team_id not in self._access:+ self._access[team_id] = TeamAccessRules(team_id)+ return self._access[team_id]++ def _plan_upload(+ self, table: DataWarehouseTable, hosted: HostedUpload, access: SourceAccessPlan+ ) -> _TablePlan | str:"""A table that reads a file the team uploaded gets a file upload source of its own."""source = self._upload_sourceif self._team_lacks_flag(table.team, source):@@ -415,22 +442,9 @@ class _Planner:group_key=f"upload/{table.pk}",description=f"Uploaded file for {table.name}"[:400],shares_source=False,+ access=access,)- def _has_access_rules(self, table: DataWarehouseTable) -> bool:- if table.team_id not in self._tables_with_rules:- self._tables_with_rules[table.team_id] = object_ids_with_rules(- team_id=table.team_id, resource="warehouse_table"- )- return str(table.id) in self._tables_with_rules[table.team_id]-- def _has_source_rules(self, table: DataWarehouseTable) -> bool:- if table.team_id not in self._teams_with_source_rules:- self._teams_with_source_rules[table.team_id] = resource_has_rules(- team_id=table.team_id, resource="external_data_source"- ) or resource_has_rules(team_id=table.team_id, resource="warehouse_objects")- return self._teams_with_source_rules[table.team_id]-def _team_lacks_flag(self, team: Team, source: ObjectStorageSource[Any] | FileUploadSource) -> bool:key = (team.pk, str(source.source_type))if key not in self._flag_missing:@@ -560,12 +574,17 @@ def _idle_reason(schema: ExternalDataSchema) -> str:return "syncing is off"-def _awaiting_reason(table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None) -> str:+def _awaiting_reason(+ table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None, planner: "_Planner"+) -> str:ref = self_managed_table_ref(schema)if schema.table_id is not None:return "the table changed after the migration, so its schema syncs a separate table"if ref is None or (table.url_pattern, table.format) != (ref.url_pattern, ref.format):return "the table changed after the migration, so a sync makes a separate table"+ refusal = adoption_access_refusal(table, schema.source, planner.team_access(table.team_id))+ if refusal is not None:+ return f"{refusal}, so the first sync will fail and leave the table as it is"if job is None:return _idle_reason(schema)if job.status == ExternalDataJobStatus.RUNNING:@@ -603,7 +622,7 @@ def _evaluate(if table.external_data_source_id is not None:return _Evaluation(table=table, status=status(TableMigrationState.MIGRATED), schema=live, job=job)if live is not None:- reason = _awaiting_reason(table, live, job)+ reason = _awaiting_reason(table, live, job, planner)return _Evaluation(table=table, status=status(TableMigrationState.AWAITING_SYNC, reason), schema=live, job=job)if schemas:refs = [ref for ref in (self_managed_table_ref(schema) for schema in schemas) if ref is not None]@@ -612,7 +631,11 @@ def _evaluate(planned = planner.plan(table)if isinstance(planned, _TablePlan):- return _Evaluation(table=table, status=status(TableMigrationState.PENDING, "not linked yet"), plan=planned)+ pending = dataclasses.replace(+ status(TableMigrationState.PENDING, "not linked yet"),+ planned_access_rules=planned.access.describe() or None,+ )+ return _Evaluation(table=table, status=pending, plan=planned)return _Evaluation(table=table, status=status(TableMigrationState.SKIPPED, planned))@@ -652,15 +675,20 @@ def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:return candidate-def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:- """The group's source for the table's team, bucket and key pair, the oldest when there are more.+def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan, access: TeamAccessRules) -> ExternalDataSource:+ """The group's source for the table's team, bucket, key pair and access rules, the oldest when there are more.- Runs under the group lock.+ A new source gets its access rules in the same transaction. An existing source joins only while+ its rules are still the ones the table needs. Runs under the group lock."""+ signature = plan.access.signatureif not plan.shares_source:- return create_source_for_upload(- table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: {"group_key": plan.group_key}}- )+ marker: dict[str, Any] = {"group_key": plan.group_key}+ if signature:+ marker["access_signature"] = signature+ source = create_source_for_upload(table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: marker})+ _write_access_rules(source, plan)+ return sourcecandidates = list(ExternalDataSource.objects.filter(team_id=table.team_id,@@ -671,9 +699,17 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter.order_by("created_at", "id"))fingerprints = _source_fingerprints(candidates)- existing = next((s for s in candidates if fingerprints.get(s.pk) == plan.credential_fingerprint), None)+ in_group = [+ s+ for s in candidates+ if fingerprints.get(s.pk) == plan.credential_fingerprint and _access_signature(s) == signature+ ]+ existing = next((s for s in in_group if access.source_rules(str(s.pk)) == plan.access.rules), None)if existing is not None:return existing+ if in_group:+ # Someone changed the rules on the group's source, so a table that joins it now would resolve differently.+ raise UnmappableTableError("the access rules on the source for this bucket changed, so the table can't join it")source = SourceInputsStore.create(prefix=_free_prefix(table.team_id, plan),source_id=str(uuid.uuid4()),@@ -691,15 +727,22 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter"group_key": plan.group_key,"credential_fingerprint": plan.credential_fingerprint,"credential_id": str(table.credential_id),+ **({"access_signature": signature} if signature else {}),}},)ExternalDataSourceDestination.objects.for_team(table.team_id).create(team_id=table.team_id, source_id=source.pk, destination=get_or_create_warehouse_destination(table.team_id))+ _write_access_rules(source, plan)return source+def _write_access_rules(source: ExternalDataSource, plan: _TablePlan) -> None:+ if plan.access.rules:+ write_source_rules(source.team_id, str(source.pk), plan.access)++def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:"""The table's migration schema, created with its source when there is none yet.@@ -709,7 +752,13 @@ def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataS"""with transaction.atomic():if plan.shares_source:- _lock_group(table.team_id, str(plan.source.source_type), plan.group_key, plan.credential_fingerprint)+ _lock_group(+ table.team_id,+ str(plan.source.source_type),+ plan.group_key,+ plan.credential_fingerprint,+ plan.access.signature,+ )else:DataWarehouseTable.raw_objects.select_for_update().get(team_id=table.team_id, pk=table.pk)schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])@@ -718,7 +767,11 @@ def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSreturn liveif schemas:raise UnmappableTableError("the table's migration was abandoned")- source = _find_or_create_source(table, plan)+ # The plan read the rules before the lock, so read them again for the rules the source gets.+ access = TeamAccessRules(table.team_id)+ if access.plan_for(str(table.id)) != plan.access:+ raise UnmappableTableError("the access rules changed during the run")+ source = _find_or_create_source(table, plan, access)if (ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk, name=table.name).exclude(deleted=True)@@ -875,6 +928,8 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:.exists()):source.soft_delete()+ # The rules the migration gave the source have nothing left to protect.+ remove_source_rules(source.team_id, str(source.pk))removed_source_id = str(source.pk)if evaluation.schema is not None:# A schedule exists only when the user turned syncing on before, so it is paused now.@@ -896,7 +951,8 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:)-_GroupId = tuple[int, str, str, str]+# Team, source type, group key, credential fingerprint and access signature.+_GroupId = tuple[int, str, str, str, str]def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False) -> list[SourceMergeOutcome]:@@ -904,8 +960,9 @@ def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False)Earlier runs grouped tables by credential row, so tables with the same key in one bucket got asource each. A duplicate moves only while all of its tables still wait for their first sync,- because nothing has read through it then. The empty duplicate is then deleted. An uploaded- table's source never merges.+ because nothing has read through it then, and only when both sources have the same access rules,+ so the moved tables keep their access. The empty duplicate and its rules are then deleted. An+ uploaded table's source never merges."""queryset = (ExternalDataSource.objects.filter(connection_metadata__has_key=MIGRATION_MARKER_KEY)@@ -922,7 +979,9 @@ def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False)fingerprint = fingerprints.get(source.pk)if not isinstance(group_key, str) or group_key.startswith("upload/") or fingerprint is None:continue- groups[(source.team_id, str(source.source_type), group_key, fingerprint)].append(source)+ groups[(source.team_id, str(source.source_type), group_key, fingerprint, _access_signature(source))].append(+ source+ )outcomes: list[SourceMergeOutcome] = []for group, (kept, *duplicates) in groups.items():@@ -954,12 +1013,10 @@ def _merge_refusal()if adopted is not None:return f"the table '{adopted}' is already adopted by a source"- if object_ids_with_rules(team_id=kept.team_id, resource="external_data_source") & {- str(kept.pk),- str(duplicate.pk),- }:- # A moved schema would take the kept source's rules, so the duplicate's rules would stop applying.- return "the kept or the duplicate source has its own access rules"+ access = TeamAccessRules(kept.team_id)+ if access.source_rules(str(kept.pk)) != access.source_rules(str(duplicate.pk)):+ # A moved table would resolve through the kept source's rules, so the duplicate's rules would stop applying.+ return "the kept and the duplicate source have different access rules"if _destination_ids(kept) != _destination_ids(duplicate):return "the kept and the duplicate source send data to different destinations"if SourceInputsStore.load(kept) != SourceInputsStore.load(duplicate):@@ -1032,6 +1089,7 @@ def _merge_source(schema.save(update_fields=["source", "updated_at"])ExternalDataSourceDestination.objects.for_team(duplicate.team_id).filter(source_id=duplicate.pk).delete()duplicate.soft_delete()+ remove_source_rules(duplicate.team_id, str(duplicate.pk))except Exception as e:# A failure on one source must not stop the batch or hide its outcomes.logger.warning("Could not merge a duplicate migration source", source_id=str(duplicate.pk), exc_info=True)@@ -1057,8 +1115,11 @@ def describe_merge(outcome: SourceMergeOutcome) -> str:def describe_status(status: TableMigrationStatus) -> str:reason = f" ({status.reason})" if status.reason else ""columns = status.nested_text_column_types or {}- note = ""+ notes = []if columns:changed = ", ".join(f"{name} ({column_type})" for name, column_type in columns.items())- note = f" note=(after the first sync these columns are stored as JSON text: {changed})"+ notes.append(f"after the first sync these columns are stored as JSON text: {changed}")+ if status.planned_access_rules:+ notes.append(f"the source gets these access rules: {status.planned_access_rules}")+ note = "".join(f" note=({note})" for note in notes)return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}{note}"diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex 7470fdf2252f..d939807c0a64 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -52,6 +52,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (update_should_sync,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERRORfrom products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (EmitSignalsActivityInputs,PersonPropertySyncActivityInputs,@@ -229,6 +230,10 @@ Any_Source_Errors: dict[str, str | None] = {# it here so every other source stops retrying too. The enriched message names the offending column# and shows example cells, so keep the raw error rather than replacing it with a generic one."must be real number, not str": None,+ # Raised before extraction, and again before the table changes, when a migration source must not+ # adopt a self-managed table because the access rules changed. Only a person can fix the rules,+ # and the message names the table and the next step, so keep it.+ ADOPTION_ACCESS_ERROR: None,# Raised by botocore (`is_valid_endpoint_url`) when the object storage endpoint URL has a# hostname it rejects — most often an underscore, invalid in a DNS hostname but common in a# self-hosted deployment's OBJECT_STORAGE_ENDPOINT (e.g. a docker service name). The endpoint isdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.pyindex 400dee20c57c..d327235bbf60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_type+from products.warehouse_sources.backend.self_managed_access import assert_adoption_keeps_accessfrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (HISTORY_NAMING,LEGACY_HISTORY_NAMING,@@ -523,6 +524,10 @@ async def validate_schema_and_update_table(update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]if table.external_data_source_id is None:# A self-managed table this schema replaces reads its own bucket until now.+ # Adoption makes the table resolve through the source's rules. Checked here, after+ # the row count that can block for minutes, so a rule change has the least time+ # to land between the check and the save.+ assert_adoption_keeps_access(table, job.pipeline)update_fields.extend(table.adopt_into_source(job.pipeline))logger.info(f"Adopted self-managed table {table.id} into source {job.pipeline.id}")if live_size_mib is not None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.pyindex 0c52ff0e8951..0fb082e62842 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (ExternalDataSchema,update_should_sync,)+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERRORfrom products.warehouse_sources.backend.temporal.data_imports.metrics import (LOCK_TAKEOVER_LATEST_ERROR,TERMINAL_JOB_STATUSES,@@ -173,6 +174,9 @@ DISABLE_SCHEMA_ERROR_PATTERNS: tuple[str, ...] = (# incoming values no longer fit the stored Delta column type# (SchemaColumnTypeChangedException) — only a reset and full re-sync can fix it"Source column type changed",+ # the access rules changed after the migration linked a self-managed table, so the first sync+ # must not replace it — only an admin can fix the rules+ ADOPTION_ACCESS_ERROR,)# The schema or job row was deleted mid-sync — no retry can bring it back, and no more of thatdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex c1697a7dddd9..c55946998ee0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (SCHEMA_DELETED_JOB_ERROR,SYNC_DISABLED_JOB_ERROR,)+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERRORfrom products.warehouse_sources.backend.temporal.data_imports.metrics import LOCK_TAKEOVER_LATEST_ERRORfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,@@ -307,6 +308,7 @@ class TestProcessSingle:"[Errno 5] An error occurred (XMinioStorageFull) when calling the CopyObject operation: ""Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",str(DestinationConfigurationError("Prod", "The host name does not exist.")),+ f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table or of its source changed",],)@pytest.mark.asyncio@@ -2137,6 +2139,9 @@ class TestFailRun:),pytest.param("Decimal value is too large to store in a Decimal128", True, id="decimal_overflow"),pytest.param("Primary key required for incremental syncs", True, id="missing_primary_key"),+ pytest.param(+ f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules changed", True, id="adoption_refused"+ ),pytest.param("XMinioStorageFull: storage backend has reached its minimum free drive threshold",False,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.pyindex 0b58bd73de30..169ddea14878 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py@@ -14,6 +14,7 @@ from parameterized import parameterizedfrom posthog.models import Team+from products.access_control.backend.facade.testing import create_access_controlfrom products.warehouse_sources.backend.models.credential import DataWarehouseCredentialfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import (@@ -23,6 +24,7 @@ from products.warehouse_sources.backend.models.external_data_schema import ()from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable+from products.warehouse_sources.backend.self_managed_access import MIGRATION_MARKER_KEY, AdoptionAccessErrorfrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_NAMING, LEGACY_HISTORY_NAMINGfrom products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (HistoryTableNameTakenError,@@ -740,6 +742,40 @@ class TestValidateSchemaAndUpdateTable:job.refresh_from_db()assert job.storage_delta_mib == 3.0+ @pytest.mark.parametrize("rule_added", [False, True])+ def test_a_migration_source_adopts_the_table_only_while_its_access_rules_still_fit(+ self, team: Team, rule_added: bool+ ) -> None:+ schema, job = self._schema_and_job(team)+ ExternalDataSource.objects.filter(pk=job.pipeline_id).update(+ connection_metadata={MIGRATION_MARKER_KEY: {"group_key": "s3/us-east-1/exports"}}+ )+ job.refresh_from_db()+ table = self._self_managed_table(team, schema)+ if rule_added:+ # Added after the migration linked the table, so the source has no rule that matches it.+ create_access_control(+ team_id=team.pk, resource="warehouse_table", resource_id=str(table.id), access_level="none"+ )++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ if rule_added:+ with pytest.raises(AdoptionAccessError, match="The table still reads its own files"):+ self._publish(team, schema, job)+ else:+ self._publish(team, schema, job)++ table.refresh_from_db()+ schema.refresh_from_db()+ if rule_added:+ assert (table.external_data_source_id, schema.table_id) == (None, None)+ assert table.credential_id is not None+ else:+ assert (table.external_data_source_id, schema.table_id) == (job.pipeline_id, table.id)+@pytest.mark.parametrize("change",[diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 87d6cda14e4e..876de3dd9176 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -45,10 +45,12 @@ from products.warehouse_sources.backend.models.external_data_schema import (prefix_promotion_rows_synced,process_incremental_value,resolve_sync_type,+ self_managed_table_ref,staged_handoff_resume_point,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.self_managed_access import AdoptionAccessError, adoption_access_errorfrom products.warehouse_sources.backend.temporal.data_imports.destinations.enablement import (external_destination_ids_for,)@@ -90,6 +92,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.typings import PipelineResultfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync import (PipelineInputs,+ find_replaced_self_managed_table,relink_orphan_table,)from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTS@@ -208,6 +211,21 @@ def _get_external_data_schema(schema_id: uuid.UUID, team_id: int) -> ExternalDat)+def self_managed_adoption_error(schema: ExternalDataSchema) -> str | None:+ """The error that stops this run before it extracts anything, or None.++ The first sync of a migration schema replaces a self-managed table. When the access rules changed+ since the migration linked the table, the publish step would refuse to replace it, so the copy+ would be wasted.+ """+ if schema.table_id is not None or self_managed_table_ref(schema) is None:+ return None+ table = find_replaced_self_managed_table(schema, schema.source)+ if table is None or table.external_data_source_id is not None:+ return None+ return adoption_access_error(table, schema.source)++@database_sync_to_async_pooldef _has_completed_schema_job(schema_id: uuid.UUID, team_id: int) -> bool:return ExternalDataJob.objects.filter(@@ -643,6 +661,13 @@ async def _import_data_with_reporting(except ExternalDataSchema.DoesNotExist as e:await _handle_import_error(job_inputs, logger, e)+ # Checked before anything is extracted, so a refused move costs no copy. The publish step+ # checks again before it changes the table.+ adoption_error = await database_sync_to_async_pool(self_managed_adoption_error)(schema)+ if adoption_error is not None:+ # The workflow reads the message from the cause, so chain the refusal.+ raise NonRetryableException(adoption_error) from AdoptionAccessError(adoption_error)+reset_pipeline = _resolve_reset_pipeline(inputs, schema, job_created_at=model.created_at)await logger.adebug(f"schema.sync_type_config = {schema.sync_type_config}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex 19390a000980..56eba6e34677 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -36,8 +36,11 @@ from products.warehouse_sources.backend.models.external_data_schema import ()from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERRORfrom products.warehouse_sources.backend.temporal.data_imports.external_data_job import (+ IMPORT_NON_RETRYABLE_ERROR_TYPES,NEW_TABLE_NOT_READY_MESSAGE,+ Any_Source_Errors,_transient_error_message,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig@@ -57,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.basResumableSource,SimpleSource,SourceExtractionNotImplementedError,+ error_message_matches,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import TemporaryHostResolutionErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (@@ -523,6 +527,29 @@ async def test_schema_deleted_mid_sync_routes_through_handler():source.parse_config.assert_not_called()+async def test_a_refused_self_managed_adoption_stops_the_run_before_extraction():+ message = f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table or of its source changed"+ source = _make_source(AssertionError("must not extract"), {})++ with (+ _patched_activity(source) as handle_mock,+ mock.patch.object(module, "self_managed_adoption_error", return_value=message),+ ):+ with pytest.raises(NonRetryableException, match="access rules of the table") as exc_info:+ await import_data_activity_sync(_inputs())++ source.parse_config.assert_not_called()+ source.source_for_pipeline.assert_not_called()+ handle_mock.assert_not_awaited()+ # Temporal doesn't retry the activity, and the job's finalizer pauses the schema with the message.+ assert type(exc_info.value).__name__ in IMPORT_NON_RETRYABLE_ERROR_TYPES+ assert error_message_matches(str(exc_info.value), Any_Source_Errors.keys())+ assert Any_Source_Errors[ADOPTION_ACCESS_ERROR] is None+ # The workflow records `e.cause.cause` as the job's error, so the refusal must be the cause.+ assert str(exc_info.value.__cause__) == message++@pytest.mark.asyncioasync def test_job_not_found_on_first_read_is_non_reportable():# A run_id can point at a job that was never created, e.g. an older worker'sdiff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.pyindex 5a45bccdc44e..436933a82545 100644--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py@@ -17,9 +17,14 @@ from temporalio.service import RPCError, RPCStatusCodefrom posthog.hogql.database.database import Database-from posthog.models.organization import OrganizationMembership+from posthog.constants import AvailableFeature+from posthog.models.organization import Organization, OrganizationMembershipfrom posthog.models.team import Team+from posthog.models.user import User+from products.access_control.backend.facade.subject_access_control import SubjectAccessControl+from products.access_control.backend.facade.testing import add_role_member, create_access_control, create_role+from products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.access_control.backend.models.access_control import AccessControlfrom products.warehouse_sources.backend.logic.source_inputs import SourceInputsStorefrom products.warehouse_sources.backend.models.credential import DataWarehouseCredential@@ -33,6 +38,12 @@ from products.warehouse_sources.backend.models.external_data_schema import ()from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.self_managed_access import (+ ADOPTION_ACCESS_ERROR,+ AdoptionAccessError,+ TeamAccessRules,+ assert_adoption_keeps_access,+)from products.warehouse_sources.backend.self_managed_migration import (MIGRATION_MARKER_KEY,MigrationOptions,@@ -52,6 +63,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objparse_self_managed_url,table_definition_for,)+from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.import_data_sync import (+ self_managed_adoption_error,+)from products.warehouse_sources.backend.types import DataWarehouseTableFormat, ExternalDataJobStatusMODULE = "products.warehouse_sources.backend.self_managed_migration"@@ -447,64 +461,6 @@ class TestSelfManagedMigration(_MigrationTest):assert "dwh-object-storage-sources" in (outcome.status.reason or "")assert not ExternalDataSchema.objects.filter(team=self.team).exists()- @parameterized.expand(- [- ("object_rule_restricts", True, "none", TableMigrationState.SKIPPED),- ("object_rule_grants", True, "editor", TableMigrationState.SKIPPED),- ("rule_on_another_table", False, "none", TableMigrationState.AWAITING_SYNC),- ]- )- def test_a_table_with_its_own_access_rules_is_skipped(- self, _name: str, on_this_table: bool, access_level: str, expected: TableMigrationState- ) -> None:- table = self._table()- other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")- AccessControl.objects.create(- team=self.team,- resource="warehouse_table",- resource_id=str(table.id if on_this_table else other.id),- access_level=access_level,- organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),- )-- outcome = next(o for o in self._migrate() if o.status.table_id == str(table.id))-- assert outcome.status.state == expected- if expected == TableMigrationState.SKIPPED:- assert "own access rules" in (outcome.status.reason or "")- assert _schemas_for(table) == []-- @parameterized.expand(- [- ("member_rule_on_sources", "external_data_source", True),- ("default_rule_on_sources", "external_data_source", False),- ("member_rule_on_the_warehouse", "warehouse_objects", True),- ("default_rule_on_the_warehouse", "warehouse_objects", False),- ]- )- def test_a_project_with_access_rules_for_sources_is_skipped(- self, _name: str, resource: str, for_member: bool- ) -> None:- table = self._table()- AccessControl.objects.create(- team=self.team,- resource=resource,- resource_id=None,- access_level="none",- organization_member=(- OrganizationMembership.objects.get(user=self.user, organization=self.organization)- if for_member- else None- ),- )-- [outcome] = self._migrate()-- assert outcome.status.table_id == str(table.id)- assert outcome.status.state == TableMigrationState.SKIPPED- assert "access rules for sources" in (outcome.status.reason or "")- assert _schemas_for(table) == []-def test_status_reports_every_state(self) -> None:pending = self._table("pending_table")self._table("skipped_table", url="https://minio.example.com/exports/a.csv")@@ -793,7 +749,7 @@ class TestMergeDuplicateSources(_MigrationTest):("job_ran", "a sync already ran for the source"),("table_adopted", "the table 'table_1' is already adopted by a source"),("name_taken_in_kept_source", "the kept source already has a table named 'table_1'"),- ("source_has_rules", "the kept or the duplicate source has its own access rules"),+ ("source_has_rules", "the kept and the duplicate source have different access rules"),("other_destinations", "the kept and the duplicate source send data to different destinations"),("inputs_edited", "the kept and the duplicate source connect with different settings"),]@@ -1111,3 +1067,576 @@ class TestUploadedTableMigration(_MigrationTest):# Inside the team folder but not one upload, so it is listed and never planned."nested_key": TableMigrationState.SKIPPED,}+++_LEVELS = ("none", "viewer", "editor", "manager")+_ROLE_NAMES = ("A", "B")+# Each member stands for one way a member can relate to the rules.+_MEMBERS: dict[str, tuple[tuple[str, ...], OrganizationMembership.Level]] = {+ "plain": ((), OrganizationMembership.Level.MEMBER),+ "named": ((), OrganizationMembership.Level.MEMBER),+ "in_role": (("A",), OrganizationMembership.Level.MEMBER),+ "named_in_roles": (("A", "B"), OrganizationMembership.Level.MEMBER),+ "in_both_roles": (("A", "B"), OrganizationMembership.Level.MEMBER),+ "admin": ((), OrganizationMembership.Level.ADMIN),+ "project_admin": ((), OrganizationMembership.Level.MEMBER),+ "creator": ((), OrganizationMembership.Level.MEMBER),+}+# Where a rule can be: "<scope>" is the scope's default rule, "<scope>:<member or role>" names a subject.+_SCOPES = {+ "table": ("warehouse_table", True),+ "warehouse": ("warehouse_objects", False),+ "sources": ("external_data_source", False),+ # Resolution reads the rules for all warehouse tables from warehouse_objects, so this rule never applies.+ "all_tables": ("warehouse_table", False),+}+++class _AccessRulesTest(_MigrationTest):+ def setUp(self) -> None:+ super().setUp()+ self.organization.available_product_features = [+ {"key": feature, "name": feature}+ for feature in (AvailableFeature.ACCESS_CONTROL, AvailableFeature.ROLE_BASED_ACCESS)+ ]+ self.organization.save()+ self.roles = {+ name: create_role(organization_id=self.organization.id, name=f"Role {name}") for name in _ROLE_NAMES+ }+ self.users: dict[str, User] = {}+ self.memberships: dict[str, OrganizationMembership] = {}+ for name, (roles, level) in _MEMBERS.items():+ user = User.objects.create_and_join(self.organization, f"{name}@example.com", None, level=level)+ self.users[name] = user+ self.memberships[name] = OrganizationMembership.objects.get(user=user, organization=self.organization)+ for role in roles:+ add_role_member(role_id=self.roles[role], user_id=user.id)+ create_access_control(+ team_id=self.team.pk,+ resource="project",+ resource_id=str(self.team.pk),+ access_level="admin",+ organization_member_id=self.memberships["project_admin"].id,+ )++ def _rule(self, slot: str, level: str, table: DataWarehouseTable | None = None) -> None:+ scope, _, subject = slot.partition(":")+ resource, on_table = _SCOPES[scope]+ create_access_control(+ team_id=self.team.pk,+ resource=resource,+ resource_id=str(table.id) if on_table and table is not None else None,+ access_level=level,+ organization_member_id=self.memberships[subject].id if subject in self.memberships else None,+ role_id=self.roles[subject] if subject in self.roles else None,+ )++ def _apply(self, rules: dict[str, str], table: DataWarehouseTable) -> None:+ for slot, level in rules.items():+ self._rule(slot, level, table)++ def _access(self, obj: Any) -> dict[str, str | None]:+ """Each subject's access to the object, from the access checks that serve requests."""+ result: dict[str, str | None] = {+ name: UserAccessControl(user, self.team).get_user_access_level(obj) for name, user in self.users.items()+ }+ asker = UserAccessControl(self.users["admin"], self.team)+ for name, role_id in self.roles.items():+ result[f"role {name}"] = SubjectAccessControl.for_role(+ asker, self.team, str(role_id)+ ).get_user_access_level(obj)+ result["everyone"] = SubjectAccessControl.for_default(asker, self.team).get_user_access_level(obj)+ return result++ def _in_both_orders(self, obj: Any) -> dict[bool, dict[str, str | None]]:+ by_order = {}+ for most_specific in (False, True):+ self.organization.uses_most_specific_access_resolution = most_specific+ self.organization.save()+ by_order[most_specific] = self._access(obj)+ return by_order++ def _source_rules(self, source: ExternalDataSource) -> set[tuple[str, str]]:+ names = {str(m.id): name for name, m in self.memberships.items()} | {str(r): n for n, r in self.roles.items()}+ return {+ (names.get(str(rule.organization_member_id or rule.role_id or ""), ""), rule.access_level)+ for rule in AccessControl.objects.filter(+ team=self.team, resource="external_data_source", resource_id=str(source.pk)+ )+ }++ def _assert_move_keeps_access(self, table: DataWarehouseTable) -> ExternalDataSource:+ """Adopt the linked table, and check every subject's access to it and to its source under both orders."""+ before = self._in_both_orders(table)+ source = _schema_for(table).source+ self._adopt(table)+ table.refresh_from_db()+ assert table.external_data_source_id == source.pk+ after = self._in_both_orders(table)+ on_source = self._in_both_orders(source)+ assert after == before+ for most_specific, levels in on_source.items():+ for subject, level in levels.items():+ table_level = after[most_specific][subject]+ assert level is not None and table_level is not None+ assert _LEVELS.index(level) <= _LEVELS.index(table_level), (most_specific, subject, level, table_level)+ return source+++class TestAccessRulesMoveWithTheTable(_AccessRulesTest):+ @parameterized.expand(+ [+ ("no_rules", {}, set()),+ ("table_private", {"table": "none"}, {("", "none")}),+ ("table_read_only", {"table": "viewer"}, {("", "viewer")}),+ ("table_hidden_from_a_member", {"table:named": "none"}, {("named", "none")}),+ (+ "table_private_but_shared_with_a_member",+ {"table": "none", "table:named": "editor"},+ {("named", "editor"), ("", "none")},+ ),+ ("table_rule_for_a_role", {"table:A": "viewer"}, {("A", "viewer")}),+ (+ "member_rule_and_role_rule_for_one_member",+ {"table:named_in_roles": "viewer", "table:A": "editor", "table:B": "none"},+ {("named_in_roles", "viewer"), ("A", "editor"), ("B", "none")},+ ),+ ("warehouse_read_only", {"warehouse": "viewer"}, {("", "viewer")}),+ ("warehouse_hidden", {"warehouse": "none"}, {("", "none")}),+ ("warehouse_managers", {"warehouse": "manager"}, set()),+ # The table gives more than the default for sources, so the source must grant it.+ ("sources_read_only", {"sources": "viewer"}, {("", "editor")}),+ # The table gives less than the default for sources, so the source must be restricted.+ ("private_table_and_sources_open", {"table": "none", "sources": "editor"}, {("", "none")}),+ ("private_table_and_warehouse_read_only", {"table": "none", "warehouse": "viewer"}, {("", "none")}),+ ("rule_for_all_tables_never_applies", {"all_tables": "none"}, set()),+ ("rule_for_the_project_admin", {"table:project_admin": "none"}, {("project_admin", "none")}),+ ("rule_for_the_creator", {"table:creator": "none"}, {("creator", "none")}),+ ]+ )+ def test_a_migrated_table_keeps_every_subjects_access(+ self, _name: str, rules: dict[str, str], source_rules: set[tuple[str, str]]+ ) -> None:+ table = self._table(created_by=self.users["creator"])+ self._apply(rules, table)+ before = self._in_both_orders(table)++ [outcome] = self._migrate()++ assert outcome.action == "linked", outcome.error or outcome.status.reason+ assert self._source_rules(_schema_for(table).source) == source_rules+ # Until the first sync the table has no source, so the source's rules don't reach it.+ assert self._in_both_orders(table) == before+ self._assert_move_keeps_access(table)++ @parameterized.expand(+ [+ # A grant on a source can't widen "none" for all sources under most-specific resolution.+ ("sources_hidden", {"sources": "none"}, "access rules for sources"),+ ("sources_hidden_from_a_member", {"sources:named": "none"}, "access rules for sources"),+ ("table_read_only_and_sources_hidden", {"table": "viewer", "sources": "none"}, "access rules for sources"),+ # Everyone could manage the source, and through its rules change the table's access.+ ("sources_managed_by_everyone", {"sources": "manager"}, "access rules for sources"),+ # Members of one role see more of the warehouse than the others, and one default can't say that.+ ("warehouse_open_to_a_role", {"warehouse": "viewer", "warehouse:A": "editor"}, "access rules for sources"),+ ("warehouse_hidden_from_a_member", {"warehouse:named": "none"}, "access rules for sources"),+ (+ "table_rule_and_sources_hidden_from_a_role",+ {"table:named": "viewer", "sources:A": "none"},+ "access rules for sources",+ ),+ ]+ )+ def test_a_table_whose_access_no_source_rules_can_keep_stays_self_managed(+ self, _name: str, rules: dict[str, str], reason: str+ ) -> None:+ table = self._table()+ self._apply(rules, table)++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.SKIPPED+ assert reason in (outcome.status.reason or "")+ assert _schemas_for(table) == []+ assert not ExternalDataSource.objects.filter(team=self.team).exists()++ @parameterized.expand(+ [+ (f"table_{table}_warehouse_{warehouse}_sources_{sources}", table, warehouse, sources)+ for table in (None, *_LEVELS[:3])+ for warehouse in (None, *_LEVELS[:3])+ for sources in (None, *_LEVELS[:3])+ ]+ )+ def test_every_move_of_a_table_with_explicit_rules_keeps_access(+ self, _name: str, table_default: str | None, warehouse: str | None, sources: str | None+ ) -> None:+ table = self._table(created_by=self.users["creator"])+ rules = {"table:named": "viewer", "table:B": "none"}+ for slot, level in (("table", table_default), ("warehouse", warehouse), ("sources", sources)):+ if level is not None:+ rules[slot] = level+ self._apply(rules, table)++ [outcome] = self._migrate()++ if outcome.action != "linked":+ assert outcome.status.state == TableMigrationState.SKIPPED+ assert outcome.status.reason in (+ "the table has its own access rules, which the source PostHog would create can't carry",+ "the project has access rules for sources or for the warehouse, which the move would change",+ )+ return+ self._assert_move_keeps_access(table)++ def test_tables_of_one_bucket_share_a_source_only_when_they_need_the_same_rules(self) -> None:+ open_table = self._table("open", url="https://exports.s3.eu-west-1.amazonaws.com/open/*.csv")+ private = self._table("private", url="https://exports.s3.eu-west-1.amazonaws.com/private/*.csv")+ also_private = self._table("also_private", url="https://exports.s3.eu-west-1.amazonaws.com/also/*.csv")+ read_only = self._table("read_only", url="https://exports.s3.eu-west-1.amazonaws.com/read/*.csv")+ self._rule("table", "none", private)+ self._rule("table", "none", also_private)+ self._rule("table", "viewer", read_only)++ self._migrate()++ sources = {table.name: _schema_for(table).source for table in (open_table, private, also_private, read_only)}+ assert sources["private"].pk == sources["also_private"].pk+ assert len({source.pk for source in sources.values()}) == 3+ assert self._source_rules(sources["open"]) == set()+ assert self._source_rules(sources["private"]) == {("", "none")}+ assert self._source_rules(sources["read_only"]) == {("", "viewer")}+ assert "access_signature" not in (sources["open"].connection_metadata or {})[MIGRATION_MARKER_KEY]+ for table in (private, also_private, read_only):+ self._assert_move_keeps_access(table)++ def test_rules_that_never_apply_are_not_copied(self) -> None:+ other_organization = Organization.objects.create(name="Elsewhere")+ foreign_role = create_role(organization_id=other_organization.id, name="Foreign")+ table = self._table()+ create_access_control(+ team_id=self.team.pk,+ resource="warehouse_table",+ resource_id=str(table.id),+ access_level="none",+ role_id=foreign_role,+ )+ before = self._in_both_orders(table)++ [outcome] = self._migrate()++ assert outcome.action == "linked"+ assert self._source_rules(_schema_for(table).source) == set()+ assert self._in_both_orders(table) == before+ self._assert_move_keeps_access(table)++ def test_a_rule_for_an_inactive_member_is_copied(self) -> None:+ User.objects.filter(pk=self.users["named"].pk).update(is_active=False)+ table = self._table()+ self._rule("table:named", "manager", table)++ [outcome] = self._migrate()++ assert outcome.action == "linked"+ assert self._source_rules(_schema_for(table).source) == {("named", "manager")}++ def test_rules_of_an_organization_without_access_control_are_copied_for_when_it_comes_back(self) -> None:+ table = self._table()+ self._apply({"table": "none", "table:named": "viewer"}, table)+ with_access_control = self._in_both_orders(table)+ self.organization.available_product_features = []+ self.organization.save()++ [outcome] = self._migrate()++ assert outcome.action == "linked"+ assert self._source_rules(_schema_for(table).source) == {("named", "viewer"), ("", "none")}+ self._assert_move_keeps_access(table)+ self.organization.available_product_features = [+ {"key": feature, "name": feature}+ for feature in (AvailableFeature.ACCESS_CONTROL, AvailableFeature.ROLE_BASED_ACCESS)+ ]+ self.organization.save()+ assert self._in_both_orders(table) == with_access_control++ @parameterized.expand(+ [+ ("unknown_level", {"table": "superuser"}, "the level 'superuser'"),+ ]+ )+ def test_rules_the_planner_cant_read_keep_the_table_self_managed(+ self, _name: str, rules: dict[str, str], reason: str+ ) -> None:+ table = self._table()+ self._apply(rules, table)++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.SKIPPED+ assert reason in (outcome.status.reason or "")++ def test_rules_that_name_too_many_roles_keep_the_table_self_managed(self) -> None:+ table = self._table()+ for i in range(11):+ role = create_role(organization_id=self.organization.id, name=f"Extra {i}")+ create_access_control(+ team_id=self.team.pk,+ resource="warehouse_table",+ resource_id=str(table.id),+ access_level=_LEVELS[i % 4],+ role_id=role,+ )+ create_access_control(+ team_id=self.team.pk, resource="warehouse_objects", access_level=_LEVELS[i // 4], role_id=role+ )++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.SKIPPED+ assert outcome.status.reason == "the access rules name too many roles to check"++ def test_status_and_dry_run_report_the_rules_the_source_gets_and_write_nothing(self) -> None:+ table = self._table()+ self._apply({"table": "none", "table:named": "editor"}, table)+ member_id = self.memberships["named"].id+ note = f"note=(the source gets these access rules: member {member_id}=editor, everyone=none)"++ for options in ({"status": True}, {}):+ out = StringIO()+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=out, **options)+ assert note in out.getvalue()++ assert not ExternalDataSource.objects.filter(team=self.team).exists()+ assert not AccessControl.objects.filter(resource="external_data_source").exists()++ def test_an_uploaded_table_gets_the_same_rules_on_its_own_source(self) -> None:+ with override_settings(DATAWAREHOUSE_BUCKET=UPLOAD_BUCKET, DATAWAREHOUSE_BUCKET_DOMAIN=UPLOAD_DOMAIN):+ table = self._table(+ "uploaded",+ url=f"https://{UPLOAD_DOMAIN}/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",+ created_by=self.users["creator"],+ )+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(+ credential=None, options={"csv_allow_double_quotes": False}+ )+ table.refresh_from_db()+ self._apply({"table": "none", "table:A": "viewer"}, table)++ [outcome] = self._migrate()++ assert outcome.action == "linked", outcome.error or outcome.status.reason+ source = _schema_for(table).source+ assert source.source_type == "FileUpload"+ assert self._source_rules(source) == {("A", "viewer"), ("", "none")}+ assert (source.connection_metadata or {})[MIGRATION_MARKER_KEY]["access_signature"]+ self._assert_move_keeps_access(table)+++class TestAccessRulesAfterLinking(_AccessRulesTest):+ def _linked(self, rules: dict[str, str], name: str = "orders") -> DataWarehouseTable:+ table = self._table(name, url=f"https://exports.s3.eu-west-1.amazonaws.com/{name}/*.csv")+ self._apply(rules, table)+ [outcome] = self._migrate(table_ids=(str(table.id),))+ assert outcome.action == "linked"+ return table++ @parameterized.expand(+ [+ ("no_change", None, None),+ ("table_rule_added", "table:named", "the access rules of the table or of its source changed"),+ ("table_rule_widened", "table", "the access rules of the table or of its source changed"),+ ("source_rule_edited", "source", "the access rules of the table or of its source changed"),+ ("sources_hidden", "sources", "now the project has access rules for sources"),+ ]+ )+ def test_the_first_sync_refuses_to_adopt_a_table_whose_rules_changed(+ self, _name: str, change: str | None, refusal: str | None+ ) -> None:+ table = self._linked({"table": "viewer"})+ source = _schema_for(table).source+ if change == "table:named":+ self._rule("table:named", "viewer", table)+ elif change == "table":+ AccessControl.objects.filter(resource="warehouse_table", resource_id=str(table.id)).update(+ access_level="editor"+ )+ elif change == "source":+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(source.pk)).update(+ access_level="none"+ )+ elif change == "sources":+ self._rule("sources", "none")++ status = self._status(table)+ if refusal is None:+ assert_adoption_keeps_access(table, source)+ assert status.reason == "syncing is off"+ return+ with self.assertRaisesRegex(AdoptionAccessError, refusal):+ assert_adoption_keeps_access(table, source)+ assert status.state == TableMigrationState.AWAITING_SYNC+ assert refusal in (status.reason or "")+ assert "the first sync will fail and leave the table as it is" in (status.reason or "")++ @parameterized.expand([("unchanged", False, False), ("rule_added", True, False), ("already_adopted", True, True)])+ def test_the_import_stops_before_extraction_only_when_the_rules_no_longer_fit(+ self, _name: str, rule_added: bool, adopted: bool+ ) -> None:+ table = self._linked({"table": "viewer"})+ if rule_added:+ self._rule("table:named", "viewer", table)+ if adopted:+ self._adopt(table)++ error = self_managed_adoption_error(_schema_for(table) if not adopted else self._unlinked_schema(table))++ if not rule_added or adopted:+ assert error is None+ return+ assert error is not None+ assert error.startswith(f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table")+ assert "an organization admin must make the access rules of this source the same" in error++ def _unlinked_schema(self, table: DataWarehouseTable) -> ExternalDataSchema:+ schema = _schema_for(table)+ ExternalDataSchema.objects.filter(pk=schema.pk).update(table=None)+ schema.refresh_from_db()+ return schema++ def test_a_source_made_before_access_signatures_takes_only_tables_without_rules(self) -> None:+ old = self._linked({})+ source = _schema_for(old).source+ assert MIGRATION_MARKER_KEY in (source.connection_metadata or {})+ assert "access_signature" not in (source.connection_metadata or {})[MIGRATION_MARKER_KEY]+ plain = self._linked({}, name="plain")+ private = self._linked({"table": "none"}, name="private")++ assert _schema_for(plain).source_id == source.pk+ assert _schema_for(private).source_id != source.pk+ assert self._source_rules(source) == set()++ def test_a_table_does_not_join_a_source_whose_rules_someone_changed(self) -> None:+ first = self._linked({"table": "none"}, name="first")+ source = _schema_for(first).source+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(source.pk)).update(+ access_level="viewer"+ )+ second = self._table("second", url="https://exports.s3.eu-west-1.amazonaws.com/second/*.csv")+ self._rule("table", "none", second)++ [outcome] = self._migrate(table_ids=(str(second.id),))++ assert outcome.action == "link_failed"+ assert "the access rules on the source for this bucket changed" in (outcome.error or "")+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 1++ @parameterized.expand([("last_table", False), ("other_tables_stay", True)])+ def test_abandon_removes_the_rules_only_with_the_source(self, _name: str, keep_another: bool) -> None:+ table = self._linked({"table": "none"})+ source = _schema_for(table).source+ if keep_another:+ other = self._linked({"table": "none"}, name="other")+ assert _schema_for(other).source_id == source.pk++ assert self._abandon(table).action == "abandoned"++ source.refresh_from_db()+ assert source.deleted is not keep_another+ assert self._source_rules(source) == (({("", "none")}) if keep_another else set())++ def test_abandon_of_a_table_never_linked_leaves_no_rules(self) -> None:+ table = self._table()+ self._rule("table", "none", table)++ assert self._abandon(table).action == "abandoned"++ assert not AccessControl.objects.filter(resource="external_data_source").exists()++ def test_a_rule_written_while_the_run_plans_fails_the_link(self) -> None:+ table = self._table()+ planned_reads: list[TeamAccessRules] = []++ def rules_read_then_changed(team_id: int) -> TeamAccessRules:+ rules = TeamAccessRules(team_id)+ if not planned_reads:+ self._rule("table", "none", table)+ planned_reads.append(rules)+ return rules++ with patch(f"{MODULE}.TeamAccessRules", side_effect=rules_read_then_changed):+ [outcome] = self._migrate()++ assert (outcome.action, outcome.error) == ("link_failed", "the access rules changed during the run")+ assert not ExternalDataSource.objects.filter(team=self.team).exists()+++class TestMergeKeepsAccessRules(_AccessRulesTest):+ def _duplicate_pair(self, rules: dict[str, str]) -> tuple[DataWarehouseTable, DataWarehouseTable]:+ """Two sources of one group with the same rules, as earlier runs could leave them."""+ tables = []+ for i in range(2):+ table = self._table(+ f"table_{i}",+ url=f"https://exports.s3.eu-west-1.amazonaws.com/t{i}/*.csv",+ credential=self._credential(f"AKIATEMP{i}"),+ )+ self._apply(rules, table)+ tables.append(table)+ self._migrate()+ for i, table in enumerate(tables):+ credential = table.credential+ assert credential is not None+ credential.access_key = "AKIAFIRST"+ credential.save()+ source = _schema_for(table).source+ inputs = SourceInputsStore.load(source)+ SourceInputsStore.save(+ source, {key: "AKIAFIRST" if value == f"AKIATEMP{i}" else value for key, value in inputs.items()}+ )+ signature = (source.connection_metadata or {})[MIGRATION_MARKER_KEY].get("access_signature")+ ExternalDataSource.objects.filter(pk=source.pk).update(+ created_at=datetime(2024, 1, 1 + i, tzinfo=UTC),+ connection_metadata={+ MIGRATION_MARKER_KEY: {+ "group_key": "s3/eu-west-1/exports",+ "credential_id": str(credential.pk),+ **({"access_signature": signature} if signature else {}),+ }+ },+ )+ assert _schema_for(tables[0]).source_id != _schema_for(tables[1]).source_id+ return tables[0], tables[1]++ def test_sources_with_the_same_rules_merge_and_the_duplicates_rules_go(self) -> None:+ first, second = self._duplicate_pair({"table": "none"})+ kept, duplicate = _schema_for(first).source, _schema_for(second).source++ [outcome] = merge_duplicate_sources(team_id=self.team.pk, live_run=True)++ assert (outcome.action, outcome.source_id) == ("merged", str(duplicate.pk))+ assert _schema_for(second).source_id == kept.pk+ assert self._source_rules(kept) == {("", "none")}+ assert self._source_rules(duplicate) == set()+ self._assert_move_keeps_access(second)++ @parameterized.expand([("rules_differ", "the kept and the duplicate source have different access rules")])+ def test_sources_whose_rules_differ_never_merge(self, _name: str, reason: str) -> None:+ first, second = self._duplicate_pair({"table": "none"})+ duplicate = _schema_for(second).source+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(duplicate.pk)).update(+ access_level="viewer"+ )++ [outcome] = merge_duplicate_sources(team_id=self.team.pk, live_run=True)++ assert (outcome.action, outcome.reason) == ("skipped", reason)+ assert _schema_for(second).source_id == duplicate.pk++ def test_sources_with_other_access_signatures_are_never_grouped(self) -> None:+ open_table = self._table("open", url="https://exports.s3.eu-west-1.amazonaws.com/open/*.csv")+ private = self._table("private", url="https://exports.s3.eu-west-1.amazonaws.com/private/*.csv")+ self._rule("table", "none", private)+ self._migrate()+ assert _schema_for(open_table).source_id != _schema_for(private).source_id++ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []