Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): copy access rules when migrating self-managed tables (#129218)

Tom Owersbe5a8bcff30f
diff --git a/products/access_control/backend/facade/api.py b/products/access_control/backend/facade/api.py
index 3d7c3d8e73e2..f9d5b56697f1 100644
--- a/products/access_control/backend/facade/api.py
+++ b/products/access_control/backend/facade/api.py
@@ -19,6 +19,7 @@ Do NOT:
from __future__ import annotations
from collections import defaultdict
+from collections.abc import Collection, Sequence
from copy import copy
from typing import cast
from uuid import UUID
@@ -418,6 +419,100 @@ def resource_has_rules(*, team_id: int, resource: APIScopeObject) -> bool:
return AccessControl.objects.filter(team_id=team_id, resource=resource, resource_id__isnull=True).exists()
+def access_rules(*, team_id: int, resources: Collection[str]) -> list[contracts.AccessRule]:
+ """Every rule of the team on these resources, object-level and resource-level, whatever its level.
+
+ For a caller that must predict what resolution decides after it changes which rules reach an
+ object. Like `object_ids_with_rules`, it ignores whether the organization enforces rules today,
+ because the rules apply again when it does.
+ """
+ team = Team.objects.only("id", "organization_id").get(id=team_id)
+ rows = AccessControl.objects.filter(team_id=team_id, resource__in=list(resources)).values_list(
+ "resource",
+ "resource_id",
+ "access_level",
+ "organization_member_id",
+ "role_id",
+ "organization_member__organization_id",
+ "role__organization_id",
+ )
+ return [
+ contracts.AccessRule(
+ resource=resource,
+ resource_id=resource_id,
+ access_level=access_level,
+ organization_member_id=member_id,
+ role_id=role_id,
+ # Resolution reads a member's rules only in the member's own organization, and a role's
+ # rules only for a role of that organization (see UserAccessControl._filter_options).
+ applies=(member_id is None or role_id is None)
+ and (member_id is None or member_organization_id == team.organization_id)
+ and (role_id is None or role_organization_id == team.organization_id),
+ )
+ for (
+ resource,
+ resource_id,
+ access_level,
+ member_id,
+ role_id,
+ member_organization_id,
+ role_organization_id,
+ ) in rows
+ ]
+
+
+def replace_object_access_rules(
+ *, team_id: int, resource: str, resource_id: str, rules: Sequence[contracts.ObjectAccessRuleInput]
+) -> None:
+ """Make the rules on one object exactly `rules`, the object's default rule included.
+
+ A rule that is already there stays. The others are created or deleted one at a time, so each
+ change goes into the activity log. An empty `rules` removes every rule on the object. The
+ caller decides whether the change is allowed.
+ """
+ team = Team.objects.only("id", "organization_id").get(id=team_id)
+ wanted: dict[tuple[UUID | None, UUID | None], str] = {}
+ for rule in rules:
+ _validate_resource_access_level(resource, rule.access_level)
+ key = (rule.organization_member_id, rule.role_id)
+ if key in wanted:
+ raise InvalidObjectAccessControlError("Provide each subject only once.")
+ wanted[key] = rule.access_level
+ member_ids = {member_id for member_id, _ in wanted if member_id is not None}
+ role_ids = {role_id for _, role_id in wanted if role_id is not None}
+ if member_ids != set(
+ OrganizationMembership.objects.filter(organization_id=team.organization_id, id__in=member_ids).values_list(
+ "id", flat=True
+ )
+ ):
+ raise InvalidObjectAccessControlError("organization_member does not belong to this team's organization.")
+ if role_ids != set(
+ Role.objects.filter(organization_id=team.organization_id, id__in=role_ids).values_list("id", flat=True)
+ ):
+ raise InvalidObjectAccessControlError("role does not belong to this team's organization.")
+
+ with transaction.atomic():
+ kept: set[tuple[UUID | None, UUID | None]] = set()
+ for existing in AccessControl.objects.select_for_update().filter(
+ team_id=team_id, resource=resource, resource_id=resource_id
+ ):
+ key = (existing.organization_member_id, existing.role_id)
+ if key not in kept and wanted.get(key) == existing.access_level:
+ kept.add(key)
+ else:
+ existing.delete()
+ for (member_id, role_id), access_level in wanted.items():
+ if (member_id, role_id) not in kept:
+ AccessControl.objects.create(
+ team_id=team_id,
+ resource=resource,
+ resource_id=resource_id,
+ access_level=access_level,
+ organization_member_id=member_id,
+ role_id=role_id,
+ )
+
+
def role_belongs_to_organization(*, role_id: str | UUID, organization_id: UUID) -> bool:
"""Whether the role is a role of that organization.
@@ -571,20 +666,23 @@ def _to_object_rule(rule: AccessControl) -> contracts.ObjectAccessControlRule:
def _validate_object_access_control_level(input: contracts.SetObjectAccessControlInput) -> APIScopeObject:
- resource = input.resource
+ return _validate_resource_access_level(input.resource, input.access_level)
+
+
+def _validate_resource_access_level(resource: str, access_level: str | None) -> APIScopeObject:
if resource not in API_SCOPE_OBJECTS or resource in INTERNAL_API_SCOPE_OBJECTS:
raise InvalidObjectAccessControlError(f"{resource} is not an access controlled resource.")
scoped_resource = cast(APIScopeObject, resource)
- if input.access_level is not None:
+ if access_level is not None:
levels = ordered_access_levels(scoped_resource)
- if input.access_level not in levels:
+ if access_level not in levels:
raise InvalidObjectAccessControlError(
f"Invalid access level for {resource}. Must be one of: {', '.join(levels)}."
)
# `ordered_access_levels` can include levels the settings UI never offers for a resource
# (for example "none" on a project), so the bounds apply here as well as in the API.
- level_index = levels.index(input.access_level)
+ level_index = levels.index(cast(AccessControlLevel, access_level))
if level_index < levels.index(minimum_access_level(scoped_resource)):
raise InvalidObjectAccessControlError(f"Access level is below the minimum for {resource}.")
if level_index > levels.index(highest_access_level(scoped_resource)):
diff --git a/products/access_control/backend/facade/contracts.py b/products/access_control/backend/facade/contracts.py
index f8a4e53adf72..c755f088a98b 100644
--- a/products/access_control/backend/facade/contracts.py
+++ b/products/access_control/backend/facade/contracts.py
@@ -149,6 +149,35 @@ class ObjectAccessControlRule:
role_id: UUID | None
+@frozen
+class AccessRule:
+ """One stored rule, object-level or resource-level, as resolution reads it.
+
+ `applies` is False for a rule that resolution never uses: it names a member or a role of
+ another organization, or it names both a member and a role.
+ """
+
+ resource: str
+ resource_id: str | None
+ access_level: str
+ organization_member_id: UUID | None
+ role_id: UUID | None
+ applies: bool
+
+
+@frozen
+class ObjectAccessRuleInput:
+ """One rule to keep on an object. With no member and no role, it is the object's default rule."""
+
+ access_level: str
+ organization_member_id: UUID | None = None
+ role_id: UUID | None = None
+
+ def __post_init__(self) -> None:
+ if self.organization_member_id is not None and self.role_id is not None:
+ raise ValueError("Set at most one of organization_member_id or role_id.")
+
+
@frozen
class TerraformLock:
"""The state of the Terraform lock of one project. has_terraform_account is true when Terraform
diff --git a/products/access_control/backend/tests/test_api.py b/products/access_control/backend/tests/test_api.py
index c9904bb3a828..5c7a7a1c4d85 100644
--- a/products/access_control/backend/tests/test_api.py
+++ b/products/access_control/backend/tests/test_api.py
@@ -17,6 +17,7 @@ from products.access_control.backend.facade.api import (
bulk_set_object_access_controls,
every_member_has_resource_access,
object_ids_restricted_from_any_member,
+ replace_object_access_rules,
role_belongs_to_organization,
set_object_access_control,
valid_role_member_user_ids,
@@ -27,7 +28,7 @@ from products.access_control.backend.models.role import Role, RoleMembership
from products.dashboards.backend.models.dashboard import Dashboard
-class TestSetObjectAccessControl(BaseTest):
+class _DashboardRulesTest(BaseTest):
def setUp(self) -> None:
super().setUp()
self.organization.available_product_features = [
@@ -40,6 +41,11 @@ class TestSetObjectAccessControl(BaseTest):
self.dashboard = Dashboard.objects.create(team=self.team, created_by=self.user)
AccessControl.objects.create(team=self.team, resource="dashboard", resource_id=None, access_level="none")
+ def _other_user_level(self) -> str | None:
+ return UserAccessControl(user=self.other_user, team=self.team).get_user_access_level(self.dashboard)
+
+
+class TestSetObjectAccessControl(_DashboardRulesTest):
def _grant(
self, access_level: str | None = "viewer", **overrides: object
) -> contracts.ObjectAccessControlRule | None:
@@ -54,9 +60,6 @@ class TestSetObjectAccessControl(BaseTest):
team_id=self.team.id, input=contracts.SetObjectAccessControlInput(**input_kwargs)
)
- def _other_user_level(self) -> str | None:
- return UserAccessControl(user=self.other_user, team=self.team).get_user_access_level(self.dashboard)
-
def test_grant_update_and_revoke_are_visible_through_user_access_control(self) -> None:
assert self._other_user_level() == "none"
@@ -246,6 +249,76 @@ class TestSetObjectAccessControl(BaseTest):
assert not AccessControl.objects.filter(team=self.team, resource_id=str(self.dashboard.id)).exists()
+class TestReplaceObjectAccessRules(_DashboardRulesTest):
+ def _replace(self, *rules: contracts.ObjectAccessRuleInput) -> None:
+ replace_object_access_rules(
+ team_id=self.team.id, resource="dashboard", resource_id=str(self.dashboard.id), rules=list(rules)
+ )
+
+ def _rules(self) -> dict[tuple[object, object], tuple[object, str]]:
+ return {
+ (rule.organization_member_id, rule.role_id): (rule.id, rule.access_level)
+ for rule in AccessControl.objects.filter(team=self.team, resource_id=str(self.dashboard.id))
+ }
+
+ def test_the_object_ends_with_exactly_the_given_rules(self) -> None:
+ self._replace(
+ contracts.ObjectAccessRuleInput(access_level="viewer"),
+ contracts.ObjectAccessRuleInput(access_level="none", organization_member_id=self.other_membership.id),
+ )
+ kept_id = self._rules()[(None, None)][0]
+ assert self._other_user_level() == "none"
+
+ self._replace(
+ contracts.ObjectAccessRuleInput(access_level="viewer"),
+ contracts.ObjectAccessRuleInput(access_level="editor", organization_member_id=self.other_membership.id),
+ )
+
+ rules = self._rules()
+ assert rules[(None, None)] == (kept_id, "viewer")
+ assert rules[(self.other_membership.id, None)][1] == "editor"
+ assert len(rules) == 2
+ assert self._other_user_level() == "editor"
+ logged = ActivityLog.objects.filter(
+ scope="AccessControl", detail__name=f"dashboard {self.dashboard.id}"
+ ).values_list("activity", flat=True)
+ assert sorted(logged) == ["created", "created", "created", "deleted"]
+
+ self._replace()
+
+ assert self._rules() == {}
+
+ @parameterized.expand([("unknown_level",), ("foreign_member",), ("foreign_role",), ("same_subject_twice",)])
+ def test_rejects_invalid_rules_before_writing(self, case: str) -> None:
+ other_org = Organization.objects.create(name="Other")
+ outsider = User.objects.create_and_join(other_org, "[email protected]", "testtest")
+ rules = {
+ "unknown_level": [contracts.ObjectAccessRuleInput(access_level="owner")],
+ "foreign_member": [
+ contracts.ObjectAccessRuleInput(
+ access_level="viewer",
+ organization_member_id=OrganizationMembership.objects.get(user=outsider).id,
+ )
+ ],
+ "foreign_role": [
+ contracts.ObjectAccessRuleInput(
+ access_level="viewer", role_id=Role.objects.create(name="Outside", organization=other_org).id
+ )
+ ],
+ "same_subject_twice": [
+ contracts.ObjectAccessRuleInput(access_level="viewer"),
+ contracts.ObjectAccessRuleInput(access_level="none"),
+ ],
+ }[case]
+ self._replace(contracts.ObjectAccessRuleInput(access_level="editor"))
+ before = self._rules()
+
+ with pytest.raises(InvalidObjectAccessControlError):
+ self._replace(*rules)
+
+ assert self._rules() == before
+
+
class TestEveryMemberHasResourceAccess(BaseTest):
"""`account` inherits from `customer_analytics`, so every rule here is written on the parent,
which is where resolution reads them from."""
diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
index fe7193012757..8daffb22829e 100644
--- a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
@@ -29,7 +29,8 @@ class Command(BaseCommand):
"in a source for its bucket, and nothing syncs: the table keeps reading its own bucket until the user "
"turns syncing on, and the first sync then replaces it with the same id and name. A table made from an "
"uploaded file gets a file upload source of its own, and its next uploaded file is the first sync. "
- "Runs only for teams "
+ "Each source gets the access rules that keep every member's access to its tables, or the table stays "
+ "self-managed. Runs only for teams "
"with the object storage sources feature flag. Without --live-run the command only reports what it "
"would do."
)
diff --git a/products/warehouse_sources/backend/self_managed_access.py b/products/warehouse_sources/backend/self_managed_access.py
new file mode 100644
index 000000000000..0859efa2dadf
--- /dev/null
+++ b/products/warehouse_sources/backend/self_managed_access.py
@@ -0,0 +1,412 @@
+"""The access rules a migration source needs, so that moving a self-managed table into it changes nobody's access.
+
+A self-managed table has no source, so access to it resolves from the table's own rules and the rules for the
+whole warehouse (`warehouse_objects`). Once a source adopts the table, access also falls back to the rules on
+that source and then to the rules for all sources (`RESOURCE_FALLBACK_MAP` in the access control facade). The
+source also has access of its own: to its settings, its schemas and its syncs, and through them to the files
+the table reads.
+
+So each migration source gets rules that make two things true for every member, whatever roles the member
+has, under both resolution orders an organization can use:
+
+- the member's access to each table of the source is the same before and after the source adopts it, and
+- the member's access to the source is never more than their access to each of its tables.
+
+The rules copy the table's member and role rules, which decide on the table before anything reaches the
+source, and add a default rule on the source when one is needed. When no such rules exist, the table can't
+move. Tables of one bucket share a source only when they need the same rules (see `SourceAccessPlan.signature`).
+
+The prediction mirrors `UserAccessControl` (`_object_access_level_from_rows` and
+`_most_specific_object_access_from_rows`). The tests check it against that code.
+"""
+
+import json
+import hashlib
+import itertools
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from typing import Literal
+from uuid import UUID
+
+from posthog.dataclasses import frozen
+
+from products.access_control.backend.facade.api import access_rules, replace_object_access_rules
+from products.access_control.backend.facade.contracts import AccessRule, ObjectAccessRuleInput
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+
+# The `connection_metadata` key of a source the migration created.
+MIGRATION_MARKER_KEY = "self_managed_migration"
+
+TABLE_RESOURCE = "warehouse_table"
+WAREHOUSE_RESOURCE = "warehouse_objects"
+SOURCE_RESOURCE = "external_data_source"
+
+LEVELS: tuple[str, ...] = ("none", "viewer", "editor", "manager")
+# The level a table or a source resolves to when no rule decides (`default_access_level`).
+_SYSTEM_DEFAULT = "editor"
+# Each role profile doubles the subjects to check, so a rule set that names more is refused.
+_MAX_ROLE_PROFILES = 10
+
+# The start of the error a sync raises when it must not replace a self-managed table. The import
+# classifies it as permanent, so the job fails once and syncing stops until someone fixes the rules.
+ADOPTION_ACCESS_ERROR = "PostHog did not replace the self-managed table"
+
+TABLE_RULES_REASON = "the table has its own access rules, which the source PostHog would create can't carry"
+PROJECT_RULES_REASON = "the project has access rules for sources or for the warehouse, which the move would change"
+ROLES_REASON = "the access rules name too many roles to check"
+
+_Subject = Literal["member", "role", "default"]
+# Most specific first, the order most-specific resolution reads one scope's rules in.
+_SUBJECTS: tuple[_Subject, ...] = ("member", "role", "default")
+_Scope = Literal["table", "warehouse", "sources", "source"]
+_SCOPES: tuple[_Scope, ...] = ("table", "warehouse", "sources", "source")
+
+
+@frozen
+class SourceRule:
+ """One rule on a migration source. With no member and no role, it is the source's default rule."""
+
+ access_level: str
+ organization_member_id: str | None = None
+ role_id: str | None = None
+
+ @property
+ def subject(self) -> _Subject:
+ if self.organization_member_id is not None:
+ return "member"
+ if self.role_id is not None:
+ return "role"
+ return "default"
+
+
+@frozen
+class SourceAccessPlan:
+ """The rules a migration source needs for its tables to keep their access."""
+
+ rules: tuple[SourceRule, ...] = ()
+
+ @property
+ def signature(self) -> str:
+ """Identifies the rules, so only tables that need the same rules share a source. Empty without rules."""
+ if not self.rules:
+ return ""
+ payload = json.dumps([[r.access_level, r.organization_member_id, r.role_id] for r in self.rules])
+ return hashlib.sha256(payload.encode()).hexdigest()[:32]
+
+ def describe(self) -> str:
+ parts = []
+ for rule in self.rules:
+ subject = {
+ "member": f"member {rule.organization_member_id}",
+ "role": f"role {rule.role_id}",
+ "default": "everyone",
+ }[rule.subject]
+ parts.append(f"{subject}={rule.access_level}")
+ return ", ".join(parts)
+
+
+@frozen
+class _Row:
+ level: str
+ subject: _Subject
+
+
+class AdoptionAccessError(Exception):
+ """The access rules changed after the migration, so the first sync must not replace the table."""
+
+
+def _rank(level: str) -> int:
+ return LEVELS.index(level)
+
+
+def _highest(rows: Iterable[_Row]) -> str:
+ return max((row.level for row in rows), key=_rank)
+
+
+def _decision(rows: Sequence[_Row]) -> _Row | None:
+ """The deciding row of one scope under most-specific resolution: the member's, then the roles', then the default."""
+ for subject in _SUBJECTS:
+ group = [row for row in rows if row.subject == subject]
+ if group:
+ return _Row(level=_highest(group), subject=subject)
+ return None
+
+
+def _legacy_level(
+ object_rows: Sequence[_Row],
+ parent_rows: Sequence[_Row] | None,
+ resource_rows: Sequence[_Row],
+ parent_resource_rows: Sequence[_Row],
+) -> str:
+ """`UserAccessControl._object_access_level_from_rows`, for a member who is no admin and not the creator.
+
+ `parent_rows` is None for an object without a fallback parent: a self-managed table, or a source.
+ """
+ explicit = [row for row in object_rows if row.subject != "default"]
+ if explicit:
+ return _highest(explicit)
+ if object_rows and _highest(object_rows) == "none":
+ return "none"
+ if parent_rows:
+ return _highest(parent_rows)
+ if resource_rows:
+ return _highest(resource_rows)
+ if parent_rows is not None and parent_resource_rows:
+ return _highest(parent_resource_rows)
+ if object_rows:
+ return _highest(object_rows)
+ return _SYSTEM_DEFAULT
+
+
+def _most_specific_level(
+ object_rows: Sequence[_Row],
+ parent_rows: Sequence[_Row] | None,
+ resource_rows: Sequence[_Row],
+ parent_resource_rows: Sequence[_Row],
+) -> str:
+ """`UserAccessControl._most_specific_object_access_from_rows`, for a member who is no admin and not the creator."""
+
+ def resource_level() -> str:
+ if resource_rows:
+ return _decided_level(resource_rows)
+ if parent_rows is not None and parent_resource_rows:
+ return _decided_level(parent_resource_rows)
+ return _SYSTEM_DEFAULT
+
+ for rows in (object_rows, parent_rows or ()):
+ row = _decision(rows)
+ if row is not None:
+ # A default grant on the object or its parent can't widen "none" for the resource.
+ if row.subject == "default" and row.level != "none" and resource_level() == "none":
+ return "none"
+ return row.level
+ return resource_level()
+
+
+def _decided_level(rows: Sequence[_Row]) -> str:
+ row = _decision(rows)
+ assert row is not None
+ return row.level
+
+
+class TeamAccessRules:
+ """The rules of one team that decide access to its warehouse tables and its sources.
+
+ Rules that resolution never applies (a member or a role of another organization) are left out.
+ Several rows for one subject on one object act as their highest level, so they count as one.
+ """
+
+ def __init__(self, team_id: int) -> None:
+ self.team_id = team_id
+ rows = [
+ rule
+ for rule in access_rules(team_id=team_id, resources=(TABLE_RESOURCE, WAREHOUSE_RESOURCE, SOURCE_RESOURCE))
+ if rule.applies
+ ]
+ self._warehouse = _subject_rules(r for r in rows if r.resource == WAREHOUSE_RESOURCE and r.resource_id is None)
+ self._sources = _subject_rules(r for r in rows if r.resource == SOURCE_RESOURCE and r.resource_id is None)
+ by_object: dict[tuple[str, str], list[AccessRule]] = defaultdict(list)
+ for rule in rows:
+ if rule.resource_id is not None and rule.resource in (TABLE_RESOURCE, SOURCE_RESOURCE):
+ by_object[(rule.resource, rule.resource_id)].append(rule)
+ self._objects = {key: _subject_rules(rules) for key, rules in by_object.items()}
+ self._plans: dict[tuple[SourceRule, ...], SourceAccessPlan | str] = {}
+
+ def source_rules(self, source_id: str) -> tuple[SourceRule, ...]:
+ """The rules on a source as they are now, in the order a plan lists them."""
+ return self._objects.get((SOURCE_RESOURCE, source_id), ())
+
+ def plan_for(self, table_id: str) -> SourceAccessPlan | str:
+ """The rules a source needs for the table to keep its access, or the reason no rules can."""
+ table_rules = self._objects.get((TABLE_RESOURCE, table_id), ())
+ if table_rules not in self._plans:
+ self._plans[table_rules] = self._plan(table_rules)
+ return self._plans[table_rules]
+
+ def _plan(self, table_rules: tuple[SourceRule, ...]) -> SourceAccessPlan | str:
+ unknown = next(
+ (
+ rule.access_level
+ for rule in (*table_rules, *self._warehouse, *self._sources)
+ if rule.access_level not in LEVELS
+ ),
+ None,
+ )
+ if unknown is not None:
+ return f"an access rule has the level '{unknown}', which PostHog doesn't know"
+ found = self._search(table_rules)
+ if found is not None:
+ return found
+ # When the same table without rules of its own could move, its own rules are the cause.
+ if table_rules and isinstance(self._search(()), SourceAccessPlan):
+ return TABLE_RULES_REASON
+ return PROJECT_RULES_REASON
+
+ def _search(self, table_rules: tuple[SourceRule, ...]) -> SourceAccessPlan | str | None:
+ """The first source rules that keep every member's access, or None when no rules can."""
+ # A member or role rule on the table decides before anything reaches the source. So a copy of
+ # it on the source changes nothing on the table, and gives that subject equal access to both.
+ copies = tuple(rule for rule in table_rules if rule.subject != "default")
+ table_default = next((rule.access_level for rule in table_rules if rule.subject == "default"), None)
+ # The table's own default comes first, then no default, so the source follows the table where it can.
+ for source_default in dict.fromkeys([table_default, None, *LEVELS]):
+ rules = _sorted([*copies, *([SourceRule(access_level=source_default)] if source_default else [])])
+ verdict = self._keeps_access(table_rules, rules)
+ if verdict is True:
+ return SourceAccessPlan(rules=rules)
+ if isinstance(verdict, str):
+ return verdict
+ return None
+
+ def _keeps_access(self, table_rules: tuple[SourceRule, ...], source_rules: tuple[SourceRule, ...]) -> bool | str:
+ """Whether every member keeps their access to the table, and gets no more to the source than to the table.
+
+ Checked for each kind of member (the rules that name them) with each combination of roles,
+ because a member's roles can change at any time, and under both resolution orders, because an
+ organization can change its order. Admins and the table's creator get the highest level before
+ any rule is read, so the move can't change their access.
+ """
+ scopes: dict[_Scope, tuple[SourceRule, ...]] = {
+ "table": table_rules,
+ "warehouse": self._warehouse,
+ "sources": self._sources,
+ "source": source_rules,
+ }
+ defaults = {
+ scope: [_Row(level=rule.access_level, subject="default") for rule in rules if rule.subject == "default"]
+ for scope, rules in scopes.items()
+ }
+ role_profiles = [profile for profile in _profiles(scopes, "role") if profile != _EMPTY_PROFILE]
+ if len(role_profiles) > _MAX_ROLE_PROFILES:
+ return ROLES_REASON
+ for member_profile in _profiles(scopes, "member"):
+ for count in range(len(role_profiles) + 1):
+ for roles in itertools.combinations(role_profiles, count):
+ rows = {
+ scope: [
+ *defaults[scope],
+ *(
+ [_Row(level=member_level, subject="member")]
+ if (member_level := member_profile[i])
+ else []
+ ),
+ *(_Row(level=level, subject="role") for role in roles if (level := role[i]) is not None),
+ ]
+ for i, scope in enumerate(_SCOPES)
+ }
+ for resolve in (_legacy_level, _most_specific_level):
+ before = resolve(rows["table"], None, rows["warehouse"], rows["sources"])
+ after = resolve(rows["table"], rows["source"], rows["warehouse"], rows["sources"])
+ source = resolve(rows["source"], None, rows["sources"], ())
+ if after != before or _rank(source) > _rank(before):
+ return False
+ return True
+
+
+_Profile = tuple[str | None, ...]
+_EMPTY_PROFILE: _Profile = (None, None, None, None)
+
+
+def _profiles(scopes: dict[_Scope, tuple[SourceRule, ...]], subject: Literal["member", "role"]) -> list[_Profile]:
+ """The level each subject of one kind has in each scope, once per distinct set of levels.
+
+ Subjects with the same levels resolve the same, so each set is checked once. The empty set stands
+ for a member that no rule names, or for having no role.
+ """
+ levels: dict[str, dict[_Scope, str]] = defaultdict(dict)
+ for scope, rules in scopes.items():
+ for rule in rules:
+ if rule.subject == subject:
+ subject_id = rule.organization_member_id if subject == "member" else rule.role_id
+ assert subject_id is not None
+ levels[subject_id][scope] = rule.access_level
+ profiles = {tuple(by_scope.get(scope) for scope in _SCOPES) for by_scope in levels.values()}
+ profiles.add(_EMPTY_PROFILE)
+ return sorted(profiles, key=repr)
+
+
+def _subject_rules(rules: Iterable[AccessRule]) -> tuple[SourceRule, ...]:
+ highest: dict[tuple[str | None, str | None], str] = {}
+ for rule in rules:
+ key = (
+ str(rule.organization_member_id) if rule.organization_member_id else None,
+ str(rule.role_id) if rule.role_id else None,
+ )
+ current, level = highest.get(key), rule.access_level
+ # An unknown level is kept, so the plan refuses the table instead of guessing what it means.
+ if current is None or level not in LEVELS or (current in LEVELS and _rank(level) > _rank(current)):
+ highest[key] = level
+ return _sorted(
+ SourceRule(access_level=level, organization_member_id=member_id, role_id=role_id)
+ for (member_id, role_id), level in highest.items()
+ )
+
+
+def _sorted(rules: Iterable[SourceRule]) -> tuple[SourceRule, ...]:
+ return tuple(
+ sorted(
+ rules,
+ key=lambda r: (_SUBJECTS.index(r.subject), r.organization_member_id or "", r.role_id or "", r.access_level),
+ )
+ )
+
+
+def write_source_rules(team_id: int, source_id: str, plan: SourceAccessPlan) -> None:
+ replace_object_access_rules(
+ team_id=team_id,
+ resource=SOURCE_RESOURCE,
+ resource_id=source_id,
+ rules=[
+ ObjectAccessRuleInput(
+ access_level=rule.access_level,
+ organization_member_id=UUID(rule.organization_member_id) if rule.organization_member_id else None,
+ role_id=UUID(rule.role_id) if rule.role_id else None,
+ )
+ for rule in plan.rules
+ ],
+ )
+
+
+def remove_source_rules(team_id: int, source_id: str) -> None:
+ replace_object_access_rules(team_id=team_id, resource=SOURCE_RESOURCE, resource_id=source_id, rules=[])
+
+
+def adoption_access_refusal(
+ table: DataWarehouseTable, source: ExternalDataSource, rules: TeamAccessRules | None = None
+) -> str | None:
+ """Why the source must not adopt the table now, or None when the move keeps everyone's access.
+
+ The rules may have changed since the migration linked the table: on the table, on the source, or
+ for the whole warehouse or all sources. The source's rules must still be the ones the table needs.
+ """
+ rules = rules or TeamAccessRules(table.team_id)
+ planned = rules.plan_for(str(table.id))
+ if isinstance(planned, str):
+ return f"the access rules changed after the migration, and now {planned}"
+ if rules.source_rules(str(source.pk)) != planned.rules:
+ return "the access rules of the table or of its source changed after the migration"
+ return None
+
+
+def adoption_access_error(table: DataWarehouseTable, source: ExternalDataSource) -> str | None:
+ """The error to show when a migration source must not adopt the table, or None when it may.
+
+ Sources the migration didn't make are not checked.
+ """
+ if MIGRATION_MARKER_KEY not in (source.connection_metadata or {}):
+ return None
+ refusal = adoption_access_refusal(table, source)
+ if refusal is None:
+ return None
+ return (
+ f"{ADOPTION_ACCESS_ERROR} '{table.name}', because {refusal}. The table still reads its own files. "
+ "To continue, an organization admin must make the access rules of this source the same as the "
+ "rules of the table, then turn syncing on again. PostHog support can help with this."
+ )
+
+
+def assert_adoption_keeps_access(table: DataWarehouseTable, source: ExternalDataSource) -> None:
+ error = adoption_access_error(table, source)
+ if error is not None:
+ raise AdoptionAccessError(error)
diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.py
index effa281d2d1e..e5b6de97eef8 100644
--- a/products/warehouse_sources/backend/self_managed_migration.py
+++ b/products/warehouse_sources/backend/self_managed_migration.py
@@ -15,6 +15,11 @@ upload folder, and nothing is copied. The next upload for the table points the s
and starts a normal billable sync, which adopts the table. The old file is deleted after that sync,
when nothing else reads it (see `file_upload_sources.replace_file_upload`).
+A table in a source also resolves access through the source's rules, so each source gets the access
+rules that keep every member's access to its tables, and tables share a source only when they need the
+same rules. A table whose access no such rules can keep stays self-managed. The first sync checks the
+rules again before it replaces the table (see `self_managed_access`).
+
The state of each table comes from the rows themselves: a migration source carries a marker in its
`connection_metadata`, and its schemas name their tables. Running the migration again therefore
continues where it stopped.
@@ -43,7 +48,6 @@ from botocore.exceptions import BotoCoreError, ClientError
from posthog.dataclasses import frozen
from posthog.models.team import Team
-from products.access_control.backend.facade.api import object_ids_with_rules, resource_has_rules
from products.data_warehouse.backend.facade.api import delete_external_data_schedule
from products.warehouse_sources.backend.facade.contracts import UploadReadSettings
from products.warehouse_sources.backend.file_upload_sources import (
@@ -77,6 +81,14 @@ from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
source_feature_flag_missing,
)
+from products.warehouse_sources.backend.self_managed_access import (
+ MIGRATION_MARKER_KEY,
+ SourceAccessPlan,
+ TeamAccessRules,
+ adoption_access_refusal,
+ remove_source_rules,
+ write_source_rules,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
UnsupportedCsvColumnError,
)
@@ -115,8 +127,6 @@ from products.warehouse_sources.backend.types import ExternalDataJobStatus
logger = structlog.get_logger(__name__)
-# The `connection_metadata` key of a source the migration created.
-MIGRATION_MARKER_KEY = "self_managed_migration"
DEFAULT_ABANDON_REASON = "Left self-managed by an operator."
_DELETED_REASON = "its migration schema or source was deleted"
@@ -149,6 +159,8 @@ class TableMigrationStatus:
source_id: str | None = None
schema_id: str | None = None
nested_text_column_types: dict[str, str] | None = None
+ # The access rules the table's source gets, for a table not linked yet.
+ planned_access_rules: str | None = None
@frozen
@@ -199,6 +211,8 @@ class _TablePlan:
# The bucket a shared source reads, which names its table prefix when one is needed.
bucket: str = ""
credential_fingerprint: str = dataclasses.field(default="", repr=False)
+ # The rules the source gets. Only tables that need the same rules share a source.
+ access: SourceAccessPlan = SourceAccessPlan()
@frozen
@@ -313,13 +327,23 @@ def _source_fingerprints(sources: Sequence[ExternalDataSource]) -> dict[uuid.UUI
return fingerprints
-def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str) -> None:
+def _access_signature(source: ExternalDataSource) -> str:
+ """The access signature in a source's marker. A source made before markers held one has no rules."""
+ signature = _marker(source).get("access_signature")
+ return signature if isinstance(signature, str) else ""
+
+
+def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str, access_signature: str) -> None:
"""Hold the group's lock until the transaction ends.
Every table of one source shares the lock, so two runs can't create two sources for the group,
- and a merge can't move schemas while a run adds to the group.
+ and a merge can't move schemas while a run adds to the group. A group without access rules keeps
+ the lock name it had before groups had access signatures.
"""
- name = "\n".join((MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint))
+ parts = [MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint]
+ if access_signature:
+ parts.append(access_signature)
+ name = "\n".join(parts)
lock_id = int.from_bytes(hashlib.blake2b(name.encode(), digest_size=8).digest(), "big", signed=True)
with connection.cursor() as cursor:
cursor.execute("SELECT pg_advisory_xact_lock(%s)", [lock_id])
@@ -334,8 +358,7 @@ class _Planner:
self._upload_source = file_upload_source()
self._flag_missing: dict[tuple[int, str], bool] = {}
self._regions: dict[tuple[str, str], str | None] = {}
- self._tables_with_rules: dict[int, set[str]] = {}
- self._teams_with_source_rules: dict[int, bool] = {}
+ self._access: dict[int, TeamAccessRules] = {}
def plan(self, table: DataWarehouseTable) -> _TablePlan | str:
"""The plan for a table, or the reason the table can't be migrated."""
@@ -343,18 +366,14 @@ class _Planner:
hosted = table.hosted_upload if credential is None else None
if credential is None and hosted is None:
return "the table has no credential"
- if self._has_access_rules(table):
- # The new source would get default access, and a member the table's rules shut out
- # could read the same files through a schema of that source.
- return "the table has its own access rules, which the source PostHog would create can't carry"
- if self._has_source_rules(table):
- # A table in a source falls back to the rules for sources, which never reached a
- # self-managed table, so moving it could take access away from members who have it.
- # Rules for the whole warehouse don't reach the new source, so a member they shut out
- # could read the table's files through it.
- return "the project has access rules for sources or for the warehouse, which the move would change"
+ # A table in a source falls back to the source's rules and to the rules for all sources, and a
+ # member who can use the source can read the table's files through it. So the source gets rules
+ # that keep each member's access, or the table stays self-managed.
+ access = self.team_access(table.team_id).plan_for(str(table.id))
+ if isinstance(access, str):
+ return access
if hosted is not None:
- return self._plan_upload(table, hosted)
+ return self._plan_upload(table, hosted, access)
assert credential is not None
fingerprint = _credential_fingerprint(credential)
try:
@@ -383,9 +402,17 @@ class _Planner:
description=f"Self-managed tables in {location.bucket}",
bucket=location.bucket,
credential_fingerprint=fingerprint,
+ access=access,
)
- def _plan_upload(self, table: DataWarehouseTable, hosted: HostedUpload) -> _TablePlan | str:
+ def team_access(self, team_id: int) -> TeamAccessRules:
+ if team_id not in self._access:
+ self._access[team_id] = TeamAccessRules(team_id)
+ return self._access[team_id]
+
+ def _plan_upload(
+ self, table: DataWarehouseTable, hosted: HostedUpload, access: SourceAccessPlan
+ ) -> _TablePlan | str:
"""A table that reads a file the team uploaded gets a file upload source of its own."""
source = self._upload_source
if self._team_lacks_flag(table.team, source):
@@ -415,22 +442,9 @@ class _Planner:
group_key=f"upload/{table.pk}",
description=f"Uploaded file for {table.name}"[:400],
shares_source=False,
+ access=access,
)
- def _has_access_rules(self, table: DataWarehouseTable) -> bool:
- if table.team_id not in self._tables_with_rules:
- self._tables_with_rules[table.team_id] = object_ids_with_rules(
- team_id=table.team_id, resource="warehouse_table"
- )
- return str(table.id) in self._tables_with_rules[table.team_id]
-
- def _has_source_rules(self, table: DataWarehouseTable) -> bool:
- if table.team_id not in self._teams_with_source_rules:
- self._teams_with_source_rules[table.team_id] = resource_has_rules(
- team_id=table.team_id, resource="external_data_source"
- ) or resource_has_rules(team_id=table.team_id, resource="warehouse_objects")
- return self._teams_with_source_rules[table.team_id]
-
def _team_lacks_flag(self, team: Team, source: ObjectStorageSource[Any] | FileUploadSource) -> bool:
key = (team.pk, str(source.source_type))
if key not in self._flag_missing:
@@ -560,12 +574,17 @@ def _idle_reason(schema: ExternalDataSchema) -> str:
return "syncing is off"
-def _awaiting_reason(table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None) -> str:
+def _awaiting_reason(
+ table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None, planner: "_Planner"
+) -> str:
ref = self_managed_table_ref(schema)
if schema.table_id is not None:
return "the table changed after the migration, so its schema syncs a separate table"
if ref is None or (table.url_pattern, table.format) != (ref.url_pattern, ref.format):
return "the table changed after the migration, so a sync makes a separate table"
+ refusal = adoption_access_refusal(table, schema.source, planner.team_access(table.team_id))
+ if refusal is not None:
+ return f"{refusal}, so the first sync will fail and leave the table as it is"
if job is None:
return _idle_reason(schema)
if job.status == ExternalDataJobStatus.RUNNING:
@@ -603,7 +622,7 @@ def _evaluate(
if table.external_data_source_id is not None:
return _Evaluation(table=table, status=status(TableMigrationState.MIGRATED), schema=live, job=job)
if live is not None:
- reason = _awaiting_reason(table, live, job)
+ reason = _awaiting_reason(table, live, job, planner)
return _Evaluation(table=table, status=status(TableMigrationState.AWAITING_SYNC, reason), schema=live, job=job)
if schemas:
refs = [ref for ref in (self_managed_table_ref(schema) for schema in schemas) if ref is not None]
@@ -612,7 +631,11 @@ def _evaluate(
planned = planner.plan(table)
if isinstance(planned, _TablePlan):
- return _Evaluation(table=table, status=status(TableMigrationState.PENDING, "not linked yet"), plan=planned)
+ pending = dataclasses.replace(
+ status(TableMigrationState.PENDING, "not linked yet"),
+ planned_access_rules=planned.access.describe() or None,
+ )
+ return _Evaluation(table=table, status=pending, plan=planned)
return _Evaluation(table=table, status=status(TableMigrationState.SKIPPED, planned))
@@ -652,15 +675,20 @@ def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:
return candidate
-def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:
- """The group's source for the table's team, bucket and key pair, the oldest when there are more.
+def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan, access: TeamAccessRules) -> ExternalDataSource:
+ """The group's source for the table's team, bucket, key pair and access rules, the oldest when there are more.
- Runs under the group lock.
+ A new source gets its access rules in the same transaction. An existing source joins only while
+ its rules are still the ones the table needs. Runs under the group lock.
"""
+ signature = plan.access.signature
if not plan.shares_source:
- return create_source_for_upload(
- table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: {"group_key": plan.group_key}}
- )
+ marker: dict[str, Any] = {"group_key": plan.group_key}
+ if signature:
+ marker["access_signature"] = signature
+ source = create_source_for_upload(table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: marker})
+ _write_access_rules(source, plan)
+ return source
candidates = list(
ExternalDataSource.objects.filter(
team_id=table.team_id,
@@ -671,9 +699,17 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter
.order_by("created_at", "id")
)
fingerprints = _source_fingerprints(candidates)
- existing = next((s for s in candidates if fingerprints.get(s.pk) == plan.credential_fingerprint), None)
+ in_group = [
+ s
+ for s in candidates
+ if fingerprints.get(s.pk) == plan.credential_fingerprint and _access_signature(s) == signature
+ ]
+ existing = next((s for s in in_group if access.source_rules(str(s.pk)) == plan.access.rules), None)
if existing is not None:
return existing
+ if in_group:
+ # Someone changed the rules on the group's source, so a table that joins it now would resolve differently.
+ raise UnmappableTableError("the access rules on the source for this bucket changed, so the table can't join it")
source = SourceInputsStore.create(
prefix=_free_prefix(table.team_id, plan),
source_id=str(uuid.uuid4()),
@@ -691,15 +727,22 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter
"group_key": plan.group_key,
"credential_fingerprint": plan.credential_fingerprint,
"credential_id": str(table.credential_id),
+ **({"access_signature": signature} if signature else {}),
}
},
)
ExternalDataSourceDestination.objects.for_team(table.team_id).create(
team_id=table.team_id, source_id=source.pk, destination=get_or_create_warehouse_destination(table.team_id)
)
+ _write_access_rules(source, plan)
return source
+def _write_access_rules(source: ExternalDataSource, plan: _TablePlan) -> None:
+ if plan.access.rules:
+ write_source_rules(source.team_id, str(source.pk), plan.access)
+
+
def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:
"""The table's migration schema, created with its source when there is none yet.
@@ -709,7 +752,13 @@ def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataS
"""
with transaction.atomic():
if plan.shares_source:
- _lock_group(table.team_id, str(plan.source.source_type), plan.group_key, plan.credential_fingerprint)
+ _lock_group(
+ table.team_id,
+ str(plan.source.source_type),
+ plan.group_key,
+ plan.credential_fingerprint,
+ plan.access.signature,
+ )
else:
DataWarehouseTable.raw_objects.select_for_update().get(team_id=table.team_id, pk=table.pk)
schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])
@@ -718,7 +767,11 @@ def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataS
return live
if schemas:
raise UnmappableTableError("the table's migration was abandoned")
- source = _find_or_create_source(table, plan)
+ # The plan read the rules before the lock, so read them again for the rules the source gets.
+ access = TeamAccessRules(table.team_id)
+ if access.plan_for(str(table.id)) != plan.access:
+ raise UnmappableTableError("the access rules changed during the run")
+ source = _find_or_create_source(table, plan, access)
if (
ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk, name=table.name)
.exclude(deleted=True)
@@ -875,6 +928,8 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:
.exists()
):
source.soft_delete()
+ # The rules the migration gave the source have nothing left to protect.
+ remove_source_rules(source.team_id, str(source.pk))
removed_source_id = str(source.pk)
if evaluation.schema is not None:
# A schedule exists only when the user turned syncing on before, so it is paused now.
@@ -896,7 +951,8 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:
)
-_GroupId = tuple[int, str, str, str]
+# Team, source type, group key, credential fingerprint and access signature.
+_GroupId = tuple[int, str, str, str, str]
def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False) -> list[SourceMergeOutcome]:
@@ -904,8 +960,9 @@ def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False)
Earlier runs grouped tables by credential row, so tables with the same key in one bucket got a
source each. A duplicate moves only while all of its tables still wait for their first sync,
- because nothing has read through it then. The empty duplicate is then deleted. An uploaded
- table's source never merges.
+ because nothing has read through it then, and only when both sources have the same access rules,
+ so the moved tables keep their access. The empty duplicate and its rules are then deleted. An
+ uploaded table's source never merges.
"""
queryset = (
ExternalDataSource.objects.filter(connection_metadata__has_key=MIGRATION_MARKER_KEY)
@@ -922,7 +979,9 @@ def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False)
fingerprint = fingerprints.get(source.pk)
if not isinstance(group_key, str) or group_key.startswith("upload/") or fingerprint is None:
continue
- groups[(source.team_id, str(source.source_type), group_key, fingerprint)].append(source)
+ groups[(source.team_id, str(source.source_type), group_key, fingerprint, _access_signature(source))].append(
+ source
+ )
outcomes: list[SourceMergeOutcome] = []
for group, (kept, *duplicates) in groups.items():
@@ -954,12 +1013,10 @@ def _merge_refusal(
)
if adopted is not None:
return f"the table '{adopted}' is already adopted by a source"
- if object_ids_with_rules(team_id=kept.team_id, resource="external_data_source") & {
- str(kept.pk),
- str(duplicate.pk),
- }:
- # A moved schema would take the kept source's rules, so the duplicate's rules would stop applying.
- return "the kept or the duplicate source has its own access rules"
+ access = TeamAccessRules(kept.team_id)
+ if access.source_rules(str(kept.pk)) != access.source_rules(str(duplicate.pk)):
+ # A moved table would resolve through the kept source's rules, so the duplicate's rules would stop applying.
+ return "the kept and the duplicate source have different access rules"
if _destination_ids(kept) != _destination_ids(duplicate):
return "the kept and the duplicate source send data to different destinations"
if SourceInputsStore.load(kept) != SourceInputsStore.load(duplicate):
@@ -1032,6 +1089,7 @@ def _merge_source(
schema.save(update_fields=["source", "updated_at"])
ExternalDataSourceDestination.objects.for_team(duplicate.team_id).filter(source_id=duplicate.pk).delete()
duplicate.soft_delete()
+ remove_source_rules(duplicate.team_id, str(duplicate.pk))
except Exception as e:
# A failure on one source must not stop the batch or hide its outcomes.
logger.warning("Could not merge a duplicate migration source", source_id=str(duplicate.pk), exc_info=True)
@@ -1057,8 +1115,11 @@ def describe_merge(outcome: SourceMergeOutcome) -> str:
def describe_status(status: TableMigrationStatus) -> str:
reason = f" ({status.reason})" if status.reason else ""
columns = status.nested_text_column_types or {}
- note = ""
+ notes = []
if columns:
changed = ", ".join(f"{name} ({column_type})" for name, column_type in columns.items())
- note = f" note=(after the first sync these columns are stored as JSON text: {changed})"
+ notes.append(f"after the first sync these columns are stored as JSON text: {changed}")
+ if status.planned_access_rules:
+ notes.append(f"the source gets these access rules: {status.planned_access_rules}")
+ note = "".join(f" note=({note})" for note in notes)
return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}{note}"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index 7470fdf2252f..d939807c0a64 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -52,6 +52,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
update_should_sync,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERROR
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
EmitSignalsActivityInputs,
PersonPropertySyncActivityInputs,
@@ -229,6 +230,10 @@ Any_Source_Errors: dict[str, str | None] = {
# it here so every other source stops retrying too. The enriched message names the offending column
# and shows example cells, so keep the raw error rather than replacing it with a generic one.
"must be real number, not str": None,
+ # Raised before extraction, and again before the table changes, when a migration source must not
+ # adopt a self-managed table because the access rules changed. Only a person can fix the rules,
+ # and the message names the table and the next step, so keep it.
+ ADOPTION_ACCESS_ERROR: None,
# Raised by botocore (`is_valid_endpoint_url`) when the object storage endpoint URL has a
# hostname it rejects — most often an underscore, invalid in a DNS hostname but common in a
# self-hosted deployment's OBJECT_STORAGE_ENDPOINT (e.g. a docker service name). The endpoint is
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
index 400dee20c57c..d327235bbf60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_type
+from products.warehouse_sources.backend.self_managed_access import assert_adoption_keeps_access
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
HISTORY_NAMING,
LEGACY_HISTORY_NAMING,
@@ -523,6 +524,10 @@ async def validate_schema_and_update_table(
update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]
if table.external_data_source_id is None:
# A self-managed table this schema replaces reads its own bucket until now.
+ # Adoption makes the table resolve through the source's rules. Checked here, after
+ # the row count that can block for minutes, so a rule change has the least time
+ # to land between the check and the save.
+ assert_adoption_keeps_access(table, job.pipeline)
update_fields.extend(table.adopt_into_source(job.pipeline))
logger.info(f"Adopted self-managed table {table.id} into source {job.pipeline.id}")
if live_size_mib is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
index 0c52ff0e8951..0fb082e62842 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
ExternalDataSchema,
update_should_sync,
)
+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERROR
from products.warehouse_sources.backend.temporal.data_imports.metrics import (
LOCK_TAKEOVER_LATEST_ERROR,
TERMINAL_JOB_STATUSES,
@@ -173,6 +174,9 @@ DISABLE_SCHEMA_ERROR_PATTERNS: tuple[str, ...] = (
# incoming values no longer fit the stored Delta column type
# (SchemaColumnTypeChangedException) — only a reset and full re-sync can fix it
"Source column type changed",
+ # the access rules changed after the migration linked a self-managed table, so the first sync
+ # must not replace it — only an admin can fix the rules
+ ADOPTION_ACCESS_ERROR,
)
# The schema or job row was deleted mid-sync — no retry can bring it back, and no more of that
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index c1697a7dddd9..c55946998ee0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
SCHEMA_DELETED_JOB_ERROR,
SYNC_DISABLED_JOB_ERROR,
)
+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERROR
from products.warehouse_sources.backend.temporal.data_imports.metrics import LOCK_TAKEOVER_LATEST_ERROR
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
@@ -307,6 +308,7 @@ class TestProcessSingle:
"[Errno 5] An error occurred (XMinioStorageFull) when calling the CopyObject operation: "
"Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
str(DestinationConfigurationError("Prod", "The host name does not exist.")),
+ f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table or of its source changed",
],
)
@pytest.mark.asyncio
@@ -2137,6 +2139,9 @@ class TestFailRun:
),
pytest.param("Decimal value is too large to store in a Decimal128", True, id="decimal_overflow"),
pytest.param("Primary key required for incremental syncs", True, id="missing_primary_key"),
+ pytest.param(
+ f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules changed", True, id="adoption_refused"
+ ),
pytest.param(
"XMinioStorageFull: storage backend has reached its minimum free drive threshold",
False,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
index 0b58bd73de30..169ddea14878 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
@@ -14,6 +14,7 @@ from parameterized import parameterized
from posthog.models import Team
+from products.access_control.backend.facade.testing import create_access_control
from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import (
@@ -23,6 +24,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable
+from products.warehouse_sources.backend.self_managed_access import MIGRATION_MARKER_KEY, AdoptionAccessError
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_NAMING, LEGACY_HISTORY_NAMING
from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (
HistoryTableNameTakenError,
@@ -740,6 +742,40 @@ class TestValidateSchemaAndUpdateTable:
job.refresh_from_db()
assert job.storage_delta_mib == 3.0
+ @pytest.mark.parametrize("rule_added", [False, True])
+ def test_a_migration_source_adopts_the_table_only_while_its_access_rules_still_fit(
+ self, team: Team, rule_added: bool
+ ) -> None:
+ schema, job = self._schema_and_job(team)
+ ExternalDataSource.objects.filter(pk=job.pipeline_id).update(
+ connection_metadata={MIGRATION_MARKER_KEY: {"group_key": "s3/us-east-1/exports"}}
+ )
+ job.refresh_from_db()
+ table = self._self_managed_table(team, schema)
+ if rule_added:
+ # Added after the migration linked the table, so the source has no rule that matches it.
+ create_access_control(
+ team_id=team.pk, resource="warehouse_table", resource_id=str(table.id), access_level="none"
+ )
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ if rule_added:
+ with pytest.raises(AdoptionAccessError, match="The table still reads its own files"):
+ self._publish(team, schema, job)
+ else:
+ self._publish(team, schema, job)
+
+ table.refresh_from_db()
+ schema.refresh_from_db()
+ if rule_added:
+ assert (table.external_data_source_id, schema.table_id) == (None, None)
+ assert table.credential_id is not None
+ else:
+ assert (table.external_data_source_id, schema.table_id) == (job.pipeline_id, table.id)
+
@pytest.mark.parametrize(
"change",
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 87d6cda14e4e..876de3dd9176 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -45,10 +45,12 @@ from products.warehouse_sources.backend.models.external_data_schema import (
prefix_promotion_rows_synced,
process_incremental_value,
resolve_sync_type,
+ self_managed_table_ref,
staged_handoff_resume_point,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.self_managed_access import AdoptionAccessError, adoption_access_error
from products.warehouse_sources.backend.temporal.data_imports.destinations.enablement import (
external_destination_ids_for,
)
@@ -90,6 +92,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.rep
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.typings import PipelineResult
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync import (
PipelineInputs,
+ find_replaced_self_managed_table,
relink_orphan_table,
)
from products.warehouse_sources.backend.temporal.data_imports.retry_limits import PROGRESSLESS_RESUMABLE_ATTEMPTS
@@ -208,6 +211,21 @@ def _get_external_data_schema(schema_id: uuid.UUID, team_id: int) -> ExternalDat
)
+def self_managed_adoption_error(schema: ExternalDataSchema) -> str | None:
+ """The error that stops this run before it extracts anything, or None.
+
+ The first sync of a migration schema replaces a self-managed table. When the access rules changed
+ since the migration linked the table, the publish step would refuse to replace it, so the copy
+ would be wasted.
+ """
+ if schema.table_id is not None or self_managed_table_ref(schema) is None:
+ return None
+ table = find_replaced_self_managed_table(schema, schema.source)
+ if table is None or table.external_data_source_id is not None:
+ return None
+ return adoption_access_error(table, schema.source)
+
+
@database_sync_to_async_pool
def _has_completed_schema_job(schema_id: uuid.UUID, team_id: int) -> bool:
return ExternalDataJob.objects.filter(
@@ -643,6 +661,13 @@ async def _import_data_with_reporting(
except ExternalDataSchema.DoesNotExist as e:
await _handle_import_error(job_inputs, logger, e)
+ # Checked before anything is extracted, so a refused move costs no copy. The publish step
+ # checks again before it changes the table.
+ adoption_error = await database_sync_to_async_pool(self_managed_adoption_error)(schema)
+ if adoption_error is not None:
+ # The workflow reads the message from the cause, so chain the refusal.
+ raise NonRetryableException(adoption_error) from AdoptionAccessError(adoption_error)
+
reset_pipeline = _resolve_reset_pipeline(inputs, schema, job_created_at=model.created_at)
await logger.adebug(f"schema.sync_type_config = {schema.sync_type_config}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index 19390a000980..56eba6e34677 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -36,8 +36,11 @@ from products.warehouse_sources.backend.models.external_data_schema import (
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.self_managed_access import ADOPTION_ACCESS_ERROR
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import (
+ IMPORT_NON_RETRYABLE_ERROR_TYPES,
NEW_TABLE_NOT_READY_MESSAGE,
+ Any_Source_Errors,
_transient_error_message,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.preemption import PreemptionConfig
@@ -57,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
ResumableSource,
SimpleSource,
SourceExtractionNotImplementedError,
+ error_message_matches,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import TemporaryHostResolutionError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
@@ -523,6 +527,29 @@ async def test_schema_deleted_mid_sync_routes_through_handler():
source.parse_config.assert_not_called()
+async def test_a_refused_self_managed_adoption_stops_the_run_before_extraction():
+ message = f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table or of its source changed"
+ source = _make_source(AssertionError("must not extract"), {})
+
+ with (
+ _patched_activity(source) as handle_mock,
+ mock.patch.object(module, "self_managed_adoption_error", return_value=message),
+ ):
+ with pytest.raises(NonRetryableException, match="access rules of the table") as exc_info:
+ await import_data_activity_sync(_inputs())
+
+ source.parse_config.assert_not_called()
+ source.source_for_pipeline.assert_not_called()
+ handle_mock.assert_not_awaited()
+ # Temporal doesn't retry the activity, and the job's finalizer pauses the schema with the message.
+ assert type(exc_info.value).__name__ in IMPORT_NON_RETRYABLE_ERROR_TYPES
+ assert error_message_matches(str(exc_info.value), Any_Source_Errors.keys())
+ assert Any_Source_Errors[ADOPTION_ACCESS_ERROR] is None
+ # The workflow records `e.cause.cause` as the job's error, so the refusal must be the cause.
+ assert str(exc_info.value.__cause__) == message
+
+
@pytest.mark.asyncio
async def test_job_not_found_on_first_read_is_non_reportable():
# A run_id can point at a job that was never created, e.g. an older worker's
diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
index 5a45bccdc44e..436933a82545 100644
--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py
+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
@@ -17,9 +17,14 @@ from temporalio.service import RPCError, RPCStatusCode
from posthog.hogql.database.database import Database
-from posthog.models.organization import OrganizationMembership
+from posthog.constants import AvailableFeature
+from posthog.models.organization import Organization, OrganizationMembership
from posthog.models.team import Team
+from posthog.models.user import User
+from products.access_control.backend.facade.subject_access_control import SubjectAccessControl
+from products.access_control.backend.facade.testing import add_role_member, create_access_control, create_role
+from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.access_control.backend.models.access_control import AccessControl
from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStore
from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
@@ -33,6 +38,12 @@ from products.warehouse_sources.backend.models.external_data_schema import (
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.self_managed_access import (
+ ADOPTION_ACCESS_ERROR,
+ AdoptionAccessError,
+ TeamAccessRules,
+ assert_adoption_keeps_access,
+)
from products.warehouse_sources.backend.self_managed_migration import (
MIGRATION_MARKER_KEY,
MigrationOptions,
@@ -52,6 +63,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
parse_self_managed_url,
table_definition_for,
)
+from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.import_data_sync import (
+ self_managed_adoption_error,
+)
from products.warehouse_sources.backend.types import DataWarehouseTableFormat, ExternalDataJobStatus
MODULE = "products.warehouse_sources.backend.self_managed_migration"
@@ -447,64 +461,6 @@ class TestSelfManagedMigration(_MigrationTest):
assert "dwh-object-storage-sources" in (outcome.status.reason or "")
assert not ExternalDataSchema.objects.filter(team=self.team).exists()
- @parameterized.expand(
- [
- ("object_rule_restricts", True, "none", TableMigrationState.SKIPPED),
- ("object_rule_grants", True, "editor", TableMigrationState.SKIPPED),
- ("rule_on_another_table", False, "none", TableMigrationState.AWAITING_SYNC),
- ]
- )
- def test_a_table_with_its_own_access_rules_is_skipped(
- self, _name: str, on_this_table: bool, access_level: str, expected: TableMigrationState
- ) -> None:
- table = self._table()
- other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")
- AccessControl.objects.create(
- team=self.team,
- resource="warehouse_table",
- resource_id=str(table.id if on_this_table else other.id),
- access_level=access_level,
- organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),
- )
-
- outcome = next(o for o in self._migrate() if o.status.table_id == str(table.id))
-
- assert outcome.status.state == expected
- if expected == TableMigrationState.SKIPPED:
- assert "own access rules" in (outcome.status.reason or "")
- assert _schemas_for(table) == []
-
- @parameterized.expand(
- [
- ("member_rule_on_sources", "external_data_source", True),
- ("default_rule_on_sources", "external_data_source", False),
- ("member_rule_on_the_warehouse", "warehouse_objects", True),
- ("default_rule_on_the_warehouse", "warehouse_objects", False),
- ]
- )
- def test_a_project_with_access_rules_for_sources_is_skipped(
- self, _name: str, resource: str, for_member: bool
- ) -> None:
- table = self._table()
- AccessControl.objects.create(
- team=self.team,
- resource=resource,
- resource_id=None,
- access_level="none",
- organization_member=(
- OrganizationMembership.objects.get(user=self.user, organization=self.organization)
- if for_member
- else None
- ),
- )
-
- [outcome] = self._migrate()
-
- assert outcome.status.table_id == str(table.id)
- assert outcome.status.state == TableMigrationState.SKIPPED
- assert "access rules for sources" in (outcome.status.reason or "")
- assert _schemas_for(table) == []
-
def test_status_reports_every_state(self) -> None:
pending = self._table("pending_table")
self._table("skipped_table", url="https://minio.example.com/exports/a.csv")
@@ -793,7 +749,7 @@ class TestMergeDuplicateSources(_MigrationTest):
("job_ran", "a sync already ran for the source"),
("table_adopted", "the table 'table_1' is already adopted by a source"),
("name_taken_in_kept_source", "the kept source already has a table named 'table_1'"),
- ("source_has_rules", "the kept or the duplicate source has its own access rules"),
+ ("source_has_rules", "the kept and the duplicate source have different access rules"),
("other_destinations", "the kept and the duplicate source send data to different destinations"),
("inputs_edited", "the kept and the duplicate source connect with different settings"),
]
@@ -1111,3 +1067,576 @@ class TestUploadedTableMigration(_MigrationTest):
# Inside the team folder but not one upload, so it is listed and never planned.
"nested_key": TableMigrationState.SKIPPED,
}
+
+
+_LEVELS = ("none", "viewer", "editor", "manager")
+_ROLE_NAMES = ("A", "B")
+# Each member stands for one way a member can relate to the rules.
+_MEMBERS: dict[str, tuple[tuple[str, ...], OrganizationMembership.Level]] = {
+ "plain": ((), OrganizationMembership.Level.MEMBER),
+ "named": ((), OrganizationMembership.Level.MEMBER),
+ "in_role": (("A",), OrganizationMembership.Level.MEMBER),
+ "named_in_roles": (("A", "B"), OrganizationMembership.Level.MEMBER),
+ "in_both_roles": (("A", "B"), OrganizationMembership.Level.MEMBER),
+ "admin": ((), OrganizationMembership.Level.ADMIN),
+ "project_admin": ((), OrganizationMembership.Level.MEMBER),
+ "creator": ((), OrganizationMembership.Level.MEMBER),
+}
+# Where a rule can be: "<scope>" is the scope's default rule, "<scope>:<member or role>" names a subject.
+_SCOPES = {
+ "table": ("warehouse_table", True),
+ "warehouse": ("warehouse_objects", False),
+ "sources": ("external_data_source", False),
+ # Resolution reads the rules for all warehouse tables from warehouse_objects, so this rule never applies.
+ "all_tables": ("warehouse_table", False),
+}
+
+
+class _AccessRulesTest(_MigrationTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.organization.available_product_features = [
+ {"key": feature, "name": feature}
+ for feature in (AvailableFeature.ACCESS_CONTROL, AvailableFeature.ROLE_BASED_ACCESS)
+ ]
+ self.organization.save()
+ self.roles = {
+ name: create_role(organization_id=self.organization.id, name=f"Role {name}") for name in _ROLE_NAMES
+ }
+ self.users: dict[str, User] = {}
+ self.memberships: dict[str, OrganizationMembership] = {}
+ for name, (roles, level) in _MEMBERS.items():
+ user = User.objects.create_and_join(self.organization, f"{name}@example.com", None, level=level)
+ self.users[name] = user
+ self.memberships[name] = OrganizationMembership.objects.get(user=user, organization=self.organization)
+ for role in roles:
+ add_role_member(role_id=self.roles[role], user_id=user.id)
+ create_access_control(
+ team_id=self.team.pk,
+ resource="project",
+ resource_id=str(self.team.pk),
+ access_level="admin",
+ organization_member_id=self.memberships["project_admin"].id,
+ )
+
+ def _rule(self, slot: str, level: str, table: DataWarehouseTable | None = None) -> None:
+ scope, _, subject = slot.partition(":")
+ resource, on_table = _SCOPES[scope]
+ create_access_control(
+ team_id=self.team.pk,
+ resource=resource,
+ resource_id=str(table.id) if on_table and table is not None else None,
+ access_level=level,
+ organization_member_id=self.memberships[subject].id if subject in self.memberships else None,
+ role_id=self.roles[subject] if subject in self.roles else None,
+ )
+
+ def _apply(self, rules: dict[str, str], table: DataWarehouseTable) -> None:
+ for slot, level in rules.items():
+ self._rule(slot, level, table)
+
+ def _access(self, obj: Any) -> dict[str, str | None]:
+ """Each subject's access to the object, from the access checks that serve requests."""
+ result: dict[str, str | None] = {
+ name: UserAccessControl(user, self.team).get_user_access_level(obj) for name, user in self.users.items()
+ }
+ asker = UserAccessControl(self.users["admin"], self.team)
+ for name, role_id in self.roles.items():
+ result[f"role {name}"] = SubjectAccessControl.for_role(
+ asker, self.team, str(role_id)
+ ).get_user_access_level(obj)
+ result["everyone"] = SubjectAccessControl.for_default(asker, self.team).get_user_access_level(obj)
+ return result
+
+ def _in_both_orders(self, obj: Any) -> dict[bool, dict[str, str | None]]:
+ by_order = {}
+ for most_specific in (False, True):
+ self.organization.uses_most_specific_access_resolution = most_specific
+ self.organization.save()
+ by_order[most_specific] = self._access(obj)
+ return by_order
+
+ def _source_rules(self, source: ExternalDataSource) -> set[tuple[str, str]]:
+ names = {str(m.id): name for name, m in self.memberships.items()} | {str(r): n for n, r in self.roles.items()}
+ return {
+ (names.get(str(rule.organization_member_id or rule.role_id or ""), ""), rule.access_level)
+ for rule in AccessControl.objects.filter(
+ team=self.team, resource="external_data_source", resource_id=str(source.pk)
+ )
+ }
+
+ def _assert_move_keeps_access(self, table: DataWarehouseTable) -> ExternalDataSource:
+ """Adopt the linked table, and check every subject's access to it and to its source under both orders."""
+ before = self._in_both_orders(table)
+ source = _schema_for(table).source
+ self._adopt(table)
+ table.refresh_from_db()
+ assert table.external_data_source_id == source.pk
+ after = self._in_both_orders(table)
+ on_source = self._in_both_orders(source)
+ assert after == before
+ for most_specific, levels in on_source.items():
+ for subject, level in levels.items():
+ table_level = after[most_specific][subject]
+ assert level is not None and table_level is not None
+ assert _LEVELS.index(level) <= _LEVELS.index(table_level), (most_specific, subject, level, table_level)
+ return source
+
+
+class TestAccessRulesMoveWithTheTable(_AccessRulesTest):
+ @parameterized.expand(
+ [
+ ("no_rules", {}, set()),
+ ("table_private", {"table": "none"}, {("", "none")}),
+ ("table_read_only", {"table": "viewer"}, {("", "viewer")}),
+ ("table_hidden_from_a_member", {"table:named": "none"}, {("named", "none")}),
+ (
+ "table_private_but_shared_with_a_member",
+ {"table": "none", "table:named": "editor"},
+ {("named", "editor"), ("", "none")},
+ ),
+ ("table_rule_for_a_role", {"table:A": "viewer"}, {("A", "viewer")}),
+ (
+ "member_rule_and_role_rule_for_one_member",
+ {"table:named_in_roles": "viewer", "table:A": "editor", "table:B": "none"},
+ {("named_in_roles", "viewer"), ("A", "editor"), ("B", "none")},
+ ),
+ ("warehouse_read_only", {"warehouse": "viewer"}, {("", "viewer")}),
+ ("warehouse_hidden", {"warehouse": "none"}, {("", "none")}),
+ ("warehouse_managers", {"warehouse": "manager"}, set()),
+ # The table gives more than the default for sources, so the source must grant it.
+ ("sources_read_only", {"sources": "viewer"}, {("", "editor")}),
+ # The table gives less than the default for sources, so the source must be restricted.
+ ("private_table_and_sources_open", {"table": "none", "sources": "editor"}, {("", "none")}),
+ ("private_table_and_warehouse_read_only", {"table": "none", "warehouse": "viewer"}, {("", "none")}),
+ ("rule_for_all_tables_never_applies", {"all_tables": "none"}, set()),
+ ("rule_for_the_project_admin", {"table:project_admin": "none"}, {("project_admin", "none")}),
+ ("rule_for_the_creator", {"table:creator": "none"}, {("creator", "none")}),
+ ]
+ )
+ def test_a_migrated_table_keeps_every_subjects_access(
+ self, _name: str, rules: dict[str, str], source_rules: set[tuple[str, str]]
+ ) -> None:
+ table = self._table(created_by=self.users["creator"])
+ self._apply(rules, table)
+ before = self._in_both_orders(table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.action == "linked", outcome.error or outcome.status.reason
+ assert self._source_rules(_schema_for(table).source) == source_rules
+ # Until the first sync the table has no source, so the source's rules don't reach it.
+ assert self._in_both_orders(table) == before
+ self._assert_move_keeps_access(table)
+
+ @parameterized.expand(
+ [
+ # A grant on a source can't widen "none" for all sources under most-specific resolution.
+ ("sources_hidden", {"sources": "none"}, "access rules for sources"),
+ ("sources_hidden_from_a_member", {"sources:named": "none"}, "access rules for sources"),
+ ("table_read_only_and_sources_hidden", {"table": "viewer", "sources": "none"}, "access rules for sources"),
+ # Everyone could manage the source, and through its rules change the table's access.
+ ("sources_managed_by_everyone", {"sources": "manager"}, "access rules for sources"),
+ # Members of one role see more of the warehouse than the others, and one default can't say that.
+ ("warehouse_open_to_a_role", {"warehouse": "viewer", "warehouse:A": "editor"}, "access rules for sources"),
+ ("warehouse_hidden_from_a_member", {"warehouse:named": "none"}, "access rules for sources"),
+ (
+ "table_rule_and_sources_hidden_from_a_role",
+ {"table:named": "viewer", "sources:A": "none"},
+ "access rules for sources",
+ ),
+ ]
+ )
+ def test_a_table_whose_access_no_source_rules_can_keep_stays_self_managed(
+ self, _name: str, rules: dict[str, str], reason: str
+ ) -> None:
+ table = self._table()
+ self._apply(rules, table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert reason in (outcome.status.reason or "")
+ assert _schemas_for(table) == []
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+
+ @parameterized.expand(
+ [
+ (f"table_{table}_warehouse_{warehouse}_sources_{sources}", table, warehouse, sources)
+ for table in (None, *_LEVELS[:3])
+ for warehouse in (None, *_LEVELS[:3])
+ for sources in (None, *_LEVELS[:3])
+ ]
+ )
+ def test_every_move_of_a_table_with_explicit_rules_keeps_access(
+ self, _name: str, table_default: str | None, warehouse: str | None, sources: str | None
+ ) -> None:
+ table = self._table(created_by=self.users["creator"])
+ rules = {"table:named": "viewer", "table:B": "none"}
+ for slot, level in (("table", table_default), ("warehouse", warehouse), ("sources", sources)):
+ if level is not None:
+ rules[slot] = level
+ self._apply(rules, table)
+
+ [outcome] = self._migrate()
+
+ if outcome.action != "linked":
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert outcome.status.reason in (
+ "the table has its own access rules, which the source PostHog would create can't carry",
+ "the project has access rules for sources or for the warehouse, which the move would change",
+ )
+ return
+ self._assert_move_keeps_access(table)
+
+ def test_tables_of_one_bucket_share_a_source_only_when_they_need_the_same_rules(self) -> None:
+ open_table = self._table("open", url="https://exports.s3.eu-west-1.amazonaws.com/open/*.csv")
+ private = self._table("private", url="https://exports.s3.eu-west-1.amazonaws.com/private/*.csv")
+ also_private = self._table("also_private", url="https://exports.s3.eu-west-1.amazonaws.com/also/*.csv")
+ read_only = self._table("read_only", url="https://exports.s3.eu-west-1.amazonaws.com/read/*.csv")
+ self._rule("table", "none", private)
+ self._rule("table", "none", also_private)
+ self._rule("table", "viewer", read_only)
+
+ self._migrate()
+
+ sources = {table.name: _schema_for(table).source for table in (open_table, private, also_private, read_only)}
+ assert sources["private"].pk == sources["also_private"].pk
+ assert len({source.pk for source in sources.values()}) == 3
+ assert self._source_rules(sources["open"]) == set()
+ assert self._source_rules(sources["private"]) == {("", "none")}
+ assert self._source_rules(sources["read_only"]) == {("", "viewer")}
+ assert "access_signature" not in (sources["open"].connection_metadata or {})[MIGRATION_MARKER_KEY]
+ for table in (private, also_private, read_only):
+ self._assert_move_keeps_access(table)
+
+ def test_rules_that_never_apply_are_not_copied(self) -> None:
+ other_organization = Organization.objects.create(name="Elsewhere")
+ foreign_role = create_role(organization_id=other_organization.id, name="Foreign")
+ table = self._table()
+ create_access_control(
+ team_id=self.team.pk,
+ resource="warehouse_table",
+ resource_id=str(table.id),
+ access_level="none",
+ role_id=foreign_role,
+ )
+ before = self._in_both_orders(table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.action == "linked"
+ assert self._source_rules(_schema_for(table).source) == set()
+ assert self._in_both_orders(table) == before
+ self._assert_move_keeps_access(table)
+
+ def test_a_rule_for_an_inactive_member_is_copied(self) -> None:
+ User.objects.filter(pk=self.users["named"].pk).update(is_active=False)
+ table = self._table()
+ self._rule("table:named", "manager", table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.action == "linked"
+ assert self._source_rules(_schema_for(table).source) == {("named", "manager")}
+
+ def test_rules_of_an_organization_without_access_control_are_copied_for_when_it_comes_back(self) -> None:
+ table = self._table()
+ self._apply({"table": "none", "table:named": "viewer"}, table)
+ with_access_control = self._in_both_orders(table)
+ self.organization.available_product_features = []
+ self.organization.save()
+
+ [outcome] = self._migrate()
+
+ assert outcome.action == "linked"
+ assert self._source_rules(_schema_for(table).source) == {("named", "viewer"), ("", "none")}
+ self._assert_move_keeps_access(table)
+ self.organization.available_product_features = [
+ {"key": feature, "name": feature}
+ for feature in (AvailableFeature.ACCESS_CONTROL, AvailableFeature.ROLE_BASED_ACCESS)
+ ]
+ self.organization.save()
+ assert self._in_both_orders(table) == with_access_control
+
+ @parameterized.expand(
+ [
+ ("unknown_level", {"table": "superuser"}, "the level 'superuser'"),
+ ]
+ )
+ def test_rules_the_planner_cant_read_keep_the_table_self_managed(
+ self, _name: str, rules: dict[str, str], reason: str
+ ) -> None:
+ table = self._table()
+ self._apply(rules, table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert reason in (outcome.status.reason or "")
+
+ def test_rules_that_name_too_many_roles_keep_the_table_self_managed(self) -> None:
+ table = self._table()
+ for i in range(11):
+ role = create_role(organization_id=self.organization.id, name=f"Extra {i}")
+ create_access_control(
+ team_id=self.team.pk,
+ resource="warehouse_table",
+ resource_id=str(table.id),
+ access_level=_LEVELS[i % 4],
+ role_id=role,
+ )
+ create_access_control(
+ team_id=self.team.pk, resource="warehouse_objects", access_level=_LEVELS[i // 4], role_id=role
+ )
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert outcome.status.reason == "the access rules name too many roles to check"
+
+ def test_status_and_dry_run_report_the_rules_the_source_gets_and_write_nothing(self) -> None:
+ table = self._table()
+ self._apply({"table": "none", "table:named": "editor"}, table)
+ member_id = self.memberships["named"].id
+ note = f"note=(the source gets these access rules: member {member_id}=editor, everyone=none)"
+
+ for options in ({"status": True}, {}):
+ out = StringIO()
+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=out, **options)
+ assert note in out.getvalue()
+
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+ assert not AccessControl.objects.filter(resource="external_data_source").exists()
+
+ def test_an_uploaded_table_gets_the_same_rules_on_its_own_source(self) -> None:
+ with override_settings(DATAWAREHOUSE_BUCKET=UPLOAD_BUCKET, DATAWAREHOUSE_BUCKET_DOMAIN=UPLOAD_DOMAIN):
+ table = self._table(
+ "uploaded",
+ url=f"https://{UPLOAD_DOMAIN}/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",
+ created_by=self.users["creator"],
+ )
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(
+ credential=None, options={"csv_allow_double_quotes": False}
+ )
+ table.refresh_from_db()
+ self._apply({"table": "none", "table:A": "viewer"}, table)
+
+ [outcome] = self._migrate()
+
+ assert outcome.action == "linked", outcome.error or outcome.status.reason
+ source = _schema_for(table).source
+ assert source.source_type == "FileUpload"
+ assert self._source_rules(source) == {("A", "viewer"), ("", "none")}
+ assert (source.connection_metadata or {})[MIGRATION_MARKER_KEY]["access_signature"]
+ self._assert_move_keeps_access(table)
+
+
+class TestAccessRulesAfterLinking(_AccessRulesTest):
+ def _linked(self, rules: dict[str, str], name: str = "orders") -> DataWarehouseTable:
+ table = self._table(name, url=f"https://exports.s3.eu-west-1.amazonaws.com/{name}/*.csv")
+ self._apply(rules, table)
+ [outcome] = self._migrate(table_ids=(str(table.id),))
+ assert outcome.action == "linked"
+ return table
+
+ @parameterized.expand(
+ [
+ ("no_change", None, None),
+ ("table_rule_added", "table:named", "the access rules of the table or of its source changed"),
+ ("table_rule_widened", "table", "the access rules of the table or of its source changed"),
+ ("source_rule_edited", "source", "the access rules of the table or of its source changed"),
+ ("sources_hidden", "sources", "now the project has access rules for sources"),
+ ]
+ )
+ def test_the_first_sync_refuses_to_adopt_a_table_whose_rules_changed(
+ self, _name: str, change: str | None, refusal: str | None
+ ) -> None:
+ table = self._linked({"table": "viewer"})
+ source = _schema_for(table).source
+ if change == "table:named":
+ self._rule("table:named", "viewer", table)
+ elif change == "table":
+ AccessControl.objects.filter(resource="warehouse_table", resource_id=str(table.id)).update(
+ access_level="editor"
+ )
+ elif change == "source":
+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(source.pk)).update(
+ access_level="none"
+ )
+ elif change == "sources":
+ self._rule("sources", "none")
+
+ status = self._status(table)
+ if refusal is None:
+ assert_adoption_keeps_access(table, source)
+ assert status.reason == "syncing is off"
+ return
+ with self.assertRaisesRegex(AdoptionAccessError, refusal):
+ assert_adoption_keeps_access(table, source)
+ assert status.state == TableMigrationState.AWAITING_SYNC
+ assert refusal in (status.reason or "")
+ assert "the first sync will fail and leave the table as it is" in (status.reason or "")
+
+ @parameterized.expand([("unchanged", False, False), ("rule_added", True, False), ("already_adopted", True, True)])
+ def test_the_import_stops_before_extraction_only_when_the_rules_no_longer_fit(
+ self, _name: str, rule_added: bool, adopted: bool
+ ) -> None:
+ table = self._linked({"table": "viewer"})
+ if rule_added:
+ self._rule("table:named", "viewer", table)
+ if adopted:
+ self._adopt(table)
+
+ error = self_managed_adoption_error(_schema_for(table) if not adopted else self._unlinked_schema(table))
+
+ if not rule_added or adopted:
+ assert error is None
+ return
+ assert error is not None
+ assert error.startswith(f"{ADOPTION_ACCESS_ERROR} 'orders', because the access rules of the table")
+ assert "an organization admin must make the access rules of this source the same" in error
+
+ def _unlinked_schema(self, table: DataWarehouseTable) -> ExternalDataSchema:
+ schema = _schema_for(table)
+ ExternalDataSchema.objects.filter(pk=schema.pk).update(table=None)
+ schema.refresh_from_db()
+ return schema
+
+ def test_a_source_made_before_access_signatures_takes_only_tables_without_rules(self) -> None:
+ old = self._linked({})
+ source = _schema_for(old).source
+ assert MIGRATION_MARKER_KEY in (source.connection_metadata or {})
+ assert "access_signature" not in (source.connection_metadata or {})[MIGRATION_MARKER_KEY]
+ plain = self._linked({}, name="plain")
+ private = self._linked({"table": "none"}, name="private")
+
+ assert _schema_for(plain).source_id == source.pk
+ assert _schema_for(private).source_id != source.pk
+ assert self._source_rules(source) == set()
+
+ def test_a_table_does_not_join_a_source_whose_rules_someone_changed(self) -> None:
+ first = self._linked({"table": "none"}, name="first")
+ source = _schema_for(first).source
+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(source.pk)).update(
+ access_level="viewer"
+ )
+ second = self._table("second", url="https://exports.s3.eu-west-1.amazonaws.com/second/*.csv")
+ self._rule("table", "none", second)
+
+ [outcome] = self._migrate(table_ids=(str(second.id),))
+
+ assert outcome.action == "link_failed"
+ assert "the access rules on the source for this bucket changed" in (outcome.error or "")
+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 1
+
+ @parameterized.expand([("last_table", False), ("other_tables_stay", True)])
+ def test_abandon_removes_the_rules_only_with_the_source(self, _name: str, keep_another: bool) -> None:
+ table = self._linked({"table": "none"})
+ source = _schema_for(table).source
+ if keep_another:
+ other = self._linked({"table": "none"}, name="other")
+ assert _schema_for(other).source_id == source.pk
+
+ assert self._abandon(table).action == "abandoned"
+
+ source.refresh_from_db()
+ assert source.deleted is not keep_another
+ assert self._source_rules(source) == (({("", "none")}) if keep_another else set())
+
+ def test_abandon_of_a_table_never_linked_leaves_no_rules(self) -> None:
+ table = self._table()
+ self._rule("table", "none", table)
+
+ assert self._abandon(table).action == "abandoned"
+
+ assert not AccessControl.objects.filter(resource="external_data_source").exists()
+
+ def test_a_rule_written_while_the_run_plans_fails_the_link(self) -> None:
+ table = self._table()
+ planned_reads: list[TeamAccessRules] = []
+
+ def rules_read_then_changed(team_id: int) -> TeamAccessRules:
+ rules = TeamAccessRules(team_id)
+ if not planned_reads:
+ self._rule("table", "none", table)
+ planned_reads.append(rules)
+ return rules
+
+ with patch(f"{MODULE}.TeamAccessRules", side_effect=rules_read_then_changed):
+ [outcome] = self._migrate()
+
+ assert (outcome.action, outcome.error) == ("link_failed", "the access rules changed during the run")
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+
+
+class TestMergeKeepsAccessRules(_AccessRulesTest):
+ def _duplicate_pair(self, rules: dict[str, str]) -> tuple[DataWarehouseTable, DataWarehouseTable]:
+ """Two sources of one group with the same rules, as earlier runs could leave them."""
+ tables = []
+ for i in range(2):
+ table = self._table(
+ f"table_{i}",
+ url=f"https://exports.s3.eu-west-1.amazonaws.com/t{i}/*.csv",
+ credential=self._credential(f"AKIATEMP{i}"),
+ )
+ self._apply(rules, table)
+ tables.append(table)
+ self._migrate()
+ for i, table in enumerate(tables):
+ credential = table.credential
+ assert credential is not None
+ credential.access_key = "AKIAFIRST"
+ credential.save()
+ source = _schema_for(table).source
+ inputs = SourceInputsStore.load(source)
+ SourceInputsStore.save(
+ source, {key: "AKIAFIRST" if value == f"AKIATEMP{i}" else value for key, value in inputs.items()}
+ )
+ signature = (source.connection_metadata or {})[MIGRATION_MARKER_KEY].get("access_signature")
+ ExternalDataSource.objects.filter(pk=source.pk).update(
+ created_at=datetime(2024, 1, 1 + i, tzinfo=UTC),
+ connection_metadata={
+ MIGRATION_MARKER_KEY: {
+ "group_key": "s3/eu-west-1/exports",
+ "credential_id": str(credential.pk),
+ **({"access_signature": signature} if signature else {}),
+ }
+ },
+ )
+ assert _schema_for(tables[0]).source_id != _schema_for(tables[1]).source_id
+ return tables[0], tables[1]
+
+ def test_sources_with_the_same_rules_merge_and_the_duplicates_rules_go(self) -> None:
+ first, second = self._duplicate_pair({"table": "none"})
+ kept, duplicate = _schema_for(first).source, _schema_for(second).source
+
+ [outcome] = merge_duplicate_sources(team_id=self.team.pk, live_run=True)
+
+ assert (outcome.action, outcome.source_id) == ("merged", str(duplicate.pk))
+ assert _schema_for(second).source_id == kept.pk
+ assert self._source_rules(kept) == {("", "none")}
+ assert self._source_rules(duplicate) == set()
+ self._assert_move_keeps_access(second)
+
+ @parameterized.expand([("rules_differ", "the kept and the duplicate source have different access rules")])
+ def test_sources_whose_rules_differ_never_merge(self, _name: str, reason: str) -> None:
+ first, second = self._duplicate_pair({"table": "none"})
+ duplicate = _schema_for(second).source
+ AccessControl.objects.filter(resource="external_data_source", resource_id=str(duplicate.pk)).update(
+ access_level="viewer"
+ )
+
+ [outcome] = merge_duplicate_sources(team_id=self.team.pk, live_run=True)
+
+ assert (outcome.action, outcome.reason) == ("skipped", reason)
+ assert _schema_for(second).source_id == duplicate.pk
+
+ def test_sources_with_other_access_signatures_are_never_grouped(self) -> None:
+ open_table = self._table("open", url="https://exports.s3.eu-west-1.amazonaws.com/open/*.csv")
+ private = self._table("private", url="https://exports.s3.eu-west-1.amazonaws.com/private/*.csv")
+ self._rule("table", "none", private)
+ self._migrate()
+ assert _schema_for(open_table).source_id != _schema_for(private).source_id
+
+ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []