:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): group migrated self-managed tables by credential value (#128399)
Tom Owersd5aaa0bcaf71
diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.pyindex 58094d04c201..fe7193012757 100644--- a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py@@ -7,8 +7,10 @@ from products.warehouse_sources.backend.self_managed_migration import (DEFAULT_ABANDON_REASON,MigrationOptions,abandon_tables,+ describe_merge,describe_status,evaluate_tables,+ merge_duplicate_sources,migrate_self_managed_tables,)@@ -53,6 +55,12 @@ class Command(BaseCommand):default=DEFAULT_ABANDON_REASON,help="The reason recorded for each abandoned table.",)+ parser.add_argument(+ "--merge-sources",+ action="store_true",+ help="Move the tables of extra sources for one bucket and key into the oldest source, and delete "+ "the extra sources. A source with a table that started to sync stays as it is.",+ )parser.add_argument("--live-run", action="store_true", help="Act. Without this the command only reports what it would do.")@@ -60,8 +68,17 @@ class Command(BaseCommand):def handle(self, *args: Any, **options: Any) -> None:if options["limit"] is not None and options["limit"] < 1:raise CommandError("--limit must be at least 1.")- if options["status"] and options["abandon"] is not None:- raise CommandError("Pass only one of --status and --abandon.")+ modes = [+ flag+ for flag, chosen in (+ ("--status", options["status"]),+ ("--abandon", options["abandon"] is not None),+ ("--merge-sources", options["merge_sources"]),+ )+ if chosen+ ]+ if len(modes) > 1:+ raise CommandError(f"Pass only one of {', '.join(modes)}.")team_id: int | None = options["team_id"]table_ids = _split_ids(options["table_ids"])@@ -71,6 +88,8 @@ class Command(BaseCommand):self._report_status(team_id, table_ids)elif options["abandon"] is not None:self._abandon(_split_ids(options["abandon"]), options["abandon_reason"], team_id, live_run)+ elif options["merge_sources"]:+ self._merge_sources(team_id, live_run)else:self._migrate(team_id, table_ids, live_run, options["limit"])@@ -90,6 +109,19 @@ class Command(BaseCommand):if not live_run:self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to abandon these tables."))+ def _merge_sources(self, team_id: int | None, live_run: bool) -> None:+ outcomes = merge_duplicate_sources(team_id=team_id, live_run=live_run)+ for outcome in outcomes:+ self.stdout.write(describe_merge(outcome))+ actions = Counter(outcome.action for outcome in outcomes)+ moved = sum(outcome.moved_schemas for outcome in outcomes)+ self.stdout.write(+ f"\n{len(outcomes)} duplicate source(s), {moved} schema(s): "+ + ", ".join(f"{n} {a}" for a, n in sorted(actions.items()))+ )+ if not live_run:+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to merge these sources."))+def _migrate(self, team_id: int | None, table_ids: list[str], live_run: bool, limit: int | None) -> None:outcomes = migrate_self_managed_tables(MigrationOptions(team_id=team_id, table_ids=tuple(table_ids), live_run=live_run, limit=limit)diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.pyindex 0c69bbefbc54..afa9e25739f3 100644--- a/products/warehouse_sources/backend/self_managed_migration.py+++ b/products/warehouse_sources/backend/self_managed_migration.py@@ -21,15 +21,17 @@ continues where it stopped."""import re+import hmacimport uuid+import hashlibimport dataclassesfrom collections import defaultdict-from collections.abc import Callable, Iterable, Mapping+from collections.abc import Callable, Iterable, Mapping, Sequencefrom enum import StrEnumfrom typing import Any, Literal, castfrom django.conf import settings-from django.db import transaction+from django.db import connection, transactionfrom django.db.models import CharField, Q, Valuefrom django.db.models.functions import Cast, Concat@@ -128,6 +130,7 @@ class TableMigrationState(StrEnum):MigrationAction = Literal["none", "would_link", "linked", "link_failed", "limit_reached"]AbandonAction = Literal["would_abandon", "abandoned", "refused"]+MergeAction = Literal["would_merge", "merged", "skipped"]@frozen@@ -156,6 +159,16 @@ class AbandonOutcome:reason: str | None = None+@frozen+class SourceMergeOutcome:+ source_id: str+ team_id: int+ kept_source_id: str+ action: MergeAction+ moved_schemas: int = 0+ reason: str | None = None++@frozenclass MigrationOptions:team_id: int | None = None@@ -172,13 +185,14 @@ class _TablePlan:source: ObjectStorageSource[Any] | FileUploadSourceschema_metadata: dict[str, Any]inputs: dict[str, Any] = dataclasses.field(repr=False)- # Tables with the same group key and credential share one source.+ # Tables with the same group key and credential fingerprint share one source.group_key: strdescription: str# An uploaded table has no credential, and gets a source of its own.shares_source: bool = True# The bucket a shared source reads, which names its table prefix when one is needed.bucket: str = ""+ credential_fingerprint: str = dataclasses.field(default="", repr=False)@frozen@@ -234,9 +248,71 @@ def _credential_of(table: DataWarehouseTable) -> SelfManagedCredential | None:credential = table.credentialif credential is None:return None+ return _self_managed_credential(credential)+++def _self_managed_credential(credential: DataWarehouseCredential) -> SelfManagedCredential:return SelfManagedCredential(access_key=credential.access_key or "", access_secret=credential.access_secret or "")+def _credential_fingerprint(credential: SelfManagedCredential) -> str:+ """Identifies a key pair by its value, so tables whose credential rows hold the same key share one source.++ The app saves a new credential row for each table, also when the key is the same, so the row id+ can't group them. An HMAC keeps the key pair itself out of the stored marker.+ """+ message = f"{credential.access_key.strip()}\n{credential.access_secret.strip()}".encode()+ return hmac.new(settings.SECRET_KEY.encode(), message, hashlib.sha256).hexdigest()+++def _marker(source: ExternalDataSource) -> dict[str, Any]:+ marker = (source.connection_metadata or {}).get(MIGRATION_MARKER_KEY)+ return marker if isinstance(marker, dict) else {}+++def _source_fingerprints(sources: Sequence[ExternalDataSource]) -> dict[uuid.UUID, str]:+ """The credential fingerprint of each migration source.++ A source made before the marker held a fingerprint names its credential row instead. When that+ row is gone, the source has no fingerprint and no table joins it.+ """+ fingerprints: dict[uuid.UUID, str] = {}+ legacy: dict[uuid.UUID, tuple[int, uuid.UUID]] = {}+ for source in sources:+ marker = _marker(source)+ fingerprint = marker.get("credential_fingerprint")+ if isinstance(fingerprint, str) and fingerprint:+ fingerprints[source.pk] = fingerprint+ continue+ try:+ legacy[source.pk] = (source.team_id, uuid.UUID(str(marker.get("credential_id"))))+ except ValueError:+ continue+ credential_ids_by_team: dict[int, set[uuid.UUID]] = defaultdict(set)+ for team_id, credential_id in legacy.values():+ credential_ids_by_team[team_id].add(credential_id)+ credentials: dict[tuple[int, uuid.UUID], DataWarehouseCredential] = {}+ for team_id, credential_ids in credential_ids_by_team.items():+ for credential in DataWarehouseCredential.objects.filter(team_id=team_id, pk__in=credential_ids):+ credentials[(team_id, credential.pk)] = credential+ for source_id, key in legacy.items():+ if (found := credentials.get(key)) is not None:+ fingerprints[source_id] = _credential_fingerprint(_self_managed_credential(found))+ return fingerprints+++def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str) -> None:+ """Hold the group's lock until the transaction ends.++ Every table of one source shares the lock, so two runs can't create two sources for the group,+ and a merge can't move schemas while a run adds to the group.+ """+ name = "\n".join((MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint))+ lock_id = int.from_bytes(hashlib.blake2b(name.encode(), digest_size=8).digest(), "big", signed=True)+ with connection.cursor() as cursor:+ cursor.execute("SELECT pg_advisory_xact_lock(%s)", [lock_id])++class _Planner:"""Maps tables onto sources, caching the per-team flag check and the per-bucket region lookup."""@@ -268,6 +344,7 @@ class _Planner:if hosted is not None:return self._plan_upload(table, hosted)assert credential is not None+ fingerprint = _credential_fingerprint(credential)try:location = parse_self_managed_url(table.url_pattern)source = self._sources.get(location.service)@@ -276,7 +353,7 @@ class _Planner:if self._team_lacks_flag(table.team, source):return f"the team doesn't have the '{source.requires_team_feature_flag}' feature flag"if location.service == "s3" and location.region is None:- region = self._region(table, location, credential)+ region = self._region(table, location, credential, fingerprint)if region is None:return "the URL doesn't name the bucket's region, and S3 didn't return it"location = dataclasses.replace(location, region=region)@@ -293,6 +370,7 @@ class _Planner:group_key=location.group_key,description=f"Self-managed tables in {location.bucket}",bucket=location.bucket,+ credential_fingerprint=fingerprint,)def _plan_upload(self, table: DataWarehouseTable, hosted: HostedUpload) -> _TablePlan | str:@@ -348,9 +426,13 @@ class _Planner:return self._flag_missing[key]def _region(- self, table: DataWarehouseTable, location: SelfManagedLocation, credential: SelfManagedCredential+ self,+ table: DataWarehouseTable,+ location: SelfManagedLocation,+ credential: SelfManagedCredential,+ fingerprint: str,) -> str | None:- key = (str(table.credential_id), location.bucket)+ key = (fingerprint, location.bucket)if key not in self._regions:try:self._regions[key] = self._resolve_region(location.bucket, credential)@@ -559,22 +641,25 @@ def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:- """The group's source for the table's team, credential and bucket. Runs under the group lock."""+ """The group's source for the table's team, bucket and key pair, the oldest when there are more.++ Runs under the group lock.+ """if not plan.shares_source:return create_source_for_upload(table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: {"group_key": plan.group_key}})- existing = (+ candidates = list(ExternalDataSource.objects.filter(team_id=table.team_id,source_type=plan.source.source_type,connection_metadata__self_managed_migration__group_key=plan.group_key,- connection_metadata__self_managed_migration__credential_id=str(table.credential_id),).exclude(deleted=True)- .order_by("created_at")- .first()+ .order_by("created_at", "id"))+ fingerprints = _source_fingerprints(candidates)+ existing = next((s for s in candidates if fingerprints.get(s.pk) == plan.credential_fingerprint), None)if existing is not None:return existingsource = SourceInputsStore.create(@@ -592,6 +677,7 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exterconnection_metadata={MIGRATION_MARKER_KEY: {"group_key": plan.group_key,+ "credential_fingerprint": plan.credential_fingerprint,"credential_id": str(table.credential_id),}},@@ -602,22 +688,16 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exterreturn source-def _credential_id(table: DataWarehouseTable) -> uuid.UUID:- if table.credential_id is None:- raise ValueError(f"Table {table.id} has no credential to migrate with")- return table.credential_id--def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:"""The table's migration schema, created with its source when there is none yet.- The credential row is the group lock: every table of one source shares it, and nothing else- locks it, so two runs can't create two sources or two schemas for the same table. An uploaded- table has no credential and a source of its own, so its own row is the lock.+ A shared source takes the group's advisory lock (see `_lock_group`), so two runs can't create+ two sources or two schemas for the same table. An uploaded table has no credential and a source+ of its own, so its own row is the lock."""with transaction.atomic():if plan.shares_source:- DataWarehouseCredential.objects.select_for_update().get(team_id=table.team_id, pk=_credential_id(table))+ _lock_group(table.team_id, str(plan.source.source_type), plan.group_key, plan.credential_fingerprint)else:DataWarehouseTable.raw_objects.select_for_update().get(team_id=table.team_id, pk=table.pk)schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])@@ -804,6 +884,164 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:)+_GroupId = tuple[int, str, str, str]+++def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False) -> list[SourceMergeOutcome]:+ """Move the schemas of each extra migration source of a group into the group's oldest source.++ Earlier runs grouped tables by credential row, so tables with the same key in one bucket got a+ source each. A duplicate moves only while all of its tables still wait for their first sync,+ because nothing has read through it then. The empty duplicate is then deleted. An uploaded+ table's source never merges.+ """+ queryset = (+ ExternalDataSource.objects.filter(connection_metadata__has_key=MIGRATION_MARKER_KEY)+ .exclude(deleted=True)+ .order_by("created_at", "id")+ )+ if team_id is not None:+ queryset = queryset.filter(team_id=team_id)+ sources = list(queryset)+ fingerprints = _source_fingerprints(sources)+ groups: dict[_GroupId, list[ExternalDataSource]] = defaultdict(list)+ for source in sources:+ group_key = _marker(source).get("group_key")+ fingerprint = fingerprints.get(source.pk)+ if not isinstance(group_key, str) or group_key.startswith("upload/") or fingerprint is None:+ continue+ groups[(source.team_id, str(source.source_type), group_key, fingerprint)].append(source)++ outcomes: list[SourceMergeOutcome] = []+ for group, (kept, *duplicates) in groups.items():+ for duplicate in duplicates:+ outcomes.append(_merge_source(group, kept, duplicate, live_run))+ return outcomes+++def _merge_refusal(+ kept: ExternalDataSource, duplicate: ExternalDataSource, schemas: list[ExternalDataSchema]+) -> str | None:+ if ExternalDataJob.objects.filter(team_id=duplicate.team_id, pipeline_id=duplicate.pk).exists():+ return "a sync already ran for the source"+ table_ids: list[str] = []+ for schema in schemas:+ ref = self_managed_table_ref(schema)+ if ref is None:+ return f"the schema '{schema.name}' doesn't wait for a self-managed table"+ if schema.should_sync:+ return f"syncing is on for '{schema.name}'"+ if schema.table_id is not None:+ return f"the schema '{schema.name}' syncs a separate table"+ table_ids.append(ref.table_id)+ adopted = (+ DataWarehouseTable.raw_objects.filter(team_id=duplicate.team_id, id__in=table_ids)+ .exclude(external_data_source__isnull=True)+ .values_list("name", flat=True)+ .first()+ )+ if adopted is not None:+ return f"the table '{adopted}' is already adopted by a source"+ if object_ids_with_rules(team_id=kept.team_id, resource="external_data_source") & {+ str(kept.pk),+ str(duplicate.pk),+ }:+ # A moved schema would take the kept source's rules, so the duplicate's rules would stop applying.+ return "the kept or the duplicate source has its own access rules"+ if _destination_ids(kept) != _destination_ids(duplicate):+ return "the kept and the duplicate source send data to different destinations"+ if SourceInputsStore.load(kept) != SourceInputsStore.load(duplicate):+ # The marker stays when a user edits the connection, so the inputs are the real check.+ return "the kept and the duplicate source connect with different settings"+ clash = (+ ExternalDataSchema.objects.filter(+ team_id=kept.team_id, source_id=kept.pk, name__in=[schema.name for schema in schemas]+ )+ .exclude(deleted=True)+ .order_by("name")+ .values_list("name", flat=True)+ .first()+ )+ if clash is not None:+ return f"the kept source already has a table named '{clash}'"+ return None+++def _destination_ids(source: ExternalDataSource) -> set[Any]:+ return set(+ ExternalDataSourceDestination.objects.for_team(source.team_id)+ .filter(source_id=source.pk)+ .values_list("destination_id", flat=True)+ )+++def _merge_source(+ group: _GroupId, kept: ExternalDataSource, duplicate: ExternalDataSource, live_run: bool+) -> SourceMergeOutcome:+ def outcome(action: MergeAction, moved: int = 0, reason: str | None = None) -> SourceMergeOutcome:+ return SourceMergeOutcome(+ source_id=str(duplicate.pk),+ team_id=duplicate.team_id,+ kept_source_id=str(kept.pk),+ action=action,+ moved_schemas=moved,+ reason=reason,+ )++ def live_schemas(lock: bool) -> list[ExternalDataSchema]:+ queryset = ExternalDataSchema.objects.filter(team_id=duplicate.team_id, source_id=duplicate.pk)+ if lock:+ queryset = queryset.select_for_update()+ return list(queryset.exclude(deleted=True).order_by("created_at", "id"))++ if not live_run:+ schemas = live_schemas(lock=False)+ refusal = _merge_refusal(kept, duplicate, schemas)+ return outcome("skipped", reason=refusal) if refusal else outcome("would_merge", len(schemas))+ try:+ with transaction.atomic():+ _lock_group(*group)+ if (+ ExternalDataSource.objects.filter(+ team_id=duplicate.team_id, pk__in=[kept.pk, duplicate.pk], deleted=False+ ).count()+ != 2+ ):+ return outcome("skipped", reason="the kept or the duplicate source was deleted during the run")+ schemas = live_schemas(lock=True)+ refusal = _merge_refusal(kept, duplicate, schemas)+ if refusal is not None:+ return outcome("skipped", reason=refusal)+ for schema in schemas:+ # Turning syncing on resumes a paused schedule, which still names the duplicate source.+ # With no schedule, it makes a new one from the schema's current source.+ delete_external_data_schedule(str(schema.id))+ schema.source = kept+ schema.save(update_fields=["source", "updated_at"])+ ExternalDataSourceDestination.objects.for_team(duplicate.team_id).filter(source_id=duplicate.pk).delete()+ duplicate.soft_delete()+ except Exception as e:+ # A failure on one source must not stop the batch or hide its outcomes.+ logger.warning("Could not merge a duplicate migration source", source_id=str(duplicate.pk), exc_info=True)+ return outcome("skipped", reason=f"{type(e).__name__}: {e}")+ logger.info(+ "Merged a duplicate self-managed migration source",+ team_id=duplicate.team_id,+ source_id=str(duplicate.pk),+ kept_source_id=str(kept.pk),+ moved_schemas=len(schemas),+ )+ return outcome("merged", len(schemas))+++def describe_merge(outcome: SourceMergeOutcome) -> str:+ detail = f" ({outcome.reason})" if outcome.reason else ""+ return (+ f"source={outcome.source_id} team={outcome.team_id} kept={outcome.kept_source_id} "+ f"{outcome.action} schemas={outcome.moved_schemas}{detail}"+ )++def describe_status(status: TableMigrationStatus) -> str:reason = f" ({status.reason})" if status.reason else ""columns = status.nested_text_column_types or {}diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.pyindex 6565b45caf8f..7b2d5d304879 100644--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py@@ -1,3 +1,5 @@+import json+import uuidfrom datetime import UTC, datetimefrom io import StringIOfrom typing import Any@@ -18,6 +20,7 @@ from posthog.models.organization import OrganizationMembershipfrom posthog.models.team import Teamfrom products.access_control.backend.models.access_control import AccessControl+from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStorefrom products.warehouse_sources.backend.models.credential import DataWarehouseCredentialfrom products.warehouse_sources.backend.models.external_data_destination import ExternalDataSourceDestinationfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob@@ -35,6 +38,7 @@ from products.warehouse_sources.backend.self_managed_migration import (TableMigrationState,abandon_tables,evaluate_tables,+ merge_duplicate_sources,migrate_self_managed_tables,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base@@ -83,8 +87,10 @@ class _MigrationTest(BaseTest):self.flag_missing, self.delete_schedule, _ = mocksself.credential = self._credential("AKIAFIRST")- def _credential(self, access_key: str) -> DataWarehouseCredential:- return DataWarehouseCredential.objects.create(access_key=access_key, access_secret="c2VjcmV0", team=self.team)+ def _credential(self, access_key: str, access_secret: str = "c2VjcmV0") -> DataWarehouseCredential:+ return DataWarehouseCredential.objects.create(+ access_key=access_key, access_secret=access_secret, team=self.team+ )def _table(self,@@ -165,10 +171,67 @@ class TestSelfManagedMigration(_MigrationTest):assert set(prefixes) - {""} <= {"exports", "archive"}assert source_of(other_bucket).job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/archive/"assert source_of(other_key).job_inputs["access_key_id"] == "AKIASECOND"- assert (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY] == {- "group_key": "s3/eu-west-1/exports",- "credential_id": str(self.credential.pk),+ marker = (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY]+ assert marker["group_key"] == "s3/eu-west-1/exports"+ assert marker["credential_id"] == str(self.credential.pk)+ assert len(marker["credential_fingerprint"]) == 64+ assert "AKIAFIRST" not in json.dumps(marker) and "c2VjcmV0" not in json.dumps(marker)++ @parameterized.expand(+ [+ ("same_credential_row", None, True),+ ("equal_valued_credential_row", ("AKIAFIRST", "c2VjcmV0"), True),+ ("equal_after_trimming_whitespace", (" AKIAFIRST", "c2VjcmV0 "), True),+ ("different_secret", ("AKIAFIRST", "b3RoZXI="), False),+ ("different_key", ("AKIASECOND", "c2VjcmV0"), False),+ ]+ )+ def test_tables_in_one_bucket_share_a_source_when_their_credentials_hold_the_same_key(+ self, _name: str, second_credential: tuple[str, str] | None, shared: bool+ ) -> None:+ first = self._table("orders")+ second = self._table(+ "refunds",+ url="https://exports.s3.eu-west-1.amazonaws.com/refunds/*.csv",+ credential=self._credential(*second_credential) if second_credential else None,+ )++ outcomes = self._migrate()++ assert [o.action for o in outcomes] == ["linked", "linked"]+ assert (_schema_for(first).source_id == _schema_for(second).source_id) is shared+ assert ExternalDataSource.objects.filter(team=self.team).count() == (1 if shared else 2)++ @parameterized.expand(+ [+ ("same_value", "c2VjcmV0", False, True),+ ("different_value", "b3RoZXI=", False, False),+ ("credential_row_gone", "c2VjcmV0", True, False),+ ]+ )+ def test_a_source_with_a_credential_id_marker_takes_tables_whose_credential_holds_its_key(+ self, _name: str, secret: str, credential_row_gone: bool, shared: bool+ ) -> None:+ first = self._table("orders")+ self._migrate()+ old_source = _schema_for(first).source+ old_source.connection_metadata = {+ MIGRATION_MARKER_KEY: {+ "group_key": "s3/eu-west-1/exports",+ "credential_id": str(uuid.uuid4()) if credential_row_gone else str(self.credential.pk),+ }}+ old_source.save()+ second = self._table(+ "refunds",+ url="https://exports.s3.eu-west-1.amazonaws.com/refunds/*.csv",+ credential=self._credential("AKIAFIRST", secret),+ )++ [outcome] = self._migrate(table_ids=(str(second.id),))++ assert outcome.action == "linked"+ assert (_schema_for(second).source_id == old_source.pk) is shareddef test_a_live_run_creates_a_turned_off_schema_and_changes_nothing_else(self) -> None:created_at = datetime(2024, 3, 1, tzinfo=UTC)@@ -454,12 +517,22 @@ class TestSelfManagedMigration(_MigrationTest):assert [(o.status.name, o.action) for o in outcomes][1:] == [("dupe", "link_failed"), ("fine", "linked")]assert "already uses the name 'dupe'" in (outcomes[1].error or "")- @parameterized.expand([("named_region", "eu-west-1", "linked"), ("unknown_region", None, "none")])+ @parameterized.expand(+ [+ ("named_region", "eu-west-1", "linked", False),+ ("unknown_region", None, "none", False),+ ("named_region_equal_valued_credential_rows", "eu-west-1", "linked", True),+ ]+ )def test_a_bucket_url_without_a_region_asks_s3_once_per_bucket(- self, _name: str, region: str | None, expected_action: str+ self, _name: str, region: str | None, expected_action: str, second_row: bool) -> None:self._table("a", url="https://exports.s3.amazonaws.com/a/*.csv")- self._table("b", url="https://s3.amazonaws.com/exports/b/*.csv")+ self._table(+ "b",+ url="https://s3.amazonaws.com/exports/b/*.csv",+ credential=self._credential("AKIAFIRST") if second_row else None,+ )resolver = MagicMock(return_value=region)outcomes = self._migrate(resolve_region=resolver)@@ -591,11 +664,139 @@ class TestAbandon(_MigrationTest):self.delete_schedule.assert_not_called()+class TestMergeDuplicateSources(_MigrationTest):+ def _duplicates(self, count: int = 3) -> list[DataWarehouseTable]:+ tables = [+ self._table(+ f"table_{i}",+ url=f"https://exports.s3.eu-west-1.amazonaws.com/t{i}/*.csv",+ credential=self._credential(f"AKIATEMP{i}"),+ )+ for i in range(count)+ ]+ self._migrate()+ # Earlier runs keyed each source by its credential row, so equal keys in separate rows made a source each.+ for i, table in enumerate(tables):+ credential = table.credential+ assert credential is not None+ credential.access_key = "AKIAFIRST"+ credential.save()+ source = _schema_for(table).source+ inputs = SourceInputsStore.load(source)+ SourceInputsStore.save(+ source, {key: "AKIAFIRST" if value == f"AKIATEMP{i}" else value for key, value in inputs.items()}+ )+ ExternalDataSource.objects.filter(pk=source.pk).update(+ created_at=datetime(2024, 1, 1 + i, tzinfo=UTC),+ connection_metadata={+ MIGRATION_MARKER_KEY: {"group_key": "s3/eu-west-1/exports", "credential_id": str(credential.pk)}+ },+ )+ assert len({_schema_for(table).source_id for table in tables}) == count+ return tables++ def test_duplicates_move_into_the_oldest_source_and_are_deleted(self) -> None:+ tables = self._duplicates()+ kept = _schema_for(tables[0]).source+ duplicates = [_schema_for(table).source for table in tables[1:]]+ moved_schema_ids = [str(_schema_for(table).id) for table in tables[1:]]++ outcomes = merge_duplicate_sources(team_id=self.team.pk, live_run=True)++ assert [(o.source_id, o.kept_source_id, o.action, o.moved_schemas) for o in outcomes] == [+ (str(source.pk), str(kept.pk), "merged", 1) for source in duplicates+ ]+ assert {_schema_for(table).source_id for table in tables} == {kept.pk}+ for source in duplicates:+ source.refresh_from_db()+ assert source.deleted is True+ assert not ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=source).exists()+ assert ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=kept).exists()+ assert sorted(call.args[0] for call in self.delete_schedule.call_args_list) == sorted(moved_schema_ids)+ assert {self._status(table).source_id for table in tables} == {str(kept.pk)}+ assert {self._status(table).state for table in tables} == {TableMigrationState.AWAITING_SYNC}+ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []+ added = self._table("late", url="https://exports.s3.eu-west-1.amazonaws.com/late/*.csv")+ self._migrate(table_ids=(str(added.id),))+ assert _schema_for(added).source_id == kept.pk++ def test_a_dry_run_reports_the_merge_and_changes_nothing(self) -> None:+ tables = self._duplicates(count=2)+ before = {table.pk: _schema_for(table).source_id for table in tables}+ out = StringIO()++ call_command("migrate_self_managed_tables", team_id=self.team.pk, merge_sources=True, stdout=out)++ duplicate_id = before[tables[1].pk]+ lines = out.getvalue().splitlines()+ assert lines[0] == (+ f"source={duplicate_id} team={self.team.pk} kept={before[tables[0].pk]} would_merge schemas=1"+ )+ assert "1 duplicate source(s), 1 schema(s): 1 would_merge" in out.getvalue()+ assert {table.pk: _schema_for(table).source_id for table in tables} == before+ assert ExternalDataSource.objects.get(pk=duplicate_id).deleted is False+ self.delete_schedule.assert_not_called()++ @parameterized.expand(+ [+ ("syncing_on", "syncing is on for 'table_1'"),+ ("job_ran", "a sync already ran for the source"),+ ("table_adopted", "the table 'table_1' is already adopted by a source"),+ ("name_taken_in_kept_source", "the kept source already has a table named 'table_1'"),+ ("source_has_rules", "the kept or the duplicate source has its own access rules"),+ ("other_destinations", "the kept and the duplicate source send data to different destinations"),+ ("inputs_edited", "the kept and the duplicate source connect with different settings"),+ ]+ )+ def test_a_duplicate_that_is_not_safe_to_move_is_skipped(self, case: str, reason: str) -> None:+ tables = self._duplicates()+ kept = _schema_for(tables[0]).source+ unsafe = _schema_for(tables[1])+ safe_source_id = _schema_for(tables[2]).source_id+ if case == "syncing_on":+ ExternalDataSchema.objects.filter(pk=unsafe.pk).update(should_sync=True)+ elif case == "job_ran":+ self._job(tables[1], ExternalDataJobStatus.FAILED)+ elif case == "table_adopted":+ DataWarehouseTable.raw_objects.filter(pk=tables[1].pk).update(external_data_source=unsafe.source)+ elif case == "name_taken_in_kept_source":+ ExternalDataSchema.objects.create(name="table_1", team=self.team, source=kept)+ elif case == "source_has_rules":+ AccessControl.objects.create(+ team=self.team, resource="external_data_source", resource_id=str(unsafe.source_id), access_level="none"+ )+ elif case == "other_destinations":+ ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source_id=unsafe.source_id).delete()+ else:+ source = unsafe.source+ SourceInputsStore.save(source, {**SourceInputsStore.load(source), "file_format": "csv"})++ outcomes = {o.source_id: o for o in merge_duplicate_sources(team_id=self.team.pk, live_run=True)}++ skipped = outcomes[str(unsafe.source_id)]+ assert (skipped.action, skipped.reason, skipped.moved_schemas) == ("skipped", reason, 0)+ assert ExternalDataSchema.objects.get(pk=unsafe.pk).source_id == unsafe.source_id+ assert ExternalDataSource.objects.get(pk=unsafe.source_id).deleted is False+ assert outcomes[str(safe_source_id)].action == "merged"+ assert _schema_for(tables[2]).source_id == kept.pk++ def test_sources_with_another_key_or_bucket_stay_apart(self) -> None:+ self._duplicates(count=1)+ self._table("other_key", credential=self._credential("AKIAFIRST", "b3RoZXI="))+ self._table("other_bucket", url="https://archive.s3.eu-west-1.amazonaws.com/a.csv")+ self._migrate()++ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 3++class TestMigrateSelfManagedTablesCommand(BaseTest):@parameterized.expand([("zero_limit", {"limit": 0}, "--limit"),("status_and_abandon", {"status": True, "abandon": "x"}, "only one"),+ ("status_and_merge", {"status": True, "merge_sources": True}, "only one"),+ ("abandon_and_merge", {"abandon": "x", "merge_sources": True}, "only one"),("empty_abandon", {"abandon": " , "}, "at least one table id"),])@@ -823,6 +1024,14 @@ class TestUploadedTableMigration(_MigrationTest):assert schema_metadata is not Noneassert schema_metadata["csv"]["column_types"] == {"day": stored}+ def test_uploaded_tables_sources_never_merge(self) -> None:+ self._uploaded("orders", "managed/team_{team}/orders.csv")+ self._uploaded("refunds", "managed/team_{team}/refunds.csv")+ self._migrate()++ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 2+def test_teams_without_the_flag_keep_their_uploaded_tables(self) -> None:self.flag_missing.return_value = Trueself._uploaded("orders", "file_uploads/team_{team}/{upload}/orders.csv")