Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): group migrated self-managed tables by credential value (#128399)

Tom Owersd5aaa0bcaf71
diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
index 58094d04c201..fe7193012757 100644
--- a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
@@ -7,8 +7,10 @@ from products.warehouse_sources.backend.self_managed_migration import (
DEFAULT_ABANDON_REASON,
MigrationOptions,
abandon_tables,
+ describe_merge,
describe_status,
evaluate_tables,
+ merge_duplicate_sources,
migrate_self_managed_tables,
)
@@ -53,6 +55,12 @@ class Command(BaseCommand):
default=DEFAULT_ABANDON_REASON,
help="The reason recorded for each abandoned table.",
)
+ parser.add_argument(
+ "--merge-sources",
+ action="store_true",
+ help="Move the tables of extra sources for one bucket and key into the oldest source, and delete "
+ "the extra sources. A source with a table that started to sync stays as it is.",
+ )
parser.add_argument(
"--live-run", action="store_true", help="Act. Without this the command only reports what it would do."
)
@@ -60,8 +68,17 @@ class Command(BaseCommand):
def handle(self, *args: Any, **options: Any) -> None:
if options["limit"] is not None and options["limit"] < 1:
raise CommandError("--limit must be at least 1.")
- if options["status"] and options["abandon"] is not None:
- raise CommandError("Pass only one of --status and --abandon.")
+ modes = [
+ flag
+ for flag, chosen in (
+ ("--status", options["status"]),
+ ("--abandon", options["abandon"] is not None),
+ ("--merge-sources", options["merge_sources"]),
+ )
+ if chosen
+ ]
+ if len(modes) > 1:
+ raise CommandError(f"Pass only one of {', '.join(modes)}.")
team_id: int | None = options["team_id"]
table_ids = _split_ids(options["table_ids"])
@@ -71,6 +88,8 @@ class Command(BaseCommand):
self._report_status(team_id, table_ids)
elif options["abandon"] is not None:
self._abandon(_split_ids(options["abandon"]), options["abandon_reason"], team_id, live_run)
+ elif options["merge_sources"]:
+ self._merge_sources(team_id, live_run)
else:
self._migrate(team_id, table_ids, live_run, options["limit"])
@@ -90,6 +109,19 @@ class Command(BaseCommand):
if not live_run:
self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to abandon these tables."))
+ def _merge_sources(self, team_id: int | None, live_run: bool) -> None:
+ outcomes = merge_duplicate_sources(team_id=team_id, live_run=live_run)
+ for outcome in outcomes:
+ self.stdout.write(describe_merge(outcome))
+ actions = Counter(outcome.action for outcome in outcomes)
+ moved = sum(outcome.moved_schemas for outcome in outcomes)
+ self.stdout.write(
+ f"\n{len(outcomes)} duplicate source(s), {moved} schema(s): "
+ + ", ".join(f"{n} {a}" for a, n in sorted(actions.items()))
+ )
+ if not live_run:
+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to merge these sources."))
+
def _migrate(self, team_id: int | None, table_ids: list[str], live_run: bool, limit: int | None) -> None:
outcomes = migrate_self_managed_tables(
MigrationOptions(team_id=team_id, table_ids=tuple(table_ids), live_run=live_run, limit=limit)
diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.py
index 0c69bbefbc54..afa9e25739f3 100644
--- a/products/warehouse_sources/backend/self_managed_migration.py
+++ b/products/warehouse_sources/backend/self_managed_migration.py
@@ -21,15 +21,17 @@ continues where it stopped.
"""
import re
+import hmac
import uuid
+import hashlib
import dataclasses
from collections import defaultdict
-from collections.abc import Callable, Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping, Sequence
from enum import StrEnum
from typing import Any, Literal, cast
from django.conf import settings
-from django.db import transaction
+from django.db import connection, transaction
from django.db.models import CharField, Q, Value
from django.db.models.functions import Cast, Concat
@@ -128,6 +130,7 @@ class TableMigrationState(StrEnum):
MigrationAction = Literal["none", "would_link", "linked", "link_failed", "limit_reached"]
AbandonAction = Literal["would_abandon", "abandoned", "refused"]
+MergeAction = Literal["would_merge", "merged", "skipped"]
@frozen
@@ -156,6 +159,16 @@ class AbandonOutcome:
reason: str | None = None
+@frozen
+class SourceMergeOutcome:
+ source_id: str
+ team_id: int
+ kept_source_id: str
+ action: MergeAction
+ moved_schemas: int = 0
+ reason: str | None = None
+
+
@frozen
class MigrationOptions:
team_id: int | None = None
@@ -172,13 +185,14 @@ class _TablePlan:
source: ObjectStorageSource[Any] | FileUploadSource
schema_metadata: dict[str, Any]
inputs: dict[str, Any] = dataclasses.field(repr=False)
- # Tables with the same group key and credential share one source.
+ # Tables with the same group key and credential fingerprint share one source.
group_key: str
description: str
# An uploaded table has no credential, and gets a source of its own.
shares_source: bool = True
# The bucket a shared source reads, which names its table prefix when one is needed.
bucket: str = ""
+ credential_fingerprint: str = dataclasses.field(default="", repr=False)
@frozen
@@ -234,9 +248,71 @@ def _credential_of(table: DataWarehouseTable) -> SelfManagedCredential | None:
credential = table.credential
if credential is None:
return None
+ return _self_managed_credential(credential)
+
+
+def _self_managed_credential(credential: DataWarehouseCredential) -> SelfManagedCredential:
return SelfManagedCredential(access_key=credential.access_key or "", access_secret=credential.access_secret or "")
+def _credential_fingerprint(credential: SelfManagedCredential) -> str:
+ """Identifies a key pair by its value, so tables whose credential rows hold the same key share one source.
+
+ The app saves a new credential row for each table, also when the key is the same, so the row id
+ can't group them. An HMAC keeps the key pair itself out of the stored marker.
+ """
+ message = f"{credential.access_key.strip()}\n{credential.access_secret.strip()}".encode()
+ return hmac.new(settings.SECRET_KEY.encode(), message, hashlib.sha256).hexdigest()
+
+
+def _marker(source: ExternalDataSource) -> dict[str, Any]:
+ marker = (source.connection_metadata or {}).get(MIGRATION_MARKER_KEY)
+ return marker if isinstance(marker, dict) else {}
+
+
+def _source_fingerprints(sources: Sequence[ExternalDataSource]) -> dict[uuid.UUID, str]:
+ """The credential fingerprint of each migration source.
+
+ A source made before the marker held a fingerprint names its credential row instead. When that
+ row is gone, the source has no fingerprint and no table joins it.
+ """
+ fingerprints: dict[uuid.UUID, str] = {}
+ legacy: dict[uuid.UUID, tuple[int, uuid.UUID]] = {}
+ for source in sources:
+ marker = _marker(source)
+ fingerprint = marker.get("credential_fingerprint")
+ if isinstance(fingerprint, str) and fingerprint:
+ fingerprints[source.pk] = fingerprint
+ continue
+ try:
+ legacy[source.pk] = (source.team_id, uuid.UUID(str(marker.get("credential_id"))))
+ except ValueError:
+ continue
+ credential_ids_by_team: dict[int, set[uuid.UUID]] = defaultdict(set)
+ for team_id, credential_id in legacy.values():
+ credential_ids_by_team[team_id].add(credential_id)
+ credentials: dict[tuple[int, uuid.UUID], DataWarehouseCredential] = {}
+ for team_id, credential_ids in credential_ids_by_team.items():
+ for credential in DataWarehouseCredential.objects.filter(team_id=team_id, pk__in=credential_ids):
+ credentials[(team_id, credential.pk)] = credential
+ for source_id, key in legacy.items():
+ if (found := credentials.get(key)) is not None:
+ fingerprints[source_id] = _credential_fingerprint(_self_managed_credential(found))
+ return fingerprints
+
+
+def _lock_group(team_id: int, source_type: str, group_key: str, fingerprint: str) -> None:
+ """Hold the group's lock until the transaction ends.
+
+ Every table of one source shares the lock, so two runs can't create two sources for the group,
+ and a merge can't move schemas while a run adds to the group.
+ """
+ name = "\n".join((MIGRATION_MARKER_KEY, str(team_id), source_type, group_key, fingerprint))
+ lock_id = int.from_bytes(hashlib.blake2b(name.encode(), digest_size=8).digest(), "big", signed=True)
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT pg_advisory_xact_lock(%s)", [lock_id])
+
+
class _Planner:
"""Maps tables onto sources, caching the per-team flag check and the per-bucket region lookup."""
@@ -268,6 +344,7 @@ class _Planner:
if hosted is not None:
return self._plan_upload(table, hosted)
assert credential is not None
+ fingerprint = _credential_fingerprint(credential)
try:
location = parse_self_managed_url(table.url_pattern)
source = self._sources.get(location.service)
@@ -276,7 +353,7 @@ class _Planner:
if self._team_lacks_flag(table.team, source):
return f"the team doesn't have the '{source.requires_team_feature_flag}' feature flag"
if location.service == "s3" and location.region is None:
- region = self._region(table, location, credential)
+ region = self._region(table, location, credential, fingerprint)
if region is None:
return "the URL doesn't name the bucket's region, and S3 didn't return it"
location = dataclasses.replace(location, region=region)
@@ -293,6 +370,7 @@ class _Planner:
group_key=location.group_key,
description=f"Self-managed tables in {location.bucket}",
bucket=location.bucket,
+ credential_fingerprint=fingerprint,
)
def _plan_upload(self, table: DataWarehouseTable, hosted: HostedUpload) -> _TablePlan | str:
@@ -348,9 +426,13 @@ class _Planner:
return self._flag_missing[key]
def _region(
- self, table: DataWarehouseTable, location: SelfManagedLocation, credential: SelfManagedCredential
+ self,
+ table: DataWarehouseTable,
+ location: SelfManagedLocation,
+ credential: SelfManagedCredential,
+ fingerprint: str,
) -> str | None:
- key = (str(table.credential_id), location.bucket)
+ key = (fingerprint, location.bucket)
if key not in self._regions:
try:
self._regions[key] = self._resolve_region(location.bucket, credential)
@@ -559,22 +641,25 @@ def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:
def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:
- """The group's source for the table's team, credential and bucket. Runs under the group lock."""
+ """The group's source for the table's team, bucket and key pair, the oldest when there are more.
+
+ Runs under the group lock.
+ """
if not plan.shares_source:
return create_source_for_upload(
table, plan.inputs, connection_metadata={MIGRATION_MARKER_KEY: {"group_key": plan.group_key}}
)
- existing = (
+ candidates = list(
ExternalDataSource.objects.filter(
team_id=table.team_id,
source_type=plan.source.source_type,
connection_metadata__self_managed_migration__group_key=plan.group_key,
- connection_metadata__self_managed_migration__credential_id=str(table.credential_id),
)
.exclude(deleted=True)
- .order_by("created_at")
- .first()
+ .order_by("created_at", "id")
)
+ fingerprints = _source_fingerprints(candidates)
+ existing = next((s for s in candidates if fingerprints.get(s.pk) == plan.credential_fingerprint), None)
if existing is not None:
return existing
source = SourceInputsStore.create(
@@ -592,6 +677,7 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter
connection_metadata={
MIGRATION_MARKER_KEY: {
"group_key": plan.group_key,
+ "credential_fingerprint": plan.credential_fingerprint,
"credential_id": str(table.credential_id),
}
},
@@ -602,22 +688,16 @@ def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> Exter
return source
-def _credential_id(table: DataWarehouseTable) -> uuid.UUID:
- if table.credential_id is None:
- raise ValueError(f"Table {table.id} has no credential to migrate with")
- return table.credential_id
-
-
def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:
"""The table's migration schema, created with its source when there is none yet.
- The credential row is the group lock: every table of one source shares it, and nothing else
- locks it, so two runs can't create two sources or two schemas for the same table. An uploaded
- table has no credential and a source of its own, so its own row is the lock.
+ A shared source takes the group's advisory lock (see `_lock_group`), so two runs can't create
+ two sources or two schemas for the same table. An uploaded table has no credential and a source
+ of its own, so its own row is the lock.
"""
with transaction.atomic():
if plan.shares_source:
- DataWarehouseCredential.objects.select_for_update().get(team_id=table.team_id, pk=_credential_id(table))
+ _lock_group(table.team_id, str(plan.source.source_type), plan.group_key, plan.credential_fingerprint)
else:
DataWarehouseTable.raw_objects.select_for_update().get(team_id=table.team_id, pk=table.pk)
schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])
@@ -804,6 +884,164 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:
)
+_GroupId = tuple[int, str, str, str]
+
+
+def merge_duplicate_sources(team_id: int | None = None, live_run: bool = False) -> list[SourceMergeOutcome]:
+ """Move the schemas of each extra migration source of a group into the group's oldest source.
+
+ Earlier runs grouped tables by credential row, so tables with the same key in one bucket got a
+ source each. A duplicate moves only while all of its tables still wait for their first sync,
+ because nothing has read through it then. The empty duplicate is then deleted. An uploaded
+ table's source never merges.
+ """
+ queryset = (
+ ExternalDataSource.objects.filter(connection_metadata__has_key=MIGRATION_MARKER_KEY)
+ .exclude(deleted=True)
+ .order_by("created_at", "id")
+ )
+ if team_id is not None:
+ queryset = queryset.filter(team_id=team_id)
+ sources = list(queryset)
+ fingerprints = _source_fingerprints(sources)
+ groups: dict[_GroupId, list[ExternalDataSource]] = defaultdict(list)
+ for source in sources:
+ group_key = _marker(source).get("group_key")
+ fingerprint = fingerprints.get(source.pk)
+ if not isinstance(group_key, str) or group_key.startswith("upload/") or fingerprint is None:
+ continue
+ groups[(source.team_id, str(source.source_type), group_key, fingerprint)].append(source)
+
+ outcomes: list[SourceMergeOutcome] = []
+ for group, (kept, *duplicates) in groups.items():
+ for duplicate in duplicates:
+ outcomes.append(_merge_source(group, kept, duplicate, live_run))
+ return outcomes
+
+
+def _merge_refusal(
+ kept: ExternalDataSource, duplicate: ExternalDataSource, schemas: list[ExternalDataSchema]
+) -> str | None:
+ if ExternalDataJob.objects.filter(team_id=duplicate.team_id, pipeline_id=duplicate.pk).exists():
+ return "a sync already ran for the source"
+ table_ids: list[str] = []
+ for schema in schemas:
+ ref = self_managed_table_ref(schema)
+ if ref is None:
+ return f"the schema '{schema.name}' doesn't wait for a self-managed table"
+ if schema.should_sync:
+ return f"syncing is on for '{schema.name}'"
+ if schema.table_id is not None:
+ return f"the schema '{schema.name}' syncs a separate table"
+ table_ids.append(ref.table_id)
+ adopted = (
+ DataWarehouseTable.raw_objects.filter(team_id=duplicate.team_id, id__in=table_ids)
+ .exclude(external_data_source__isnull=True)
+ .values_list("name", flat=True)
+ .first()
+ )
+ if adopted is not None:
+ return f"the table '{adopted}' is already adopted by a source"
+ if object_ids_with_rules(team_id=kept.team_id, resource="external_data_source") & {
+ str(kept.pk),
+ str(duplicate.pk),
+ }:
+ # A moved schema would take the kept source's rules, so the duplicate's rules would stop applying.
+ return "the kept or the duplicate source has its own access rules"
+ if _destination_ids(kept) != _destination_ids(duplicate):
+ return "the kept and the duplicate source send data to different destinations"
+ if SourceInputsStore.load(kept) != SourceInputsStore.load(duplicate):
+ # The marker stays when a user edits the connection, so the inputs are the real check.
+ return "the kept and the duplicate source connect with different settings"
+ clash = (
+ ExternalDataSchema.objects.filter(
+ team_id=kept.team_id, source_id=kept.pk, name__in=[schema.name for schema in schemas]
+ )
+ .exclude(deleted=True)
+ .order_by("name")
+ .values_list("name", flat=True)
+ .first()
+ )
+ if clash is not None:
+ return f"the kept source already has a table named '{clash}'"
+ return None
+
+
+def _destination_ids(source: ExternalDataSource) -> set[Any]:
+ return set(
+ ExternalDataSourceDestination.objects.for_team(source.team_id)
+ .filter(source_id=source.pk)
+ .values_list("destination_id", flat=True)
+ )
+
+
+def _merge_source(
+ group: _GroupId, kept: ExternalDataSource, duplicate: ExternalDataSource, live_run: bool
+) -> SourceMergeOutcome:
+ def outcome(action: MergeAction, moved: int = 0, reason: str | None = None) -> SourceMergeOutcome:
+ return SourceMergeOutcome(
+ source_id=str(duplicate.pk),
+ team_id=duplicate.team_id,
+ kept_source_id=str(kept.pk),
+ action=action,
+ moved_schemas=moved,
+ reason=reason,
+ )
+
+ def live_schemas(lock: bool) -> list[ExternalDataSchema]:
+ queryset = ExternalDataSchema.objects.filter(team_id=duplicate.team_id, source_id=duplicate.pk)
+ if lock:
+ queryset = queryset.select_for_update()
+ return list(queryset.exclude(deleted=True).order_by("created_at", "id"))
+
+ if not live_run:
+ schemas = live_schemas(lock=False)
+ refusal = _merge_refusal(kept, duplicate, schemas)
+ return outcome("skipped", reason=refusal) if refusal else outcome("would_merge", len(schemas))
+ try:
+ with transaction.atomic():
+ _lock_group(*group)
+ if (
+ ExternalDataSource.objects.filter(
+ team_id=duplicate.team_id, pk__in=[kept.pk, duplicate.pk], deleted=False
+ ).count()
+ != 2
+ ):
+ return outcome("skipped", reason="the kept or the duplicate source was deleted during the run")
+ schemas = live_schemas(lock=True)
+ refusal = _merge_refusal(kept, duplicate, schemas)
+ if refusal is not None:
+ return outcome("skipped", reason=refusal)
+ for schema in schemas:
+ # Turning syncing on resumes a paused schedule, which still names the duplicate source.
+ # With no schedule, it makes a new one from the schema's current source.
+ delete_external_data_schedule(str(schema.id))
+ schema.source = kept
+ schema.save(update_fields=["source", "updated_at"])
+ ExternalDataSourceDestination.objects.for_team(duplicate.team_id).filter(source_id=duplicate.pk).delete()
+ duplicate.soft_delete()
+ except Exception as e:
+ # A failure on one source must not stop the batch or hide its outcomes.
+ logger.warning("Could not merge a duplicate migration source", source_id=str(duplicate.pk), exc_info=True)
+ return outcome("skipped", reason=f"{type(e).__name__}: {e}")
+ logger.info(
+ "Merged a duplicate self-managed migration source",
+ team_id=duplicate.team_id,
+ source_id=str(duplicate.pk),
+ kept_source_id=str(kept.pk),
+ moved_schemas=len(schemas),
+ )
+ return outcome("merged", len(schemas))
+
+
+def describe_merge(outcome: SourceMergeOutcome) -> str:
+ detail = f" ({outcome.reason})" if outcome.reason else ""
+ return (
+ f"source={outcome.source_id} team={outcome.team_id} kept={outcome.kept_source_id} "
+ f"{outcome.action} schemas={outcome.moved_schemas}{detail}"
+ )
+
+
def describe_status(status: TableMigrationStatus) -> str:
reason = f" ({status.reason})" if status.reason else ""
columns = status.nested_text_column_types or {}
diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
index 6565b45caf8f..7b2d5d304879 100644
--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py
+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
@@ -1,3 +1,5 @@
+import json
+import uuid
from datetime import UTC, datetime
from io import StringIO
from typing import Any
@@ -18,6 +20,7 @@ from posthog.models.organization import OrganizationMembership
from posthog.models.team import Team
from products.access_control.backend.models.access_control import AccessControl
+from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStore
from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
from products.warehouse_sources.backend.models.external_data_destination import ExternalDataSourceDestination
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
@@ -35,6 +38,7 @@ from products.warehouse_sources.backend.self_managed_migration import (
TableMigrationState,
abandon_tables,
evaluate_tables,
+ merge_duplicate_sources,
migrate_self_managed_tables,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base
@@ -83,8 +87,10 @@ class _MigrationTest(BaseTest):
self.flag_missing, self.delete_schedule, _ = mocks
self.credential = self._credential("AKIAFIRST")
- def _credential(self, access_key: str) -> DataWarehouseCredential:
- return DataWarehouseCredential.objects.create(access_key=access_key, access_secret="c2VjcmV0", team=self.team)
+ def _credential(self, access_key: str, access_secret: str = "c2VjcmV0") -> DataWarehouseCredential:
+ return DataWarehouseCredential.objects.create(
+ access_key=access_key, access_secret=access_secret, team=self.team
+ )
def _table(
self,
@@ -165,10 +171,67 @@ class TestSelfManagedMigration(_MigrationTest):
assert set(prefixes) - {""} <= {"exports", "archive"}
assert source_of(other_bucket).job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/archive/"
assert source_of(other_key).job_inputs["access_key_id"] == "AKIASECOND"
- assert (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY] == {
- "group_key": "s3/eu-west-1/exports",
- "credential_id": str(self.credential.pk),
+ marker = (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY]
+ assert marker["group_key"] == "s3/eu-west-1/exports"
+ assert marker["credential_id"] == str(self.credential.pk)
+ assert len(marker["credential_fingerprint"]) == 64
+ assert "AKIAFIRST" not in json.dumps(marker) and "c2VjcmV0" not in json.dumps(marker)
+
+ @parameterized.expand(
+ [
+ ("same_credential_row", None, True),
+ ("equal_valued_credential_row", ("AKIAFIRST", "c2VjcmV0"), True),
+ ("equal_after_trimming_whitespace", (" AKIAFIRST", "c2VjcmV0 "), True),
+ ("different_secret", ("AKIAFIRST", "b3RoZXI="), False),
+ ("different_key", ("AKIASECOND", "c2VjcmV0"), False),
+ ]
+ )
+ def test_tables_in_one_bucket_share_a_source_when_their_credentials_hold_the_same_key(
+ self, _name: str, second_credential: tuple[str, str] | None, shared: bool
+ ) -> None:
+ first = self._table("orders")
+ second = self._table(
+ "refunds",
+ url="https://exports.s3.eu-west-1.amazonaws.com/refunds/*.csv",
+ credential=self._credential(*second_credential) if second_credential else None,
+ )
+
+ outcomes = self._migrate()
+
+ assert [o.action for o in outcomes] == ["linked", "linked"]
+ assert (_schema_for(first).source_id == _schema_for(second).source_id) is shared
+ assert ExternalDataSource.objects.filter(team=self.team).count() == (1 if shared else 2)
+
+ @parameterized.expand(
+ [
+ ("same_value", "c2VjcmV0", False, True),
+ ("different_value", "b3RoZXI=", False, False),
+ ("credential_row_gone", "c2VjcmV0", True, False),
+ ]
+ )
+ def test_a_source_with_a_credential_id_marker_takes_tables_whose_credential_holds_its_key(
+ self, _name: str, secret: str, credential_row_gone: bool, shared: bool
+ ) -> None:
+ first = self._table("orders")
+ self._migrate()
+ old_source = _schema_for(first).source
+ old_source.connection_metadata = {
+ MIGRATION_MARKER_KEY: {
+ "group_key": "s3/eu-west-1/exports",
+ "credential_id": str(uuid.uuid4()) if credential_row_gone else str(self.credential.pk),
+ }
}
+ old_source.save()
+ second = self._table(
+ "refunds",
+ url="https://exports.s3.eu-west-1.amazonaws.com/refunds/*.csv",
+ credential=self._credential("AKIAFIRST", secret),
+ )
+
+ [outcome] = self._migrate(table_ids=(str(second.id),))
+
+ assert outcome.action == "linked"
+ assert (_schema_for(second).source_id == old_source.pk) is shared
def test_a_live_run_creates_a_turned_off_schema_and_changes_nothing_else(self) -> None:
created_at = datetime(2024, 3, 1, tzinfo=UTC)
@@ -454,12 +517,22 @@ class TestSelfManagedMigration(_MigrationTest):
assert [(o.status.name, o.action) for o in outcomes][1:] == [("dupe", "link_failed"), ("fine", "linked")]
assert "already uses the name 'dupe'" in (outcomes[1].error or "")
- @parameterized.expand([("named_region", "eu-west-1", "linked"), ("unknown_region", None, "none")])
+ @parameterized.expand(
+ [
+ ("named_region", "eu-west-1", "linked", False),
+ ("unknown_region", None, "none", False),
+ ("named_region_equal_valued_credential_rows", "eu-west-1", "linked", True),
+ ]
+ )
def test_a_bucket_url_without_a_region_asks_s3_once_per_bucket(
- self, _name: str, region: str | None, expected_action: str
+ self, _name: str, region: str | None, expected_action: str, second_row: bool
) -> None:
self._table("a", url="https://exports.s3.amazonaws.com/a/*.csv")
- self._table("b", url="https://s3.amazonaws.com/exports/b/*.csv")
+ self._table(
+ "b",
+ url="https://s3.amazonaws.com/exports/b/*.csv",
+ credential=self._credential("AKIAFIRST") if second_row else None,
+ )
resolver = MagicMock(return_value=region)
outcomes = self._migrate(resolve_region=resolver)
@@ -591,11 +664,139 @@ class TestAbandon(_MigrationTest):
self.delete_schedule.assert_not_called()
+class TestMergeDuplicateSources(_MigrationTest):
+ def _duplicates(self, count: int = 3) -> list[DataWarehouseTable]:
+ tables = [
+ self._table(
+ f"table_{i}",
+ url=f"https://exports.s3.eu-west-1.amazonaws.com/t{i}/*.csv",
+ credential=self._credential(f"AKIATEMP{i}"),
+ )
+ for i in range(count)
+ ]
+ self._migrate()
+ # Earlier runs keyed each source by its credential row, so equal keys in separate rows made a source each.
+ for i, table in enumerate(tables):
+ credential = table.credential
+ assert credential is not None
+ credential.access_key = "AKIAFIRST"
+ credential.save()
+ source = _schema_for(table).source
+ inputs = SourceInputsStore.load(source)
+ SourceInputsStore.save(
+ source, {key: "AKIAFIRST" if value == f"AKIATEMP{i}" else value for key, value in inputs.items()}
+ )
+ ExternalDataSource.objects.filter(pk=source.pk).update(
+ created_at=datetime(2024, 1, 1 + i, tzinfo=UTC),
+ connection_metadata={
+ MIGRATION_MARKER_KEY: {"group_key": "s3/eu-west-1/exports", "credential_id": str(credential.pk)}
+ },
+ )
+ assert len({_schema_for(table).source_id for table in tables}) == count
+ return tables
+
+ def test_duplicates_move_into_the_oldest_source_and_are_deleted(self) -> None:
+ tables = self._duplicates()
+ kept = _schema_for(tables[0]).source
+ duplicates = [_schema_for(table).source for table in tables[1:]]
+ moved_schema_ids = [str(_schema_for(table).id) for table in tables[1:]]
+
+ outcomes = merge_duplicate_sources(team_id=self.team.pk, live_run=True)
+
+ assert [(o.source_id, o.kept_source_id, o.action, o.moved_schemas) for o in outcomes] == [
+ (str(source.pk), str(kept.pk), "merged", 1) for source in duplicates
+ ]
+ assert {_schema_for(table).source_id for table in tables} == {kept.pk}
+ for source in duplicates:
+ source.refresh_from_db()
+ assert source.deleted is True
+ assert not ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=source).exists()
+ assert ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=kept).exists()
+ assert sorted(call.args[0] for call in self.delete_schedule.call_args_list) == sorted(moved_schema_ids)
+ assert {self._status(table).source_id for table in tables} == {str(kept.pk)}
+ assert {self._status(table).state for table in tables} == {TableMigrationState.AWAITING_SYNC}
+ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []
+ added = self._table("late", url="https://exports.s3.eu-west-1.amazonaws.com/late/*.csv")
+ self._migrate(table_ids=(str(added.id),))
+ assert _schema_for(added).source_id == kept.pk
+
+ def test_a_dry_run_reports_the_merge_and_changes_nothing(self) -> None:
+ tables = self._duplicates(count=2)
+ before = {table.pk: _schema_for(table).source_id for table in tables}
+ out = StringIO()
+
+ call_command("migrate_self_managed_tables", team_id=self.team.pk, merge_sources=True, stdout=out)
+
+ duplicate_id = before[tables[1].pk]
+ lines = out.getvalue().splitlines()
+ assert lines[0] == (
+ f"source={duplicate_id} team={self.team.pk} kept={before[tables[0].pk]} would_merge schemas=1"
+ )
+ assert "1 duplicate source(s), 1 schema(s): 1 would_merge" in out.getvalue()
+ assert {table.pk: _schema_for(table).source_id for table in tables} == before
+ assert ExternalDataSource.objects.get(pk=duplicate_id).deleted is False
+ self.delete_schedule.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("syncing_on", "syncing is on for 'table_1'"),
+ ("job_ran", "a sync already ran for the source"),
+ ("table_adopted", "the table 'table_1' is already adopted by a source"),
+ ("name_taken_in_kept_source", "the kept source already has a table named 'table_1'"),
+ ("source_has_rules", "the kept or the duplicate source has its own access rules"),
+ ("other_destinations", "the kept and the duplicate source send data to different destinations"),
+ ("inputs_edited", "the kept and the duplicate source connect with different settings"),
+ ]
+ )
+ def test_a_duplicate_that_is_not_safe_to_move_is_skipped(self, case: str, reason: str) -> None:
+ tables = self._duplicates()
+ kept = _schema_for(tables[0]).source
+ unsafe = _schema_for(tables[1])
+ safe_source_id = _schema_for(tables[2]).source_id
+ if case == "syncing_on":
+ ExternalDataSchema.objects.filter(pk=unsafe.pk).update(should_sync=True)
+ elif case == "job_ran":
+ self._job(tables[1], ExternalDataJobStatus.FAILED)
+ elif case == "table_adopted":
+ DataWarehouseTable.raw_objects.filter(pk=tables[1].pk).update(external_data_source=unsafe.source)
+ elif case == "name_taken_in_kept_source":
+ ExternalDataSchema.objects.create(name="table_1", team=self.team, source=kept)
+ elif case == "source_has_rules":
+ AccessControl.objects.create(
+ team=self.team, resource="external_data_source", resource_id=str(unsafe.source_id), access_level="none"
+ )
+ elif case == "other_destinations":
+ ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source_id=unsafe.source_id).delete()
+ else:
+ source = unsafe.source
+ SourceInputsStore.save(source, {**SourceInputsStore.load(source), "file_format": "csv"})
+
+ outcomes = {o.source_id: o for o in merge_duplicate_sources(team_id=self.team.pk, live_run=True)}
+
+ skipped = outcomes[str(unsafe.source_id)]
+ assert (skipped.action, skipped.reason, skipped.moved_schemas) == ("skipped", reason, 0)
+ assert ExternalDataSchema.objects.get(pk=unsafe.pk).source_id == unsafe.source_id
+ assert ExternalDataSource.objects.get(pk=unsafe.source_id).deleted is False
+ assert outcomes[str(safe_source_id)].action == "merged"
+ assert _schema_for(tables[2]).source_id == kept.pk
+
+ def test_sources_with_another_key_or_bucket_stay_apart(self) -> None:
+ self._duplicates(count=1)
+ self._table("other_key", credential=self._credential("AKIAFIRST", "b3RoZXI="))
+ self._table("other_bucket", url="https://archive.s3.eu-west-1.amazonaws.com/a.csv")
+ self._migrate()
+
+ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []
+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 3
+
+
class TestMigrateSelfManagedTablesCommand(BaseTest):
@parameterized.expand(
[
("zero_limit", {"limit": 0}, "--limit"),
("status_and_abandon", {"status": True, "abandon": "x"}, "only one"),
+ ("status_and_merge", {"status": True, "merge_sources": True}, "only one"),
+ ("abandon_and_merge", {"abandon": "x", "merge_sources": True}, "only one"),
("empty_abandon", {"abandon": " , "}, "at least one table id"),
]
)
@@ -823,6 +1024,14 @@ class TestUploadedTableMigration(_MigrationTest):
assert schema_metadata is not None
assert schema_metadata["csv"]["column_types"] == {"day": stored}
+ def test_uploaded_tables_sources_never_merge(self) -> None:
+ self._uploaded("orders", "managed/team_{team}/orders.csv")
+ self._uploaded("refunds", "managed/team_{team}/refunds.csv")
+ self._migrate()
+
+ assert merge_duplicate_sources(team_id=self.team.pk, live_run=True) == []
+ assert ExternalDataSource.objects.filter(team=self.team, deleted=False).count() == 2
+
def test_teams_without_the_flag_keep_their_uploaded_tables(self) -> None:
self.flag_missing.return_value = True
self._uploaded("orders", "file_uploads/team_{team}/{upload}/orders.csv")