:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): mirror deleted and rewritten files in incremental object storage syncs (#121282)
Tom Owers9d441be844fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.pyindex 3c42e62ca32f..9f4b2d801d0f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py@@ -6,6 +6,7 @@ from pathlib import Pathfrom typing import Any, castimport pytest+from unittest.mock import patchimport pyarrow as paimport structlog@@ -13,8 +14,10 @@ from deltalake import write_deltalakefrom fsspec import AbstractFileSystemfrom fsspec.implementations.local import LocalFileSystem+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import validate_incremental_syncfrom products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Sourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_basefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (ObjectStoreConfig,StoreConfig,@@ -27,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStorefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputs+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (AmazonS3SourceConfig,)@@ -206,16 +210,31 @@ def test_validate_credentials(@pytest.mark.parametrize(- "sync_type, last_value, expected_files, expected_keys",+ "sync_type, last_value, expected_files, expected_keys, mirrors",[- (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None),- (ExternalDataSchemaSyncType.INCREMENTAL, None, ["c.csv", "b.csv", "a.csv"], ["_file_name", "_file_row_number"]),+ (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None, False),+ (+ ExternalDataSchemaSyncType.INCREMENTAL,+ None,+ ["c.csv", "b.csv", "a.csv"],+ ["_file_name", "_file_row_number"],+ False,+ ),# The file at the watermark is read again, and the merge key replaces its rows.(ExternalDataSchemaSyncType.INCREMENTAL,FIRST_MODIFIED_AT + timedelta(minutes=1),["b.csv", "a.csv"],["_file_name", "_file_row_number"],+ False,+ ),+ # With the flag the run replaces the rows of the files it reads, so it needs no key.+ (+ ExternalDataSchemaSyncType.INCREMENTAL,+ FIRST_MODIFIED_AT + timedelta(minutes=1),+ ["b.csv", "a.csv"],+ None,+ True,),# The pipeline can hand the watermark back without a timezone.(@@ -223,10 +242,13 @@ def test_validate_credentials((FIRST_MODIFIED_AT + timedelta(minutes=2)).replace(tzinfo=None),["a.csv"],["_file_name", "_file_row_number"],+ False,),# An append has no key, so the file at the watermark must not be read again.- (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None),- (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None),+ (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, False),+ (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None, False),+ # The flag does not change an append.+ (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, True),],)def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(@@ -235,6 +257,7 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(last_value: datetime | None,expected_files: list[str],expected_keys: list[str] | None,+ mirrors: bool,) -> None:# Path order is the reverse of age order, so the test tells the two apart.for minutes, name in enumerate(["c.csv", "b.csv", "a.csv"]):@@ -251,14 +274,65 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge()inputs.schema_metadata = schema.schema_metadatainputs.sync_type = sync_type+ inputs.source_cursor = SourceCursorManager.from_sync_type_config(source, {}, structlog.get_logger())- response = source.source_for_pipeline(config, inputs)- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]+ with patch.object(source_base, "_mirrors_files", return_value=mirrors):+ response = source.source_for_pipeline(config, inputs)+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]assert [(row["_file_name"], row["_file_row_number"]) for row in rows] == [(f"orders/{name}", row_number) for name in expected_files for row_number in (0, 1)]assert response.primary_keys == expected_keys+ mirrored = mirrors and sync_type == ExternalDataSchemaSyncType.INCREMENTAL+ assert response.replace_scope == (+ ReplaceScope(+ column="_file_name",+ replaced=frozenset(f"orders/{name}" for name in expected_files),+ retained=frozenset({"orders/a.csv", "orders/b.csv", "orders/c.csv"}),+ )+ if mirrored+ else None+ )+++def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path: Path) -> None:+ for minutes, name in enumerate(["b.csv", "a.csv"]):+ write(tmp_path, f"exports/orders/{name}", b"id\n1\n", FIRST_MODIFIED_AT + timedelta(minutes=minutes))+ source = _LocalAmazonS3Source(tmp_path)+ config = make_config()+ [schema] = source.get_schemas(config, team_id=1)++ def sync(stored: dict[str, Any]) -> tuple[SourceResponse, int, SourceCursorManager]:+ inputs = source_inputs(+ schema.name,+ incremental_field=schema.incremental_fields[0]["field"],+ incremental_field_type=IncrementalFieldType.DateTime,+ db_incremental_field_last_value=FIRST_MODIFIED_AT + timedelta(minutes=1),+ )+ inputs.schema_metadata = schema.schema_metadata+ inputs.sync_type = ExternalDataSchemaSyncType.INCREMENTAL+ cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())+ inputs.source_cursor = cursor+ with patch.object(source_base, "_mirrors_files", return_value=True):+ response = source.source_for_pipeline(config, inputs)+ rows = sum(table.num_rows for table in cast(Iterable[pa.Table], response.items()))+ return response, rows, cursor++ first, first_rows, cursor = sync({})+ unchanged, unchanged_rows, _ = sync({"source_cursor": cursor.staged_payload()})+ write(tmp_path, "exports/orders/b.csv", b"id\n1\n2\n", FIRST_MODIFIED_AT + timedelta(minutes=2))+ rewritten, rewritten_rows, _ = sync({"source_cursor": cursor.staged_payload()})++ # The file at the watermark is read again on every run, so only the stored listing can tell+ # that nothing changed.+ assert (first.replace_scope is not None, first_rows) == (True, 1)+ assert (unchanged.replace_scope, unchanged_rows) == (None, 0)+ validate_incremental_sync(True, unchanged, is_first_sync=False)+ assert (rewritten.replace_scope is not None and rewritten.replace_scope.replaced, rewritten_rows) == (+ frozenset({"orders/a.csv", "orders/b.csv"}),+ 3,+ )@pytest.mark.parametrize(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pyindex 586f8d8be9f9..cfb372f75734 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -43,13 +43,15 @@ DELTA_VERSION_COLUMN = "_delta_version"@frozenclass ObjectStorageDeltaCursor:- """The Delta table version the destination holds."""+ """What the destination holds: a version of a Delta table, or a listing of a files table."""cursor_kind: ClassVar[str] = "object_storage_delta_v1"- version: int+ version: int | None = None# Delta gives a table a new ID when it is dropped and created again, which restarts its versions.- table_id: str+ table_id: str | None = None+ # The digest of a files table's listing (see `listing_digest`). None for a Delta table.+ files_digest: str | None = None@contextmanager@@ -153,12 +155,13 @@ def plan_delta_read("""with _delta_errors(table_name):table = DeltaTable(uri, storage_options=storage_options)- cursor = ObjectStorageDeltaCursor(version=table.version(), table_id=str(table.metadata().id))+ version = table.version()+ cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id))dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())if logger is not None:logger.info("Reading Delta table snapshot", table=table_name, delta_version=cursor.version)- if stored is None:+ if stored is None or stored.version is None:return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor)if stored.table_id == cursor.table_id and stored.version == cursor.version:return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=())@@ -167,7 +170,7 @@ def plan_delta_read(added_files: list[ds.FileFragment] = []if stored.table_id != cursor.table_id:reset_reason = "the table was created again"- elif stored.version > cursor.version:+ elif stored.version > version:reset_reason = "the table is at an earlier version than the last sync"else:with _delta_errors(table_name):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.pyindex 6a86e21ef8bf..35f72152fa63 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py@@ -1,4 +1,5 @@-from collections.abc import Iterable+import hashlib+from collections.abc import Iterable, Sequencefrom datetime import UTC, datetime, timedeltafrom typing import Any, Literal@@ -9,7 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (APPEND_SETTLE_SECONDS,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceInputsfrom products.warehouse_sources.backend.types import ExternalDataSchemaSyncTypeFILE_ROW_NUMBER_COLUMN = "_file_row_number"@@ -69,3 +70,45 @@ class FileSelection:READ_ALL = FileSelection()+++def listing_digest(files: Iterable[StoredObject]) -> str:+ """A 64-character digest of the path, size and modification time of each file."""+ digest = hashlib.sha256()+ for stored in sorted(files, key=lambda stored: stored.path):+ digest.update(f"{stored.path}\t{stored.size}\t{stored.modified_at.isoformat()}\n".encode())+ return digest.hexdigest()+++@frozen+class MirrorPlan:+ """One incremental read that makes a files table match the bucket.++ The changed files are read again and replace their own rows, and the rows of a file that is no+ longer in the bucket go (see `ReplaceScope`).+ """++ files: list[StoredObject]+ """The files to read, oldest first."""+ replace_scope: ReplaceScope | None+ """None when nothing changed since the last sync, so the run reads and deletes nothing."""+ digest: str+ """The digest of the listing that this read brings the table up to."""+++def plan_mirror(+ listed: Sequence[StoredObject], selection: FileSelection, stored_digest: str | None, now: datetime | None = None+) -> MirrorPlan:+ """Plan the read from the current listing and the digest of the listing at the last sync."""+ digest = listing_digest(listed)+ if digest == stored_digest:+ # The files at the watermark are read again on every run, so the watermark alone can never+ # tell that a listing has not changed.+ return MirrorPlan(files=[], replace_scope=None, digest=digest)+ files = selection.select(listed, now=now)+ scope = ReplaceScope(+ column=FILE_PATH_COLUMN,+ replaced=frozenset(stored.path for stored in files),+ retained=frozenset(stored.path for stored in listed),+ )+ return MirrorPlan(files=files, replace_scope=scope, digest=digest)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 684ac2d114d8..d58802c78a0f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -7,12 +7,18 @@ import pyarrow as pafrom botocore.exceptions import ClientErrorfrom structlog.types import FilteringBoundLogger+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import (+ REPLACE_ROWS_BY_FILE_FLAG,+ is_schema_flag_enabled,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, SimpleSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import CursorSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_MODIFIED_AT_COLUMNfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (DiscoveryOptions,+ FileFormat,StoreConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (@@ -43,6 +49,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (ObjectStore,+ StoredObject,open_store,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (@@ -52,6 +59,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objFILE_PRIMARY_KEYS,READ_ALL,FileSelection,+ plan_mirror,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (CHUNK_SIZE_BYTES,@@ -97,9 +105,12 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe"""Imports the files a bucket URL selects: CSV, JSON Lines and Parquet files, and Delta tables.A full refresh reads each table in full, so the table matches the bucket at the time of the sync.- The incremental and append sync types read only the files modified since the last sync. They- never remove rows, so the rows of a file deleted from the bucket stay in the table. An append sync- of a Delta table reads only the data files committed since the version of the last sync.+ The incremental and append sync types read only the files modified since the last sync. An append+ never removes rows, so the rows of a file deleted from the bucket stay in the table. An incremental+ sync with `REPLACE_ROWS_BY_FILE_FLAG` on also deletes the rows of a deleted file, and a changed file+ replaces its rows. Without the flag it merges on the file and row number and keeps the rows of+ deleted files. An append sync of a Delta table reads only the data files committed since the+ version of the last sync."""@abstractmethod@@ -210,6 +221,8 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDeif table.kind == "delta":return self._delta_response(store, table, inputs)selection = FileSelection.from_inputs(inputs)+ if selection.merges and _mirrors_files(inputs):+ return self._mirrored_files_response(store, table, inputs, selection)def items() -> Iterator[pa.Table]:yield from iter_table(store, table, inputs.logger, selection)@@ -222,6 +235,37 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDechunk_size_bytes=CHUNK_SIZE_BYTES,)+ def _mirrored_files_response(+ self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs, selection: FileSelection+ ) -> SourceResponse:+ """An incremental read that also deletes the rows of deleted files and of rewritten files.++ The listing is made here and not in `items`, because the response has to name the files+ whose rows the run replaces before the first batch loads.+ """+ cursor_manager = self.get_cursor_manager(inputs)+ stored = cursor_manager.load()+ # Taken before listing, as in `iter_table`.+ listing_started_at = datetime.now(UTC)+ resolved = _resolve_files(store, table, inputs.logger)+ listed = [stored_file for stored_file in resolved.files if _file_format(resolved, stored_file) is not None]+ plan = plan_mirror(+ listed, selection, stored.files_digest if stored is not None else None, now=listing_started_at+ )++ def items() -> Iterator[pa.Table]:+ cursor_manager.stage(ObjectStorageDeltaCursor(files_digest=plan.digest))+ yield from _iter_files(store, resolved, plan.files, inputs.logger)++ return SourceResponse(+ name=inputs.schema_name,+ items=items,+ # A run with no scope reads no rows, but the pipeline still needs a key or a scope for an incremental run.+ primary_keys=FILE_PRIMARY_KEYS if plan.replace_scope is None else None,+ chunk_size_bytes=CHUNK_SIZE_BYTES,+ replace_scope=plan.replace_scope,+ )+def _delta_response(self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs) -> SourceResponse:cursor_manager = self.get_cursor_manager(inputs)appends = inputs.should_use_incremental_field@@ -294,12 +338,26 @@ def iter_table(# Taken before listing: a file that lands during a slow listing can share a timestamp with a listed# file, so the append settle window has to count from before the listing started.listing_started_at = datetime.now(UTC)+ resolved = _resolve_files(store, table, logger)+ yield from _iter_files(store, resolved, selection.select(resolved.files, now=listing_started_at), logger)+++def _resolve_files(store: ObjectStore, table: TableDefinition, logger: FilteringBoundLogger) -> TableDefinition:try:- resolved = resolve_table_files(store, table, logger=logger)+ return resolve_table_files(store, table, logger=logger)except Exception as e:_raise_translated(e, store.root)- for stored in selection.select(resolved.files, now=listing_started_at):- file_format = resolve_file_format(stored.path, resolved.file_format)+++def _file_format(resolved: TableDefinition, stored: StoredObject) -> FileFormat | None:+ return resolve_file_format(stored.path, resolved.file_format)+++def _iter_files(+ store: ObjectStore, resolved: TableDefinition, files: list[StoredObject], logger: FilteringBoundLogger+) -> Iterator[pa.Table]:+ for stored in files:+ file_format = _file_format(resolved, stored)if file_format is None:continuetry:@@ -307,3 +365,11 @@ def iter_table(yield from iter_file_tables(stream, stored, file_format, resolved.csv, logger)except PermissionError as e:_raise_translated(e, store.root)+++def _mirrors_files(inputs: SourceInputs) -> bool:+ """Whether an incremental read deletes the rows of deleted and rewritten files (see `ReplaceScope`)."""+ schema = (+ ExternalDataSchema.objects.filter(team_id=inputs.team_id, id=inputs.schema_id).select_related("source").first()+ )+ return schema is not None and is_schema_flag_enabled(schema, REPLACE_ROWS_BY_FILE_FLAG)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex f6bb421cf9fc..4ef4d5f00e87 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -66,6 +66,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.incremental import (FileSelection,+ listing_digest,+ plan_mirror,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (JSONL_CHUNK_ROWS,@@ -76,6 +78,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objiter_table,validate_store_location,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScopeMODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)LOGGER = structlog.get_logger()@@ -819,6 +822,53 @@ class TestFileSelection:assert {row["_file_name"] for row in rows} == {"settled.csv"}+ @parameterized.expand(+ [+ ("an_unchanged_listing_reads_and_deletes_nothing", [("a.csv", 0, 10), ("b.csv", 1, 10)], None),+ (+ "a_deleted_file_loses_its_rows",+ [("b.csv", 1, 10)],+ (["b.csv"], {"b.csv"}, {"b.csv"}),+ ),+ (+ "a_shrunk_file_replaces_its_rows",+ [("a.csv", 0, 10), ("b.csv", 2, 4)],+ (["b.csv"], {"b.csv"}, {"a.csv", "b.csv"}),+ ),+ (+ "a_renamed_file_moves_its_rows",+ [("renamed.csv", 2, 10), ("b.csv", 1, 10)],+ (["b.csv", "renamed.csv"], {"b.csv", "renamed.csv"}, {"b.csv", "renamed.csv"}),+ ),+ ]+ )+ def test_a_mirrored_read_replaces_the_changed_files_and_drops_the_deleted_ones(+ self,+ _name: str,+ listing: list[tuple[str, int, int]],+ expected: tuple[list[str], set[str], set[str]] | None,+ ) -> None:+ def files(entries: list[tuple[str, int, int]]) -> list[StoredObject]:+ return [+ StoredObject(path=path, size=size, modified_at=MODIFIED_AT + timedelta(minutes=minutes))+ for path, minutes, size in entries+ ]++ last_sync = listing_digest(files([("a.csv", 0, 10), ("b.csv", 1, 10)]))+ selection = FileSelection(mode="changed", watermark=MODIFIED_AT + timedelta(minutes=1))++ plan = plan_mirror(files(listing), selection, last_sync)++ assert len(plan.digest) == 64+ if expected is None:+ assert (plan.files, plan.replace_scope) == ([], None)+ return+ read, replaced, retained = expected+ assert [stored.path for stored in plan.files] == read+ assert plan.replace_scope == ReplaceScope(+ column="_file_name", replaced=frozenset(replaced), retained=frozenset(retained)+ )+def test_row_numbers_continue_across_the_chunks_of_one_file(self) -> None:rows = JSONL_CHUNK_ROWS + 3content = b"".join(b'{"n":%d}\n' % number for number in range(rows))