:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): sync delta tables incrementally by data file (#121283)
Tom Owers33b760ea24af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex a336b90553e0..6be1ab90b5dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -1054,10 +1054,12 @@ class PipelineV3(Generic[ResumableData]):pa_table = normalize_table_column_names(pa_table)if self._uses_delta_write_column_selection:+ # The loader deletes rows by the replace scope's column, so a column selection must keep it.+ replace_scope = self._resource.replace_scopepa_table = await observe_and_project_table(pa_table,self._schema.enabled_columns,- self._resource.primary_keys,+ [*(self._resource.primary_keys or []), *([replace_scope.column] if replace_scope else [])],self._schema.incremental_field,[*(self._schema.partitioning_keys_override or []),diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex fe5537f59fd2..0c855e24726d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1008,6 +1008,22 @@ class TestLaneFanOut:for writer in writers:cast(MagicMock, writer.pg_producer.hold_batch).assert_called_once()+ async def test_a_column_selection_keeps_the_replace_scope_column(self) -> None:+ writer = _lane_writer("orders")+ pipeline = self._pipeline([writer])+ pipeline._uses_delta_write_column_selection = True+ cast(MagicMock, pipeline._schema).configure_mock(enabled_columns=["name"])+ pipeline._resource.primary_keys = None+ pipeline._resource.replace_scope = ReplaceScope(+ column="_file_name", replaced=frozenset({"a.csv"}), retained=frozenset({"a.csv"})+ )++ await self._process(pipeline, pa.table({"id": [1], "name": ["x"], "_file_name": ["a.csv"]}))++ [written] = [call.args[0] for call in cast(MagicMock, writer.s3_batch_writer.write_batch).call_args_list]+ assert {"name", "_file_name"} <= set(written.column_names)+ assert "id" not in written.column_names+async def test_only_the_billable_lane_counts_towards_usage(self) -> None:# One read of a change stream is one sync however many tables it keeps.writers = [_lane_writer("users"), _lane_writer("users_cdc", billable=False)]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.pyindex 9f4b2d801d0f..f0de6a2b06b7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py@@ -336,49 +336,57 @@ def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path:@pytest.mark.parametrize(- "sync_type, has_stored_cursor, expected_ids, expected_columns",+ "sync_type, has_stored_cursor, mirrors, expected_ids, expected_columns, expected_reset",[- (ExternalDataSchemaSyncType.APPEND, True, [2], {"id", "_delta_version"}),- (ExternalDataSchemaSyncType.APPEND, False, [1, 2], {"id", "_delta_version"}),+ (ExternalDataSchemaSyncType.APPEND, True, False, [2], {"id", "_delta_version"}, False),+ (ExternalDataSchemaSyncType.APPEND, False, False, [1, 2], {"id", "_delta_version"}, False),# A full refresh ignores the stored cursor, and still records the version it wrote.- (ExternalDataSchemaSyncType.FULL_REFRESH, True, [1, 2], {"id"}),+ (ExternalDataSchemaSyncType.FULL_REFRESH, True, False, [1, 2], {"id"}, False),+ # An incremental sync names each row's data file, so the loader can replace rows by file.+ (ExternalDataSchemaSyncType.INCREMENTAL, True, True, [2], {"id", "_delta_version", "_file_name"}, False),+ # Without the flag the loader can't delete by file, so the run rebuilds the table.+ (ExternalDataSchemaSyncType.INCREMENTAL, True, False, [1, 2], {"id", "_delta_version", "_file_name"}, True),],)def test_delta_sync_reads_from_the_stored_cursor_and_stages_the_next_one(tmp_path: Path,sync_type: ExternalDataSchemaSyncType,has_stored_cursor: bool,+ mirrors: bool,expected_ids: list[int],expected_columns: set[str],+ expected_reset: bool,) -> None:table_path = str(tmp_path / "customer-data" / "exports" / "sales")write_deltalake(table_path, pa.table({"id": [1]}))source = _LocalAmazonS3Source(tmp_path)config = make_config()[schema] = source.get_schemas(config, team_id=1)- first = source_inputs(schema.name)- first.schema_metadata = schema.schema_metadata- first_cursor = SourceCursorManager.from_sync_type_config(source, {}, structlog.get_logger())- first.source_cursor = first_cursor- list(cast(Iterable[pa.Table], source.source_for_pipeline(config, first).items()))+ reads_versions = sync_type != ExternalDataSchemaSyncType.FULL_REFRESH++ def sync(stored: dict[str, Any]) -> tuple[SourceResponse, list[dict[str, Any]], SourceCursorManager]:+ inputs = source_inputs(+ schema.name,+ incremental_field=schema.incremental_fields[0]["field"] if reads_versions else None,+ incremental_field_type=IncrementalFieldType.Integer if reads_versions else None,+ )+ inputs.schema_metadata = schema.schema_metadata+ inputs.sync_type = sync_type+ cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())+ inputs.source_cursor = cursor+ with patch.object(source_base, "_mirrors_files", return_value=mirrors):+ response = source.source_for_pipeline(config, inputs)+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]+ return response, rows, cursor++ _, _, first_cursor = sync({})stored = {"source_cursor": first_cursor.staged_payload()} if has_stored_cursor else {}write_deltalake(table_path, pa.table({"id": [2]}), mode="append")-- appends = sync_type == ExternalDataSchemaSyncType.APPEND- inputs = source_inputs(- schema.name,- incremental_field=schema.incremental_fields[0]["field"] if appends else None,- incremental_field_type=IncrementalFieldType.Integer if appends else None,- )- inputs.schema_metadata = schema.schema_metadata- inputs.sync_type = sync_type- cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())- inputs.source_cursor = cursor- response = source.source_for_pipeline(config, inputs)- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]+ response, rows, cursor = sync(stored)assert sorted(row["id"] for row in rows) == expected_idsassert all(set(row) == expected_columns for row in rows)- assert response.destination_reset_required is False+ assert response.destination_reset_required is expected_reset+ assert (response.replace_scope is not None) == (sync_type == ExternalDataSchemaSyncType.INCREMENTAL)staged = cursor.staged_payload()assert staged is not None and staged["data"]["version"] == 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pyindex cfb372f75734..ff92fe7aff30 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -12,6 +12,7 @@ from structlog.types import FilteringBoundLoggerfrom posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_PATH_COLUMNfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (ACCESS_DENIED_ERROR,DELTA_UNSUPPORTED_ERROR,@@ -22,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (DELTA_BATCH_ROWS,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope# Whole words only, so a status code doesn't match inside an object key such as `sales403/part.parquet`._ACCESS_DENIED_MARKERS = re.compile(r"\b(403|AccessDenied|Forbidden|InvalidAccessKeyId|SignatureDoesNotMatch)\b")@@ -52,6 +54,8 @@ class ObjectStorageDeltaCursor:table_id: str | None = None# The digest of a files table's listing (see `listing_digest`). None for a Delta table.files_digest: str | None = None+ # Whether each row the destination holds names its data file, which a read by file needs.+ file_names: bool = False@contextmanager@@ -85,24 +89,33 @@ class DeltaRead:added_files: tuple[ds.FileFragment, ...] | None = dataclasses.field(default=None, repr=False)# The stored cursor can't be extended by appending, so the destination must be rebuilt.reset_required: bool = False+ # Set for a read by data file: the rows of the files it reads, and of the files that are gone, go+ # before the read loads. None when nothing changed.+ replace_scope: ReplaceScope | None = dataclasses.field(default=None, repr=False)- def iter_tables(self, batch_rows: int = DELTA_BATCH_ROWS, with_version: bool = False) -> Iterator[pa.Table]:- """Stream the planned rows. The scan reads one fragment at a time on one thread to bound memory."""- dataset: ds.Dataset = self.dataset- if self.added_files is not None:- if not self.added_files:- return- # Built from the snapshot's own fragments, so partition columns come back as in a full read.- dataset = ds.FileSystemDataset(- list(self.added_files), self.dataset.schema, self.dataset.format, self.dataset.filesystem- )- scanner = dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)+ def iter_tables(+ self, batch_rows: int = DELTA_BATCH_ROWS, with_version: bool = False, with_file_names: bool = False+ ) -> Iterator[pa.Table]:+ """Stream the planned rows. The scan reads one fragment at a time on one thread to bound memory.++ `with_file_names` adds the path of each row's data file, which a replace scope names rows by.+ """+ fragments = list(self.added_files) if self.added_files is not None else None+ if fragments is not None and not fragments:+ return+ scans: list[tuple[str | None, list[ds.FileFragment] | None]]+ if with_file_names:+ # One scan per file, because a scan does not say which file a batch came from.+ scans = [(fragment.path, [fragment]) for fragment in fragments or self.dataset.get_fragments()]+ else:+ scans = [(None, fragments)]yielded = Falsetry:- for batch in scanner.to_batches():- if batch.num_rows:- yielded = True- yield self._with_version(pa.Table.from_batches([batch]), with_version)+ for file_name, scan_fragments in scans:+ for batch in self._scanner(scan_fragments, batch_rows).to_batches():+ if batch.num_rows:+ yielded = True+ yield self._tagged(pa.Table.from_batches([batch]), with_version, with_file_names, file_name)except pa.ArrowInvalid as e:raise ObjectStorageError(f"{FILE_PARSE_ERROR} a data file of the Delta table '{self.table_name}': {e}"@@ -114,16 +127,30 @@ class DeltaRead:if not yielded and self.added_files is None:# An empty snapshot still has to reach the pipeline as a schema, or the table is dropped and# never recreated.- yield self._with_version(self.dataset.schema.empty_table(), with_version)-- def _with_version(self, table: pa.Table, with_version: bool) -> pa.Table:- if not with_version:- return table- if DELTA_VERSION_COLUMN in table.column_names:- table = table.drop_columns([DELTA_VERSION_COLUMN])- return table.append_column(- DELTA_VERSION_COLUMN, pa.repeat(pa.scalar(self.cursor.version, pa.int64()), table.num_rows)- )+ yield self._tagged(self.dataset.schema.empty_table(), with_version, with_file_names, None)++ def _scanner(self, fragments: list[ds.FileFragment] | None, batch_rows: int) -> ds.Scanner:+ dataset: ds.Dataset = self.dataset+ if fragments is not None:+ # Built from the snapshot's own fragments, so partition columns come back as in a full read.+ dataset = ds.FileSystemDataset(+ cast(list[ds.Fragment], fragments), self.dataset.schema, self.dataset.format, self.dataset.filesystem+ )+ return dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)++ def _tagged(self, table: pa.Table, with_version: bool, with_file_names: bool, file_name: str | None) -> pa.Table:+ columns: list[tuple[str, pa.Array]] = []+ if with_version:+ columns.append(+ (DELTA_VERSION_COLUMN, pa.repeat(pa.scalar(self.cursor.version, pa.int64()), table.num_rows))+ )+ if with_file_names:+ columns.append((FILE_PATH_COLUMN, pa.repeat(pa.scalar(file_name, pa.string()), table.num_rows)))+ for name, values in columns:+ if name in table.column_names:+ table = table.drop_columns([name])+ table = table.append_column(name, values)+ return tabledef _files_at_version(uri: str, storage_options: dict[str, str], version: int) -> set[str] | None:@@ -144,41 +171,65 @@ def plan_delta_read(table_name: str,stored: ObjectStorageDeltaCursor | None = None,logger: FilteringBoundLogger | None = None,+ by_file: bool = False,) -> DeltaRead:"""Pin the table's current version and decide what to read to bring the destination up to it.Without a stored cursor the whole snapshot is read. With one, only the data files added since- the cursor's version are read, and only when every file of that version is still in the table.- A file that is gone was removed by a delete, an update, a merge, an overwrite or a compaction,- all of which change or move rows the destination already holds. No append can repair that, so- the plan then asks for a rebuild from the whole snapshot.+ the cursor's version are read. A file that is gone was removed by a delete, an update, a merge,+ an overwrite or a compaction, all of which change or move rows the destination already holds.++ An append can't repair that, so the plan then asks for a rebuild from the whole snapshot. A read+ `by_file` can: each row names its data file, and the plan's replace scope deletes the rows of+ the files that are gone and of the files it reads. Such a read still rebuilds when the table was+ created again, when it is at an earlier version, when the stored version can't be read, and when+ the stored rows don't name their files."""with _delta_errors(table_name):table = DeltaTable(uri, storage_options=storage_options)version = table.version()- cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id))+ cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id), file_names=by_file)dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())if logger is not None:logger.info("Reading Delta table snapshot", table=table_name, delta_version=cursor.version)++ def planned(added: list[ds.FileFragment] | None, *, reset: bool = False) -> DeltaRead:+ scope: ReplaceScope | None = None+ # Also when nothing changed: a keyless incremental run without a scope fails before it reads.+ if by_file:+ current = frozenset(fragment.path for fragment in dataset.get_fragments())+ replaced = current if added is None else frozenset(fragment.path for fragment in added)+ scope = ReplaceScope(column=FILE_PATH_COLUMN, replaced=replaced, retained=current)+ return DeltaRead(+ table_name=table_name,+ dataset=dataset,+ cursor=cursor,+ added_files=None if added is None else tuple(added),+ reset_required=reset,+ replace_scope=scope,+ )+if stored is None or stored.version is None:- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor)- if stored.table_id == cursor.table_id and stored.version == cursor.version:- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=())+ # Rows from a sync that did not name their files can't be deleted by file, so they go with a rebuild.+ return planned(None, reset=by_file)reset_reason: str | None = Noneadded_files: list[ds.FileFragment] = []- if stored.table_id != cursor.table_id:+ if by_file and not stored.file_names:+ reset_reason = "the rows of the last sync do not name their data files"+ elif stored.table_id != cursor.table_id:reset_reason = "the table was created again"elif stored.version > version:reset_reason = "the table is at an earlier version than the last sync"- else:+ elif stored.version < version:with _delta_errors(table_name):earlier_files = _files_at_version(uri, storage_options, stored.version)fragments: list[ds.FileFragment] = list(dataset.get_fragments())+ current_files = {fragment.path for fragment in fragments}if earlier_files is None:reset_reason = "the version of the last sync can no longer be read"- elif not earlier_files <= {fragment.path for fragment in fragments}:+ elif not earlier_files <= current_files and not by_file:reset_reason = "a commit since the last sync removed or rewrote data files"else:added_files = [fragment for fragment in fragments if fragment.path not in earlier_files]@@ -192,8 +243,8 @@ def plan_delta_read(stored_version=stored.version,delta_version=cursor.version,)- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, reset_required=True)- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=tuple(added_files))+ return planned(None, reset=True)+ return planned(added_files)def iter_delta_tables(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.pyindex 3a5c4a265148..a333827863fa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py@@ -54,8 +54,9 @@ def source_caption(store: str) -> str:return (f"Import CSV, JSON Lines and Parquet files, and Delta Lake tables, from {store}. PostHog creates one ""table per folder, one per file directly in the folder the URL points to, and one per Delta table. A full "- "refresh reads every file again, so the table matches the files. An incremental or append-only sync "- "reads only new and changed files, and keeps the rows of files you delete."+ "refresh reads every file again, so the table matches the files. An incremental sync reads only new "+ "and changed files, and removes the rows of files you delete. For a Delta table, it also applies "+ "deleted and updated rows. An append-only sync reads only new files and keeps every row it imported.")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 565bdf28d8a4..56acc9068c71 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -77,7 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sch)from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_messagefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse-from products.warehouse_sources.backend.types import IncrementalFieldType+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType, IncrementalFieldTypedef _raise_translated(error: Exception, location: str) -> NoReturn:@@ -116,7 +116,9 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDesync with `REPLACE_ROWS_BY_FILE_FLAG` on also deletes the rows of a deleted file, and a changed filereplaces its rows. Without the flag it merges on the file and row number and keeps the rows ofdeleted files. An append sync of a Delta table reads only the data files committed since the- version of the last sync.+ version of the last sync. An incremental sync of a Delta table also replaces the rows of each data+ file that a commit removed, so deletes, updates, merges and compactions reach the table. Without+ the flag it rebuilds the table on every run."""@abstractmethod@@ -305,21 +307,26 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDedef _delta_response(self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs) -> SourceResponse:cursor_manager = self.get_cursor_manager(inputs)appends = inputs.should_use_incremental_field+ by_file = appends and inputs.sync_type == ExternalDataSchemaSyncType.INCREMENTAL+ # Without the flag the loader can't delete by file, so an incremental sync rebuilds the+ # table from the snapshot on every run.+ stored = cursor_manager.load() if appends and (not by_file or _mirrors_files(inputs)) else None# The plan is made here and not in `items`, because the response has to say up front# whether the destination must be rebuilt.read = plan_delta_read(store.delta_uri(table.path),store.delta_storage_options,table.name,- stored=cursor_manager.load() if appends else None,+ stored=stored,logger=inputs.logger,+ by_file=by_file,)def items() -> Iterator[pa.Table]:# A full refresh stages the cursor too. A later switch to append then continues from# the version the table holds, and not from the cursor of an earlier append.cursor_manager.stage(read.cursor)- yield from read.iter_tables(with_version=appends)+ yield from read.iter_tables(with_version=appends, with_file_names=by_file)return SourceResponse(name=inputs.schema_name,@@ -328,6 +335,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDechunk_size_bytes=CHUNK_SIZE_BYTES,destination_reset_required=read.reset_required,preserve_column_names=table.preserve_column_names,+ replace_scope=read.replace_scope,)def _find_table(@@ -347,8 +355,8 @@ def _source_schema(table: TableDefinition) -> SourceSchema:is_files = table.kind == "files"return SourceSchema(name=table.name,- # Delta rows have no key to merge on, so a Delta table can only append.- supports_incremental=is_files,+ # Delta rows have no key to merge on, so an incremental sync of a Delta table replaces rows by data file.+ supports_incremental=True,supports_append=True,incremental_fields=[incremental_field(FILE_MODIFIED_AT_COLUMN)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex ffab85de6375..8b92bdd8e84c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -686,18 +686,60 @@ def _append_then_compact(path: str) -> None:DeltaTable(path).optimize.compact()+def _update(path: str) -> None:+ DeltaTable(path).update(updates={"id": "10"}, predicate="id = 2")+++def _merge(path: str) -> None:+ (+ DeltaTable(path)+ .merge(pa.table({"id": [2, 4], "day": ["b", "c"]}), predicate="t.id = s.id", source_alias="s", target_alias="t")+ .when_matched_update(updates={"id": "s.id + 20"})+ .when_not_matched_insert_all()+ .execute()+ )+++def _files_and_ids(path: str, version: int | None = None) -> list[tuple[str, int]]:+ dataset = DeltaTable(path, version=version).to_pyarrow_dataset()+ return sorted(+ (fragment.path, row_id)+ for fragment in dataset.get_fragments()+ for row_id in fragment.to_table(columns=["id"]).column("id").to_pylist()+ )++class TestDeltaCursor:@parameterized.expand([- ("no_cursor_reads_the_snapshot", None, [_append], False, [1, 2, 3], 1),- ("appends_read_only_the_new_files", "same", [_append], False, [3], 1),- ("an_unchanged_table_reads_nothing", "same", [], False, [], 0),- ("an_overwrite_rebuilds", "same", [_overwrite], True, [9], 1),- ("a_delete_rebuilds", "same", [_delete], True, [2], 1),+ ("no_cursor_reads_the_snapshot", None, [_append], False, False, [1, 2, 3], 1),+ ("appends_read_only_the_new_files", "same", [_append], False, False, [3], 1),+ ("an_unchanged_table_reads_nothing", "same", [], False, False, [], 0),+ ("an_overwrite_rebuilds", "same", [_overwrite], False, True, [9], 1),+ ("a_delete_rebuilds", "same", [_delete], False, True, [2], 1),# Compaction keeps the rows but moves them to a new file, which an append would read twice.- ("a_compaction_rebuilds", "same", [_append_then_compact], True, [1, 2, 3], 2),- ("a_recreated_table_rebuilds", "other", [_append], True, [1, 2, 3], 1),- ("a_cursor_ahead_of_the_table_rebuilds", "ahead", [], True, [1, 2], 0),+ ("a_compaction_rebuilds", "same", [_append_then_compact], False, True, [1, 2, 3], 2),+ ("a_recreated_table_rebuilds", "other", [_append], False, True, [1, 2, 3], 1),+ ("a_cursor_ahead_of_the_table_rebuilds", "ahead", [], False, True, [1, 2], 0),+ # By file, a commit that removes files only replaces the rows of the files it touched.+ ("by_file_an_append_reads_the_new_file", "same", [_append], True, False, [3], 1),+ ("by_file_an_unchanged_table_reads_nothing", "same", [], True, False, [], 0),+ ("by_file_a_delete_reads_nothing_and_drops_the_file", "same", [_delete], True, False, [], 1),+ ("by_file_an_update_reads_the_rewritten_file", "same", [_update], True, False, [10], 1),+ ("by_file_a_merge_reads_the_files_it_wrote", "same", [_merge], True, False, [4, 22], 1),+ ("by_file_an_overwrite_reads_the_new_file", "same", [_overwrite], True, False, [9], 1),+ ("by_file_a_compaction_reads_the_compacted_files", "same", [_append_then_compact], True, False, [2, 3], 2),+ ("by_file_a_recreated_table_rebuilds", "other", [_append], True, True, [1, 2, 3], 1),+ ("by_file_a_cursor_ahead_of_the_table_rebuilds", "ahead", [], True, True, [1, 2], 0),+ (+ "by_file_a_sync_that_named_no_files_rebuilds",+ "same_without_file_names",+ [_append],+ True,+ True,+ [1, 2, 3],+ 1,+ ),])def test_plan(@@ -705,6 +747,7 @@ class TestDeltaCursor:_name: str,stored_cursor: str | None,commits: list[Any],+ by_file: bool,expected_reset: bool,expected_ids: list[int],expected_version: int,@@ -715,33 +758,58 @@ class TestDeltaCursor:table_id = str(DeltaTable(path).metadata().id)stored = {None: None,- "same": ObjectStorageDeltaCursor(version=0, table_id=table_id),- "other": ObjectStorageDeltaCursor(version=0, table_id="another-table"),- "ahead": ObjectStorageDeltaCursor(version=5, table_id=table_id),+ "same": ObjectStorageDeltaCursor(version=0, table_id=table_id, file_names=by_file),+ "same_without_file_names": ObjectStorageDeltaCursor(version=0, table_id=table_id),+ "other": ObjectStorageDeltaCursor(version=0, table_id="another-table", file_names=by_file),+ "ahead": ObjectStorageDeltaCursor(version=5, table_id=table_id, file_names=by_file),}[stored_cursor]for commit in commits:commit(path)- read = plan_delta_read(path, {}, "sales", stored=stored)- rows = [row for table in read.iter_tables(with_version=True) for row in table.to_pylist()]+ read = plan_delta_read(path, {}, "sales", stored=stored, by_file=by_file)+ rows = [+ row+ for table in read.iter_tables(with_version=True, with_file_names=by_file)+ for row in table.to_pylist()+ ]+ if by_file and not expected_reset:+ # The destination after the load: the rows of the last sync, less the rows the scope+ # deletes, plus the rows read. It must match the table's current snapshot exactly.+ held = _files_and_ids(path, version=0)+ kept = [(file, row_id) for file, row_id in held if not self._deleted(read.replace_scope, file)]+ loaded = [(row["_file_name"], row["id"]) for row in rows]+ assert sorted(kept + loaded) == _files_and_ids(path)assert read.reset_required is expected_reset- assert read.cursor == ObjectStorageDeltaCursor(version=expected_version, table_id=table_id)+ assert read.cursor == ObjectStorageDeltaCursor(version=expected_version, table_id=table_id, file_names=by_file)assert sorted(row["id"] for row in rows) == expected_ids# Rows read from a subset of the files keep the partition column and carry the version.- assert all(set(row) == {"id", "day", "_delta_version"} for row in rows)+ assert all(set(row) == {"id", "day", "_delta_version", *(["_file_name"] if by_file else [])} for row in rows)assert {row["_delta_version"] for row in rows} <= {expected_version}+ assert (read.replace_scope is not None) == by_file- def test_a_cleaned_up_version_rebuilds(self, tmp_path: Path) -> None:- path = str(tmp_path / "sales")- write_deltalake(path, pa.table({"id": [1, 2], "day": ["a", "b"]}))- table_id = str(DeltaTable(path).metadata().id)- _append(path)- DeltaTable(path).create_checkpoint()- # Log cleanup removes the entries before the checkpoint, so version 0 can no longer be loaded.- (Path(path) / "_delta_log" / f"{0:020d}.json").unlink()+ @staticmethod+ def _deleted(scope: ReplaceScope | None, file: str) -> bool:+ return scope is not None and (file in scope.replaced or file not in scope.retained)- read = plan_delta_read(path, {}, "sales", stored=ObjectStorageDeltaCursor(version=0, table_id=table_id))+ @parameterized.expand([("append", False), ("by_file", True)])+ def test_a_cleaned_up_version_rebuilds(self, _name: str, by_file: bool) -> None:+ with tempfile.TemporaryDirectory() as folder:+ path = str(Path(folder) / "sales")+ write_deltalake(path, pa.table({"id": [1, 2], "day": ["a", "b"]}))+ table_id = str(DeltaTable(path).metadata().id)+ _append(path)+ DeltaTable(path).create_checkpoint()+ # Log cleanup removes the entries before the checkpoint, so version 0 can no longer be loaded.+ (Path(path) / "_delta_log" / f"{0:020d}.json").unlink()++ read = plan_delta_read(+ path,+ {},+ "sales",+ stored=ObjectStorageDeltaCursor(version=0, table_id=table_id, file_names=by_file),+ by_file=by_file,+ )assert read.reset_required is True