Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): sync delta tables incrementally by data file (#121283)

Tom Owers33b760ea24af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index a336b90553e0..6be1ab90b5dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -1054,10 +1054,12 @@ class PipelineV3(Generic[ResumableData]):
pa_table = normalize_table_column_names(pa_table)
if self._uses_delta_write_column_selection:
+ # The loader deletes rows by the replace scope's column, so a column selection must keep it.
+ replace_scope = self._resource.replace_scope
pa_table = await observe_and_project_table(
pa_table,
self._schema.enabled_columns,
- self._resource.primary_keys,
+ [*(self._resource.primary_keys or []), *([replace_scope.column] if replace_scope else [])],
self._schema.incremental_field,
[
*(self._schema.partitioning_keys_override or []),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index fe5537f59fd2..0c855e24726d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1008,6 +1008,22 @@ class TestLaneFanOut:
for writer in writers:
cast(MagicMock, writer.pg_producer.hold_batch).assert_called_once()
+ async def test_a_column_selection_keeps_the_replace_scope_column(self) -> None:
+ writer = _lane_writer("orders")
+ pipeline = self._pipeline([writer])
+ pipeline._uses_delta_write_column_selection = True
+ cast(MagicMock, pipeline._schema).configure_mock(enabled_columns=["name"])
+ pipeline._resource.primary_keys = None
+ pipeline._resource.replace_scope = ReplaceScope(
+ column="_file_name", replaced=frozenset({"a.csv"}), retained=frozenset({"a.csv"})
+ )
+
+ await self._process(pipeline, pa.table({"id": [1], "name": ["x"], "_file_name": ["a.csv"]}))
+
+ [written] = [call.args[0] for call in cast(MagicMock, writer.s3_batch_writer.write_batch).call_args_list]
+ assert {"name", "_file_name"} <= set(written.column_names)
+ assert "id" not in written.column_names
+
async def test_only_the_billable_lane_counts_towards_usage(self) -> None:
# One read of a change stream is one sync however many tables it keeps.
writers = [_lane_writer("users"), _lane_writer("users_cdc", billable=False)]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
index 9f4b2d801d0f..f0de6a2b06b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -336,49 +336,57 @@ def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path:
@pytest.mark.parametrize(
- "sync_type, has_stored_cursor, expected_ids, expected_columns",
+ "sync_type, has_stored_cursor, mirrors, expected_ids, expected_columns, expected_reset",
[
- (ExternalDataSchemaSyncType.APPEND, True, [2], {"id", "_delta_version"}),
- (ExternalDataSchemaSyncType.APPEND, False, [1, 2], {"id", "_delta_version"}),
+ (ExternalDataSchemaSyncType.APPEND, True, False, [2], {"id", "_delta_version"}, False),
+ (ExternalDataSchemaSyncType.APPEND, False, False, [1, 2], {"id", "_delta_version"}, False),
# A full refresh ignores the stored cursor, and still records the version it wrote.
- (ExternalDataSchemaSyncType.FULL_REFRESH, True, [1, 2], {"id"}),
+ (ExternalDataSchemaSyncType.FULL_REFRESH, True, False, [1, 2], {"id"}, False),
+ # An incremental sync names each row's data file, so the loader can replace rows by file.
+ (ExternalDataSchemaSyncType.INCREMENTAL, True, True, [2], {"id", "_delta_version", "_file_name"}, False),
+ # Without the flag the loader can't delete by file, so the run rebuilds the table.
+ (ExternalDataSchemaSyncType.INCREMENTAL, True, False, [1, 2], {"id", "_delta_version", "_file_name"}, True),
],
)
def test_delta_sync_reads_from_the_stored_cursor_and_stages_the_next_one(
tmp_path: Path,
sync_type: ExternalDataSchemaSyncType,
has_stored_cursor: bool,
+ mirrors: bool,
expected_ids: list[int],
expected_columns: set[str],
+ expected_reset: bool,
) -> None:
table_path = str(tmp_path / "customer-data" / "exports" / "sales")
write_deltalake(table_path, pa.table({"id": [1]}))
source = _LocalAmazonS3Source(tmp_path)
config = make_config()
[schema] = source.get_schemas(config, team_id=1)
- first = source_inputs(schema.name)
- first.schema_metadata = schema.schema_metadata
- first_cursor = SourceCursorManager.from_sync_type_config(source, {}, structlog.get_logger())
- first.source_cursor = first_cursor
- list(cast(Iterable[pa.Table], source.source_for_pipeline(config, first).items()))
+ reads_versions = sync_type != ExternalDataSchemaSyncType.FULL_REFRESH
+
+ def sync(stored: dict[str, Any]) -> tuple[SourceResponse, list[dict[str, Any]], SourceCursorManager]:
+ inputs = source_inputs(
+ schema.name,
+ incremental_field=schema.incremental_fields[0]["field"] if reads_versions else None,
+ incremental_field_type=IncrementalFieldType.Integer if reads_versions else None,
+ )
+ inputs.schema_metadata = schema.schema_metadata
+ inputs.sync_type = sync_type
+ cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())
+ inputs.source_cursor = cursor
+ with patch.object(source_base, "_mirrors_files", return_value=mirrors):
+ response = source.source_for_pipeline(config, inputs)
+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+ return response, rows, cursor
+
+ _, _, first_cursor = sync({})
stored = {"source_cursor": first_cursor.staged_payload()} if has_stored_cursor else {}
write_deltalake(table_path, pa.table({"id": [2]}), mode="append")
-
- appends = sync_type == ExternalDataSchemaSyncType.APPEND
- inputs = source_inputs(
- schema.name,
- incremental_field=schema.incremental_fields[0]["field"] if appends else None,
- incremental_field_type=IncrementalFieldType.Integer if appends else None,
- )
- inputs.schema_metadata = schema.schema_metadata
- inputs.sync_type = sync_type
- cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())
- inputs.source_cursor = cursor
- response = source.source_for_pipeline(config, inputs)
- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+ response, rows, cursor = sync(stored)
assert sorted(row["id"] for row in rows) == expected_ids
assert all(set(row) == expected_columns for row in rows)
- assert response.destination_reset_required is False
+ assert response.destination_reset_required is expected_reset
+ assert (response.replace_scope is not None) == (sync_type == ExternalDataSchemaSyncType.INCREMENTAL)
staged = cursor.staged_payload()
assert staged is not None and staged["data"]["version"] == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
index cfb372f75734..ff92fe7aff30 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -12,6 +12,7 @@ from structlog.types import FilteringBoundLogger
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_PATH_COLUMN
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
ACCESS_DENIED_ERROR,
DELTA_UNSUPPORTED_ERROR,
@@ -22,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
DELTA_BATCH_ROWS,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope
# Whole words only, so a status code doesn't match inside an object key such as `sales403/part.parquet`.
_ACCESS_DENIED_MARKERS = re.compile(r"\b(403|AccessDenied|Forbidden|InvalidAccessKeyId|SignatureDoesNotMatch)\b")
@@ -52,6 +54,8 @@ class ObjectStorageDeltaCursor:
table_id: str | None = None
# The digest of a files table's listing (see `listing_digest`). None for a Delta table.
files_digest: str | None = None
+ # Whether each row the destination holds names its data file, which a read by file needs.
+ file_names: bool = False
@contextmanager
@@ -85,24 +89,33 @@ class DeltaRead:
added_files: tuple[ds.FileFragment, ...] | None = dataclasses.field(default=None, repr=False)
# The stored cursor can't be extended by appending, so the destination must be rebuilt.
reset_required: bool = False
+ # Set for a read by data file: the rows of the files it reads, and of the files that are gone, go
+ # before the read loads. None when nothing changed.
+ replace_scope: ReplaceScope | None = dataclasses.field(default=None, repr=False)
- def iter_tables(self, batch_rows: int = DELTA_BATCH_ROWS, with_version: bool = False) -> Iterator[pa.Table]:
- """Stream the planned rows. The scan reads one fragment at a time on one thread to bound memory."""
- dataset: ds.Dataset = self.dataset
- if self.added_files is not None:
- if not self.added_files:
- return
- # Built from the snapshot's own fragments, so partition columns come back as in a full read.
- dataset = ds.FileSystemDataset(
- list(self.added_files), self.dataset.schema, self.dataset.format, self.dataset.filesystem
- )
- scanner = dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)
+ def iter_tables(
+ self, batch_rows: int = DELTA_BATCH_ROWS, with_version: bool = False, with_file_names: bool = False
+ ) -> Iterator[pa.Table]:
+ """Stream the planned rows. The scan reads one fragment at a time on one thread to bound memory.
+
+ `with_file_names` adds the path of each row's data file, which a replace scope names rows by.
+ """
+ fragments = list(self.added_files) if self.added_files is not None else None
+ if fragments is not None and not fragments:
+ return
+ scans: list[tuple[str | None, list[ds.FileFragment] | None]]
+ if with_file_names:
+ # One scan per file, because a scan does not say which file a batch came from.
+ scans = [(fragment.path, [fragment]) for fragment in fragments or self.dataset.get_fragments()]
+ else:
+ scans = [(None, fragments)]
yielded = False
try:
- for batch in scanner.to_batches():
- if batch.num_rows:
- yielded = True
- yield self._with_version(pa.Table.from_batches([batch]), with_version)
+ for file_name, scan_fragments in scans:
+ for batch in self._scanner(scan_fragments, batch_rows).to_batches():
+ if batch.num_rows:
+ yielded = True
+ yield self._tagged(pa.Table.from_batches([batch]), with_version, with_file_names, file_name)
except pa.ArrowInvalid as e:
raise ObjectStorageError(
f"{FILE_PARSE_ERROR} a data file of the Delta table '{self.table_name}': {e}"
@@ -114,16 +127,30 @@ class DeltaRead:
if not yielded and self.added_files is None:
# An empty snapshot still has to reach the pipeline as a schema, or the table is dropped and
# never recreated.
- yield self._with_version(self.dataset.schema.empty_table(), with_version)
-
- def _with_version(self, table: pa.Table, with_version: bool) -> pa.Table:
- if not with_version:
- return table
- if DELTA_VERSION_COLUMN in table.column_names:
- table = table.drop_columns([DELTA_VERSION_COLUMN])
- return table.append_column(
- DELTA_VERSION_COLUMN, pa.repeat(pa.scalar(self.cursor.version, pa.int64()), table.num_rows)
- )
+ yield self._tagged(self.dataset.schema.empty_table(), with_version, with_file_names, None)
+
+ def _scanner(self, fragments: list[ds.FileFragment] | None, batch_rows: int) -> ds.Scanner:
+ dataset: ds.Dataset = self.dataset
+ if fragments is not None:
+ # Built from the snapshot's own fragments, so partition columns come back as in a full read.
+ dataset = ds.FileSystemDataset(
+ cast(list[ds.Fragment], fragments), self.dataset.schema, self.dataset.format, self.dataset.filesystem
+ )
+ return dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)
+
+ def _tagged(self, table: pa.Table, with_version: bool, with_file_names: bool, file_name: str | None) -> pa.Table:
+ columns: list[tuple[str, pa.Array]] = []
+ if with_version:
+ columns.append(
+ (DELTA_VERSION_COLUMN, pa.repeat(pa.scalar(self.cursor.version, pa.int64()), table.num_rows))
+ )
+ if with_file_names:
+ columns.append((FILE_PATH_COLUMN, pa.repeat(pa.scalar(file_name, pa.string()), table.num_rows)))
+ for name, values in columns:
+ if name in table.column_names:
+ table = table.drop_columns([name])
+ table = table.append_column(name, values)
+ return table
def _files_at_version(uri: str, storage_options: dict[str, str], version: int) -> set[str] | None:
@@ -144,41 +171,65 @@ def plan_delta_read(
table_name: str,
stored: ObjectStorageDeltaCursor | None = None,
logger: FilteringBoundLogger | None = None,
+ by_file: bool = False,
) -> DeltaRead:
"""Pin the table's current version and decide what to read to bring the destination up to it.
Without a stored cursor the whole snapshot is read. With one, only the data files added since
- the cursor's version are read, and only when every file of that version is still in the table.
- A file that is gone was removed by a delete, an update, a merge, an overwrite or a compaction,
- all of which change or move rows the destination already holds. No append can repair that, so
- the plan then asks for a rebuild from the whole snapshot.
+ the cursor's version are read. A file that is gone was removed by a delete, an update, a merge,
+ an overwrite or a compaction, all of which change or move rows the destination already holds.
+
+ An append can't repair that, so the plan then asks for a rebuild from the whole snapshot. A read
+ `by_file` can: each row names its data file, and the plan's replace scope deletes the rows of
+ the files that are gone and of the files it reads. Such a read still rebuilds when the table was
+ created again, when it is at an earlier version, when the stored version can't be read, and when
+ the stored rows don't name their files.
"""
with _delta_errors(table_name):
table = DeltaTable(uri, storage_options=storage_options)
version = table.version()
- cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id))
+ cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id), file_names=by_file)
dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())
if logger is not None:
logger.info("Reading Delta table snapshot", table=table_name, delta_version=cursor.version)
+
+ def planned(added: list[ds.FileFragment] | None, *, reset: bool = False) -> DeltaRead:
+ scope: ReplaceScope | None = None
+ # Also when nothing changed: a keyless incremental run without a scope fails before it reads.
+ if by_file:
+ current = frozenset(fragment.path for fragment in dataset.get_fragments())
+ replaced = current if added is None else frozenset(fragment.path for fragment in added)
+ scope = ReplaceScope(column=FILE_PATH_COLUMN, replaced=replaced, retained=current)
+ return DeltaRead(
+ table_name=table_name,
+ dataset=dataset,
+ cursor=cursor,
+ added_files=None if added is None else tuple(added),
+ reset_required=reset,
+ replace_scope=scope,
+ )
+
if stored is None or stored.version is None:
- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor)
- if stored.table_id == cursor.table_id and stored.version == cursor.version:
- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=())
+ # Rows from a sync that did not name their files can't be deleted by file, so they go with a rebuild.
+ return planned(None, reset=by_file)
reset_reason: str | None = None
added_files: list[ds.FileFragment] = []
- if stored.table_id != cursor.table_id:
+ if by_file and not stored.file_names:
+ reset_reason = "the rows of the last sync do not name their data files"
+ elif stored.table_id != cursor.table_id:
reset_reason = "the table was created again"
elif stored.version > version:
reset_reason = "the table is at an earlier version than the last sync"
- else:
+ elif stored.version < version:
with _delta_errors(table_name):
earlier_files = _files_at_version(uri, storage_options, stored.version)
fragments: list[ds.FileFragment] = list(dataset.get_fragments())
+ current_files = {fragment.path for fragment in fragments}
if earlier_files is None:
reset_reason = "the version of the last sync can no longer be read"
- elif not earlier_files <= {fragment.path for fragment in fragments}:
+ elif not earlier_files <= current_files and not by_file:
reset_reason = "a commit since the last sync removed or rewrote data files"
else:
added_files = [fragment for fragment in fragments if fragment.path not in earlier_files]
@@ -192,8 +243,8 @@ def plan_delta_read(
stored_version=stored.version,
delta_version=cursor.version,
)
- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, reset_required=True)
- return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=tuple(added_files))
+ return planned(None, reset=True)
+ return planned(added_files)
def iter_delta_tables(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
index 3a5c4a265148..a333827863fa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
@@ -54,8 +54,9 @@ def source_caption(store: str) -> str:
return (
f"Import CSV, JSON Lines and Parquet files, and Delta Lake tables, from {store}. PostHog creates one "
"table per folder, one per file directly in the folder the URL points to, and one per Delta table. A full "
- "refresh reads every file again, so the table matches the files. An incremental or append-only sync "
- "reads only new and changed files, and keeps the rows of files you delete."
+ "refresh reads every file again, so the table matches the files. An incremental sync reads only new "
+ "and changed files, and removes the rows of files you delete. For a Delta table, it also applies "
+ "deleted and updated rows. An append-only sync reads only new files and keeps every row it imported."
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 565bdf28d8a4..56acc9068c71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -77,7 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sch
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.types import IncrementalFieldType
+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType, IncrementalFieldType
def _raise_translated(error: Exception, location: str) -> NoReturn:
@@ -116,7 +116,9 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
sync with `REPLACE_ROWS_BY_FILE_FLAG` on also deletes the rows of a deleted file, and a changed file
replaces its rows. Without the flag it merges on the file and row number and keeps the rows of
deleted files. An append sync of a Delta table reads only the data files committed since the
- version of the last sync.
+ version of the last sync. An incremental sync of a Delta table also replaces the rows of each data
+ file that a commit removed, so deletes, updates, merges and compactions reach the table. Without
+ the flag it rebuilds the table on every run.
"""
@abstractmethod
@@ -305,21 +307,26 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
def _delta_response(self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs) -> SourceResponse:
cursor_manager = self.get_cursor_manager(inputs)
appends = inputs.should_use_incremental_field
+ by_file = appends and inputs.sync_type == ExternalDataSchemaSyncType.INCREMENTAL
+ # Without the flag the loader can't delete by file, so an incremental sync rebuilds the
+ # table from the snapshot on every run.
+ stored = cursor_manager.load() if appends and (not by_file or _mirrors_files(inputs)) else None
# The plan is made here and not in `items`, because the response has to say up front
# whether the destination must be rebuilt.
read = plan_delta_read(
store.delta_uri(table.path),
store.delta_storage_options,
table.name,
- stored=cursor_manager.load() if appends else None,
+ stored=stored,
logger=inputs.logger,
+ by_file=by_file,
)
def items() -> Iterator[pa.Table]:
# A full refresh stages the cursor too. A later switch to append then continues from
# the version the table holds, and not from the cursor of an earlier append.
cursor_manager.stage(read.cursor)
- yield from read.iter_tables(with_version=appends)
+ yield from read.iter_tables(with_version=appends, with_file_names=by_file)
return SourceResponse(
name=inputs.schema_name,
@@ -328,6 +335,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
chunk_size_bytes=CHUNK_SIZE_BYTES,
destination_reset_required=read.reset_required,
preserve_column_names=table.preserve_column_names,
+ replace_scope=read.replace_scope,
)
def _find_table(
@@ -347,8 +355,8 @@ def _source_schema(table: TableDefinition) -> SourceSchema:
is_files = table.kind == "files"
return SourceSchema(
name=table.name,
- # Delta rows have no key to merge on, so a Delta table can only append.
- supports_incremental=is_files,
+ # Delta rows have no key to merge on, so an incremental sync of a Delta table replaces rows by data file.
+ supports_incremental=True,
supports_append=True,
incremental_fields=[
incremental_field(FILE_MODIFIED_AT_COLUMN)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index ffab85de6375..8b92bdd8e84c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -686,18 +686,60 @@ def _append_then_compact(path: str) -> None:
DeltaTable(path).optimize.compact()
+def _update(path: str) -> None:
+ DeltaTable(path).update(updates={"id": "10"}, predicate="id = 2")
+
+
+def _merge(path: str) -> None:
+ (
+ DeltaTable(path)
+ .merge(pa.table({"id": [2, 4], "day": ["b", "c"]}), predicate="t.id = s.id", source_alias="s", target_alias="t")
+ .when_matched_update(updates={"id": "s.id + 20"})
+ .when_not_matched_insert_all()
+ .execute()
+ )
+
+
+def _files_and_ids(path: str, version: int | None = None) -> list[tuple[str, int]]:
+ dataset = DeltaTable(path, version=version).to_pyarrow_dataset()
+ return sorted(
+ (fragment.path, row_id)
+ for fragment in dataset.get_fragments()
+ for row_id in fragment.to_table(columns=["id"]).column("id").to_pylist()
+ )
+
+
class TestDeltaCursor:
@parameterized.expand(
[
- ("no_cursor_reads_the_snapshot", None, [_append], False, [1, 2, 3], 1),
- ("appends_read_only_the_new_files", "same", [_append], False, [3], 1),
- ("an_unchanged_table_reads_nothing", "same", [], False, [], 0),
- ("an_overwrite_rebuilds", "same", [_overwrite], True, [9], 1),
- ("a_delete_rebuilds", "same", [_delete], True, [2], 1),
+ ("no_cursor_reads_the_snapshot", None, [_append], False, False, [1, 2, 3], 1),
+ ("appends_read_only_the_new_files", "same", [_append], False, False, [3], 1),
+ ("an_unchanged_table_reads_nothing", "same", [], False, False, [], 0),
+ ("an_overwrite_rebuilds", "same", [_overwrite], False, True, [9], 1),
+ ("a_delete_rebuilds", "same", [_delete], False, True, [2], 1),
# Compaction keeps the rows but moves them to a new file, which an append would read twice.
- ("a_compaction_rebuilds", "same", [_append_then_compact], True, [1, 2, 3], 2),
- ("a_recreated_table_rebuilds", "other", [_append], True, [1, 2, 3], 1),
- ("a_cursor_ahead_of_the_table_rebuilds", "ahead", [], True, [1, 2], 0),
+ ("a_compaction_rebuilds", "same", [_append_then_compact], False, True, [1, 2, 3], 2),
+ ("a_recreated_table_rebuilds", "other", [_append], False, True, [1, 2, 3], 1),
+ ("a_cursor_ahead_of_the_table_rebuilds", "ahead", [], False, True, [1, 2], 0),
+ # By file, a commit that removes files only replaces the rows of the files it touched.
+ ("by_file_an_append_reads_the_new_file", "same", [_append], True, False, [3], 1),
+ ("by_file_an_unchanged_table_reads_nothing", "same", [], True, False, [], 0),
+ ("by_file_a_delete_reads_nothing_and_drops_the_file", "same", [_delete], True, False, [], 1),
+ ("by_file_an_update_reads_the_rewritten_file", "same", [_update], True, False, [10], 1),
+ ("by_file_a_merge_reads_the_files_it_wrote", "same", [_merge], True, False, [4, 22], 1),
+ ("by_file_an_overwrite_reads_the_new_file", "same", [_overwrite], True, False, [9], 1),
+ ("by_file_a_compaction_reads_the_compacted_files", "same", [_append_then_compact], True, False, [2, 3], 2),
+ ("by_file_a_recreated_table_rebuilds", "other", [_append], True, True, [1, 2, 3], 1),
+ ("by_file_a_cursor_ahead_of_the_table_rebuilds", "ahead", [], True, True, [1, 2], 0),
+ (
+ "by_file_a_sync_that_named_no_files_rebuilds",
+ "same_without_file_names",
+ [_append],
+ True,
+ True,
+ [1, 2, 3],
+ 1,
+ ),
]
)
def test_plan(
@@ -705,6 +747,7 @@ class TestDeltaCursor:
_name: str,
stored_cursor: str | None,
commits: list[Any],
+ by_file: bool,
expected_reset: bool,
expected_ids: list[int],
expected_version: int,
@@ -715,33 +758,58 @@ class TestDeltaCursor:
table_id = str(DeltaTable(path).metadata().id)
stored = {
None: None,
- "same": ObjectStorageDeltaCursor(version=0, table_id=table_id),
- "other": ObjectStorageDeltaCursor(version=0, table_id="another-table"),
- "ahead": ObjectStorageDeltaCursor(version=5, table_id=table_id),
+ "same": ObjectStorageDeltaCursor(version=0, table_id=table_id, file_names=by_file),
+ "same_without_file_names": ObjectStorageDeltaCursor(version=0, table_id=table_id),
+ "other": ObjectStorageDeltaCursor(version=0, table_id="another-table", file_names=by_file),
+ "ahead": ObjectStorageDeltaCursor(version=5, table_id=table_id, file_names=by_file),
}[stored_cursor]
for commit in commits:
commit(path)
- read = plan_delta_read(path, {}, "sales", stored=stored)
- rows = [row for table in read.iter_tables(with_version=True) for row in table.to_pylist()]
+ read = plan_delta_read(path, {}, "sales", stored=stored, by_file=by_file)
+ rows = [
+ row
+ for table in read.iter_tables(with_version=True, with_file_names=by_file)
+ for row in table.to_pylist()
+ ]
+ if by_file and not expected_reset:
+ # The destination after the load: the rows of the last sync, less the rows the scope
+ # deletes, plus the rows read. It must match the table's current snapshot exactly.
+ held = _files_and_ids(path, version=0)
+ kept = [(file, row_id) for file, row_id in held if not self._deleted(read.replace_scope, file)]
+ loaded = [(row["_file_name"], row["id"]) for row in rows]
+ assert sorted(kept + loaded) == _files_and_ids(path)
assert read.reset_required is expected_reset
- assert read.cursor == ObjectStorageDeltaCursor(version=expected_version, table_id=table_id)
+ assert read.cursor == ObjectStorageDeltaCursor(version=expected_version, table_id=table_id, file_names=by_file)
assert sorted(row["id"] for row in rows) == expected_ids
# Rows read from a subset of the files keep the partition column and carry the version.
- assert all(set(row) == {"id", "day", "_delta_version"} for row in rows)
+ assert all(set(row) == {"id", "day", "_delta_version", *(["_file_name"] if by_file else [])} for row in rows)
assert {row["_delta_version"] for row in rows} <= {expected_version}
+ assert (read.replace_scope is not None) == by_file
- def test_a_cleaned_up_version_rebuilds(self, tmp_path: Path) -> None:
- path = str(tmp_path / "sales")
- write_deltalake(path, pa.table({"id": [1, 2], "day": ["a", "b"]}))
- table_id = str(DeltaTable(path).metadata().id)
- _append(path)
- DeltaTable(path).create_checkpoint()
- # Log cleanup removes the entries before the checkpoint, so version 0 can no longer be loaded.
- (Path(path) / "_delta_log" / f"{0:020d}.json").unlink()
+ @staticmethod
+ def _deleted(scope: ReplaceScope | None, file: str) -> bool:
+ return scope is not None and (file in scope.replaced or file not in scope.retained)
- read = plan_delta_read(path, {}, "sales", stored=ObjectStorageDeltaCursor(version=0, table_id=table_id))
+ @parameterized.expand([("append", False), ("by_file", True)])
+ def test_a_cleaned_up_version_rebuilds(self, _name: str, by_file: bool) -> None:
+ with tempfile.TemporaryDirectory() as folder:
+ path = str(Path(folder) / "sales")
+ write_deltalake(path, pa.table({"id": [1, 2], "day": ["a", "b"]}))
+ table_id = str(DeltaTable(path).metadata().id)
+ _append(path)
+ DeltaTable(path).create_checkpoint()
+ # Log cleanup removes the entries before the checkpoint, so version 0 can no longer be loaded.
+ (Path(path) / "_delta_log" / f"{0:020d}.json").unlink()
+
+ read = plan_delta_read(
+ path,
+ {},
+ "sales",
+ stored=ObjectStorageDeltaCursor(version=0, table_id=table_id, file_names=by_file),
+ by_file=by_file,
+ )
assert read.reset_required is True