Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): mirror deleted and rewritten files in incremental object storage syncs (#121282)

Tom Owers9d441be844fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
index 3c42e62ca32f..9f4b2d801d0f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -6,6 +6,7 @@ from pathlib import Path
from typing import Any, cast
import pytest
+from unittest.mock import patch
import pyarrow as pa
import structlog
@@ -13,8 +14,10 @@ from deltalake import write_deltalake
from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import validate_incremental_sync
from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
ObjectStoreConfig,
StoreConfig,
@@ -27,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStore
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputs
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (
AmazonS3SourceConfig,
)
@@ -206,16 +210,31 @@ def test_validate_credentials(
@pytest.mark.parametrize(
- "sync_type, last_value, expected_files, expected_keys",
+ "sync_type, last_value, expected_files, expected_keys, mirrors",
[
- (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None),
- (ExternalDataSchemaSyncType.INCREMENTAL, None, ["c.csv", "b.csv", "a.csv"], ["_file_name", "_file_row_number"]),
+ (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None, False),
+ (
+ ExternalDataSchemaSyncType.INCREMENTAL,
+ None,
+ ["c.csv", "b.csv", "a.csv"],
+ ["_file_name", "_file_row_number"],
+ False,
+ ),
# The file at the watermark is read again, and the merge key replaces its rows.
(
ExternalDataSchemaSyncType.INCREMENTAL,
FIRST_MODIFIED_AT + timedelta(minutes=1),
["b.csv", "a.csv"],
["_file_name", "_file_row_number"],
+ False,
+ ),
+ # With the flag the run replaces the rows of the files it reads, so it needs no key.
+ (
+ ExternalDataSchemaSyncType.INCREMENTAL,
+ FIRST_MODIFIED_AT + timedelta(minutes=1),
+ ["b.csv", "a.csv"],
+ None,
+ True,
),
# The pipeline can hand the watermark back without a timezone.
(
@@ -223,10 +242,13 @@ def test_validate_credentials(
(FIRST_MODIFIED_AT + timedelta(minutes=2)).replace(tzinfo=None),
["a.csv"],
["_file_name", "_file_row_number"],
+ False,
),
# An append has no key, so the file at the watermark must not be read again.
- (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None),
- (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None),
+ (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, False),
+ (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None, False),
+ # The flag does not change an append.
+ (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, True),
],
)
def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
@@ -235,6 +257,7 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
last_value: datetime | None,
expected_files: list[str],
expected_keys: list[str] | None,
+ mirrors: bool,
) -> None:
# Path order is the reverse of age order, so the test tells the two apart.
for minutes, name in enumerate(["c.csv", "b.csv", "a.csv"]):
@@ -251,14 +274,65 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
)
inputs.schema_metadata = schema.schema_metadata
inputs.sync_type = sync_type
+ inputs.source_cursor = SourceCursorManager.from_sync_type_config(source, {}, structlog.get_logger())
- response = source.source_for_pipeline(config, inputs)
- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+ with patch.object(source_base, "_mirrors_files", return_value=mirrors):
+ response = source.source_for_pipeline(config, inputs)
+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
assert [(row["_file_name"], row["_file_row_number"]) for row in rows] == [
(f"orders/{name}", row_number) for name in expected_files for row_number in (0, 1)
]
assert response.primary_keys == expected_keys
+ mirrored = mirrors and sync_type == ExternalDataSchemaSyncType.INCREMENTAL
+ assert response.replace_scope == (
+ ReplaceScope(
+ column="_file_name",
+ replaced=frozenset(f"orders/{name}" for name in expected_files),
+ retained=frozenset({"orders/a.csv", "orders/b.csv", "orders/c.csv"}),
+ )
+ if mirrored
+ else None
+ )
+
+
+def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path: Path) -> None:
+ for minutes, name in enumerate(["b.csv", "a.csv"]):
+ write(tmp_path, f"exports/orders/{name}", b"id\n1\n", FIRST_MODIFIED_AT + timedelta(minutes=minutes))
+ source = _LocalAmazonS3Source(tmp_path)
+ config = make_config()
+ [schema] = source.get_schemas(config, team_id=1)
+
+ def sync(stored: dict[str, Any]) -> tuple[SourceResponse, int, SourceCursorManager]:
+ inputs = source_inputs(
+ schema.name,
+ incremental_field=schema.incremental_fields[0]["field"],
+ incremental_field_type=IncrementalFieldType.DateTime,
+ db_incremental_field_last_value=FIRST_MODIFIED_AT + timedelta(minutes=1),
+ )
+ inputs.schema_metadata = schema.schema_metadata
+ inputs.sync_type = ExternalDataSchemaSyncType.INCREMENTAL
+ cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())
+ inputs.source_cursor = cursor
+ with patch.object(source_base, "_mirrors_files", return_value=True):
+ response = source.source_for_pipeline(config, inputs)
+ rows = sum(table.num_rows for table in cast(Iterable[pa.Table], response.items()))
+ return response, rows, cursor
+
+ first, first_rows, cursor = sync({})
+ unchanged, unchanged_rows, _ = sync({"source_cursor": cursor.staged_payload()})
+ write(tmp_path, "exports/orders/b.csv", b"id\n1\n2\n", FIRST_MODIFIED_AT + timedelta(minutes=2))
+ rewritten, rewritten_rows, _ = sync({"source_cursor": cursor.staged_payload()})
+
+ # The file at the watermark is read again on every run, so only the stored listing can tell
+ # that nothing changed.
+ assert (first.replace_scope is not None, first_rows) == (True, 1)
+ assert (unchanged.replace_scope, unchanged_rows) == (None, 0)
+ validate_incremental_sync(True, unchanged, is_first_sync=False)
+ assert (rewritten.replace_scope is not None and rewritten.replace_scope.replaced, rewritten_rows) == (
+ frozenset({"orders/a.csv", "orders/b.csv"}),
+ 3,
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
index 586f8d8be9f9..cfb372f75734 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -43,13 +43,15 @@ DELTA_VERSION_COLUMN = "_delta_version"
@frozen
class ObjectStorageDeltaCursor:
- """The Delta table version the destination holds."""
+ """What the destination holds: a version of a Delta table, or a listing of a files table."""
cursor_kind: ClassVar[str] = "object_storage_delta_v1"
- version: int
+ version: int | None = None
# Delta gives a table a new ID when it is dropped and created again, which restarts its versions.
- table_id: str
+ table_id: str | None = None
+ # The digest of a files table's listing (see `listing_digest`). None for a Delta table.
+ files_digest: str | None = None
@contextmanager
@@ -153,12 +155,13 @@ def plan_delta_read(
"""
with _delta_errors(table_name):
table = DeltaTable(uri, storage_options=storage_options)
- cursor = ObjectStorageDeltaCursor(version=table.version(), table_id=str(table.metadata().id))
+ version = table.version()
+ cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id))
dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())
if logger is not None:
logger.info("Reading Delta table snapshot", table=table_name, delta_version=cursor.version)
- if stored is None:
+ if stored is None or stored.version is None:
return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor)
if stored.table_id == cursor.table_id and stored.version == cursor.version:
return DeltaRead(table_name=table_name, dataset=dataset, cursor=cursor, added_files=())
@@ -167,7 +170,7 @@ def plan_delta_read(
added_files: list[ds.FileFragment] = []
if stored.table_id != cursor.table_id:
reset_reason = "the table was created again"
- elif stored.version > cursor.version:
+ elif stored.version > version:
reset_reason = "the table is at an earlier version than the last sync"
else:
with _delta_errors(table_name):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py
index 6a86e21ef8bf..35f72152fa63 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/incremental.py
@@ -1,4 +1,5 @@
-from collections.abc import Iterable
+import hashlib
+from collections.abc import Iterable, Sequence
from datetime import UTC, datetime, timedelta
from typing import Any, Literal
@@ -9,7 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
APPEND_SETTLE_SECONDS,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceInputs
from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType
FILE_ROW_NUMBER_COLUMN = "_file_row_number"
@@ -69,3 +70,45 @@ class FileSelection:
READ_ALL = FileSelection()
+
+
+def listing_digest(files: Iterable[StoredObject]) -> str:
+ """A 64-character digest of the path, size and modification time of each file."""
+ digest = hashlib.sha256()
+ for stored in sorted(files, key=lambda stored: stored.path):
+ digest.update(f"{stored.path}\t{stored.size}\t{stored.modified_at.isoformat()}\n".encode())
+ return digest.hexdigest()
+
+
+@frozen
+class MirrorPlan:
+ """One incremental read that makes a files table match the bucket.
+
+ The changed files are read again and replace their own rows, and the rows of a file that is no
+ longer in the bucket go (see `ReplaceScope`).
+ """
+
+ files: list[StoredObject]
+ """The files to read, oldest first."""
+ replace_scope: ReplaceScope | None
+ """None when nothing changed since the last sync, so the run reads and deletes nothing."""
+ digest: str
+ """The digest of the listing that this read brings the table up to."""
+
+
+def plan_mirror(
+ listed: Sequence[StoredObject], selection: FileSelection, stored_digest: str | None, now: datetime | None = None
+) -> MirrorPlan:
+ """Plan the read from the current listing and the digest of the listing at the last sync."""
+ digest = listing_digest(listed)
+ if digest == stored_digest:
+ # The files at the watermark are read again on every run, so the watermark alone can never
+ # tell that a listing has not changed.
+ return MirrorPlan(files=[], replace_scope=None, digest=digest)
+ files = selection.select(listed, now=now)
+ scope = ReplaceScope(
+ column=FILE_PATH_COLUMN,
+ replaced=frozenset(stored.path for stored in files),
+ retained=frozenset(stored.path for stored in listed),
+ )
+ return MirrorPlan(files=files, replace_scope=scope, digest=digest)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 684ac2d114d8..d58802c78a0f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -7,12 +7,18 @@ import pyarrow as pa
from botocore.exceptions import ClientError
from structlog.types import FilteringBoundLogger
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern
+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import (
+ REPLACE_ROWS_BY_FILE_FLAG,
+ is_schema_flag_enabled,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, SimpleSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import CursorSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_MODIFIED_AT_COLUMN
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
DiscoveryOptions,
+ FileFormat,
StoreConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (
@@ -43,6 +49,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
ObjectStore,
+ StoredObject,
open_store,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (
@@ -52,6 +59,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
FILE_PRIMARY_KEYS,
READ_ALL,
FileSelection,
+ plan_mirror,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
CHUNK_SIZE_BYTES,
@@ -97,9 +105,12 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
"""Imports the files a bucket URL selects: CSV, JSON Lines and Parquet files, and Delta tables.
A full refresh reads each table in full, so the table matches the bucket at the time of the sync.
- The incremental and append sync types read only the files modified since the last sync. They
- never remove rows, so the rows of a file deleted from the bucket stay in the table. An append sync
- of a Delta table reads only the data files committed since the version of the last sync.
+ The incremental and append sync types read only the files modified since the last sync. An append
+ never removes rows, so the rows of a file deleted from the bucket stay in the table. An incremental
+ sync with `REPLACE_ROWS_BY_FILE_FLAG` on also deletes the rows of a deleted file, and a changed file
+ replaces its rows. Without the flag it merges on the file and row number and keeps the rows of
+ deleted files. An append sync of a Delta table reads only the data files committed since the
+ version of the last sync.
"""
@abstractmethod
@@ -210,6 +221,8 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
if table.kind == "delta":
return self._delta_response(store, table, inputs)
selection = FileSelection.from_inputs(inputs)
+ if selection.merges and _mirrors_files(inputs):
+ return self._mirrored_files_response(store, table, inputs, selection)
def items() -> Iterator[pa.Table]:
yield from iter_table(store, table, inputs.logger, selection)
@@ -222,6 +235,37 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
chunk_size_bytes=CHUNK_SIZE_BYTES,
)
+ def _mirrored_files_response(
+ self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs, selection: FileSelection
+ ) -> SourceResponse:
+ """An incremental read that also deletes the rows of deleted files and of rewritten files.
+
+ The listing is made here and not in `items`, because the response has to name the files
+ whose rows the run replaces before the first batch loads.
+ """
+ cursor_manager = self.get_cursor_manager(inputs)
+ stored = cursor_manager.load()
+ # Taken before listing, as in `iter_table`.
+ listing_started_at = datetime.now(UTC)
+ resolved = _resolve_files(store, table, inputs.logger)
+ listed = [stored_file for stored_file in resolved.files if _file_format(resolved, stored_file) is not None]
+ plan = plan_mirror(
+ listed, selection, stored.files_digest if stored is not None else None, now=listing_started_at
+ )
+
+ def items() -> Iterator[pa.Table]:
+ cursor_manager.stage(ObjectStorageDeltaCursor(files_digest=plan.digest))
+ yield from _iter_files(store, resolved, plan.files, inputs.logger)
+
+ return SourceResponse(
+ name=inputs.schema_name,
+ items=items,
+ # A run with no scope reads no rows, but the pipeline still needs a key or a scope for an incremental run.
+ primary_keys=FILE_PRIMARY_KEYS if plan.replace_scope is None else None,
+ chunk_size_bytes=CHUNK_SIZE_BYTES,
+ replace_scope=plan.replace_scope,
+ )
+
def _delta_response(self, store: ObjectStore, table: TableDefinition, inputs: SourceInputs) -> SourceResponse:
cursor_manager = self.get_cursor_manager(inputs)
appends = inputs.should_use_incremental_field
@@ -294,12 +338,26 @@ def iter_table(
# Taken before listing: a file that lands during a slow listing can share a timestamp with a listed
# file, so the append settle window has to count from before the listing started.
listing_started_at = datetime.now(UTC)
+ resolved = _resolve_files(store, table, logger)
+ yield from _iter_files(store, resolved, selection.select(resolved.files, now=listing_started_at), logger)
+
+
+def _resolve_files(store: ObjectStore, table: TableDefinition, logger: FilteringBoundLogger) -> TableDefinition:
try:
- resolved = resolve_table_files(store, table, logger=logger)
+ return resolve_table_files(store, table, logger=logger)
except Exception as e:
_raise_translated(e, store.root)
- for stored in selection.select(resolved.files, now=listing_started_at):
- file_format = resolve_file_format(stored.path, resolved.file_format)
+
+
+def _file_format(resolved: TableDefinition, stored: StoredObject) -> FileFormat | None:
+ return resolve_file_format(stored.path, resolved.file_format)
+
+
+def _iter_files(
+ store: ObjectStore, resolved: TableDefinition, files: list[StoredObject], logger: FilteringBoundLogger
+) -> Iterator[pa.Table]:
+ for stored in files:
+ file_format = _file_format(resolved, stored)
if file_format is None:
continue
try:
@@ -307,3 +365,11 @@ def iter_table(
yield from iter_file_tables(stream, stored, file_format, resolved.csv, logger)
except PermissionError as e:
_raise_translated(e, store.root)
+
+
+def _mirrors_files(inputs: SourceInputs) -> bool:
+ """Whether an incremental read deletes the rows of deleted and rewritten files (see `ReplaceScope`)."""
+ schema = (
+ ExternalDataSchema.objects.filter(team_id=inputs.team_id, id=inputs.schema_id).select_related("source").first()
+ )
+ return schema is not None and is_schema_flag_enabled(schema, REPLACE_ROWS_BY_FILE_FLAG)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index f6bb421cf9fc..4ef4d5f00e87 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -66,6 +66,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.incremental import (
FileSelection,
+ listing_digest,
+ plan_mirror,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
JSONL_CHUNK_ROWS,
@@ -76,6 +78,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
iter_table,
validate_store_location,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope
MODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)
LOGGER = structlog.get_logger()
@@ -819,6 +822,53 @@ class TestFileSelection:
assert {row["_file_name"] for row in rows} == {"settled.csv"}
+ @parameterized.expand(
+ [
+ ("an_unchanged_listing_reads_and_deletes_nothing", [("a.csv", 0, 10), ("b.csv", 1, 10)], None),
+ (
+ "a_deleted_file_loses_its_rows",
+ [("b.csv", 1, 10)],
+ (["b.csv"], {"b.csv"}, {"b.csv"}),
+ ),
+ (
+ "a_shrunk_file_replaces_its_rows",
+ [("a.csv", 0, 10), ("b.csv", 2, 4)],
+ (["b.csv"], {"b.csv"}, {"a.csv", "b.csv"}),
+ ),
+ (
+ "a_renamed_file_moves_its_rows",
+ [("renamed.csv", 2, 10), ("b.csv", 1, 10)],
+ (["b.csv", "renamed.csv"], {"b.csv", "renamed.csv"}, {"b.csv", "renamed.csv"}),
+ ),
+ ]
+ )
+ def test_a_mirrored_read_replaces_the_changed_files_and_drops_the_deleted_ones(
+ self,
+ _name: str,
+ listing: list[tuple[str, int, int]],
+ expected: tuple[list[str], set[str], set[str]] | None,
+ ) -> None:
+ def files(entries: list[tuple[str, int, int]]) -> list[StoredObject]:
+ return [
+ StoredObject(path=path, size=size, modified_at=MODIFIED_AT + timedelta(minutes=minutes))
+ for path, minutes, size in entries
+ ]
+
+ last_sync = listing_digest(files([("a.csv", 0, 10), ("b.csv", 1, 10)]))
+ selection = FileSelection(mode="changed", watermark=MODIFIED_AT + timedelta(minutes=1))
+
+ plan = plan_mirror(files(listing), selection, last_sync)
+
+ assert len(plan.digest) == 64
+ if expected is None:
+ assert (plan.files, plan.replace_scope) == ([], None)
+ return
+ read, replaced, retained = expected
+ assert [stored.path for stored in plan.files] == read
+ assert plan.replace_scope == ReplaceScope(
+ column="_file_name", replaced=frozenset(replaced), retained=frozenset(retained)
+ )
+
def test_row_numbers_continue_across_the_chunks_of_one_file(self) -> None:
rows = JSONL_CHUNK_ROWS + 3
content = b"".join(b'{"n":%d}\n' % number for number in range(rows))