Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): stop rebuilding incremental delta tables on every sync (#125497)

Tom Owers40e2a5a492fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md b/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md
index 03d327ca0970..abd045cfc592 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md
@@ -39,7 +39,6 @@ The append keeps one row for each key in a batch, not in the run: a key that the
A source that has no primary key can name the origin of each row instead, for example the file it came from.
It sets `SourceResponse.replace_scope` (`ReplaceScope` in `sources/common/typings.py`): a column, the values this run reads again (`replaced`), and the values that are live now (`retained`).
-A source sets a scope only while the `warehouse-replace-rows-by-file` flag is on for the schema, because an older loader ignores the scope and fails the keyless batch.
- The pipeline writes the two value sets to `replace_scope.parquet` in the run's folder (`pipeline_v3/replace_scope.py`). Each queue row carries only the column and the manifest path, under `replace_scope` in its metadata.
- Before batch 0 of a fresh attempt writes, the loader deletes each row whose value is in `replaced` or not in `retained` (`pipeline_v3/load/replace_rows.py`). It reads the values in the table from one column, and deletes in commits of up to 1,000 values. Then every batch of the run appends.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
index f9e5726ca407..92441c8bafdf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
@@ -19,12 +19,6 @@ if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-# A source sends a replace scope (see `ReplaceScope`) only while this flag is on for the schema. The
-# flag stays off until the loaders that apply a scope are live, because an older loader ignores the
-# scope and fails the keyless batch.
-REPLACE_ROWS_BY_FILE_FLAG = "warehouse-replace-rows-by-file"
-
-
def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:
"""Evaluate a rollout flag for this schema.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
index d22e65c410bd..a92cfe08a4b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -17,7 +17,6 @@ from fsspec.implementations.local import LocalFileSystem
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import validate_incremental_sync
from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
ObjectStoreConfig,
StoreConfig,
@@ -255,45 +254,22 @@ def test_validate_credentials(
@pytest.mark.parametrize(
- "sync_type, last_value, expected_files, expected_keys, mirrors",
+ "sync_type, last_value, expected_files, expected_keys",
[
- (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None, False),
- (
- ExternalDataSchemaSyncType.INCREMENTAL,
- None,
- ["c.csv", "b.csv", "a.csv"],
- ["_file_name", "_file_row_number"],
- False,
- ),
- # The file at the watermark is read again, and the merge key replaces its rows.
- (
- ExternalDataSchemaSyncType.INCREMENTAL,
- FIRST_MODIFIED_AT + timedelta(minutes=1),
- ["b.csv", "a.csv"],
- ["_file_name", "_file_row_number"],
- False,
- ),
- # With the flag the run replaces the rows of the files it reads, so it needs no key.
- (
- ExternalDataSchemaSyncType.INCREMENTAL,
- FIRST_MODIFIED_AT + timedelta(minutes=1),
- ["b.csv", "a.csv"],
- None,
- True,
- ),
+ (ExternalDataSchemaSyncType.FULL_REFRESH, None, ["a.csv", "b.csv", "c.csv"], None),
+ (ExternalDataSchemaSyncType.INCREMENTAL, None, ["c.csv", "b.csv", "a.csv"], None),
+ # The file at the watermark is read again, and the run replaces its rows.
+ (ExternalDataSchemaSyncType.INCREMENTAL, FIRST_MODIFIED_AT + timedelta(minutes=1), ["b.csv", "a.csv"], None),
# The pipeline can hand the watermark back without a timezone.
(
ExternalDataSchemaSyncType.INCREMENTAL,
(FIRST_MODIFIED_AT + timedelta(minutes=2)).replace(tzinfo=None),
["a.csv"],
- ["_file_name", "_file_row_number"],
- False,
+ None,
),
# An append has no key, so the file at the watermark must not be read again.
- (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, False),
- (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None, False),
- # The flag does not change an append.
- (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None, True),
+ (ExternalDataSchemaSyncType.APPEND, FIRST_MODIFIED_AT + timedelta(minutes=1), ["a.csv"], None),
+ (ExternalDataSchemaSyncType.APPEND, None, ["c.csv", "b.csv", "a.csv"], None),
],
)
def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
@@ -302,7 +278,6 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
last_value: datetime | None,
expected_files: list[str],
expected_keys: list[str] | None,
- mirrors: bool,
) -> None:
# Path order is the reverse of age order, so the test tells the two apart.
for minutes, name in enumerate(["c.csv", "b.csv", "a.csv"]):
@@ -321,22 +296,20 @@ def test_sync_type_decides_which_files_are_read_and_whether_rows_merge(
inputs.sync_type = sync_type
inputs.source_cursor = SourceCursorManager.from_sync_type_config(source, {}, structlog.get_logger())
- with patch.object(source_base, "_mirrors_files", return_value=mirrors):
- response = source.source_for_pipeline(config, inputs)
- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+ response = source.source_for_pipeline(config, inputs)
+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
assert [(row["_file_name"], row["_file_row_number"]) for row in rows] == [
(f"orders/{name}", row_number) for name in expected_files for row_number in (0, 1)
]
assert response.primary_keys == expected_keys
- mirrored = mirrors and sync_type == ExternalDataSchemaSyncType.INCREMENTAL
assert response.replace_scope == (
ReplaceScope(
column="_file_name",
replaced=frozenset(f"orders/{name}" for name in expected_files),
retained=frozenset({"orders/a.csv", "orders/b.csv", "orders/c.csv"}),
)
- if mirrored
+ if sync_type == ExternalDataSchemaSyncType.INCREMENTAL
else None
)
@@ -359,9 +332,8 @@ def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path:
inputs.sync_type = ExternalDataSchemaSyncType.INCREMENTAL
cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())
inputs.source_cursor = cursor
- with patch.object(source_base, "_mirrors_files", return_value=True):
- response = source.source_for_pipeline(config, inputs)
- rows = sum(table.num_rows for table in cast(Iterable[pa.Table], response.items()))
+ response = source.source_for_pipeline(config, inputs)
+ rows = sum(table.num_rows for table in cast(Iterable[pa.Table], response.items()))
return response, rows, cursor
first, first_rows, cursor = sync({})
@@ -381,23 +353,20 @@ def test_a_mirrored_sync_of_a_bucket_that_did_not_change_reads_nothing(tmp_path:
@pytest.mark.parametrize(
- "sync_type, has_stored_cursor, mirrors, expected_ids, expected_columns, expected_reset",
+ "sync_type, has_stored_cursor, expected_ids, expected_columns, expected_reset",
[
- (ExternalDataSchemaSyncType.APPEND, True, False, [2], {"id", "_delta_version"}, False),
- (ExternalDataSchemaSyncType.APPEND, False, False, [1, 2], {"id", "_delta_version"}, False),
+ (ExternalDataSchemaSyncType.APPEND, True, [2], {"id", "_delta_version"}, False),
+ (ExternalDataSchemaSyncType.APPEND, False, [1, 2], {"id", "_delta_version"}, False),
# A full refresh ignores the stored cursor, and still records the version it wrote.
- (ExternalDataSchemaSyncType.FULL_REFRESH, True, False, [1, 2], {"id"}, False),
+ (ExternalDataSchemaSyncType.FULL_REFRESH, True, [1, 2], {"id"}, False),
# An incremental sync names each row's data file, so the loader can replace rows by file.
- (ExternalDataSchemaSyncType.INCREMENTAL, True, True, [2], {"id", "_delta_version", "_file_name"}, False),
- # Without the flag the loader can't delete by file, so the run rebuilds the table.
- (ExternalDataSchemaSyncType.INCREMENTAL, True, False, [1, 2], {"id", "_delta_version", "_file_name"}, True),
+ (ExternalDataSchemaSyncType.INCREMENTAL, True, [2], {"id", "_delta_version", "_file_name"}, False),
],
)
def test_delta_sync_reads_from_the_stored_cursor_and_stages_the_next_one(
tmp_path: Path,
sync_type: ExternalDataSchemaSyncType,
has_stored_cursor: bool,
- mirrors: bool,
expected_ids: list[int],
expected_columns: set[str],
expected_reset: bool,
@@ -419,9 +388,8 @@ def test_delta_sync_reads_from_the_stored_cursor_and_stages_the_next_one(
inputs.sync_type = sync_type
cursor = SourceCursorManager.from_sync_type_config(source, stored, structlog.get_logger())
inputs.source_cursor = cursor
- with patch.object(source_base, "_mirrors_files", return_value=mirrors):
- response = source.source_for_pipeline(config, inputs)
- rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+ response = source.source_for_pipeline(config, inputs)
+ rows = [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
return response, rows, cursor
_, _, first_cursor = sync({})
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index eb4df59d4a84..1440fcae8f4f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -7,12 +7,7 @@ import pyarrow as pa
from botocore.exceptions import ClientError
from structlog.types import FilteringBoundLogger
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern
-from products.warehouse_sources.backend.temporal.data_imports.schema_flags import (
- REPLACE_ROWS_BY_FILE_FLAG,
- is_schema_flag_enabled,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, SimpleSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import CursorSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_MODIFIED_AT_COLUMN
@@ -113,12 +108,10 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
A full refresh reads each table in full, so the table matches the bucket at the time of the sync.
The incremental and append sync types read only the files modified since the last sync. An append
never removes rows, so the rows of a file deleted from the bucket stay in the table. An incremental
- sync with `REPLACE_ROWS_BY_FILE_FLAG` on also deletes the rows of a deleted file, and a changed file
- replaces its rows. Without the flag it merges on the file and row number and keeps the rows of
- deleted files. An append sync of a Delta table reads only the data files committed since the
- version of the last sync. An incremental sync of a Delta table also replaces the rows of each data
- file that a commit removed, so deletes, updates, merges and compactions reach the table. Without
- the flag it rebuilds the table on every run.
+ sync also deletes the rows of a deleted file, and a changed file replaces its rows. An append sync
+ of a Delta table reads only the data files committed since the version of the last sync. An
+ incremental sync of a Delta table also replaces the rows of each data file that a commit removed,
+ so deletes, updates, merges and compactions reach the table.
"""
@abstractmethod
@@ -258,7 +251,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
if table.kind == "delta":
return self._delta_response(store, table, inputs)
selection = FileSelection.from_inputs(inputs)
- if selection.merges and _mirrors_files(inputs):
+ if selection.merges:
return self._mirrored_files_response(store, table, inputs, selection)
def items() -> Iterator[pa.Table]:
@@ -267,8 +260,8 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
return SourceResponse(
name=inputs.schema_name,
items=items,
- # Only a merge needs a key. Without one, a full refresh replaces the table and an append adds to it.
- primary_keys=FILE_PRIMARY_KEYS if selection.merges else None,
+ # A full refresh replaces the table and an append adds to it, so neither needs a key.
+ primary_keys=None,
chunk_size_bytes=CHUNK_SIZE_BYTES,
preserve_column_names=table.preserve_column_names,
)
@@ -308,9 +301,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
cursor_manager = self.get_cursor_manager(inputs)
appends = inputs.should_use_incremental_field
by_file = appends and inputs.sync_type == ExternalDataSchemaSyncType.INCREMENTAL
- # Without the flag the loader can't delete by file, so an incremental sync rebuilds the
- # table from the snapshot on every run.
- stored = cursor_manager.load() if appends and (not by_file or _mirrors_files(inputs)) else None
+ stored = cursor_manager.load() if appends else None
# The plan is made here and not in `items`, because the response has to say up front
# whether the destination must be rebuilt.
read = plan_delta_read(
@@ -412,11 +403,3 @@ def _iter_files(
yield from iter_file_tables(stream, stored, file_format, resolved.csv, logger)
except PermissionError as e:
_raise_translated(e, store.root)
-
-
-def _mirrors_files(inputs: SourceInputs) -> bool:
- """Whether an incremental read deletes the rows of deleted and rewritten files (see `ReplaceScope`)."""
- schema = (
- ExternalDataSchema.objects.filter(team_id=inputs.team_id, id=inputs.schema_id).select_related("source").first()
- )
- return schema is not None and is_schema_flag_enabled(schema, REPLACE_ROWS_BY_FILE_FLAG)