:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): retry opening a delta table that is briefly missing (#129140)
Tom Owersd3d77b5077b1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pyindex 1e7f56375ffb..b4d62e145cb2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -1,4 +1,5 @@import re+import timeimport dataclassesfrom collections.abc import Iterable, Iteratorfrom contextlib import contextmanager@@ -25,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (DELTA_BATCH_ROWS,+ DELTA_MISSING_RETRY_DELAYS_SECONDS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope@@ -199,7 +201,18 @@ def plan_delta_read(the stored rows don't name their files."""with _delta_errors(table_name):- table = DeltaTable(uri, storage_options=storage_options)+ # A writer that replaces the table deletes its folder and writes it again, so a missing+ # table can come back seconds later.+ for attempt, delay in enumerate(DELTA_MISSING_RETRY_DELAYS_SECONDS, start=1):+ try:+ table = DeltaTable(uri, storage_options=storage_options)+ break+ except TableNotFoundError:+ if logger is not None:+ logger.info("Delta table not found, retrying", table=table_name, attempt=attempt)+ time.sleep(delay)+ else:+ table = DeltaTable(uri, storage_options=storage_options)version = table.version()cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id), file_names=by_file)dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.pyindex 03387b79e0f2..57ae656cb9cd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py@@ -29,6 +29,7 @@ MAX_FIRST_CSV_RECORD_BYTES = 1024 * 1024MAX_UNPACKED_PARQUET_BYTES = 2 * 1024 * 1024 * 1024PARQUET_BATCH_ROWS = 10_000DELTA_BATCH_ROWS = 16_384+DELTA_MISSING_RETRY_DELAYS_SECONDS: tuple[float, ...] = (5.0, 10.0, 20.0)# Arrow batches from wide files can be large, so the pipeline's batcher flushes before this many bytes.CHUNK_SIZE_BYTES = 100 * 1024 * 1024diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 173c3b382d49..0e6d357b2694 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -10,7 +10,7 @@ from types import SimpleNamespacefrom typing import Anyimport pytest-from unittest.mock import MagicMock+from unittest.mock import MagicMock, call, patchfrom django.test import override_settings@@ -21,6 +21,7 @@ import pyarrow.parquet as pqfrom botocore.exceptions import ClientErrorfrom botocore.stub import Stubberfrom deltalake import DeltaTable, WriterProperties, write_deltalake+from deltalake.exceptions import DeltaError, TableNotFoundErrorfrom fsspec.implementations.local import LocalFileSystemfrom parameterized import parameterized@@ -800,6 +801,59 @@ def _files_and_ids(path: str, version: int | None = None) -> list[tuple[str, intclass TestDeltaCursor:+ @parameterized.expand(+ [+ ("recovers", TableNotFoundError, 2, (5.0, 10.0), None, None),+ (+ "stays_missing",+ TableNotFoundError,+ 4,+ (5.0, 10.0, 20.0),+ ObjectStorageError,+ "no longer at its folder",+ ),+ ("other_delta_error", DeltaError, 1, (), DeltaError, "other failure"),+ ]+ )+ def test_open_retries_only_when_the_delta_table_is_missing(+ self,+ _name: str,+ error_type: type[Exception],+ failed_opens: int,+ expected_delays: tuple[float, ...],+ expected_error: type[Exception] | None,+ expected_message: str | None,+ ) -> None:+ with tempfile.TemporaryDirectory() as folder:+ path = str(Path(folder) / "sales")+ write_deltalake(path, pa.table({"id": [1]}))+ real_table = DeltaTable(path)+ errors = [+ error_type("other failure" if error_type is DeltaError else "missing") for _ in range(failed_opens)+ ]+ logger = MagicMock()++ with (+ patch.object(delta, "DeltaTable", side_effect=[*errors, real_table]) as open_table,+ patch.object(delta.time, "sleep") as sleep,+ ):+ if expected_error is None:+ read = plan_delta_read(path, {}, "sales", logger=logger)+ assert [row for batch in read.iter_tables() for row in batch.to_pylist()] == [{"id": 1}]+ else:+ with pytest.raises(expected_error, match=expected_message):+ plan_delta_read(path, {}, "sales", logger=logger)++ assert open_table.call_count == failed_opens + (expected_error is None)+ assert sleep.call_args_list == [call(delay) for delay in expected_delays]+ retry_logs = [+ entry for entry in logger.info.call_args_list if entry.args[0] == "Delta table not found, retrying"+ ]+ assert retry_logs == [+ call("Delta table not found, retrying", table="sales", attempt=attempt)+ for attempt in range(1, len(expected_delays) + 1)+ ]+@parameterized.expand([("no_cursor_reads_the_snapshot", None, [_append], False, False, [1, 2, 3], 1),