Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): retry opening a delta table that is briefly missing (#129140)

Tom Owersd3d77b5077b1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
index 1e7f56375ffb..b4d62e145cb2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -1,4 +1,5 @@
import re
+import time
import dataclasses
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
@@ -25,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
DELTA_BATCH_ROWS,
+ DELTA_MISSING_RETRY_DELAYS_SECONDS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope
@@ -199,7 +201,18 @@ def plan_delta_read(
the stored rows don't name their files.
"""
with _delta_errors(table_name):
- table = DeltaTable(uri, storage_options=storage_options)
+ # A writer that replaces the table deletes its folder and writes it again, so a missing
+ # table can come back seconds later.
+ for attempt, delay in enumerate(DELTA_MISSING_RETRY_DELAYS_SECONDS, start=1):
+ try:
+ table = DeltaTable(uri, storage_options=storage_options)
+ break
+ except TableNotFoundError:
+ if logger is not None:
+ logger.info("Delta table not found, retrying", table=table_name, attempt=attempt)
+ time.sleep(delay)
+ else:
+ table = DeltaTable(uri, storage_options=storage_options)
version = table.version()
cursor = ObjectStorageDeltaCursor(version=version, table_id=str(table.metadata().id), file_names=by_file)
dataset = cast(ds.FileSystemDataset, table.to_pyarrow_dataset())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py
index 03387b79e0f2..57ae656cb9cd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py
@@ -29,6 +29,7 @@ MAX_FIRST_CSV_RECORD_BYTES = 1024 * 1024
MAX_UNPACKED_PARQUET_BYTES = 2 * 1024 * 1024 * 1024
PARQUET_BATCH_ROWS = 10_000
DELTA_BATCH_ROWS = 16_384
+DELTA_MISSING_RETRY_DELAYS_SECONDS: tuple[float, ...] = (5.0, 10.0, 20.0)
# Arrow batches from wide files can be large, so the pipeline's batcher flushes before this many bytes.
CHUNK_SIZE_BYTES = 100 * 1024 * 1024
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 173c3b382d49..0e6d357b2694 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -10,7 +10,7 @@ from types import SimpleNamespace
from typing import Any
import pytest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, call, patch
from django.test import override_settings
@@ -21,6 +21,7 @@ import pyarrow.parquet as pq
from botocore.exceptions import ClientError
from botocore.stub import Stubber
from deltalake import DeltaTable, WriterProperties, write_deltalake
+from deltalake.exceptions import DeltaError, TableNotFoundError
from fsspec.implementations.local import LocalFileSystem
from parameterized import parameterized
@@ -800,6 +801,59 @@ def _files_and_ids(path: str, version: int | None = None) -> list[tuple[str, int
class TestDeltaCursor:
+ @parameterized.expand(
+ [
+ ("recovers", TableNotFoundError, 2, (5.0, 10.0), None, None),
+ (
+ "stays_missing",
+ TableNotFoundError,
+ 4,
+ (5.0, 10.0, 20.0),
+ ObjectStorageError,
+ "no longer at its folder",
+ ),
+ ("other_delta_error", DeltaError, 1, (), DeltaError, "other failure"),
+ ]
+ )
+ def test_open_retries_only_when_the_delta_table_is_missing(
+ self,
+ _name: str,
+ error_type: type[Exception],
+ failed_opens: int,
+ expected_delays: tuple[float, ...],
+ expected_error: type[Exception] | None,
+ expected_message: str | None,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as folder:
+ path = str(Path(folder) / "sales")
+ write_deltalake(path, pa.table({"id": [1]}))
+ real_table = DeltaTable(path)
+ errors = [
+ error_type("other failure" if error_type is DeltaError else "missing") for _ in range(failed_opens)
+ ]
+ logger = MagicMock()
+
+ with (
+ patch.object(delta, "DeltaTable", side_effect=[*errors, real_table]) as open_table,
+ patch.object(delta.time, "sleep") as sleep,
+ ):
+ if expected_error is None:
+ read = plan_delta_read(path, {}, "sales", logger=logger)
+ assert [row for batch in read.iter_tables() for row in batch.to_pylist()] == [{"id": 1}]
+ else:
+ with pytest.raises(expected_error, match=expected_message):
+ plan_delta_read(path, {}, "sales", logger=logger)
+
+ assert open_table.call_count == failed_opens + (expected_error is None)
+ assert sleep.call_args_list == [call(delay) for delay in expected_delays]
+ retry_logs = [
+ entry for entry in logger.info.call_args_list if entry.args[0] == "Delta table not found, retrying"
+ ]
+ assert retry_logs == [
+ call("Delta table not found, retrying", table="sales", attempt=attempt)
+ for attempt in range(1, len(expected_delays) + 1)
+ ]
+
@parameterized.expand(
[
("no_cursor_reads_the_snapshot", None, [_append], False, False, [1, 2, 3], 1),