:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): stop size splits of deltalite merge tables (#119687)
Tom Owers16c147c4037c
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 5fae06142dbe..2501f6dda2d0 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -90,6 +90,16 @@ DATA_WAREHOUSE_TARGET_PARTITION_BYTES = get_from_env("DATA_WAREHOUSE_TARGET_PARTITION_BYTES", 500_000_000, type_cast=int)+# The size-driven split budget for tables that upsert on a primary key (incremental, webhook, xmin).+# Those upserts go through deltalite, whose peak memory does not grow with partition size, while on an+# md5 table every extra partition adds one more rewritten file to each merge. The delta-rs MERGE runs on+# such a table only for a batch that deltalite fails, and this larger cap bounds partition size for that+# fallback. The OOM-history trigger still splits a table whose fallback runs out of memory.+# A value below DATA_WAREHOUSE_TARGET_PARTITION_BYTES has no effect.+DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES = get_from_env(+ "DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES", 2_000_000_000, type_cast=int+)+# How often each sync activity self-reports its workload (phase, buffer bytes, RSS) to the warehouse# Redis, for post-mortem enrichment of silent worker deaths. Zero or negative disables reporting# entirely (the fleet kill switch — hooks become no-ops and no thread starts).diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex 9de070267e01..c5de0957e4de 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -321,7 +321,7 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djan# See `sources/common/history_window.py`. A column rather than a `sync_type_config` key# because it has to outlive a reset, and clearing that blob is what a reset is for.history_start = models.DateTimeField(null=True, blank=True)- # { "incremental_field": string, "incremental_field_type": string, "incremental_field_last_value": any, "incremental_field_earliest_value": any, "incremental_field_lookback_seconds": int | None, "reset_pipeline": bool, "partitioning_enabled": bool, "partition_count": int, "partition_size": int, "partition_mode": str, "partitioning_keys": list[str], "chunk_size_override": int | None, "primary_key_columns": list[str] | None, "verified_primary_keys": list[str] | None, "source_cursor": { "kind": str, "data": dict }, "max_partition_bytes": int, "partition_measurement": { "job_id": str, "phase": "pre_extraction" | "post_load", "budget": int, "healthy": bool }, "last_repartition_at": iso8601 str, "repartition_pending": { "partition_mode": str, "partition_format": str | None, "partition_count": int | None, "partition_size": int | None, "partition_keys": list[str], "trigger_reason": str }, "repartition_swap": { "state": "ready", "temp_uri": str, "live_uri": str }, "repartition_rewrite": { "temp_uri": str, "rows_written": int, "target": dict }, "query_folder_state": { "<table>__query": { "active": str, "active_since": iso8601 str, "active_job_id": str, "history_since": iso8601 str, "inactive_since": { str: iso8601 str } } }, "registered_schema_fingerprint": str }+ # { "incremental_field": string, "incremental_field_type": string, "incremental_field_last_value": any, "incremental_field_earliest_value": any, "incremental_field_lookback_seconds": int | None, "reset_pipeline": bool, "partitioning_enabled": bool, "partition_count": int, "partition_size": int, "partition_mode": str, "partitioning_keys": list[str], "chunk_size_override": int | None, "primary_key_columns": list[str] | None, "verified_primary_keys": list[str] | None, "source_cursor": { "kind": str, "data": dict }, "max_partition_bytes": int, "partition_measurement": { "job_id": str, "phase": "pre_extraction" | "post_load", "budget": int, "split_budget": int, "healthy": bool }, "last_repartition_at": iso8601 str, "repartition_pending": { "partition_mode": str, "partition_format": str | None, "partition_count": int | None, "partition_size": int | None, "partition_keys": list[str], "trigger_reason": str }, "repartition_swap": { "state": "ready", "temp_uri": str, "live_uri": str }, "repartition_rewrite": { "temp_uri": str, "rows_written": int, "target": dict }, "query_folder_state": { "<table>__query": { "active": str, "active_since": iso8601 str, "active_job_id": str, "history_since": iso8601 str, "inactive_since": { str: iso8601 str } } }, "registered_schema_fingerprint": str }sync_type_config = models.JSONField(default=dict,blank=True,diff --git a/products/warehouse_sources/backend/temporal/data_imports/README.md b/products/warehouse_sources/backend/temporal/data_imports/README.mdindex 121a544f83b8..1813c46c2307 100644--- a/products/warehouse_sources/backend/temporal/data_imports/README.md+++ b/products/warehouse_sources/backend/temporal/data_imports/README.md@@ -61,6 +61,7 @@ Tuning and gating:- **Repairing the existing backlog.** The automatic coarsening path will not reach a table that keeps recording OOM occurrences, and the tables most in need of coarsening are exactly the ones the unreliable OOM signal keeps firing on. `./manage.py stage_warehouse_coarsening` is the way in: it nominates tables by setting a `coarsen_requested` marker, and the next sync evaluates them. A nomination skips the policy gates (OOM history, layout age, minimum partition count) because an operator has made that call, but never the safety check: the controller still measures the live layout and refuses any target that would not fit the budget, so a nomination can only ever be a no-op. Nominated rewrites report `trigger_reason=coarsening_requested`. Dry run by default, `--limit` defaults low, and it is worth keeping low: a rewrite blocks that table's sync for as long as it runs, which is seconds for small tables and hours for the largest.- **Putting back a table that was split too fine.** A table split from just over the budget cannot pass the coarsening safety check, because its coarser partitions are larger than half the budget. `./manage.py stage_warehouse_repartition --schema-id <id> --format <tier>` covers that case for datetime tables. The operator names the coarser tier, and the command measures the live table and refuses a target whose largest partition would be over the full budget. It then stages the same `repartition_pending` target as the admin page (`trigger_reason=admin`), and the next scheduled sync runs the rewrite. A scheduled full refresh skips the repartition step, so the target then waits for the sync after it. `--schema-id` repeats, and it is a dry run by default.- The budget is tunable via the `DATA_WAREHOUSE_TARGET_PARTITION_BYTES` setting (default ~0.5 GB at-rest → ~10 GB worst-case merge). Worker pods are multi-tenant, so the budget leaves headroom for concurrent merges under the 29 GB pod limit rather than sizing to a single merge.+- Tables that upsert on a primary key (incremental, webhook and xmin sync types) use a larger size budget for the `proactive_threshold` trigger: `DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES` (default ~2 GB at-rest, never below the base budget). These upserts go through deltalite, whose peak memory does not grow with partition size. On an md5 table each upsert touches every partition and rewrites at least one file in each, so a split there only makes every later merge slower. The delta-rs MERGE runs on such a table only for a batch that deltalite fails, and the engine is chosen per batch, so the controller cannot know it in advance. When deltalite fails on a partition above the base budget, the writer refuses the delta-rs MERGE and raises a retryable error, so the larger cap cannot lead to an oversized MERGE (metric outcome `fallback_refused`). The OOM trigger, its floor and coarsening keep the base budget. Full-refresh tables keep the base budget too.- The OOM-history override is tunable via `DATA_WAREHOUSE_REPARTITION_OOM_THRESHOLD` (default 3) and `DATA_WAREHOUSE_REPARTITION_OOM_WINDOW_DAYS` (default 7). An OOM-triggered rewrite of an under-budget table steps one tier finer per cooldown cycle, converging as the (still-recorded) OOMs continue.- The split floor and the coarsening trigger are both derived from the budget rather than tuned separately (`COARSEN_TRIGGER_DIVISOR`), so the two directions stay consistent by construction.- The coarsening gate's own tunables are `COARSEN_OOM_FREE_DAYS` (default 14, deliberately longer than the split window) and `DATA_WAREHOUSE_COARSEN_BLOCK_MERGE_PEAK_BYTES` (default 1 MB), the peak above which a merge-phase death blocks regardless of how the classification rules explained it. `DELTA_COARSEN_DECLINE_TOTAL` records which gate declined, labelled by reason.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex c2efa4fa8ffc..126784a5def4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.deltable_is_misaligned,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import (+ DeltaliteFallbackRefusedError,DeltaWriter,_committed_file_count_change,_committed_version,@@ -52,6 +53,8 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delcommit_covers_batch,)+_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition"+_CONTROLLER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller"_WRITER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer"@@ -1339,6 +1342,78 @@ class TestDeltaliteWritePath:wrote = await self._call(helper)assert wrote is False # deltalite blew up -> caller falls through to the delta-rs MERGE+ @parameterized.expand(+ [+ ("under_budget", {"a": 100, "b": 200}, None, False, "fallback"),+ ("over_budget", {"a": 100, "b": 900}, None, True, "fallback_refused"),+ ("untouched_partition_is_ignored", {"a": 100, "b": 900}, None, False, "fallback"),+ ("size_unknown", None, RuntimeError("log unreadable"), False, "fallback"),+ ]+ )+ @pytest.mark.asyncio+ async def test_fallback_is_refused_above_the_merge_budget(+ self, name: str, sizes: dict | None, measure_error: Exception | None, refused: bool, outcome: str+ ):+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.metrics import (+ DELTALITE_WRITE_TOTAL,+ )++ helper = self._helper()+ fake_table = MagicMock()+ deltalite_error = RuntimeError("commit conflict, retries exhausted")+ fake_table.upsert.side_effect = deltalite_error+ fake_deltalite = MagicMock()+ fake_deltalite.DeltaLiteTable.open.return_value = fake_table+ touched = ["b"] if name == "over_budget" else ["a"]+ data = pa.table({"id": pa.array([1], pa.int64()), PARTITION_KEY: touched})+ before = DELTALITE_WRITE_TOTAL.labels(outcome=outcome)._value.get()+ with (+ patch.dict("sys.modules", {"deltalite": fake_deltalite}),+ patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),+ patch.object(helper, "_get_credentials", return_value={}),+ patch(f"{_REPARTITION_MODULE}.measure_partition_bytes", side_effect=measure_error, return_value=sizes),+ patch(f"{_CONTROLLER_MODULE}.target_partition_bytes", return_value=500),+ ):+ write = DeltaWriter(helper)._write_via_deltalite(+ existing_delta_table=MagicMock(),+ data=data,+ normalized_primary_keys=["id"],+ use_partitioning=True,+ commit_metadata=None,+ )+ if refused:+ with pytest.raises(DeltaliteFallbackRefusedError) as exc_info:+ await write+ assert exc_info.value.__cause__ is deltalite_error+ else:+ assert await write is False+ assert DELTALITE_WRITE_TOTAL.labels(outcome=outcome)._value.get() == before + 1++ @pytest.mark.asyncio+ async def test_no_size_check_when_deltalite_succeeds(self):+ helper = self._helper()+ fake_table = MagicMock()+ fake_table.upsert.return_value = SimpleNamespace(version=5)+ fake_deltalite = MagicMock()+ fake_deltalite.DeltaLiteTable.open.return_value = fake_table+ existing = MagicMock()+ existing.version.return_value = 4+ with (+ patch.dict("sys.modules", {"deltalite": fake_deltalite}),+ patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),+ patch.object(helper, "_get_credentials", return_value={}),+ patch(f"{_REPARTITION_MODULE}.measure_partition_bytes") as measure,+ ):+ wrote = await DeltaWriter(helper)._write_via_deltalite(+ existing_delta_table=existing,+ data=pa.table({"id": pa.array([1], pa.int64())}),+ normalized_primary_keys=["id"],+ use_partitioning=False,+ commit_metadata=None,+ )+ assert wrote is True+ measure.assert_not_called()+@parameterized.expand([("log",), ("metric",)])@pytest.mark.asyncioasync def test_post_commit_failure_does_not_fall_back(self, failing_step: str):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 47d405a8e24b..cc64bd8d8169 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -36,6 +36,13 @@ if TYPE_CHECKING:from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef+class DeltaliteFallbackRefusedError(Exception):+ """Deltalite failed on a partition too large for the delta-rs MERGE fallback.++ Retryable: the retry gives deltalite another attempt, and the MERGE would risk an OOM that kills the worker.+ """++def _write_deltalake(table_or_uri: str | deltalake.DeltaTable,table_data: pa.Table,@@ -229,6 +236,33 @@ class DeltaWriter:)return deduped+ async def _fallback_partition_size(+ self, existing_delta_table: deltalake.DeltaTable, data: pa.Table, use_partitioning: bool+ ) -> tuple[int | None, int]:+ """Largest at-rest partition the delta-rs MERGE would touch, and the budget it must fit.++ Reads the in-memory Delta log only, and runs only when deltalite has already failed. The+ size is None when it cannot be measured, and the caller then keeps the fallback.+ """+ # Lazy: the controller pulls in the Django models and the pipeline_v3 chain — circular.+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition import (+ measure_partition_bytes,+ )+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller import (+ target_partition_bytes,+ )++ budget_bytes = target_partition_bytes()+ try:+ sizes = await asyncio.to_thread(measure_partition_bytes, existing_delta_table)+ if use_partitioning and PARTITION_KEY in data.column_names:+ touched = {str(v) for v in pc.unique(data[PARTITION_KEY]).to_pylist()}+ sizes = {k: v for k, v in sizes.items() if k is not None and k in touched}+ return max(sizes.values(), default=0), budget_bytes+ except Exception as e: # noqa: BLE001 - a failed measurement keeps the fallback+ await self._logger.awarning(f"could not measure the merge partition size before the fallback: {e}")+ return None, budget_bytes+async def _write_via_deltalite(self,*,@@ -323,6 +357,23 @@ class DeltaWriter:stats = await asyncio.to_thread(_upsert)duration_s = time.perf_counter() - startedexcept Exception as e: # noqa: BLE001 - pre-commit failure: nothing committed, fall back to MERGE+ partition_bytes, budget_bytes = await self._fallback_partition_size(+ existing_delta_table, data, use_partitioning+ )+ if partition_bytes is not None and partition_bytes > budget_bytes:+ await self._logger.awarning(+ "deltalite write failed; refusing the delta-rs MERGE fallback above the merge budget",+ schema_id=str(self._table.job.schema_id),+ partition_bytes=partition_bytes,+ budget_bytes=budget_bytes,+ error=str(e),+ )+ with contextlib.suppress(Exception):+ DELTALITE_WRITE_TOTAL.labels(outcome="fallback_refused").inc()+ raise DeltaliteFallbackRefusedError(+ f"deltalite write failed and the largest touched partition ({partition_bytes} bytes) "+ f"is above the delta-rs MERGE budget ({budget_bytes} bytes)"+ ) from eawait self._logger.awarning(f"deltalite write failed; falling back to delta-rs MERGE (sync unaffected): {e}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.pyindex 39915a5980d8..3376caba263c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py@@ -85,6 +85,39 @@ def target_partition_bytes() -> int:return int(getattr(settings, "DATA_WAREHOUSE_TARGET_PARTITION_BYTES", 500_000_000))+# The loader writes these sync types as incremental upserts on the primary key (CDC also upserts, but+# detection excludes it).+KEYED_MERGE_SYNC_TYPES = frozenset(+ {ExternalDataSchema.SyncType.INCREMENTAL, ExternalDataSchema.SyncType.WEBHOOK, ExternalDataSchema.SyncType.XMIN}+)+++def merge_target_partition_bytes() -> int:+ return max(+ target_partition_bytes(), int(getattr(settings, "DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES", 2_000_000_000))+ )+++def split_budget_bytes(schema: ExternalDataSchema) -> int:+ """The largest partition the size-driven split leaves alone on this table.++ `target_partition_bytes` was sized for the delta-rs MERGE, whose working set grows with the+ partition it merges into. Every keyed upsert now goes through deltalite first, and deltalite's+ peak memory follows the shape of the files it rewrites, not the partition size. On an md5 table+ each upsert touches every partition and rewrites at least one file in each, so a split there+ makes every later merge slower and saves no memory.++ Which engine writes is decided per batch: the delta-rs MERGE runs on a batch that deltalite+ fails, and nothing records that per table. So a keyed-merge table still gets a budget, sized for+ that fallback (`merge_target_partition_bytes`), rather than none. The writer refuses the+ fallback on a partition above the base budget (`target_partition_bytes`) and retries the batch,+ so the larger cap cannot lead to an oversized delta-rs MERGE.+ """+ if schema.sync_type in KEYED_MERGE_SYNC_TYPES:+ return merge_target_partition_bytes()+ return target_partition_bytes()++def min_splittable_partition_bytes() -> int:"""The smallest partition an OOM-triggered split is allowed to produce.@@ -174,6 +207,8 @@ def pre_extraction_measurement_is_redundant(schema: ExternalDataSchema, job: Extreturn Falseif measurement.get("budget") != target_partition_bytes():return False+ if measurement.get("split_budget") != split_budget_bytes(schema):+ return Falseif (not schema.is_appendand ExternalDataSchemaOOMEvent.recent_count(schema, days=repartition_oom_window_days())@@ -476,7 +511,9 @@ async def maybe_flag_for_repartition(# split only rewrites every row for no gain. Coarsening still applies, because fewer files# make reads of the table cheaper.split_applies = not schema.is_append- over_budget = split_applies and max_bytes > budget+ # The size trigger only. The OOM trigger, its floor and coarsening stay on `budget`.+ size_split_budget = split_budget_bytes(schema)+ over_budget = split_applies and max_bytes > size_split_budget# Hybrid trigger: a table that has actually OOM'd repeatedly is repartitioned even when its# largest partition looks within budget — the compressed at-rest size under-counts the merge's@@ -506,6 +543,7 @@ async def maybe_flag_for_repartition("job_id": str(job.id),"phase": phase,"budget": budget,+ "split_budget": size_split_budget,"healthy": not over_budget and not oom_triggered and not coarsen_candidate,},)@@ -514,11 +552,11 @@ async def maybe_flag_for_repartition(verdict = "within budget and no repeated OOMs" if split_applies else "append table, a split does not apply"await logger.adebug(f"repartition: not needed, {verdict} schema_id={schema.id} "- f"max_partition_bytes={max_bytes} budget_bytes={budget} recent_oom_count={oom_count} "+ f"max_partition_bytes={max_bytes} budget_bytes={size_split_budget} recent_oom_count={oom_count} "f"partition_count={len(partition_bytes)}",schema_id=str(schema.id),max_partition_bytes=max_bytes,- budget_bytes=budget,+ budget_bytes=size_split_budget,recent_oom_count=oom_count,partition_count=len(partition_bytes),)@@ -532,7 +570,7 @@ async def maybe_flag_for_repartition(# killing a table whose partitions are already that small, and without this guard oom_history# drives the scheme finer tier by tier until it bottoms out (e.g. datetime at hour) and then# re-measures and re-emits the skip on every cooldown expiry forever.- split_budget = budget if over_budget else max(1, max_bytes // 2)+ split_budget = size_split_budget if over_budget else max(1, max_bytes // 2)floor = min_splittable_partition_bytes()if not over_budget and split_budget < floor:await logger.adebug(@@ -599,7 +637,7 @@ async def maybe_flag_for_repartition()return- # `split_budget` was computed with the floor check above: an over-budget table targets the+ # `split_budget` was computed with the floor check above: an over-budget table targets its size# budget, while an OOM-triggered one targets roughly half its current largest partition to force# a meaningfully finer scheme (md5 grows buckets, numerical halves the row-size, datetime steps# one tier finer).diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.pyindex 6bcdb2b4df74..d7f989fcd0f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py@@ -317,6 +317,72 @@ class TestRepartitionDetection:assert schema.last_repartition_at is Noneassert capture.call_count == 0+ @pytest.mark.parametrize(+ "sync_type, base_budget, merge_budget, recent_ooms, expected_trigger",+ [+ (ExternalDataSchema.SyncType.INCREMENTAL, 1, 10**12, 0, None),+ (ExternalDataSchema.SyncType.WEBHOOK, 1, 10**12, 0, None),+ (ExternalDataSchema.SyncType.XMIN, 1, 10**12, 0, None),+ (ExternalDataSchema.SyncType.INCREMENTAL, 1, 2, 0, "proactive_threshold"),+ (ExternalDataSchema.SyncType.INCREMENTAL, 1, 0, 0, "proactive_threshold"),+ (ExternalDataSchema.SyncType.INCREMENTAL, 1, 10**12, 3, "oom_history"),+ (ExternalDataSchema.SyncType.FULL_REFRESH, 1, 10**12, 0, "proactive_threshold"),+ (None, 1, 10**12, 0, "proactive_threshold"),+ ],+ ids=[+ "incremental_over_base_budget_is_not_split",+ "webhook_over_base_budget_is_not_split",+ "xmin_over_base_budget_is_not_split",+ "merge_table_over_its_fallback_cap_still_splits",+ "merge_cap_below_base_budget_falls_back_to_base",+ "merge_table_with_repeated_ooms_still_splits",+ "full_refresh_keeps_the_base_budget",+ "unknown_sync_type_keeps_the_base_budget",+ ],+ )+ def test_keyed_merge_tables_split_on_size_only_past_the_merge_budget(+ self,+ team,+ sync_type: str | None,+ base_budget: int,+ merge_budget: int,+ recent_ooms: int,+ expected_trigger: str | None,+ ):+ schema = _make_schema(+ team,+ {"partitioning_enabled": True, "partition_mode": "md5", "partition_count": 2, "partitioning_keys": ["id"]},+ )+ schema.sync_type = sync_type+ schema.save()+ for _ in range(recent_ooms):+ ExternalDataSchemaOOMEvent.objects.for_team(schema.team_id).create(team_id=schema.team_id, schema=schema)++ with tempfile.TemporaryDirectory() as d:+ delta = _write_partitioned_delta(f"{d}/t", ["0", "0", "1", "1"])+ with (+ override_settings(+ DATA_WAREHOUSE_TARGET_PARTITION_BYTES=base_budget,+ DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES=merge_budget,+ ),+ patch.object(ctrl, "repartition_oom_threshold", return_value=3),+ patch.object(ctrl, "min_splittable_partition_bytes", return_value=1),+ patch.object(ctrl, "capture_repartition_event"),+ ):+ self._detect(team, schema, delta)++ schema.refresh_from_db()+ pending = schema.repartition_pending+ if expected_trigger is None:+ assert pending is None+ assert schema.partition_measurement is not None+ assert schema.partition_measurement["healthy"] is True+ else:+ assert pending is not None+ assert pending["trigger_reason"] == expected_trigger+ assert pending["partition_mode"] == "md5"+ assert pending["partition_count"] > 2+class TestPartitionMeasurementRecord:@pytest.mark.parametrize(@@ -345,6 +411,7 @@ class TestPartitionMeasurementRecord:"job_id": str(job.id),"phase": phase,"budget": budget,+ "split_budget": budget,"healthy": healthy,}@@ -383,13 +450,16 @@ class TestPartitionMeasurementRecord:assert measurement is not None and measurement["healthy"] is False@pytest.mark.parametrize(- "previous_status, nominated, measured_budget, recent_ooms, redundant",+ "previous_status, nominated, sync_type, measured_budget, measured_split_budget, recent_ooms, redundant",[- (ExternalDataJob.Status.COMPLETED, False, 500, 0, True),- (ExternalDataJob.Status.FAILED, False, 500, 0, False),- (ExternalDataJob.Status.COMPLETED, True, 500, 0, False),- (ExternalDataJob.Status.COMPLETED, False, 400, 0, False),- (ExternalDataJob.Status.COMPLETED, False, 500, 3, False),+ (ExternalDataJob.Status.COMPLETED, False, None, 500, 500, 0, True),+ (ExternalDataJob.Status.FAILED, False, None, 500, 500, 0, False),+ (ExternalDataJob.Status.COMPLETED, True, None, 500, 500, 0, False),+ (ExternalDataJob.Status.COMPLETED, False, None, 400, 500, 0, False),+ (ExternalDataJob.Status.COMPLETED, False, None, 500, 500, 3, False),+ (ExternalDataJob.Status.COMPLETED, False, ExternalDataSchema.SyncType.INCREMENTAL, 500, 2000, 0, True),+ (ExternalDataJob.Status.COMPLETED, False, ExternalDataSchema.SyncType.INCREMENTAL, 500, 3000, 0, False),+ (ExternalDataJob.Status.COMPLETED, False, None, 500, None, 0, False),],ids=["previous_run_completed",@@ -397,12 +467,24 @@ class TestPartitionMeasurementRecord:"operator_nomination_waits","budget_changed_since_the_measurement","repeated_oom_kills_since_the_measurement",+ "merge_table_measured_against_its_split_budget",+ "split_budget_changed_since_the_measurement",+ "measurement_without_a_split_budget",],)def test_gate_reads_what_can_change_without_a_write(- self, team, previous_status: str, nominated: bool, measured_budget: int, recent_ooms: int, redundant: bool+ self,+ team,+ previous_status: str,+ nominated: bool,+ sync_type: str | None,+ measured_budget: int,+ measured_split_budget: int | None,+ recent_ooms: int,+ redundant: bool,):schema = _make_schema(team, {})+ schema.sync_type = sync_typeother_schema = ExternalDataSchema.objects.create(name="other", team=team, source=schema.source, sync_type_config={})@@ -423,13 +505,16 @@ class TestPartitionMeasurementRecord:"job_id": str(previous.id),"phase": "post_load","budget": measured_budget,+ "split_budget": measured_split_budget,"healthy": True,},**({"coarsen_requested": _NOMINATION} if nominated else {}),}with (- patch.object(ctrl, "target_partition_bytes", return_value=500),+ override_settings(+ DATA_WAREHOUSE_TARGET_PARTITION_BYTES=500, DATA_WAREHOUSE_MERGE_TARGET_PARTITION_BYTES=2000+ ),patch.object(ctrl.ExternalDataSchemaOOMEvent, "recent_count", return_value=recent_ooms),):assert ctrl.pre_extraction_measurement_is_redundant(schema, current) is redundantdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.pyindex 4ea7b7c079b3..ef9989b6b3f6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py@@ -101,6 +101,8 @@ CDC_DELETE_ENRICHMENT_VIOLATIONS_TOTAL = Counter(# deltalite real-write path (phase 2). `outcome` is one of:# written - deltalite performed the incremental merge and committed# fallback - deltalite was enabled but failed (or refused), so the delta-rs MERGE ran instead+# fallback_refused - deltalite failed and the largest touched partition is above the base budget, so+# the batch raised for a retry instead of running the delta-rs MERGEDELTALITE_WRITE_TOTAL = Counter("warehouse_load_deltalite_write_total","Incremental merges routed to deltalite's real write path, by outcome",