:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): let long deltalite upserts borrow idle pod memory (#121886)
Tom Owers38fb74dfdd7e
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex a2e823b6724f..4f03724d71f9 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -191,6 +191,15 @@ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env("DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool)+# Lets a deltalite upsert whose slice-sized plan is narrower than the widest plan borrow idle pod memory,+# up to DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES slices, and run more partition workers and+# readers. A borrow takes at most half of the free pod memory and never happens while an upsert waits for+# room. "advisory" logs the borrowed plan and keeps the slice-sized plan. "enforce" runs the borrowed plan,+# and needs DELTALITE_GOVERNOR_MODE=enforce. "off" computes nothing.+DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW = get_from_env("DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW", "advisory")+DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES = get_from_env(+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES", 4.0, type_cast=float+)# deltalite keeps the primary keys of the data files that a merge reads or writes in a per-process cache,# so a later merge into the same table decides most files without a read from object storage. The cache# size comes from the DELTALITE_PROBE_CACHE_BYTES environment variable (default 128 MiB per process). False makes every merge read eachdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.pyindex 13e32cce7810..86ad2c34034d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py@@ -26,6 +26,10 @@ Two layers, by design:governor predicted. The governor aims never to reach it; the backstop guarantees safety when aprediction is wrong.+The slices guarantee room for ``max_concurrent`` upserts, but a pod mostly runs far fewer. So an+upsert whose slice keeps its plan narrow can borrow idle memory, up to a few slices, to run more+partition workers. ``MemoryGovernor._borrowable_mb`` holds the rule that keeps a borrow fair.+The memory model follows the shape of the files a merge rewrites, not their total size. An upsertin deltalite 0.1.10 holds three things at its peak:@@ -183,6 +187,11 @@ _WAIT_POLL_S = 0.5_ROOM_TOLERANCE_MB = 1.0#: How often the RSS sampler reads the process RSS while an upsert runs._DEFAULT_RSS_SAMPLE_MS = 100.0+#: Most slices one upsert may borrow when the pod has idle memory. 4 slices let a wide-row merge+#: run 4 partition workers, and beyond 4 workers the gains vanish.+_DEFAULT_BORROW_MAX_SLICES = 4.0+#: deltalite's own default for ``DELTALITE_PROBE_CACHE_BYTES``.+_DEFAULT_PROBE_CACHE_BYTES = 128 * MBdef _rss_retention(environ: Mapping[str, str] = os.environ) -> float:@@ -830,6 +839,13 @@ class GovernorConfig:rss_sample_ms: float = _DEFAULT_RSS_SAMPLE_MS#: RSS per byte of predicted in-use memory. Read from the allocator settings by default.rss_retention: float = _RSS_RETENTION_GLIBC_DEFAULT+ #: Rollout gate for borrowing idle pod memory, with the same three values as ``mode``. Advisory+ #: computes and logs the borrowed plan. Enforce runs it, and only when ``mode`` is enforce too.+ borrow_mode: Mode = "advisory"+ #: Most slices one borrowed plan may reserve.+ borrow_max_slices: float = _DEFAULT_BORROW_MAX_SLICES+ #: Cap of deltalite's process-wide probe cache. No slice holds it, so a borrow leaves it free.+ probe_cache_mb: float = _DEFAULT_PROBE_CACHE_BYTES / MB@staticmethoddef from_env() -> GovernorConfig:@@ -847,8 +863,26 @@ class GovernorConfig:max_wait_s=max(0.0, _env_float("DELTALITE_GOVERNOR_MAX_WAIT_S", 60.0)),rss_sample_ms=max(0.0, _env_float("DELTALITE_GOVERNOR_RSS_SAMPLE_MS", _DEFAULT_RSS_SAMPLE_MS)),rss_retention=_rss_retention(),+ borrow_mode=GovernorConfig._resolve_borrow_mode(),+ borrow_max_slices=max(1.0, float(settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES)),+ probe_cache_mb=GovernorConfig._resolve_probe_cache_mb(),)+ @staticmethod+ def _resolve_borrow_mode() -> Mode:+ mode = str(settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW).strip().lower()+ if mode not in ("off", "advisory", "enforce"):+ logger.warning("invalid DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW=%r; defaulting to advisory", mode)+ return "advisory"+ return mode # type: ignore[return-value]++ @staticmethod+ def _resolve_probe_cache_mb() -> float:+ """The probe cache cap, read from the variable deltalite reads. 0 when the writer turns the cache off."""+ if not settings.DATA_WAREHOUSE_DELTALITE_PROBE_CACHE:+ return 0.0+ return max(0.0, _env_float("DELTALITE_PROBE_CACHE_BYTES", _DEFAULT_PROBE_CACHE_BYTES)) / MB+@staticmethoddef _resolve_max_concurrent() -> int:"""Max concurrent upserts on this process, from the same source of truth as the worker.@@ -924,6 +958,13 @@ class Admission:#: Time spent waiting for room before the write, and whether the wait ran out.wait_ms: int = 0wait_timed_out: bool = False+ #: The borrowed plan: the slices it reserves and its knobs. None when no borrow was possible:+ #: borrowing off, the slice-sized plan already the widest, or too little idle pod memory.+ borrow_slots: float | None = None+ borrow_mpp: int | None = None+ borrow_mpf: int | None = None+ #: True when this upsert runs the borrowed plan and holds its reservation.+ borrow_applied: bool = False# --- The governor ----------------------------------------------------------------------------@@ -1049,6 +1090,56 @@ class MemoryGovernor:if ticket in self._waiters:self._waiters.remove(ticket)+ # -- borrowing idle pod memory -------------------------------------------------------------++ def _borrowable_mb(self, usable_mb: float) -> float:+ """Memory one borrow may reserve now: half of the free pod memory, and none while an upsert waits.++ Free memory is the usable pod (the limit after the safety derate and the reserve) less the+ reservations in flight and the probe cache cap. The half rule leaves at least as much free as a+ borrower takes, so the next upsert finds room without a wait, and a series of borrowers fills+ the pod in shrinking steps instead of at once. A waiter means the pod is already full, so a+ borrow then would only make the wait longer. Call with ``self._lock`` held.+ """+ if self._waiters:+ return 0.0+ free_mb = usable_mb - self._reserved_mb - self.config.probe_cache_mb+ return max(0.0, free_mb) / 2++ def _borrowed_plan(+ self,+ slice_plan: UpsertPlan,+ budget_mb: float,+ usable_mb: float,+ source_mb: float,+ n_partitions: int | None,+ rewrite: RewriteProfile,+ ) -> UpsertPlan | None:+ """A plan wider than ``slice_plan`` sized to the borrowable pod memory, or None when there is none.++ The pod state can change before the caller reserves, so ``_take_borrowed`` checks it again.+ """+ if self.config.borrow_mode == "off" or is_widest_plan(slice_plan, rewrite, n_partitions):+ return None+ with self._lock:+ borrow_mb = min(self.config.borrow_max_slices * budget_mb, self._borrowable_mb(usable_mb))+ if borrow_mb <= budget_mb:+ return None+ plan = size_upsert(borrow_mb, source_mb, n_partitions, rewrite, self.config.rss_retention)+ widens = (plan.max_parallel_partitions, plan.max_parallel_files) > (+ slice_plan.max_parallel_partitions,+ slice_plan.max_parallel_files,+ )+ return plan if plan.fits and widens else None++ def _take_borrowed(self, reserve_mb: float, usable_mb: float) -> bool:+ """Reserve a borrowed plan if it still fits the borrow rule. A borrow never waits."""+ with self._lock:+ if reserve_mb > self._borrowable_mb(usable_mb) + _ROOM_TOLERANCE_MB:+ return False+ self._take(reserve_mb)+ return True+@contextlib.asynccontextmanagerasync def admit(self,@@ -1064,6 +1155,10 @@ class MemoryGovernor:mode an upsert predicted above its slice reserves up to the whole usable pod, and entrywaits (bounded by ``max_wait_s``) until the reservation fits beside the ones in flight.In every mode but off, the process RSS is sampled from the end of any wait until exit.++ When the slice-sized plan is narrower than the widest plan, the upsert can borrow idle pod+ memory (``_borrowable_mb``) up to ``borrow_max_slices`` slices. Borrow advisory only logs the+ borrowed plan. Borrow enforce, with enforce mode, runs it and holds its reservation."""source_mb = source_bytes / MB@@ -1073,6 +1168,7 @@ class MemoryGovernor:limit_mb = self.pod.limit_mb()plan: UpsertPlan | None = None+ borrowed: UpsertPlan | None = Nonebudget_mb: float | None = Noneusable_mb = reserve_mb = 0.0# No cgroup limit visible (local dev / unreadable): we can't size a slice, so use deltalite's@@ -1083,27 +1179,41 @@ class MemoryGovernor:plan = size_upsert(budget_mb, source_mb, n_partitions, rewrite, self.config.rss_retention)# Capped at the usable pod so an upsert alone always fits and never waits for itself.reserve_mb = min(plan.predicted_peak_mb, usable_mb)+ # An unknown shape is sized by a guess, so it never borrows on top of it.+ if rewrite is not None:+ borrowed = self._borrowed_plan(plan, budget_mb, usable_mb, source_mb, n_partitions, rewrite)# Only enforce reserves against the budget (and may wait for room); advisory is a pure# no-op on the accounting.admitted = False+ borrow_applied = Falsewaited_s, timed_out = 0.0, Falseif plan is not None and self.config.mode == "enforce":- waited_s, timed_out = await self._reserve(reserve_mb, usable_mb)+ if (+ borrowed is not None+ and self.config.borrow_mode == "enforce"+ and self._take_borrowed(borrowed.predicted_peak_mb, usable_mb)+ ):+ borrow_applied = True+ reserve_mb = borrowed.predicted_peak_mb+ else:+ waited_s, timed_out = await self._reserve(reserve_mb, usable_mb)admitted = True+ applied = borrowed if borrow_applied else planwith self._lock:self._active += 1concurrent = self._activeadm = Admission(- upsert_kwargs=plan.as_upsert_kwargs() if plan is not None and admitted else {},+ upsert_kwargs=applied.as_upsert_kwargs() if applied is not None and admitted else {},mode=self.config.mode,- predicted_peak_mb=plan.predicted_peak_mb if plan is not None else None,+ predicted_peak_mb=applied.predicted_peak_mb if applied is not None else None,budget_mb=round(budget_mb, 1) if budget_mb is not None else None,- planned_mpp=plan.max_parallel_partitions if plan is not None else None,- planned_mpf=plan.max_parallel_files if plan is not None else None,+ planned_mpp=applied.max_parallel_partitions if applied is not None else None,+ planned_mpf=applied.max_parallel_files if applied is not None else None,+ # Measured against the slice, so the ops signal keeps its meaning when the upsert borrows.capacity_exceeded=plan is not None and not plan.fits,- estimate=plan.estimate if plan is not None else None,+ estimate=applied.estimate if applied is not None else None,max_row_group_mb=round(rewrite.max_row_group_mb, 1) if rewrite is not None else None,row_group_sized_files=rewrite.row_group_sized_files if rewrite is not None else None,footers_read=rewrite.footers_read if rewrite is not None else None,@@ -1114,6 +1224,12 @@ class MemoryGovernor:reserved_slots=round(reserve_mb / budget_mb, 2) if budget_mb else None,wait_ms=round(waited_s * 1000),wait_timed_out=timed_out,+ borrow_slots=round(borrowed.predicted_peak_mb / budget_mb, 2)+ if borrowed is not None and budget_mb+ else None,+ borrow_mpp=borrowed.max_parallel_partitions if borrowed is not None else None,+ borrow_mpf=borrowed.max_parallel_files if borrowed is not None else None,+ borrow_applied=borrow_applied,)if plan is not None:self._emit_decision(adm, source_mb)@@ -1139,13 +1255,16 @@ class MemoryGovernor:if adm.capacity_exceeded:logger.warning("deltalite governor: source %.0f MB with %.0f MB row groups exceeds its %.0f MB "- "per-upsert slice (max_concurrent=%d); running deltalite at mpp=1, mpf=1 and "- "reserving %.2f slices (waited %d ms, timed out: %s).",+ "per-upsert slice (max_concurrent=%d); running deltalite at mpp=%s, mpf=%s and "+ "reserving %.2f slices (borrowed: %s, waited %d ms, timed out: %s).",source_mb,adm.max_row_group_mb or 0.0,adm.budget_mb or 0.0,self.config.max_concurrent,+ adm.planned_mpp,+ adm.planned_mpf,adm.reserved_slots or 0.0,+ adm.borrow_applied,adm.wait_ms,adm.wait_timed_out,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.pyindex 6d29068ad0c5..631afa37de5e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py@@ -1064,3 +1064,173 @@ class TestOverSlotReservation:assert adm.upsert_kwargs == {} and adm.reserved_slots == 4.0assert (gov._reserved_mb, gov._inflight, fake.sleeps) == (0.0, 0, 0)await self._exit(cm)+++class TestBorrowIdleMemory:+ # 16 slices of 500 MB. Wide-row merges: 8 partitions of 33 MB files under a 250 MB source, so the+ # slice-sized plan is (1, 1) at 748.9 MB, above the slice. 4 slices fit (4, 4) at 1504.5 MB.+ _WIDE_ROWS = _profile(*([[33.0] * 8] * 8), target_mb=32.0, columns=30)+ _SOURCE = 250 * MB+ _SLICE_PLAN_MB = 748.9++ @staticmethod+ def _gov(+ mode: str = "enforce",+ borrow_mode: str = "enforce",+ *,+ limit_mb: float = 8_000.0,+ max_concurrent: int = 16,+ clock=None,+ sleep=None,+ **cfg,+ ) -> MemoryGovernor:+ config = GovernorConfig(+ **{+ "mode": mode,+ "borrow_mode": borrow_mode,+ "safety": 1.0,+ "reserve_mb": 0.0,+ "max_concurrent": max_concurrent,+ "probe_cache_mb": 0.0,+ "rss_sample_ms": 0.0,+ **cfg,+ }+ )+ timing = {k: v for k, v in (("clock", clock), ("sleep", sleep)) if v is not None}+ return MemoryGovernor(config, _FakePod(limit_mb, 1_000.0), **timing)++ @parameterized.expand(+ [+ ("idle_pod_borrows_up_to_k_slices", {}, 0, _WIDE_ROWS, (4, 4)),+ ("k_caps_the_borrow", {"borrow_max_slices": 2.0}, 0, _WIDE_ROWS, (1, 2)),+ ("half_of_free_memory_caps_the_borrow", {}, 10, _WIDE_ROWS, (3, 5)),+ ("busy_pod_borrows_nothing", {}, 13, _WIDE_ROWS, None),+ ("probe_cache_is_not_borrowed", {"probe_cache_mb": 1_000.0}, 10, _WIDE_ROWS, (1, 2)),+ ("pod_reserve_is_not_borrowed", {"limit_mb": 10_000.0, "reserve_mb": 2_000.0}, 10, _WIDE_ROWS, (3, 5)),+ ("safety_margin_is_not_borrowed", {"limit_mb": 10_000.0, "safety": 0.8}, 10, _WIDE_ROWS, (3, 5)),+ ("widest_slice_plan_borrows_nothing", {"limit_mb": 30_000.0}, 0, _SMALL, None),+ ("unknown_shape_borrows_nothing", {}, 0, None, None),+ ]+ )+ async def test_borrow(self, _name, cfg, reserved_slices, rewrite, expected):+ gov = self._gov(**cfg)+ budget_mb = gov.slot_budget_mb()+ assert budget_mb is not None+ usable_mb = budget_mb * 16+ gov._reserved_mb = reserved_slices * budget_mb+ free_mb = usable_mb - gov._reserved_mb - gov.config.probe_cache_mb+ async with gov.admit(source_bytes=self._SOURCE, rewrite=rewrite) as adm:+ reserved_mb = gov._reserved_mb - reserved_slices * budget_mb+ knobs = (adm.upsert_kwargs["max_parallel_partitions"], adm.upsert_kwargs["max_parallel_files"])+ if expected is None:+ assert (adm.borrow_mpp, adm.borrow_slots, adm.borrow_applied) == (None, None, False)+ assert reserved_mb == pytest.approx(adm.predicted_peak_mb)+ else:+ assert (adm.borrow_mpp, adm.borrow_mpf) == knobs == expected+ assert adm.borrow_applied is True and adm.capacity_exceeded is True+ assert reserved_mb == pytest.approx(adm.predicted_peak_mb)+ assert reserved_mb <= free_mb / 2 and reserved_mb <= gov.config.borrow_max_slices * budget_mb+ assert adm.reserved_slots == adm.borrow_slots == round(reserved_mb / budget_mb, 2)+ assert gov._reserved_mb == pytest.approx(reserved_slices * budget_mb)++ @parameterized.expand(+ [+ ("borrow_off", "enforce", "off", (1, 1), None, _SLICE_PLAN_MB),+ ("borrow_advisory_keeps_the_slice_plan", "enforce", "advisory", (1, 1), (4, 4), _SLICE_PLAN_MB),+ ("borrow_enforce_widens_the_plan", "enforce", "enforce", (4, 4), (4, 4), 1_504.5),+ ("governor_advisory_never_applies_a_borrow", "advisory", "enforce", None, (4, 4), 0.0),+ ]+ )+ async def test_modes(self, _name, mode, borrow_mode, expected_knobs, expected_borrow, expected_reserved_mb):+ gov = self._gov(mode, borrow_mode)+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as adm:+ knobs = (+ (adm.upsert_kwargs["max_parallel_partitions"], adm.upsert_kwargs["max_parallel_files"])+ if adm.upsert_kwargs+ else None+ )+ assert knobs == expected_knobs+ assert (adm.borrow_mpp, adm.borrow_mpf) == (expected_borrow or (None, None))+ assert adm.borrow_applied is (expected_knobs == expected_borrow)+ assert gov._reserved_mb == pytest.approx(expected_reserved_mb)+ if expected_knobs is not None:+ assert (adm.planned_mpp, adm.planned_mpf) == expected_knobs+ assert gov._reserved_mb == 0.0 and gov._inflight == 0++ async def test_the_next_upsert_finds_room_without_a_wait(self):+ async def _no_sleep(_seconds: float) -> None:+ raise AssertionError("the upsert after a borrower must not wait")++ # 4 slices: the first borrower may take only half of the 2000 MB pod.+ gov = self._gov(limit_mb=2_000.0, max_concurrent=4, sleep=_no_sleep)+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as first:+ assert first.borrow_applied is True and (first.planned_mpp, first.planned_mpf) == (1, 2)+ assert first.predicted_peak_mb is not None and first.predicted_peak_mb <= 1_000.0+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as second:+ assert second.borrow_applied is False and second.wait_ms == 0+ assert (second.planned_mpp, second.planned_mpf) == (1, 1)+ assert gov._reserved_mb == pytest.approx(first.predicted_peak_mb + self._SLICE_PLAN_MB)+ assert gov._reserved_mb == pytest.approx(0.0)++ async def test_a_borrower_never_overtakes_a_waiter(self):+ fake = _FakeTime()+ gov = self._gov(clock=fake.clock, sleep=fake.sleep)+ held = gov.admit(source_bytes=50 * MB, rewrite=_SMALL)+ await held.__aenter__()+ # Even (1, 1) of 16 GB files is above the pod, so it waits for the pod to empty.+ pod_sized = gov.admit(source_bytes=50 * MB, rewrite=_profile([16_384.0] * 4))+ waiter = asyncio.create_task(pod_sized.__aenter__())+ await _settle()+ borrower_cm = gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS)+ borrower = asyncio.create_task(borrower_cm.__aenter__())+ await _settle()+ assert not waiter.done() and not borrower.done()++ await held.__aexit__(None, None, None)+ await _settle()+ assert waiter.done() and not borrower.done()+ await pod_sized.__aexit__(None, None, None)+ await _settle()+ adm = borrower.result()+ assert adm.borrow_applied is False and (adm.planned_mpp, adm.planned_mpf) == (1, 1)+ await borrower_cm.__aexit__(None, None, None)+ assert gov._reserved_mb == 0.0 and gov._inflight == 0+++class TestBorrowConfigFromEnv:+ @parameterized.expand(+ [+ ("defaults", {}, {}, ("advisory", 4.0, 128.0)),+ (+ "settings_and_cache_size",+ {+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW": "ENFORCE",+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES": 2.5,+ },+ {"DELTALITE_PROBE_CACHE_BYTES": str(256 * MB)},+ ("enforce", 2.5, 256.0),+ ),+ (+ "invalid_mode_is_advisory",+ {"DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW": "nonsense"},+ {},+ ("advisory", 4.0, 128.0),+ ),+ (+ "at_least_one_slice",+ {"DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES": 0.0},+ {},+ ("advisory", 1.0, 128.0),+ ),+ (+ "cache_turned_off_holds_nothing",+ {"DATA_WAREHOUSE_DELTALITE_PROBE_CACHE": False},+ {},+ ("advisory", 4.0, 0.0),+ ),+ ]+ )+ def test_reads_borrow_settings(self, _name, overrides, env, expected):+ with patch.dict("os.environ", env, clear=True), override_settings(**overrides):+ cfg = GovernorConfig.from_env()+ assert (cfg.borrow_mode, cfg.borrow_max_slices, cfg.probe_cache_mb) == expecteddiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex 1e8e52715108..4c481b16b144 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -1448,6 +1448,8 @@ class TestDeltaliteWritePath:assert log_kwargs["governor_max_row_group_mb"] == round(partition_a_bytes / (1024 * 1024), 1)assert log_kwargs["governor_reader_mb"] is not None and log_kwargs["governor_writer_mb"] is not Noneassert log_kwargs["governor_reserved_slots"] is not None and log_kwargs["governor_wait_ms"] == 0+ # (1, 8) is already the widest plan here, so nothing is borrowed.+ assert (log_kwargs["governor_borrow_mpp"], log_kwargs["governor_borrow_applied"]) == (None, False)# The measured peak sits next to the prediction, with the concurrency to filter solo runs.assert (log_kwargs["governor_peak_rss_mb"], log_kwargs["governor_rss_delta_mb"]) == (650.0, 150.0)assert log_kwargs["governor_concurrent_upserts"] == log_kwargs["governor_max_concurrent_upserts"] == 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex a6faa1e45750..16ef8b7523ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -455,6 +455,10 @@ class DeltaWriter:governor_reserved_slots=adm.reserved_slots,governor_wait_ms=adm.wait_ms,governor_wait_timed_out=adm.wait_timed_out,+ governor_borrow_slots=adm.borrow_slots,+ governor_borrow_mpp=adm.borrow_mpp,+ governor_borrow_mpf=adm.borrow_mpf,+ governor_borrow_applied=adm.borrow_applied,# Process-wide RSS while this upsert ran; filter on max_concurrent_upserts == 1 for# a clean per-upsert signal.governor_peak_rss_mb=adm.rss.peak_mb if adm.rss else None,