Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): let long deltalite upserts borrow idle pod memory (#121886)

Tom Owers38fb74dfdd7e
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index a2e823b6724f..4f03724d71f9 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -191,6 +191,15 @@ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(
DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env(
"DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool
)
+# Lets a deltalite upsert whose slice-sized plan is narrower than the widest plan borrow idle pod memory,
+# up to DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES slices, and run more partition workers and
+# readers. A borrow takes at most half of the free pod memory and never happens while an upsert waits for
+# room. "advisory" logs the borrowed plan and keeps the slice-sized plan. "enforce" runs the borrowed plan,
+# and needs DELTALITE_GOVERNOR_MODE=enforce. "off" computes nothing.
+DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW = get_from_env("DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW", "advisory")
+DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES = get_from_env(
+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES", 4.0, type_cast=float
+)
# deltalite keeps the primary keys of the data files that a merge reads or writes in a per-process cache,
# so a later merge into the same table decides most files without a read from object storage. The cache
# size comes from the DELTALITE_PROBE_CACHE_BYTES environment variable (default 128 MiB per process). False makes every merge read each
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
index 13e32cce7810..86ad2c34034d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
@@ -26,6 +26,10 @@ Two layers, by design:
governor predicted. The governor aims never to reach it; the backstop guarantees safety when a
prediction is wrong.
+The slices guarantee room for ``max_concurrent`` upserts, but a pod mostly runs far fewer. So an
+upsert whose slice keeps its plan narrow can borrow idle memory, up to a few slices, to run more
+partition workers. ``MemoryGovernor._borrowable_mb`` holds the rule that keeps a borrow fair.
+
The memory model follows the shape of the files a merge rewrites, not their total size. An upsert
in deltalite 0.1.10 holds three things at its peak:
@@ -183,6 +187,11 @@ _WAIT_POLL_S = 0.5
_ROOM_TOLERANCE_MB = 1.0
#: How often the RSS sampler reads the process RSS while an upsert runs.
_DEFAULT_RSS_SAMPLE_MS = 100.0
+#: Most slices one upsert may borrow when the pod has idle memory. 4 slices let a wide-row merge
+#: run 4 partition workers, and beyond 4 workers the gains vanish.
+_DEFAULT_BORROW_MAX_SLICES = 4.0
+#: deltalite's own default for ``DELTALITE_PROBE_CACHE_BYTES``.
+_DEFAULT_PROBE_CACHE_BYTES = 128 * MB
def _rss_retention(environ: Mapping[str, str] = os.environ) -> float:
@@ -830,6 +839,13 @@ class GovernorConfig:
rss_sample_ms: float = _DEFAULT_RSS_SAMPLE_MS
#: RSS per byte of predicted in-use memory. Read from the allocator settings by default.
rss_retention: float = _RSS_RETENTION_GLIBC_DEFAULT
+ #: Rollout gate for borrowing idle pod memory, with the same three values as ``mode``. Advisory
+ #: computes and logs the borrowed plan. Enforce runs it, and only when ``mode`` is enforce too.
+ borrow_mode: Mode = "advisory"
+ #: Most slices one borrowed plan may reserve.
+ borrow_max_slices: float = _DEFAULT_BORROW_MAX_SLICES
+ #: Cap of deltalite's process-wide probe cache. No slice holds it, so a borrow leaves it free.
+ probe_cache_mb: float = _DEFAULT_PROBE_CACHE_BYTES / MB
@staticmethod
def from_env() -> GovernorConfig:
@@ -847,8 +863,26 @@ class GovernorConfig:
max_wait_s=max(0.0, _env_float("DELTALITE_GOVERNOR_MAX_WAIT_S", 60.0)),
rss_sample_ms=max(0.0, _env_float("DELTALITE_GOVERNOR_RSS_SAMPLE_MS", _DEFAULT_RSS_SAMPLE_MS)),
rss_retention=_rss_retention(),
+ borrow_mode=GovernorConfig._resolve_borrow_mode(),
+ borrow_max_slices=max(1.0, float(settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES)),
+ probe_cache_mb=GovernorConfig._resolve_probe_cache_mb(),
)
+ @staticmethod
+ def _resolve_borrow_mode() -> Mode:
+ mode = str(settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW).strip().lower()
+ if mode not in ("off", "advisory", "enforce"):
+ logger.warning("invalid DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW=%r; defaulting to advisory", mode)
+ return "advisory"
+ return mode # type: ignore[return-value]
+
+ @staticmethod
+ def _resolve_probe_cache_mb() -> float:
+ """The probe cache cap, read from the variable deltalite reads. 0 when the writer turns the cache off."""
+ if not settings.DATA_WAREHOUSE_DELTALITE_PROBE_CACHE:
+ return 0.0
+ return max(0.0, _env_float("DELTALITE_PROBE_CACHE_BYTES", _DEFAULT_PROBE_CACHE_BYTES)) / MB
+
@staticmethod
def _resolve_max_concurrent() -> int:
"""Max concurrent upserts on this process, from the same source of truth as the worker.
@@ -924,6 +958,13 @@ class Admission:
#: Time spent waiting for room before the write, and whether the wait ran out.
wait_ms: int = 0
wait_timed_out: bool = False
+ #: The borrowed plan: the slices it reserves and its knobs. None when no borrow was possible:
+ #: borrowing off, the slice-sized plan already the widest, or too little idle pod memory.
+ borrow_slots: float | None = None
+ borrow_mpp: int | None = None
+ borrow_mpf: int | None = None
+ #: True when this upsert runs the borrowed plan and holds its reservation.
+ borrow_applied: bool = False
# --- The governor ----------------------------------------------------------------------------
@@ -1049,6 +1090,56 @@ class MemoryGovernor:
if ticket in self._waiters:
self._waiters.remove(ticket)
+ # -- borrowing idle pod memory -------------------------------------------------------------
+
+ def _borrowable_mb(self, usable_mb: float) -> float:
+ """Memory one borrow may reserve now: half of the free pod memory, and none while an upsert waits.
+
+ Free memory is the usable pod (the limit after the safety derate and the reserve) less the
+ reservations in flight and the probe cache cap. The half rule leaves at least as much free as a
+ borrower takes, so the next upsert finds room without a wait, and a series of borrowers fills
+ the pod in shrinking steps instead of at once. A waiter means the pod is already full, so a
+ borrow then would only make the wait longer. Call with ``self._lock`` held.
+ """
+ if self._waiters:
+ return 0.0
+ free_mb = usable_mb - self._reserved_mb - self.config.probe_cache_mb
+ return max(0.0, free_mb) / 2
+
+ def _borrowed_plan(
+ self,
+ slice_plan: UpsertPlan,
+ budget_mb: float,
+ usable_mb: float,
+ source_mb: float,
+ n_partitions: int | None,
+ rewrite: RewriteProfile,
+ ) -> UpsertPlan | None:
+ """A plan wider than ``slice_plan`` sized to the borrowable pod memory, or None when there is none.
+
+ The pod state can change before the caller reserves, so ``_take_borrowed`` checks it again.
+ """
+ if self.config.borrow_mode == "off" or is_widest_plan(slice_plan, rewrite, n_partitions):
+ return None
+ with self._lock:
+ borrow_mb = min(self.config.borrow_max_slices * budget_mb, self._borrowable_mb(usable_mb))
+ if borrow_mb <= budget_mb:
+ return None
+ plan = size_upsert(borrow_mb, source_mb, n_partitions, rewrite, self.config.rss_retention)
+ widens = (plan.max_parallel_partitions, plan.max_parallel_files) > (
+ slice_plan.max_parallel_partitions,
+ slice_plan.max_parallel_files,
+ )
+ return plan if plan.fits and widens else None
+
+ def _take_borrowed(self, reserve_mb: float, usable_mb: float) -> bool:
+ """Reserve a borrowed plan if it still fits the borrow rule. A borrow never waits."""
+ with self._lock:
+ if reserve_mb > self._borrowable_mb(usable_mb) + _ROOM_TOLERANCE_MB:
+ return False
+ self._take(reserve_mb)
+ return True
+
@contextlib.asynccontextmanager
async def admit(
self,
@@ -1064,6 +1155,10 @@ class MemoryGovernor:
mode an upsert predicted above its slice reserves up to the whole usable pod, and entry
waits (bounded by ``max_wait_s``) until the reservation fits beside the ones in flight.
In every mode but off, the process RSS is sampled from the end of any wait until exit.
+
+ When the slice-sized plan is narrower than the widest plan, the upsert can borrow idle pod
+ memory (``_borrowable_mb``) up to ``borrow_max_slices`` slices. Borrow advisory only logs the
+ borrowed plan. Borrow enforce, with enforce mode, runs it and holds its reservation.
"""
source_mb = source_bytes / MB
@@ -1073,6 +1168,7 @@ class MemoryGovernor:
limit_mb = self.pod.limit_mb()
plan: UpsertPlan | None = None
+ borrowed: UpsertPlan | None = None
budget_mb: float | None = None
usable_mb = reserve_mb = 0.0
# No cgroup limit visible (local dev / unreadable): we can't size a slice, so use deltalite's
@@ -1083,27 +1179,41 @@ class MemoryGovernor:
plan = size_upsert(budget_mb, source_mb, n_partitions, rewrite, self.config.rss_retention)
# Capped at the usable pod so an upsert alone always fits and never waits for itself.
reserve_mb = min(plan.predicted_peak_mb, usable_mb)
+ # An unknown shape is sized by a guess, so it never borrows on top of it.
+ if rewrite is not None:
+ borrowed = self._borrowed_plan(plan, budget_mb, usable_mb, source_mb, n_partitions, rewrite)
# Only enforce reserves against the budget (and may wait for room); advisory is a pure
# no-op on the accounting.
admitted = False
+ borrow_applied = False
waited_s, timed_out = 0.0, False
if plan is not None and self.config.mode == "enforce":
- waited_s, timed_out = await self._reserve(reserve_mb, usable_mb)
+ if (
+ borrowed is not None
+ and self.config.borrow_mode == "enforce"
+ and self._take_borrowed(borrowed.predicted_peak_mb, usable_mb)
+ ):
+ borrow_applied = True
+ reserve_mb = borrowed.predicted_peak_mb
+ else:
+ waited_s, timed_out = await self._reserve(reserve_mb, usable_mb)
admitted = True
+ applied = borrowed if borrow_applied else plan
with self._lock:
self._active += 1
concurrent = self._active
adm = Admission(
- upsert_kwargs=plan.as_upsert_kwargs() if plan is not None and admitted else {},
+ upsert_kwargs=applied.as_upsert_kwargs() if applied is not None and admitted else {},
mode=self.config.mode,
- predicted_peak_mb=plan.predicted_peak_mb if plan is not None else None,
+ predicted_peak_mb=applied.predicted_peak_mb if applied is not None else None,
budget_mb=round(budget_mb, 1) if budget_mb is not None else None,
- planned_mpp=plan.max_parallel_partitions if plan is not None else None,
- planned_mpf=plan.max_parallel_files if plan is not None else None,
+ planned_mpp=applied.max_parallel_partitions if applied is not None else None,
+ planned_mpf=applied.max_parallel_files if applied is not None else None,
+ # Measured against the slice, so the ops signal keeps its meaning when the upsert borrows.
capacity_exceeded=plan is not None and not plan.fits,
- estimate=plan.estimate if plan is not None else None,
+ estimate=applied.estimate if applied is not None else None,
max_row_group_mb=round(rewrite.max_row_group_mb, 1) if rewrite is not None else None,
row_group_sized_files=rewrite.row_group_sized_files if rewrite is not None else None,
footers_read=rewrite.footers_read if rewrite is not None else None,
@@ -1114,6 +1224,12 @@ class MemoryGovernor:
reserved_slots=round(reserve_mb / budget_mb, 2) if budget_mb else None,
wait_ms=round(waited_s * 1000),
wait_timed_out=timed_out,
+ borrow_slots=round(borrowed.predicted_peak_mb / budget_mb, 2)
+ if borrowed is not None and budget_mb
+ else None,
+ borrow_mpp=borrowed.max_parallel_partitions if borrowed is not None else None,
+ borrow_mpf=borrowed.max_parallel_files if borrowed is not None else None,
+ borrow_applied=borrow_applied,
)
if plan is not None:
self._emit_decision(adm, source_mb)
@@ -1139,13 +1255,16 @@ class MemoryGovernor:
if adm.capacity_exceeded:
logger.warning(
"deltalite governor: source %.0f MB with %.0f MB row groups exceeds its %.0f MB "
- "per-upsert slice (max_concurrent=%d); running deltalite at mpp=1, mpf=1 and "
- "reserving %.2f slices (waited %d ms, timed out: %s).",
+ "per-upsert slice (max_concurrent=%d); running deltalite at mpp=%s, mpf=%s and "
+ "reserving %.2f slices (borrowed: %s, waited %d ms, timed out: %s).",
source_mb,
adm.max_row_group_mb or 0.0,
adm.budget_mb or 0.0,
self.config.max_concurrent,
+ adm.planned_mpp,
+ adm.planned_mpf,
adm.reserved_slots or 0.0,
+ adm.borrow_applied,
adm.wait_ms,
adm.wait_timed_out,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
index 6d29068ad0c5..631afa37de5e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
@@ -1064,3 +1064,173 @@ class TestOverSlotReservation:
assert adm.upsert_kwargs == {} and adm.reserved_slots == 4.0
assert (gov._reserved_mb, gov._inflight, fake.sleeps) == (0.0, 0, 0)
await self._exit(cm)
+
+
+class TestBorrowIdleMemory:
+ # 16 slices of 500 MB. Wide-row merges: 8 partitions of 33 MB files under a 250 MB source, so the
+ # slice-sized plan is (1, 1) at 748.9 MB, above the slice. 4 slices fit (4, 4) at 1504.5 MB.
+ _WIDE_ROWS = _profile(*([[33.0] * 8] * 8), target_mb=32.0, columns=30)
+ _SOURCE = 250 * MB
+ _SLICE_PLAN_MB = 748.9
+
+ @staticmethod
+ def _gov(
+ mode: str = "enforce",
+ borrow_mode: str = "enforce",
+ *,
+ limit_mb: float = 8_000.0,
+ max_concurrent: int = 16,
+ clock=None,
+ sleep=None,
+ **cfg,
+ ) -> MemoryGovernor:
+ config = GovernorConfig(
+ **{
+ "mode": mode,
+ "borrow_mode": borrow_mode,
+ "safety": 1.0,
+ "reserve_mb": 0.0,
+ "max_concurrent": max_concurrent,
+ "probe_cache_mb": 0.0,
+ "rss_sample_ms": 0.0,
+ **cfg,
+ }
+ )
+ timing = {k: v for k, v in (("clock", clock), ("sleep", sleep)) if v is not None}
+ return MemoryGovernor(config, _FakePod(limit_mb, 1_000.0), **timing)
+
+ @parameterized.expand(
+ [
+ ("idle_pod_borrows_up_to_k_slices", {}, 0, _WIDE_ROWS, (4, 4)),
+ ("k_caps_the_borrow", {"borrow_max_slices": 2.0}, 0, _WIDE_ROWS, (1, 2)),
+ ("half_of_free_memory_caps_the_borrow", {}, 10, _WIDE_ROWS, (3, 5)),
+ ("busy_pod_borrows_nothing", {}, 13, _WIDE_ROWS, None),
+ ("probe_cache_is_not_borrowed", {"probe_cache_mb": 1_000.0}, 10, _WIDE_ROWS, (1, 2)),
+ ("pod_reserve_is_not_borrowed", {"limit_mb": 10_000.0, "reserve_mb": 2_000.0}, 10, _WIDE_ROWS, (3, 5)),
+ ("safety_margin_is_not_borrowed", {"limit_mb": 10_000.0, "safety": 0.8}, 10, _WIDE_ROWS, (3, 5)),
+ ("widest_slice_plan_borrows_nothing", {"limit_mb": 30_000.0}, 0, _SMALL, None),
+ ("unknown_shape_borrows_nothing", {}, 0, None, None),
+ ]
+ )
+ async def test_borrow(self, _name, cfg, reserved_slices, rewrite, expected):
+ gov = self._gov(**cfg)
+ budget_mb = gov.slot_budget_mb()
+ assert budget_mb is not None
+ usable_mb = budget_mb * 16
+ gov._reserved_mb = reserved_slices * budget_mb
+ free_mb = usable_mb - gov._reserved_mb - gov.config.probe_cache_mb
+ async with gov.admit(source_bytes=self._SOURCE, rewrite=rewrite) as adm:
+ reserved_mb = gov._reserved_mb - reserved_slices * budget_mb
+ knobs = (adm.upsert_kwargs["max_parallel_partitions"], adm.upsert_kwargs["max_parallel_files"])
+ if expected is None:
+ assert (adm.borrow_mpp, adm.borrow_slots, adm.borrow_applied) == (None, None, False)
+ assert reserved_mb == pytest.approx(adm.predicted_peak_mb)
+ else:
+ assert (adm.borrow_mpp, adm.borrow_mpf) == knobs == expected
+ assert adm.borrow_applied is True and adm.capacity_exceeded is True
+ assert reserved_mb == pytest.approx(adm.predicted_peak_mb)
+ assert reserved_mb <= free_mb / 2 and reserved_mb <= gov.config.borrow_max_slices * budget_mb
+ assert adm.reserved_slots == adm.borrow_slots == round(reserved_mb / budget_mb, 2)
+ assert gov._reserved_mb == pytest.approx(reserved_slices * budget_mb)
+
+ @parameterized.expand(
+ [
+ ("borrow_off", "enforce", "off", (1, 1), None, _SLICE_PLAN_MB),
+ ("borrow_advisory_keeps_the_slice_plan", "enforce", "advisory", (1, 1), (4, 4), _SLICE_PLAN_MB),
+ ("borrow_enforce_widens_the_plan", "enforce", "enforce", (4, 4), (4, 4), 1_504.5),
+ ("governor_advisory_never_applies_a_borrow", "advisory", "enforce", None, (4, 4), 0.0),
+ ]
+ )
+ async def test_modes(self, _name, mode, borrow_mode, expected_knobs, expected_borrow, expected_reserved_mb):
+ gov = self._gov(mode, borrow_mode)
+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as adm:
+ knobs = (
+ (adm.upsert_kwargs["max_parallel_partitions"], adm.upsert_kwargs["max_parallel_files"])
+ if adm.upsert_kwargs
+ else None
+ )
+ assert knobs == expected_knobs
+ assert (adm.borrow_mpp, adm.borrow_mpf) == (expected_borrow or (None, None))
+ assert adm.borrow_applied is (expected_knobs == expected_borrow)
+ assert gov._reserved_mb == pytest.approx(expected_reserved_mb)
+ if expected_knobs is not None:
+ assert (adm.planned_mpp, adm.planned_mpf) == expected_knobs
+ assert gov._reserved_mb == 0.0 and gov._inflight == 0
+
+ async def test_the_next_upsert_finds_room_without_a_wait(self):
+ async def _no_sleep(_seconds: float) -> None:
+ raise AssertionError("the upsert after a borrower must not wait")
+
+ # 4 slices: the first borrower may take only half of the 2000 MB pod.
+ gov = self._gov(limit_mb=2_000.0, max_concurrent=4, sleep=_no_sleep)
+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as first:
+ assert first.borrow_applied is True and (first.planned_mpp, first.planned_mpf) == (1, 2)
+ assert first.predicted_peak_mb is not None and first.predicted_peak_mb <= 1_000.0
+ async with gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS) as second:
+ assert second.borrow_applied is False and second.wait_ms == 0
+ assert (second.planned_mpp, second.planned_mpf) == (1, 1)
+ assert gov._reserved_mb == pytest.approx(first.predicted_peak_mb + self._SLICE_PLAN_MB)
+ assert gov._reserved_mb == pytest.approx(0.0)
+
+ async def test_a_borrower_never_overtakes_a_waiter(self):
+ fake = _FakeTime()
+ gov = self._gov(clock=fake.clock, sleep=fake.sleep)
+ held = gov.admit(source_bytes=50 * MB, rewrite=_SMALL)
+ await held.__aenter__()
+ # Even (1, 1) of 16 GB files is above the pod, so it waits for the pod to empty.
+ pod_sized = gov.admit(source_bytes=50 * MB, rewrite=_profile([16_384.0] * 4))
+ waiter = asyncio.create_task(pod_sized.__aenter__())
+ await _settle()
+ borrower_cm = gov.admit(source_bytes=self._SOURCE, rewrite=self._WIDE_ROWS)
+ borrower = asyncio.create_task(borrower_cm.__aenter__())
+ await _settle()
+ assert not waiter.done() and not borrower.done()
+
+ await held.__aexit__(None, None, None)
+ await _settle()
+ assert waiter.done() and not borrower.done()
+ await pod_sized.__aexit__(None, None, None)
+ await _settle()
+ adm = borrower.result()
+ assert adm.borrow_applied is False and (adm.planned_mpp, adm.planned_mpf) == (1, 1)
+ await borrower_cm.__aexit__(None, None, None)
+ assert gov._reserved_mb == 0.0 and gov._inflight == 0
+
+
+class TestBorrowConfigFromEnv:
+ @parameterized.expand(
+ [
+ ("defaults", {}, {}, ("advisory", 4.0, 128.0)),
+ (
+ "settings_and_cache_size",
+ {
+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW": "ENFORCE",
+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES": 2.5,
+ },
+ {"DELTALITE_PROBE_CACHE_BYTES": str(256 * MB)},
+ ("enforce", 2.5, 256.0),
+ ),
+ (
+ "invalid_mode_is_advisory",
+ {"DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW": "nonsense"},
+ {},
+ ("advisory", 4.0, 128.0),
+ ),
+ (
+ "at_least_one_slice",
+ {"DATA_WAREHOUSE_DELTALITE_GOVERNOR_BORROW_MAX_SLICES": 0.0},
+ {},
+ ("advisory", 1.0, 128.0),
+ ),
+ (
+ "cache_turned_off_holds_nothing",
+ {"DATA_WAREHOUSE_DELTALITE_PROBE_CACHE": False},
+ {},
+ ("advisory", 4.0, 0.0),
+ ),
+ ]
+ )
+ def test_reads_borrow_settings(self, _name, overrides, env, expected):
+ with patch.dict("os.environ", env, clear=True), override_settings(**overrides):
+ cfg = GovernorConfig.from_env()
+ assert (cfg.borrow_mode, cfg.borrow_max_slices, cfg.probe_cache_mb) == expected
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index 1e8e52715108..4c481b16b144 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -1448,6 +1448,8 @@ class TestDeltaliteWritePath:
assert log_kwargs["governor_max_row_group_mb"] == round(partition_a_bytes / (1024 * 1024), 1)
assert log_kwargs["governor_reader_mb"] is not None and log_kwargs["governor_writer_mb"] is not None
assert log_kwargs["governor_reserved_slots"] is not None and log_kwargs["governor_wait_ms"] == 0
+ # (1, 8) is already the widest plan here, so nothing is borrowed.
+ assert (log_kwargs["governor_borrow_mpp"], log_kwargs["governor_borrow_applied"]) == (None, False)
# The measured peak sits next to the prediction, with the concurrency to filter solo runs.
assert (log_kwargs["governor_peak_rss_mb"], log_kwargs["governor_rss_delta_mb"]) == (650.0, 150.0)
assert log_kwargs["governor_concurrent_upserts"] == log_kwargs["governor_max_concurrent_upserts"] == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index a6faa1e45750..16ef8b7523ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -455,6 +455,10 @@ class DeltaWriter:
governor_reserved_slots=adm.reserved_slots,
governor_wait_ms=adm.wait_ms,
governor_wait_timed_out=adm.wait_timed_out,
+ governor_borrow_slots=adm.borrow_slots,
+ governor_borrow_mpp=adm.borrow_mpp,
+ governor_borrow_mpf=adm.borrow_mpf,
+ governor_borrow_applied=adm.borrow_applied,
# Process-wide RSS while this upsert ran; filter on max_concurrent_upserts == 1 for
# a clean per-upsert signal.
governor_peak_rss_mb=adm.rss.peak_mb if adm.rss else None,