Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, performance] perf(kernel): single-pass state snapshots, bounded restore, post-restore snapshot skip (#2478)

Sebastian Müller02e5babb4d90
diff --git a/packages/coding-agent/.changes/kernel-snapshot-singlepass.md b/packages/coding-agent/.changes/kernel-snapshot-singlepass.md
new file mode 100644
index 0000000000..ff1bc9bda3
--- /dev/null
+++ b/packages/coding-agent/.changes/kernel-snapshot-singlepass.md
@@ -0,0 +1 @@
+- Reduced Python kernel snapshot cost: auto-snapshots serialize each variable exactly once into a length-framed payload instead of a second whole-namespace dump (and no prefix re-dump at the aggregate cap), compaction re-serializes each variable once (bounded by the per-variable cap) and prunes only the ones that still measure oversized, so a value shrunk in place or redefined after an earlier snapshot is never deleted on a stale size, and a resumed kernel skips the auto-snapshot that would rewrite the just-restored namespace until a real cell changes it.
diff --git a/packages/coding-agent/src/core/kernel/repl-manager.ts b/packages/coding-agent/src/core/kernel/repl-manager.ts
index 7c7ef2c6f6..5246f7d025 100644
--- a/packages/coding-agent/src/core/kernel/repl-manager.ts
+++ b/packages/coding-agent/src/core/kernel/repl-manager.ts
@@ -52,6 +52,7 @@ import {
parseAttachmentDisplay,
parseDiffDisplay,
parseSentAgentMessage,
+ RESTORE_EXECUTION_TIMEOUT_MS,
raceStartupWithAbort,
SNAPSHOT_EXECUTION_TIMEOUT_MS,
} from "./shared.js";
@@ -88,6 +89,15 @@ const KERNEL_STDERR_LOG_DIR_MODE = 0o700;
// Owner-only file bits; kernel stderr can carry exception payloads.
const KERNEL_STDERR_LOG_MODE = 0o600;
+function manifestStatOf(path: string): { mtimeMs: number; size: number } | null {
+ try {
+ const stat = statSync(path);
+ return { mtimeMs: stat.mtimeMs, size: stat.size };
+ } catch {
+ return null;
+ }
+}
+
/** fs.writeSync may write fewer bytes than asked (partial ENOSPC, signals); loop until done. */
function writeFullySync(fd: number, data: Buffer): void {
let offset = 0;
@@ -235,6 +245,13 @@ export class ReplKernelManager {
private pendingRebootstrap = false;
/** Restore the saved namespace on that fresh start too (false when the snapshot itself is the declared culprit). */
private pendingRestore = false;
+ private completedExecutions = 0;
+ /** Tri-state: undefined = no non-repair restore attempted yet, null = manifest was missing at that attempt. */
+ private restoredManifestStat?: { mtimeMs: number; size: number } | null;
+ private restoredNamespaceSkip?: {
+ manifestStat: { mtimeMs: number; size: number } | null;
+ completedExecutions: number;
+ };
private rebootstrapPromise?: Promise<boolean>;
private teardownInFlight = 0;
@@ -1160,6 +1177,7 @@ export class ReplKernelManager {
}
if (!execution.settled) {
execution.settled = true;
+ this.completedExecutions += 1;
if (execution.opts.onLateSentAgentMessage) {
this.registerLateSentAgentMessageHandler(execution.requestId, execution.opts.onLateSentAgentMessage);
}
@@ -1630,17 +1648,22 @@ export class ReplKernelManager {
private async performRestore(protocolRepair: boolean): Promise<RestoreResult | null> {
const cfg = this.options.snapshot;
if (!cfg) return null;
+ // Before the attempt, so a failed or timed-out restore still arms the skip;
+ // repair retries (reprovision after a failed first restore) keep the non-repair stat.
+ if (!protocolRepair) this.restoredManifestStat = manifestStatOf(cfg.manifestPath);
try {
const r = await this.enqueueRequest(
{ type: "restore", path: cfg.path },
"",
{ internal: true, protocolRepair },
- protocolRepair ? REPAIR_STEP_TIMEOUT_MS : undefined,
+ protocolRepair ? REPAIR_STEP_TIMEOUT_MS : RESTORE_EXECUTION_TIMEOUT_MS,
);
if (r.status !== "ok" || !r.doneFields) {
this.appendKernelDiagnostic(
`state restore ${r.status === "aborted" ? "timed out" : "failed"}: ${r.error?.evalue ?? r.stderr}`,
);
+ // The namespace never got the saved state, so the on-disk payload must stay the fresher copy.
+ if (!protocolRepair) this.pendingRestore = true;
return null;
}
this.pendingRestore = false;
@@ -1651,10 +1674,26 @@ export class ReplKernelManager {
};
} catch (error) {
this.appendKernelDiagnostic(`state restore error: ${errorMessage(error)}`);
+ if (!protocolRepair) this.pendingRestore = true;
return null;
}
}
+ /**
+ * Arm the one-shot post-restore snapshot skip: the bootstrap-scheduled snapshot would
+ * rewrite identical content, or after a failed restore clobber the healthy on-disk
+ * copy with a skills-only payload. Call after the bootstrap succeeds — its own
+ * settled execution must not defeat the arm.
+ */
+ markRestoredNamespaceFresh(): void {
+ if (this.restoredManifestStat === undefined) return; // no attempted non-repair restore to match
+ this.restoredNamespaceSkip = {
+ manifestStat: this.restoredManifestStat,
+ completedExecutions: this.completedExecutions,
+ };
+ this.restoredManifestStat = undefined;
+ }
+
/** Live user-defined top-level names, or null if the kernel isn't running. Never throws. */
async listNamespaceNames(signal?: AbortSignal): Promise<string[] | null> {
if (!this.isRunning) return null;
@@ -1677,6 +1716,7 @@ export class ReplKernelManager {
if (this.snapshotTimer) clearTimeout(this.snapshotTimer);
this.snapshotTimer = globalThis.setTimeout(() => {
this.snapshotTimer = undefined;
+ if (this.consumeRestoredSnapshotSkip()) return;
void this.captureSnapshot({ executionTimeoutMs: SNAPSHOT_EXECUTION_TIMEOUT_MS });
}, cfg.debounceMs ?? DEFAULT_SNAPSHOT_DEBOUNCE_MS);
if (this.snapshotTimer && typeof this.snapshotTimer === "object" && "unref" in this.snapshotTimer) {
@@ -1684,6 +1724,16 @@ export class ReplKernelManager {
}
}
+ private consumeRestoredSnapshotSkip(): boolean {
+ const skip = this.restoredNamespaceSkip;
+ this.restoredNamespaceSkip = undefined; // one-shot: consumed whether or not it fires
+ if (!skip || !this.options.snapshot) return false;
+ if (this.completedExecutions !== skip.completedExecutions) return false;
+ const stat = manifestStatOf(this.options.snapshot.manifestPath);
+ if (stat === null || skip.manifestStat === null) return stat === skip.manifestStat;
+ return stat.mtimeMs === skip.manifestStat.mtimeMs && stat.size === skip.manifestStat.size;
+ }
+
private clearSnapshotTimer(): void {
if (this.snapshotTimer) {
clearTimeout(this.snapshotTimer);
diff --git a/packages/coding-agent/src/core/kernel/shared.ts b/packages/coding-agent/src/core/kernel/shared.ts
index d4bea5e423..6bedd67d9b 100644
--- a/packages/coding-agent/src/core/kernel/shared.ts
+++ b/packages/coding-agent/src/core/kernel/shared.ts
@@ -7,6 +7,8 @@ export const HOST_REQUEST_SHUTDOWN_TIMEOUT_MS = 5000;
export const KERNEL_SHUTDOWN_TIMEOUT_MS = 5000;
export const DEFAULT_SNAPSHOT_DEBOUNCE_MS = 1500;
export const SNAPSHOT_EXECUTION_TIMEOUT_MS = 5000;
+/** Restore deserializes everything a snapshot serializes: bound like the repair step. */
+export const RESTORE_EXECUTION_TIMEOUT_MS = 30_000;
export const KERNEL_ABORT_GRACE_MS = 1000;
export const KERNEL_BUSY_REUSE_WAIT_MS = 5000;
export const KERNEL_BUSY_INTERRUPT_INTERVAL_MS = 500;
diff --git a/packages/coding-agent/src/core/tools/ipython.ts b/packages/coding-agent/src/core/tools/ipython.ts
index 92980ab98a..4c5bf910de 100644
--- a/packages/coding-agent/src/core/tools/ipython.ts
+++ b/packages/coding-agent/src/core/tools/ipython.ts
@@ -537,6 +537,7 @@ export class IpythonKernelProvisioner {
bootstrapCode,
});
let pendingRestore: RestoreResult | undefined;
+ let snapshotExisted = false;
try {
// Emitted synchronously (before the permit await) so a listener attaching
// mid-flight can replay the current stage.
@@ -557,7 +558,7 @@ export class IpythonKernelProvisioner {
// Revive a prior session's namespace before the bootstrap, so the bootstrap
// then overwrites live handles (rlm, skills) on top of anything restored.
if (snapshotDir) {
- const snapshotExisted = existsSync(snapshotPathIn(snapshotDir));
+ snapshotExisted = existsSync(snapshotPathIn(snapshotDir));
this.emitStartupProgress("Restoring Python state...");
const restore = await raceWithAbort(m.restoreState(), startupSignal);
if (snapshotExisted) {
@@ -572,6 +573,9 @@ export class IpythonKernelProvisioner {
const details = [bootstrap.stderr, bootstrap.error?.traceback.join("\n")].filter(Boolean).join("\n");
throw new Error(`Failed to initialize rlm runtime in the Python kernel:\n${details}`);
}
+ if (snapshotExisted) {
+ m.markRestoredNamespaceFresh();
+ }
// Broken skills stay importable-looking placeholders; report them so the
// model learns before its first call, not from the placeholder's error.
const unavailableSkills = parseUnavailablePythonSkills(bootstrap.stdout);
diff --git a/packages/coding-agent/test/repl-kernel-state-roundtrip.test.ts b/packages/coding-agent/test/repl-kernel-state-roundtrip.test.ts
index e08ece79a5..4179beab83 100644
--- a/packages/coding-agent/test/repl-kernel-state-roundtrip.test.ts
+++ b/packages/coding-agent/test/repl-kernel-state-roundtrip.test.ts
@@ -1,8 +1,8 @@
import { spawnSync } from "node:child_process";
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { join, resolve } from "node:path";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { ReplKernelManager } from "../src/core/kernel/index.js";
function resolveReplPython(): string | null {
@@ -191,4 +191,67 @@ describeIfKernel("repl kernel state snapshot round-trip (real runtime)", { tags:
rmSync(autoDir, { recursive: true, force: true });
}
}, 60_000);
+
+ it("skips the post-restore auto-snapshot when nothing changed, and snapshots again after a user cell", async () => {
+ const writer = newManager();
+ await writer.execute("restored_var = 'from-disk'");
+ await writer.shutdown({ snapshot: true, drainHostRequests: true });
+
+ const reader = new ReplKernelManager({
+ python: python as string,
+ cwd: dir,
+ snapshot: { path: snapshotPath, manifestPath, debounceMs: 50 },
+ });
+ try {
+ expect((await reader.restoreState())?.restored).toContain("restored_var");
+ vi.useFakeTimers();
+ // Production order: the execute schedules the debounced snapshot, then the arm.
+ await reader.execute("pass");
+ reader.markRestoredNamespaceFresh();
+ const statBefore = statSync(manifestPath);
+ await vi.advanceTimersByTimeAsync(300);
+ // A capture the debounce enqueued anyway would settle ahead of this request.
+ await reader.listNamespaceNames();
+ expect(statSync(manifestPath).mtimeMs).toBe(statBefore.mtimeMs);
+ expect(statSync(manifestPath).size).toBe(statBefore.size);
+
+ await reader.execute("user_var = 7");
+ await vi.advanceTimersByTimeAsync(300);
+ await reader.listNamespaceNames();
+ const manifestAfter = JSON.parse(readFileSync(manifestPath, "utf8")) as { savedNames: string[] };
+ expect(manifestAfter.savedNames).toEqual(expect.arrayContaining(["restored_var", "user_var"]));
+
+ // A failed restore must arm too, or the debounced auto-snapshot clobbers the on-disk copy.
+ writeFileSync(snapshotPath, "not-a-snapshot");
+ expect(await reader.restoreState()).toBeNull();
+ await reader.execute("pass");
+ reader.markRestoredNamespaceFresh();
+ await vi.advanceTimersByTimeAsync(300);
+ await reader.listNamespaceNames();
+ expect(readFileSync(snapshotPath, "utf8")).toBe("not-a-snapshot");
+ } finally {
+ vi.useRealTimers();
+ await reader.shutdown({ snapshot: true, drainHostRequests: true });
+ }
+ });
+
+ it("keeps the on-disk payload when a restore failed or timed out and the kernel shuts down (PR 2478)", async () => {
+ const failDir = mkdtempSync(join(tmpdir(), "prime-agent-repl-restore-fail-"));
+ const failPath = join(failDir, "session.dill");
+ const manager = new ReplKernelManager({
+ python: python as string,
+ cwd: failDir,
+ snapshot: { path: failPath, manifestPath: join(failDir, "session.json") },
+ });
+ try {
+ // An unloadable payload takes the same failed-restore branch as a timeout.
+ writeFileSync(failPath, "not-a-snapshot");
+ expect(await manager.restoreState()).toBeNull();
+ await manager.shutdown({ snapshot: true, drainHostRequests: true });
+ expect(readFileSync(failPath, "utf8")).toBe("not-a-snapshot");
+ expect(existsSync(join(failDir, "session.json"))).toBe(false);
+ } finally {
+ rmSync(failDir, { recursive: true, force: true });
+ }
+ });
});
diff --git a/prime-agent-runtime/src/rlm/repl.py b/prime-agent-runtime/src/rlm/repl.py
index be0fdf1af4..a53f89f83f 100644
--- a/prime-agent-runtime/src/rlm/repl.py
+++ b/prime-agent-runtime/src/rlm/repl.py
@@ -34,6 +34,10 @@ PROTOCOL_VERSION = 3
DEFAULT_SNAPSHOT_MAX_BYTES = 256 * 1024 * 1024
DEFAULT_SNAPSHOT_MAX_VARIABLE_BYTES = 16 * 1024 * 1024
+# Plain ASCII, never a pickle start: _restore_state sniffs it to tell v2 framed
+# payloads from legacy (single dill-pickled dict) ones.
+_SNAPSHOT_MAGIC = b"PRIME-AGENT-KERNEL-SNAPSHOT-V2\n"
+
# Stream writes must fit one protocol frame: the host buffers whole lines
# before its per-execution truncation, and raw fd writes already arrive as
# 64 KiB pump chunks.
@@ -681,6 +685,32 @@ class _CappedWriter:
return size
+def _read_snapshot_records(fh: Any) -> dict[str, bytes]:
+ """Framing damage is a corrupt snapshot: a restore error, never a partial namespace.
+ Length fields are bounds-checked before their reads, so a corrupt header cannot force a huge allocation."""
+ fh.seek(0, os.SEEK_END)
+ size = fh.tell()
+ fh.seek(len(_SNAPSHOT_MAGIC))
+ records: dict[str, bytes] = {}
+ while fh.tell() < size:
+ header = fh.read(4)
+ if len(header) < 4:
+ raise ValueError("truncated snapshot record")
+ name_len = int.from_bytes(header, "little")
+ if fh.tell() + name_len + 8 > size:
+ raise ValueError("truncated snapshot record")
+ name = fh.read(name_len)
+ raw_len = fh.read(8)
+ blob_len = int.from_bytes(raw_len, "little")
+ if len(raw_len) < 8 or fh.tell() + blob_len > size:
+ raise ValueError("truncated snapshot record")
+ blob = fh.read(blob_len)
+ if len(blob) < blob_len:
+ raise ValueError("truncated snapshot record")
+ records[name.decode("utf-8")] = blob
+ return records
+
+
def _snapshot_state(
ns: dict[str, Any],
path: str,
@@ -699,41 +729,10 @@ def _snapshot_state(
return {"error": f"dill unavailable: {err}"}
dill.settings["recurse"] = True
- payload: dict[str, bytes] = {}
+ saved: list[str] = []
skipped: list[dict[str, str]] = []
oversized: list[str] = []
- total = 0
missing = object()
- for name in list(ns.keys()):
- if name.startswith("_") or name in _ALWAYS_SKIP:
- continue
- value = ns.get(name, missing)
- if value is missing:
- # A background thread deleted the name after the key listing.
- skipped.append({"name": name, "reason": "deleted during snapshot"})
- continue
- remaining = max_bytes - total
- limit = max_variable_bytes if prune_oversized else min(max_variable_bytes, remaining)
- buffer = io.BytesIO()
- try:
- dill.dump(value, _CappedWriter(buffer, limit))
- blob = buffer.getvalue()
- except _SnapshotSizeLimitExceeded:
- if not prune_oversized and remaining < max_variable_bytes:
- skipped.append({"name": name, "reason": "exceeds aggregate snapshot size cap"})
- else:
- skipped.append({"name": name, "reason": "exceeds per-variable snapshot size cap"})
- oversized.append(name)
- continue
- except Exception as err: # noqa: BLE001 - one unpicklable name must not abort the snapshot
- skipped.append({"name": name, "reason": f"{type(err).__name__}: {_safe_str(err)[:200]}"})
- continue
- if total + len(blob) > max_bytes:
- skipped.append({"name": name, "reason": "exceeds aggregate snapshot size cap"})
- continue
- payload[name] = blob
- total += len(blob)
-
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
temps: list[str] = []
@@ -765,56 +764,69 @@ def _snapshot_state(
previous = None
try:
try:
+ if max_bytes < len(_SNAPSHOT_MAGIC):
+ # Even the header alone busts the cap: keep the committed-payload <= cap invariant.
+ return {"error": "write failed: snapshot exceeds aggregate snapshot size cap"}
fh, tmp = stage_temp(path, "wb")
with fh:
- def dump_to_temp(candidate: dict[str, bytes]) -> int | None:
- writer = _CappedWriter(fh, max_bytes)
+ # Single pass: each variable is dill-serialized exactly once, streamed
+ # into the staged temp. The record header is charged against the aggregate
+ # cap up front, so a completed record can never overflow it (no prefix re-dump).
+ total = fh.write(_SNAPSHOT_MAGIC)
+ for name in list(ns.keys()):
+ if name.startswith("_") or name in _ALWAYS_SKIP:
+ continue
+ value = ns.get(name, missing)
+ if value is missing:
+ # A background thread deleted the name after the key listing.
+ skipped.append({"name": name, "reason": "deleted during snapshot"})
+ continue
+ encoded = name.encode("utf-8")
+ # Record header: 4-byte name length + 8-byte blob length, plus the name itself.
+ budget = max_bytes - total - 12 - len(encoded)
+ # Prune mode measures at the full per-variable cap: only that cap decides
+ # pruned-ness, and the write always re-measures — in-place mutation
+ # defeats any name-based size tracking from an earlier dump.
+ limit = max_variable_bytes if prune_oversized else min(max_variable_bytes, budget)
+ buffer = io.BytesIO()
try:
- dill.dump(candidate, writer)
+ dill.dump(value, _CappedWriter(buffer, limit))
+ blob = buffer.getvalue()
except _SnapshotSizeLimitExceeded:
- return None
- return writer.written
-
- def redump_to_temp(candidate: dict[str, bytes]) -> int | None:
- fh.seek(0)
- fh.truncate()
- return dump_to_temp(candidate)
-
- bytes_written = dump_to_temp(payload)
- if bytes_written is None:
- # Prefix pickle size is monotonic because each prefix only adds a string key and bytes value.
- items = list(payload.items())
- if redump_to_temp({}) is None:
- return {"error": "write failed: snapshot exceeds aggregate snapshot size cap"}
- low, high = 0, len(items) - 1
- while low < high:
- mid = (low + high + 1) // 2
- if redump_to_temp(dict(items[:mid])) is None:
- high = mid - 1
+ if not prune_oversized and budget < max_variable_bytes:
+ skipped.append({"name": name, "reason": "exceeds aggregate snapshot size cap"})
else:
- low = mid
- for name, _ in items[low:]:
+ skipped.append({"name": name, "reason": "exceeds per-variable snapshot size cap"})
+ oversized.append(name)
+ continue
+ except Exception as err: # noqa: BLE001 - one unpicklable name must not abort the snapshot
+ skipped.append({"name": name, "reason": f"{type(err).__name__}: {_safe_str(err)[:200]}"})
+ continue
+ if total + 12 + len(encoded) + len(blob) > max_bytes:
+ # Only reachable in prune mode, where the measurement cap ignores the budget.
skipped.append({"name": name, "reason": "exceeds aggregate snapshot size cap"})
- payload = dict(items[:low])
- # The search's last attempt may have overflowed the temp; rewrite the chosen prefix.
- bytes_written = redump_to_temp(payload)
- if bytes_written is None:
- return {"error": "write failed: snapshot exceeds aggregate snapshot size cap"}
- saved = sorted(payload.keys())
- pruned = sorted(name for name in oversized if name in ns) if prune_oversized else []
- manifest = {
- "version": 1,
- "savedNames": saved,
- "skipped": skipped,
- "pruned": pruned,
- "bytes": bytes_written,
- "pythonVersion": sys.version.split()[0],
- "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
- }
- stage = "manifest write"
- fh, manifest_tmp = stage_temp(manifest_path, "w")
- with fh:
- json.dump(manifest, fh)
+ continue
+ fh.write(len(encoded).to_bytes(4, "little"))
+ fh.write(encoded)
+ fh.write(len(blob).to_bytes(8, "little"))
+ fh.write(blob)
+ total += 12 + len(encoded) + len(blob)
+ saved.append(name)
+ saved.sort()
+ pruned = sorted(name for name in oversized if name in ns) if prune_oversized else []
+ manifest = {
+ "version": 1,
+ "savedNames": saved,
+ "skipped": skipped,
+ "pruned": pruned,
+ "bytes": total,
+ "pythonVersion": sys.version.split()[0],
+ "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+ }
+ stage = "manifest write"
+ fh, manifest_tmp = stage_temp(manifest_path, "w")
+ with fh:
+ json.dump(manifest, fh)
except BaseException as err: # noqa: BLE001 - Exception -> error dict, rest propagates
if not isinstance(err, Exception):
raise # e.g. KeyboardInterrupt: clean up (outer finally), then propagate
@@ -836,7 +848,7 @@ def _snapshot_state(
return {"error": f"manifest write failed: {err}"}
for name in pruned:
ns.pop(name, None)
- result = {"saved": saved, "skipped": skipped, "pruned": pruned, "bytes": bytes_written}
+ result = {"saved": saved, "skipped": skipped, "pruned": pruned, "bytes": total}
# Publish while still parked: a later KeyboardInterrupt into this task finds the committed result (see _handle_state).
if committed is not None:
committed.append(result)
@@ -868,7 +880,12 @@ def _restore_state(
return {"error": f"dill unavailable: {err}"}
try:
with open(path, "rb") as fh:
- payload = dill.load(fh)
+ if fh.read(len(_SNAPSHOT_MAGIC)) == _SNAPSHOT_MAGIC:
+ payload = _read_snapshot_records(fh)
+ else:
+ # Legacy: one dill-pickled dict; old snapshot files must keep restoring.
+ fh.seek(0)
+ payload = dill.load(fh)
except Exception as err: # noqa: BLE001 - a corrupt snapshot yields an empty restore
return {"error": f"load failed: {_safe_str(err)}"}
if not isinstance(payload, dict):
diff --git a/prime-agent-runtime/test/test_repl.py b/prime-agent-runtime/test/test_repl.py
index 396b439071..7e57221ab8 100644
--- a/prime-agent-runtime/test/test_repl.py
+++ b/prime-agent-runtime/test/test_repl.py
@@ -682,6 +682,23 @@ class ReplTest(unittest.TestCase):
events = self.repl.execute("p3", "'big' in dir()")
self.assertEqual(one(events, "result")["text"], "False")
+ def test_prune_measures_the_current_value_2478(self):
+ # Redefinition or in-place shrink after an oversized skip: prune must re-measure.
+ for cell in ("def big(): pass", "big.clear()"):
+ with self.subTest(cell=cell):
+ with tempfile.TemporaryDirectory() as tmp:
+ base = {"type": "snapshot", "path": os.path.join(tmp, "kernel-state.dill"),
+ "manifest_path": os.path.join(tmp, "kernel-state.json"), "max_variable_bytes": 1024}
+ self.repl.execute("pk1", "big = bytearray(b'x' * 100_000)\nkeep = 1")
+ self.repl.send({"id": "pk2", **base})
+ self.assertEqual(one(self.repl.until_done("pk2"), "done")["status"], "ok")
+ self.repl.execute("pk3", cell)
+ self.repl.send({"id": "pk4", "prune_oversized": True, **base})
+ done = one(self.repl.until_done("pk4"), "done")
+ self.assertEqual(done["status"], "ok")
+ self.assertEqual(done["pruned"], [])
+ self.assertEqual(done["saved"], ["big", "keep"])
+
def test_emit_display(self):
payloads = {
"application/vnd.prime-agent.diff+json": {
@@ -2126,11 +2143,10 @@ class SnapshotTempCleanupTest(unittest.TestCase):
real_dump = dill.dump
- def interrupted_dump(payload, fh):
- if isinstance(payload, dict): # the complete payload, not a per-variable value
- fh.write(b"partial")
- raise KeyboardInterrupt
- return real_dump(payload, fh)
+ def interrupted_dump(value, fh):
+ # Interrupt mid-dump, after partial bytes landed in the staged temp.
+ fh.write(b"partial")
+ raise KeyboardInterrupt
with unittest_mock.patch.object(dill, "dump", interrupted_dump):
with self.assertRaises(KeyboardInterrupt):
@@ -2222,19 +2238,22 @@ class SnapshotPairConsistencyTest(unittest.TestCase):
def test_near_cap_payload_skips_tail_instead_of_failing_snapshot(self):
import dill
+ from rlm.repl import _SNAPSHOT_MAGIC, _restore_state
+
dill.settings["recurse"] = True
source = {"a": "first", "b": "second"}
blobs = {name: dill.dumps(value) for name, value in source.items()}
- cap = len(dill.dumps(blobs)) - 1
- self.assertLessEqual(sum(map(len, blobs.values())), cap)
- self.assertLessEqual(len(dill.dumps({"a": blobs["a"]})), cap)
+ # One byte short of the full single-pass payload (magic + both records),
+ # so "a" fits exactly and "b" overflows the remaining aggregate budget.
+ cap = len(_SNAPSHOT_MAGIC) + 13 + len(blobs["a"]) + 13 + len(blobs["b"]) - 1
result = self._snap(source, max_bytes=cap, max_variable_bytes=cap)
self.assertEqual(result["saved"], ["a"])
self.assertEqual(result["skipped"], [{"name": "b", "reason": "exceeds aggregate snapshot size cap"}])
self.assertLessEqual(result["bytes"], cap)
- with open(self.path, "rb") as fh:
- self.assertEqual(list(dill.load(fh)), ["a"])
+ ns: dict = {}
+ self.assertNotIn("error", _restore_state(ns, self.path))
+ self.assertEqual(ns, {"a": "first"})
def test_manifest_write_failure_preserves_prior_pair(self):
old_payload, old_manifest = self._old_pair()
@@ -2419,6 +2438,28 @@ class SnapshotPairConsistencyTest(unittest.TestCase):
self.assertEqual(json.load(fh)["savedNames"], ["keep"])
self.assertEqual(sorted(os.listdir(d)), sorted([payload_name, manifest_name]))
+ def test_each_variable_serialized_once_and_payload_is_not_a_pickle(self):
+ import dill
+
+ from rlm.repl import _SNAPSHOT_MAGIC
+
+ real_dump = dill.dump
+ dumped: list[object] = []
+
+ def counting_dump(value, writer):
+ dumped.append(value)
+ return real_dump(value, writer)
+
+ with mock.patch.object(dill, "dump", counting_dump):
+ result = self._snap({"a": 1, "b": 2, "c": 3})
+ self.assertEqual(result["saved"], ["a", "b", "c"])
+ self.assertEqual(dumped, [1, 2, 3])
+ self.assertEqual(result["bytes"], os.path.getsize(self.path))
+ with open(self.path, "rb") as fh:
+ self.assertEqual(fh.read(len(_SNAPSHOT_MAGIC)), _SNAPSHOT_MAGIC)
+ with self.assertRaises(Exception):
+ dill.load(fh)
+
class OwnerWatchdogTest(unittest.TestCase):
def test_owner_watchdog_exits_busy_runtime(self):