The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(run-logs): skip durable log restore for a proven-new run (#17367)
Nicky Leach2e3864934181
diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.mdindex 1e8bc2acf2..ed505bc771 100644--- a/packages/paperclip-runner/docs/durable-recovery.md+++ b/packages/paperclip-runner/docs/durable-recovery.md@@ -292,6 +292,7 @@ model or rely on an LLM choosing the desired timing.| Execution and cleanup both fail | Original execution error retained; cleanup error attached | `native-session-runtime.test.ts` || Retry in same process or after local log loss | Earlier attempts survive locally and in the durable mirror | Server `run-log-store.test.ts` || Two restores overlap and one appends before the other publishes | Complete prefix is published without replacement; both attempts survive | Server `run-log-store.test.ts` |+| New run whose row has no log reference, output sequence, or log size | No durable-mirror request; a later attempt with a recorded log reference still restores after local loss | Server `run-log-store.test.ts`, `heartbeat/run-output.test.ts` |## Residual local trust and revocation windowdiff --git a/server/src/services/heartbeat/run-output.test.ts b/server/src/services/heartbeat/run-output.test.tsnew file mode 100644index 0000000000..bcfb7c3c2b--- /dev/null+++ b/server/src/services/heartbeat/run-output.test.ts@@ -0,0 +1,123 @@+import { promises as fs } from "node:fs";+import os from "node:os";+import path from "node:path";+import { Readable } from "node:stream";+import type { Db, heartbeatRuns } from "@paperclipai/db";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { createDurableRunLogStore } from "../run-log-store.js";+import type { StorageProvider } from "../../storage/types.js";+import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js";++vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));++type Run = typeof heartbeatRuns.$inferSelect;++function createMemoryProvider() {+ const objects = new Map<string, Buffer>();+ const calls = { head: 0, get: 0 };+ const provider: StorageProvider = {+ id: "s3",+ async putObject(input) {+ const chunks: Buffer[] = [];+ for await (const chunk of Readable.from(input.body)) chunks.push(Buffer.from(chunk));+ objects.set(input.objectKey, Buffer.concat(chunks));+ },+ async getObject(input) {+ calls.get++;+ const buf = objects.get(input.objectKey)!;+ return { stream: Readable.from(buf), contentLength: buf.length };+ },+ async headObject(input) {+ calls.head++;+ const buf = objects.get(input.objectKey);+ return buf ? { exists: true, contentLength: buf.length } : { exists: false };+ },+ async deleteObject(input) {+ objects.delete(input.objectKey);+ },+ };+ return { provider, objects, calls };+}++// The recorder persists the handle with update().set().where(); capture it the+// way the run row would carry it into a later attempt.+function createRunRowDb(row: Partial<Run>) {+ return {+ update: () => ({+ set: (values: Partial<Run>) => ({+ where: async () => { Object.assign(row, values); },+ }),+ }),+ } as unknown as Db;+}++const newRunRow = {+ id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running",+ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0, lastOutputAt: null,+} as Partial<Run>;++async function beginAttempt(store: ReturnType<typeof createDurableRunLogStore>, row: Partial<Run>) {+ const output = createHeartbeatRunOutput(createRunRowDb(row), {+ run: row as Run,+ runId: row.id!,+ issueId: null,+ runLogStore: store,+ identityRedactor: { chunk: (_stream, chunk) => chunk, redact: (value) => value },+ secretKeys: new Set(),+ appendRunEvent: vi.fn(),+ getCurrentUserRedactionOptions: async () => ({ enabled: false }),+ });+ const { onLog } = await output.begin({ ...row } as Run);+ return { output, onLog };+}++let baseDir: string;+beforeEach(async () => {+ baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "run-output-test-"));+});+afterEach(async () => {+ await fs.rm(baseDir, { recursive: true, force: true });+});++describe("heartbeat run output log recovery", () => {+ it("treats only a run row with no recorded log or output as new", () => {+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(true);+ expect(isNewRunLog({ logStore: "local_file", logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);+ expect(isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(false);+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(false);+ });++ it("makes no remote log recovery request for a new run", async () => {+ const { provider, calls } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const row = { ...newRunRow };+ const { onLog } = await beginAttempt(store, row);+ await onLog("stdout", "hello\n");+ expect(calls).toEqual({ head: 0, get: 0 });+ expect(row.logRef).toBe(path.join("company-1", "agent-1", "run-1.ndjson"));+ });++ it("restores an existing attempt's durable log after local state is lost", async () => {+ const { provider, objects, calls } = createMemoryProvider();+ const row = { ...newRunRow };+ const firstStore = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const first = await beginAttempt(firstStore, row);+ await first.onLog("stdout", "attempt one\n");+ await firstStore.finalize(first.output.handle!);+ expect(calls).toEqual({ head: 0, get: 0 });++ // Process restart on a new pod: local logs and in-memory state are gone,+ // but the run row still records the earlier attempt's log.+ await fs.rm(baseDir, { recursive: true, force: true });+ const restartedStore = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const second = await beginAttempt(restartedStore, row);+ expect(calls).toEqual({ head: 1, get: 1 });+ await second.onLog("stdout", "attempt two\n");+ await restartedStore.finalize(second.output.handle!);++ const lines = objects.get(row.logRef!)!.toString().trim().split("\n").map((line) => JSON.parse(line));+ expect(lines.map((line) => line.chunk)).toEqual(["attempt one\n", "attempt two\n"]);+ expect(lines.map((line) => line.seq)).toEqual([1, 2]);+ });+});diff --git a/server/src/services/heartbeat/run-output.ts b/server/src/services/heartbeat/run-output.tsindex 1b1091b905..2b285ef829 100644--- a/server/src/services/heartbeat/run-output.ts+++ b/server/src/services/heartbeat/run-output.ts@@ -26,6 +26,21 @@ export interface HeartbeatRunOutputInput {getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;}+/**+ * True only when the persisted run row proves no earlier attempt began its log.+ * The first begin() records logStore/logRef before any output can be appended+ * or mirrored to object storage; output sequence and final size are recorded+ * after that. A row with none of them has no durable log prefix to restore.+ */+export function isNewRunLog(+ run: Pick<Run, "logStore" | "logRef" | "logBytes" | "lastOutputSeq">,+): boolean {+ return run.logStore == null+ && run.logRef == null+ && run.logBytes == null+ && Number(run.lastOutputSeq ?? 0) === 0;+}+/** One recorder per execution attempt; the log store itself stays service-scoped. */export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) {const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input;@@ -74,6 +89,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)companyId: run.companyId,agentId: run.agentId,runId,+ newRun: isNewRunLog(run),});await dbdiff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.tsindex 75b7efda35..c4ad5900fe 100644--- a/server/src/services/run-log-store.test.ts+++ b/server/src/services/run-log-store.test.ts@@ -175,6 +175,24 @@ describe("createDurableRunLogStore", () => {} finally { spy.mockRestore(); }});+ it("skips the durable restore request only for a proven-new run", async () => {+ const { provider, objects, calls } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const fresh = await store.begin({ ...begin, newRun: true });+ expect(calls).toMatchObject({ head: 0, get: 0 });+ await store.append(fresh, { stream: "stdout", chunk: "first attempt", ts: "t1" });+ await store.finalize(fresh);++ // Lose local state, then begin again without the proof: the prefix restores.+ await fs.rm(baseDir, { recursive: true, force: true });+ const recovered = await store.begin(begin);+ expect(calls).toMatchObject({ head: 1, get: 1 });+ await store.append(recovered, { stream: "stdout", chunk: "second attempt", ts: "t2" });+ await store.finalize(recovered);+ expect(objects.get(fresh.logRef)!.toString().trim().split("\n").map((line) => JSON.parse(line).chunk))+ .toEqual(["first attempt", "second attempt"]);+ });+it("uploads the complete log to S3 on finalize", async () => {const { provider, objects, calls } = createMemoryProvider();const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.tsindex 85a51ba1e0..7e26b15261 100644--- a/server/src/services/run-log-store.ts+++ b/server/src/services/run-log-store.ts@@ -35,8 +35,18 @@ export interface RunLogFinalizeSummary {compressed: boolean;}+export interface RunLogBeginInput {+ companyId: string;+ agentId: string;+ runId: string;+ // Set only when persisted run lifecycle state proves no earlier attempt+ // began this log, so no durable prefix can exist to restore. Omitted means+ // the run may be a retry or recovery and the durable mirror is checked.+ newRun?: boolean;+}+export interface RunLogStore {- begin(input: { companyId: string; agentId: string; runId: string }): Promise<RunLogHandle>;+ begin(input: RunLogBeginInput): Promise<RunLogHandle>;append(handle: RunLogHandle,event: { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string; seq?: number },@@ -300,11 +310,12 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ruawait retireInflightMirror(relPath);// Retries share a run ID. Never truncate their earlier evidence. On a// new pod, restore the durable prefix before opening another attempt.+ // A proven-new run has no prefix, so skip the remote round trip.const exists = await fs.stat(absPath).then(() => true, (error) => {if (error.code === "ENOENT") return false;throw error;});- if (!exists && s3) {+ if (!exists && s3 && input.newRun !== true) {const key = s3Key(relPath);const head = await s3.provider.headObject({ objectKey: key });if (head.exists) {