Skip to content

The open-source app everyone uses to manage agents at work

feat(run-logs): durable run-log store via object-storage mirror (#8984)

Jannes Stubbemannb4e7ba51436c
diff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.ts
new file mode 100644
index 0000000000..029262ee07
--- /dev/null
+++ b/server/src/services/run-log-store.test.ts
@@ -0,0 +1,168 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { Readable } from "node:stream";
+import { createDurableRunLogStore } from "./run-log-store.js";
+import type { StorageProvider } from "../storage/types.js";
+
+// In-memory StorageProvider stand-in: durable, survives the "pod roll" (local
+// dir wipe) the same way Cubbit does. Records calls so we can assert behaviour.
+function createMemoryProvider() {
+ const objects = new Map<string, Buffer>();
+ const calls = { put: 0, get: 0, head: 0 };
+ const provider: StorageProvider = {
+ id: "s3",
+ async putObject(input) {
+ calls.put++;
+ if (Buffer.isBuffer(input.body)) {
+ objects.set(input.objectKey, Buffer.from(input.body));
+ return;
+ }
+ const chunks: Buffer[] = [];
+ for await (const chunk of input.body) {
+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+ }
+ objects.set(input.objectKey, Buffer.concat(chunks));
+ },
+ async getObject(input) {
+ calls.get++;
+ const buf = objects.get(input.objectKey);
+ if (!buf) {
+ const err = new Error("Object not found") as Error & { name: string };
+ err.name = "NoSuchKey";
+ throw err;
+ }
+ const slice = input.range ? buf.subarray(input.range.start, input.range.end + 1) : buf;
+ return { stream: Readable.from(slice), contentLength: slice.length };
+ },
+ async headObject(input) {
+ calls.head++;
+ const buf = objects.get(input.objectKey);
+ return buf ? { exists: true, contentLength: buf.length } : { exists: false };
+ },
+ async deleteObject(input) {
+ objects.delete(input.objectKey);
+ },
+ };
+ return { provider, objects, calls };
+}
+
+let baseDir: string;
+beforeEach(async () => {
+ baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "run-log-store-test-"));
+});
+afterEach(async () => {
+ await fs.rm(baseDir, { recursive: true, force: true });
+});
+
+const begin = { companyId: "co1", agentId: "ag1", runId: "run1" };
+
+describe("createDurableRunLogStore", () => {
+ it("keeps store id 'local_file' so downstream coupling (feedback, casts) is unchanged", async () => {
+ const { provider } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });
+ const handle = await store.begin(begin);
+ expect(handle.store).toBe("local_file");
+ });
+
+ it("appends locally and reads back during a run WITHOUT hitting S3 (fast live tail)", async () => {
+ const { provider, calls } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });
+ const handle = await store.begin(begin);
+ await store.append(handle, { stream: "stdout", chunk: "hello", ts: "t1" });
+ const res = await store.read(handle);
+ expect(res.content).toContain("hello");
+ expect(calls.get).toBe(0); // local file present -> no S3 read
+ });
+
+ it("uploads the complete log to S3 on finalize", async () => {
+ const { provider, objects, calls } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });
+ const handle = await store.begin(begin);
+ await store.append(handle, { stream: "stdout", chunk: "line-A", ts: "t1" });
+ await store.append(handle, { stream: "stdout", chunk: "line-B", ts: "t2" });
+ const summary = await store.finalize(handle);
+ expect(calls.put).toBe(1);
+ expect(summary.bytes).toBeGreaterThan(0);
+ // keyed by prefix + the handle's logRef so read can find it later
+ const key = `run-logs/${handle.logRef}`;
+ expect(objects.has(key)).toBe(true);
+ expect(objects.get(key)!.toString("utf8")).toContain("line-A");
+ expect(objects.get(key)!.toString("utf8")).toContain("line-B");
+ });
+
+ it("falls back to S3 when the local file is gone (the pod-roll case that caused 'Run log not found')", async () => {
+ const { provider } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });
+ const handle = await store.begin(begin);
+ await store.append(handle, { stream: "stdout", chunk: "persisted-line", ts: "t1" });
+ await store.finalize(handle);
+ // Simulate a pod restart wiping the emptyDir.
+ await fs.rm(baseDir, { recursive: true, force: true });
+ const res = await store.read(handle);
+ expect(res.content).toContain("persisted-line");
+ });
+
+ it("S3 fallback honours offset/limitBytes (range read) and reports nextOffset", async () => {
+ const { provider } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "p" } });
+ const handle = await store.begin(begin);
+ // one line; persisted bytes = JSON line + "\n"
+ await store.append(handle, { stream: "stdout", chunk: "0123456789", ts: "t" });
+ await store.finalize(handle);
+ const full = await store.read(handle); // from local, to learn total size
+ const total = Buffer.byteLength(full.content, "utf8");
+ await fs.rm(baseDir, { recursive: true, force: true }); // force S3 path
+ const firstHalf = await store.read(handle, { offset: 0, limitBytes: 5 });
+ expect(Buffer.byteLength(firstHalf.content, "utf8")).toBe(5);
+ expect(firstHalf.nextOffset).toBe(5);
+ const tail = await store.read(handle, { offset: total - 3, limitBytes: 100 });
+ expect(Buffer.byteLength(tail.content, "utf8")).toBe(3);
+ expect(tail.nextOffset).toBeUndefined();
+ });
+
+ it("falls back to S3 when the local file vanishes between stat() and open (TOCTOU race)", async () => {
+ const { provider } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });
+ const handle = await store.begin(begin);
+ await store.append(handle, { stream: "stdout", chunk: "raced-line", ts: "t1" });
+ await store.finalize(handle);
+ // Delete the local file DURING stat(), i.e. after it reports the file
+ // present but before createReadStream opens it -> the open hits ENOENT.
+ const realStat = fs.stat.bind(fs);
+ const statSpy = vi.spyOn(fs, "stat").mockImplementation(async (target, ...rest) => {
+ const result = await realStat(target as Parameters<typeof realStat>[0], ...(rest as []));
+ if (String(target).endsWith(".ndjson")) {
+ await fs.rm(target as string, { force: true });
+ }
+ return result;
+ });
+ try {
+ const res = await store.read(handle);
+ expect(res.content).toContain("raced-line");
+ } finally {
+ statSpy.mockRestore();
+ }
+ });
+
+ it("throws notFound when neither local nor S3 has the log (pre-S3 run after a roll)", async () => {
+ const { provider } = createMemoryProvider();
+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });
+ const handle = await store.begin(begin);
+ await fs.rm(baseDir, { recursive: true, force: true }); // never finalized -> never uploaded
+ await expect(store.read(handle)).rejects.toThrow(/not found/i);
+ });
+
+ it("without S3 configured behaves exactly like the local-only store (safe degrade)", async () => {
+ const store = createDurableRunLogStore({ basePath: baseDir });
+ const handle = await store.begin(begin);
+ await store.append(handle, { stream: "stdout", chunk: "local-only", ts: "t1" });
+ await store.finalize(handle);
+ const res = await store.read(handle);
+ expect(res.content).toContain("local-only");
+ // and a roll loses it (documented limitation; this is the pre-fix behaviour)
+ await fs.rm(baseDir, { recursive: true, force: true });
+ await expect(store.read(handle)).rejects.toThrow(/not found/i);
+ });
+});
diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.ts
index 0d3b508c9d..8bf10d1ed4 100644
--- a/server/src/services/run-log-store.ts
+++ b/server/src/services/run-log-store.ts
@@ -3,6 +3,8 @@ import path from "node:path";
import { createHash } from "node:crypto";
import { notFound } from "../errors.js";
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
+import { createS3StorageProvider } from "../storage/s3-provider.js";
+import type { StorageProvider } from "../storage/types.js";
export type RunLogStoreType = "local_file";
@@ -50,35 +52,97 @@ function resolveWithin(basePath: string, relativePath: string) {
return resolved;
}
-function createLocalFileRunLogStore(basePath: string): RunLogStore {
+function normalizeKeyPrefix(prefix: string | undefined): string {
+ if (!prefix) return "";
+ return prefix.trim().replace(/^\/+/, "").replace(/\/+$/, "");
+}
+
+export interface DurableRunLogStoreOptions {
+ basePath: string;
+ // When provided, completed logs are mirrored to object storage on finalize and
+ // served from there on read whenever the local file is missing (e.g. the pod
+ // rolled and wiped the emptyDir). When omitted, the store is local-only (the
+ // historical behaviour: a restart loses the log).
+ s3?: { provider: StorageProvider; keyPrefix?: string };
+}
+
+// Run-log store with TRANSPARENT durability. The store id stays "local_file" so
+// nothing downstream (feedback.ts, the heartbeat read cast, fixtures) changes;
+// the S3 mirror is keyed by the same logRef and is purely an implementation
+// detail. Live append/tail stays on the pod-local file (fast, no per-chunk PUT);
+// on finalize the complete .ndjson is uploaded to object storage; on read we try
+// local first and fall back to S3 when the local file is gone. This is the fix
+// for "Run log not found" after a deploy/restart (the /paperclip data dir is an
+// emptyDir in cloud_tenant mode -- persistence is disabled to avoid the
+// operator's privileged selinux-relabel init container in our hardened ns).
+export function createDurableRunLogStore(options: DurableRunLogStoreOptions): RunLogStore {
+ const { basePath } = options;
+ const s3 = options.s3;
+ const s3Prefix = normalizeKeyPrefix(s3?.keyPrefix);
+
+ function s3Key(logRef: string): string {
+ return s3Prefix ? `${s3Prefix}/${logRef}` : logRef;
+ }
+
async function ensureDir(relativeDir: string) {
const dir = resolveWithin(basePath, relativeDir);
await fs.mkdir(dir, { recursive: true });
}
- async function readFileRange(filePath: string, offset: number, limitBytes: number): Promise<RunLogReadResult> {
+ async function readLocalRange(
+ filePath: string,
+ offset: number,
+ limitBytes: number,
+ ): Promise<RunLogReadResult | null> {
const stat = await fs.stat(filePath).catch(() => null);
- if (!stat) throw notFound("Run log not found");
-
+ if (!stat) return null;
const start = Math.max(0, Math.min(offset, stat.size));
const end = Math.max(start, Math.min(start + limitBytes - 1, stat.size - 1));
+ if (start > end) return { content: "", nextOffset: start };
- if (start > end) {
- return { content: "", nextOffset: start };
+ const chunks: Buffer[] = [];
+ try {
+ await new Promise<void>((resolve, reject) => {
+ const stream = createReadStream(filePath, { start, end });
+ stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
+ stream.on("error", reject);
+ stream.on("end", () => resolve());
+ });
+ } catch (err) {
+ // File deleted between stat() and open (pod-roll cleanup racing a read):
+ // treat as missing so the caller falls through to the S3 mirror instead
+ // of surfacing the very "Run log not found" this store exists to prevent.
+ if ((err as NodeJS.ErrnoException | null)?.code === "ENOENT") return null;
+ throw err;
}
+ const content = Buffer.concat(chunks).toString("utf8");
+ const nextOffset = end + 1 < stat.size ? end + 1 : undefined;
+ return { content, nextOffset };
+ }
+
+ async function readS3Range(
+ logRef: string,
+ offset: number,
+ limitBytes: number,
+ ): Promise<RunLogReadResult> {
+ if (!s3) throw notFound("Run log not found");
+ const key = s3Key(logRef);
+ const head = await s3.provider.headObject({ objectKey: key });
+ if (!head.exists) throw notFound("Run log not found");
+ const total = head.contentLength ?? 0;
+ const start = Math.max(0, Math.min(offset, total));
+ const end = Math.max(start, Math.min(start + limitBytes - 1, total - 1));
+ if (start > end || total === 0) return { content: "", nextOffset: start < total ? start : undefined };
+ const result = await s3.provider.getObject({ objectKey: key, range: { start, end } });
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
- const stream = createReadStream(filePath, { start, end });
- stream.on("data", (chunk) => {
- chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
- });
- stream.on("error", reject);
- stream.on("end", () => resolve());
+ result.stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
+ result.stream.on("error", reject);
+ result.stream.on("end", () => resolve());
});
-
const content = Buffer.concat(chunks).toString("utf8");
- const nextOffset = end + 1 < stat.size ? end + 1 : undefined;
+ const nextOffset = end + 1 < total ? end + 1 : undefined;
return { content, nextOffset };
}
@@ -99,10 +163,8 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore {
const relDir = path.join(companyId, agentId);
const relPath = path.join(relDir, `${runId}.ndjson`);
await ensureDir(relDir);
-
const absPath = resolveWithin(basePath, relPath);
await fs.writeFile(absPath, "", "utf8");
-
return { store: "local_file", logRef: relPath };
},
@@ -124,38 +186,79 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore {
},
async finalize(handle) {
- if (handle.store !== "local_file") {
- return { bytes: 0, compressed: false };
- }
+ if (handle.store !== "local_file") return { bytes: 0, compressed: false };
const absPath = resolveWithin(basePath, handle.logRef);
const stat = await fs.stat(absPath).catch(() => null);
if (!stat) throw notFound("Run log not found");
-
const hash = await sha256File(absPath);
- return {
- bytes: stat.size,
- sha256: hash,
- compressed: false,
- };
+
+ // Mirror the completed log to object storage so it survives a pod roll.
+ // Best-effort upload failures must NOT fail run finalization (which also
+ // records cost/usage); the local copy still serves reads until the pod
+ // rolls, and a failed mirror only loses durability for that one run.
+ if (s3) {
+ try {
+ // Stream from disk instead of buffering the whole .ndjson in the
+ // heap; long agent sessions can produce large logs. The file is
+ // complete at this point, so stat.size is the exact content length.
+ await s3.provider.putObject({
+ objectKey: s3Key(handle.logRef),
+ body: createReadStream(absPath),
+ contentType: "application/x-ndjson",
+ contentLength: stat.size,
+ });
+ } catch (err) {
+ // Best-effort: finalization must not break, but a persistently
+ // failing mirror (bad creds/bucket/endpoint) should be visible to
+ // operators before a pod roll makes the logs unreadable.
+ console.warn(
+ `[run-log-store] Failed to mirror run log to object storage (key: ${s3Key(handle.logRef)}):`,
+ err,
+ );
+ }
+ }
+
+ return { bytes: stat.size, sha256: hash, compressed: false };
},
async read(handle, opts) {
- if (handle.store !== "local_file") {
- throw notFound("Run log not found");
- }
+ if (handle.store !== "local_file") throw notFound("Run log not found");
const absPath = resolveWithin(basePath, handle.logRef);
const offset = opts?.offset ?? 0;
const limitBytes = opts?.limitBytes ?? 256_000;
- return readFileRange(absPath, offset, limitBytes);
+ const local = await readLocalRange(absPath, offset, limitBytes);
+ if (local) return local;
+ // Local file gone (pod rolled) -> serve from the S3 mirror if configured.
+ return readS3Range(handle.logRef, offset, limitBytes);
},
};
}
+// Build the run-log S3 mirror from dedicated RUN_LOG_S3_* env. Deliberately
+// separate from PAPERCLIP_STORAGE_PROVIDER so enabling durable run logs does
+// NOT redirect the product's workspace/file storage (smaller blast radius).
+// Unset RUN_LOG_S3_BUCKET -> no mirror -> local-only (safe degrade). Creds come
+// from the standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY chain.
+function resolveRunLogS3(): { provider: StorageProvider; keyPrefix?: string } | undefined {
+ const bucket = process.env.RUN_LOG_S3_BUCKET?.trim();
+ if (!bucket) return undefined;
+ const provider = createS3StorageProvider({
+ bucket,
+ region: process.env.RUN_LOG_S3_REGION?.trim() || "us-east-1",
+ endpoint: process.env.RUN_LOG_S3_ENDPOINT?.trim() || undefined,
+ prefix: undefined, // prefixing is handled by keyPrefix below (kept off the provider)
+ forcePathStyle: process.env.RUN_LOG_S3_FORCE_PATH_STYLE
+ ? process.env.RUN_LOG_S3_FORCE_PATH_STYLE === "true"
+ : true, // Cubbit (and most S3-compatible endpoints) need path-style
+ });
+ return { provider, keyPrefix: process.env.RUN_LOG_S3_PREFIX?.trim() || "run-logs" };
+}
+
let cachedStore: RunLogStore | null = null;
export function getRunLogStore() {
if (cachedStore) return cachedStore;
const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs");
- cachedStore = createLocalFileRunLogStore(basePath);
+ cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() });
return cachedStore;
}
diff --git a/server/src/storage/types.ts b/server/src/storage/types.ts
index efeadd8ba8..dab23fa4c0 100644
--- a/server/src/storage/types.ts
+++ b/server/src/storage/types.ts
@@ -3,7 +3,9 @@ import type { Readable } from "node:stream";
export interface PutObjectInput {
objectKey: string;
- body: Buffer;
+ // Readable bodies stream straight to the backend (contentLength must be the
+ // exact byte size); Buffer stays supported for small payloads.
+ body: Buffer | Readable;
contentType: string;
contentLength: number;
}