The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(run-logs): durable run-log store via object-storage mirror (#8984)
Jannes Stubbemannb4e7ba51436c
diff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.tsnew file mode 100644index 0000000000..029262ee07--- /dev/null+++ b/server/src/services/run-log-store.test.ts@@ -0,0 +1,168 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { promises as fs } from "node:fs";+import path from "node:path";+import os from "node:os";+import { Readable } from "node:stream";+import { createDurableRunLogStore } from "./run-log-store.js";+import type { StorageProvider } from "../storage/types.js";++// In-memory StorageProvider stand-in: durable, survives the "pod roll" (local+// dir wipe) the same way Cubbit does. Records calls so we can assert behaviour.+function createMemoryProvider() {+ const objects = new Map<string, Buffer>();+ const calls = { put: 0, get: 0, head: 0 };+ const provider: StorageProvider = {+ id: "s3",+ async putObject(input) {+ calls.put++;+ if (Buffer.isBuffer(input.body)) {+ objects.set(input.objectKey, Buffer.from(input.body));+ return;+ }+ const chunks: Buffer[] = [];+ for await (const chunk of input.body) {+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));+ }+ objects.set(input.objectKey, Buffer.concat(chunks));+ },+ async getObject(input) {+ calls.get++;+ const buf = objects.get(input.objectKey);+ if (!buf) {+ const err = new Error("Object not found") as Error & { name: string };+ err.name = "NoSuchKey";+ throw err;+ }+ const slice = input.range ? buf.subarray(input.range.start, input.range.end + 1) : buf;+ return { stream: Readable.from(slice), contentLength: slice.length };+ },+ async headObject(input) {+ calls.head++;+ const buf = objects.get(input.objectKey);+ return buf ? { exists: true, contentLength: buf.length } : { exists: false };+ },+ async deleteObject(input) {+ objects.delete(input.objectKey);+ },+ };+ return { provider, objects, calls };+}++let baseDir: string;+beforeEach(async () => {+ baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "run-log-store-test-"));+});+afterEach(async () => {+ await fs.rm(baseDir, { recursive: true, force: true });+});++const begin = { companyId: "co1", agentId: "ag1", runId: "run1" };++describe("createDurableRunLogStore", () => {+ it("keeps store id 'local_file' so downstream coupling (feedback, casts) is unchanged", async () => {+ const { provider } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const handle = await store.begin(begin);+ expect(handle.store).toBe("local_file");+ });++ it("appends locally and reads back during a run WITHOUT hitting S3 (fast live tail)", async () => {+ const { provider, calls } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const handle = await store.begin(begin);+ await store.append(handle, { stream: "stdout", chunk: "hello", ts: "t1" });+ const res = await store.read(handle);+ expect(res.content).toContain("hello");+ expect(calls.get).toBe(0); // local file present -> no S3 read+ });++ it("uploads the complete log to S3 on finalize", async () => {+ const { provider, objects, calls } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });+ const handle = await store.begin(begin);+ await store.append(handle, { stream: "stdout", chunk: "line-A", ts: "t1" });+ await store.append(handle, { stream: "stdout", chunk: "line-B", ts: "t2" });+ const summary = await store.finalize(handle);+ expect(calls.put).toBe(1);+ expect(summary.bytes).toBeGreaterThan(0);+ // keyed by prefix + the handle's logRef so read can find it later+ const key = `run-logs/${handle.logRef}`;+ expect(objects.has(key)).toBe(true);+ expect(objects.get(key)!.toString("utf8")).toContain("line-A");+ expect(objects.get(key)!.toString("utf8")).toContain("line-B");+ });++ it("falls back to S3 when the local file is gone (the pod-roll case that caused 'Run log not found')", async () => {+ const { provider } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });+ const handle = await store.begin(begin);+ await store.append(handle, { stream: "stdout", chunk: "persisted-line", ts: "t1" });+ await store.finalize(handle);+ // Simulate a pod restart wiping the emptyDir.+ await fs.rm(baseDir, { recursive: true, force: true });+ const res = await store.read(handle);+ expect(res.content).toContain("persisted-line");+ });++ it("S3 fallback honours offset/limitBytes (range read) and reports nextOffset", async () => {+ const { provider } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "p" } });+ const handle = await store.begin(begin);+ // one line; persisted bytes = JSON line + "\n"+ await store.append(handle, { stream: "stdout", chunk: "0123456789", ts: "t" });+ await store.finalize(handle);+ const full = await store.read(handle); // from local, to learn total size+ const total = Buffer.byteLength(full.content, "utf8");+ await fs.rm(baseDir, { recursive: true, force: true }); // force S3 path+ const firstHalf = await store.read(handle, { offset: 0, limitBytes: 5 });+ expect(Buffer.byteLength(firstHalf.content, "utf8")).toBe(5);+ expect(firstHalf.nextOffset).toBe(5);+ const tail = await store.read(handle, { offset: total - 3, limitBytes: 100 });+ expect(Buffer.byteLength(tail.content, "utf8")).toBe(3);+ expect(tail.nextOffset).toBeUndefined();+ });++ it("falls back to S3 when the local file vanishes between stat() and open (TOCTOU race)", async () => {+ const { provider } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } });+ const handle = await store.begin(begin);+ await store.append(handle, { stream: "stdout", chunk: "raced-line", ts: "t1" });+ await store.finalize(handle);+ // Delete the local file DURING stat(), i.e. after it reports the file+ // present but before createReadStream opens it -> the open hits ENOENT.+ const realStat = fs.stat.bind(fs);+ const statSpy = vi.spyOn(fs, "stat").mockImplementation(async (target, ...rest) => {+ const result = await realStat(target as Parameters<typeof realStat>[0], ...(rest as []));+ if (String(target).endsWith(".ndjson")) {+ await fs.rm(target as string, { force: true });+ }+ return result;+ });+ try {+ const res = await store.read(handle);+ expect(res.content).toContain("raced-line");+ } finally {+ statSpy.mockRestore();+ }+ });++ it("throws notFound when neither local nor S3 has the log (pre-S3 run after a roll)", async () => {+ const { provider } = createMemoryProvider();+ const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } });+ const handle = await store.begin(begin);+ await fs.rm(baseDir, { recursive: true, force: true }); // never finalized -> never uploaded+ await expect(store.read(handle)).rejects.toThrow(/not found/i);+ });++ it("without S3 configured behaves exactly like the local-only store (safe degrade)", async () => {+ const store = createDurableRunLogStore({ basePath: baseDir });+ const handle = await store.begin(begin);+ await store.append(handle, { stream: "stdout", chunk: "local-only", ts: "t1" });+ await store.finalize(handle);+ const res = await store.read(handle);+ expect(res.content).toContain("local-only");+ // and a roll loses it (documented limitation; this is the pre-fix behaviour)+ await fs.rm(baseDir, { recursive: true, force: true });+ await expect(store.read(handle)).rejects.toThrow(/not found/i);+ });+});diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.tsindex 0d3b508c9d..8bf10d1ed4 100644--- a/server/src/services/run-log-store.ts+++ b/server/src/services/run-log-store.ts@@ -3,6 +3,8 @@ import path from "node:path";import { createHash } from "node:crypto";import { notFound } from "../errors.js";import { resolvePaperclipInstanceRoot } from "../home-paths.js";+import { createS3StorageProvider } from "../storage/s3-provider.js";+import type { StorageProvider } from "../storage/types.js";export type RunLogStoreType = "local_file";@@ -50,35 +52,97 @@ function resolveWithin(basePath: string, relativePath: string) {return resolved;}-function createLocalFileRunLogStore(basePath: string): RunLogStore {+function normalizeKeyPrefix(prefix: string | undefined): string {+ if (!prefix) return "";+ return prefix.trim().replace(/^\/+/, "").replace(/\/+$/, "");+}++export interface DurableRunLogStoreOptions {+ basePath: string;+ // When provided, completed logs are mirrored to object storage on finalize and+ // served from there on read whenever the local file is missing (e.g. the pod+ // rolled and wiped the emptyDir). When omitted, the store is local-only (the+ // historical behaviour: a restart loses the log).+ s3?: { provider: StorageProvider; keyPrefix?: string };+}++// Run-log store with TRANSPARENT durability. The store id stays "local_file" so+// nothing downstream (feedback.ts, the heartbeat read cast, fixtures) changes;+// the S3 mirror is keyed by the same logRef and is purely an implementation+// detail. Live append/tail stays on the pod-local file (fast, no per-chunk PUT);+// on finalize the complete .ndjson is uploaded to object storage; on read we try+// local first and fall back to S3 when the local file is gone. This is the fix+// for "Run log not found" after a deploy/restart (the /paperclip data dir is an+// emptyDir in cloud_tenant mode -- persistence is disabled to avoid the+// operator's privileged selinux-relabel init container in our hardened ns).+export function createDurableRunLogStore(options: DurableRunLogStoreOptions): RunLogStore {+ const { basePath } = options;+ const s3 = options.s3;+ const s3Prefix = normalizeKeyPrefix(s3?.keyPrefix);++ function s3Key(logRef: string): string {+ return s3Prefix ? `${s3Prefix}/${logRef}` : logRef;+ }+async function ensureDir(relativeDir: string) {const dir = resolveWithin(basePath, relativeDir);await fs.mkdir(dir, { recursive: true });}- async function readFileRange(filePath: string, offset: number, limitBytes: number): Promise<RunLogReadResult> {+ async function readLocalRange(+ filePath: string,+ offset: number,+ limitBytes: number,+ ): Promise<RunLogReadResult | null> {const stat = await fs.stat(filePath).catch(() => null);- if (!stat) throw notFound("Run log not found");-+ if (!stat) return null;const start = Math.max(0, Math.min(offset, stat.size));const end = Math.max(start, Math.min(start + limitBytes - 1, stat.size - 1));+ if (start > end) return { content: "", nextOffset: start };- if (start > end) {- return { content: "", nextOffset: start };+ const chunks: Buffer[] = [];+ try {+ await new Promise<void>((resolve, reject) => {+ const stream = createReadStream(filePath, { start, end });+ stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));+ stream.on("error", reject);+ stream.on("end", () => resolve());+ });+ } catch (err) {+ // File deleted between stat() and open (pod-roll cleanup racing a read):+ // treat as missing so the caller falls through to the S3 mirror instead+ // of surfacing the very "Run log not found" this store exists to prevent.+ if ((err as NodeJS.ErrnoException | null)?.code === "ENOENT") return null;+ throw err;}+ const content = Buffer.concat(chunks).toString("utf8");+ const nextOffset = end + 1 < stat.size ? end + 1 : undefined;+ return { content, nextOffset };+ }++ async function readS3Range(+ logRef: string,+ offset: number,+ limitBytes: number,+ ): Promise<RunLogReadResult> {+ if (!s3) throw notFound("Run log not found");+ const key = s3Key(logRef);+ const head = await s3.provider.headObject({ objectKey: key });+ if (!head.exists) throw notFound("Run log not found");+ const total = head.contentLength ?? 0;+ const start = Math.max(0, Math.min(offset, total));+ const end = Math.max(start, Math.min(start + limitBytes - 1, total - 1));+ if (start > end || total === 0) return { content: "", nextOffset: start < total ? start : undefined };+ const result = await s3.provider.getObject({ objectKey: key, range: { start, end } });const chunks: Buffer[] = [];await new Promise<void>((resolve, reject) => {- const stream = createReadStream(filePath, { start, end });- stream.on("data", (chunk) => {- chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));- });- stream.on("error", reject);- stream.on("end", () => resolve());+ result.stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));+ result.stream.on("error", reject);+ result.stream.on("end", () => resolve());});-const content = Buffer.concat(chunks).toString("utf8");- const nextOffset = end + 1 < stat.size ? end + 1 : undefined;+ const nextOffset = end + 1 < total ? end + 1 : undefined;return { content, nextOffset };}@@ -99,10 +163,8 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore {const relDir = path.join(companyId, agentId);const relPath = path.join(relDir, `${runId}.ndjson`);await ensureDir(relDir);-const absPath = resolveWithin(basePath, relPath);await fs.writeFile(absPath, "", "utf8");-return { store: "local_file", logRef: relPath };},@@ -124,38 +186,79 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore {},async finalize(handle) {- if (handle.store !== "local_file") {- return { bytes: 0, compressed: false };- }+ if (handle.store !== "local_file") return { bytes: 0, compressed: false };const absPath = resolveWithin(basePath, handle.logRef);const stat = await fs.stat(absPath).catch(() => null);if (!stat) throw notFound("Run log not found");-const hash = await sha256File(absPath);- return {- bytes: stat.size,- sha256: hash,- compressed: false,- };++ // Mirror the completed log to object storage so it survives a pod roll.+ // Best-effort upload failures must NOT fail run finalization (which also+ // records cost/usage); the local copy still serves reads until the pod+ // rolls, and a failed mirror only loses durability for that one run.+ if (s3) {+ try {+ // Stream from disk instead of buffering the whole .ndjson in the+ // heap; long agent sessions can produce large logs. The file is+ // complete at this point, so stat.size is the exact content length.+ await s3.provider.putObject({+ objectKey: s3Key(handle.logRef),+ body: createReadStream(absPath),+ contentType: "application/x-ndjson",+ contentLength: stat.size,+ });+ } catch (err) {+ // Best-effort: finalization must not break, but a persistently+ // failing mirror (bad creds/bucket/endpoint) should be visible to+ // operators before a pod roll makes the logs unreadable.+ console.warn(+ `[run-log-store] Failed to mirror run log to object storage (key: ${s3Key(handle.logRef)}):`,+ err,+ );+ }+ }++ return { bytes: stat.size, sha256: hash, compressed: false };},async read(handle, opts) {- if (handle.store !== "local_file") {- throw notFound("Run log not found");- }+ if (handle.store !== "local_file") throw notFound("Run log not found");const absPath = resolveWithin(basePath, handle.logRef);const offset = opts?.offset ?? 0;const limitBytes = opts?.limitBytes ?? 256_000;- return readFileRange(absPath, offset, limitBytes);+ const local = await readLocalRange(absPath, offset, limitBytes);+ if (local) return local;+ // Local file gone (pod rolled) -> serve from the S3 mirror if configured.+ return readS3Range(handle.logRef, offset, limitBytes);},};}+// Build the run-log S3 mirror from dedicated RUN_LOG_S3_* env. Deliberately+// separate from PAPERCLIP_STORAGE_PROVIDER so enabling durable run logs does+// NOT redirect the product's workspace/file storage (smaller blast radius).+// Unset RUN_LOG_S3_BUCKET -> no mirror -> local-only (safe degrade). Creds come+// from the standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY chain.+function resolveRunLogS3(): { provider: StorageProvider; keyPrefix?: string } | undefined {+ const bucket = process.env.RUN_LOG_S3_BUCKET?.trim();+ if (!bucket) return undefined;+ const provider = createS3StorageProvider({+ bucket,+ region: process.env.RUN_LOG_S3_REGION?.trim() || "us-east-1",+ endpoint: process.env.RUN_LOG_S3_ENDPOINT?.trim() || undefined,+ prefix: undefined, // prefixing is handled by keyPrefix below (kept off the provider)+ forcePathStyle: process.env.RUN_LOG_S3_FORCE_PATH_STYLE+ ? process.env.RUN_LOG_S3_FORCE_PATH_STYLE === "true"+ : true, // Cubbit (and most S3-compatible endpoints) need path-style+ });+ return { provider, keyPrefix: process.env.RUN_LOG_S3_PREFIX?.trim() || "run-logs" };+}+let cachedStore: RunLogStore | null = null;export function getRunLogStore() {if (cachedStore) return cachedStore;const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs");- cachedStore = createLocalFileRunLogStore(basePath);+ cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() });return cachedStore;}diff --git a/server/src/storage/types.ts b/server/src/storage/types.tsindex efeadd8ba8..dab23fa4c0 100644--- a/server/src/storage/types.ts+++ b/server/src/storage/types.ts@@ -3,7 +3,9 @@ import type { Readable } from "node:stream";export interface PutObjectInput {objectKey: string;- body: Buffer;+ // Readable bodies stream straight to the backend (contentLength must be the+ // exact byte size); Buffer stays supported for small payloads.+ body: Buffer | Readable;contentType: string;contentLength: number;}