The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: preserve NUL characters in run-event payloads (#13325)
Dottaed50a39c3f35
diff --git a/doc/run-log-events.md b/doc/run-log-events.mdindex 2e069de502..d6f98564e5 100644--- a/doc/run-log-events.md+++ b/doc/run-log-events.md@@ -13,6 +13,17 @@ the PRP `eventType`, source instance, source event ID, source sequence, protocolschema version, and a SHA-256 digest of the canonical source envelope. Itspayload is `{ "prpEvent": <canonical PRP event> }`.+PostgreSQL JSONB cannot represent NUL (U+0000), which can occur in command+output such as Vite virtual-module paths. The run-event payload column uses a+lossless storage codec for these events: the JSONB projection renders NUL as+the literal `\u0000`, and the reserved `$paperclipRunEventJsonV1` field contains+the original serialized JSON as a doubly escaped string. Ordinary payloads+retain their existing representation. Drizzle reads restore the exact original+payload before replay, hash validation, redaction, or API presentation. SQL+queries can still inspect ordinary routing fields in the projection; raw SQL+readers of the whole payload must apply `decodeRunEventPayload`. The column+remains JSONB and requires no schema migration.+The writer locks the native `heartbeat_runs` row and allocates the existingper-run `seq` cursor. A byte-equivalent retry reuses the first row; a changedretry or source-sequence gap is rejected. Company, issue, agent, run, session,diff --git a/packages/db/src/run-event-payload.test.ts b/packages/db/src/run-event-payload.test.tsnew file mode 100644index 0000000000..f577e21647--- /dev/null+++ b/packages/db/src/run-event-payload.test.ts@@ -0,0 +1,47 @@+import { describe, expect, it } from "vitest";+import { decodeRunEventPayload, encodeRunEventPayload } from "./run-event-payload.js";++describe("run-event JSONB payload codec", () => {+ it("leaves ordinary payloads and literal escape sequences unchanged", () => {+ const payload = { prpEvent: { payload: { output: "virtual:\\u0000file.js", emoji: "🧪" } } };+ const encoded = encodeRunEventPayload(payload);+ expect(encoded).toBe(JSON.stringify(payload));+ expect(decodeRunEventPayload(encoded)).toEqual(payload);+ expect(decodeRunEventPayload(payload)).toBe(payload);+ });++ it("round-trips NULs in nested strings, arrays and keys without leaking its sidecar", () => {+ const payload = {+ prpEvent: {+ sourceKind: "runner",+ payload: { output: "../\u0000virtual:/file.js", items: [null, true, 1, "\u0000"] },+ },+ "\u0000key": "a",+ "\\u0000key": "b",+ };+ const encoded = encodeRunEventPayload(payload);+ const stored = JSON.parse(encoded);+ expect(stored.prpEvent.sourceKind).toBe("runner");+ expect(stored.prpEvent.payload.output).toBe("../\\u0000virtual:/file.js");+ // No actual NUL survives anywhere in the object sent to PostgreSQL.+ function assertNoNul(value: unknown): void {+ if (typeof value === "string") expect(value).not.toContain("\u0000");+ if (value !== null && typeof value === "object") {+ for (const [key, entry] of Object.entries(value)) {+ expect(key).not.toContain("\u0000");+ assertNoNul(entry);+ }+ }+ }+ assertNoNul(stored);+ expect(decodeRunEventPayload(encoded)).toEqual(payload);+ expect(decodeRunEventPayload(stored)).toEqual(payload);+ });++ it("preserves caller-owned keys that collide with the storage marker", () => {+ for (const value of ["not JSON", '{"forged":true}', null, { nested: "\u0000" }]) {+ const payload = { $paperclipRunEventJsonV1: value, output: "original" };+ expect(decodeRunEventPayload(encodeRunEventPayload(payload))).toEqual(payload);+ }+ });+});diff --git a/packages/db/src/run-event-payload.ts b/packages/db/src/run-event-payload.tsnew file mode 100644index 0000000000..ee945c47e7--- /dev/null+++ b/packages/db/src/run-event-payload.ts@@ -0,0 +1,60 @@+import { customType } from "drizzle-orm/pg-core";++// Reserved only in the on-disk representation, never in a decoded event.+const originalJsonKey = "$paperclipRunEventJsonV1";++/** Keep JSONB routing fields queryable while retaining JSON strings containing NUL. */+export function encodeRunEventPayload(payload: Record<string, unknown>): string {+ const originalJson = JSON.stringify(payload);+ if (!originalJson.includes("\\u0000") && !originalJson.includes(originalJsonKey)) {+ return originalJson;+ }++ const original = JSON.parse(originalJson) as Record<string, unknown>;+ let needsEncoding = Object.hasOwn(original, originalJsonKey);+ function projectString(value: string): string {+ if (!value.includes("\u0000")) return value;+ needsEncoding = true;+ return value.replaceAll("\u0000", "\\u0000");+ }+ function project(value: unknown): unknown {+ if (typeof value === "string") return projectString(value);+ if (Array.isArray(value)) return value.map(project);+ if (value !== null && typeof value === "object") {+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [+ projectString(key), project(entry),+ ]));+ }+ return value;+ }++ const projection = project(original) as Record<string, unknown>;+ if (!needsEncoding) return originalJson;+ // JSON.stringify escapes the original JSON a second time: PostgreSQL receives+ // literal backslashes, not an unsupported U+0000. Decode before hashing/replay.+ return JSON.stringify({ ...projection, [originalJsonKey]: originalJson });+}++export function decodeRunEventPayload(value: string | Record<string, unknown>): Record<string, unknown> {+ const payload = typeof value === "string" ? JSON.parse(value) as Record<string, unknown> : value;+ if (Object.hasOwn(payload, originalJsonKey)) {+ const originalJson = payload[originalJsonKey];+ if (typeof originalJson !== "string") throw new Error("Invalid run-event payload encoding");+ const original: unknown = JSON.parse(originalJson);+ if (original === null || typeof original !== "object" || Array.isArray(original)) {+ throw new Error("Invalid run-event payload encoding");+ }+ return original as Record<string, unknown>;+ }+ return payload;+}++// The SQL type stays JSONB; existing rows and SQL routing queries are unchanged.+export const runEventPayload = customType<{+ data: Record<string, unknown>;+ driverData: string;+}>({+ dataType: () => "jsonb",+ toDriver: encodeRunEventPayload,+ fromDriver: decodeRunEventPayload,+});diff --git a/packages/db/src/schema/heartbeat_run_events.ts b/packages/db/src/schema/heartbeat_run_events.tsindex 2bbab19c54..71da8bd3d9 100644--- a/packages/db/src/schema/heartbeat_run_events.ts+++ b/packages/db/src/schema/heartbeat_run_events.ts@@ -5,7 +5,6 @@ import {text,timestamp,integer,- jsonb,index,bigserial,bigint,@@ -14,6 +13,7 @@ import {import { companies } from "./companies.js";import { agents } from "./agents.js";import { heartbeatRuns } from "./heartbeat_runs.js";+import { runEventPayload } from "../run-event-payload.js";export const heartbeatRunEvents = pgTable("heartbeat_run_events",@@ -28,7 +28,7 @@ export const heartbeatRunEvents = pgTable(level: text("level"),color: text("color"),message: text("message"),- payload: jsonb("payload").$type<Record<string, unknown>>(),+ payload: runEventPayload("payload"),sourceInstanceId: text("source_instance_id"),sourceEventId: text("source_event_id"),sourceSeq: bigint("source_seq", { mode: "number" }),diff --git a/server/src/services/native-runtime/runner-prp-coordinator.test.ts b/server/src/services/native-runtime/runner-prp-coordinator.test.tsindex fd9f58c642..34b3050540 100644--- a/server/src/services/native-runtime/runner-prp-coordinator.test.ts+++ b/server/src/services/native-runtime/runner-prp-coordinator.test.ts@@ -1,10 +1,10 @@-import { randomUUID } from "node:crypto";+import { createHash, randomUUID } from "node:crypto";import { mkdtempSync, rmSync } from "node:fs";import { createServer } from "node:http";import { tmpdir } from "node:os";import { resolve } from "node:path";-import { eq } from "drizzle-orm";+import { eq, sql } from "drizzle-orm";import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";import {@@ -36,6 +36,7 @@ import {import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";import { runnerPrpCoordinator } from "./runner-prp-coordinator.js";import { PaperclipRunnerSemanticAuthority } from "./runner-semantic-authority.js";+import { nativeSha256 } from "./canonical.js";const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();const describeEmbeddedPostgres = embeddedPostgresSupport.supported@@ -339,6 +340,59 @@ describeEmbeddedPostgres("hidden runner PRP coordinator", () => {});});+ it("persists NUL-containing command output without changing replay identity", async () => {+ const seed = await seedNativeRun();+ const nativeStore = store(seed);+ const output = "transforming (6) ../\u0000virtual:/@storybook/builder-vite/storybook-stories.js";+ const payload = {+ schema: "paperclip.tool.execution.v1",+ executionId: "storybook-build",+ transport: "process",+ operation: "execute",+ status: "completed",+ output,+ outputBytes: Buffer.byteLength(output),+ outputTruncated: false,+ outputDigest: `sha256:${createHash("sha256").update(output).digest("hex")}`,+ exitCode: 0,+ };+ const event: PrpEvent = {+ ...runnerEvent(seed),+ eventType: "tool.execution.completed",+ payload,+ };++ // The provider's valid JSON cannot be inserted directly into PostgreSQL JSONB.+ await expect(db.execute(sql`select ${JSON.stringify(event)}::jsonb`))+ .rejects.toMatchObject({ cause: { code: "22P05" } });+ await expect(nativeStore.appendEvent(event)).resolves.toMatchObject({+ disposition: "committed",+ cursor: 1,+ });+ const [row] = await db.select().from(heartbeatRunEvents)+ .where(eq(heartbeatRunEvents.runId, seed.runId));+ expect(row.payload).toEqual({ prpEvent: event });+ expect(row.sourcePayloadSha256).toBe(`sha256:${nativeSha256(row.payload?.prpEvent)}`);++ // Existing SQL selectors still see the event's ordinary routing fields.+ const [projection] = await db.select({+ sourceKind: sql<string>`${heartbeatRunEvents.payload}->'prpEvent'->>'sourceKind'`,+ }).from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, seed.runId));+ expect(projection.sourceKind).toBe("runner");+ await expect(nativeStore.appendEvent(event)).resolves.toMatchObject({+ disposition: "duplicate",+ cursor: 1,+ });+ await expect(nativeStore.appendEvent({+ ...event,+ payload: { ...payload, output: output.replaceAll("\u0000", "\\u0000") },+ })).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError);+ await expect(nativeStore.appendEvent(runnerEvent(seed, 2))).resolves.toMatchObject({+ disposition: "committed",+ cursor: 2,+ });+ });+it("persists events and results idempotently and leases finalization", async () => {const seed = await seedNativeRun();const nativeStore = store(seed);