The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
adapter-claude-local: recover from poisoned previous_message_id 400 (detect + clearSession) (#5972)
Danial Jawaid8ee3987d12d5
diff --git a/docs/adapters/claude-local.md b/docs/adapters/claude-local.mdindex fc64fcf8d8..1cc39e219c 100644--- a/docs/adapters/claude-local.md+++ b/docs/adapters/claude-local.md@@ -43,6 +43,28 @@ Session resume is cwd-aware: if the agent's working directory changed since theIf resume fails with an unknown session error, the adapter automatically retries with a fresh session.+### Poisoned `previous_message_id` (recovery)++Symptom in logs / issue thread:++```+API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)+```++What it means: the on-disk Claude Code transcript JSONL for that session contains a malformed (non-`msg_`-prefixed) `previous_message_id`. Anthropic's `/v1/messages` rejects every resume attempt against that transcript with a deterministic 400. Without guards, Paperclip would re-persist the same poisoned session id and the issue is stranded permanently — see [RED-976](../../../) / [RED-978](../../../).++What the adapter does automatically:++1. **Auto-rotate on resume.** If a `--resume` attempt returns this 400, the adapter retries once with a fresh session, deletes the poisoned `<session>.jsonl` from the local Claude config dir (best effort), and uses the fresh session id going forward.+2. **Validate-before-persist.** A result that carries this 400 never gets its `session_id` written back to the task session store, even if Claude Code emits one in the result event. The adapter returns `sessionId: null`, `sessionParams: null`, and `errorCode: "claude_poisoned_previous_message_id"`.+3. **Clear-on-error.** The adapter sets `clearSession: true` on the result, which causes the heartbeat service to drop any persisted session row for that issue (`clearTaskSessions`). The next continuation starts from a clean slate.++On-call checklist if you see this in production:++- Confirm `errorCode` is `claude_poisoned_previous_message_id` in the run row — that means the guards fired correctly and the issue auto-recovers on the next heartbeat.+- If the same issue still loops after one heartbeat, check that `agentTaskSessions` for that `(agentId, taskKey)` was cleared. If not, the adapter return value was lost (e.g. a malformed run finalization) — escalate; do **not** manually edit the row, file a child issue with the run id.+- For remote execution targets (sandbox/SSH), the poisoned JSONL is on the remote and the adapter only logs the cleanup intent. The fresh-session retry still succeeds because it uses a new session id, and the server-side `clearSession: true` is authoritative regardless of remote disk state.+## Skills InjectionThe adapter creates a temporary directory with symlinks to Paperclip skills and passes it via `--add-dir`. This makes skills discoverable without polluting the agent's working directory.diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.tsindex 067f68cdbc..99fdd99582 100644--- a/packages/adapters/claude-local/src/server/execute.ts+++ b/packages/adapters/claude-local/src/server/execute.ts@@ -54,8 +54,9 @@ import {isClaudeMaxTurnsResult,isClaudeTransientUpstreamError,isClaudeUnknownSessionError,+ isClaudePoisonedPreviousMessageIdError,} from "./parse.js";-import { prepareClaudeConfigSeed } from "./claude-config.js";+import { prepareClaudeConfigSeed, resolveSharedClaudeConfigDir } from "./claude-config.js";import { resolveClaudeDesiredSkillNames } from "./skills.js";import { isBedrockModelId } from "./models.js";import { prepareClaudePromptBundle } from "./prompt-cache.js";@@ -860,9 +861,22 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec};})();- const resolvedSessionId =+ const rawResolvedSessionId =parsedStream.sessionId ??(asString(parsed.session_id, opts.fallbackSessionId ?? "") || opts.fallbackSessionId);+ const clearSessionForMaxTurns = isClaudeMaxTurnsResult(parsed);+ const poisonedPreviousMessageId = isClaudePoisonedPreviousMessageIdError(parsed);+ const parsedIsError = asBoolean(parsed.is_error, false);+ const failed = (proc.exitCode ?? 0) !== 0 || parsedIsError;+ // Validate-before-persist guard: never persist a sessionId whose transcript+ // is known-poisoned. The Claude CLI keeps an on-disk JSONL keyed by the+ // session id; if the last entry contains a non-`msg_`-prefixed+ // `previous_message_id`, every subsequent `--resume` hits a 400 from+ // /v1/messages and the issue is permanently unrecoverable until the+ // sessionId is dropped server-side. Drop here so resolveNextSessionState+ // calls clearTaskSessions on the next heartbeat. See RED-978 / RED-976.+ const shouldDropSessionForPoison = poisonedPreviousMessageId;+ const resolvedSessionId = shouldDropSessionForPoison ? null : rawResolvedSessionId;const resolvedSessionParams = resolvedSessionId? ({sessionId: resolvedSessionId,@@ -878,9 +892,6 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec...(workspaceRepoRef ? { repoRef: workspaceRepoRef } : {}),} as Record<string, unknown>): null;- const clearSessionForMaxTurns = isClaudeMaxTurnsResult(parsed);- const parsedIsError = asBoolean(parsed.is_error, false);- const failed = (proc.exitCode ?? 0) !== 0 || parsedIsError;const errorMessage = failed? describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`: null;@@ -888,6 +899,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecfailed &&!loginMeta.requiresLogin &&!clearSessionForMaxTurns &&+ !poisonedPreviousMessageId &&isClaudeTransientUpstreamError({parsed,stdout: proc.stdout,@@ -906,12 +918,15 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec? "claude_auth_required": failed && clearSessionForMaxTurns? "max_turns_exhausted"+ : failed && poisonedPreviousMessageId+ ? "claude_poisoned_previous_message_id": transientUpstream? "claude_transient_upstream": null;const mergedResultJson: Record<string, unknown> = {...parsed,...(failed && clearSessionForMaxTurns ? { stopReason: "max_turns_exhausted" } : {}),+ ...(failed && poisonedPreviousMessageId ? { stopReason: "claude_poisoned_previous_message_id" } : {}),...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),@@ -937,23 +952,59 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExeccostUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),resultJson: mergedResultJson,summary: parsedStream.summary || asString(parsed.result, ""),- clearSession: clearSessionForMaxTurns || Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),+ clearSession:+ clearSessionForMaxTurns ||+ // Clear-on-error: a poisoned previous_message_id is a deterministic+ // state error. Force the server to drop persisted session state for+ // this issue so the next continuation starts from a clean slate.+ poisonedPreviousMessageId ||+ Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),};};try {const initial = await runAttempt(sessionId ?? null);- if (+ const sessionErrorKind =sessionId &&!initial.proc.timedOut &&(initial.proc.exitCode ?? 0) !== 0 &&- initial.parsed &&- isClaudeUnknownSessionError(initial.parsed)- ) {+ initial.parsed+ ? isClaudeUnknownSessionError(initial.parsed)+ ? "unknown"+ : isClaudePoisonedPreviousMessageIdError(initial.parsed)+ ? "poisoned"+ : null+ : null;++ if (sessionErrorKind !== null) {+ const reason =+ sessionErrorKind === "poisoned"+ ? "returned a poisoned message-id"+ : "is unavailable";await onLog("stdout",- `[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,+ `[paperclip] Claude resume session "${sessionId}" ${reason}; retrying with a fresh session.\n`,);+ if (sessionErrorKind === "poisoned" && !executionTargetIsRemote) {+ const claudeConfigDir = resolveSharedClaudeConfigDir(effectiveEnv);+ // Mirrors Claude Code's project-dir encoding: non-alphanumeric chars become "-"; existing hyphens pass through.+ const encodedCwd = effectiveExecutionCwd.replace(/[^a-zA-Z0-9-]/g, "-");+ const poisonedJsonlPath = path.join(claudeConfigDir, "projects", encodedCwd, `${sessionId}.jsonl`);+ let unlinked = false;+ try {+ await fs.unlink(poisonedJsonlPath);+ unlinked = true;+ } catch {+ // best-effort; session is cleared server-side regardless+ }+ if (unlinked) {+ try {+ await onLog("stdout", `[paperclip] Removed poisoned session file: ${poisonedJsonlPath}\n`);+ } catch {+ // log stream may be closed; the unlink already succeeded+ }+ }+ }const retry = await runAttempt(null);return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });}diff --git a/packages/adapters/claude-local/src/server/parse.test.ts b/packages/adapters/claude-local/src/server/parse.test.tsindex 3f27e40ec3..2c88106f8f 100644--- a/packages/adapters/claude-local/src/server/parse.test.ts+++ b/packages/adapters/claude-local/src/server/parse.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";import {extractClaudeRetryNotBefore,isClaudeTransientUpstreamError,+ isClaudePoisonedPreviousMessageIdError,} from "./parse.js";describe("isClaudeTransientUpstreamError", () => {@@ -94,6 +95,53 @@ describe("isClaudeTransientUpstreamError", () => {}),).toBe(false);});++ it("does not classify poisoned previous_message_id errors as transient", () => {+ expect(+ isClaudeTransientUpstreamError({+ parsed: {+ subtype: "success",+ is_error: true,+ result: "API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)",+ },+ }),+ ).toBe(false);+ });+});++describe("isClaudePoisonedPreviousMessageIdError", () => {+ it("detects the previous_message_id 400 error in the result field", () => {+ expect(+ isClaudePoisonedPreviousMessageIdError({+ subtype: "success",+ is_error: true,+ result: "API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)",+ }),+ ).toBe(true);+ });++ it("detects the error in the errors array", () => {+ expect(+ isClaudePoisonedPreviousMessageIdError({+ is_error: true,+ result: "",+ errors: [{ message: "400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)" }],+ }),+ ).toBe(true);+ });++ it("returns false for unrelated errors", () => {+ expect(+ isClaudePoisonedPreviousMessageIdError({+ is_error: true,+ result: "No conversation found with session id abc-123",+ }),+ ).toBe(false);+ });++ it("returns false for empty parsed result", () => {+ expect(isClaudePoisonedPreviousMessageIdError({})).toBe(false);+ });});describe("extractClaudeRetryNotBefore", () => {diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.tsindex f645c4f27a..9664f66bc2 100644--- a/packages/adapters/claude-local/src/server/parse.ts+++ b/packages/adapters/claude-local/src/server/parse.ts@@ -196,6 +196,17 @@ export function isClaudeUnknownSessionError(parsed: Record<string, unknown>): bo);}+export function isClaudePoisonedPreviousMessageIdError(parsed: Record<string, unknown>): boolean {+ const resultText = asString(parsed.result, "").trim();+ const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)]+ .map((msg) => msg.trim())+ .filter(Boolean);++ return allMessages.some((msg) =>+ /diagnostics\.previous_message_id.*starts with `msg_`/i.test(msg),+ );+}+function buildClaudeTransientHaystack(input: {parsed?: Record<string, unknown> | null;stdout?: string | null;@@ -375,7 +386,7 @@ export function isClaudeTransientUpstreamError(input: {}): boolean {const parsed = input.parsed ?? null;// Deterministic failures are handled by their own classifiers.- if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed))) {+ if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed) || isClaudePoisonedPreviousMessageIdError(parsed))) {return false;}const loginMeta = detectClaudeLoginRequired({diff --git a/packages/db/scripts/clean-poisoned-claude-sessions.test.ts b/packages/db/scripts/clean-poisoned-claude-sessions.test.tsnew file mode 100644index 0000000000..5fcdc20d20--- /dev/null+++ b/packages/db/scripts/clean-poisoned-claude-sessions.test.ts@@ -0,0 +1,166 @@+import path from "node:path";+import { describe, expect, it } from "vitest";+import {+ classifyJsonlText,+ classifyRow,+ encodeClaudeCwd,+ jsonlPathFor,+ resolveClaudeConfigDir,+} from "./clean-poisoned-claude-sessions.js";++describe("classifyJsonlText", () => {+ it("returns healthy when the last assistant message.id starts with msg_", () => {+ const text = [+ JSON.stringify({ type: "user" }),+ JSON.stringify({ type: "assistant", message: { id: "msg_first" } }),+ JSON.stringify({ type: "user" }),+ JSON.stringify({ type: "assistant", message: { id: "msg_last" } }),+ ].join("\n");+ const result = classifyJsonlText(text);+ expect(result).toEqual({ kind: "healthy", lastAssistantId: "msg_last" });+ });++ it("returns poisoned when the last assistant message.id is a synthetic UUID", () => {+ const text = [+ JSON.stringify({ type: "user" }),+ JSON.stringify({ type: "assistant", message: { id: "msg_real" } }),+ JSON.stringify({+ type: "assistant",+ message: { id: "f649efd6-1837-4c52-981f-b81fdc0c2ddb" },+ }),+ ].join("\n");+ const result = classifyJsonlText(text);+ expect(result).toEqual({+ kind: "poisoned",+ lastAssistantId: "f649efd6-1837-4c52-981f-b81fdc0c2ddb",+ });+ });++ it("ignores trailing whitespace, blank lines, and malformed JSON", () => {+ const text = [+ "",+ " ",+ "this-is-not-json",+ JSON.stringify({ type: "assistant", message: { id: "msg_only" } }),+ "",+ ].join("\n");+ expect(classifyJsonlText(text)).toEqual({+ kind: "healthy",+ lastAssistantId: "msg_only",+ });+ });++ it("returns no_assistant_entries when the transcript has no assistant rows", () => {+ const text = [+ JSON.stringify({ type: "user" }),+ JSON.stringify({ type: "attachment" }),+ JSON.stringify({ type: "last-prompt", lastPrompt: "x" }),+ ].join("\n");+ expect(classifyJsonlText(text)).toEqual({ kind: "no_assistant_entries" });+ });++ it("treats a CRLF-encoded transcript the same as LF", () => {+ const text = [+ JSON.stringify({ type: "assistant", message: { id: "msg_first" } }),+ JSON.stringify({ type: "assistant", message: { id: "synthetic-uuid" } }),+ ].join("\r\n");+ expect(classifyJsonlText(text)).toEqual({+ kind: "poisoned",+ lastAssistantId: "synthetic-uuid",+ });+ });+});++describe("encodeClaudeCwd", () => {+ it("mirrors the claude-local adapter encoding rule", () => {+ expect(encodeClaudeCwd("/Users/dj/.paperclip/instances/default/workspaces/abc")).toBe(+ "-Users-dj--paperclip-instances-default-workspaces-abc",+ );+ });+});++describe("jsonlPathFor", () => {+ it("returns <root>/projects/<encoded-cwd>/<sessionId>.jsonl", () => {+ const result = jsonlPathFor({+ claudeConfigDir: "/tmp/.claude",+ cwd: "/Users/dj/work",+ sessionId: "abc-123",+ });+ expect(result).toBe(path.join("/tmp/.claude", "projects", "-Users-dj-work", "abc-123.jsonl"));+ });+});++describe("resolveClaudeConfigDir", () => {+ it("prefers the explicit override", () => {+ expect(resolveClaudeConfigDir({}, "/tmp/custom")).toBe(path.resolve("/tmp/custom"));+ });+ it("uses CLAUDE_CONFIG_DIR when no override is given", () => {+ expect(resolveClaudeConfigDir({ CLAUDE_CONFIG_DIR: "/tmp/from-env" } as NodeJS.ProcessEnv)).toBe(+ path.resolve("/tmp/from-env"),+ );+ });+ it("falls back to ~/.claude when no override and no env var", () => {+ const result = resolveClaudeConfigDir({} as NodeJS.ProcessEnv);+ expect(result.endsWith(`${path.sep}.claude`)).toBe(true);+ });+});++describe("classifyRow", () => {+ const claudeConfigDir = "/tmp/.claude";+ const cwd = "/Users/dj/work";+ const sessionId = "session-1";+ const expectedJsonl = jsonlPathFor({ claudeConfigDir, cwd, sessionId });++ it("returns skipped_no_session_id when sessionId is null", () => {+ const result = classifyRow(+ { sessionId: null, cwd, isRemote: false },+ { claudeConfigDir, readJsonl: () => null },+ );+ expect(result.kind).toBe("skipped_no_session_id");+ });++ it("returns skipped_remote when the row has remoteExecution params", () => {+ const result = classifyRow(+ { sessionId, cwd, isRemote: true },+ { claudeConfigDir, readJsonl: () => null },+ );+ expect(result.kind).toBe("skipped_remote");+ });++ it("returns missing_jsonl with the expected path when the file does not exist", () => {+ const result = classifyRow(+ { sessionId, cwd, isRemote: false },+ { claudeConfigDir, readJsonl: () => null },+ );+ expect(result.kind).toBe("missing_jsonl");+ if (result.kind === "missing_jsonl") {+ expect(result.jsonlPath).toBe(expectedJsonl);+ }+ });++ it("flags rows whose JSONL last assistant id is non-msg_", () => {+ const jsonl = [+ JSON.stringify({ type: "assistant", message: { id: "msg_first" } }),+ JSON.stringify({ type: "assistant", message: { id: "synthetic-uuid" } }),+ ].join("\n");+ const result = classifyRow(+ { sessionId, cwd, isRemote: false },+ { claudeConfigDir, readJsonl: () => jsonl },+ );+ expect(result.kind).toBe("poisoned");+ if (result.kind === "poisoned") {+ expect(result.lastAssistantId).toBe("synthetic-uuid");+ }+ });++ it("leaves valid msg_-prefixed sessions classified as healthy", () => {+ const jsonl = [+ JSON.stringify({ type: "assistant", message: { id: "msg_only" } }),+ ].join("\n");+ const result = classifyRow(+ { sessionId, cwd, isRemote: false },+ { claudeConfigDir, readJsonl: () => jsonl },+ );+ expect(result.kind).toBe("healthy");+ });+});diff --git a/packages/db/scripts/clean-poisoned-claude-sessions.ts b/packages/db/scripts/clean-poisoned-claude-sessions.tsnew file mode 100644index 0000000000..2a26a9309e--- /dev/null+++ b/packages/db/scripts/clean-poisoned-claude-sessions.ts@@ -0,0 +1,462 @@+#!/usr/bin/env tsx+/**+ * One-shot cleaner for `agent_task_sessions` rows whose persisted `claude_local`+ * session is poisoned by a non-`msg_*` `previous_message_id`.+ *+ * The poison shape: the Claude CLI keeps an on-disk JSONL transcript keyed by+ * sessionId at `<claude-config-dir>/projects/<encoded-cwd>/<sessionId>.jsonl`.+ * If the last `assistant` entry has a `message.id` that does NOT start with+ * `msg_`, every subsequent `--resume` against /v1/messages 400s with+ * `diagnostics.previous_message_id` and the issue is permanently stranded+ * until the persisted session is dropped. See RED-877 / RED-978 / RED-991 for+ * the upstream adapter fix; this script retroactively heals rows that were+ * stranded BEFORE that adapter shipped.+ *+ * Usage:+ * tsx packages/db/scripts/clean-poisoned-claude-sessions.ts \+ * --config /path/to/paperclip/config.json \+ * [--claude-config-dir ~/.claude] \+ * [--dry-run] [--json]+ *+ * Or in a Paperclip checkout shell:+ * pnpm --filter @paperclipai/db exec tsx scripts/clean-poisoned-claude-sessions.ts --dry-run+ *+ * Exits 0 on success even when nothing was healed. Idempotent.+ */+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import { and, eq, isNotNull, sql } from "drizzle-orm";+import { createDb } from "../src/client.js";+import { agentTaskSessions } from "../src/schema/index.js";++const CLAUDE_ADAPTER_TYPE = "claude_local";++export type SessionClassification =+ | { kind: "healthy"; lastAssistantId: string }+ | { kind: "poisoned"; lastAssistantId: string }+ | { kind: "missing_jsonl"; jsonlPath: string }+ | { kind: "no_assistant_entries" }+ | { kind: "skipped_remote" }+ | { kind: "skipped_no_session_id" }+ | { kind: "unreadable_jsonl"; error: string };++export interface ClaudeSessionRowInput {+ sessionId: string | null;+ cwd: string | null;+ isRemote: boolean;+}++/**+ * Pure classifier — given JSONL text, returns whether the session is poisoned.+ * A session is "poisoned" iff its last `assistant` entry has a `message.id`+ * that does not start with `msg_`. Sessions with zero assistant entries are+ * left alone (they can be resumed safely from scratch).+ */+export function classifyJsonlText(text: string): SessionClassification {+ let lastAssistantId: string | null = null;+ for (const rawLine of text.split(/\r?\n/)) {+ const line = rawLine.trim();+ if (!line) continue;+ let obj: unknown;+ try {+ obj = JSON.parse(line);+ } catch {+ continue;+ }+ if (!obj || typeof obj !== "object") continue;+ const entry = obj as Record<string, unknown>;+ if (entry.type !== "assistant") continue;+ const msg = entry.message;+ if (!msg || typeof msg !== "object") continue;+ const id = (msg as Record<string, unknown>).id;+ if (typeof id === "string" && id.length > 0) {+ lastAssistantId = id;+ }+ }+ if (lastAssistantId === null) {+ return { kind: "no_assistant_entries" };+ }+ if (lastAssistantId.startsWith("msg_")) {+ return { kind: "healthy", lastAssistantId };+ }+ return { kind: "poisoned", lastAssistantId };+}++export function encodeClaudeCwd(cwd: string): string {+ // Mirrors `effectiveExecutionCwd.replace(/[^a-zA-Z0-9-]/g, "-")` from the+ // claude-local adapter (`packages/adapters/claude-local/src/server/execute.ts`).+ return cwd.replace(/[^a-zA-Z0-9-]/g, "-");+}++export function resolveClaudeConfigDir(env: NodeJS.ProcessEnv = process.env, override?: string): string {+ if (override && override.trim().length > 0) return path.resolve(override);+ const fromEnv = env.CLAUDE_CONFIG_DIR;+ if (typeof fromEnv === "string" && fromEnv.trim().length > 0) {+ return path.resolve(fromEnv);+ }+ return path.join(os.homedir(), ".claude");+}++export function jsonlPathFor(input: { claudeConfigDir: string; cwd: string; sessionId: string }): string {+ return path.join(+ input.claudeConfigDir,+ "projects",+ encodeClaudeCwd(input.cwd),+ `${input.sessionId}.jsonl`,+ );+}++export function classifyRow(+ row: ClaudeSessionRowInput,+ opts: { claudeConfigDir: string; readJsonl: (filePath: string) => string | null },+): SessionClassification & { jsonlPath?: string } {+ if (!row.sessionId) return { kind: "skipped_no_session_id" };+ if (row.isRemote) return { kind: "skipped_remote" };+ if (!row.cwd) return { kind: "skipped_no_session_id" };+ const jsonlPath = jsonlPathFor({+ claudeConfigDir: opts.claudeConfigDir,+ cwd: row.cwd,+ sessionId: row.sessionId,+ });+ let text: string | null;+ try {+ text = opts.readJsonl(jsonlPath);+ } catch (error) {+ return {+ kind: "unreadable_jsonl",+ error: error instanceof Error ? error.message : String(error),+ jsonlPath,+ };+ }+ if (text === null) {+ return { kind: "missing_jsonl", jsonlPath };+ }+ const classification = classifyJsonlText(text);+ return { ...classification, jsonlPath };+}++interface CliArgs {+ configPath: string | null;+ claudeConfigDir: string | null;+ dryRun: boolean;+ json: boolean;+ databaseUrl: string | null;+ deleteMissing: boolean;+}++function parseArgs(argv: string[]): CliArgs {+ const args: CliArgs = {+ configPath: null,+ claudeConfigDir: null,+ dryRun: false,+ json: false,+ databaseUrl: null,+ deleteMissing: false,+ };+ for (let i = 0; i < argv.length; i++) {+ const token = argv[i];+ switch (token) {+ case "--config":+ args.configPath = argv[++i] ?? null;+ break;+ case "--claude-config-dir":+ args.claudeConfigDir = argv[++i] ?? null;+ break;+ case "--database-url":+ args.databaseUrl = argv[++i] ?? null;+ break;+ case "--dry-run":+ args.dryRun = true;+ break;+ case "--json":+ args.json = true;+ break;+ case "--delete-missing-jsonl":+ args.deleteMissing = true;+ break;+ case "--help":+ case "-h":+ process.stdout.write(USAGE);+ process.exit(0);+ break;+ default:+ if (token.startsWith("--")) {+ throw new Error(`Unknown flag: ${token}`);+ }+ }+ }+ return args;+}++const USAGE = `Usage:+ tsx packages/db/scripts/clean-poisoned-claude-sessions.ts [flags]++Flags:+ --config <path> Path to paperclip config.json (defaults to+ $PAPERCLIP_HOME/instances/default/config.json or+ the standard locations).+ --database-url <url> Override DB connection string entirely.+ --claude-config-dir <path> Override Claude CLI config dir (default:+ $CLAUDE_CONFIG_DIR or ~/.claude).+ --dry-run Report poisoned rows but do not delete.+ --delete-missing-jsonl Also delete rows whose sessionId references a+ JSONL that no longer exists on disk.+ --json Emit a single JSON summary on stdout instead of+ human-readable lines.+ -h, --help Print this usage.+`;++function readDatabaseUrlFromConfig(configPath: string): string {+ const raw = fs.readFileSync(configPath, "utf8");+ const parsed = JSON.parse(raw) as {+ database?: {+ mode?: string;+ embeddedPostgresPort?: number;+ connectionString?: string;+ };+ };+ if (parsed.database?.mode === "postgres" && parsed.database.connectionString) {+ return parsed.database.connectionString;+ }+ const port = parsed.database?.embeddedPostgresPort ?? 54329;+ return `postgres://paperclip:[email protected]:${port}/paperclip`;+}++function defaultConfigPath(): string | null {+ const candidates: string[] = [];+ const home = os.homedir();+ if (process.env.PAPERCLIP_HOME) {+ candidates.push(path.join(process.env.PAPERCLIP_HOME, "config.json"));+ }+ candidates.push(path.join(home, ".paperclip", "instances", "default", "config.json"));+ for (const candidate of candidates) {+ if (fs.existsSync(candidate)) return candidate;+ }+ return null;+}++function readJsonlSafe(filePath: string): string | null {+ try {+ return fs.readFileSync(filePath, "utf8");+ } catch (error) {+ if (error && typeof error === "object" && "code" in error && (error as { code: string }).code === "ENOENT") {+ return null;+ }+ throw error;+ }+}++interface Row {+ id: string;+ agentId: string;+ taskKey: string;+ sessionDisplayId: string | null;+ sessionParamsJson: Record<string, unknown> | null;+}++function extractSessionParams(row: Row): ClaudeSessionRowInput {+ const params = row.sessionParamsJson ?? {};+ const sessionId =+ typeof params.sessionId === "string" && params.sessionId.length > 0+ ? (params.sessionId as string)+ : null;+ const cwd =+ typeof params.cwd === "string" && params.cwd.length > 0 ? (params.cwd as string) : null;+ const isRemote =+ "remoteExecution" in params &&+ params.remoteExecution !== null &&+ typeof params.remoteExecution === "object";+ return { sessionId, cwd, isRemote };+}++interface HealedPair {+ rowId: string;+ agentId: string;+ taskKey: string;+ sessionId: string | null;+ lastAssistantId: string | null;+ jsonlPath: string | null;+ reason: "poisoned" | "missing_jsonl";+}++async function main(argv: string[]): Promise<void> {+ const args = parseArgs(argv);+ const configPath = args.configPath ?? defaultConfigPath();+ const databaseUrl =+ args.databaseUrl ??+ process.env.DATABASE_URL ??+ (configPath ? readDatabaseUrlFromConfig(configPath) : null);+ if (!databaseUrl) {+ throw new Error(+ "Unable to resolve database URL. Pass --database-url or --config <paperclip config.json>.",+ );+ }+ const claudeConfigDir = resolveClaudeConfigDir(process.env, args.claudeConfigDir ?? undefined);++ const db = createDb(databaseUrl);+ const closableDb = db as typeof db & {+ $client?: { end?: (options?: { timeout?: number }) => Promise<void> };+ };+ const summary = {+ scanned: 0,+ poisoned: 0,+ missingJsonl: 0,+ noAssistantEntries: 0,+ healthy: 0,+ skippedRemote: 0,+ skippedNoSession: 0,+ unreadable: 0,+ deleted: 0,+ dryRun: args.dryRun,+ healedPairs: [] as HealedPair[],+ skippedRemotePairs: [] as Array<{ rowId: string; agentId: string; taskKey: string }>,+ };++ const log = (line: string) => {+ if (!args.json) process.stdout.write(`${line}\n`);+ };++ try {+ const rows: Row[] = await db+ .select({+ id: agentTaskSessions.id,+ agentId: agentTaskSessions.agentId,+ taskKey: agentTaskSessions.taskKey,+ sessionDisplayId: agentTaskSessions.sessionDisplayId,+ sessionParamsJson: agentTaskSessions.sessionParamsJson,+ })+ .from(agentTaskSessions)+ .where(+ and(+ eq(agentTaskSessions.adapterType, CLAUDE_ADAPTER_TYPE),+ isNotNull(agentTaskSessions.sessionParamsJson),+ ),+ );++ log(+ `Scanning ${rows.length} claude_local rows; claudeConfigDir=${claudeConfigDir}; dryRun=${args.dryRun}`,+ );++ const toDelete: HealedPair[] = [];+ for (const row of rows) {+ summary.scanned += 1;+ const input = extractSessionParams(row);+ const classification = classifyRow(input, {+ claudeConfigDir,+ readJsonl: readJsonlSafe,+ });+ switch (classification.kind) {+ case "healthy":+ summary.healthy += 1;+ break;+ case "no_assistant_entries":+ summary.noAssistantEntries += 1;+ break;+ case "skipped_remote":+ summary.skippedRemote += 1;+ summary.skippedRemotePairs.push({+ rowId: row.id,+ agentId: row.agentId,+ taskKey: row.taskKey,+ });+ break;+ case "skipped_no_session_id":+ summary.skippedNoSession += 1;+ break;+ case "unreadable_jsonl":+ summary.unreadable += 1;+ log(+ `[warn] unreadable jsonl for row=${row.id} agent=${row.agentId} task=${row.taskKey} error=${classification.error}`,+ );+ break;+ case "missing_jsonl": {+ summary.missingJsonl += 1;+ if (args.deleteMissing) {+ toDelete.push({+ rowId: row.id,+ agentId: row.agentId,+ taskKey: row.taskKey,+ sessionId: input.sessionId,+ lastAssistantId: null,+ jsonlPath: classification.jsonlPath ?? null,+ reason: "missing_jsonl",+ });+ } else {+ log(+ `[skip] missing jsonl (use --delete-missing-jsonl to clear) row=${row.id} agent=${row.agentId} task=${row.taskKey} jsonl=${classification.jsonlPath}`,+ );+ }+ break;+ }+ case "poisoned":+ summary.poisoned += 1;+ toDelete.push({+ rowId: row.id,+ agentId: row.agentId,+ taskKey: row.taskKey,+ sessionId: input.sessionId,+ lastAssistantId: classification.lastAssistantId,+ jsonlPath: classification.jsonlPath ?? null,+ reason: "poisoned",+ });+ break;+ }+ }++ if (toDelete.length === 0) {+ log("No poisoned rows found.");+ } else {+ log(`Found ${toDelete.length} rows to clear:`);+ for (const pair of toDelete) {+ log(+ ` - row=${pair.rowId} agent=${pair.agentId} task=${pair.taskKey} sessionId=${pair.sessionId} reason=${pair.reason} lastAssistantId=${pair.lastAssistantId ?? "n/a"}`,+ );+ }+ }++ if (!args.dryRun && toDelete.length > 0) {+ // Single-statement delete with ids list; safer than a per-row loop and+ // avoids races against concurrent heartbeat writes that might re-touch+ // the same (agentId, taskKey).+ const ids = toDelete.map((pair) => pair.rowId);+ const result = await db+ .delete(agentTaskSessions)+ .where(sql`${agentTaskSessions.id} IN (${sql.join(ids.map((id) => sql`${id}`), sql`, `)})`)+ .returning({ id: agentTaskSessions.id });+ summary.deleted = result.length;+ log(`Deleted ${result.length} rows.`);+ }++ summary.healedPairs = toDelete;++ if (args.json) {+ process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);+ } else {+ log("--- Summary ---");+ log(+ `scanned=${summary.scanned} healthy=${summary.healthy} poisoned=${summary.poisoned} missingJsonl=${summary.missingJsonl} noAssistantEntries=${summary.noAssistantEntries} skippedRemote=${summary.skippedRemote} skippedNoSession=${summary.skippedNoSession} unreadable=${summary.unreadable} deleted=${summary.deleted} dryRun=${summary.dryRun}`,+ );+ }+ } finally {+ await closableDb.$client?.end?.({ timeout: 5 }).catch(() => undefined);+ }+}++// Allow `import { ... } from "./clean-poisoned-claude-sessions.js"` from tests+// without triggering the CLI side-effect. Node sets `import.meta.main` on+// direct invocation; fall back to argv[1] comparison for older runtimes.+const invokedDirectly =+ typeof process !== "undefined" &&+ Array.isArray(process.argv) &&+ process.argv[1] !== undefined &&+ path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);++if (invokedDirectly) {+ main(process.argv.slice(2)).catch((error) => {+ process.stderr.write(+ `${error instanceof Error ? error.stack ?? error.message : String(error)}\n`,+ );+ process.exit(1);+ });+}diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.tsindex 8cda4f0f57..525270512b 100644--- a/server/src/__tests__/claude-local-execute.test.ts+++ b/server/src/__tests__/claude-local-execute.test.ts@@ -90,6 +90,72 @@ type CapturePayload = {appendedSystemPromptFileContents?: string | null;};+async function writePoisonedMessageIdClaudeCommand(commandPath: string): Promise<void> {+ const script = `#!/usr/bin/env node+const fs = require("node:fs");++const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH;+const statePath = process.env.PAPERCLIP_TEST_STATE_PATH;+const payload = {+ argv: process.argv.slice(2),+ prompt: fs.readFileSync(0, "utf8"),+};+if (capturePath) {+ const entries = fs.existsSync(capturePath) ? JSON.parse(fs.readFileSync(capturePath, "utf8")) : [];+ entries.push(payload);+ fs.writeFileSync(capturePath, JSON.stringify(entries), "utf8");+}+const resumed = process.argv.includes("--resume");+const shouldFailResume = resumed && statePath && !fs.existsSync(statePath);+if (shouldFailResume) {+ fs.writeFileSync(statePath, "retried", "utf8");+ console.log(JSON.stringify({+ type: "result",+ subtype: "success",+ session_id: "claude-session-poisoned",+ is_error: true,+ result: "API Error: 400 diagnostics.previous_message_id: must be the \`id\` from a prior /v1/messages response (starts with \`msg_\`)",+ }));+ process.exit(1);+}+console.log(JSON.stringify({ type: "system", subtype: "init", session_id: "claude-session-new", model: "claude-sonnet" }));+console.log(JSON.stringify({ type: "assistant", session_id: "claude-session-new", message: { content: [{ type: "text", text: "hello" }] } }));+console.log(JSON.stringify({ type: "result", session_id: "claude-session-new", result: "hello", usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 } }));+`;+ await fs.writeFile(commandPath, script, "utf8");+ await fs.chmod(commandPath, 0o755);+}++async function writeAlwaysPoisonedMessageIdClaudeCommand(commandPath: string): Promise<void> {+ const script = `#!/usr/bin/env node+const fs = require("node:fs");++const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH;+const payload = {+ argv: process.argv.slice(2),+ prompt: fs.readFileSync(0, "utf8"),+};+if (capturePath) {+ const entries = fs.existsSync(capturePath) ? JSON.parse(fs.readFileSync(capturePath, "utf8")) : [];+ entries.push(payload);+ fs.writeFileSync(capturePath, JSON.stringify(entries), "utf8");+}+// Both --resume and fresh attempts emit the poisoned previous_message_id result.+// The fresh attempt still carries a session_id in the result; the adapter must+// NOT persist it, otherwise the next continuation re-resumes a known-bad transcript.+console.log(JSON.stringify({+ type: "result",+ subtype: "success",+ session_id: "claude-session-poisoned-fresh",+ is_error: true,+ result: "API Error: 400 diagnostics.previous_message_id: must be the \`id\` from a prior /v1/messages response (starts with \`msg_\`)",+}));+process.exit(1);+`;+ await fs.writeFile(commandPath, script, "utf8");+ await fs.chmod(commandPath, 0o755);+}+async function writeRetryThenSucceedClaudeCommand(commandPath: string): Promise<void> {const script = `#!/usr/bin/env nodeconst fs = require("node:fs");@@ -1116,4 +1182,136 @@ describe("claude execute", () => {await fs.rm(root, { recursive: true, force: true });}});++ it("auto-rotates session on previous_message_id 400 (synthetic-msg poisoning) and succeeds on retry", async () => {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-msgid-"));+ const { workspace, commandPath, capturePath, statePath, restore } = await setupExecuteEnv(root, {+ commandWriter: writePoisonedMessageIdClaudeCommand,+ });+ const logs: string[] = [];+ try {+ const result = await execute({+ runId: "run-poisoned-msgid",+ agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },+ runtime: { sessionId: "claude-session-poisoned", sessionParams: null, sessionDisplayId: null, taskKey: null },+ config: {+ command: commandPath,+ cwd: workspace,+ env: {+ PAPERCLIP_TEST_CAPTURE_PATH: capturePath,+ PAPERCLIP_TEST_STATE_PATH: statePath,+ },+ promptTemplate: "Do work.",+ },+ context: {},+ authToken: "tok",+ onLog: async (_stream, chunk) => { logs.push(chunk); },+ });++ const captured: Array<{ argv: string[] }> = JSON.parse(await fs.readFile(capturePath, "utf-8"));+ // First attempt resumes, second attempt starts fresh+ expect(captured).toHaveLength(2);+ expect(captured[0]?.argv).toContain("--resume");+ expect(captured[1]?.argv).not.toContain("--resume");+ // Result comes from the fresh retry+ expect(result.sessionId).toBe("claude-session-new");+ expect(result.errorCode).toBeNull();+ // Adapter logged the fallback reason+ expect(logs.some((l) => l.includes("poisoned message-id"))).toBe(true);+ } finally {+ restore();+ await fs.rm(root, { recursive: true, force: true });+ }+ });++ /**+ * Regression for RED-978: the adapter must not persist a sessionId from a+ * run that ended with a poisoned previous_message_id error. Otherwise the+ * next continuation auto-resumes a known-bad transcript and Anthropic+ * /v1/messages returns 400 again, permanently stranding the issue.+ */+ it("drops sessionId and forces clearSession when a fresh run reports a poisoned previous_message_id", async () => {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-fresh-"));+ const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root, {+ commandWriter: writeAlwaysPoisonedMessageIdClaudeCommand,+ });+ try {+ const result = await execute({+ runId: "run-poisoned-fresh",+ agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },+ runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null },+ config: {+ command: commandPath,+ cwd: workspace,+ env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath },+ promptTemplate: "Do work.",+ },+ context: {},+ authToken: "tok",+ onLog: async () => {},+ });++ // The fake CLI emits a session_id in its poisoned result; the adapter+ // must not propagate it. The server uses clearSession=true to wipe+ // any previously-persisted session state for this issue/task.+ expect(result.sessionId).toBeNull();+ expect(result.sessionParams).toBeNull();+ expect(result.sessionDisplayId).toBeNull();+ expect(result.clearSession).toBe(true);+ expect(result.errorCode).toBe("claude_poisoned_previous_message_id");+ expect(result.errorMessage ?? "").toContain("previous_message_id");+ } finally {+ restore();+ await fs.rm(root, { recursive: true, force: true });+ }+ });++ /**+ * Regression for RED-978: if the auto-retry after a poisoned resume *also*+ * fails with a poisoned previous_message_id, the adapter must still emit+ * clearSession=true so the next heartbeat starts from a clean transcript.+ * Before this fix, the retry result's session_id ("claude-session-poisoned-fresh")+ * was persisted and every subsequent continuation hit the same 400 again.+ */+ it("forces clearSession when the recovery retry also reports a poisoned previous_message_id", async () => {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-retry-"));+ const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root, {+ commandWriter: writeAlwaysPoisonedMessageIdClaudeCommand,+ });+ try {+ const result = await execute({+ runId: "run-poisoned-retry",+ agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} },+ runtime: {+ sessionId: "claude-session-already-poisoned",+ sessionParams: null,+ sessionDisplayId: null,+ taskKey: null,+ },+ config: {+ command: commandPath,+ cwd: workspace,+ env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath },+ promptTemplate: "Do work.",+ },+ context: {},+ authToken: "tok",+ onLog: async () => {},+ });++ const captured: Array<{ argv: string[] }> = JSON.parse(await fs.readFile(capturePath, "utf-8"));+ // Resume attempt + fresh recovery attempt, both poisoned.+ expect(captured).toHaveLength(2);+ expect(captured[0]?.argv).toContain("--resume");+ expect(captured[1]?.argv).not.toContain("--resume");+ // Crucially: do NOT persist the retry's reported sessionId.+ expect(result.sessionId).toBeNull();+ expect(result.sessionParams).toBeNull();+ expect(result.clearSession).toBe(true);+ expect(result.errorCode).toBe("claude_poisoned_previous_message_id");+ } finally {+ restore();+ await fs.rm(root, { recursive: true, force: true });+ }+ });});