The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(recovery): project context evidence for latest-run checks (#17463)
Devin Foley7bd449dbb664
diff --git a/doc/DATABASE.md b/doc/DATABASE.mdindex 2698c899e5..3ab9490ee1 100644--- a/doc/DATABASE.md+++ b/doc/DATABASE.md@@ -49,6 +49,13 @@ and leaves all existing cases passive. Source identities are not grants.### Task query performance+Recovery's latest-run queries return the context fields used by recovery+decisions. Prompts, execution continuation bodies, and workspace setup payloads+remain in the stored run and its full detail reads. These queries also return+the PostgreSQL row version. Native blocked-wait repair compares that version+against a fresh full source read before applying changes. Any intervening row+update, including an edit outside the projection, defers repair to a fresh sweep.+Run-context lookups use expression indexes for both `context_snapshot.issueId`and `context_snapshot.paperclipIssue.id`. Task creation provenance has its owncompany-scoped expression index that follows the query's native issue ID anddiff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 3af86f73e7..ca2576a5b5 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -15676,6 +15676,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {"comment","wake","run",+ "run_context",+ "run_metadata","attention",] as const)("does not clobber a newer %s while native blocker repair waits on the issue lock",@@ -15750,6 +15752,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {continuationPolicy: "wake_assignee",payload: { version: 1, questions: [] },});+ if (change === "run_context")+ await tx.update(heartbeatRuns).set({+ contextSnapshot: sql`jsonb_set(${heartbeatRuns.contextSnapshot}, '{paperclipTaskMarkdown}', '"Changed execution instructions"'::jsonb)`,+ }).where(eq(heartbeatRuns.id, runId));+ if (change === "run_metadata")+ await tx.update(heartbeatRuns).set({ stdoutExcerpt: "Late execution output" }).where(eq(heartbeatRuns.id, runId));[snapshot] = await tx.select().from(issues)@@ -15793,6 +15801,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {),).toHaveLength(0);expect(mockAdapterExecute).not.toHaveBeenCalled();+ if (change === "run_context" || change === "run_metadata") {+ expect((await heartbeat.reconcileStrandedAssignedIssues()).escalated).toBe(1);+ expect(mockAdapterExecute).not.toHaveBeenCalled();+ }},);diff --git a/server/src/services/recovery/run-context-read.test.ts b/server/src/services/recovery/run-context-read.test.tsnew file mode 100644index 0000000000..682aea7ab9--- /dev/null+++ b/server/src/services/recovery/run-context-read.test.ts@@ -0,0 +1,84 @@+import { randomBytes, randomUUID } from "node:crypto";+import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";+import { eq } from "drizzle-orm";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import { isExternalChatPresentationContext } from "../heartbeat-run-summary.js";+import { legacyDispositionEpisode } from "./legacy-continuation.js";+import { decideIssueReviewPathRecovery } from "./review-path-recovery.js";+import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js";++const support = await getEmbeddedPostgresTestSupport();++describe.skipIf(!support.supported)("recovery context PostgreSQL projection", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: ReturnType<typeof createDb>;+ let companyId: string;+ let agentId: string;+ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("recovery-context-read-");+ db = createDb(database.connectionString);+ const [company] = await db.insert(companies).values({ name: "Recovery fixture", issuePrefix: "RC" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Recovery agent", adapterType: "process" }).returning();+ companyId = company.id;+ agentId = agent.id;+ }, 60_000);+ afterAll(async () => {+ await db?.$client.end();+ await database?.cleanup();+ });++ async function read(contextSnapshot: unknown) {+ const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, contextSnapshot: contextSnapshot as Record<string, unknown> | null }).returning();+ const [selected] = await db.select({ context: recoveryRunContextColumn, version: recoveryRunVersionColumn })+ .from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ return { run, selected };+ }++ it("retains typed recovery evidence while excluding large execution data", async () => {+ const issueId = randomUUID();+ const payload = randomBytes(128 * 1024).toString("base64");+ const evidence = {+ issueId, taskId: issueId, retryReason: "issue_continuation_needed", source: "chat:slack",+ wakeReason: "assignment", handoffReason: "successful_run_missing_state", handoffRequired: true,+ handoffAttempt: 2, maxHandoffAttempts: 3, sourceRunId: randomUUID(), resumeFromRunId: randomUUID(),+ missingDisposition: "clear_next_step", providerQuotaRetryNotBefore: "2099-01-01T00:00:00Z",+ transientRetryNotBefore: "2099-01-02T00:00:00Z", dispositionRepairAttempt: 2,+ dispositionRepairFingerprint: "fingerprint", legacyDispositionEpisode: { id: "episode", attempt: 1, maxAttempts: 2 },+ reviewPathRecoveryAttempt: 1, reviewPathConsumedRef: "reference", interactionId: randomUUID(),+ approvalId: randomUUID(), clearReason: "resolved", monitorAttemptCount: 4,+ wakeCommentIds: [randomUUID(), randomUUID()], wakeCommentId: randomUUID(), commentId: randomUUID(),+ externalChatContinuation: true,+ };+ const context = { ...evidence, paperclipWake: { externalInteractionContinuation: true, prompt: payload },+ paperclipWorkspace: { mode: "shared_workspace", instructions: payload }, executionContinuation: { prompt: payload }, paperclipTaskMarkdown: payload };+ const { run, selected } = await read(context);+ expect(selected.context).toEqual({ ...evidence, paperclipWake: { externalInteractionContinuation: true }, paperclipWorkspace: { mode: "shared_workspace" } });+ expect(Buffer.byteLength(JSON.stringify(selected))).toBeLessThan(4096);+ expect(isExternalChatPresentationContext(selected.context)).toBe(isExternalChatPresentationContext(context));+ expect(legacyDispositionEpisode({ ...run, contextSnapshot: selected.context })).toEqual(legacyDispositionEpisode(run));+ const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ expect(stored.contextSnapshot).toEqual(context);+ const [serialized] = await db.$client.unsafe("SELECT xmin::text AS version FROM heartbeat_runs WHERE id = $1", [run.id]);+ expect(selected.version).toBe(serialized.version);+ await db.update(heartbeatRuns).set({ contextSnapshot: { ...context, paperclipTaskMarkdown: "changed" } }).where(eq(heartbeatRuns.id, run.id));+ const [changed] = await db.select({ context: recoveryRunContextColumn, version: recoveryRunVersionColumn }).from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ expect(changed.context).toEqual(selected.context);+ expect(changed.version).not.toBe(selected.version);+ });++ it.each([null, [], ["unexpected"], "scalar", 42, { issueId: 42, taskId: false, paperclipWake: [], paperclipWorkspace: null }].map(context => ({ context })))("preserves decisions for malformed context $context", async ({ context }) => {+ const { selected } = await read(context);+ expect(isExternalChatPresentationContext(selected.context)).toBe(isExternalChatPresentationContext(context));+ expect(legacyDispositionEpisode({ id: "run", contextSnapshot: selected.context })).toEqual(legacyDispositionEpisode({ id: "run", contextSnapshot: context }));+ });++ it("retains external-chat review recovery message references and monitor identity", async () => {+ const context = { source: "chat:slack", wakeReason: "issue_monitor_fired", clearReason: "fired", monitorAttemptCount: 3,+ wakeCommentIds: [randomUUID(), randomUUID()], prompt: randomBytes(64 * 1024).toString("base64") };+ const { selected } = await read(context);+ const input = { issueId: randomUUID(), sourceRunId: randomUUID(), assigneeAgentId: agentId, reviewAttention: { state: "stalled" } as Parameters<typeof decideIssueReviewPathRecovery>[0]["reviewAttention"], existingWake: false };+ expect(decideIssueReviewPathRecovery({ ...input, contextSnapshot: selected.context }))+ .toEqual(decideIssueReviewPathRecovery({ ...input, contextSnapshot: context }));+ });+});diff --git a/server/src/services/recovery/run-context-read.ts b/server/src/services/recovery/run-context-read.tsnew file mode 100644index 0000000000..b453aa79d0--- /dev/null+++ b/server/src/services/recovery/run-context-read.ts@@ -0,0 +1,28 @@+import { heartbeatRuns } from "@paperclipai/db";+import { sql } from "drizzle-orm";++const recoveryContextKeys = [+ "issueId", "taskId", "retryReason", "source", "wakeReason",+ "handoffReason", "handoffRequired", "handoffAttempt", "maxHandoffAttempts",+ "sourceRunId", "resumeFromRunId", "missingDisposition",+ "providerQuotaRetryNotBefore", "transientRetryNotBefore",+ "dispositionRepairAttempt", "dispositionRepairFingerprint", "legacyDispositionEpisode",+ "reviewPathRecoveryAttempt", "reviewPathConsumedRef", "interactionId", "approvalId",+ "clearReason", "monitorAttemptCount", "wakeCommentIds", "wakeCommentId", "commentId",+ "externalChatContinuation", "paperclipWake", "paperclipWorkspace",+] as const;++export const recoveryRunContextColumn = sql<Record<string, unknown> | null>`+ case when ${heartbeatRuns.contextSnapshot} is null then null else (+ select coalesce(jsonb_object_agg(field.key, case field.key+ when 'paperclipWake' then jsonb_build_object(+ 'externalInteractionContinuation', field.value->'externalInteractionContinuation')+ when 'paperclipWorkspace' then jsonb_build_object('mode', field.value->'mode')+ else field.value end), '{}'::jsonb)+ from jsonb_each(case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object'+ then ${heartbeatRuns.contextSnapshot} else '{}'::jsonb end) as field(key, value)+ where field.key in (${sql.join(recoveryContextKeys.map(key => sql`${key}`), sql`, `)})+ ) end+`;++export const recoveryRunVersionColumn = sql<string>`${heartbeatRuns}.xmin::text`;diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex 42765bb360..e517b69ec6 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -1,3 +1,4 @@+import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js";import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";@@ -25,6 +26,7 @@ import {desc,eq,gt,+ getTableColumns,gte,inArray,isNotNull,@@ -247,6 +249,7 @@ type LatestIssueRun =| "createdAt"> & {resultJson?: unknown;+ rowVersion?: string;})| null;type SuccessfulLatestIssueRun = NonNullable<LatestIssueRun> & {@@ -977,7 +980,8 @@ export function recoveryService(status: heartbeatRuns.status,error: heartbeatRuns.error,errorCode: heartbeatRuns.errorCode,- contextSnapshot: heartbeatRuns.contextSnapshot,+ contextSnapshot: recoveryRunContextColumn,+ rowVersion: recoveryRunVersionColumn,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,startedAt: heartbeatRuns.startedAt,@@ -1007,7 +1011,8 @@ export function recoveryService(status: heartbeatRuns.status,error: heartbeatRuns.error,errorCode: heartbeatRuns.errorCode,- contextSnapshot: heartbeatRuns.contextSnapshot,+ contextSnapshot: recoveryRunContextColumn,+ rowVersion: recoveryRunVersionColumn,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,startedAt: heartbeatRuns.startedAt,@@ -1618,7 +1623,10 @@ export function recoveryService(return false;const source = await tx- .select()+ .select({+ ...getTableColumns(heartbeatRuns),+ rowVersion: recoveryRunVersionColumn,+ }).from(heartbeatRuns).where(and(@@ -1634,8 +1642,9 @@ export function recoveryService(source.id !== latestRun.id ||source.agentId !== current.assigneeAgentId ||source.status !== "succeeded" ||- JSON.stringify(source.contextSnapshot) !==- JSON.stringify(latestRun.contextSnapshot)+ (latestRun.rowVersion !== undefined+ ? source.rowVersion !== latestRun.rowVersion+ : JSON.stringify(source.contextSnapshot) !== JSON.stringify(latestRun.contextSnapshot)))return false;const proof = await nativeBlockedUnblockAction(@@ -1903,7 +1912,8 @@ export function recoveryService(status: heartbeatRuns.status,error: heartbeatRuns.error,errorCode: heartbeatRuns.errorCode,- contextSnapshot: heartbeatRuns.contextSnapshot,+ contextSnapshot: recoveryRunContextColumn,+ rowVersion: recoveryRunVersionColumn,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,startedAt: heartbeatRuns.startedAt,