The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(server): skip settled committed finalizations in unscoped passes (#16485)
Devin Foleyc7ea5c0216b8
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex e2c160a225..04f1d2c477 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -859,6 +859,14 @@ On startup and on the periodic recovery loop, Paperclip performs the following rThe stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output.+Native finalization reconciliation (part of native recovery and the orphan+reap) skips settled committed finalizations in unscoped passes: the run is+succeeded, failed, cancelled or timed out, it finished more than an hour ago,+and its decision has no pending or failed effect. Each unscoped pass still+visits up to 10 settled rows in a rotating order, so presentation and evidence+upkeep reaches every row over time. A pass scoped to explicit runs reconciles+those runs whatever their state.+Startup health (`/api/health`) reports `starting` until these recovery passes andthe other startup steps that serving requests depends on finish. Housekeepingthat every scheduler tick repeats does not delay it: the custom-image setupdiff --git a/server/src/__tests__/native-finalization-settled-candidates.test.ts b/server/src/__tests__/native-finalization-settled-candidates.test.tsnew file mode 100644index 0000000000..cdf1f486a6--- /dev/null+++ b/server/src/__tests__/native-finalization-settled-candidates.test.ts@@ -0,0 +1,175 @@+import { randomUUID } from "node:crypto";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { sql } from "drizzle-orm";+import {+ agents,+ companies,+ createDb,+ heartbeatRuns,+ issues,+ nativeRunFinalizations,+ statusDecisionEffects,+} from "@paperclipai/db";+import {+ getEmbeddedPostgresTestSupport,+ startEmbeddedPostgresTestDatabase,+} from "./helpers/embedded-postgres.js";++// The pass records its candidate count on its timing object, then starts the+// per-row loop with phase("candidates"). The mock captures the count and stops+// the pass there, so these tests check candidate selection only.+const captured = vi.hoisted(() => ({ counts: [] as number[] }));+class StopAfterCandidateScan extends Error {}+vi.mock("../services/native-runtime/native-reconciliation-timing.js", () => ({+ createNativeReconciliationPassTiming: () => ({+ phase: (name: string) => {+ if (name === "candidates") throw new StopAfterCandidateScan();+ },+ candidates: (count: number) => { captured.counts.push(count); },+ finish: () => undefined,+ }),+}));++const {+ NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,+ reconcileNativeFinalizations,+ resetSettledCommittedUpkeepForTests,+} = await import("../services/native-runtime/native-finalization-reconciler.js");++const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;++const HOUR_MS = 60 * 60 * 1000;++async function candidateCount(pass: Promise<unknown>) {+ await expect(pass).rejects.toBeInstanceOf(StopAfterCandidateScan);+ return captured.counts.at(-1);+}++describeEmbeddedPostgres("native finalization candidate selection", () => {+ let db!: ReturnType<typeof createDb>;+ let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let companyId: string;+ let agentId: string;+ let issueId: string;++ beforeAll(async () => {+ temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-settled-candidates-");+ db = createDb(temporary.connectionString);+ }, 30_000);++ beforeEach(async () => {+ captured.counts = [];+ resetSettledCommittedUpkeepForTests();+ companyId = randomUUID();+ agentId = randomUUID();+ issueId = randomUUID();+ await db.insert(companies).values({ id: companyId, name: "Settled", issuePrefix: `S${companyId.slice(0, 6)}` });+ await db.insert(agents).values({+ id: agentId, companyId, name: "Native", role: "engineer", status: "active",+ adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {},+ });+ await db.insert(issues).values({ id: issueId, companyId, title: "Settled work", status: "done" });+ });++ afterEach(async () => {+ await db.delete(statusDecisionEffects);+ await db.delete(nativeRunFinalizations);+ await db.delete(heartbeatRuns);+ await db.delete(issues);+ await db.delete(agents);+ await db.delete(companies);+ });++ afterAll(async () => {+ await temporary?.cleanup();+ });++ async function committedRun(options: { status?: string; finishedAgoMs?: number | null; decisionId?: string } = {}) {+ const runId = randomUUID();+ const finishedAgoMs = options.finishedAgoMs === undefined ? 2 * HOUR_MS : options.finishedAgoMs;+ await db.insert(heartbeatRuns).values({+ id: runId, companyId, agentId,+ status: options.status ?? "succeeded",+ runtimeMode: "native",+ nativeIssueId: issueId,+ finishedAt: finishedAgoMs === null ? null : new Date(Date.now() - finishedAgoMs),+ contextSnapshot: { issueId },+ });+ // Result, assessment and decision rows need full contract chains. These+ // tests check candidate selection only, so foreign-key triggers are off+ // for this insert; check constraints still apply.+ await db.transaction(async (tx) => {+ await tx.execute(sql`set local session_replication_role = replica`);+ await tx.insert(nativeRunFinalizations).values({+ runId, companyId, issueId, phase: "committed",+ resultId: randomUUID(),+ ...(options.decisionId ? { assessmentId: randomUUID(), decisionId: options.decisionId } : {}),+ });+ });+ return runId;+ }++ it("skips settled committed rows outside the upkeep rotation and rotates through them", async () => {+ const settled = NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS * 2 + 5;+ for (let index = 0; index < settled; index++) await committedRun();++ const counts = [];+ for (let pass = 0; pass < 4; pass++) counts.push(await candidateCount(reconcileNativeFinalizations(db)));++ // Two full upkeep pages, the remainder, then the rotation wraps.+ expect(counts).toEqual([+ NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,+ NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,+ 5,+ NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,+ ]);+ });++ // More settled rows than one upkeep page, so the page is full and a+ // protected row can only be a candidate through its own protection.+ async function fillUpkeepPage() {+ for (let index = 0; index < NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS + 2; index++) await committedRun();+ }++ it("keeps committed rows that finished within the grace window or have not finished", async () => {+ await fillUpkeepPage();+ const protectedRows = 12;+ for (let index = 0; index < protectedRows; index++) {+ const variant = index % 3;+ if (variant === 0) await committedRun({ finishedAgoMs: 10 * 60 * 1000 });+ else if (variant === 1) await committedRun({ status: "running", finishedAgoMs: null });+ else await committedRun({ status: "succeeded", finishedAgoMs: null });+ }++ expect(await candidateCount(reconcileNativeFinalizations(db)))+ .toBe(NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS + protectedRows);+ });++ it("keeps settled committed rows whose decision has an undelivered effect", async () => {+ await fillUpkeepPage();+ const protectedRows = 12;+ for (let index = 0; index < protectedRows; index++) {+ const decisionId = randomUUID();+ await committedRun({ decisionId });+ // As above, the decision row is not created; foreign-key triggers are off.+ await db.transaction(async (tx) => {+ await tx.execute(sql`set local session_replication_role = replica`);+ await tx.insert(statusDecisionEffects).values({+ companyId, issueId, decisionId, ordinal: 0, effectKind: "wake", targetType: "agent",+ idempotencyKey: randomUUID(), payload: {}, deliveryState: index % 2 === 0 ? "failed" : "pending",+ });+ });+ }++ expect(await candidateCount(reconcileNativeFinalizations(db)))+ .toBe(NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS + protectedRows);+ });++ it("reconciles a settled row when a pass names it explicitly", async () => {+ const runId = await committedRun();+ await committedRun();++ expect(await candidateCount(reconcileNativeFinalizations(db, [runId]))).toBe(1);+ });+});diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.tsindex 29f1ff7e1d..c64b785d1c 100644--- a/server/src/services/native-runtime/native-finalization-reconciler.ts+++ b/server/src/services/native-runtime/native-finalization-reconciler.ts@@ -3,7 +3,7 @@ import { hasPendingNativeChildCompletion } from "./native-child-completion-delivimport { dismissAutomaticCompletionReviews, decisionHasRetiredAutomaticReview } from "./automatic-completion-reviews.js";import { logger } from "../../middleware/logger.js";import { createHash, randomUUID } from "node:crypto";-import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, lte, ne, notInArray, or, sql } from "drizzle-orm";+import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, lt, lte, ne, not, notInArray, or, sql, type SQL } from "drizzle-orm";import type { Db } from "@paperclipai/db";import {completionContracts,@@ -634,6 +634,62 @@ type NativeReconciliationOptions = {type NativeReconciliationPassTiming = ReturnType<typeof createNativeReconciliationPassTiming>;+// A committed finalization whose run ended this long ago, with no undelivered+// decision effect, is settled: its result is projected and its effects are+// delivered, so an unscoped pass has nothing to finish. Re-reading every such+// row on every pass cost dozens of sequential queries per row, and startup+// runs the pass twice before readiness. Recent rows stay in every pass, so the+// crash-window repairs still run promptly.+export const NATIVE_SETTLED_COMMITTED_GRACE_MS = 60 * 60 * 1000;+// Settled rows still get the presentation and evidence upkeep, a few per pass+// in a rotating order, so a late repair is delayed, never skipped.+export const NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS = 10;+const NATIVE_SETTLED_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"];+let settledCommittedUpkeepCursor: string | null = null;++/** Test hook: restart the settled-row upkeep rotation from the beginning. */+export function resetSettledCommittedUpkeepForTests() {+ settledCommittedUpkeepCursor = null;+}++function settledCommittedFinalization(now: Date): SQL {+ return and(+ eq(nativeRunFinalizations.phase, "committed"),+ inArray(heartbeatRuns.status, NATIVE_SETTLED_RUN_STATUSES),+ isNotNull(heartbeatRuns.finishedAt),+ lt(heartbeatRuns.finishedAt, new Date(now.getTime() - NATIVE_SETTLED_COMMITTED_GRACE_MS)),+ sql<boolean>`not exists (+ select 1 from ${statusDecisionEffects} effect+ where effect.company_id = ${nativeRunFinalizations.companyId}+ and effect.issue_id = ${nativeRunFinalizations.issueId}+ and effect.decision_id = ${nativeRunFinalizations.decisionId}+ and effect.delivery_state in ('pending', 'failed')+ )`,+ )!;+}++/** The next settled committed runs in the upkeep rotation, wrapping at the end. */+async function nextSettledCommittedUpkeep(db: Db, now: Date): Promise<string[]> {+ const page = (after: string | null) => db+ .select({ runId: heartbeatRuns.id })+ .from(heartbeatRuns)+ .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))+ .where(and(+ eq(heartbeatRuns.runtimeMode, "native"),+ isNotNull(nativeRunFinalizations.resultId),+ settledCommittedFinalization(now),+ ...(after ? [gt(heartbeatRuns.id, after)] : []),+ ))+ .orderBy(asc(heartbeatRuns.id))+ .limit(NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS);+ let rows = await page(settledCommittedUpkeepCursor);+ if (rows.length === 0 && settledCommittedUpkeepCursor !== null) rows = await page(null);+ settledCommittedUpkeepCursor = rows.length === NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS+ ? rows[rows.length - 1]!.runId+ : null;+ return rows.map((row) => row.runId);+}+/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */export async function reconcileNativeFinalizations(db: Db,@@ -675,6 +731,11 @@ async function reconcileNativeFinalizationsPass(await dismissAutomaticCompletionReviews(db);}timing.phase("candidate_scan");+ const now = new Date();+ // An unscoped pass skips settled committed rows, except the few in this+ // pass's upkeep rotation. A scoped pass reconciles exactly the given runs.+ const unscoped = !(runIds && runIds.length > 0);+ const upkeepRunIds = unscoped ? await nextSettledCommittedUpkeep(db, now) : [];const rows = await db.select({runId: heartbeatRuns.id,@@ -711,6 +772,11 @@ async function reconcileNativeFinalizationsPass(lte(nativeRunFinalizations.leaseExpiresAt, new Date()),),...(runIds && runIds.length > 0 ? [inArray(heartbeatRuns.id, runIds)] : []),+ ...(unscoped+ ? [upkeepRunIds.length > 0+ ? or(not(settledCommittedFinalization(now)), inArray(heartbeatRuns.id, upkeepRunIds))!+ : not(settledCommittedFinalization(now))]+ : []),));timing.candidates(rows.length);timing.phase("candidates");