The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(skills): unblock retention for legacy run selections (#16852)
Dotta06c5bc471d68
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 727e3469f3..dfec311ad6 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -2196,6 +2196,14 @@ with an active or unsettled run whose pins are unknown is skipped. Optional(default `60`). Separate `__version_access__` records track use; existing cacheswithout records receive a full observation period before expiration.+Finished interrupted runs release their pins only after controller ownership,+native finalization, and environment cleanup have settled. Older native runs+without a pin map can supply their complete selection from the saved execution+input. The recovery parser validates its digests, and its company, run, agent,+and issue binding must match. Captured version IDs remain protected; unversioned+skills use separate runtime asset bundles. Missing or invalid evidence still+blocks cleanup. This fallback does not write pins or change recovery records.+Preparation, durable run pins, audits, and eviction share a company-scopedPostgreSQL lock across processes. Eligible directories are atomically renamedto unique managed `.trash-<UUID>` names. Deletion runs outside the preparationdiff --git a/server/src/__tests__/skill-version-cache.test.ts b/server/src/__tests__/skill-version-cache.test.tsindex 67e19206e3..725006cc4a 100644--- a/server/src/__tests__/skill-version-cache.test.ts+++ b/server/src/__tests__/skill-version-cache.test.ts@@ -4,12 +4,14 @@ import os from "node:os";import path from "node:path";import { eq } from "drizzle-orm";import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";-import { agents, heartbeatRuns, issues, nativeRunFinalizations, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb } from "@paperclipai/db";+import { agents, heartbeatRuns, issues, nativeRunFinalizations, environmentLeases, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb } from "@paperclipai/db";import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";import { companySkillService } from "../services/company-skills.js";import { skillVersionCachePaths, touchSkillVersionCache, withSkillVersionCacheLock } from "../services/skill-version-cache.js";import { skillVersionRetentionService } from "../services/skill-version-retention.js";+import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js";+import { canonicalNativeRuntimeContextDigest, parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js";async function sweepSkillVersionCache(db: ReturnType<typeof createDb>, options: { companyId: string; now?: number; dryRun?: boolean }) {const result = await skillVersionRetentionService(db).pruneCompany(options.companyId, { dryRun: options.dryRun, now: options.now === undefined ? undefined : new Date(options.now) });return { ...result, evicted: result.removed };@@ -136,6 +138,88 @@ describePostgres("immutable skill version cache", () => {expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).evicted).toBe(1);});+ it("releases settled interrupted runs but preserves unfinished execution and cleanup", async () => {+ const f = await fixture();+ await f.resolve(f.versions[0]!.id);+ await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000);+ const run = await activeRun(f.companyId, null);+ await db.update(heartbeatRuns).set({ status: "interrupted" }).where(eq(heartbeatRuns.id, run.id));+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).skipped.unknown_run_pins).toBe(1);+ await db.update(heartbeatRuns).set({ finishedAt: new Date(), controllerLeaseExpiresAt: new Date(Date.now() + 60_000) })+ .where(eq(heartbeatRuns.id, run.id));+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).skipped.unknown_run_pins).toBe(1);+ await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(0) }).where(eq(heartbeatRuns.id, run.id));+ const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run.id,+ status: "pending_cleanup", cleanupStatus: "failed" }).returning();+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).skipped.unknown_run_pins).toBe(1);+ await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date() })+ .where(eq(environmentLeases.id, lease!.id));+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).evicted).toBe(1);+ });++ async function legacyNativeRun(companyId: string, skills: Array<{ key: string; versionId: string | null }> = []) {+ const run = await activeRun(companyId, null);+ const [issue] = await db.insert(issues).values({ companyId, title: "Saved native selection" }).returning();+ const runtimeContext = nativeRuntimeContextFixture();+ runtimeContext.skills = skills.map((skill, index) => ({ ...skill, runtimeName: `skill-${index}`, bundle: runtimeContext.instructions.bundle }));+ runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext);+ const execution = parseNativeExecutionInput({+ schema: "paperclip.native-execution-input.v3",+ binding: { companyId, runId: run.id, agentId: run.agentId, issueId: issue!.id, executionWorkspaceId: run.id },+ task: { identifier: "CACHE-1", title: "Saved selection", description: null, prompt: "Read saved skills", workMode: "standard" },+ workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null },+ session: { normalizedSessionId: null, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },+ provider: { kind: "codex", model: null },+ completionContract: { id: "contract", sha256: "saved", schemaVersion: "paperclip.completion-contract.v1",+ contract: { revision: "1", objective: "Read saved skills", criteria: [{ id: "read", requirement: "Read skills" }] } },+ interactionResponses: [], credentialBindings: [], executionMode: "default", planningContext: null, runtimeContext,+ });+ await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue!.id, status: "failed", finishedAt: new Date(),+ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, run.id));+ await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue!.id, runId: run.id,+ phase: "terminal_failure", recoveryState: "blocked" });+ return { run, execution };+ }++ it.each([false, true])("uses the legacy native manifest without changing unresolved recovery (bundled skill: %s)", async bundled => {+ const f = await fixture();+ const old = await f.resolve(f.versions[0]!.id);+ await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000);+ const { run } = await legacyNativeRun(f.companyId, bundled ? [{ key: "paperclipai/paperclip/slack", versionId: null }] : []);+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).evicted).toBe(1);+ expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0])+ .toMatchObject({ phase: "terminal_failure", recoveryState: "blocked" });+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]!.contextSnapshot).toBeNull();+ vi.stubGlobal("fetch", vi.fn(() => Promise.reject(new Error("Upstream unavailable"))));+ expect(await f.resolve(f.versions[0]!.id)).toMatchObject({ source: old.source });+ });++ it("preserves exact legacy manifest pins", async () => {+ const f = await fixture();+ await f.resolve(f.versions[0]!.id);+ await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000);+ await legacyNativeRun(f.companyId, [{ key: f.key, versionId: f.versions[0]!.id }]);+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).skipped.protected_version).toBe(1);+ });++ it.each(["companyId", "runId", "agentId", "issueId", "digest", "missing", "invalid_version", "invalid_pins", "legacy_adapter"])(+ "fails closed for unusable native selection evidence (%s)", async invalid => {+ const f = await fixture();+ await f.resolve(f.versions[0]!.id);+ await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000);+ const { run, execution } = await legacyNativeRun(f.companyId,+ invalid === "invalid_version" ? [{ key: f.key, versionId: "not-a-version" }] : []);+ if (["companyId", "runId", "agentId", "issueId"].includes(invalid)) Object.assign(execution.binding, { [invalid]: randomUUID() });+ if (invalid === "digest" && "runtimeContext" in execution) execution.runtimeContext.aggregateDigest = "f".repeat(64);+ await db.update(heartbeatRuns).set({+ runnerProfileJson: invalid === "missing" ? {} : { nativeExecutionInput: execution },+ ...(invalid === "invalid_pins" ? { contextSnapshot: { skillVersions: null } } : {}),+ ...(invalid === "legacy_adapter" ? { runtimeMode: "legacy" } : {}),+ }).where(eq(heartbeatRuns.id, run.id));+ expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).skipped.unknown_run_pins).toBe(1);+ },+ );+it("serializes eviction behind preparation and durable pinning across database clients", async () => {const f = await fixture();await f.resolve(f.versions[0]!.id);diff --git a/server/src/services/skill-version-cache.ts b/server/src/services/skill-version-cache.tsindex 50ee27f846..2f3ec15c5a 100644--- a/server/src/services/skill-version-cache.ts+++ b/server/src/services/skill-version-cache.ts@@ -9,6 +9,7 @@ import { resolvePaperclipInstanceRoot } from "../home-paths.js";import { assertSkillSnapshotPath } from "./skill-snapshot.js";import { repositorySkillFrontmatter } from "./skill-repository-package.js";import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared";+import { parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js";export const SKILL_VERSION_CACHE_IDLE_MS = 60 * 60 * 1000;const locks = new AsyncLocalStorage<Array<{ db: Db; companyId: string }>>();@@ -91,9 +92,16 @@ export async function lastUsedAt(filename: string): Promise<number | null> {// A terminal run can still own provider execution or recovery. Only fully settled// runs release their pins; resume preparation reacquires the same cache lock.export async function protectedVersions(db: Db, companyId: string): Promise<Set<string> | null> {- const rows = await db.select({ pins: sql<unknown>`${heartbeatRuns.contextSnapshot}->'skillVersions'` })+ const rows = await db.select({+ id: heartbeatRuns.id, agentId: heartbeatRuns.agentId, nativeIssueId: heartbeatRuns.nativeIssueId,+ pins: sql<unknown>`${heartbeatRuns.contextSnapshot}->'skillVersions'`,+ // Avoid loading large execution inputs for runs with the modern pin map.+ legacyInput: sql<unknown>`case when ${heartbeatRuns.runtimeMode} = 'native'+ and not coalesce(${heartbeatRuns.contextSnapshot} ? 'skillVersions', false)+ then ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput' end`,+ }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), sql`(- ${heartbeatRuns.status} not in ('succeeded', 'failed', 'timed_out', 'cancelled')+ ${heartbeatRuns.status} not in ('succeeded', 'failed', 'timed_out', 'cancelled', 'interrupted')or ${heartbeatRuns.finishedAt} is nullor ${heartbeatRuns.controllerLeaseExpiresAt} > now()or exists (select 1 from ${nativeRunFinalizations} f@@ -107,7 +115,27 @@ export async function protectedVersions(db: Db, companyId: string): Promise<Set<and l.lease_policy = 'reuse_by_environment' and l.cleanup_status = 'success', false)))))`));const versions = new Set<string>();- for (const { pins } of rows) {+ for (const { pins, legacyInput, id, agentId, nativeIssueId } of rows) {+ if (legacyInput != null) {+ // Older native runs captured the complete selection in their immutable+ // execution input before contextSnapshot.skillVersions was introduced.+ // Use the recovery parser (including digest validation) and the exact run+ // binding. Never repair a malformed modern pin map with a different source.+ try {+ const input = parseNativeExecutionInput(legacyInput);+ if (!("runtimeContext" in input) || input.binding.companyId !== companyId+ || input.binding.runId !== id || input.binding.agentId !== agentId+ || input.binding.issueId !== nativeIssueId) return null;+ for (const skill of input.runtimeContext.skills) {+ // Unversioned skills have their own captured asset bundle; they do+ // not reference a directory under __versions__.+ if (skill.versionId === null) continue;+ if (!uuid.test(skill.versionId)) return null;+ versions.add(skill.versionId);+ }+ continue;+ } catch { return null; }+ }// Legacy/incomplete preparation is not proof that no skill is in use.if (!pins || typeof pins !== "object" || Array.isArray(pins)) return null;for (const version of Object.values(pins)) {diff --git a/server/src/services/skill-version-retention.ts b/server/src/services/skill-version-retention.tsindex 4dc46985d0..ab507fd8ee 100644--- a/server/src/services/skill-version-retention.ts+++ b/server/src/services/skill-version-retention.ts@@ -33,9 +33,6 @@ export const SKILL_VERSION_RETENTION_DEFAULT_MAX_REMOVALS = 50;export const SKILL_VERSION_RETENTION_INTERVAL_MS = 60 * 60 * 1000;export const SKILL_VERSION_RETENTION_STARTUP_DELAY_MS = 60 * 1000;-/** A run in one of these states can still read (or re-read) its captured skill versions. */-export const SKILL_VERSION_LIVE_RUN_STATUSES = ["queued", "scheduled_retry", "running", "interrupted"] as const;-const VERSIONS_DIR = "__versions__";const TRASH_PREFIX = ".trash-";const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;