The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(skills): coalesce inventory before run cache locking (#16576)
Dotta06689c1b3410
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 7edd08fde9..bd7d1acdfb 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1506,6 +1506,12 @@ as they are available, so later setup failures still clean up acquired resourcesBusy-account deferral returns explicitly before session/workspace preparation.Public-path tests are in `server/src/__tests__/heartbeat-execution-configuration.test.ts`.+Concurrent skill preparations join the company inventory refresh before taking+the version-cache lock. Each preparation then reads its actor's current skill+visibility under that lock. Snapshot materialization and the durable run version+pins stay in the same locked callback. Only the in-flight inventory work is+shared; completed refreshes and permission results are not cached.+Browser Use skill assignment first checks for an enabled, active Browser Useconnection in the run's company. With none, it skips full tool discovery.With one, the existing task, tool-policy and credential-grant checks still apply.diff --git a/maps/agent-access-map/skills.md b/maps/agent-access-map/skills.mdindex e0b1943090..6106b0eb54 100644--- a/maps/agent-access-map/skills.md+++ b/maps/agent-access-map/skills.md@@ -102,6 +102,13 @@ exclusion and immutable version pinning. This changes no agent endpoint or grantthe [configuration tests](../../server/src/__tests__/heartbeat-execution-configuration.test.ts)cover the held-lock boundary with real PostgreSQL.+Concurrent preparations share only in-flight inventory reconciliation. After+that work, each actor's visible skill rows are read again under the company cache+lock. Snapshot delivery and durable run pins complete before the lock is released.+No permission result is shared or kept between requests. The+[concurrency tests](../../server/src/__tests__/company-skills-runtime-concurrency.test.ts)+cover separate actors and project-access revocation; this adds no agent API.+## `policy`Review: `baseline`.diff --git a/maps/feature-map/skills.md b/maps/feature-map/skills.mdindex 7c7e19a564..1b5a206d2b 100644--- a/maps/feature-map/skills.md+++ b/maps/feature-map/skills.md@@ -63,6 +63,12 @@ later assignments. Selected skills and the legacy operational skill retain cacheexclusion through version pinning. The [configuration regression](../../server/src/__tests__/heartbeat-execution-configuration.test.ts)uses a real PostgreSQL lock; live provider latency remains a separate check.+Concurrent selected-skill preparations share one in-flight company inventory+refresh before cache locking. Each run retains its own selection and durable+version pins. The [concurrency regression](../../server/src/__tests__/company-skills-runtime-concurrency.test.ts)+checks shared work and separate actor access; the cache tests check that cleanup+cannot remove a snapshot before the run stores its pins.+## Gotchas- The release picker requires `enableBetaSkills`. Do not invent an Instructionsdiff --git a/server/src/__tests__/company-skills-runtime-concurrency.test.ts b/server/src/__tests__/company-skills-runtime-concurrency.test.tsnew file mode 100644index 0000000000..db6b464e4f--- /dev/null+++ b/server/src/__tests__/company-skills-runtime-concurrency.test.ts@@ -0,0 +1,80 @@+import { randomUUID } from "node:crypto";+import { promises as fs } from "node:fs";+import os from "node:os";+import path from "node:path";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";+import { agents, companies, companyMemberships, companySkills, companySkillSources, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,+ projectAccessMembers, projects, projectSkillSources } from "@paperclipai/db";+import { and, eq } from "drizzle-orm";+import { PAPERCLIP_FEEDBACK_SKILL_KEYS, PAPERCLIP_OPERATIONAL_SKILL_KEY } from "@paperclipai/adapter-utils/server-utils";+import { companySkillService } from "../services/company-skills.js";+import { withSkillActor } from "../services/skill-access.js";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";++const support = await getEmbeddedPostgresTestSupport();+const describeDatabase = support.supported ? describe : describe.skip;++describeDatabase("concurrent runtime skill preparation", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: ReturnType<typeof createDb>;+ let home: string;+ const previousHome = process.env.PAPERCLIP_HOME;++ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("runtime-skill-concurrency-");+ db = createDb(database.connectionString);+ home = await fs.mkdtemp(path.join(os.tmpdir(), "runtime-skill-concurrency-"));+ process.env.PAPERCLIP_HOME = home;+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);++ afterAll(async () => {+ if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;+ else process.env.PAPERCLIP_HOME = previousHome;+ await db?.$client.end({ timeout: 0 });+ await database?.cleanup();+ if (home) await fs.rm(home, { recursive: true, force: true });+ });++ it("shares an in-flight inventory refresh across service instances before snapshot locking", async () => {+ const companyId = randomUUID();+ await db.insert(companies).values({ id: companyId, name: "Concurrent skill readers", issuePrefix: `SC${companyId.slice(0, 6)}` });+ const select = vi.spyOn(db, "select");+ try {+ const selections = await Promise.all(Array.from({ length: 6 }, () => companySkillService(db).listRuntimeSkillEntries(companyId, {+ selectionConfig: { paperclipSkillSync: { desiredSkills: [] } }, includeLegacy: true, snapshotLatest: true,+ })));+ for (const entries of selections) {+ expect(entries.map(entry => entry.key).sort()).toEqual([PAPERCLIP_OPERATIONAL_SKILL_KEY, ...PAPERCLIP_FEEDBACK_SKILL_KEYS].sort());+ expect(entries.every(entry => entry.sourceStatus === "available" && entry.versionId)).toBe(true);+ }+ expect(select.mock.calls.filter(([fields]) => fields && Object.keys(fields).length === 1 && fields.id === companies.id)).toHaveLength(1);+ await companySkillService(db).listRuntimeSkillEntries(companyId, {+ selectionConfig: { paperclipSkillSync: { desiredSkills: [] } }, includeLegacy: true, snapshotLatest: true,+ });+ expect(select.mock.calls.filter(([fields]) => fields && Object.keys(fields).length === 1 && fields.id === companies.id)).toHaveLength(2);+ } finally { select.mockRestore(); }+ }, 30_000);++ it("shares inventory work without sharing actor visibility or retaining revoked project access", async () => {+ const companyId = randomUUID();+ await db.insert(companies).values({ id: companyId, name: "Separate skill readers", issuePrefix: `SV${companyId.slice(0, 6)}` });+ const readers = await db.insert(agents).values(["Allowed", "Denied"].map(name => ({ companyId, name, adapterType: "codex_local" }))).returning();+ await db.insert(companyMemberships).values(readers.map(reader => ({ companyId, principalType: "agent", principalId: reader.id, membershipRole: "member" })));+ const [project] = await db.insert(projects).values({ companyId, name: "Private skill source", visibility: "private" }).returning();+ const [source] = await db.insert(companySkillSources).values({ companyId, repositoryUrl: "https://github.com/fixture/private-skills",+ fullName: "fixture/private-skills", trackingRef: "main", accessScope: "projects" }).returning();+ await db.insert(projectSkillSources).values({ companyId, projectId: project!.id, sourceId: source!.id,+ repositoryUrl: source!.repositoryUrl, trackingRef: "main", status: "ready" });+ await db.insert(projectAccessMembers).values({ companyId, projectId: project!.id, subjectType: "agent", subjectId: readers[0]!.id });+ const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Private local skill" });+ await db.update(companySkills).set({ accessSourceId: source!.id }).where(eq(companySkills.id, skill.id));+ const read = (agentId: string) => withSkillActor({ type: "agent", companyId, agentId, source: "agent_jwt" },+ () => companySkillService(db).listRuntimeSkillEntries(companyId, { onlyKeys: new Set([skill.key]), snapshotLatest: true }));+ const [allowed, denied] = await Promise.all(readers.map(reader => read(reader.id)));+ expect(allowed!.map(entry => entry.key)).toEqual([skill.key]);+ expect(allowed![0]!.sourceStatus).toBe("available");+ expect(denied).toEqual([]);+ await db.delete(projectAccessMembers).where(and(eq(projectAccessMembers.projectId, project!.id), eq(projectAccessMembers.subjectId, readers[0]!.id)));+ expect(await read(readers[0]!.id)).toEqual([]);+ });+});diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.tsindex 644532e830..0a040d0787 100644--- a/server/src/__tests__/heartbeat-execution-configuration.test.ts+++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts@@ -49,10 +49,12 @@ vi.mock("../services/company-skills.js", async importOriginal => {const actual = await importOriginal<typeof import("../services/company-skills.js")>();return { ...actual, companySkillService: (...args: Parameters<typeof actual.companySkillService>) => {const service = actual.companySkillService(...args);- return { ...service, listRuntimeSkillEntries: async (...input: Parameters<typeof service.listRuntimeSkillEntries>) => {- const failure = skillFailures.get(input[0]);+ return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,+ options: Parameters<typeof service.withRuntimeSkillEntries>[1],+ consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {+ const failure = skillFailures.get(companyId);if (failure) throw new Error(failure);- return service.listRuntimeSkillEntries(...input);+ return service.withRuntimeSkillEntries(companyId, options, consume);} };} };});diff --git a/server/src/__tests__/skill-version-cache.test.ts b/server/src/__tests__/skill-version-cache.test.tsindex bd6b1f6cd4..67e19206e3 100644--- a/server/src/__tests__/skill-version-cache.test.ts+++ b/server/src/__tests__/skill-version-cache.test.ts@@ -142,8 +142,10 @@ describePostgres("immutable skill version cache", () => {const run = await activeRun(f.companyId, null);const ready = Promise.withResolvers<void>();const release = Promise.withResolvers<void>();- const preparing = withSkillVersionCacheLock(db, f.companyId, async () => {- await f.resolve(f.versions[0]!.id);+ const preparing = companySkillService(db).withRuntimeSkillEntries(f.companyId, {+ onlyKeys: new Set([f.key]), versionSelections: new Map([[f.key, f.versions[0]!.id]]),+ }, async entries => {+ expect(entries).toMatchObject([{ key: f.key, versionId: f.versions[0]!.id, sourceStatus: "available" }]);await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000);ready.resolve();await release.promise;@@ -159,6 +161,21 @@ describePostgres("immutable skill version cache", () => {} finally { release.resolve(); }await preparing;expect(await sweepSkillVersionCache(other, { companyId: f.companyId })).toMatchObject({ evicted: 0, skipped: { protected_version: 1 } });+ await other.$client.end({ timeout: 0 });+ });++ it("releases the cache lock after a runtime selection consumer fails", async () => {+ const f = await fixture();+ await expect(companySkillService(db).withRuntimeSkillEntries(f.companyId, {+ onlyKeys: new Set([f.key]), versionSelections: new Map([[f.key, f.versions[0]!.id]]),+ }, async entries => {+ expect(entries[0]!.sourceStatus).toBe("available");+ throw new Error("durable pin write failed");+ })).rejects.toThrow("durable pin write failed");+ const other = createDb(database.connectionString);+ try {+ expect(await withSkillVersionCacheLock(other, f.companyId, async () => "released", true)).toBe("released");+ } finally { await other.$client.end({ timeout: 0 }); }});it("reads company-wide protection twice per batch regardless of version count", async () => {diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.tsindex 2637721cf6..37d3b14d78 100644--- a/server/src/services/company-skills.ts+++ b/server/src/services/company-skills.ts@@ -3383,6 +3383,11 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActoasync function listFull(companyId: string): Promise<CompanySkill[]> {if (!visibility().canReadCompany(companyId)) return [];await ensureSkillInventoryCurrent(companyId);+ return readFullSkillRows(companyId);+ }++ async function readFullSkillRows(companyId: string): Promise<CompanySkill[]> {+ if (!visibility().canReadCompany(companyId)) return [];const rows = await db.select(selectCompanySkillColumns()).from(companySkills)@@ -6152,16 +6157,25 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActocompanyId: string,options: RuntimeSkillEntryOptions = {},): Promise<PaperclipSkillEntry[]> {+ return withRuntimeSkillEntries(companyId, options, entries => entries);+ }++ async function withRuntimeSkillEntries<T>(+ companyId: string,+ options: RuntimeSkillEntryOptions,+ consume: (entries: PaperclipSkillEntry[]) => T | Promise<T>,+ ): Promise<T> {if (options.onlyKeys?.size === 0 || (!options.onlyKeys && options.selectionConfig && !options.includeLegacy- && readPaperclipSkillSyncPreference(options.selectionConfig).desiredSkills.length === 0)) return [];- return withSkillVersionCacheLock(db, companyId, () => listRuntimeSkillEntriesLocked(companyId, options));+ && readPaperclipSkillSyncPreference(options.selectionConfig).desiredSkills.length === 0)) return consume([]);+ await ensureSkillInventoryCurrent(companyId);+ return withSkillVersionCacheLock(db, companyId, async () => consume(await listRuntimeSkillEntriesLocked(companyId, options)));}async function listRuntimeSkillEntriesLocked(companyId: string,options: RuntimeSkillEntryOptions,): Promise<PaperclipSkillEntry[]> {- const skills = await listFull(companyId);+ const skills = await readFullSkillRows(companyId);const selectedKeys = options.onlyKeys ?? (options.selectionConfig ? new Set((options.includeLegacy ? resolveLegacyPaperclipDesiredSkillNames : resolvePaperclipDesiredSkillNames)(options.selectionConfig, skills.map(skill => ({ key: skill.key, runtimeName: buildSkillRuntimeName(skill.key, skill.slug) }))),@@ -7442,5 +7456,6 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActoinstallUpdate,resetSkill,listRuntimeSkillEntries,+ withRuntimeSkillEntries,};}diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.tsindex 1f92076b29..f10acfda75 100644--- a/server/src/services/heartbeat/execution-configuration.ts+++ b/server/src/services/heartbeat/execution-configuration.ts@@ -1,5 +1,4 @@import { withSkillActor } from "../skill-access.js";-import { withSkillVersionCacheLock } from "../skill-version-cache.js";import type { AuthorizationActor } from "../authorization.js";import { eq, sql } from "drizzle-orm";import { heartbeatRuns, type Db, type agents, type projects } from "@paperclipai/db";@@ -70,7 +69,7 @@ export interface HeartbeatExecutionConfigurationInput {"selectedEnvironmentId" | "selectedEnvironmentForConfig" | "resolvedInstanceSettings" | "persistedNativeExecutionInput">;services: {secretsSvc: Parameters<typeof resolveExecutionRunAdapterConfig>[0]["secretsSvc"];- companySkills: Pick<ReturnType<typeof companySkillService>, "listRuntimeSkillEntries">;+ companySkills: Pick<ReturnType<typeof companySkillService>, "withRuntimeSkillEntries">;appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];finalizeAiConnectionBusyDeferral: ReturnType<typeof createHeartbeatRetries>["finalizeAiConnectionBusyDeferral"];};@@ -224,9 +223,6 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hearlet skillDeliveryConfig = effectiveResolvedConfig;const storedVersions = parseObject(run.contextSnapshot?.skillVersions);const hasStoredVersions = Object.prototype.hasOwnProperty.call(run.contextSnapshot ?? {}, "skillVersions");- const emptySkillSelection = hasStoredVersions- ? run.contextSnapshot?.skillSelectionSnapshot === true && Object.keys(storedVersions).length === 0- : agent.adapterType === "paperclip_runner" && runtimeSkillPreference.desiredSkills.length === 0;const prepareSkills = async () => {try {@@ -235,40 +231,41 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear: skillVersionSelectionMap(runtimeSkillPreference.desiredSkillEntries, {versionPinsEnabled: resolvedInstanceSettings.experimental.enableBetaSkills === true,});- const entries = await withSkillActor(skillActor, () => companySkills.listRuntimeSkillEntries(agent.companyId, {+ return await withSkillActor(skillActor, () => companySkills.withRuntimeSkillEntries(agent.companyId, {versionSelections, snapshotLatest: true,...(hasStoredVersions && run.contextSnapshot?.skillSelectionSnapshot === true? { onlyKeys: new Set(Object.keys(storedVersions)) }: { selectionConfig: effectiveResolvedConfig, includeLegacy: agent.adapterType !== "paperclip_runner" }),- }));- const available = hasStoredVersions ? entries.filter(entry => Object.prototype.hasOwnProperty.call(storedVersions, entry.key)) : entries;- const availableKeys = new Set(available.map(entry => entry.key));- const desiredKeys = hasStoredVersions && run.contextSnapshot?.skillSelectionSnapshot === true- ? Object.keys(storedVersions)- : agent.adapterType === "paperclip_runner"- ? resolvePaperclipDesiredSkillNames(effectiveResolvedConfig, available)- : resolveLegacyPaperclipDesiredSkillNames(effectiveResolvedConfig, available);- const selectedKeys = new Set(desiredKeys.filter(key => availableKeys.has(key)));- const resolved = available.filter(entry => selectedKeys.has(entry.key));- // Optional assignments can outlive project membership. Drop inaccessible- // assignments from delivery; explicit task mentions remain errors below.- skillDeliveryConfig = writePaperclipSkillSyncPreference(effectiveResolvedConfig,- resolved.map(entry => ({ key: entry.key, versionId: entry.versionId ?? null })));- for (const key of runScopedMentionedSkillKeys) {- const entry = resolved.find(candidate => candidate.key === key);- if (!entry || entry.sourceStatus === "missing") {- throw new ConfigurationIncompleteFailure("A referenced skill is unavailable. Restore its installed version or update the task's skill reference.", {- configurationIncomplete: { reason: "skill_unavailable", skillKey: key, actionUrl: "/skills" },- });+ }, async entries => {+ const available = hasStoredVersions ? entries.filter(entry => Object.prototype.hasOwnProperty.call(storedVersions, entry.key)) : entries;+ const availableKeys = new Set(available.map(entry => entry.key));+ const desiredKeys = hasStoredVersions && run.contextSnapshot?.skillSelectionSnapshot === true+ ? Object.keys(storedVersions)+ : agent.adapterType === "paperclip_runner"+ ? resolvePaperclipDesiredSkillNames(effectiveResolvedConfig, available)+ : resolveLegacyPaperclipDesiredSkillNames(effectiveResolvedConfig, available);+ const selectedKeys = new Set(desiredKeys.filter(key => availableKeys.has(key)));+ const resolved = available.filter(entry => selectedKeys.has(entry.key));+ // Optional assignments can outlive project membership. Drop inaccessible+ // assignments from delivery; explicit task mentions remain errors below.+ skillDeliveryConfig = writePaperclipSkillSyncPreference(effectiveResolvedConfig,+ resolved.map(entry => ({ key: entry.key, versionId: entry.versionId ?? null })));+ for (const key of runScopedMentionedSkillKeys) {+ const entry = resolved.find(candidate => candidate.key === key);+ if (!entry || entry.sourceStatus === "missing") {+ throw new ConfigurationIncompleteFailure("A referenced skill is unavailable. Restore its installed version or update the task's skill reference.", {+ configurationIncomplete: { reason: "skill_unavailable", skillKey: key, actionUrl: "/skills" },+ });+ }}- }- if (!hasStoredVersions) {- const skillVersions = Object.fromEntries(resolved.map(entry => [entry.key, entry.versionId ?? entry.currentVersionId ?? null]));- context.skillVersions = skillVersions;- context.skillSelectionSnapshot = true;- await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));- }- return resolved;+ if (!hasStoredVersions) {+ const skillVersions = Object.fromEntries(resolved.map(entry => [entry.key, entry.versionId ?? entry.currentVersionId ?? null]));+ context.skillVersions = skillVersions;+ context.skillSelectionSnapshot = true;+ await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));+ }+ return resolved;+ }));} catch (error) {if (agent.adapterType === "paperclip_runner") {await recordFailedSkillPreparation({@@ -282,9 +279,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hearthrow error;}};- const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", () => emptySkillSelection- ? prepareSkills()- : withSkillVersionCacheLock(db, agent.companyId, prepareSkills));+ const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);nativeRunnerPreparationSpans.push({name: "skills.prepare",parentName: "task.prepare",