The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: preserve repository cache negotiation refs (#16687)
Dotta023f4a5d0cd4
diff --git a/doc/observability.md b/doc/observability.mdindex 1052b71878..3bfa51657d 100644--- a/doc/observability.md+++ b/doc/observability.md@@ -321,6 +321,24 @@ Asset writes and verification may add only non-negative integer`paperclip.execution.fileCount` and `paperclip.execution.bytes` attributes.The closed keys are declared in `EXECUTION_PREPARATION_SPAN_ATTRS` in`packages/adapter-utils/src/acpx-engine/startup-timing.ts`.+Repository checkout cache phases also carry the closed+`paperclip.execution.repositoryCache.outcome` value: `disabled`, `hit`, `miss`,+`published`, `already_present`, `contended`, `capacity`, `budget`, `invalid`, or `error`.+The clone span carries the boolean+`paperclip.execution.repositoryCache.reference_supplied`, including `false` for+a fallback clone. A lookup hit means a validated cache was found, and supplying+a reference does not prove Git avoided a download. These attributes carry no+cache keys, remote URLs, paths, refs, credentials, or error text.++Pristine repository-object cache version 2 mirrors the fresh checkout's refs,+including remote-tracking refs. Version 1 copied their objects into a bare+repository but omitted the remote-tracking tips. Git could then download those+objects again during negotiation. The version is part of the cache key, so the+first checkout creates a new generation and later checkouts can reuse its tips.+Existing generations remain under the same storage limits. Cached refs stay+private to the company and credential scope. Task checkouts still authenticate+online, use the current remote refs, and dissociate their objects from the cache.+Other execution phases carry only the existing hashed run ID; manual boundarieshave no attributes. Exceptions set status without exporting their messages.diff --git a/packages/adapter-utils/src/acpx-engine/startup-timing.ts b/packages/adapter-utils/src/acpx-engine/startup-timing.tsindex 590d1f4f2a..2c3fbe8f4c 100644--- a/packages/adapter-utils/src/acpx-engine/startup-timing.ts+++ b/packages/adapter-utils/src/acpx-engine/startup-timing.ts@@ -51,6 +51,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = {runId: "paperclip.task.run.run_id",fileCount: "paperclip.execution.fileCount",bytes: "paperclip.execution.bytes",+ repositoryCacheOutcome: "paperclip.execution.repositoryCache.outcome",+ repositoryCacheReferenceSupplied: "paperclip.execution.repositoryCache.reference_supplied",} as const;/**diff --git a/server/src/__tests__/repository-object-cache.test.ts b/server/src/__tests__/repository-object-cache.test.tsindex 4c5454d6f8..5dc9eb9683 100644--- a/server/src/__tests__/repository-object-cache.test.ts+++ b/server/src/__tests__/repository-object-cache.test.ts@@ -11,11 +11,13 @@ import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-resimport { materializeManagedProjectWorkspace } from "../services/managed-repository-checkout.js";import { assertReproducibleBaseGitStorage } from "../services/git-workspace-restoration.js";import { readRepositoryObjectCache, resolveRepositoryObjectCache, publishRepositoryObjectCache } from "../services/repository-object-cache.js";+import * as preparationTiming from "../services/execution-preparation-timing.js";const execFile = promisify(execute);const roots: string[] = [];afterEach(async () => {vi.unstubAllEnvs();+ vi.restoreAllMocks();await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true })));});async function fixture() {@@ -33,6 +35,30 @@ async function fixture() {return { root, source, git, company, repoUrl, prepare };}+it("retains remote branch tips so a primed clone transfers no existing objects", async () => {+ const f = await fixture();+ await f.git(f.source, "checkout", "-b", "feature");+ await fs.writeFile(path.join(f.source, "feature.txt"), "unmerged feature content\n");+ await f.git(f.source, "add", ".");+ await f.git(f.source, "commit", "-m", "feature");+ const featureHead = await f.git(f.source, "rev-parse", "HEAD");+ await f.git(f.source, "checkout", "main");+ await f.prepare("first");+ const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!;+ expect(await f.git(cache.directory, "rev-parse", "refs/remotes/origin/feature")).toBe(featureHead);++ const packTrace = path.join(f.root, "received.pack");+ vi.stubEnv("GIT_TRACE_PACKFILE", packTrace);+ await f.prepare("second");+ expect(await fs.stat(packTrace).then(stat => stat.size, () => 0)).toBe(0);+ const target = path.join(f.root, "second");+ expect(await f.git(target, "rev-parse", "origin/feature")).toBe(featureHead);+ await expect(fs.stat(path.join(target, ".git/objects/info/alternates"))).rejects.toMatchObject({ code: "ENOENT" });+ await fs.rm(f.company, { recursive: true });+ expect(await f.git(target, "show", "origin/feature:feature.txt")).toBe("unmerged feature content");+ await f.git(target, "fsck", "--full");+});+it("uses current remote refs and produces an independent checkout without private cached refs", async () => {const f = await fixture();await f.git(f.source, "checkout", "-b", "private");@@ -157,11 +183,23 @@ it("skips oversized repositories without creating a cache generation", async ()const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!;const oversized = path.join(f.source, ".git/oversized");await fs.writeFile(oversized, ""); await fs.truncate(oversized, 2 * 1024 ** 3 + 1);+ const detail = vi.spyOn(preparationTiming, "recordRepositoryObjectCacheDetail");await publishRepositoryObjectCache(cache, f.source);+ expect(detail).toHaveBeenLastCalledWith({ outcome: "capacity" });expect(await fs.readdir(f.company)).toEqual([]);expect(await f.git(f.source, "show", "HEAD:work.txt")).toBe("initial");});+it("reports a symlink found during storage scanning as an invalid cache skip", async () => {+ const f = await fixture();+ const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!;+ await fs.symlink(f.source, path.join(f.source, ".git/unexpected-link"));+ const detail = vi.spyOn(preparationTiming, "recordRepositoryObjectCacheDetail");+ await publishRepositoryObjectCache(cache, f.source);+ expect(detail).toHaveBeenLastCalledWith({ outcome: "invalid" });+ expect(await fs.readdir(f.company)).toEqual([]);+});+it.each(["https://user:[email protected]/fixture/repo", "https://github.com/fixture/repo?token=secret", "https://github.com/fixture/repo#secret"])("does not persist credential-bearing URLs: %s", async (repoUrl) => {const f = await fixture();expect(await resolveRepositoryObjectCache(f.company, repoUrl, null)).toBeNull();diff --git a/server/src/services/execution-preparation-timing.test.ts b/server/src/services/execution-preparation-timing.test.tsindex 17bda2f419..0579890170 100644--- a/server/src/services/execution-preparation-timing.test.ts+++ b/server/src/services/execution-preparation-timing.test.ts@@ -1,11 +1,11 @@import { afterEach, beforeEach, expect, it, vi } from "vitest";-import { createRunPreparationTiming, measureRunPhase, withRunExecutionTiming } from "./execution-preparation-timing.js";+import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming } from "./execution-preparation-timing.js";const handles = vi.hoisted(() => ({ getTracer: vi.fn(), startActiveSpan: vi.fn() }));-vi.mock("@opentelemetry/api", () => ({- ROOT_CONTEXT: "root", SpanStatusCode: { ERROR: 2 },- trace: { getTracer: handles.getTracer, getActiveSpan: () => undefined },-}));+vi.mock("@opentelemetry/api", async importOriginal => {+ const api = await importOriginal<typeof import("@opentelemetry/api")>();+ return { ...api, trace: { ...api.trace, getTracer: handles.getTracer, getActiveSpan: () => undefined } };+});beforeEach(() => {vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");handles.getTracer.mockReturnValue({ startActiveSpan: handles.startActiveSpan });@@ -43,7 +43,10 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", asyncawait expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation"] as const) {const result = {};- const work = vi.fn(() => result);+ const work = vi.fn(() => {+ recordRepositoryObjectCacheDetail({ outcome: "hit", referenceSupplied: true });+ return result;+ });await expect(withRunExecutionTiming("run", () => measureRunPhase(phase, work))).resolves.toBe(result);expect(work).toHaveBeenCalledOnce();const original = new Error("original startup failure");@@ -59,3 +62,46 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", asyncerror.mockRestore();}});++it("keeps cache outcomes and clone references on their owning concurrent phases", async () => {+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> = [];+ handles.startActiveSpan.mockImplementation((name, options, _context, callback) => {+ const record = { name, attributes: { ...options.attributes } };+ spans.push(record);+ return callback({+ end: vi.fn(), setStatus: vi.fn(),+ setAttribute: (key: string, value: unknown) => { record.attributes[key] = value; },+ });+ });+ await Promise.all([+ withRunExecutionTiming("first", () => measureRunPhase("workspace.checkout.cache_lookup", async () => {+ await Promise.resolve();+ recordRepositoryObjectCacheDetail({ outcome: "hit" });+ })),+ withRunExecutionTiming("second", () => measureRunPhase("workspace.checkout.git_clone", async () => {+ await Promise.resolve();+ recordRepositoryObjectCacheDetail({ referenceSupplied: false });+ })),+ ]);+ expect(spans.find(span => span.name === "workspace.checkout.cache_lookup")?.attributes).toMatchObject({+ "paperclip.execution.repositoryCache.outcome": "hit",+ });+ expect(spans.find(span => span.name === "workspace.checkout.git_clone")?.attributes).toMatchObject({+ "paperclip.execution.repositoryCache.reference_supplied": false,+ });+ expect(spans.filter(span => span.name === "task.execution_attempt").every(span =>+ Object.keys(span.attributes).every(key => key === "paperclip.task.run.run_id"))).toBe(true);+});++it("does not let cache attribute failures replace the operation result", async () => {+ handles.startActiveSpan.mockImplementation((_name, _options, _context, callback) => callback({+ end: vi.fn(), setStatus: vi.fn(),+ setAttribute: () => { throw new Error("tracer failed"); },+ }));+ const work = vi.fn(() => {+ recordRepositoryObjectCacheDetail({ outcome: "published", referenceSupplied: true });+ return 42;+ });+ await expect(withRunExecutionTiming("run", () => measureRunPhase("workspace.checkout.cache_publish", work))).resolves.toBe(42);+ expect(work).toHaveBeenCalledOnce();+});diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.tsindex 94f1a67862..e92e42ee98 100644--- a/server/src/services/execution-preparation-timing.ts+++ b/server/src/services/execution-preparation-timing.ts@@ -84,6 +84,26 @@ type RunScope = { runId: string; parent?: Context; ended: boolean; details: Map<const ownership = new AsyncLocalStorage<RunScope>();const endpointEnabled = () => Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim());+const repositoryCacheOutcomes = ["disabled", "hit", "miss", "published", "already_present", "contended", "capacity", "budget", "invalid", "error"] as const;+export type RepositoryObjectCacheOutcome = typeof repositoryCacheOutcomes[number];++export function recordRepositoryObjectCacheDetail(detail: {+ outcome?: RepositoryObjectCacheOutcome;+ referenceSupplied?: boolean;+}): void {+ const owned = ownership.getStore();+ if (!endpointEnabled() || !owned?.parent || owned.ended) return;+ try {+ const span = trace.getSpan(owned.parent);+ if (detail.outcome && repositoryCacheOutcomes.includes(detail.outcome)) {+ span?.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.repositoryCacheOutcome, detail.outcome);+ }+ if (typeof detail.referenceSupplied === "boolean") {+ span?.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.repositoryCacheReferenceSupplied, detail.referenceSupplied);+ }+ } catch {}+}+export function getRunExecutionTraceContext(runId: string): Context | undefined {const owned = ownership.getStore();return endpointEnabled() && owned?.runId === runId && !owned.ended ? owned.parent : undefined;diff --git a/server/src/services/managed-repository-checkout.ts b/server/src/services/managed-repository-checkout.tsindex 1e6205381b..3d5dd027c0 100644--- a/server/src/services/managed-repository-checkout.ts+++ b/server/src/services/managed-repository-checkout.ts@@ -8,7 +8,7 @@ import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js";import { describeGitAuthFailure, scrubGitCredentialText, type GitRemoteAuthProvider } from "./git-credentials.js";import { classifyGitCloneFailure, GitConnectionFailureError, readGitConnectionFailure } from "./git-connection-failure.js";import { isWorkspaceGitScanError, WorkspaceGitScanError } from "./workspace-git-operation-scheduler.js";-import { measureRunPhase } from "./execution-preparation-timing.js";+import { measureRunPhase, recordRepositoryObjectCacheDetail } from "./execution-preparation-timing.js";import { resolveRepositoryObjectCache, readRepositoryObjectCache, publishRepositoryObjectCache } from "./repository-object-cache.js";const execFile = promisify(execFileCallback);@@ -65,23 +65,34 @@ export async function materializeManagedProjectWorkspace(: `${cwd}.clone-`);const cloneSource = input.localSource ?? input.repoUrl;let objectCache = input.objectCacheCompanyDirectory && !input.localSource- ? await measureRunPhase("workspace.checkout.cache_lookup", () => resolveRepositoryObjectCache(input.objectCacheCompanyDirectory!, input.repoUrl!, auth)) : null;- const reference = objectCache ? await measureRunPhase("workspace.checkout.cache_lookup", () => readRepositoryObjectCache(objectCache!)) : null;+ ? await measureRunPhase("workspace.checkout.cache_lookup", async () => {+ const cache = await resolveRepositoryObjectCache(input.objectCacheCompanyDirectory!, input.repoUrl!, auth);+ if (!cache) recordRepositoryObjectCacheDetail({ outcome: "disabled" });+ return cache;+ }) : null;+ const reference = objectCache ? await measureRunPhase("workspace.checkout.cache_lookup", async () => {+ const cached = await readRepositoryObjectCache(objectCache!);+ recordRepositoryObjectCacheDetail({ outcome: cached ? "hit" : "miss" });+ return cached;+ }) : null;try {await input.beforeCopy?.(cloneTmpDir);try {- const clone = (referencePath: string | null) => measureRunPhase("workspace.checkout.git_clone", () => execFile(- "git",- [...(auth?.configArgs ?? []), ...(referencePath ? ["-c", "repack.updateServerInfo=false"] : []), "clone", "--no-hardlinks", ...(referencePath ? ["--reference-if-able", referencePath, "--dissociate"] : []), "--", cloneSource, cloneTmpDir],- {- env: {- ...sanitizeRuntimeServiceBaseEnv(process.env),- GIT_TERMINAL_PROMPT: "0",- ...(auth?.env ?? {}),+ const clone = (referencePath: string | null) => measureRunPhase("workspace.checkout.git_clone", () => {+ recordRepositoryObjectCacheDetail({ referenceSupplied: referencePath !== null });+ return execFile(+ "git",+ [...(auth?.configArgs ?? []), ...(referencePath ? ["-c", "repack.updateServerInfo=false"] : []), "clone", "--no-hardlinks", ...(referencePath ? ["--reference-if-able", referencePath, "--dissociate"] : []), "--", cloneSource, cloneTmpDir],+ {+ env: {+ ...sanitizeRuntimeServiceBaseEnv(process.env),+ GIT_TERMINAL_PROMPT: "0",+ ...(auth?.env ?? {}),+ },+ timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS,},- timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS,- },- ));+ );+ });try { await clone(reference); }catch (error) {if (!reference) throw error;diff --git a/server/src/services/repository-object-cache.ts b/server/src/services/repository-object-cache.tsindex 9a0590739f..68dc23df1f 100644--- a/server/src/services/repository-object-cache.ts+++ b/server/src/services/repository-object-cache.ts@@ -7,13 +7,21 @@ import { tryWithDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-import { assertWorkspaceManifestDiskSpace } from "@paperclipai/adapter-utils/workspace-manifest";import type { GitAuthInvocation } from "./git-credentials.js";import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js";+import { recordRepositoryObjectCacheDetail, type RepositoryObjectCacheOutcome } from "./execution-preparation-timing.js";const execFile = promisify(execute);const REPOSITORY_BYTES = 2 * 1024 ** 3;const COMPANY_BYTES = 4 * 1024 ** 3;const COMPANY_REPOSITORIES = 64;+const CACHE_VERSION = 2;const hash = (value: string) => createHash("sha256").update(value).digest("hex");+class RepositoryObjectCacheSkip extends Error {+ constructor(readonly outcome: "capacity" | "budget" | "invalid") {+ super("Repository object cache skipped");+ }+}+export interface RepositoryObjectCache {companyDirectory: string;directory: string;@@ -36,7 +44,7 @@ export async function resolveRepositoryObjectCache(companyDirectory: string, rep}companyDirectory = await fs.realpath(companyDirectory);const scope = auth ? [auth.source, auth.secretName, auth.configArgs, Object.entries(auth.env).sort(([a], [b]) => a.localeCompare(b))] : null;- const key = hash(JSON.stringify([repoUrl, scope]));+ const key = hash(JSON.stringify([CACHE_VERSION, repoUrl, scope]));return { companyDirectory, directory: path.join(companyDirectory, key), key, repoUrl };} catch { return null; }}@@ -55,7 +63,7 @@ export async function readRepositoryObjectCache(cache: RepositoryObjectCache): Pconst stat = await fs.lstat(receiptPath);if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 4096) return null;const receipt = JSON.parse(await fs.readFile(receiptPath, "utf8"));- if (receipt.version !== 1 || receipt.key !== cache.key) return null;+ if (receipt.version !== CACHE_VERSION || receipt.key !== cache.key) return null;const options = { timeout: 10000, maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) };const bare = await execFile("git", ["-C", cache.directory, "rev-parse", "--is-bare-repository"], options);const origin = await execFile("git", ["-C", cache.directory, "config", "--local", "--null", "--get-all", "remote.origin.url"], options);@@ -67,45 +75,54 @@ export async function readRepositoryObjectCache(cache: RepositoryObjectCache): Pasync function storageBytes(directory: string, ceiling: number, budget: { entries: number; deadline: number }): Promise<number> {let bytes = 0;for await (const entry of await fs.opendir(directory)) {- if (--budget.entries < 0 || performance.now() > budget.deadline) throw new Error("Repository object cache work budget exhausted");+ if (--budget.entries < 0 || performance.now() > budget.deadline) throw new RepositoryObjectCacheSkip("budget");const target = path.join(directory, entry.name);const stat = await fs.lstat(target);- if (stat.isSymbolicLink()) throw new Error("Invalid repository object cache entry");+ if (stat.isSymbolicLink()) throw new RepositoryObjectCacheSkip("invalid");bytes += stat.isDirectory() ? await storageBytes(target, ceiling - bytes, budget) : stat.size;- if (bytes > ceiling) throw new Error("Repository object cache capacity exhausted");+ if (bytes > ceiling) throw new RepositoryObjectCacheSkip("capacity");}return bytes;}export async function publishRepositoryObjectCache(cache: RepositoryObjectCache, checkout: string): Promise<void> {try {- if (await readRepositoryObjectCache(cache)) return;- await tryWithDirectoryMergeLock(cache.companyDirectory, async () => {+ if (await readRepositoryObjectCache(cache)) {+ recordRepositoryObjectCacheDetail({ outcome: "already_present" });+ return;+ }+ const outcome = await tryWithDirectoryMergeLock(cache.companyDirectory, async (): Promise<RepositoryObjectCacheOutcome> => {const company = await fs.lstat(cache.companyDirectory);- if (!company.isDirectory() || company.isSymbolicLink() || await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return;+ if (!company.isDirectory() || company.isSymbolicLink() || await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return "invalid";const budget = { entries: 4096, deadline: performance.now() + 5000 };let repositories = 0, bytes = 0;for await (const entry of await fs.opendir(cache.companyDirectory)) {- if (++repositories >= COMPANY_REPOSITORIES) return;+ if (++repositories >= COMPANY_REPOSITORIES) return "capacity";const directory = path.join(cache.companyDirectory, entry.name);const stat = await fs.lstat(directory);- if (!stat.isDirectory() || stat.isSymbolicLink()) return;+ if (!stat.isDirectory() || stat.isSymbolicLink()) return "invalid";bytes += await storageBytes(directory, COMPANY_BYTES - bytes, budget);}- if (await fs.lstat(cache.directory).then(() => true, () => false)) return;+ if (await fs.lstat(cache.directory).then(() => true, () => false)) return "invalid";const incoming = await storageBytes(path.join(checkout, ".git"), REPOSITORY_BYTES, budget) + 65536;- if (bytes + incoming > COMPANY_BYTES) return;+ if (bytes + incoming > COMPANY_BYTES) return "capacity";assertWorkspaceManifestDiskSpace(cache.companyDirectory, incoming);const temporary = path.join(cache.companyDirectory, `.pending-${randomUUID()}`);try {const options = () => ({ timeout: Math.max(1, Math.ceil(budget.deadline - performance.now())), maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) });- await execFile("git", ["clone", "--bare", "--no-hardlinks", "--", checkout, temporary], options());+ await execFile("git", ["clone", "--mirror", "--no-hardlinks", "--", checkout, temporary], options());await execFile("git", ["-C", temporary, "remote", "set-url", "origin", cache.repoUrl], options());- await fs.writeFile(path.join(temporary, "paperclip-object-cache.json"), JSON.stringify({ version: 1, key: cache.key }));+ await fs.writeFile(path.join(temporary, "paperclip-object-cache.json"), JSON.stringify({ version: CACHE_VERSION, key: cache.key }));const actual = await storageBytes(temporary, REPOSITORY_BYTES, budget);- if (bytes + actual > COMPANY_BYTES) return;+ if (bytes + actual > COMPANY_BYTES) return "capacity";await fs.rename(temporary, cache.directory);+ return "published";} finally { await fs.rm(temporary, { recursive: true, force: true }); }});- } catch {}+ recordRepositoryObjectCacheDetail({ outcome: outcome ?? "contended" });+ } catch (error) {+ const code = (error as NodeJS.ErrnoException)?.code;+ recordRepositoryObjectCacheDetail({ outcome: error instanceof RepositoryObjectCacheSkip+ ? error.outcome : code === "ENOSPC" || code === "workspace_git_scan_failed" ? "capacity" : "error" });+ }}