The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Capture bounded workspace base-ref failure diagnostics (#15719)
Devin Foley7333e4eba8e5
diff --git a/doc/observability.md b/doc/observability.mdindex a7880e3163..9e7433cd6c 100644--- a/doc/observability.md+++ b/doc/observability.md@@ -622,6 +622,20 @@ or `restore_failed`. Unknown values are omitted. Workspace paths and arbitrarypre-restore result data are not included. A later successful run does not, byitself, establish that an earlier failed restore recovered the workspace files.+For `configuration_incomplete` with `workspace_base_ref_unresolved`,+`run_execution` includes `workspaceBaseRefRemoteLookup`, `workspaceBaseRefAuthLookup`,+`workspaceBaseRefFetch`, and `workspaceBaseRefRefResolution` when producer evidence+is available. These closed outcomes distinguish skipped operations, auth lookup+failure, and failed or completed Git commands. Optional `workspaceBaseRefFetchExitCode`+and `workspaceBaseRefRefExitCode` are integers from 0 through 255.+`workspaceBaseRefFetchFailureKind` is a coarse, bounded Git diagnostic category;+unknown, mixed, truncated, or oversized command output remains `unknown`.+A failed fetch that prints “remote ref not found” is an observation, not+authoritative proof of user error. These failures remain reportable. The fields+add no Git calls and do not change authentication, retries, task blocking,+recovery actions, or Sentry filtering. They never contain repository URLs,+configured refs, command output, paths, or credentials.+For `workspace_validation_failed` with `git_worktree_not_reusable`, `run_execution`includes `workspaceValidationReason` and an allowlisted `workspaceValidationReasonCode`:`missing_worktree`, `not_a_git_checkout`, `not_registered`, `wrong_repository_root`,diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 3cdbeb86a4..541540efe0 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -1,5 +1,6 @@import * as conversationContinuation from "../services/conversation-continuation.js";import { environmentRuntimeService } from "../services/environment-runtime.js";+import * as workspaceRuntime from "../services/workspace-runtime.js";import { ensureNativeCompletionContract } from "../services/native-runtime/completion-contracts.js";import { readNativePlanWait, hasCommittedNativePlanWait } from "../services/native-runtime/native-plan-wait.js";import { nativeSha256 } from "../services/native-runtime/canonical.js";@@ -1534,6 +1535,51 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {return { companyId, agentId, runId, wakeupRequestId, issueId };}+ it.each(["owned", "forged"] as const)("keeps unresolved-ref recovery and reporting while persisting only owned diagnostics: %s", async provenance => {+ const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();+ const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "base-ref-heartbeat-"));+ execFileSync("git", ["init", "-b", "master"], { cwd: repoRoot, stdio: "ignore" });+ execFileSync("git", ["-c", "user.name=Fixture", "-c", "[email protected]", "commit", "--allow-empty", "-m", "Fixture"], { cwd: repoRoot, stdio: "ignore" });+ const original = await workspaceRuntime.realizeExecutionWorkspace({+ base: { baseCwd: repoRoot, source: "project_primary", projectId: null, workspaceId: null, repoUrl: null, repoRef: "absent-fixture" },+ config: { workspaceStrategy: { type: "git_worktree" } },+ issue: { id: issueId, identifier: "FIXTURE-1", title: "Fixture" }, agent: { id: agentId, companyId, name: "Fixture" },+ }).catch(error => error);+ expect(original).toBeInstanceOf(workspaceRuntime.UnresolvedWorkspaceBaseRefError);+ const diagnostic = workspaceRuntime.readUnresolvedWorkspaceBaseRefDiagnostic(original);+ expect(diagnostic).toMatchObject({ remoteLookup: "failed", fetch: "not_attempted" });+ const error = provenance === "owned" ? original : Object.assign(new workspaceRuntime.UnresolvedWorkspaceBaseRefError({+ requestedRef: "absent-fixture", recoveryIdentityRef: "origin/absent-fixture", attemptedRefs: ["origin/absent-fixture"],+ }), { baseRefDiagnostic: diagnostic });+ const realize = vi.spyOn(workspaceRuntime, "realizeExecutionWorkspace").mockRejectedValueOnce(error);+ try {+ const heartbeat = heartbeatService(db);+ await heartbeat.resumeQueuedRuns();+ await waitForRunToSettle(heartbeat, runId, 5_000);+ await heartbeat.waitForRunExecutionDrain(runId);+ await waitForPendingRunFailureReports();+ expect(realize).toHaveBeenCalled();+ expect(mockAdapterExecute).not.toHaveBeenCalled();+ const failed = await heartbeat.getRun(runId);+ expect(failed).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { reason: "workspace_base_ref_unresolved" },+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },+ } });+ if (provenance === "owned") expect(failed?.resultJson?.configurationIncomplete).toHaveProperty("baseRefDiagnostic", diagnostic);+ else expect(failed?.resultJson?.configurationIncomplete).not.toHaveProperty("baseRefDiagnostic");+ expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId, errorCode: "configuration_incomplete" }));+ const captured = mockCaptureRunFailure.mock.calls.find(([entry]) => entry.runId === runId)?.[0];+ if (provenance === "owned") expect(captured?.diagnostics.execution).toMatchObject({ workspaceBaseRefRemoteLookup: "failed", workspaceBaseRefFetch: "not_attempted" });+ else expect(captured?.diagnostics.execution).not.toHaveProperty("workspaceBaseRefFetch");+ const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));+ expect(issue).toMatchObject({ status: "blocked", executionRunId: null });+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));+ expect(action).toMatchObject({ status: "active", kind: "configuration_validation", ownerType: "board" });+ const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId));+ expect(wakeup.status).toBe("failed");+ } finally { realize.mockRestore(); await fs.rm(repoRoot, { recursive: true, force: true }); }+ });+it.each(["missing_default", "credential_not_shared", "database_error", "unmarked_http_error", "unmarked_forbidden", "provider_error"] as const)("preserves AI configuration recovery and reports only unexpected causes: %s", async (cause) => {const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.tsindex 5f5e2c180e..53b6c7caea 100644--- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts+++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts@@ -42,6 +42,51 @@ afterEach(async () => {});describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", () => {+ it("sends base-ref observations through the real scrubber and transport without private payloads", async () => {+ const Sentry = sentryPackage!;+ const events: Array<Record<string, unknown>> = [];+ vi.stubEnv("SENTRY_DSN_BACKEND", "https://[email protected]/1");+ vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) }));+ vi.doMock("@sentry/node", () => ({ ...Sentry,+ init: (options: Record<string, unknown>) => Sentry.init({ ...options,+ transport: () => ({+ send: async (envelope: [unknown, Array<[{ type: string }, Record<string, unknown>]>]) => {+ for (const [header, payload] of envelope[1]) if (header.type === "event") events.push(payload);+ return {};+ }, flush: async () => true,+ }),+ }),+ }));+ vi.resetModules();+ const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js");+ await sentryReady;+ const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "private-base-ref",+ fetchError: "private-base-stderr", baseRefDiagnostic: { schemaVersion: 1, remoteLookup: "resolved", authLookup: "failed",+ fetch: "failed", fetchExitCode: 128, fetchFailureKind: "dns_failure", refResolution: "failed", refExitCode: 128,+ repoUrl: "https://private-base.example.invalid", credential: "private-base-token", stderr: "private-base-output",+ },+ },+ } } as unknown as typeof heartbeatRuns.$inferSelect, {}));+ captureRunFailure({ taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222",+ errorMessage: "Configured workspace base ref could not be resolved.", errorCode: "configuration_incomplete",+ agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: null, signal: null, diagnostics,+ });+ captureException(new Error("Unrelated base-ref fixture"));+ await Sentry.flush(2000);+ expect(events).toHaveLength(2);+ const event = events.find(entry => (entry.tags as Record<string, unknown>)?.error_code === "configuration_incomplete");+ expect(event).toMatchObject({ contexts: { run_execution: { workspaceBaseRefRemoteLookup: "resolved", workspaceBaseRefAuthLookup: "failed",+ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchExitCode: 128, workspaceBaseRefFetchFailureKind: "dns_failure",+ workspaceBaseRefRefResolution: "failed", workspaceBaseRefRefExitCode: 128,+ } } });+ expect(JSON.stringify(events)).not.toContain("private-base-");+ const unrelated = events.find(entry => entry !== event);+ expect(unrelated).not.toHaveProperty("contexts.run_execution");+ expect(unrelated).not.toHaveProperty("contexts.run_failure");+ expect(unrelated).not.toHaveProperty("tags.error_code");+ });+it("sends only bounded process-loss evidence without leaking it into another capture", async () => {const Sentry = sentryPackage!;const events: Array<Record<string, unknown>> = [];diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.tsindex 112c30243a..969635018c 100644--- a/server/src/__tests__/workspace-runtime.test.ts+++ b/server/src/__tests__/workspace-runtime.test.ts@@ -42,6 +42,7 @@ import {refreshRemoteTrackingBaseRef,releaseRuntimeServicesForRun,UnresolvedWorkspaceBaseRefError,+ readUnresolvedWorkspaceBaseRefDiagnostic,resetRuntimeServicesForTests,resetRuntimeServicePortReservationsForTests,MANAGED_RUNTIME_PUBLIC_URL_ENV,@@ -291,8 +292,9 @@ async function pushRemoteOnlyBranch(return sha;}-function realizeWorktreeForTest(repoRoot: string, repoRef: string | null) {+function realizeWorktreeForTest(repoRoot: string, repoRef: string | null, resolveGitAuth?: Parameters<typeof realizeExecutionWorkspace>[0]["resolveGitAuth"]) {return realizeExecutionWorkspace({+ resolveGitAuth,base: {baseCwd: repoRoot,source: "project_primary",@@ -1239,6 +1241,15 @@ describe("realizeExecutionWorkspace", () => {expect(unresolved.requestedRef).toBe("fix/does-not-exist");expect(unresolved.recoveryIdentityRef).toBe("origin/fix/does-not-exist");expect(unresolved.attemptedRefs).toEqual(["origin/fix/does-not-exist"]);+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(Object.freeze(unresolved));+ expect(diagnostic).toEqual({ schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested",+ fetch: "failed", fetchExitCode: 128, fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128 });+ diagnostic!.fetch = "succeeded";+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved)?.fetch).toBe("failed");+ expect(JSON.stringify(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved))).not.toContain("fix/");+ const forged = Object.assign(new UnresolvedWorkspaceBaseRefError({ requestedRef: "private-ref",+ recoveryIdentityRef: "private-ref", attemptedRefs: [] }), { baseRefDiagnostic: diagnostic });+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(forged)).toBeNull();// No worktree directory was created for the fresh-create path.await expect(fs.stat(path.join(repoRoot, ".paperclip", "worktrees", "PAP-447-add-worktree-support")),@@ -1299,6 +1310,78 @@ describe("realizeExecutionWorkspace", () => {expect(unresolved.fetchError).toEqual(expect.stringContaining("Could not refresh base ref origin/fix/unreachable"),);+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved)).toEqual({ schemaVersion: 1,+ remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed", fetchExitCode: 128,+ fetchFailureKind: "unknown", refResolution: "failed", refExitCode: 128 });+ });++ it.each(["missing", "empty"] as const)("records the actual lookup and fetch outcome for a %s origin", async (mode) => {+ const { repoRoot } = await createClonedRepoWithRemote();+ await runGit(repoRoot, mode === "missing" ? ["remote", "remove", "origin"] : ["config", "remote.origin.url", ""]);+ let authCalls = 0;+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => { authCalls++; return null; }).catch(error => error);+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);+ if (mode === "missing") {+ expect(error.fetchError).toBeNull();+ expect(authCalls).toBe(0);+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(error)).toEqual({ schemaVersion: 1,+ remoteLookup: "failed", authLookup: "not_requested", fetch: "not_attempted", refResolution: "failed", refExitCode: 128 });+ } else {+ // Git returns the remote name when its URL is empty, then the fetch fails.+ expect(error.fetchError).toBeTruthy();+ expect(authCalls).toBe(1);+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(error)).toEqual({ schemaVersion: 1,+ remoteLookup: "resolved", authLookup: "unavailable", fetch: "failed", fetchExitCode: 128,+ fetchFailureKind: "unknown", refResolution: "failed", refExitCode: 128 });+ }+ });++ it("isolates rejected and unavailable auth lookup evidence during concurrent real Git failures", async () => {+ const results = await Promise.all(["failed", "unavailable"].map(async authLookup => {+ const { repoRoot } = await createClonedRepoWithRemote();+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => {+ if (authLookup === "failed") throw new Error("private-auth-lookup-failure");+ return null;+ }).catch(error => error);+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);+ expect(error.fetchError).not.toContain("private-auth-");+ return { authLookup, diagnostic: readUnresolvedWorkspaceBaseRefDiagnostic(error) };+ }));+ for (const { authLookup, diagnostic } of results) expect(diagnostic).toEqual({ schemaVersion: 1,+ remoteLookup: "resolved", authLookup, fetch: "failed", fetchExitCode: 128,+ fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128 });+ });++ it.each([+ ["authentication_failed", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\n", 128],+ ["dns_failure", "fatal: unable to access 'https://fixture.example.invalid/private-repo': Could not resolve host: fixture.example.invalid\n", 128],+ ["unknown", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\nfatal: unable to create local lock\n", 128],+ ["unknown", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\n" + "private-output".repeat(700), 128],+ [undefined, "", 0],+ ] as const)("records only bounded fetch process evidence (%s)", async (kind, stderr, code) => {+ const { repoRoot } = await createClonedRepoWithRemote();+ await runGit(repoRoot, ["remote", "set-url", "origin", "https://fixture.example.invalid/private-repo"]);+ const bin = path.join(repoRoot, "fixture-bin"), calls = path.join(repoRoot, "fixture-calls.jsonl");+ await fs.mkdir(bin);+ await fs.writeFile(path.join(bin, "git"), `#!${process.execPath}+const fs = require("node:fs");+fs.appendFileSync(${JSON.stringify(calls)}, JSON.stringify(process.argv.slice(2)) + "\\n");+process.stderr.write(${JSON.stringify(stderr)}, () => { process.exitCode = ${code}; });+`);+ await fs.chmod(path.join(bin, "git"), 0o700);+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => ({+ configArgs: [], env: { PATH: `${bin}:${process.env.PATH}` }, source: "server_env",+ })).catch(error => error);+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);+ expect(diagnostic).toEqual({ schemaVersion: 1, remoteLookup: "resolved", authLookup: "resolved",+ fetch: code === 0 ? "succeeded" : "failed", fetchExitCode: code,+ ...(kind ? { fetchFailureKind: kind } : {}), refResolution: "failed", refExitCode: 128 });+ expect(JSON.stringify(diagnostic)).not.toContain("private-");+ expect(JSON.stringify(diagnostic)).not.toContain("example.invalid");+ expect((await fs.readFile(calls, "utf8")).trim().split("\n").map(line => JSON.parse(line))).toEqual([+ ["fetch", "--prune", "origin", "+refs/heads/absent-fixture:refs/remotes/origin/absent-fixture"],+ ]);});it("rejects reusing an empty directory that only looks like a worktree because it sits inside the repo", async () => {diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.tsindex 868e1841e1..0a6708766c 100644--- a/server/src/services/__tests__/run-failure-diagnostics.test.ts+++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts@@ -7,6 +7,49 @@ const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overridesconst collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });describe("run failure diagnostics", () => {+ it("exports only bounded workspace base-ref observations without treating them as proof", () => {+ const resultJson = { configurationIncomplete: { reason: "workspace_base_ref_unresolved",+ requestedRef: "private-ref", fetchError: "private-stderr", baseRefDiagnostic: {+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "failed", fetch: "failed", fetchExitCode: 128,+ fetchFailureKind: "authentication_failed", refResolution: "failed", refExitCode: 128,+ repoUrl: "https://private.example.invalid", stderr: "private-token", credential: "private-credential",+ } } };+ const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson }), {}).execution;+ expect(execution).toEqual({ workspaceBaseRefRemoteLookup: "resolved", workspaceBaseRefAuthLookup: "failed",+ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchExitCode: 128, workspaceBaseRefFetchFailureKind: "authentication_failed",+ workspaceBaseRefRefResolution: "failed", workspaceBaseRefRefExitCode: 128 });+ expect(JSON.stringify(execution)).not.toContain("private");+ expect(collectRunFailureDiagnostics(run({ errorCode: "setup_failed", resultJson }), {}).execution).toEqual({});+ expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { ...resultJson.configurationIncomplete, reason: "other" },+ } }), {}).execution).toEqual({});+ });++ it.each([null, -1, 256, Infinity, NaN, 1.5, "private-value", {}])("omits invalid base-ref process codes (%j)", code => {+ const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: {+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed", fetchExitCode: code,+ refResolution: "failed", refExitCode: code, fetchFailureKind: "private-value",+ } },+ } }), {}).execution;+ expect(execution).not.toHaveProperty("workspaceBaseRefFetchExitCode");+ expect(execution).not.toHaveProperty("workspaceBaseRefRefExitCode");+ expect(execution).not.toHaveProperty("workspaceBaseRefFetchFailureKind");+ });++ it("rejects forged or unknown base-ref shapes without reading accessor payloads", () => {+ const base = { schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed" };+ let reads = 0;+ const accessor = Object.defineProperty({}, "schemaVersion", { get() { reads++; throw new Error("private-accessor"); } });+ for (const baseRefDiagnostic of [null, {}, [], accessor, Object.create(base), { ...base, schemaVersion: 2 },+ { ...base, remoteLookup: "private-url" }, { ...base, authLookup: "private-token" }, { ...base, fetch: "private-command" }]) {+ expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic },+ } }), {}).execution).toEqual({});+ }+ expect(reads).toBe(0);+ });+it("exports only fixed workspace validation codes and bounded inspection evidence", () => {const resultJson = { workspaceValidation: { reason: "git_worktree_not_reusable", reasonCode: "git_inspection_failed",worktreePath: "/private/path", executionWorkspaceId: "private-id", repository: "private-url", message: "private-message",diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.tsindex cf62998235..0c1d83de7b 100644--- a/server/src/services/__tests__/run-failure-report.test.ts+++ b/server/src/services/__tests__/run-failure-report.test.ts@@ -117,6 +117,24 @@ describeEmbeddedPostgres("reportRunFailure", () => {} as unknown as typeof heartbeatRuns.$inferSelect;}+ it.each(["fresh", "resume", "provider_started"] as const)("keeps workspace base-ref observations reportable: %s", async mode => {+ await seedCompanyAndAgent();+ const run = buildRun({ errorCode: "configuration_incomplete", executionStage: "preparing", exitCode: null, signal: null,+ runtimeMode: mode === "resume" ? "native" : "legacy", runnerProfileJson: mode === "resume" ? { nativeExecutionInput: {} } : null,+ resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: {+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "resolved", fetch: "failed", fetchExitCode: 128,+ fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128,+ } }, executionRecovery: { kind: "bootstrap", providerWorkStarted: mode === "provider_started" } },+ });+ const before = structuredClone(run);+ await reportRunFailure(db, run, { phase: "setup" });+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);+ expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ diagnostics: expect.objectContaining({+ execution: expect.objectContaining({ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchFailureKind: "remote_ref_not_found" }),+ }) }));+ expect(run).toEqual(before);+ });+const gitConnection = { schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" };const hermesConnection = { schemaVersion: 1, provider: "hermes_gateway", operation: "create_run", reason: "connection_refused" };diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 89089ed5ff..8e514e11ea 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -589,6 +589,7 @@ import {realizeExecutionWorkspace,releaseRuntimeServicesForRun,isUnresolvedWorkspaceBaseRefError,+ readUnresolvedWorkspaceBaseRefDiagnostic,type ExecutionWorkspaceInput,type RealizedExecutionWorkspace,type RuntimeServiceRef,@@ -850,9 +851,11 @@ function buildUnresolvedWorkspaceBaseRefResultJson(error: UnresolvedWorkspaceBaseRefError,): Record<string, unknown> {const context = parseObject(run.contextSnapshot);+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);return {configurationIncomplete: {reason: "workspace_base_ref_unresolved",+ ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),companyId: run.companyId,agentId: run.agentId,issueId: readNonEmptyString(context.issueId) ?? null,diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.tsindex 71188371e3..a1d7a2a6a3 100644--- a/server/src/services/run-failure-diagnostics.ts+++ b/server/src/services/run-failure-diagnostics.ts@@ -1,3 +1,4 @@+import { readWorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";import type { heartbeatRuns } from "@paperclipai/db";import { readRunCancellation } from "./run-cancellation.js";import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js";@@ -148,6 +149,15 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportif (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;}const result = run.resultJson;+ const configuration = read(result, "configurationIncomplete");+ if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {+ const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));+ if (diagnostic) {+ for (const [key, value] of Object.entries(diagnostic)) {+ if (key !== "schemaVersion") execution[`workspaceBaseRef${key[0]!.toUpperCase()}${key.slice(1)}`] = value;+ }+ }+ }const workspaceValidation = read(result, "workspaceValidation");if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "git_worktree_not_reusable") {execution.workspaceValidationReason = "git_worktree_not_reusable";diff --git a/server/src/services/workspace-base-ref-diagnostics.ts b/server/src/services/workspace-base-ref-diagnostics.tsnew file mode 100644index 0000000000..2a246bfc51--- /dev/null+++ b/server/src/services/workspace-base-ref-diagnostics.ts@@ -0,0 +1,46 @@+import { readConnectionFailure, type GitConnectionFailure } from "@paperclipai/adapter-utils/connection-failure";++/** Observations only: none of these outcomes proves that a ref is absent. */+export interface WorkspaceBaseRefDiagnostic {+ schemaVersion: 1;+ remoteLookup: "not_attempted" | "failed" | "empty" | "resolved";+ authLookup: "not_requested" | "failed" | "unavailable" | "resolved";+ fetch: "not_attempted" | "failed" | "succeeded";+ fetchExitCode?: number;+ fetchFailureKind?: GitConnectionFailure["reason"] | "remote_ref_not_found" | "unknown";+ refResolution?: "failed" | "succeeded" | "spawn_failed";+ refExitCode?: number;+}++function read(value: unknown, key: string): unknown {+ try {+ if (!value || typeof value !== "object") return undefined;+ const descriptor = Object.getOwnPropertyDescriptor(value, key);+ return descriptor && "value" in descriptor ? descriptor.value : undefined;+ } catch { return undefined; }+}++/** Persist and export only closed values, never arguments, output or credentials. */+export function readWorkspaceBaseRefDiagnostic(value: unknown): WorkspaceBaseRefDiagnostic | null {+ if (read(value, "schemaVersion") !== 1) return null;+ const remoteLookup = read(value, "remoteLookup"), authLookup = read(value, "authLookup"), fetch = read(value, "fetch");+ if (remoteLookup !== "not_attempted" && remoteLookup !== "failed" && remoteLookup !== "empty" && remoteLookup !== "resolved") return null;+ if (authLookup !== "not_requested" && authLookup !== "failed" && authLookup !== "unavailable" && authLookup !== "resolved") return null;+ if (fetch !== "not_attempted" && fetch !== "failed" && fetch !== "succeeded") return null;+ const result: WorkspaceBaseRefDiagnostic = { schemaVersion: 1, remoteLookup, authLookup, fetch };+ const resolution = read(value, "refResolution");+ if (resolution === "failed" || resolution === "succeeded" || resolution === "spawn_failed") result.refResolution = resolution;+ for (const key of ["fetchExitCode", "refExitCode"] as const) {+ if (key === "fetchExitCode" && fetch === "not_attempted") continue;+ if (key === "refExitCode" && resolution !== "failed" && resolution !== "succeeded") continue;+ const code = read(value, key);+ if (typeof code === "number" && Number.isInteger(code) && code >= 0 && code <= 255) result[key] = code;+ }+ const kind = read(value, "fetchFailureKind");+ const transport = readConnectionFailure({ schemaVersion: 1, provider: "git", operation: "clone", reason: kind });+ if (fetch === "failed") {+ if (kind === "remote_ref_not_found" || kind === "unknown") result.fetchFailureKind = kind;+ else if (transport?.provider === "git") result.fetchFailureKind = transport.reason;+ }+ return result;+}diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.tsindex f0cfe40289..3adc9581e2 100644--- a/server/src/services/workspace-runtime.ts+++ b/server/src/services/workspace-runtime.ts@@ -67,6 +67,8 @@ import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.import { logActivity } from "./activity-log.js";import { readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js";import { workspaceGitOperationScheduler } from "./workspace-git-operation-scheduler.js";+import { classifyGitCloneFailure } from "./git-connection-failure.js";+import { readWorkspaceBaseRefDiagnostic, type WorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";import {MANAGED_GIT_WORKTREE_REASON_CODES,readManagedGitInspectionDiagnostic,@@ -914,13 +916,17 @@ async function executeProcess(input: {};}-async function runGit(args: string[], cwd: string, opts?: { env?: NodeJS.ProcessEnv }): Promise<string> {+async function runGit(args: string[], cwd: string, opts?: {+ env?: NodeJS.ProcessEnv;+ observe?: (result: Awaited<ReturnType<typeof executeProcess>>) => void;+}): Promise<string> {const proc = await executeProcess({command: "git",args,cwd,env: opts?.env,});+ opts?.observe?.(proc);if (proc.code !== 0) {throw new Error(proc.stderr.trim() || proc.stdout.trim() || `git ${args.join(" ")} failed`);}@@ -971,15 +977,32 @@ export async function refreshRemoteTrackingBaseRef(baseRef: string,resolveGitAuth?: GitRemoteAuthProvider | null,): Promise<string[]> {+ return (await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, baseRef, resolveGitAuth)).warnings;+}++async function refreshRemoteTrackingBaseRefWithDiagnostic(+ repoRoot: string,+ baseRef: string,+ resolveGitAuth?: GitRemoteAuthProvider | null,+): Promise<{ warnings: string[]; diagnostic: WorkspaceBaseRefDiagnostic }> {+ const diagnostic: WorkspaceBaseRefDiagnostic = {+ schemaVersion: 1, remoteLookup: "not_attempted", authLookup: "not_requested", fetch: "not_attempted",+ };const remoteTracking = parseRemoteTrackingRef(baseRef);- if (!remoteTracking) return [];+ if (!remoteTracking) return { warnings: [], diagnostic };const remoteUrl = await runGit(["remote", "get-url", remoteTracking.remote], repoRoot)- .then((value) => value.trim() || null)- .catch(() => null);- if (!remoteUrl) return [];+ .then((value) => {+ diagnostic.remoteLookup = value.trim() ? "resolved" : "empty";+ return value.trim() || null;+ })+ .catch(() => { diagnostic.remoteLookup = "failed"; return null; });+ if (!remoteUrl) return { warnings: [], diagnostic };- const auth = resolveGitAuth ? await resolveGitAuth(remoteUrl).catch(() => null) : null;+ const auth = resolveGitAuth ? await resolveGitAuth(remoteUrl).then((value) => {+ diagnostic.authLookup = value ? "resolved" : "unavailable";+ return value;+ }).catch(() => { diagnostic.authLookup = "failed"; return null; }) : null;try {await runGit([...(auth?.configArgs ?? []),@@ -987,24 +1010,52 @@ export async function refreshRemoteTrackingBaseRef("--prune",remoteTracking.remote,`+refs/heads/${remoteTracking.branch}:refs/remotes/${remoteTracking.remote}/${remoteTracking.branch}`,- ], repoRoot, auth ? { env: { ...process.env, ...auth.env } } : undefined);- return [];+ ], repoRoot, {+ ...(auth ? { env: { ...process.env, ...auth.env } } : {}),+ observe: (proc) => {+ diagnostic.fetch = proc.code === 0 ? "succeeded" : "failed";+ if (proc.code !== null) diagnostic.fetchExitCode = proc.code;+ if (proc.code === 0) return;+ diagnostic.fetchFailureKind = "unknown";+ // Diagnostic parsing must not add unbounded work to an already failed+ // fetch. Keep the existing captured warning/output limits unchanged.+ if (proc.stdoutTruncated || proc.stderrTruncated || proc.stderrBytes > 8192 || proc.stdoutBytes > 8192) return;+ // Reuse only the closed transport diagnostic parser. This is not a+ // clone failure marker and never changes reporting or ref validation.+ const failure = Object.assign(new Error(), { code: proc.code, stderr: proc.stderr });+ diagnostic.fetchFailureKind = classifyGitCloneFailure(remoteUrl, failure)?.reason ?? "unknown";+ if (proc.code === 128 && !proc.stdout.trim() &&+ proc.stderr.trim() === `fatal: couldn't find remote ref refs/heads/${remoteTracking.branch}`) {+ diagnostic.fetchFailureKind = "remote_ref_not_found";+ }+ },+ });+ return { warnings: [], diagnostic };} catch (error) {+ diagnostic.fetch = "failed";+ diagnostic.fetchFailureKind ??= "unknown";const rawMessage = error instanceof Error ? error.message : String(error);- // Mask URL userinfo (any scheme) and whole URL query strings before the message rides- // warnings that reach run logs.+ // Keep the existing warning contract; new diagnostics never retain this text.const message = rawMessage.replace(/([a-z][a-z0-9+.-]*:\/\/)[^/@\s]+@/gi, "$1***@").replace(/([a-z][a-z0-9+.-]*:\/\/[^\s"'?]*)\?[^\s"']*/gi, "$1?***");const authNote = auth? ` The fetch authenticated with ${auth.secretName ? `the ${auth.secretName} company-secret GitHub credential` : "the server-environment GitHub credential"}, which may have been rejected.`: "";- return [`Could not refresh base ref ${baseRef} before preparing the execution workspace: ${message}${authNote}`];+ return { warnings: [`Could not refresh base ref ${baseRef} before preparing the execution workspace: ${message}${authNote}`], diagnostic };}}-async function resolveBaseRefSha(repoRoot: string, baseRef: string): Promise<string | null> {- return await runGit(["rev-parse", "--verify", `${baseRef}^{commit}`], repoRoot).catch(() => null);+async function resolveBaseRefSha(repoRoot: string, baseRef: string, diagnostic?: WorkspaceBaseRefDiagnostic): Promise<string | null> {+ return await runGit(["rev-parse", "--verify", `${baseRef}^{commit}`], repoRoot, diagnostic ? {+ observe: (proc) => {+ diagnostic.refResolution = proc.code === 0 ? "succeeded" : "failed";+ if (proc.code !== null) diagnostic.refExitCode = proc.code;+ },+ } : undefined).catch(() => {+ if (diagnostic && !diagnostic.refResolution) diagnostic.refResolution = "spawn_failed";+ return null;+ });}function readRecordedBaseRefSha(metadata: Record<string, unknown> | null | undefined): string | null {@@ -2413,6 +2464,13 @@ export class UnresolvedWorkspaceBaseRefError extends Error {}}+const unresolvedBaseRefDiagnostics = new WeakMap<Error, WorkspaceBaseRefDiagnostic>();++/** Reads only the actual resolver's private receipt, not similarly named fields. */+export function readUnresolvedWorkspaceBaseRefDiagnostic(error: unknown): WorkspaceBaseRefDiagnostic | null {+ return error instanceof Error ? readWorkspaceBaseRefDiagnostic(unresolvedBaseRefDiagnostics.get(error)) : null;+}+export function isUnresolvedWorkspaceBaseRefError(error: unknown): error is UnresolvedWorkspaceBaseRefError {return error instanceof UnresolvedWorkspaceBaseRefError;}@@ -2432,6 +2490,7 @@ type AuthoritativeBaseRefResolution =attemptedRefs: string[];warnings: string[];fetchError: string | null;+ diagnostic: WorkspaceBaseRefDiagnostic;};// Resolve the authoritative base ref for a fresh worktree. A configured local@@ -2474,9 +2533,9 @@ async function resolveAuthoritativeBaseRef(return { resolved: true, baseRef: configured, warnings, refreshed: false };}if (remoteTracking && await remoteExists(repoRoot, remoteTracking.remote)) {- const fetchWarnings = await refreshRemoteTrackingBaseRef(repoRoot, configured, resolveGitAuth);+ const { warnings: fetchWarnings, diagnostic } = await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, configured, resolveGitAuth);warnings.push(...fetchWarnings);- if (await resolveBaseRefSha(repoRoot, configured)) {+ if (await resolveBaseRefSha(repoRoot, configured, diagnostic)) {return { resolved: true, baseRef: configured, warnings, refreshed: true };}// Build the recovery identity from the parsed remote and branch. The raw@@ -2491,6 +2550,7 @@ async function resolveAuthoritativeBaseRef(attemptedRefs: [configured],warnings,fetchError: fetchWarnings[0] ?? null,+ diagnostic,};}@@ -2518,9 +2578,9 @@ async function resolveAuthoritativeBaseRef(return { resolved: true, baseRef: configured, warnings, refreshed: false };}const remoteCandidate = `origin/${configured}`;- const fetchWarnings = await refreshRemoteTrackingBaseRef(repoRoot, remoteCandidate, resolveGitAuth);+ const { warnings: fetchWarnings, diagnostic } = await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, remoteCandidate, resolveGitAuth);warnings.push(...fetchWarnings);- if (await resolveBaseRefSha(repoRoot, remoteCandidate)) {+ if (await resolveBaseRefSha(repoRoot, remoteCandidate, diagnostic)) {return { resolved: true, baseRef: remoteCandidate, warnings, refreshed: true };}return {@@ -2530,6 +2590,7 @@ async function resolveAuthoritativeBaseRef(attemptedRefs: [remoteCandidate],warnings,fetchError: fetchWarnings[0] ?? null,+ diagnostic,};}@@ -3574,12 +3635,14 @@ export async function realizeExecutionWorkspace(input: {// `fatal: invalid reference`. Stop here instead and raise a pre-dispatch// configuration failure that the setup catch routes to a human owner.if (!baseRefResolution.resolved) {- throw new UnresolvedWorkspaceBaseRefError({+ const error = new UnresolvedWorkspaceBaseRefError({requestedRef: baseRefResolution.requestedRef,recoveryIdentityRef: baseRefResolution.recoveryIdentityRef,attemptedRefs: baseRefResolution.attemptedRefs,fetchError: baseRefResolution.fetchError,});+ unresolvedBaseRefDiagnostics.set(error, readWorkspaceBaseRefDiagnostic(baseRefResolution.diagnostic)!);+ throw error;}let branchCreatedByRuntime = true;