Skip to content

The open-source app everyone uses to manage agents at work

Capture bounded workspace base-ref failure diagnostics (#15719)

Devin Foley7333e4eba8e5
diff --git a/doc/observability.md b/doc/observability.md
index a7880e3163..9e7433cd6c 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -622,6 +622,20 @@ or `restore_failed`. Unknown values are omitted. Workspace paths and arbitrary
pre-restore result data are not included. A later successful run does not, by
itself, establish that an earlier failed restore recovered the workspace files.
+For `configuration_incomplete` with `workspace_base_ref_unresolved`,
+`run_execution` includes `workspaceBaseRefRemoteLookup`, `workspaceBaseRefAuthLookup`,
+`workspaceBaseRefFetch`, and `workspaceBaseRefRefResolution` when producer evidence
+is available. These closed outcomes distinguish skipped operations, auth lookup
+failure, and failed or completed Git commands. Optional `workspaceBaseRefFetchExitCode`
+and `workspaceBaseRefRefExitCode` are integers from 0 through 255.
+`workspaceBaseRefFetchFailureKind` is a coarse, bounded Git diagnostic category;
+unknown, mixed, truncated, or oversized command output remains `unknown`.
+A failed fetch that prints “remote ref not found” is an observation, not
+authoritative proof of user error. These failures remain reportable. The fields
+add no Git calls and do not change authentication, retries, task blocking,
+recovery actions, or Sentry filtering. They never contain repository URLs,
+configured refs, command output, paths, or credentials.
+
For `workspace_validation_failed` with `git_worktree_not_reusable`, `run_execution`
includes `workspaceValidationReason` and an allowlisted `workspaceValidationReasonCode`:
`missing_worktree`, `not_a_git_checkout`, `not_registered`, `wrong_repository_root`,
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 3cdbeb86a4..541540efe0 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -1,5 +1,6 @@
import * as conversationContinuation from "../services/conversation-continuation.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
+import * as workspaceRuntime from "../services/workspace-runtime.js";
import { ensureNativeCompletionContract } from "../services/native-runtime/completion-contracts.js";
import { readNativePlanWait, hasCommittedNativePlanWait } from "../services/native-runtime/native-plan-wait.js";
import { nativeSha256 } from "../services/native-runtime/canonical.js";
@@ -1534,6 +1535,51 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, runId, wakeupRequestId, issueId };
}
+ it.each(["owned", "forged"] as const)("keeps unresolved-ref recovery and reporting while persisting only owned diagnostics: %s", async provenance => {
+ const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();
+ const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "base-ref-heartbeat-"));
+ execFileSync("git", ["init", "-b", "master"], { cwd: repoRoot, stdio: "ignore" });
+ execFileSync("git", ["-c", "user.name=Fixture", "-c", "[email protected]", "commit", "--allow-empty", "-m", "Fixture"], { cwd: repoRoot, stdio: "ignore" });
+ const original = await workspaceRuntime.realizeExecutionWorkspace({
+ base: { baseCwd: repoRoot, source: "project_primary", projectId: null, workspaceId: null, repoUrl: null, repoRef: "absent-fixture" },
+ config: { workspaceStrategy: { type: "git_worktree" } },
+ issue: { id: issueId, identifier: "FIXTURE-1", title: "Fixture" }, agent: { id: agentId, companyId, name: "Fixture" },
+ }).catch(error => error);
+ expect(original).toBeInstanceOf(workspaceRuntime.UnresolvedWorkspaceBaseRefError);
+ const diagnostic = workspaceRuntime.readUnresolvedWorkspaceBaseRefDiagnostic(original);
+ expect(diagnostic).toMatchObject({ remoteLookup: "failed", fetch: "not_attempted" });
+ const error = provenance === "owned" ? original : Object.assign(new workspaceRuntime.UnresolvedWorkspaceBaseRefError({
+ requestedRef: "absent-fixture", recoveryIdentityRef: "origin/absent-fixture", attemptedRefs: ["origin/absent-fixture"],
+ }), { baseRefDiagnostic: diagnostic });
+ const realize = vi.spyOn(workspaceRuntime, "realizeExecutionWorkspace").mockRejectedValueOnce(error);
+ try {
+ const heartbeat = heartbeatService(db);
+ await heartbeat.resumeQueuedRuns();
+ await waitForRunToSettle(heartbeat, runId, 5_000);
+ await heartbeat.waitForRunExecutionDrain(runId);
+ await waitForPendingRunFailureReports();
+ expect(realize).toHaveBeenCalled();
+ expect(mockAdapterExecute).not.toHaveBeenCalled();
+ const failed = await heartbeat.getRun(runId);
+ expect(failed).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { reason: "workspace_base_ref_unresolved" },
+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
+ } });
+ if (provenance === "owned") expect(failed?.resultJson?.configurationIncomplete).toHaveProperty("baseRefDiagnostic", diagnostic);
+ else expect(failed?.resultJson?.configurationIncomplete).not.toHaveProperty("baseRefDiagnostic");
+ expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId, errorCode: "configuration_incomplete" }));
+ const captured = mockCaptureRunFailure.mock.calls.find(([entry]) => entry.runId === runId)?.[0];
+ if (provenance === "owned") expect(captured?.diagnostics.execution).toMatchObject({ workspaceBaseRefRemoteLookup: "failed", workspaceBaseRefFetch: "not_attempted" });
+ else expect(captured?.diagnostics.execution).not.toHaveProperty("workspaceBaseRefFetch");
+ const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
+ expect(issue).toMatchObject({ status: "blocked", executionRunId: null });
+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
+ expect(action).toMatchObject({ status: "active", kind: "configuration_validation", ownerType: "board" });
+ const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId));
+ expect(wakeup.status).toBe("failed");
+ } finally { realize.mockRestore(); await fs.rm(repoRoot, { recursive: true, force: true }); }
+ });
+
it.each(["missing_default", "credential_not_shared", "database_error", "unmarked_http_error", "unmarked_forbidden", "provider_error"] as const)(
"preserves AI configuration recovery and reports only unexpected causes: %s", async (cause) => {
const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();
diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
index 5f5e2c180e..53b6c7caea 100644
--- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
+++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
@@ -42,6 +42,51 @@ afterEach(async () => {
});
describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK", () => {
+ it("sends base-ref observations through the real scrubber and transport without private payloads", async () => {
+ const Sentry = sentryPackage!;
+ const events: Array<Record<string, unknown>> = [];
+ vi.stubEnv("SENTRY_DSN_BACKEND", "https://[email protected]/1");
+ vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) }));
+ vi.doMock("@sentry/node", () => ({ ...Sentry,
+ init: (options: Record<string, unknown>) => Sentry.init({ ...options,
+ transport: () => ({
+ send: async (envelope: [unknown, Array<[{ type: string }, Record<string, unknown>]>]) => {
+ for (const [header, payload] of envelope[1]) if (header.type === "event") events.push(payload);
+ return {};
+ }, flush: async () => true,
+ }),
+ }),
+ }));
+ vi.resetModules();
+ const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js");
+ await sentryReady;
+ const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "private-base-ref",
+ fetchError: "private-base-stderr", baseRefDiagnostic: { schemaVersion: 1, remoteLookup: "resolved", authLookup: "failed",
+ fetch: "failed", fetchExitCode: 128, fetchFailureKind: "dns_failure", refResolution: "failed", refExitCode: 128,
+ repoUrl: "https://private-base.example.invalid", credential: "private-base-token", stderr: "private-base-output",
+ },
+ },
+ } } as unknown as typeof heartbeatRuns.$inferSelect, {}));
+ captureRunFailure({ taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222",
+ errorMessage: "Configured workspace base ref could not be resolved.", errorCode: "configuration_incomplete",
+ agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: null, signal: null, diagnostics,
+ });
+ captureException(new Error("Unrelated base-ref fixture"));
+ await Sentry.flush(2000);
+ expect(events).toHaveLength(2);
+ const event = events.find(entry => (entry.tags as Record<string, unknown>)?.error_code === "configuration_incomplete");
+ expect(event).toMatchObject({ contexts: { run_execution: { workspaceBaseRefRemoteLookup: "resolved", workspaceBaseRefAuthLookup: "failed",
+ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchExitCode: 128, workspaceBaseRefFetchFailureKind: "dns_failure",
+ workspaceBaseRefRefResolution: "failed", workspaceBaseRefRefExitCode: 128,
+ } } });
+ expect(JSON.stringify(events)).not.toContain("private-base-");
+ const unrelated = events.find(entry => entry !== event);
+ expect(unrelated).not.toHaveProperty("contexts.run_execution");
+ expect(unrelated).not.toHaveProperty("contexts.run_failure");
+ expect(unrelated).not.toHaveProperty("tags.error_code");
+ });
+
it("sends only bounded process-loss evidence without leaking it into another capture", async () => {
const Sentry = sentryPackage!;
const events: Array<Record<string, unknown>> = [];
diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts
index 112c30243a..969635018c 100644
--- a/server/src/__tests__/workspace-runtime.test.ts
+++ b/server/src/__tests__/workspace-runtime.test.ts
@@ -42,6 +42,7 @@ import {
refreshRemoteTrackingBaseRef,
releaseRuntimeServicesForRun,
UnresolvedWorkspaceBaseRefError,
+ readUnresolvedWorkspaceBaseRefDiagnostic,
resetRuntimeServicesForTests,
resetRuntimeServicePortReservationsForTests,
MANAGED_RUNTIME_PUBLIC_URL_ENV,
@@ -291,8 +292,9 @@ async function pushRemoteOnlyBranch(
return sha;
}
-function realizeWorktreeForTest(repoRoot: string, repoRef: string | null) {
+function realizeWorktreeForTest(repoRoot: string, repoRef: string | null, resolveGitAuth?: Parameters<typeof realizeExecutionWorkspace>[0]["resolveGitAuth"]) {
return realizeExecutionWorkspace({
+ resolveGitAuth,
base: {
baseCwd: repoRoot,
source: "project_primary",
@@ -1239,6 +1241,15 @@ describe("realizeExecutionWorkspace", () => {
expect(unresolved.requestedRef).toBe("fix/does-not-exist");
expect(unresolved.recoveryIdentityRef).toBe("origin/fix/does-not-exist");
expect(unresolved.attemptedRefs).toEqual(["origin/fix/does-not-exist"]);
+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(Object.freeze(unresolved));
+ expect(diagnostic).toEqual({ schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested",
+ fetch: "failed", fetchExitCode: 128, fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128 });
+ diagnostic!.fetch = "succeeded";
+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved)?.fetch).toBe("failed");
+ expect(JSON.stringify(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved))).not.toContain("fix/");
+ const forged = Object.assign(new UnresolvedWorkspaceBaseRefError({ requestedRef: "private-ref",
+ recoveryIdentityRef: "private-ref", attemptedRefs: [] }), { baseRefDiagnostic: diagnostic });
+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(forged)).toBeNull();
// No worktree directory was created for the fresh-create path.
await expect(
fs.stat(path.join(repoRoot, ".paperclip", "worktrees", "PAP-447-add-worktree-support")),
@@ -1299,6 +1310,78 @@ describe("realizeExecutionWorkspace", () => {
expect(unresolved.fetchError).toEqual(
expect.stringContaining("Could not refresh base ref origin/fix/unreachable"),
);
+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(unresolved)).toEqual({ schemaVersion: 1,
+ remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed", fetchExitCode: 128,
+ fetchFailureKind: "unknown", refResolution: "failed", refExitCode: 128 });
+ });
+
+ it.each(["missing", "empty"] as const)("records the actual lookup and fetch outcome for a %s origin", async (mode) => {
+ const { repoRoot } = await createClonedRepoWithRemote();
+ await runGit(repoRoot, mode === "missing" ? ["remote", "remove", "origin"] : ["config", "remote.origin.url", ""]);
+ let authCalls = 0;
+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => { authCalls++; return null; }).catch(error => error);
+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);
+ if (mode === "missing") {
+ expect(error.fetchError).toBeNull();
+ expect(authCalls).toBe(0);
+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(error)).toEqual({ schemaVersion: 1,
+ remoteLookup: "failed", authLookup: "not_requested", fetch: "not_attempted", refResolution: "failed", refExitCode: 128 });
+ } else {
+ // Git returns the remote name when its URL is empty, then the fetch fails.
+ expect(error.fetchError).toBeTruthy();
+ expect(authCalls).toBe(1);
+ expect(readUnresolvedWorkspaceBaseRefDiagnostic(error)).toEqual({ schemaVersion: 1,
+ remoteLookup: "resolved", authLookup: "unavailable", fetch: "failed", fetchExitCode: 128,
+ fetchFailureKind: "unknown", refResolution: "failed", refExitCode: 128 });
+ }
+ });
+
+ it("isolates rejected and unavailable auth lookup evidence during concurrent real Git failures", async () => {
+ const results = await Promise.all(["failed", "unavailable"].map(async authLookup => {
+ const { repoRoot } = await createClonedRepoWithRemote();
+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => {
+ if (authLookup === "failed") throw new Error("private-auth-lookup-failure");
+ return null;
+ }).catch(error => error);
+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);
+ expect(error.fetchError).not.toContain("private-auth-");
+ return { authLookup, diagnostic: readUnresolvedWorkspaceBaseRefDiagnostic(error) };
+ }));
+ for (const { authLookup, diagnostic } of results) expect(diagnostic).toEqual({ schemaVersion: 1,
+ remoteLookup: "resolved", authLookup, fetch: "failed", fetchExitCode: 128,
+ fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128 });
+ });
+
+ it.each([
+ ["authentication_failed", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\n", 128],
+ ["dns_failure", "fatal: unable to access 'https://fixture.example.invalid/private-repo': Could not resolve host: fixture.example.invalid\n", 128],
+ ["unknown", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\nfatal: unable to create local lock\n", 128],
+ ["unknown", "fatal: Authentication failed for 'https://fixture.example.invalid/private-repo'\n" + "private-output".repeat(700), 128],
+ [undefined, "", 0],
+ ] as const)("records only bounded fetch process evidence (%s)", async (kind, stderr, code) => {
+ const { repoRoot } = await createClonedRepoWithRemote();
+ await runGit(repoRoot, ["remote", "set-url", "origin", "https://fixture.example.invalid/private-repo"]);
+ const bin = path.join(repoRoot, "fixture-bin"), calls = path.join(repoRoot, "fixture-calls.jsonl");
+ await fs.mkdir(bin);
+ await fs.writeFile(path.join(bin, "git"), `#!${process.execPath}
+const fs = require("node:fs");
+fs.appendFileSync(${JSON.stringify(calls)}, JSON.stringify(process.argv.slice(2)) + "\\n");
+process.stderr.write(${JSON.stringify(stderr)}, () => { process.exitCode = ${code}; });
+`);
+ await fs.chmod(path.join(bin, "git"), 0o700);
+ const error = await realizeWorktreeForTest(repoRoot, "absent-fixture", async () => ({
+ configArgs: [], env: { PATH: `${bin}:${process.env.PATH}` }, source: "server_env",
+ })).catch(error => error);
+ expect(error).toBeInstanceOf(UnresolvedWorkspaceBaseRefError);
+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);
+ expect(diagnostic).toEqual({ schemaVersion: 1, remoteLookup: "resolved", authLookup: "resolved",
+ fetch: code === 0 ? "succeeded" : "failed", fetchExitCode: code,
+ ...(kind ? { fetchFailureKind: kind } : {}), refResolution: "failed", refExitCode: 128 });
+ expect(JSON.stringify(diagnostic)).not.toContain("private-");
+ expect(JSON.stringify(diagnostic)).not.toContain("example.invalid");
+ expect((await fs.readFile(calls, "utf8")).trim().split("\n").map(line => JSON.parse(line))).toEqual([
+ ["fetch", "--prune", "origin", "+refs/heads/absent-fixture:refs/remotes/origin/absent-fixture"],
+ ]);
});
it("rejects reusing an empty directory that only looks like a worktree because it sits inside the repo", async () => {
diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts
index 868e1841e1..0a6708766c 100644
--- a/server/src/services/__tests__/run-failure-diagnostics.test.ts
+++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts
@@ -7,6 +7,49 @@ const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides
const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });
describe("run failure diagnostics", () => {
+ it("exports only bounded workspace base-ref observations without treating them as proof", () => {
+ const resultJson = { configurationIncomplete: { reason: "workspace_base_ref_unresolved",
+ requestedRef: "private-ref", fetchError: "private-stderr", baseRefDiagnostic: {
+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "failed", fetch: "failed", fetchExitCode: 128,
+ fetchFailureKind: "authentication_failed", refResolution: "failed", refExitCode: 128,
+ repoUrl: "https://private.example.invalid", stderr: "private-token", credential: "private-credential",
+ } } };
+ const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson }), {}).execution;
+ expect(execution).toEqual({ workspaceBaseRefRemoteLookup: "resolved", workspaceBaseRefAuthLookup: "failed",
+ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchExitCode: 128, workspaceBaseRefFetchFailureKind: "authentication_failed",
+ workspaceBaseRefRefResolution: "failed", workspaceBaseRefRefExitCode: 128 });
+ expect(JSON.stringify(execution)).not.toContain("private");
+ expect(collectRunFailureDiagnostics(run({ errorCode: "setup_failed", resultJson }), {}).execution).toEqual({});
+ expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { ...resultJson.configurationIncomplete, reason: "other" },
+ } }), {}).execution).toEqual({});
+ });
+
+ it.each([null, -1, 256, Infinity, NaN, 1.5, "private-value", {}])("omits invalid base-ref process codes (%j)", code => {
+ const execution = collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: {
+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed", fetchExitCode: code,
+ refResolution: "failed", refExitCode: code, fetchFailureKind: "private-value",
+ } },
+ } }), {}).execution;
+ expect(execution).not.toHaveProperty("workspaceBaseRefFetchExitCode");
+ expect(execution).not.toHaveProperty("workspaceBaseRefRefExitCode");
+ expect(execution).not.toHaveProperty("workspaceBaseRefFetchFailureKind");
+ });
+
+ it("rejects forged or unknown base-ref shapes without reading accessor payloads", () => {
+ const base = { schemaVersion: 1, remoteLookup: "resolved", authLookup: "not_requested", fetch: "failed" };
+ let reads = 0;
+ const accessor = Object.defineProperty({}, "schemaVersion", { get() { reads++; throw new Error("private-accessor"); } });
+ for (const baseRefDiagnostic of [null, {}, [], accessor, Object.create(base), { ...base, schemaVersion: 2 },
+ { ...base, remoteLookup: "private-url" }, { ...base, authLookup: "private-token" }, { ...base, fetch: "private-command" }]) {
+ expect(collectRunFailureDiagnostics(run({ errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic },
+ } }), {}).execution).toEqual({});
+ }
+ expect(reads).toBe(0);
+ });
+
it("exports only fixed workspace validation codes and bounded inspection evidence", () => {
const resultJson = { workspaceValidation: { reason: "git_worktree_not_reusable", reasonCode: "git_inspection_failed",
worktreePath: "/private/path", executionWorkspaceId: "private-id", repository: "private-url", message: "private-message",
diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts
index cf62998235..0c1d83de7b 100644
--- a/server/src/services/__tests__/run-failure-report.test.ts
+++ b/server/src/services/__tests__/run-failure-report.test.ts
@@ -117,6 +117,24 @@ describeEmbeddedPostgres("reportRunFailure", () => {
} as unknown as typeof heartbeatRuns.$inferSelect;
}
+ it.each(["fresh", "resume", "provider_started"] as const)("keeps workspace base-ref observations reportable: %s", async mode => {
+ await seedCompanyAndAgent();
+ const run = buildRun({ errorCode: "configuration_incomplete", executionStage: "preparing", exitCode: null, signal: null,
+ runtimeMode: mode === "resume" ? "native" : "legacy", runnerProfileJson: mode === "resume" ? { nativeExecutionInput: {} } : null,
+ resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: {
+ schemaVersion: 1, remoteLookup: "resolved", authLookup: "resolved", fetch: "failed", fetchExitCode: 128,
+ fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128,
+ } }, executionRecovery: { kind: "bootstrap", providerWorkStarted: mode === "provider_started" } },
+ });
+ const before = structuredClone(run);
+ await reportRunFailure(db, run, { phase: "setup" });
+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
+ expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ diagnostics: expect.objectContaining({
+ execution: expect.objectContaining({ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchFailureKind: "remote_ref_not_found" }),
+ }) }));
+ expect(run).toEqual(before);
+ });
+
const gitConnection = { schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" };
const hermesConnection = { schemaVersion: 1, provider: "hermes_gateway", operation: "create_run", reason: "connection_refused" };
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 89089ed5ff..8e514e11ea 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -589,6 +589,7 @@ import {
realizeExecutionWorkspace,
releaseRuntimeServicesForRun,
isUnresolvedWorkspaceBaseRefError,
+ readUnresolvedWorkspaceBaseRefDiagnostic,
type ExecutionWorkspaceInput,
type RealizedExecutionWorkspace,
type RuntimeServiceRef,
@@ -850,9 +851,11 @@ function buildUnresolvedWorkspaceBaseRefResultJson(
error: UnresolvedWorkspaceBaseRefError,
): Record<string, unknown> {
const context = parseObject(run.contextSnapshot);
+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);
return {
configurationIncomplete: {
reason: "workspace_base_ref_unresolved",
+ ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),
companyId: run.companyId,
agentId: run.agentId,
issueId: readNonEmptyString(context.issueId) ?? null,
diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts
index 71188371e3..a1d7a2a6a3 100644
--- a/server/src/services/run-failure-diagnostics.ts
+++ b/server/src/services/run-failure-diagnostics.ts
@@ -1,3 +1,4 @@
+import { readWorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";
import type { heartbeatRuns } from "@paperclipai/db";
import { readRunCancellation } from "./run-cancellation.js";
import { readProcessLossDiagnostic } from "./process-loss-diagnostics.js";
@@ -148,6 +149,15 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
}
const result = run.resultJson;
+ const configuration = read(result, "configurationIncomplete");
+ if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {
+ const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));
+ if (diagnostic) {
+ for (const [key, value] of Object.entries(diagnostic)) {
+ if (key !== "schemaVersion") execution[`workspaceBaseRef${key[0]!.toUpperCase()}${key.slice(1)}`] = value;
+ }
+ }
+ }
const workspaceValidation = read(result, "workspaceValidation");
if (run.errorCode === "workspace_validation_failed" && read(workspaceValidation, "reason") === "git_worktree_not_reusable") {
execution.workspaceValidationReason = "git_worktree_not_reusable";
diff --git a/server/src/services/workspace-base-ref-diagnostics.ts b/server/src/services/workspace-base-ref-diagnostics.ts
new file mode 100644
index 0000000000..2a246bfc51
--- /dev/null
+++ b/server/src/services/workspace-base-ref-diagnostics.ts
@@ -0,0 +1,46 @@
+import { readConnectionFailure, type GitConnectionFailure } from "@paperclipai/adapter-utils/connection-failure";
+
+/** Observations only: none of these outcomes proves that a ref is absent. */
+export interface WorkspaceBaseRefDiagnostic {
+ schemaVersion: 1;
+ remoteLookup: "not_attempted" | "failed" | "empty" | "resolved";
+ authLookup: "not_requested" | "failed" | "unavailable" | "resolved";
+ fetch: "not_attempted" | "failed" | "succeeded";
+ fetchExitCode?: number;
+ fetchFailureKind?: GitConnectionFailure["reason"] | "remote_ref_not_found" | "unknown";
+ refResolution?: "failed" | "succeeded" | "spawn_failed";
+ refExitCode?: number;
+}
+
+function read(value: unknown, key: string): unknown {
+ try {
+ if (!value || typeof value !== "object") return undefined;
+ const descriptor = Object.getOwnPropertyDescriptor(value, key);
+ return descriptor && "value" in descriptor ? descriptor.value : undefined;
+ } catch { return undefined; }
+}
+
+/** Persist and export only closed values, never arguments, output or credentials. */
+export function readWorkspaceBaseRefDiagnostic(value: unknown): WorkspaceBaseRefDiagnostic | null {
+ if (read(value, "schemaVersion") !== 1) return null;
+ const remoteLookup = read(value, "remoteLookup"), authLookup = read(value, "authLookup"), fetch = read(value, "fetch");
+ if (remoteLookup !== "not_attempted" && remoteLookup !== "failed" && remoteLookup !== "empty" && remoteLookup !== "resolved") return null;
+ if (authLookup !== "not_requested" && authLookup !== "failed" && authLookup !== "unavailable" && authLookup !== "resolved") return null;
+ if (fetch !== "not_attempted" && fetch !== "failed" && fetch !== "succeeded") return null;
+ const result: WorkspaceBaseRefDiagnostic = { schemaVersion: 1, remoteLookup, authLookup, fetch };
+ const resolution = read(value, "refResolution");
+ if (resolution === "failed" || resolution === "succeeded" || resolution === "spawn_failed") result.refResolution = resolution;
+ for (const key of ["fetchExitCode", "refExitCode"] as const) {
+ if (key === "fetchExitCode" && fetch === "not_attempted") continue;
+ if (key === "refExitCode" && resolution !== "failed" && resolution !== "succeeded") continue;
+ const code = read(value, key);
+ if (typeof code === "number" && Number.isInteger(code) && code >= 0 && code <= 255) result[key] = code;
+ }
+ const kind = read(value, "fetchFailureKind");
+ const transport = readConnectionFailure({ schemaVersion: 1, provider: "git", operation: "clone", reason: kind });
+ if (fetch === "failed") {
+ if (kind === "remote_ref_not_found" || kind === "unknown") result.fetchFailureKind = kind;
+ else if (transport?.provider === "git") result.fetchFailureKind = transport.reason;
+ }
+ return result;
+}
diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts
index f0cfe40289..3adc9581e2 100644
--- a/server/src/services/workspace-runtime.ts
+++ b/server/src/services/workspace-runtime.ts
@@ -67,6 +67,8 @@ import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.
import { logActivity } from "./activity-log.js";
import { readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js";
import { workspaceGitOperationScheduler } from "./workspace-git-operation-scheduler.js";
+import { classifyGitCloneFailure } from "./git-connection-failure.js";
+import { readWorkspaceBaseRefDiagnostic, type WorkspaceBaseRefDiagnostic } from "./workspace-base-ref-diagnostics.js";
import {
MANAGED_GIT_WORKTREE_REASON_CODES,
readManagedGitInspectionDiagnostic,
@@ -914,13 +916,17 @@ async function executeProcess(input: {
};
}
-async function runGit(args: string[], cwd: string, opts?: { env?: NodeJS.ProcessEnv }): Promise<string> {
+async function runGit(args: string[], cwd: string, opts?: {
+ env?: NodeJS.ProcessEnv;
+ observe?: (result: Awaited<ReturnType<typeof executeProcess>>) => void;
+}): Promise<string> {
const proc = await executeProcess({
command: "git",
args,
cwd,
env: opts?.env,
});
+ opts?.observe?.(proc);
if (proc.code !== 0) {
throw new Error(proc.stderr.trim() || proc.stdout.trim() || `git ${args.join(" ")} failed`);
}
@@ -971,15 +977,32 @@ export async function refreshRemoteTrackingBaseRef(
baseRef: string,
resolveGitAuth?: GitRemoteAuthProvider | null,
): Promise<string[]> {
+ return (await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, baseRef, resolveGitAuth)).warnings;
+}
+
+async function refreshRemoteTrackingBaseRefWithDiagnostic(
+ repoRoot: string,
+ baseRef: string,
+ resolveGitAuth?: GitRemoteAuthProvider | null,
+): Promise<{ warnings: string[]; diagnostic: WorkspaceBaseRefDiagnostic }> {
+ const diagnostic: WorkspaceBaseRefDiagnostic = {
+ schemaVersion: 1, remoteLookup: "not_attempted", authLookup: "not_requested", fetch: "not_attempted",
+ };
const remoteTracking = parseRemoteTrackingRef(baseRef);
- if (!remoteTracking) return [];
+ if (!remoteTracking) return { warnings: [], diagnostic };
const remoteUrl = await runGit(["remote", "get-url", remoteTracking.remote], repoRoot)
- .then((value) => value.trim() || null)
- .catch(() => null);
- if (!remoteUrl) return [];
+ .then((value) => {
+ diagnostic.remoteLookup = value.trim() ? "resolved" : "empty";
+ return value.trim() || null;
+ })
+ .catch(() => { diagnostic.remoteLookup = "failed"; return null; });
+ if (!remoteUrl) return { warnings: [], diagnostic };
- const auth = resolveGitAuth ? await resolveGitAuth(remoteUrl).catch(() => null) : null;
+ const auth = resolveGitAuth ? await resolveGitAuth(remoteUrl).then((value) => {
+ diagnostic.authLookup = value ? "resolved" : "unavailable";
+ return value;
+ }).catch(() => { diagnostic.authLookup = "failed"; return null; }) : null;
try {
await runGit([
...(auth?.configArgs ?? []),
@@ -987,24 +1010,52 @@ export async function refreshRemoteTrackingBaseRef(
"--prune",
remoteTracking.remote,
`+refs/heads/${remoteTracking.branch}:refs/remotes/${remoteTracking.remote}/${remoteTracking.branch}`,
- ], repoRoot, auth ? { env: { ...process.env, ...auth.env } } : undefined);
- return [];
+ ], repoRoot, {
+ ...(auth ? { env: { ...process.env, ...auth.env } } : {}),
+ observe: (proc) => {
+ diagnostic.fetch = proc.code === 0 ? "succeeded" : "failed";
+ if (proc.code !== null) diagnostic.fetchExitCode = proc.code;
+ if (proc.code === 0) return;
+ diagnostic.fetchFailureKind = "unknown";
+ // Diagnostic parsing must not add unbounded work to an already failed
+ // fetch. Keep the existing captured warning/output limits unchanged.
+ if (proc.stdoutTruncated || proc.stderrTruncated || proc.stderrBytes > 8192 || proc.stdoutBytes > 8192) return;
+ // Reuse only the closed transport diagnostic parser. This is not a
+ // clone failure marker and never changes reporting or ref validation.
+ const failure = Object.assign(new Error(), { code: proc.code, stderr: proc.stderr });
+ diagnostic.fetchFailureKind = classifyGitCloneFailure(remoteUrl, failure)?.reason ?? "unknown";
+ if (proc.code === 128 && !proc.stdout.trim() &&
+ proc.stderr.trim() === `fatal: couldn't find remote ref refs/heads/${remoteTracking.branch}`) {
+ diagnostic.fetchFailureKind = "remote_ref_not_found";
+ }
+ },
+ });
+ return { warnings: [], diagnostic };
} catch (error) {
+ diagnostic.fetch = "failed";
+ diagnostic.fetchFailureKind ??= "unknown";
const rawMessage = error instanceof Error ? error.message : String(error);
- // Mask URL userinfo (any scheme) and whole URL query strings before the message rides
- // warnings that reach run logs.
+ // Keep the existing warning contract; new diagnostics never retain this text.
const message = rawMessage
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^/@\s]+@/gi, "$1***@")
.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s"'?]*)\?[^\s"']*/gi, "$1?***");
const authNote = auth
? ` The fetch authenticated with ${auth.secretName ? `the ${auth.secretName} company-secret GitHub credential` : "the server-environment GitHub credential"}, which may have been rejected.`
: "";
- return [`Could not refresh base ref ${baseRef} before preparing the execution workspace: ${message}${authNote}`];
+ return { warnings: [`Could not refresh base ref ${baseRef} before preparing the execution workspace: ${message}${authNote}`], diagnostic };
}
}
-async function resolveBaseRefSha(repoRoot: string, baseRef: string): Promise<string | null> {
- return await runGit(["rev-parse", "--verify", `${baseRef}^{commit}`], repoRoot).catch(() => null);
+async function resolveBaseRefSha(repoRoot: string, baseRef: string, diagnostic?: WorkspaceBaseRefDiagnostic): Promise<string | null> {
+ return await runGit(["rev-parse", "--verify", `${baseRef}^{commit}`], repoRoot, diagnostic ? {
+ observe: (proc) => {
+ diagnostic.refResolution = proc.code === 0 ? "succeeded" : "failed";
+ if (proc.code !== null) diagnostic.refExitCode = proc.code;
+ },
+ } : undefined).catch(() => {
+ if (diagnostic && !diagnostic.refResolution) diagnostic.refResolution = "spawn_failed";
+ return null;
+ });
}
function readRecordedBaseRefSha(metadata: Record<string, unknown> | null | undefined): string | null {
@@ -2413,6 +2464,13 @@ export class UnresolvedWorkspaceBaseRefError extends Error {
}
}
+const unresolvedBaseRefDiagnostics = new WeakMap<Error, WorkspaceBaseRefDiagnostic>();
+
+/** Reads only the actual resolver's private receipt, not similarly named fields. */
+export function readUnresolvedWorkspaceBaseRefDiagnostic(error: unknown): WorkspaceBaseRefDiagnostic | null {
+ return error instanceof Error ? readWorkspaceBaseRefDiagnostic(unresolvedBaseRefDiagnostics.get(error)) : null;
+}
+
export function isUnresolvedWorkspaceBaseRefError(error: unknown): error is UnresolvedWorkspaceBaseRefError {
return error instanceof UnresolvedWorkspaceBaseRefError;
}
@@ -2432,6 +2490,7 @@ type AuthoritativeBaseRefResolution =
attemptedRefs: string[];
warnings: string[];
fetchError: string | null;
+ diagnostic: WorkspaceBaseRefDiagnostic;
};
// Resolve the authoritative base ref for a fresh worktree. A configured local
@@ -2474,9 +2533,9 @@ async function resolveAuthoritativeBaseRef(
return { resolved: true, baseRef: configured, warnings, refreshed: false };
}
if (remoteTracking && await remoteExists(repoRoot, remoteTracking.remote)) {
- const fetchWarnings = await refreshRemoteTrackingBaseRef(repoRoot, configured, resolveGitAuth);
+ const { warnings: fetchWarnings, diagnostic } = await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, configured, resolveGitAuth);
warnings.push(...fetchWarnings);
- if (await resolveBaseRefSha(repoRoot, configured)) {
+ if (await resolveBaseRefSha(repoRoot, configured, diagnostic)) {
return { resolved: true, baseRef: configured, warnings, refreshed: true };
}
// Build the recovery identity from the parsed remote and branch. The raw
@@ -2491,6 +2550,7 @@ async function resolveAuthoritativeBaseRef(
attemptedRefs: [configured],
warnings,
fetchError: fetchWarnings[0] ?? null,
+ diagnostic,
};
}
@@ -2518,9 +2578,9 @@ async function resolveAuthoritativeBaseRef(
return { resolved: true, baseRef: configured, warnings, refreshed: false };
}
const remoteCandidate = `origin/${configured}`;
- const fetchWarnings = await refreshRemoteTrackingBaseRef(repoRoot, remoteCandidate, resolveGitAuth);
+ const { warnings: fetchWarnings, diagnostic } = await refreshRemoteTrackingBaseRefWithDiagnostic(repoRoot, remoteCandidate, resolveGitAuth);
warnings.push(...fetchWarnings);
- if (await resolveBaseRefSha(repoRoot, remoteCandidate)) {
+ if (await resolveBaseRefSha(repoRoot, remoteCandidate, diagnostic)) {
return { resolved: true, baseRef: remoteCandidate, warnings, refreshed: true };
}
return {
@@ -2530,6 +2590,7 @@ async function resolveAuthoritativeBaseRef(
attemptedRefs: [remoteCandidate],
warnings,
fetchError: fetchWarnings[0] ?? null,
+ diagnostic,
};
}
@@ -3574,12 +3635,14 @@ export async function realizeExecutionWorkspace(input: {
// `fatal: invalid reference`. Stop here instead and raise a pre-dispatch
// configuration failure that the setup catch routes to a human owner.
if (!baseRefResolution.resolved) {
- throw new UnresolvedWorkspaceBaseRefError({
+ const error = new UnresolvedWorkspaceBaseRefError({
requestedRef: baseRefResolution.requestedRef,
recoveryIdentityRef: baseRefResolution.recoveryIdentityRef,
attemptedRefs: baseRefResolution.attemptedRefs,
fetchError: baseRefResolution.fetchError,
});
+ unresolvedBaseRefDiagnostics.set(error, readWorkspaceBaseRefDiagnostic(baseRefResolution.diagnostic)!);
+ throw error;
}
let branchCreatedByRuntime = true;