Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): preserve typed agent startup failures (#124662)

Alessandro Pogliaghi1016d50bc54f
diff --git a/docs/internal/task-run-api.md b/docs/internal/task-run-api.md
index 1a3e54028950..e749b133285a 100644
--- a/docs/internal/task-run-api.md
+++ b/docs/internal/task-run-api.md
@@ -26,6 +26,18 @@ Keep the task ID and inspect its runs before you retry.
Do not repeat task creation to recover from a run failure.
If a database read fails after dispatch, the response still includes the created run.
+The worker persists agent startup failures in `latest_run.state.startup_failure` from the failed readiness response for that run.
+The object contains `provider`, `phase`, nullable `reason`, and `retryability` (`requires_action`, `transient`, or `unknown`).
+Only validated provider reason codes can stop startup activity retries; log text is not used to classify them.
+Failures that require action include the configuration problem and a recovery step in the run's error message.
+`requires_action` means the current attempt needs a configuration or credential change before retrying, not that recovery is impossible.
+Unknown failures keep the existing retry policy. Credit exhaustion is a separate gateway-token refusal: a new attempt rechecks credit availability after a top-up; topping up does not automatically resume a failed run.
+Temporary Codex credential-service outages remain retryable. Missing or invalid subscription credentials preserve the provider's reconnect or token-relay instructions.
+The worker records `agent_startup_attempt` immediately before a real launch and `agent_startup_success` after readiness succeeds.
+Successful initialization clears the startup failure and clears the crash message while the run is non-terminal. A terminal run keeps its final error. A later failed attempt preserves the last completed startup as recovery evidence.
+These startup state fields cannot be written, appended to, or removed through the run PATCH API, including by sandbox clients.
+Older agents may omit these fields; consumers must preserve the existing behavior when they are absent or invalid.
+
`branch` selects the base branch for the first run.
Optional run hints can be `null`.
Pi runs reject ACP runtime, model, reasoning-effort, and permission-mode hints.
diff --git a/packages/agent/packages/agent-contracts/src/startup-failure.ts b/packages/agent/packages/agent-contracts/src/startup-failure.ts
new file mode 100644
index 000000000000..4e1adff84b12
--- /dev/null
+++ b/packages/agent/packages/agent-contracts/src/startup-failure.ts
@@ -0,0 +1,40 @@
+import { z } from "zod";
+
+export const startupFailureSchema = z.object({
+ provider: z.enum(["claude", "codex", "pi"]),
+ phase: z.enum(["session_setup", "sdk_initialization", "setup_hooks"]),
+ reason: z
+ .enum([
+ "credential_unavailable",
+ "credential_service_unavailable",
+ "org_pin_api_key_conflict",
+ "org_verify_failed",
+ "org_pin_mismatch",
+ "managed_settings_invalid",
+ "remote_settings_required_unavailable",
+ "gateway_signin_required",
+ "gateway_access_denied",
+ "proxy_invalid",
+ "temp_dir_unusable",
+ "cwd_unavailable",
+ "shell_tool_missing",
+ "session_held_by_background",
+ "worktree_resume_refused",
+ "worktree_unverified",
+ "cli_version_too_old",
+ "bypass_root",
+ ])
+ .nullable(),
+ retryability: z.enum(["requires_action", "transient", "unknown"]),
+});
+
+export type StartupFailure = z.infer<typeof startupFailureSchema>;
+
+const startupErrorSchema = z.object({
+ data: z.object({ startup_failure: startupFailureSchema }),
+});
+
+export function readStartupFailure(error: unknown): StartupFailure | undefined {
+ const parsed = startupErrorSchema.safeParse(error);
+ return parsed.success ? parsed.data.data.startup_failure : undefined;
+}
diff --git a/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts b/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts
index 83fe1765557a..4f5fd13c7f65 100644
--- a/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts
+++ b/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts
@@ -3046,7 +3046,11 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
if (!error.message.startsWith("Session ")) {
error.message = `Session ${step} failed: ${error.message}`;
}
- return error;
+ if (!("startup_failure" in errorData)) return error;
+ return new RequestError(error.code, error.message, {
+ ...errorData,
+ ...(error.data && typeof error.data === "object" ? error.data : {}),
+ });
}
return new RequestError(-32603, `Session ${step} failed`, errorData);
}
@@ -3451,6 +3455,7 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
},
);
throw this.startupFailure(forkSession ? "fork" : "resumption", err, {
+ startup_failure: initialization.failure,
sessionId,
taskId,
taskRunId: meta?.taskRunId,
@@ -3462,7 +3467,10 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
// with the model config fetch below (the gateway REST call is independent).
const initStartedAt = Date.now();
const initPromise = !isResume
- ? initialization.wait(q.initializationResult())
+ ? initialization.wait(q.initializationResult()).then(
+ (result) => ({ ok: true as const, result }),
+ (error: unknown) => ({ ok: false as const, error }),
+ )
: undefined;
const [rawModelOptions] = await Promise.all([
@@ -3499,7 +3507,9 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
let timeoutMs = SESSION_VALIDATION_TIMEOUT_MS;
let cliOutput: CliOutputSummary | undefined;
try {
- const initResult = await initPromise;
+ const initializationResult = await initPromise;
+ if (!initializationResult.ok) throw initializationResult.error;
+ const initResult = initializationResult.result;
if (initResult.result === "timeout") {
initializationPhase = initResult.phase;
timeoutMs = initResult.timeoutMs;
@@ -3548,7 +3558,12 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
? "setup hooks"
: "initialization",
err,
- { sessionId, taskId, taskRunId: meta?.taskRunId },
+ {
+ sessionId,
+ taskId,
+ taskRunId: meta?.taskRunId,
+ startup_failure: initialization.failure,
+ },
);
}
}
diff --git a/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts b/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts
index d5b6049734c3..2173073e983e 100644
--- a/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts
+++ b/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts
@@ -136,4 +136,55 @@ describe("SessionInitialization", () => {
expect(stdout.listenerCount("data")).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
+
+ it.each([
+ ["proxy_invalid", "requires_action"],
+ ["worktree_unverified", "transient"],
+ ["org_verify_failed", "unknown"],
+ ["future_provider_reason", "unknown"],
+ [null, "unknown"],
+ ])(
+ "preserves structured %s through an SDK rejection",
+ async (reason, retryability) => {
+ const stdout = new PassThrough();
+ const monitor = new SessionInitialization(vi.fn());
+ monitor.observe(stdout);
+ const line = `${JSON.stringify({
+ type: "result",
+ subtype: "error_during_execution",
+ startup_failure_reason: reason,
+ })}\n`;
+ stdout.write(line.slice(0, 19));
+ stdout.write(line.slice(19));
+
+ await expect(
+ monitor.wait(Promise.reject(new Error("SDK initialization failed"))),
+ ).rejects.toThrow("SDK initialization failed");
+
+ expect(monitor.failure).toEqual({
+ provider: "claude",
+ phase: "sdk_initialization",
+ reason: reason === "future_provider_reason" ? null : reason,
+ retryability,
+ });
+ expect(stdout.listenerCount("data")).toBe(0);
+ },
+ );
+
+ it.each([
+ "proxy_invalid: the proxy setting is invalid\n",
+ `${JSON.stringify({ type: "assistant", startup_failure_reason: "proxy_invalid" })}\n`,
+ `${JSON.stringify({ type: "result", subtype: "success", startup_failure_reason: "proxy_invalid" })}\n`,
+ `${JSON.stringify({ type: "result", subtype: "error_during_execution", errors: ["proxy_invalid"] })}\n`,
+ ])("does not classify unrelated output %s", async (output) => {
+ const stdout = new PassThrough();
+ const monitor = new SessionInitialization(vi.fn());
+ monitor.observe(stdout);
+ stdout.write(output);
+ expect(monitor.failure).toMatchObject({
+ reason: null,
+ retryability: "unknown",
+ });
+ await monitor.wait(Promise.resolve());
+ });
});
diff --git a/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts b/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts
index d0286306f538..35d8c737d960 100644
--- a/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts
+++ b/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts
@@ -1,5 +1,8 @@
import type { Readable } from "node:stream";
import { StringDecoder } from "node:string_decoder";
+import type { SDKStartupFailureReason } from "@anthropic-ai/claude-agent-sdk";
+import type { StartupFailure } from "@posthog/agent-contracts/startup-failure";
+import { z } from "zod";
export type InitializationPhase = "sdk_initialization" | "setup_hooks";
@@ -22,6 +25,36 @@ type InitializationResult<Value> =
const MAX_OBSERVED_LINE_LENGTH = 256 * 1024;
const MAX_MESSAGE_TYPE_LENGTH = 100;
+const startupRetryability = {
+ org_pin_api_key_conflict: "requires_action",
+ org_verify_failed: "unknown",
+ org_pin_mismatch: "requires_action",
+ managed_settings_invalid: "requires_action",
+ remote_settings_required_unavailable: "unknown",
+ gateway_signin_required: "requires_action",
+ gateway_access_denied: "requires_action",
+ proxy_invalid: "requires_action",
+ temp_dir_unusable: "unknown",
+ cwd_unavailable: "unknown",
+ shell_tool_missing: "requires_action",
+ session_held_by_background: "transient",
+ worktree_resume_refused: "unknown",
+ worktree_unverified: "transient",
+ cli_version_too_old: "requires_action",
+ bypass_root: "requires_action",
+} satisfies Record<SDKStartupFailureReason, StartupFailure["retryability"]>;
+
+const startupResultSchema = z.object({
+ type: z.literal("result"),
+ subtype: z.literal("error_during_execution"),
+ startup_failure_reason: z.enum(
+ Object.keys(startupRetryability) as [
+ SDKStartupFailureReason,
+ ...SDKStartupFailureReason[],
+ ],
+ ),
+});
+
export class SessionInitialization {
private activeHooks = new Set<string>();
private hooksStartedAt: number | undefined;
@@ -32,6 +65,7 @@ export class SessionInitialization {
private reschedule: (() => void) | undefined;
private detach: (() => void) | undefined;
private disposed = false;
+ private failureReason: SDKStartupFailureReason | null = null;
constructor(
private onPhaseChange: (phase: InitializationPhase) => void,
@@ -43,6 +77,17 @@ export class SessionInitialization {
return this.activeHooks.size > 0 ? "setup_hooks" : "sdk_initialization";
}
+ get failure(): StartupFailure {
+ return {
+ provider: "claude",
+ phase: this.phase,
+ reason: this.failureReason,
+ retryability: this.failureReason
+ ? startupRetryability[this.failureReason]
+ : "unknown",
+ };
+ }
+
get timeoutMs(): number {
return this.phase === "setup_hooks"
? this.hooksTimeoutMs
@@ -84,6 +129,10 @@ export class SessionInitialization {
} catch {
return;
}
+ const startupResult = startupResultSchema.safeParse(message);
+ if (startupResult.success) {
+ this.failureReason = startupResult.data.startup_failure_reason;
+ }
if (
message &&
typeof message === "object" &&
diff --git a/packages/agent/packages/agent/src/adapters/claude/session/options.ts b/packages/agent/packages/agent/src/adapters/claude/session/options.ts
index 5909be0c9e1a..5b9e163ffc69 100644
--- a/packages/agent/packages/agent/src/adapters/claude/session/options.ts
+++ b/packages/agent/packages/agent/src/adapters/claude/session/options.ts
@@ -220,6 +220,7 @@ function buildEnvironment(
ENABLE_TOOL_SEARCH: "auto:0",
// Enable idle state as end-of-turn signal (required for SDK 0.2.114+)
CLAUDE_CODE_EMIT_SESSION_STATE_EVENTS: "1",
+ CLAUDE_CODE_STARTUP_FAILURE_RESULTS: "1",
...(mcpNonblocking !== undefined && {
MCP_CONNECTION_NONBLOCKING: mcpNonblocking,
}),
diff --git a/packages/agent/packages/agent/src/server/agent-server.test.ts b/packages/agent/packages/agent/src/server/agent-server.test.ts
index 4565aa42ef86..1af9176604d6 100644
--- a/packages/agent/packages/agent/src/server/agent-server.test.ts
+++ b/packages/agent/packages/agent/src/server/agent-server.test.ts
@@ -37,7 +37,10 @@ import { getSessionJsonlPath } from "../adapters/claude/session/jsonl-hydration"
import { selectProduct } from "../boot/select-product";
import type { ContextSelectionResponse } from "../context-selection/schemas";
import type { PermissionMode } from "../execution-mode";
-import type { PostHogAPIClient } from "../posthog-api";
+import {
+ CodexSubscriptionTokenError,
+ type PostHogAPIClient,
+} from "../posthog-api";
import { formatConversationForResume, type ResumeState } from "../resume";
import type { ProductLaunchConfiguration, TaskSource } from "../sdk/product";
import { SessionLogWriter } from "../session-log-writer";
@@ -59,6 +62,7 @@ import {
UPSTREAM_PROVIDER_FAILURE_MESSAGE,
} from "./agent-server";
import { ContextSelection, type HumanMessage } from "./context-selection";
+import { CredentialRelayError } from "./credential-relay";
import { type JwtPayload, SANDBOX_CONNECTION_AUDIENCE } from "./jwt";
import type { ExistingPrCheckoutResult } from "./pr-checkout";
@@ -597,6 +601,65 @@ it.each([false, true])(
},
);
+it.each([
+ ["request_failed", "transient"],
+ ["openai_unavailable", "transient"],
+ ["reauth_required", "requires_action"],
+ ["forbidden", "requires_action"],
+] as const)(
+ "classifies Codex credential failure %s",
+ async (code, retryability) => {
+ const server = new AgentServer({
+ port: 0,
+ jwtPublicKey: TEST_PUBLIC_KEY,
+ apiUrl: "http://localhost:8000",
+ apiKey: "test-api-key",
+ projectId: 1,
+ mode: "interactive",
+ taskId: "test-task-id",
+ runId: "test-run-id",
+ runtimeAdapter: "codex",
+ }) as unknown as {
+ _doInitializeSession(payload: JwtPayload): Promise<void>;
+ initializeSession(payload: JwtPayload, controller: null): Promise<void>;
+ app: { fetch(request: Request): Promise<Response> };
+ };
+ const error = new CodexSubscriptionTokenError(
+ code,
+ 503,
+ "Synthetic token failure",
+ );
+ server._doInitializeSession = vi.fn().mockRejectedValue(error);
+ await expect(
+ server.initializeSession(
+ {
+ task_id: "test-task-id",
+ run_id: "test-run-id",
+ team_id: 1,
+ user_id: 1,
+ distinct_id: "example",
+ mode: "interactive",
+ },
+ null,
+ ),
+ ).rejects.toBe(error);
+ const health = await server.app.fetch(
+ new Request("http://localhost/health"),
+ );
+ expect(await health.json()).toMatchObject({
+ readiness: "failed",
+ startupFailure: {
+ provider: "codex",
+ retryability,
+ reason:
+ retryability === "transient"
+ ? "credential_service_unavailable"
+ : "credential_unavailable",
+ },
+ });
+ },
+);
+
describe("AgentServer HTTP Mode", () => {
let repo: TestRepo;
let server: AgentServer | undefined;
@@ -676,6 +739,115 @@ describe("AgentServer HTTP Mode", () => {
);
};
+ it.each(["typed", "unknown", "credential"])(
+ "keeps startup failures available to the worker and recovers on retry (%s)",
+ async (kind) => {
+ const typed = kind === "typed";
+ const failure = {
+ provider: "claude",
+ phase: typed ? "sdk_initialization" : "session_setup",
+ reason: typed
+ ? "proxy_invalid"
+ : kind === "credential"
+ ? "credential_unavailable"
+ : null,
+ retryability:
+ typed || kind === "credential" ? "requires_action" : "unknown",
+ };
+ const s = createServer({ runtimeAdapter: "claude" });
+ const internals = s as unknown as {
+ app: { fetch(request: Request): Promise<Response> };
+ _doInitializeSession(payload: JwtPayload): Promise<void>;
+ initializeSession(payload: JwtPayload, controller: null): Promise<void>;
+ reportFatalError(error: unknown): Promise<void>;
+ reportSubscriptionTokenMissing(adapter: "claude", reason: string): void;
+ eventStreamSender: {
+ enqueue: ReturnType<typeof vi.fn>;
+ stop: ReturnType<typeof vi.fn>;
+ };
+ posthogAPI: PostHogAPIClient;
+ logger: { error: (message: string, ...args: unknown[]) => void };
+ };
+ const logError = vi.spyOn(internals.logger, "error");
+ internals.eventStreamSender = {
+ enqueue: vi.fn(),
+ stop: vi.fn(async () => {}),
+ };
+ const original = internals._doInitializeSession.bind(s);
+ const error = typed
+ ? RequestError.internalError({ startup_failure: failure })
+ : kind === "credential"
+ ? new CredentialRelayError("no_token")
+ : new Error("Instruction sync failed: ghp_syntheticSecretToken");
+ internals._doInitializeSession = vi
+ .fn()
+ .mockImplementationOnce(async () => {
+ if (kind === "credential") {
+ internals.reportSubscriptionTokenMissing("claude", "no_token");
+ }
+ throw error;
+ })
+ .mockImplementation(original);
+ const payload: JwtPayload = {
+ task_id: "test-task-id",
+ run_id: "test-run-id",
+ team_id: 1,
+ user_id: 1,
+ distinct_id: "test-distinct-id",
+ mode: "interactive",
+ };
+
+ await s.start();
+ const health = await internals.app.fetch(
+ new Request("http://localhost/health"),
+ );
+ expect(await health.json()).toMatchObject({
+ readiness: "failed",
+ hasSession: false,
+ startupFailure: failure,
+ });
+
+ if (kind === "unknown") {
+ expect(logError).toHaveBeenCalledWith(
+ "Agent session initialization failed",
+ expect.stringContaining("Instruction sync failed"),
+ );
+ expect(JSON.stringify(logError.mock.calls)).not.toContain(
+ "ghp_syntheticSecretToken",
+ );
+ }
+ if (kind !== "credential") {
+ await internals.reportFatalError(error);
+ expect(updateTaskRunCalls).toContainEqual(
+ expect.objectContaining({ error_type: "agent_server_crash" }),
+ );
+ }
+ expect(updateTaskRunCalls).not.toContainEqual(
+ expect.objectContaining({ status: "failed" }),
+ );
+ await openProduct(s);
+ internals.eventStreamSender.enqueue.mockClear();
+ await internals.initializeSession(payload, null);
+ const recovered = await internals.app.fetch(
+ new Request("http://localhost/health"),
+ );
+ expect(await recovered.json()).toMatchObject({
+ readiness: "ready",
+ hasSession: true,
+ });
+ if (kind === "credential") {
+ expect(internals.eventStreamSender.stop).not.toHaveBeenCalled();
+ expect(internals.eventStreamSender.enqueue).toHaveBeenCalled();
+ }
+ expect(updateTaskRunCalls).toContainEqual(
+ expect.objectContaining({
+ status: "in_progress",
+ error_message: null,
+ }),
+ );
+ },
+ );
+
it("replays ACP notifications emitted before cloud session assignment", () => {
const testServer = createServer() as unknown as {
session: { sseController: null } | null;
@@ -717,6 +889,7 @@ describe("AgentServer HTTP Mode", () => {
expect(response.status).toBe(200);
expect(body).toEqual({
status: "ok",
+ runId: "test-run-id",
hasSession: true,
readiness: "ready",
bootMs: expect.any(Number),
diff --git a/packages/agent/packages/agent/src/server/agent-server.ts b/packages/agent/packages/agent/src/server/agent-server.ts
index f29f93de3ae6..645e454e262e 100644
--- a/packages/agent/packages/agent/src/server/agent-server.ts
+++ b/packages/agent/packages/agent/src/server/agent-server.ts
@@ -37,6 +37,10 @@ import {
appendRichOutputPrompt,
getProjectWebUrl,
} from "@posthog/agent-contracts/rich-output-prompt";
+import {
+ readStartupFailure,
+ type StartupFailure,
+} from "@posthog/agent-contracts/startup-failure";
import { execGh } from "@posthog/git/gh";
import { getCurrentBranch, getRemoteUrl } from "@posthog/git/queries";
import { ghTokenEnv } from "@posthog/git/signed-commit";
@@ -569,6 +573,7 @@ export class AgentServer {
typeof createAcpConnection
> | null = null;
private initializationFailureCode: string | undefined;
+ private startupFailure: StartupFailure | undefined;
private initializationPhase: InitializationPhase | undefined;
private initializingSseController: SseController | null = null;
private initializingTelemetry: OtelRunTelemetry | undefined;
@@ -804,6 +809,8 @@ export class AgentServer {
readiness: boot.state,
initializationPhase: this.initializationPhase,
failureCode: this.initializationFailureCode,
+ startupFailure: this.startupFailure,
+ runId: this.config.runId,
bootMs: this.sessionReadyBootMs,
sessionInitMs: this.sessionInitMs,
boot,
@@ -1029,7 +1036,11 @@ export class AgentServer {
});
await this.startMemoryWatchdogWatcher();
- await this.autoInitializeSession();
+ try {
+ await this.autoInitializeSession();
+ } catch (error) {
+ if (!this.startupFailure) throw error;
+ }
}
private async startMemoryWatchdogWatcher(): Promise<void> {
@@ -1259,10 +1270,12 @@ export class AgentServer {
this.config.taskId,
this.config.runId,
{
- status: "failed",
+ ...(this.startupFailure ? {} : { status: "failed" as const }),
error_message: `Agent server crashed: ${errorMessage}`,
error_type: "agent_server_crash",
- state: { agent_version: this.agentVersion },
+ state: {
+ agent_version: this.agentVersion,
+ },
},
);
} catch (updateError) {
@@ -1344,10 +1357,10 @@ export class AgentServer {
return this.codexTokenClient;
}
- private async reportSubscriptionTokenMissing(
+ private reportSubscriptionTokenMissing(
adapter: "claude" | "codex",
reason: string,
- ): Promise<void> {
+ ): void {
const failure = SUBSCRIPTION_TOKEN_FAILURE[adapter];
this.initializationFailureCode = `${adapter}_credential_unavailable`;
this.logger.warn(this.initializationFailureCode);
@@ -1366,12 +1379,8 @@ export class AgentServer {
},
},
});
- await this.eventStreamSender?.stop({ complete: false });
} catch (error) {
- this.logger.error(
- "Failed to flush events after credential relay failure",
- error,
- );
+ this.logger.error("Failed to report credential relay failure", error);
}
}
@@ -2005,19 +2014,44 @@ export class AgentServer {
this.httpReadyBootMs,
this.config.launcherToProcessMs,
);
+ this.startupFailure = undefined;
+ this.initializationFailureCode = undefined;
this.initializationPromise = this._doInitializeSession(payload);
const initStartedAt = Date.now();
try {
await this.initializationPromise;
} catch (error) {
if (this.shutdownController.signal.aborted) throw error;
- this.bootTracker.markFailed();
+ if (this.bootTracker.snapshot().state === "ready") throw error;
+ this.logger.error(
+ "Agent session initialization failed",
+ redactSecrets(describeFatalError(error)),
+ );
+ this.startupFailure = readStartupFailure(error) ?? {
+ provider: this.getRuntimeAdapter(),
+ phase: "session_setup",
+ reason: null,
+ retryability: "unknown",
+ };
if (
error instanceof CredentialRelayError ||
error instanceof CodexSubscriptionTokenError
) {
+ const transient =
+ error instanceof CodexSubscriptionTokenError &&
+ (error.code === "request_failed" ||
+ error.code === "openai_unavailable");
+ this.startupFailure = {
+ provider: this.subscriptionAdapter(),
+ phase: "session_setup",
+ reason: transient
+ ? "credential_service_unavailable"
+ : "credential_unavailable",
+ retryability: transient ? "transient" : "requires_action",
+ };
this.initializationFailureCode = `${this.subscriptionAdapter()}_credential_unavailable`;
}
+ this.bootTracker.markFailed();
const telemetry = this.initializingTelemetry;
telemetry?.append(payload.run_id, {
type: "notification",
@@ -2027,6 +2061,7 @@ export class AgentServer {
method: POSTHOG_NOTIFICATIONS.INITIALIZATION_FAILED,
params: {
runtimeAdapter: this.getRuntimeAdapter(),
+ startupFailure: this.startupFailure,
initializationPhase: "session_setup",
initMs: Date.now() - initStartedAt,
requestedModel: this.config.model,
@@ -2272,7 +2307,7 @@ export class AgentServer {
if (this.shutdownController.signal.aborted) throw error;
const reason = error instanceof Error ? error.message : String(error);
this.logger.warn("Claude subscription token relay failed", { reason });
- await this.reportSubscriptionTokenMissing("claude", reason);
+ this.reportSubscriptionTokenMissing("claude", reason);
throw error;
}
}
@@ -2293,7 +2328,7 @@ export class AgentServer {
? error.message
: String(error);
this.logger.warn("ChatGPT token request failed", { reason });
- await this.reportSubscriptionTokenMissing("codex", reason);
+ this.reportSubscriptionTokenMissing("codex", reason);
throw error;
}
}
@@ -2653,16 +2688,23 @@ export class AgentServer {
this.broadcastAndPersistNotification(agentStartedProgress);
// Signal in_progress so the UI can start polling for updates
- this.posthogAPI
- .updateTaskRun(payload.task_id, payload.run_id, {
- status: "in_progress",
- state: {
- agent_version: this.agentVersion,
- ...(isBenjaminEnabled() && {
- benjamin_version: BENJAMIN_UPSTREAM_COMMIT,
- }),
+ this.startupFailure = undefined;
+ await this.posthogAPI
+ .updateTaskRun(
+ payload.task_id,
+ payload.run_id,
+ {
+ status: "in_progress",
+ error_message: null,
+ state: {
+ agent_version: this.agentVersion,
+ ...(isBenjaminEnabled() && {
+ benjamin_version: BENJAMIN_UPSTREAM_COMMIT,
+ }),
+ },
},
- })
+ AbortSignal.timeout(2_000),
+ )
.catch((err) =>
this.logger.debug("Failed to set task run to in_progress", err),
);
diff --git a/packages/agent/packages/agent/src/server/pi-agent-server.test.ts b/packages/agent/packages/agent/src/server/pi-agent-server.test.ts
index c792d7aab3be..fef6defb6fb4 100644
--- a/packages/agent/packages/agent/src/server/pi-agent-server.test.ts
+++ b/packages/agent/packages/agent/src/server/pi-agent-server.test.ts
@@ -3,7 +3,10 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import type { AgentConversationEvent } from "@posthog/agent-contracts";
+import { AgentInstructionFiles } from "@posthog/harness/extensions/agent-instructions";
import { describe, expect, it, vi } from "vitest";
+import * as localTools from "../adapters/codex-app-server/local-tools-mcp";
+import * as piRpc from "../pi/rpc-client";
import { PiAgentServer } from "./pi-agent-server";
import type { AgentServerConfig } from "./types";
@@ -29,6 +32,8 @@ describe("PiAgentServer", () => {
const server = new PiAgentServer(
config({ model: "anthropic/claude-opus-5" }),
) as unknown as {
+ start(): Promise<void>;
+ stop(): Promise<void>;
logger: { error: ReturnType<typeof vi.fn> };
createSession(sessionPayload: typeof payload): Promise<void>;
initializeSession(
@@ -36,7 +41,10 @@ describe("PiAgentServer", () => {
sseController: null,
): Promise<void>;
createRunTelemetry: ReturnType<typeof vi.fn>;
+ posthogAPI: { updateTaskRun: ReturnType<typeof vi.fn> };
+ app: { fetch(request: Request): Promise<Response> };
};
+ server.posthogAPI.updateTaskRun = vi.fn().mockResolvedValue({});
const append = vi.fn();
const shutdown = vi.fn(async () => {});
server.createRunTelemetry = vi.fn(() => ({ append, shutdown }));
@@ -45,41 +53,121 @@ describe("PiAgentServer", () => {
});
const errorSpy = vi.spyOn(server.logger, "error");
- await expect(server.initializeSession(payload, null)).rejects.toThrow(
- "Pi RPC startup failed",
- );
-
- expect(errorSpy).toHaveBeenCalledWith(
- "Pi session initialization failed",
- expect.objectContaining({
- runtimeAdapter: "pi",
- initializationPhase: "session_setup",
- initMs: expect.any(Number),
- requestedModel: "anthropic/claude-opus-5",
- gatewayConfigured: true,
- taskId: "task-1",
- taskRunId: "run-1",
- errorDetail: expect.objectContaining({
- message: "Pi RPC startup failed",
+ try {
+ await server.start();
+ const failed = await server.app.fetch(
+ new Request("http://localhost/health"),
+ );
+ expect(await failed.json()).toMatchObject({
+ readiness: "failed",
+ startupFailure: {
+ provider: "pi",
+ phase: "session_setup",
+ reason: null,
+ retryability: "unknown",
+ },
+ });
+ expect(errorSpy).toHaveBeenCalledWith(
+ "Pi session initialization failed",
+ expect.objectContaining({
+ runtimeAdapter: "pi",
+ initializationPhase: "session_setup",
+ initMs: expect.any(Number),
+ requestedModel: "anthropic/claude-opus-5",
+ gatewayConfigured: true,
+ taskId: "task-1",
+ taskRunId: "run-1",
+ errorDetail: expect.objectContaining({
+ message: "Pi RPC startup failed",
+ }),
}),
- }),
- );
- expect(append).toHaveBeenCalledWith(
- "run-1",
- expect.objectContaining({
- notification: expect.objectContaining({
- method: "_posthog/initialization_failed",
- params: expect.objectContaining({
- runtimeAdapter: "pi",
- initializationPhase: "session_setup",
- errorType: "error",
+ );
+ expect(append).toHaveBeenCalledWith(
+ "run-1",
+ expect.objectContaining({
+ notification: expect.objectContaining({
+ method: "_posthog/initialization_failed",
+ params: expect.objectContaining({
+ runtimeAdapter: "pi",
+ initializationPhase: "session_setup",
+ errorType: "error",
+ }),
}),
}),
- }),
- );
- expect(shutdown).toHaveBeenCalledOnce();
+ );
+ expect(shutdown).toHaveBeenCalledOnce();
+ } finally {
+ await server.stop();
+ }
});
+ it.each(["start", "setThinkingLevel", "getState"] as const)(
+ "stops the partial Pi client when %s fails",
+ async (operation) => {
+ const client = {
+ start: vi.fn(async () => {}),
+ setThinkingLevel: vi.fn(async () => {}),
+ getState: vi.fn(async () => ({})),
+ stop: vi.fn(async () => {}),
+ onEvent: vi.fn(() => vi.fn()),
+ onMcpToolPermissionRequest: vi.fn(() => vi.fn()),
+ };
+ const failure = new Error("Synthetic Pi setup failure");
+ client[operation].mockRejectedValueOnce(failure);
+ const createClient = vi
+ .spyOn(piRpc, "createPiRpcClient")
+ .mockReturnValue(client as unknown as piRpc.PiRpcClient);
+ const syncInstructions = vi
+ .spyOn(AgentInstructionFiles.prototype, "sync")
+ .mockResolvedValue(null);
+ const buildLocalTools = vi
+ .spyOn(localTools, "buildLocalToolsServer")
+ .mockReturnValue(null);
+ const server = new PiAgentServer(
+ config({
+ model: "anthropic/claude-opus-5",
+ reasoningEffort: "low",
+ repositoryPath: tmpdir(),
+ }),
+ ) as unknown as {
+ createSession(payload: Record<string, unknown>): Promise<void>;
+ posthogAPI: Record<string, unknown>;
+ session: unknown;
+ };
+ server.posthogAPI = {
+ getTaskSession: vi.fn(async () => ({
+ id: "session-1",
+ content_sha256: null,
+ })),
+ downloadTaskSession: vi.fn(async () => null),
+ getMcpRuntimeConfiguration: vi.fn(async () => ({
+ servers: [],
+ policies: [],
+ })),
+ getTask: vi.fn(async () => null),
+ getTaskRun: vi.fn(async () => null),
+ };
+ try {
+ await expect(
+ server.createSession({
+ task_id: "task-1",
+ run_id: "run-1",
+ team_id: 1,
+ }),
+ ).rejects.toBe(failure);
+ expect(client.stop).toHaveBeenCalledOnce();
+ expect(
+ client.onMcpToolPermissionRequest.mock.results[0].value,
+ ).toHaveBeenCalledOnce();
+ expect(server.session).toBeNull();
+ } finally {
+ createClient.mockRestore();
+ syncInstructions.mockRestore();
+ buildLocalTools.mockRestore();
+ }
+ },
+ );
+
it("emits RTK savings for a cloud Pi run", async () => {
const enqueue = vi.fn();
const server = new PiAgentServer(
@@ -1253,7 +1341,9 @@ describe("PiAgentServer", () => {
sseController: TestController,
): Promise<void>;
cancelSseController(sseController: TestController): void;
+ posthogAPI: { updateTaskRun: ReturnType<typeof vi.fn> };
};
+ server.posthogAPI.updateTaskRun = vi.fn().mockResolvedValue({});
server.createSession = vi.fn(async (sessionPayload) => {
await initializationGate;
server.session = { payload: sessionPayload, sseController: null };
diff --git a/packages/agent/packages/agent/src/server/pi-agent-server.ts b/packages/agent/packages/agent/src/server/pi-agent-server.ts
index 95f65abbed20..5c925fee4fd7 100644
--- a/packages/agent/packages/agent/src/server/pi-agent-server.ts
+++ b/packages/agent/packages/agent/src/server/pi-agent-server.ts
@@ -18,6 +18,7 @@ import {
serializeError,
type TaskRunArtifact,
} from "@posthog/agent-contracts";
+import type { StartupFailure } from "@posthog/agent-contracts/startup-failure";
import type { TaskContext } from "@posthog/agent-contracts/task-context";
import { AgentInstructionFiles } from "@posthog/harness/extensions/agent-instructions";
import { resolveContextWikiPath } from "@posthog/harness/extensions/context-wiki";
@@ -179,6 +180,7 @@ export class PiAgentServer {
private server: ServerType | null = null;
private session: PiCloudSession | null = null;
private initializationPromise: Promise<void> | null = null;
+ private startupFailure: StartupFailure | undefined;
private pendingEvents: Record<string, unknown>[] = [];
private sessionReadyBootMs?: number;
private sessionInitMs?: number;
@@ -287,7 +289,11 @@ export class PiAgentServer {
distinct_id: "pi-agent-server",
mode: this.config.mode,
};
- await this.initializeSession(payload, null);
+ try {
+ await this.initializeSession(payload, null);
+ } catch (error) {
+ if (!this.startupFailure) throw error;
+ }
}
async stop(): Promise<void> {
@@ -364,10 +370,12 @@ export class PiAgentServer {
);
await this.posthogAPI
.updateTaskRun(this.config.taskId, this.config.runId, {
- status: "failed",
+ ...(this.startupFailure ? {} : { status: "failed" as const }),
error_message: `Pi agent server crashed: ${message}`,
error_type: "agent_server_crash",
- state: { agent_version: this.agentVersion },
+ state: {
+ agent_version: this.agentVersion,
+ },
})
.catch((updateError) =>
this.logger.error(
@@ -409,6 +417,13 @@ export class PiAgentServer {
context.json({
status: "ok",
hasSession: this.session !== null,
+ readiness: this.startupFailure
+ ? "failed"
+ : this.session
+ ? "ready"
+ : "initializing",
+ startupFailure: this.startupFailure,
+ runId: this.config.runId,
bootMs: this.sessionReadyBootMs,
sessionInitMs: this.sessionInitMs,
boot: {
@@ -558,12 +573,20 @@ export class PiAgentServer {
return;
}
+ this.startupFailure = undefined;
const initializationPromise = this.createSession(payload);
this.initializationPromise = initializationPromise;
const initStartedAt = Date.now();
try {
await initializationPromise;
} catch (error) {
+ const startupFailure: StartupFailure = {
+ provider: "pi",
+ phase: "session_setup",
+ reason: null,
+ retryability: "unknown",
+ };
+ this.startupFailure = startupFailure;
const telemetry = this.createRunTelemetry(payload);
telemetry?.append(payload.run_id, {
type: "notification",
@@ -573,6 +596,7 @@ export class PiAgentServer {
method: POSTHOG_NOTIFICATIONS.INITIALIZATION_FAILED,
params: {
runtimeAdapter: "pi",
+ startupFailure,
initializationPhase: "session_setup",
initMs: Date.now() - initStartedAt,
requestedModel: this.config.model,
@@ -777,18 +801,6 @@ export class PiAgentServer {
const unsubscribeMcpPermissions = client.onMcpToolPermissionRequest(
(request) => this.handleMcpToolPermissionRequest(request),
);
- await client.start();
- if (this.config.reasoningEffort) {
- // Pi's ThinkingLevel has no ultracode notch; run it at its xhigh equivalent.
- await client.setThinkingLevel(
- this.config.reasoningEffort === "ultracode"
- ? "xhigh"
- : this.config.reasoningEffort,
- );
- }
- const runtimeState = await client.getState();
- this.applyModelContextWindow(runtimeState);
- this.sessionFile = runtimeState.sessionFile ?? restoredSessionFile ?? null;
const unsubscribe = () => {
unsubscribeConversation();
unsubscribeRuntime();
@@ -796,19 +808,57 @@ export class PiAgentServer {
unsubscribeMcpPermissions();
};
- this.session = {
- payload,
- runtime,
- sseController: null,
- unsubscribe,
- providerRoute,
- };
+ try {
+ await client.start();
+ if (this.config.reasoningEffort) {
+ // Pi's ThinkingLevel has no ultracode notch; run it at its xhigh equivalent.
+ await client.setThinkingLevel(
+ this.config.reasoningEffort === "ultracode"
+ ? "xhigh"
+ : this.config.reasoningEffort,
+ );
+ }
+ const runtimeState = await client.getState();
+ this.applyModelContextWindow(runtimeState);
+ this.sessionFile =
+ runtimeState.sessionFile ?? restoredSessionFile ?? null;
+ this.session = {
+ payload,
+ runtime,
+ sseController: null,
+ unsubscribe,
+ providerRoute,
+ };
+ } catch (error) {
+ unsubscribe();
+ await client
+ .stop()
+ .catch((stopError) =>
+ this.logger.error(
+ "Failed to stop Pi client after setup failure",
+ stopError,
+ ),
+ );
+ throw error;
+ }
this.sessionReadyBootMs = Math.round(process.uptime() * 1000);
this.sessionInitMs = Date.now() - startedAt;
- await this.posthogAPI.updateTaskRun(payload.task_id, payload.run_id, {
- status: "in_progress",
- state: { agent_version: this.agentVersion },
- });
+ await this.posthogAPI
+ .updateTaskRun(
+ payload.task_id,
+ payload.run_id,
+ {
+ status: "in_progress",
+ error_message: null,
+ state: {
+ agent_version: this.agentVersion,
+ },
+ },
+ AbortSignal.timeout(2_000),
+ )
+ .catch((error) =>
+ this.logger.error("Failed to persist agent startup success", error),
+ );
this.broadcast({
type: "pi_run_started",
timestamp: new Date().toISOString(),
diff --git a/products/tasks/backend/facade/agents.py b/products/tasks/backend/facade/agents.py
index 06677ee5406e..de74c94c78af 100644
--- a/products/tasks/backend/facade/agents.py
+++ b/products/tasks/backend/facade/agents.py
@@ -32,6 +32,7 @@ from products.tasks.backend.logic.services.local_skills import (
ENV_LOCAL_SKILLS_HOST_PATH,
LocalSkillsCache,
)
+from products.tasks.backend.logic.services.startup_failure import AgentStartupFailed
from products.tasks.backend.models import SandboxEnvironment
@@ -53,6 +54,7 @@ __all__ = [
"MCP_EXEC_SKILLS_FEATURE_FLAG",
"ENV_LOCAL_SKILLS_HOST_PATH",
"AgentTurnFailed",
+ "AgentStartupFailed",
"CustomPromptSandboxContext",
"EmptyAgentTurnError",
"LocalSkillsCache",
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index f29b519858aa..148dc2fee4e7 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -2971,6 +2971,13 @@ _PROTECTED_RUN_STATE_KEYS = frozenset(
"token_cost_incomplete",
"compute_cost",
"failure_error_type",
+ "startup_failure",
+ "startup_started_at",
+ "startup_completed_at",
+ "startup_provider",
+ "startup_runtime",
+ "agent_startup_attempt",
+ "agent_startup_success",
"unprocessed_request_ids",
TASK_OWNERSHIP_VERSION_STATE_KEY,
"pr_authorship_mode",
diff --git a/products/tasks/backend/facade/contracts.py b/products/tasks/backend/facade/contracts.py
index cf6e874a1b34..d1e7bb920263 100644
--- a/products/tasks/backend/facade/contracts.py
+++ b/products/tasks/backend/facade/contracts.py
@@ -75,6 +75,36 @@ class AgentTaskRunDTO:
workflow_id: str
+@dataclass(frozen=True, kw_only=True)
+class AgentStartupFailure:
+ provider: Literal["claude", "codex", "pi"]
+ phase: Literal["session_setup", "sdk_initialization", "setup_hooks"]
+ reason: (
+ Literal[
+ "credential_unavailable",
+ "credential_service_unavailable",
+ "org_pin_api_key_conflict",
+ "org_verify_failed",
+ "org_pin_mismatch",
+ "managed_settings_invalid",
+ "remote_settings_required_unavailable",
+ "gateway_signin_required",
+ "gateway_access_denied",
+ "proxy_invalid",
+ "temp_dir_unusable",
+ "cwd_unavailable",
+ "shell_tool_missing",
+ "session_held_by_background",
+ "worktree_resume_refused",
+ "worktree_unverified",
+ "cli_version_too_old",
+ "bypass_root",
+ ]
+ | None
+ )
+ retryability: Literal["requires_action", "transient", "unknown"]
+
+
@dataclass(frozen=True, kw_only=True)
class SessionFile:
"""A text file a custom-prompt session puts in the sandbox before its first turn.
diff --git a/products/tasks/backend/logic/services/agent_server_launcher.py b/products/tasks/backend/logic/services/agent_server_launcher.py
index aac3877aee76..4881f3e9e333 100644
--- a/products/tasks/backend/logic/services/agent_server_launcher.py
+++ b/products/tasks/backend/logic/services/agent_server_launcher.py
@@ -61,6 +61,7 @@ from products.tasks.backend.logic.services.sandbox import (
health_check_timeout_seconds,
wait_for_health_check,
)
+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError, raise_startup_failure
if TYPE_CHECKING:
from products.tasks.backend.temporal.process_task.utils import McpServerConfig
@@ -648,6 +649,13 @@ class AgentServerLaunchMixin(SandboxBase):
start_and_health_ms = int((time.perf_counter() - start_time) * 1000)
if launch_result.exit_code != 0:
health_duration_ms = _health_duration_ms(launch_result.stdout)
+ try:
+ raise_startup_failure(launch_result.stdout, self.id)
+ except AgentServerStartupError as error:
+ error.context.update(start_and_health_ms=start_and_health_ms)
+ if wait_for_health and health_duration_ms is not None:
+ error.context.update(health_poll_ms=health_duration_ms)
+ raise
if wait_for_health and health_duration_ms is not None:
diagnostics = self._diagnose_startup_failure(allowed_domains)
credential_error = self._credential_unavailable_error(
@@ -719,6 +727,7 @@ class AgentServerLaunchMixin(SandboxBase):
codex_model_access: str | None = None,
codex_run_token: str | None = None,
sandbox_runtime: str | None = None,
+ before_launch: Callable[[], None] | None = None,
) -> int | None:
"""Start the agent-server HTTP server in the sandbox.
@@ -735,6 +744,9 @@ class AgentServerLaunchMixin(SandboxBase):
if preflight.reused:
return 0 if wait_for_health else None
+ if before_launch:
+ before_launch()
+
repo_path: str | None = None
if repository:
org, repo = repository.lower().split("/")
diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.py
index 7d7abb0ccbc5..bc9c551a3ed0 100644
--- a/products/tasks/backend/logic/services/custom_prompt_internals.py
+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py
@@ -22,9 +22,13 @@ from posthog.storage import object_storage
from posthog.storage.object_storage import ObjectStorageError
from posthog.temporal.oauth import PosthogMcpScopes
-from products.tasks.backend.exceptions import is_transient_failure_error_type
from products.tasks.backend.facade.api import upload_task_run_artifacts
-from products.tasks.backend.facade.contracts import AgentTaskRunDTO, SessionFile
+from products.tasks.backend.facade.contracts import AgentStartupFailure, AgentTaskRunDTO, SessionFile
+from products.tasks.backend.logic.services.startup_failure import (
+ AgentStartupFailed,
+ SandboxRunFailed as SandboxRunFailed,
+ read_startup_failure,
+)
from products.tasks.backend.models import MCPBuiltInAgentKey, Task, TaskRun, stamp_pending_user_message_id
if TYPE_CHECKING:
@@ -274,16 +278,6 @@ class AgentTurnFailed(RuntimeError):
return self.category in UPSTREAM_RETRYABLE_ERROR_CATEGORIES
-class SandboxRunFailed(RuntimeError):
- def __init__(self, message: str, *, error_type: str | None) -> None:
- super().__init__(message)
- self.error_type = error_type
-
- @property
- def transient(self) -> bool:
- return is_transient_failure_error_type(self.error_type)
-
-
SESSION_FILE_ARTIFACT_TYPE = "context"
@@ -685,6 +679,7 @@ async def poll_for_turn(
refreshed_status=refreshed.status,
error_message=refreshed.error_message,
failure_error_type=(refreshed.state or {}).get("failure_error_type"),
+ startup_failure=read_startup_failure(refreshed.state),
printed_lines=printed_lines,
original_skip_lines=original_skip_lines,
verbose=verbose,
@@ -711,6 +706,7 @@ async def poll_for_turn(
refreshed_status=refreshed.status,
error_message=refreshed.error_message,
failure_error_type=(refreshed.state or {}).get("failure_error_type"),
+ startup_failure=read_startup_failure(refreshed.state),
printed_lines=printed_lines,
original_skip_lines=original_skip_lines,
verbose=verbose,
@@ -894,6 +890,7 @@ async def _drain_final_log(
refreshed_status: str,
error_message: str | None = None,
failure_error_type: str | None = None,
+ startup_failure: AgentStartupFailure | None = None,
printed_lines: int,
original_skip_lines: int,
verbose: bool,
@@ -910,6 +907,13 @@ async def _drain_final_log(
stale earlier-turn response in multi-turn sessions). For single-turn callers `original_skip_lines == 0`, so
the scan covers the full log as before. The walk is idempotent and only runs once at terminal status.
"""
+ if refreshed_status == TaskRun.Status.FAILED and startup_failure is not None:
+ raise AgentStartupFailed(
+ failure=startup_failure,
+ task_id=str(task_run.task_id),
+ run_id=str(task_run.id),
+ error_type=failure_error_type,
+ )
final_state = await _read_turn_log_with_retry(task_run, skip_lines=original_skip_lines, context="drain_final_log")
printed_lines = _stream_new_lines(final_state.full_log, printed_lines, verbose=verbose, output_fn=output_fn)
if final_state.last_message:
diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.py
index 9b9aa680819d..bf0155aec8bd 100644
--- a/products/tasks/backend/logic/services/sandbox.py
+++ b/products/tasks/backend/logic/services/sandbox.py
@@ -47,6 +47,7 @@ from products.tasks.backend.logic.services.sandbox_config import (
SANDBOX_TTL_SECONDS,
VM_SANDBOX_CPU_CORES,
)
+from products.tasks.backend.logic.services.startup_failure import raise_startup_failure
if TYPE_CHECKING:
from products.tasks.backend.temporal.process_task.utils import McpServerConfig
@@ -686,6 +687,7 @@ class SandboxBase(ABC):
codex_model_access: str | None = None,
codex_run_token: str | None = None,
sandbox_runtime: str | None = None,
+ before_launch: Callable[[], None] | None = None,
) -> int | None:
"""Start the agent-server HTTP server in the sandbox.
@@ -906,6 +908,8 @@ def wait_for_health_check(
"""
health_script = build_health_check_command(port, max_attempts, poll_interval, pid_file)
result = execute(health_script, timeout_seconds=health_check_timeout_seconds(max_attempts, poll_interval))
+ if result.exit_code != 0:
+ raise_startup_failure(result.stdout, sandbox_id)
if "claude_credential_unavailable" in result.stdout:
from products.tasks.backend.exceptions import ProcessTaskFatalError
@@ -954,15 +958,17 @@ def build_health_check_command(
f" body=$(curl -q -s --max-time {HEALTH_CURL_MAX_TIME_SECONDS} http://localhost:{port}/health); "
" status=$?; "
' if [ "$status" = "0" ]; then '
- ' case "$body" in *claude_credential_unavailable*) echo "claude_credential_unavailable"; exit 1;; '
- '*codex_credential_unavailable*) echo "codex_credential_unavailable"; exit 1;; esac; '
" python3 -c '"
"import json, sys; "
"payload = json.loads(sys.argv[1]); "
'ready = payload.get("status") == "ok" and payload.get("hasSession") is True; '
- 'sys.exit(0 if ready else 2 if payload.get("initializationPhase") == "setup_hooks" else 1)'
+ 'sys.exit(0 if ready else 3 if payload.get("readiness") == "failed" '
+ 'else 2 if payload.get("initializationPhase") == "setup_hooks" else 1)'
'\' "$body"; ready=$?; '
' if [ "$ready" = "0" ]; then echo "ok:$i"; exit 0; fi; '
+ ' if [ "$ready" = "3" ]; then printf "%s\\n" "$body"; exit 1; fi; '
+ ' case "$body" in *claude_credential_unavailable*) echo "claude_credential_unavailable"; exit 1;; '
+ '*codex_credential_unavailable*) echo "codex_credential_unavailable"; exit 1;; esac; '
f' if [ "$ready" = "2" ]; then max_attempts={hooks_max_attempts}; budget={hooks_budget_seconds}; fi; '
" fi; "
' if [ "$i" -ge "$max_attempts" ] || [ "$SECONDS" -ge "$budget" ]; then exit 1; fi; '
diff --git a/products/tasks/backend/logic/services/startup_failure.py b/products/tasks/backend/logic/services/startup_failure.py
new file mode 100644
index 000000000000..8ca5e083147e
--- /dev/null
+++ b/products/tasks/backend/logic/services/startup_failure.py
@@ -0,0 +1,103 @@
+import json
+from collections.abc import Mapping
+from dataclasses import replace
+from typing import Literal
+
+from pydantic import TypeAdapter, ValidationError
+
+from products.tasks.backend.exceptions import SandboxExecutionError, is_transient_failure_error_type
+from products.tasks.backend.facade.contracts import AgentStartupFailure
+
+_startup_failure_adapter = TypeAdapter(AgentStartupFailure)
+
+_ACTION_REQUIRED_MESSAGES = {
+ "org_pin_api_key_conflict": "The agent's API key conflicts with its organization settings. Use credentials for the configured organization, then retry.",
+ "org_pin_mismatch": "The agent's organization settings do not match its credentials. Check the configured organization and credentials, then retry.",
+ "managed_settings_invalid": "The agent's managed settings are invalid. Check the provider's managed settings, then retry.",
+ "gateway_signin_required": "The agent needs to sign in to its provider. Sign in to the provider, then retry.",
+ "gateway_access_denied": "The provider denied access while loading the agent's managed settings. Check the provider account's permissions, then retry.",
+ "proxy_invalid": "The agent's proxy configuration is invalid. Check the sandbox proxy URL, then retry.",
+ "shell_tool_missing": "A shell required by the agent is unavailable. Update the sandbox image to include a supported shell, then retry.",
+ "cli_version_too_old": "The agent's provider runtime is too old. Update the provider runtime in the sandbox image, then retry.",
+ "bypass_root": "The provider rejected the agent's permission configuration. Run the agent as a non-root user or update its permission settings, then retry.",
+}
+_ACTION_REQUIRED_REASONS = frozenset({"credential_unavailable", *_ACTION_REQUIRED_MESSAGES})
+_TRANSIENT_REASONS = frozenset({"credential_service_unavailable", "session_held_by_background", "worktree_unverified"})
+
+
+def read_startup_failure(state: Mapping[str, object] | None) -> AgentStartupFailure | None:
+ try:
+ return _startup_failure_adapter.validate_python((state or {}).get("startup_failure"))
+ except ValidationError:
+ return None
+
+
+class AgentServerStartupError(SandboxExecutionError):
+ def __init__(self, failure: AgentStartupFailure, run_id: str, sandbox_id: str) -> None:
+ self.startup_failure = failure
+ self.run_id = run_id
+ super().__init__(
+ _ACTION_REQUIRED_MESSAGES.get(failure.reason or "", "The agent could not initialize its session."),
+ {"sandbox_id": sandbox_id, "run_id": run_id},
+ cause=RuntimeError("Agent session initialization failed"),
+ capture=False,
+ )
+
+
+def raise_startup_failure(stdout: str, sandbox_id: str) -> None:
+ for line in stdout.splitlines():
+ if len(line) > 4096:
+ continue
+ try:
+ payload = json.loads(line)
+ except (ValueError, TypeError):
+ continue
+ if not isinstance(payload, dict) or payload.get("readiness") != "failed" or payload.get("hasSession") is True:
+ continue
+ run_id = payload.get("runId")
+ if not isinstance(run_id, str) or not run_id:
+ continue
+ try:
+ failure = _startup_failure_adapter.validate_python(payload.get("startupFailure"))
+ except ValidationError:
+ continue
+ reason = (
+ failure.reason
+ if failure.provider == "claude"
+ or failure.reason in {"credential_unavailable", "credential_service_unavailable"}
+ else None
+ )
+ retryability: Literal["requires_action", "transient", "unknown"] = (
+ "requires_action"
+ if reason in _ACTION_REQUIRED_REASONS
+ else "transient"
+ if reason in _TRANSIENT_REASONS
+ else "unknown"
+ )
+ raise AgentServerStartupError(replace(failure, reason=reason, retryability=retryability), run_id, sandbox_id)
+
+
+class SandboxRunFailed(RuntimeError):
+ def __init__(self, message: str, *, error_type: str | None) -> None:
+ super().__init__(message)
+ self.error_type = error_type
+
+ @property
+ def transient(self) -> bool:
+ return is_transient_failure_error_type(self.error_type)
+
+
+class AgentStartupFailed(SandboxRunFailed):
+ def __init__(
+ self, *, failure: AgentStartupFailure, task_id: str, run_id: str, error_type: str | None = None
+ ) -> None:
+ super().__init__("The agent failed during startup.", error_type=error_type)
+ self.failure = failure
+ self.task_id = task_id
+ self.run_id = run_id
+
+ @property
+ def transient(self) -> bool:
+ return self.failure.retryability != "requires_action" and (
+ self.failure.retryability == "transient" or super().transient
+ )
diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
index b3066022d925..d8b589f75d65 100644
--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
@@ -104,6 +104,7 @@ from products.tasks.backend.logic.services.sandbox import (
SandboxWorkload,
health_check_timeout_seconds,
)
+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError
def _agent_server_launch_command(mock_execute: Any) -> str:
@@ -1046,15 +1047,35 @@ class TestModalSandboxAgentServer:
run_id="run-456",
)
- def test_start_agent_server_raises_on_health_check_failure(self, mock_sandbox: Any):
+ @pytest.mark.parametrize("structured", [False, True])
+ def test_start_agent_server_raises_on_health_check_failure(self, mock_sandbox: Any, structured: bool):
+ payload = (
+ json.dumps(
+ {
+ "hasSession": False,
+ "readiness": "failed",
+ "runId": "run-456",
+ "startupFailure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "proxy_invalid",
+ "retryability": "requires_action",
+ },
+ }
+ )
+ + "\n"
+ if structured
+ else ""
+ )
+
def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:
if "./node_modules/.bin/agent-server" in command:
- return ExecutionResult(stdout="__posthog_agent_health_ms=120000", stderr="", exit_code=1)
+ return ExecutionResult(stdout=payload + "__posthog_agent_health_ms=120000", stderr="", exit_code=1)
return ExecutionResult(stdout="", stderr="", exit_code=0)
mock_sandbox.execute = MagicMock(side_effect=execute)
with patch.object(mock_sandbox, "_diagnose_startup_failure", return_value={"failure_reason": "not ready"}):
- with pytest.raises(SandboxExecutionError, match="Agent-server failed to start") as error:
+ with pytest.raises(AgentServerStartupError if structured else SandboxExecutionError) as error:
mock_sandbox.start_agent_server(
repository="posthog/posthog",
task_id="task-123",
@@ -1062,6 +1083,7 @@ class TestModalSandboxAgentServer:
)
assert error.value.context["health_poll_ms"] == 120000
+ assert isinstance(error.value.context["start_and_health_ms"], int)
assert mock_sandbox.supports_combined_agent_server_start_and_health() is True
@pytest.mark.parametrize(
diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py b/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py
index a8f911ed6bc2..dedaed627730 100644
--- a/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py
+++ b/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py
@@ -1,4 +1,5 @@
import os
+import json
import time
import subprocess
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -18,6 +19,7 @@ from products.tasks.backend.logic.services.sandbox import (
build_health_check_command,
health_check_budget_seconds,
)
+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError, raise_startup_failure
def _path_with_fake_curl(tmp_path: Path, body: str, delay_seconds: float) -> dict[str, str]:
@@ -63,6 +65,11 @@ def test_health_check_ignores_curl_config(tmp_path: Path) -> None:
[
('{"status":"ok","hasSession":true}', 0, "ok:1"),
('{"status":"ok","hasSession":false}', 1, ""),
+ (
+ '{"status":"ok","hasSession":false,"readiness":"failed"}',
+ 1,
+ '{"status":"ok","hasSession":false,"readiness":"failed"}',
+ ),
('{"status":"error","code":"codex_credential_unavailable"}', 1, "codex_credential_unavailable"),
],
)
@@ -183,3 +190,74 @@ def test_preflight_reports_only_the_capabilities_the_binary_carries(
if line.startswith(AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX)
]
assert sorted(reported) == sorted(capabilities)
+
+
+ ("provider", "reason", "reported_retryability", "expected_retryability"),
+ [
+ ("claude", "proxy_invalid", "unknown", "requires_action"),
+ ("claude", "worktree_unverified", "requires_action", "transient"),
+ ("claude", None, "requires_action", "unknown"),
+ ("codex", "credential_service_unavailable", "requires_action", "transient"),
+ ],
+)
+def test_health_check_carries_validated_startup_failure(
+ tmp_path: Path, provider, reason, reported_retryability, expected_retryability
+) -> None:
+ payload = {
+ "status": "ok",
+ "hasSession": False,
+ "readiness": "failed",
+ "runId": "example-run",
+ "startupFailure": {
+ "provider": provider,
+ "phase": "sdk_initialization",
+ "reason": reason,
+ "retryability": reported_retryability,
+ },
+ }
+ result = subprocess.run(
+ ["bash", "-c", build_health_check_command(port=1, max_attempts=1, poll_interval=0.0)],
+ env=_path_with_fake_curl(tmp_path, json.dumps(payload), 0.0),
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ assert result.returncode == 1
+ with pytest.raises(AgentServerStartupError) as raised:
+ raise_startup_failure(result.stdout, "example-sandbox")
+ assert raised.value.run_id == "example-run"
+ assert raised.value.startup_failure.reason == reason
+ assert raised.value.startup_failure.retryability == expected_retryability
+
+
+ "change",
+ [
+ {"runId": None},
+ {"hasSession": True},
+ {"readiness": "ready"},
+ {
+ "startupFailure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "arbitrary-reason",
+ "retryability": "requires_action",
+ }
+ },
+ ],
+)
+def test_health_check_ignores_unconfirmed_startup_failures(change) -> None:
+ payload = {
+ "hasSession": False,
+ "readiness": "failed",
+ "runId": "example-run",
+ "startupFailure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "proxy_invalid",
+ "retryability": "requires_action",
+ },
+ **change,
+ }
+ raise_startup_failure(json.dumps(payload), "example-sandbox")
diff --git a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
index 1abe6384165b..ce67fc4b731f 100644
--- a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
+++ b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
@@ -4,11 +4,11 @@ import shlex
import threading
from collections.abc import Sequence
from concurrent.futures import Future, ThreadPoolExecutor
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from django.conf import settings
-from django.db import connection
+from django.db import connection, transaction
import posthoganalytics
from temporalio import activity
@@ -41,12 +41,15 @@ from products.tasks.backend.logic.services.connection_token import (
from products.tasks.backend.logic.services.launch_preparation_metrics import launch_preparation_metric_context
from products.tasks.backend.logic.services.modal_sandbox import ModalSandbox
from products.tasks.backend.logic.services.sandbox import (
+ CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE,
+ CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE,
REPO_READY_FILE,
SUBSCRIPTION_CLI_FLAGS,
SandboxBase,
get_sandbox_class_for_sandbox_id,
sandbox_repo_path,
)
+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError
from products.tasks.backend.logic.stream.redis_stream import release_task_run_milestone_claims
from products.tasks.backend.models import Task, TaskRun
from products.tasks.backend.redis import run_uses_dedicated_stream
@@ -607,6 +610,37 @@ def _prepare_launch(ctx: TaskProcessingContext, scopes: PosthogMcpScopes, sandbo
)
+def _record_agent_startup_attempt(ctx: TaskProcessingContext, runtime: str) -> None:
+ TaskRun.update_state_atomic(
+ ctx.run_id,
+ updates={
+ "agent_startup_attempt": {
+ "started_at": datetime.now(UTC).isoformat(),
+ "repository": (ctx.repository or "").lower(),
+ "provider": "pi" if ctx.task_runtime == "pi" else ctx.runtime_adapter or "claude",
+ "runtime": runtime,
+ },
+ },
+ remove_keys=["startup_failure", "startup_completed_at", "startup_provider"],
+ )
+
+
+def _record_agent_startup_success(ctx: TaskProcessingContext) -> None:
+ with transaction.atomic():
+ state = TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).values_list("state", flat=True).first()
+ attempt = (state or {}).get("agent_startup_attempt")
+ TaskRun.update_state_atomic(
+ ctx.run_id,
+ updates={"agent_startup_success": {**attempt, "completed_at": datetime.now(UTC).isoformat()}}
+ if isinstance(attempt, dict)
+ else {},
+ remove_keys=["startup_failure"],
+ )
+ TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).exclude(
+ status__in=[TaskRun.Status.COMPLETED, TaskRun.Status.FAILED, TaskRun.Status.CANCELLED]
+ ).update(error_message=None)
+
+
def _invoke_start_agent_server(
sandbox: SandboxBase,
ctx: TaskProcessingContext,
@@ -618,6 +652,9 @@ def _invoke_start_agent_server(
try:
_enforce_subscription_support(sandbox, ctx)
health_duration_ms = sandbox.start_agent_server(
+ before_launch=lambda: _record_agent_startup_attempt(
+ ctx, sandbox_runtime_label(ctx.use_modal_vm_sandbox) if isinstance(sandbox, ModalSandbox) else "unknown"
+ ),
repository=ctx.repository if len(ctx.repositories) <= 1 else None,
task_id=ctx.task_id,
run_id=ctx.run_id,
@@ -777,6 +814,26 @@ def _read_agent_boot_metrics(sandbox: SandboxBase, input: StartAgentServerInput)
return session_init_ms, boot_phases_ms
+def _record_agent_startup_failure(ctx: TaskProcessingContext, sandbox_id: str, error: Exception) -> None:
+ if not isinstance(error, AgentServerStartupError) or error.run_id != ctx.run_id:
+ return
+ TaskRun.update_state_atomic(ctx.run_id, updates={"startup_failure": asdict(error.startup_failure)})
+ if error.startup_failure.retryability == "requires_action":
+ message = error.message
+ if error.startup_failure.reason == "credential_unavailable":
+ message = (
+ CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE
+ if error.startup_failure.provider == "codex"
+ else CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE
+ )
+ raise ProcessTaskFatalError(
+ message,
+ {"run_id": ctx.run_id, "sandbox_id": sandbox_id},
+ error,
+ capture=False,
+ ) from error
+
+
@activity.defn
@asyncify
@posthoganalytics.scoped(fresh=True, capture_exceptions=False)
@@ -843,6 +900,7 @@ def start_agent_server(input: StartAgentServerInput) -> StartAgentServerOutput:
)
invoke_status = "COMPLETED" if health_poll_ms is not None else "FAILED"
health_status = "FAILED"
+ _record_agent_startup_failure(ctx, input.sandbox_id, error)
raise
finally:
if start_and_health_ms is None:
@@ -887,14 +945,19 @@ def start_agent_server(input: StartAgentServerInput) -> StartAgentServerOutput:
origin_product=ctx.origin_product,
runtime=runtime,
) as health_timer:
- sandbox.wait_for_agent_server_ready(
- params.agentsh_domains,
- claude_model_access=ctx.model_access.access_for("claude"),
- )
+ try:
+ sandbox.wait_for_agent_server_ready(
+ params.agentsh_domains,
+ claude_model_access=ctx.model_access.access_for("claude"),
+ )
+ except Exception as error:
+ _record_agent_startup_failure(ctx, input.sandbox_id, error)
+ raise
invoke_ms = invoke_timer.elapsed_ms
health_poll_ms = health_timer.elapsed_ms
_record_agent_server_launch(sandbox, ctx, params)
+ _record_agent_startup_success(ctx)
_record_network_enforcement_observation(ctx)
log_with_activity_context(f"Agent server started at {input.sandbox_url}", run_id=ctx.run_id)
@@ -1093,6 +1156,7 @@ def await_agent_server_ready(input: StartAgentServerInput) -> StartAgentServerOu
if agentsh_domains is not None:
_emit_agentsh_log_tail(ctx, sandbox)
_emit_agent_server_log_tail(ctx, sandbox)
+ _record_agent_startup_failure(ctx, input.sandbox_id, error)
raise
if attempt > 1:
@@ -1106,6 +1170,7 @@ def await_agent_server_ready(input: StartAgentServerInput) -> StartAgentServerOu
_record_network_enforcement_observation(ctx)
+ _record_agent_startup_success(ctx)
log_with_activity_context(f"Agent server ready at {input.sandbox_url}", run_id=ctx.run_id)
activity.logger.info(f"Agent server ready at {input.sandbox_url} for task {ctx.task_id}")
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
index ab705984fffe..c099df0ad183 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
@@ -15,9 +15,11 @@ from products.tasks.backend.exceptions import (
SandboxRateLimitedError,
SandboxTimeoutError,
)
+from products.tasks.backend.facade.contracts import AgentStartupFailure
from products.tasks.backend.logic.services.launch_preparation_metrics import record_launch_preparation_ms
from products.tasks.backend.logic.services.modal_sandbox import ModalSandbox
from products.tasks.backend.logic.services.sandbox import ExecutionResult, sandbox_repo_path
+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError
from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContext
from products.tasks.backend.temporal.process_task.activities.start_agent_server import (
CollectAgentShadowResultInput,
@@ -50,6 +52,20 @@ def organization_state(mocker):
return state
[email protected](autouse=True)
+def task_run_state(mocker):
+ mocker.patch("products.tasks.backend.temporal.process_task.activities.start_agent_server.transaction.atomic")
+ mocker.patch(
+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.update_state_atomic"
+ )
+ runs = mocker.patch(
+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.objects.filter"
+ )
+ state = runs.return_value.values_list.return_value.first
+ state.return_value = None
+ return state
+
+
@time_machine.travel("2026-08-06T12:01:30Z", tick=False)
def test_record_boot_total_excludes_wizard_time_and_labels_runtime(mocker) -> None:
record_metric = mocker.patch(
@@ -551,6 +567,71 @@ async def test_await_agent_server_ready_relaunches_on_activity_retries(mocker, a
record_retry.assert_not_called()
[email protected]("boot_path", ["classic", "combined", "overlap"])
+ "retryability",
+ ["requires_action", "transient", "unknown", "other_run", "claude_credential", "codex_credential", None],
+)
+async def test_startup_failure_controls_readiness_retries(mocker, task_run_state, boot_path, retryability) -> None:
+ context = _context()
+ credential = retryability in {"claude_credential", "codex_credential"}
+ requires_action = retryability == "requires_action" or credential
+ task_run_state.return_value = {
+ "startup_failure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "proxy_invalid",
+ "retryability": "requires_action",
+ }
+ }
+ sandbox = mocker.Mock(id="sandbox-id")
+ sandbox.supports_combined_agent_server_start_and_health.return_value = boot_path == "combined"
+ failure = (
+ AgentServerStartupError(
+ AgentStartupFailure(
+ provider="codex" if retryability == "codex_credential" else "claude",
+ phase="sdk_initialization",
+ reason="credential_unavailable" if credential else "proxy_invalid",
+ retryability="requires_action" if retryability == "other_run" or credential else retryability,
+ ),
+ "another-run" if retryability == "other_run" else context.run_id,
+ sandbox.id,
+ )
+ if retryability is not None
+ else SandboxExecutionError(
+ "Agent startup failed", {"run_id": context.run_id}, RuntimeError("startup"), capture=False
+ )
+ )
+ sandbox.wait_for_agent_server_ready.side_effect = failure
+ prefix = "products.tasks.backend.temporal.process_task.activities.start_agent_server."
+ mocker.patch(prefix + "get_sandbox_class_for_sandbox_id", **{"return_value.get_by_id.return_value": sandbox})
+ mocker.patch(prefix + "current_activity_attempt", return_value=1)
+ mocker.patch(prefix + "_prepare_launch", return_value=mocker.Mock(agentsh_domains=None))
+ mocker.patch(prefix + "_launch_agent_shadow")
+ mocker.patch(prefix + "_emit_agent_server_log_tail")
+ mocker.patch(prefix + "_invoke_start_agent_server", side_effect=failure if boot_path == "combined" else None)
+ activity_fn = await_agent_server_ready if boot_path == "overlap" else start_agent_server
+
+ with pytest.raises(ProcessTaskFatalError if requires_action else SandboxExecutionError) as error:
+ await activity_fn(
+ StartAgentServerInput(
+ context=context,
+ sandbox_id=sandbox.id,
+ sandbox_url="https://sandbox.example",
+ boot_path=boot_path,
+ )
+ )
+
+ assert error.value.non_retryable is requires_action
+ if credential:
+ assert "Settings > Harness" in str(error.value)
+ assert ("credential_request" in str(error.value)) is (retryability == "claude_credential")
+ elif requires_action:
+ assert "Check the sandbox proxy URL, then retry." in str(error.value)
+ if not requires_action:
+ assert error.value is failure
+
+
@pytest.mark.parametrize("pending_deletion", [False, True])
async def test_await_agent_server_ready_records_failed_relaunch(
mocker, organization_state, pending_deletion: bool
diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
index 7f8d32ea4cb7..2a73ea11e700 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
@@ -1811,6 +1811,25 @@ class TestPosthogCodeSandboxMint:
attributes={"surface": "sandbox", "ai_product": "posthog_code", "reason": counted_reason},
)
+ @pytest.mark.django_db
+ def test_retry_after_credit_top_up_rechecks_credit_availability(self, mint_settings):
+ team = self._team(paid=True)
+ mint_settings.SANDBOX_AI_GATEWAY_PRODUCTS = "posthog_code"
+ with (
+ patch(_CREDIT_LOOKUP, side_effect=["posthog_code_credits_exhausted", None]) as credit,
+ patch(
+ "products.tasks.backend.temporal.process_task.utils.mint_scoped_token", return_value="synthetic-token"
+ ) as mint,
+ ):
+ with pytest.raises(GatewayTokenUnavailableError) as refused:
+ ai_gateway_env_vars(team_id=team.id, origin_product="user_created", model="claude-opus-5")
+ assert refused.value.non_retryable is True
+ mint.assert_not_called()
+ env = ai_gateway_env_vars(team_id=team.id, origin_product="user_created", model="claude-opus-5")
+ assert env["AI_GATEWAY_TOKEN"] == "synthetic-token"
+ assert credit.call_count == 2
+ mint.assert_called_once()
+
@pytest.mark.django_db
@pytest.mark.parametrize("setting", ["SANDBOX_AI_GATEWAY_URL", "SANDBOX_AI_GATEWAY_MINT_KEY"])
def test_a_posthog_code_run_with_the_gateway_unconfigured_fails_and_logs(self, mint_settings, setting):
diff --git a/products/tasks/backend/tests/agent_log_fixtures.py b/products/tasks/backend/tests/agent_log_fixtures.py
index 5f9bff637de7..1af617854726 100644
--- a/products/tasks/backend/tests/agent_log_fixtures.py
+++ b/products/tasks/backend/tests/agent_log_fixtures.py
@@ -164,6 +164,7 @@ def _cost_less_usage_update_line(used: int = 1000) -> str:
@dataclass
class FakeTaskRun:
id: str = "run-1"
+ task_id: str = "task-1"
log_url: str = "s3://fake/log"
status: str = "running"
error_message: str | None = None
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index b5e8f9d08882..1638708cd016 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -7288,6 +7288,43 @@ class TestTaskSummariesAPI(BaseTaskAPITest):
class TestTaskRunAPI(BaseTaskAPITest):
+ @parameterized.expand([(False,), (True,)])
+ @patch("products.tasks.backend.models.TaskRun.publish_stream_state_event")
+ def test_patch_cannot_forge_startup_outcomes(self, sandbox_agent: bool, _mock_publish) -> None:
+ task = self.create_task()
+ attempt = {
+ "started_at": django_timezone.now().isoformat(),
+ "repository": "example/research",
+ "provider": "claude",
+ "runtime": "gvisor",
+ }
+ state = {"agent_startup_attempt": attempt, "agent_startup_success": attempt}
+ run = TaskRun.objects.create(team=self.team, task=task, status=TaskRun.Status.IN_PROGRESS, state=state)
+ client = self._sandbox_oauth_client(task.id, internal_scope=True) if sandbox_agent else self.client
+ forged = {
+ "agent_startup_attempt": {**attempt, "provider": "pi"},
+ "agent_startup_success": {**attempt, "provider": "pi"},
+ "startup_failure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "proxy_invalid",
+ "retryability": "requires_action",
+ },
+ "startup_started_at": "forged",
+ "startup_completed_at": "forged",
+ "startup_provider": "pi",
+ "startup_runtime": "vm",
+ }
+ for payload in [
+ {"state": forged},
+ {"state_remove_keys": list(forged)},
+ {"state_append": forged},
+ ]:
+ response = client.patch(f"/api/projects/@current/tasks/{task.id}/runs/{run.id}/", payload, format="json")
+ assert response.status_code == status.HTTP_200_OK
+ run.refresh_from_db()
+ assert {key: run.state.get(key) for key in forged} == {key: state.get(key) for key in forged}
+
def _create_run_for_origin(self, origin_product: Task.OriginProduct) -> tuple[Task, TaskRun]:
task = Task.objects.create(
team=self.team,
diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.py
index c1cc4f5c9429..6186d0651239 100644
--- a/products/tasks/backend/tests/test_multi_turn_session.py
+++ b/products/tasks/backend/tests/test_multi_turn_session.py
@@ -39,6 +39,7 @@ from products.tasks.backend.logic.services.custom_prompt_multi_turn_runner impor
_EMPTY_TURN_RETRY_NUDGE,
MultiTurnSession,
)
+from products.tasks.backend.logic.services.startup_failure import AgentStartupFailed
from products.tasks.backend.models import Task, TaskRun
from products.tasks.backend.tests.agent_log_fixtures import (
FakeTaskRun,
@@ -1130,6 +1131,46 @@ class TestPollForTurnSurfacesAgentError:
the exception, so a caller separates a retryable provider outage from a scout defect without
matching the message text."""
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "reason,retryability,error_type,transient",
+ [
+ ("proxy_invalid", "requires_action", "SandboxExecutionError", False),
+ (None, "unknown", "SandboxExecutionError", True),
+ (None, "unknown", None, False),
+ ("session_held_by_background", "transient", None, True),
+ ],
+ )
+ async def test_startup_failure_survives_without_session_logs(self, reason, retryability, error_type, transient):
+ fake_task_run = FakeTaskRun(
+ status="failed",
+ state={
+ "failure_error_type": error_type,
+ "startup_failure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": reason,
+ "retryability": retryability,
+ },
+ },
+ )
+ with (
+ patch("posthog.storage.object_storage.read", return_value=None),
+ patch("asyncio.sleep", new=AsyncMock()),
+ patch("products.tasks.backend.logic.services.custom_prompt_internals.POLL_INTERVAL_SECONDS", 0),
+ patch("products.tasks.backend.models.TaskRun.objects.get", return_value=fake_task_run),
+ ):
+ with pytest.raises(AgentStartupFailed) as error:
+ await poll_for_turn(fake_task_run, skip_lines=0)
+
+ assert error.value.failure.reason == reason
+ assert error.value.failure.retryability == retryability
+ assert error.value.task_id == fake_task_run.task_id
+ assert error.value.run_id == fake_task_run.id
+ assert isinstance(error.value, SandboxRunFailed)
+ assert error.value.error_type == error_type
+ assert error.value.transient is transient
+
@parameterized.expand(
[
("upstream_provider_failure", "API Error: 429 rate_limit_error", True),
@@ -1233,6 +1274,7 @@ class TestPollForTurnSurfacesAgentError:
"state,expected_error_type,expected_transient",
[
(None, None, False),
+ ({"startup_failure": {"reason": "proxy_invalid", "retryability": "requires_action"}}, None, False),
({"failure_error_type": "RepositoryCloneError"}, "RepositoryCloneError", True),
({"failure_error_type": "RepositoryAccessError"}, "RepositoryAccessError", False),
],
diff --git a/products/tasks/backend/tests/test_startup_recovery.py b/products/tasks/backend/tests/test_startup_recovery.py
new file mode 100644
index 000000000000..576a2351f0ed
--- /dev/null
+++ b/products/tasks/backend/tests/test_startup_recovery.py
@@ -0,0 +1,68 @@
+from datetime import timedelta
+
+import time_machine
+from posthog.test.base import BaseTest
+
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContext
+from products.tasks.backend.temporal.process_task.activities.start_agent_server import (
+ _record_agent_startup_attempt,
+ _record_agent_startup_success,
+)
+
+
+class TestAgentStartupRecovery(BaseTest):
+ @parameterized.expand(
+ [
+ ("acp", None, "claude", TaskRun.Status.IN_PROGRESS),
+ ("acp", "codex", "codex", TaskRun.Status.IN_PROGRESS),
+ ("pi", None, "pi", TaskRun.Status.IN_PROGRESS),
+ ("acp", None, "claude", TaskRun.Status.FAILED),
+ ("acp", None, "claude", TaskRun.Status.CANCELLED),
+ ("acp", None, "claude", TaskRun.Status.COMPLETED),
+ ]
+ )
+ def test_only_a_completed_worker_attempt_is_recovery_evidence(
+ self, runtime: str, adapter: str | None, provider: str, final_status: str
+ ) -> None:
+ task = Task.objects.create(team=self.team, title="Example research", repository="example/research")
+ run = TaskRun.objects.create(team=self.team, task=task)
+ context = TaskProcessingContext(
+ task_id=str(task.id),
+ run_id=str(run.id),
+ team_id=self.team.id,
+ team_uuid=str(self.team.uuid),
+ organization_id=str(self.organization.id),
+ github_integration_id=None,
+ repository=task.repository,
+ distinct_id="example",
+ task_runtime=runtime,
+ state={"runtime_adapter": adapter} if adapter else {},
+ )
+ started_at = timezone.now()
+ with time_machine.travel(started_at, tick=False):
+ _record_agent_startup_attempt(context, "gvisor")
+ run.refresh_from_db()
+ assert "agent_startup_success" not in run.state
+ with time_machine.travel(started_at + timedelta(seconds=1), tick=False):
+ _record_agent_startup_success(context)
+ run.refresh_from_db()
+ success = run.state["agent_startup_success"]
+ assert success["started_at"] == started_at.isoformat()
+ assert success["provider"] == provider
+ with time_machine.travel(started_at + timedelta(seconds=2), tick=False):
+ _record_agent_startup_attempt(context, "gvisor")
+ run.refresh_from_db()
+ assert run.state["agent_startup_success"] == success
+ TaskRun.update_state_atomic(str(run.id), updates={"startup_failure": {"reason": "proxy_invalid"}})
+ TaskRun.objects.filter(id=run.id).update(error_message="Turn failed", status=final_status)
+ with time_machine.travel(started_at + timedelta(seconds=3), tick=False):
+ _record_agent_startup_success(context)
+ run.refresh_from_db()
+ assert "startup_failure" not in run.state
+ assert run.error_message == ("Turn failed" if run.is_terminal else None)
+ assert run.state["agent_startup_success"]["started_at"] == (started_at + timedelta(seconds=2)).isoformat()