:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): preserve typed agent startup failures (#124662)
Alessandro Pogliaghi1016d50bc54f
diff --git a/docs/internal/task-run-api.md b/docs/internal/task-run-api.mdindex 1a3e54028950..e749b133285a 100644--- a/docs/internal/task-run-api.md+++ b/docs/internal/task-run-api.md@@ -26,6 +26,18 @@ Keep the task ID and inspect its runs before you retry.Do not repeat task creation to recover from a run failure.If a database read fails after dispatch, the response still includes the created run.+The worker persists agent startup failures in `latest_run.state.startup_failure` from the failed readiness response for that run.+The object contains `provider`, `phase`, nullable `reason`, and `retryability` (`requires_action`, `transient`, or `unknown`).+Only validated provider reason codes can stop startup activity retries; log text is not used to classify them.+Failures that require action include the configuration problem and a recovery step in the run's error message.+`requires_action` means the current attempt needs a configuration or credential change before retrying, not that recovery is impossible.+Unknown failures keep the existing retry policy. Credit exhaustion is a separate gateway-token refusal: a new attempt rechecks credit availability after a top-up; topping up does not automatically resume a failed run.+Temporary Codex credential-service outages remain retryable. Missing or invalid subscription credentials preserve the provider's reconnect or token-relay instructions.+The worker records `agent_startup_attempt` immediately before a real launch and `agent_startup_success` after readiness succeeds.+Successful initialization clears the startup failure and clears the crash message while the run is non-terminal. A terminal run keeps its final error. A later failed attempt preserves the last completed startup as recovery evidence.+These startup state fields cannot be written, appended to, or removed through the run PATCH API, including by sandbox clients.+Older agents may omit these fields; consumers must preserve the existing behavior when they are absent or invalid.+`branch` selects the base branch for the first run.Optional run hints can be `null`.Pi runs reject ACP runtime, model, reasoning-effort, and permission-mode hints.diff --git a/packages/agent/packages/agent-contracts/src/startup-failure.ts b/packages/agent/packages/agent-contracts/src/startup-failure.tsnew file mode 100644index 000000000000..4e1adff84b12--- /dev/null+++ b/packages/agent/packages/agent-contracts/src/startup-failure.ts@@ -0,0 +1,40 @@+import { z } from "zod";++export const startupFailureSchema = z.object({+ provider: z.enum(["claude", "codex", "pi"]),+ phase: z.enum(["session_setup", "sdk_initialization", "setup_hooks"]),+ reason: z+ .enum([+ "credential_unavailable",+ "credential_service_unavailable",+ "org_pin_api_key_conflict",+ "org_verify_failed",+ "org_pin_mismatch",+ "managed_settings_invalid",+ "remote_settings_required_unavailable",+ "gateway_signin_required",+ "gateway_access_denied",+ "proxy_invalid",+ "temp_dir_unusable",+ "cwd_unavailable",+ "shell_tool_missing",+ "session_held_by_background",+ "worktree_resume_refused",+ "worktree_unverified",+ "cli_version_too_old",+ "bypass_root",+ ])+ .nullable(),+ retryability: z.enum(["requires_action", "transient", "unknown"]),+});++export type StartupFailure = z.infer<typeof startupFailureSchema>;++const startupErrorSchema = z.object({+ data: z.object({ startup_failure: startupFailureSchema }),+});++export function readStartupFailure(error: unknown): StartupFailure | undefined {+ const parsed = startupErrorSchema.safeParse(error);+ return parsed.success ? parsed.data.data.startup_failure : undefined;+}diff --git a/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts b/packages/agent/packages/agent/src/adapters/claude/claude-agent.tsindex 83fe1765557a..4f5fd13c7f65 100644--- a/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts+++ b/packages/agent/packages/agent/src/adapters/claude/claude-agent.ts@@ -3046,7 +3046,11 @@ export class ClaudeAcpAgent extends BaseAcpAgent {if (!error.message.startsWith("Session ")) {error.message = `Session ${step} failed: ${error.message}`;}- return error;+ if (!("startup_failure" in errorData)) return error;+ return new RequestError(error.code, error.message, {+ ...errorData,+ ...(error.data && typeof error.data === "object" ? error.data : {}),+ });}return new RequestError(-32603, `Session ${step} failed`, errorData);}@@ -3451,6 +3455,7 @@ export class ClaudeAcpAgent extends BaseAcpAgent {},);throw this.startupFailure(forkSession ? "fork" : "resumption", err, {+ startup_failure: initialization.failure,sessionId,taskId,taskRunId: meta?.taskRunId,@@ -3462,7 +3467,10 @@ export class ClaudeAcpAgent extends BaseAcpAgent {// with the model config fetch below (the gateway REST call is independent).const initStartedAt = Date.now();const initPromise = !isResume- ? initialization.wait(q.initializationResult())+ ? initialization.wait(q.initializationResult()).then(+ (result) => ({ ok: true as const, result }),+ (error: unknown) => ({ ok: false as const, error }),+ ): undefined;const [rawModelOptions] = await Promise.all([@@ -3499,7 +3507,9 @@ export class ClaudeAcpAgent extends BaseAcpAgent {let timeoutMs = SESSION_VALIDATION_TIMEOUT_MS;let cliOutput: CliOutputSummary | undefined;try {- const initResult = await initPromise;+ const initializationResult = await initPromise;+ if (!initializationResult.ok) throw initializationResult.error;+ const initResult = initializationResult.result;if (initResult.result === "timeout") {initializationPhase = initResult.phase;timeoutMs = initResult.timeoutMs;@@ -3548,7 +3558,12 @@ export class ClaudeAcpAgent extends BaseAcpAgent {? "setup hooks": "initialization",err,- { sessionId, taskId, taskRunId: meta?.taskRunId },+ {+ sessionId,+ taskId,+ taskRunId: meta?.taskRunId,+ startup_failure: initialization.failure,+ },);}}diff --git a/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts b/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.tsindex d5b6049734c3..2173073e983e 100644--- a/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts+++ b/packages/agent/packages/agent/src/adapters/claude/session/initialization.test.ts@@ -136,4 +136,55 @@ describe("SessionInitialization", () => {expect(stdout.listenerCount("data")).toBe(0);expect(vi.getTimerCount()).toBe(0);});++ it.each([+ ["proxy_invalid", "requires_action"],+ ["worktree_unverified", "transient"],+ ["org_verify_failed", "unknown"],+ ["future_provider_reason", "unknown"],+ [null, "unknown"],+ ])(+ "preserves structured %s through an SDK rejection",+ async (reason, retryability) => {+ const stdout = new PassThrough();+ const monitor = new SessionInitialization(vi.fn());+ monitor.observe(stdout);+ const line = `${JSON.stringify({+ type: "result",+ subtype: "error_during_execution",+ startup_failure_reason: reason,+ })}\n`;+ stdout.write(line.slice(0, 19));+ stdout.write(line.slice(19));++ await expect(+ monitor.wait(Promise.reject(new Error("SDK initialization failed"))),+ ).rejects.toThrow("SDK initialization failed");++ expect(monitor.failure).toEqual({+ provider: "claude",+ phase: "sdk_initialization",+ reason: reason === "future_provider_reason" ? null : reason,+ retryability,+ });+ expect(stdout.listenerCount("data")).toBe(0);+ },+ );++ it.each([+ "proxy_invalid: the proxy setting is invalid\n",+ `${JSON.stringify({ type: "assistant", startup_failure_reason: "proxy_invalid" })}\n`,+ `${JSON.stringify({ type: "result", subtype: "success", startup_failure_reason: "proxy_invalid" })}\n`,+ `${JSON.stringify({ type: "result", subtype: "error_during_execution", errors: ["proxy_invalid"] })}\n`,+ ])("does not classify unrelated output %s", async (output) => {+ const stdout = new PassThrough();+ const monitor = new SessionInitialization(vi.fn());+ monitor.observe(stdout);+ stdout.write(output);+ expect(monitor.failure).toMatchObject({+ reason: null,+ retryability: "unknown",+ });+ await monitor.wait(Promise.resolve());+ });});diff --git a/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts b/packages/agent/packages/agent/src/adapters/claude/session/initialization.tsindex d0286306f538..35d8c737d960 100644--- a/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts+++ b/packages/agent/packages/agent/src/adapters/claude/session/initialization.ts@@ -1,5 +1,8 @@import type { Readable } from "node:stream";import { StringDecoder } from "node:string_decoder";+import type { SDKStartupFailureReason } from "@anthropic-ai/claude-agent-sdk";+import type { StartupFailure } from "@posthog/agent-contracts/startup-failure";+import { z } from "zod";export type InitializationPhase = "sdk_initialization" | "setup_hooks";@@ -22,6 +25,36 @@ type InitializationResult<Value> =const MAX_OBSERVED_LINE_LENGTH = 256 * 1024;const MAX_MESSAGE_TYPE_LENGTH = 100;+const startupRetryability = {+ org_pin_api_key_conflict: "requires_action",+ org_verify_failed: "unknown",+ org_pin_mismatch: "requires_action",+ managed_settings_invalid: "requires_action",+ remote_settings_required_unavailable: "unknown",+ gateway_signin_required: "requires_action",+ gateway_access_denied: "requires_action",+ proxy_invalid: "requires_action",+ temp_dir_unusable: "unknown",+ cwd_unavailable: "unknown",+ shell_tool_missing: "requires_action",+ session_held_by_background: "transient",+ worktree_resume_refused: "unknown",+ worktree_unverified: "transient",+ cli_version_too_old: "requires_action",+ bypass_root: "requires_action",+} satisfies Record<SDKStartupFailureReason, StartupFailure["retryability"]>;++const startupResultSchema = z.object({+ type: z.literal("result"),+ subtype: z.literal("error_during_execution"),+ startup_failure_reason: z.enum(+ Object.keys(startupRetryability) as [+ SDKStartupFailureReason,+ ...SDKStartupFailureReason[],+ ],+ ),+});+export class SessionInitialization {private activeHooks = new Set<string>();private hooksStartedAt: number | undefined;@@ -32,6 +65,7 @@ export class SessionInitialization {private reschedule: (() => void) | undefined;private detach: (() => void) | undefined;private disposed = false;+ private failureReason: SDKStartupFailureReason | null = null;constructor(private onPhaseChange: (phase: InitializationPhase) => void,@@ -43,6 +77,17 @@ export class SessionInitialization {return this.activeHooks.size > 0 ? "setup_hooks" : "sdk_initialization";}+ get failure(): StartupFailure {+ return {+ provider: "claude",+ phase: this.phase,+ reason: this.failureReason,+ retryability: this.failureReason+ ? startupRetryability[this.failureReason]+ : "unknown",+ };+ }+get timeoutMs(): number {return this.phase === "setup_hooks"? this.hooksTimeoutMs@@ -84,6 +129,10 @@ export class SessionInitialization {} catch {return;}+ const startupResult = startupResultSchema.safeParse(message);+ if (startupResult.success) {+ this.failureReason = startupResult.data.startup_failure_reason;+ }if (message &&typeof message === "object" &&diff --git a/packages/agent/packages/agent/src/adapters/claude/session/options.ts b/packages/agent/packages/agent/src/adapters/claude/session/options.tsindex 5909be0c9e1a..5b9e163ffc69 100644--- a/packages/agent/packages/agent/src/adapters/claude/session/options.ts+++ b/packages/agent/packages/agent/src/adapters/claude/session/options.ts@@ -220,6 +220,7 @@ function buildEnvironment(ENABLE_TOOL_SEARCH: "auto:0",// Enable idle state as end-of-turn signal (required for SDK 0.2.114+)CLAUDE_CODE_EMIT_SESSION_STATE_EVENTS: "1",+ CLAUDE_CODE_STARTUP_FAILURE_RESULTS: "1",...(mcpNonblocking !== undefined && {MCP_CONNECTION_NONBLOCKING: mcpNonblocking,}),diff --git a/packages/agent/packages/agent/src/server/agent-server.test.ts b/packages/agent/packages/agent/src/server/agent-server.test.tsindex 4565aa42ef86..1af9176604d6 100644--- a/packages/agent/packages/agent/src/server/agent-server.test.ts+++ b/packages/agent/packages/agent/src/server/agent-server.test.ts@@ -37,7 +37,10 @@ import { getSessionJsonlPath } from "../adapters/claude/session/jsonl-hydration"import { selectProduct } from "../boot/select-product";import type { ContextSelectionResponse } from "../context-selection/schemas";import type { PermissionMode } from "../execution-mode";-import type { PostHogAPIClient } from "../posthog-api";+import {+ CodexSubscriptionTokenError,+ type PostHogAPIClient,+} from "../posthog-api";import { formatConversationForResume, type ResumeState } from "../resume";import type { ProductLaunchConfiguration, TaskSource } from "../sdk/product";import { SessionLogWriter } from "../session-log-writer";@@ -59,6 +62,7 @@ import {UPSTREAM_PROVIDER_FAILURE_MESSAGE,} from "./agent-server";import { ContextSelection, type HumanMessage } from "./context-selection";+import { CredentialRelayError } from "./credential-relay";import { type JwtPayload, SANDBOX_CONNECTION_AUDIENCE } from "./jwt";import type { ExistingPrCheckoutResult } from "./pr-checkout";@@ -597,6 +601,65 @@ it.each([false, true])(},);+it.each([+ ["request_failed", "transient"],+ ["openai_unavailable", "transient"],+ ["reauth_required", "requires_action"],+ ["forbidden", "requires_action"],+] as const)(+ "classifies Codex credential failure %s",+ async (code, retryability) => {+ const server = new AgentServer({+ port: 0,+ jwtPublicKey: TEST_PUBLIC_KEY,+ apiUrl: "http://localhost:8000",+ apiKey: "test-api-key",+ projectId: 1,+ mode: "interactive",+ taskId: "test-task-id",+ runId: "test-run-id",+ runtimeAdapter: "codex",+ }) as unknown as {+ _doInitializeSession(payload: JwtPayload): Promise<void>;+ initializeSession(payload: JwtPayload, controller: null): Promise<void>;+ app: { fetch(request: Request): Promise<Response> };+ };+ const error = new CodexSubscriptionTokenError(+ code,+ 503,+ "Synthetic token failure",+ );+ server._doInitializeSession = vi.fn().mockRejectedValue(error);+ await expect(+ server.initializeSession(+ {+ task_id: "test-task-id",+ run_id: "test-run-id",+ team_id: 1,+ user_id: 1,+ distinct_id: "example",+ mode: "interactive",+ },+ null,+ ),+ ).rejects.toBe(error);+ const health = await server.app.fetch(+ new Request("http://localhost/health"),+ );+ expect(await health.json()).toMatchObject({+ readiness: "failed",+ startupFailure: {+ provider: "codex",+ retryability,+ reason:+ retryability === "transient"+ ? "credential_service_unavailable"+ : "credential_unavailable",+ },+ });+ },+);+describe("AgentServer HTTP Mode", () => {let repo: TestRepo;let server: AgentServer | undefined;@@ -676,6 +739,115 @@ describe("AgentServer HTTP Mode", () => {);};+ it.each(["typed", "unknown", "credential"])(+ "keeps startup failures available to the worker and recovers on retry (%s)",+ async (kind) => {+ const typed = kind === "typed";+ const failure = {+ provider: "claude",+ phase: typed ? "sdk_initialization" : "session_setup",+ reason: typed+ ? "proxy_invalid"+ : kind === "credential"+ ? "credential_unavailable"+ : null,+ retryability:+ typed || kind === "credential" ? "requires_action" : "unknown",+ };+ const s = createServer({ runtimeAdapter: "claude" });+ const internals = s as unknown as {+ app: { fetch(request: Request): Promise<Response> };+ _doInitializeSession(payload: JwtPayload): Promise<void>;+ initializeSession(payload: JwtPayload, controller: null): Promise<void>;+ reportFatalError(error: unknown): Promise<void>;+ reportSubscriptionTokenMissing(adapter: "claude", reason: string): void;+ eventStreamSender: {+ enqueue: ReturnType<typeof vi.fn>;+ stop: ReturnType<typeof vi.fn>;+ };+ posthogAPI: PostHogAPIClient;+ logger: { error: (message: string, ...args: unknown[]) => void };+ };+ const logError = vi.spyOn(internals.logger, "error");+ internals.eventStreamSender = {+ enqueue: vi.fn(),+ stop: vi.fn(async () => {}),+ };+ const original = internals._doInitializeSession.bind(s);+ const error = typed+ ? RequestError.internalError({ startup_failure: failure })+ : kind === "credential"+ ? new CredentialRelayError("no_token")+ : new Error("Instruction sync failed: ghp_syntheticSecretToken");+ internals._doInitializeSession = vi+ .fn()+ .mockImplementationOnce(async () => {+ if (kind === "credential") {+ internals.reportSubscriptionTokenMissing("claude", "no_token");+ }+ throw error;+ })+ .mockImplementation(original);+ const payload: JwtPayload = {+ task_id: "test-task-id",+ run_id: "test-run-id",+ team_id: 1,+ user_id: 1,+ distinct_id: "test-distinct-id",+ mode: "interactive",+ };++ await s.start();+ const health = await internals.app.fetch(+ new Request("http://localhost/health"),+ );+ expect(await health.json()).toMatchObject({+ readiness: "failed",+ hasSession: false,+ startupFailure: failure,+ });++ if (kind === "unknown") {+ expect(logError).toHaveBeenCalledWith(+ "Agent session initialization failed",+ expect.stringContaining("Instruction sync failed"),+ );+ expect(JSON.stringify(logError.mock.calls)).not.toContain(+ "ghp_syntheticSecretToken",+ );+ }+ if (kind !== "credential") {+ await internals.reportFatalError(error);+ expect(updateTaskRunCalls).toContainEqual(+ expect.objectContaining({ error_type: "agent_server_crash" }),+ );+ }+ expect(updateTaskRunCalls).not.toContainEqual(+ expect.objectContaining({ status: "failed" }),+ );+ await openProduct(s);+ internals.eventStreamSender.enqueue.mockClear();+ await internals.initializeSession(payload, null);+ const recovered = await internals.app.fetch(+ new Request("http://localhost/health"),+ );+ expect(await recovered.json()).toMatchObject({+ readiness: "ready",+ hasSession: true,+ });+ if (kind === "credential") {+ expect(internals.eventStreamSender.stop).not.toHaveBeenCalled();+ expect(internals.eventStreamSender.enqueue).toHaveBeenCalled();+ }+ expect(updateTaskRunCalls).toContainEqual(+ expect.objectContaining({+ status: "in_progress",+ error_message: null,+ }),+ );+ },+ );+it("replays ACP notifications emitted before cloud session assignment", () => {const testServer = createServer() as unknown as {session: { sseController: null } | null;@@ -717,6 +889,7 @@ describe("AgentServer HTTP Mode", () => {expect(response.status).toBe(200);expect(body).toEqual({status: "ok",+ runId: "test-run-id",hasSession: true,readiness: "ready",bootMs: expect.any(Number),diff --git a/packages/agent/packages/agent/src/server/agent-server.ts b/packages/agent/packages/agent/src/server/agent-server.tsindex f29f93de3ae6..645e454e262e 100644--- a/packages/agent/packages/agent/src/server/agent-server.ts+++ b/packages/agent/packages/agent/src/server/agent-server.ts@@ -37,6 +37,10 @@ import {appendRichOutputPrompt,getProjectWebUrl,} from "@posthog/agent-contracts/rich-output-prompt";+import {+ readStartupFailure,+ type StartupFailure,+} from "@posthog/agent-contracts/startup-failure";import { execGh } from "@posthog/git/gh";import { getCurrentBranch, getRemoteUrl } from "@posthog/git/queries";import { ghTokenEnv } from "@posthog/git/signed-commit";@@ -569,6 +573,7 @@ export class AgentServer {typeof createAcpConnection> | null = null;private initializationFailureCode: string | undefined;+ private startupFailure: StartupFailure | undefined;private initializationPhase: InitializationPhase | undefined;private initializingSseController: SseController | null = null;private initializingTelemetry: OtelRunTelemetry | undefined;@@ -804,6 +809,8 @@ export class AgentServer {readiness: boot.state,initializationPhase: this.initializationPhase,failureCode: this.initializationFailureCode,+ startupFailure: this.startupFailure,+ runId: this.config.runId,bootMs: this.sessionReadyBootMs,sessionInitMs: this.sessionInitMs,boot,@@ -1029,7 +1036,11 @@ export class AgentServer {});await this.startMemoryWatchdogWatcher();- await this.autoInitializeSession();+ try {+ await this.autoInitializeSession();+ } catch (error) {+ if (!this.startupFailure) throw error;+ }}private async startMemoryWatchdogWatcher(): Promise<void> {@@ -1259,10 +1270,12 @@ export class AgentServer {this.config.taskId,this.config.runId,{- status: "failed",+ ...(this.startupFailure ? {} : { status: "failed" as const }),error_message: `Agent server crashed: ${errorMessage}`,error_type: "agent_server_crash",- state: { agent_version: this.agentVersion },+ state: {+ agent_version: this.agentVersion,+ },},);} catch (updateError) {@@ -1344,10 +1357,10 @@ export class AgentServer {return this.codexTokenClient;}- private async reportSubscriptionTokenMissing(+ private reportSubscriptionTokenMissing(adapter: "claude" | "codex",reason: string,- ): Promise<void> {+ ): void {const failure = SUBSCRIPTION_TOKEN_FAILURE[adapter];this.initializationFailureCode = `${adapter}_credential_unavailable`;this.logger.warn(this.initializationFailureCode);@@ -1366,12 +1379,8 @@ export class AgentServer {},},});- await this.eventStreamSender?.stop({ complete: false });} catch (error) {- this.logger.error(- "Failed to flush events after credential relay failure",- error,- );+ this.logger.error("Failed to report credential relay failure", error);}}@@ -2005,19 +2014,44 @@ export class AgentServer {this.httpReadyBootMs,this.config.launcherToProcessMs,);+ this.startupFailure = undefined;+ this.initializationFailureCode = undefined;this.initializationPromise = this._doInitializeSession(payload);const initStartedAt = Date.now();try {await this.initializationPromise;} catch (error) {if (this.shutdownController.signal.aborted) throw error;- this.bootTracker.markFailed();+ if (this.bootTracker.snapshot().state === "ready") throw error;+ this.logger.error(+ "Agent session initialization failed",+ redactSecrets(describeFatalError(error)),+ );+ this.startupFailure = readStartupFailure(error) ?? {+ provider: this.getRuntimeAdapter(),+ phase: "session_setup",+ reason: null,+ retryability: "unknown",+ };if (error instanceof CredentialRelayError ||error instanceof CodexSubscriptionTokenError) {+ const transient =+ error instanceof CodexSubscriptionTokenError &&+ (error.code === "request_failed" ||+ error.code === "openai_unavailable");+ this.startupFailure = {+ provider: this.subscriptionAdapter(),+ phase: "session_setup",+ reason: transient+ ? "credential_service_unavailable"+ : "credential_unavailable",+ retryability: transient ? "transient" : "requires_action",+ };this.initializationFailureCode = `${this.subscriptionAdapter()}_credential_unavailable`;}+ this.bootTracker.markFailed();const telemetry = this.initializingTelemetry;telemetry?.append(payload.run_id, {type: "notification",@@ -2027,6 +2061,7 @@ export class AgentServer {method: POSTHOG_NOTIFICATIONS.INITIALIZATION_FAILED,params: {runtimeAdapter: this.getRuntimeAdapter(),+ startupFailure: this.startupFailure,initializationPhase: "session_setup",initMs: Date.now() - initStartedAt,requestedModel: this.config.model,@@ -2272,7 +2307,7 @@ export class AgentServer {if (this.shutdownController.signal.aborted) throw error;const reason = error instanceof Error ? error.message : String(error);this.logger.warn("Claude subscription token relay failed", { reason });- await this.reportSubscriptionTokenMissing("claude", reason);+ this.reportSubscriptionTokenMissing("claude", reason);throw error;}}@@ -2293,7 +2328,7 @@ export class AgentServer {? error.message: String(error);this.logger.warn("ChatGPT token request failed", { reason });- await this.reportSubscriptionTokenMissing("codex", reason);+ this.reportSubscriptionTokenMissing("codex", reason);throw error;}}@@ -2653,16 +2688,23 @@ export class AgentServer {this.broadcastAndPersistNotification(agentStartedProgress);// Signal in_progress so the UI can start polling for updates- this.posthogAPI- .updateTaskRun(payload.task_id, payload.run_id, {- status: "in_progress",- state: {- agent_version: this.agentVersion,- ...(isBenjaminEnabled() && {- benjamin_version: BENJAMIN_UPSTREAM_COMMIT,- }),+ this.startupFailure = undefined;+ await this.posthogAPI+ .updateTaskRun(+ payload.task_id,+ payload.run_id,+ {+ status: "in_progress",+ error_message: null,+ state: {+ agent_version: this.agentVersion,+ ...(isBenjaminEnabled() && {+ benjamin_version: BENJAMIN_UPSTREAM_COMMIT,+ }),+ },},- })+ AbortSignal.timeout(2_000),+ ).catch((err) =>this.logger.debug("Failed to set task run to in_progress", err),);diff --git a/packages/agent/packages/agent/src/server/pi-agent-server.test.ts b/packages/agent/packages/agent/src/server/pi-agent-server.test.tsindex c792d7aab3be..fef6defb6fb4 100644--- a/packages/agent/packages/agent/src/server/pi-agent-server.test.ts+++ b/packages/agent/packages/agent/src/server/pi-agent-server.test.ts@@ -3,7 +3,10 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { pathToFileURL } from "node:url";import type { AgentConversationEvent } from "@posthog/agent-contracts";+import { AgentInstructionFiles } from "@posthog/harness/extensions/agent-instructions";import { describe, expect, it, vi } from "vitest";+import * as localTools from "../adapters/codex-app-server/local-tools-mcp";+import * as piRpc from "../pi/rpc-client";import { PiAgentServer } from "./pi-agent-server";import type { AgentServerConfig } from "./types";@@ -29,6 +32,8 @@ describe("PiAgentServer", () => {const server = new PiAgentServer(config({ model: "anthropic/claude-opus-5" }),) as unknown as {+ start(): Promise<void>;+ stop(): Promise<void>;logger: { error: ReturnType<typeof vi.fn> };createSession(sessionPayload: typeof payload): Promise<void>;initializeSession(@@ -36,7 +41,10 @@ describe("PiAgentServer", () => {sseController: null,): Promise<void>;createRunTelemetry: ReturnType<typeof vi.fn>;+ posthogAPI: { updateTaskRun: ReturnType<typeof vi.fn> };+ app: { fetch(request: Request): Promise<Response> };};+ server.posthogAPI.updateTaskRun = vi.fn().mockResolvedValue({});const append = vi.fn();const shutdown = vi.fn(async () => {});server.createRunTelemetry = vi.fn(() => ({ append, shutdown }));@@ -45,41 +53,121 @@ describe("PiAgentServer", () => {});const errorSpy = vi.spyOn(server.logger, "error");- await expect(server.initializeSession(payload, null)).rejects.toThrow(- "Pi RPC startup failed",- );-- expect(errorSpy).toHaveBeenCalledWith(- "Pi session initialization failed",- expect.objectContaining({- runtimeAdapter: "pi",- initializationPhase: "session_setup",- initMs: expect.any(Number),- requestedModel: "anthropic/claude-opus-5",- gatewayConfigured: true,- taskId: "task-1",- taskRunId: "run-1",- errorDetail: expect.objectContaining({- message: "Pi RPC startup failed",+ try {+ await server.start();+ const failed = await server.app.fetch(+ new Request("http://localhost/health"),+ );+ expect(await failed.json()).toMatchObject({+ readiness: "failed",+ startupFailure: {+ provider: "pi",+ phase: "session_setup",+ reason: null,+ retryability: "unknown",+ },+ });+ expect(errorSpy).toHaveBeenCalledWith(+ "Pi session initialization failed",+ expect.objectContaining({+ runtimeAdapter: "pi",+ initializationPhase: "session_setup",+ initMs: expect.any(Number),+ requestedModel: "anthropic/claude-opus-5",+ gatewayConfigured: true,+ taskId: "task-1",+ taskRunId: "run-1",+ errorDetail: expect.objectContaining({+ message: "Pi RPC startup failed",+ }),}),- }),- );- expect(append).toHaveBeenCalledWith(- "run-1",- expect.objectContaining({- notification: expect.objectContaining({- method: "_posthog/initialization_failed",- params: expect.objectContaining({- runtimeAdapter: "pi",- initializationPhase: "session_setup",- errorType: "error",+ );+ expect(append).toHaveBeenCalledWith(+ "run-1",+ expect.objectContaining({+ notification: expect.objectContaining({+ method: "_posthog/initialization_failed",+ params: expect.objectContaining({+ runtimeAdapter: "pi",+ initializationPhase: "session_setup",+ errorType: "error",+ }),}),}),- }),- );- expect(shutdown).toHaveBeenCalledOnce();+ );+ expect(shutdown).toHaveBeenCalledOnce();+ } finally {+ await server.stop();+ }});+ it.each(["start", "setThinkingLevel", "getState"] as const)(+ "stops the partial Pi client when %s fails",+ async (operation) => {+ const client = {+ start: vi.fn(async () => {}),+ setThinkingLevel: vi.fn(async () => {}),+ getState: vi.fn(async () => ({})),+ stop: vi.fn(async () => {}),+ onEvent: vi.fn(() => vi.fn()),+ onMcpToolPermissionRequest: vi.fn(() => vi.fn()),+ };+ const failure = new Error("Synthetic Pi setup failure");+ client[operation].mockRejectedValueOnce(failure);+ const createClient = vi+ .spyOn(piRpc, "createPiRpcClient")+ .mockReturnValue(client as unknown as piRpc.PiRpcClient);+ const syncInstructions = vi+ .spyOn(AgentInstructionFiles.prototype, "sync")+ .mockResolvedValue(null);+ const buildLocalTools = vi+ .spyOn(localTools, "buildLocalToolsServer")+ .mockReturnValue(null);+ const server = new PiAgentServer(+ config({+ model: "anthropic/claude-opus-5",+ reasoningEffort: "low",+ repositoryPath: tmpdir(),+ }),+ ) as unknown as {+ createSession(payload: Record<string, unknown>): Promise<void>;+ posthogAPI: Record<string, unknown>;+ session: unknown;+ };+ server.posthogAPI = {+ getTaskSession: vi.fn(async () => ({+ id: "session-1",+ content_sha256: null,+ })),+ downloadTaskSession: vi.fn(async () => null),+ getMcpRuntimeConfiguration: vi.fn(async () => ({+ servers: [],+ policies: [],+ })),+ getTask: vi.fn(async () => null),+ getTaskRun: vi.fn(async () => null),+ };+ try {+ await expect(+ server.createSession({+ task_id: "task-1",+ run_id: "run-1",+ team_id: 1,+ }),+ ).rejects.toBe(failure);+ expect(client.stop).toHaveBeenCalledOnce();+ expect(+ client.onMcpToolPermissionRequest.mock.results[0].value,+ ).toHaveBeenCalledOnce();+ expect(server.session).toBeNull();+ } finally {+ createClient.mockRestore();+ syncInstructions.mockRestore();+ buildLocalTools.mockRestore();+ }+ },+ );+it("emits RTK savings for a cloud Pi run", async () => {const enqueue = vi.fn();const server = new PiAgentServer(@@ -1253,7 +1341,9 @@ describe("PiAgentServer", () => {sseController: TestController,): Promise<void>;cancelSseController(sseController: TestController): void;+ posthogAPI: { updateTaskRun: ReturnType<typeof vi.fn> };};+ server.posthogAPI.updateTaskRun = vi.fn().mockResolvedValue({});server.createSession = vi.fn(async (sessionPayload) => {await initializationGate;server.session = { payload: sessionPayload, sseController: null };diff --git a/packages/agent/packages/agent/src/server/pi-agent-server.ts b/packages/agent/packages/agent/src/server/pi-agent-server.tsindex 95f65abbed20..5c925fee4fd7 100644--- a/packages/agent/packages/agent/src/server/pi-agent-server.ts+++ b/packages/agent/packages/agent/src/server/pi-agent-server.ts@@ -18,6 +18,7 @@ import {serializeError,type TaskRunArtifact,} from "@posthog/agent-contracts";+import type { StartupFailure } from "@posthog/agent-contracts/startup-failure";import type { TaskContext } from "@posthog/agent-contracts/task-context";import { AgentInstructionFiles } from "@posthog/harness/extensions/agent-instructions";import { resolveContextWikiPath } from "@posthog/harness/extensions/context-wiki";@@ -179,6 +180,7 @@ export class PiAgentServer {private server: ServerType | null = null;private session: PiCloudSession | null = null;private initializationPromise: Promise<void> | null = null;+ private startupFailure: StartupFailure | undefined;private pendingEvents: Record<string, unknown>[] = [];private sessionReadyBootMs?: number;private sessionInitMs?: number;@@ -287,7 +289,11 @@ export class PiAgentServer {distinct_id: "pi-agent-server",mode: this.config.mode,};- await this.initializeSession(payload, null);+ try {+ await this.initializeSession(payload, null);+ } catch (error) {+ if (!this.startupFailure) throw error;+ }}async stop(): Promise<void> {@@ -364,10 +370,12 @@ export class PiAgentServer {);await this.posthogAPI.updateTaskRun(this.config.taskId, this.config.runId, {- status: "failed",+ ...(this.startupFailure ? {} : { status: "failed" as const }),error_message: `Pi agent server crashed: ${message}`,error_type: "agent_server_crash",- state: { agent_version: this.agentVersion },+ state: {+ agent_version: this.agentVersion,+ },}).catch((updateError) =>this.logger.error(@@ -409,6 +417,13 @@ export class PiAgentServer {context.json({status: "ok",hasSession: this.session !== null,+ readiness: this.startupFailure+ ? "failed"+ : this.session+ ? "ready"+ : "initializing",+ startupFailure: this.startupFailure,+ runId: this.config.runId,bootMs: this.sessionReadyBootMs,sessionInitMs: this.sessionInitMs,boot: {@@ -558,12 +573,20 @@ export class PiAgentServer {return;}+ this.startupFailure = undefined;const initializationPromise = this.createSession(payload);this.initializationPromise = initializationPromise;const initStartedAt = Date.now();try {await initializationPromise;} catch (error) {+ const startupFailure: StartupFailure = {+ provider: "pi",+ phase: "session_setup",+ reason: null,+ retryability: "unknown",+ };+ this.startupFailure = startupFailure;const telemetry = this.createRunTelemetry(payload);telemetry?.append(payload.run_id, {type: "notification",@@ -573,6 +596,7 @@ export class PiAgentServer {method: POSTHOG_NOTIFICATIONS.INITIALIZATION_FAILED,params: {runtimeAdapter: "pi",+ startupFailure,initializationPhase: "session_setup",initMs: Date.now() - initStartedAt,requestedModel: this.config.model,@@ -777,18 +801,6 @@ export class PiAgentServer {const unsubscribeMcpPermissions = client.onMcpToolPermissionRequest((request) => this.handleMcpToolPermissionRequest(request),);- await client.start();- if (this.config.reasoningEffort) {- // Pi's ThinkingLevel has no ultracode notch; run it at its xhigh equivalent.- await client.setThinkingLevel(- this.config.reasoningEffort === "ultracode"- ? "xhigh"- : this.config.reasoningEffort,- );- }- const runtimeState = await client.getState();- this.applyModelContextWindow(runtimeState);- this.sessionFile = runtimeState.sessionFile ?? restoredSessionFile ?? null;const unsubscribe = () => {unsubscribeConversation();unsubscribeRuntime();@@ -796,19 +808,57 @@ export class PiAgentServer {unsubscribeMcpPermissions();};- this.session = {- payload,- runtime,- sseController: null,- unsubscribe,- providerRoute,- };+ try {+ await client.start();+ if (this.config.reasoningEffort) {+ // Pi's ThinkingLevel has no ultracode notch; run it at its xhigh equivalent.+ await client.setThinkingLevel(+ this.config.reasoningEffort === "ultracode"+ ? "xhigh"+ : this.config.reasoningEffort,+ );+ }+ const runtimeState = await client.getState();+ this.applyModelContextWindow(runtimeState);+ this.sessionFile =+ runtimeState.sessionFile ?? restoredSessionFile ?? null;+ this.session = {+ payload,+ runtime,+ sseController: null,+ unsubscribe,+ providerRoute,+ };+ } catch (error) {+ unsubscribe();+ await client+ .stop()+ .catch((stopError) =>+ this.logger.error(+ "Failed to stop Pi client after setup failure",+ stopError,+ ),+ );+ throw error;+ }this.sessionReadyBootMs = Math.round(process.uptime() * 1000);this.sessionInitMs = Date.now() - startedAt;- await this.posthogAPI.updateTaskRun(payload.task_id, payload.run_id, {- status: "in_progress",- state: { agent_version: this.agentVersion },- });+ await this.posthogAPI+ .updateTaskRun(+ payload.task_id,+ payload.run_id,+ {+ status: "in_progress",+ error_message: null,+ state: {+ agent_version: this.agentVersion,+ },+ },+ AbortSignal.timeout(2_000),+ )+ .catch((error) =>+ this.logger.error("Failed to persist agent startup success", error),+ );this.broadcast({type: "pi_run_started",timestamp: new Date().toISOString(),diff --git a/products/tasks/backend/facade/agents.py b/products/tasks/backend/facade/agents.pyindex 06677ee5406e..de74c94c78af 100644--- a/products/tasks/backend/facade/agents.py+++ b/products/tasks/backend/facade/agents.py@@ -32,6 +32,7 @@ from products.tasks.backend.logic.services.local_skills import (ENV_LOCAL_SKILLS_HOST_PATH,LocalSkillsCache,)+from products.tasks.backend.logic.services.startup_failure import AgentStartupFailedfrom products.tasks.backend.models import SandboxEnvironment@@ -53,6 +54,7 @@ __all__ = ["MCP_EXEC_SKILLS_FEATURE_FLAG","ENV_LOCAL_SKILLS_HOST_PATH","AgentTurnFailed",+ "AgentStartupFailed","CustomPromptSandboxContext","EmptyAgentTurnError","LocalSkillsCache",diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex f29b519858aa..148dc2fee4e7 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -2971,6 +2971,13 @@ _PROTECTED_RUN_STATE_KEYS = frozenset("token_cost_incomplete","compute_cost","failure_error_type",+ "startup_failure",+ "startup_started_at",+ "startup_completed_at",+ "startup_provider",+ "startup_runtime",+ "agent_startup_attempt",+ "agent_startup_success","unprocessed_request_ids",TASK_OWNERSHIP_VERSION_STATE_KEY,"pr_authorship_mode",diff --git a/products/tasks/backend/facade/contracts.py b/products/tasks/backend/facade/contracts.pyindex cf6e874a1b34..d1e7bb920263 100644--- a/products/tasks/backend/facade/contracts.py+++ b/products/tasks/backend/facade/contracts.py@@ -75,6 +75,36 @@ class AgentTaskRunDTO:workflow_id: str+@dataclass(frozen=True, kw_only=True)+class AgentStartupFailure:+ provider: Literal["claude", "codex", "pi"]+ phase: Literal["session_setup", "sdk_initialization", "setup_hooks"]+ reason: (+ Literal[+ "credential_unavailable",+ "credential_service_unavailable",+ "org_pin_api_key_conflict",+ "org_verify_failed",+ "org_pin_mismatch",+ "managed_settings_invalid",+ "remote_settings_required_unavailable",+ "gateway_signin_required",+ "gateway_access_denied",+ "proxy_invalid",+ "temp_dir_unusable",+ "cwd_unavailable",+ "shell_tool_missing",+ "session_held_by_background",+ "worktree_resume_refused",+ "worktree_unverified",+ "cli_version_too_old",+ "bypass_root",+ ]+ | None+ )+ retryability: Literal["requires_action", "transient", "unknown"]++@dataclass(frozen=True, kw_only=True)class SessionFile:"""A text file a custom-prompt session puts in the sandbox before its first turn.diff --git a/products/tasks/backend/logic/services/agent_server_launcher.py b/products/tasks/backend/logic/services/agent_server_launcher.pyindex aac3877aee76..4881f3e9e333 100644--- a/products/tasks/backend/logic/services/agent_server_launcher.py+++ b/products/tasks/backend/logic/services/agent_server_launcher.py@@ -61,6 +61,7 @@ from products.tasks.backend.logic.services.sandbox import (health_check_timeout_seconds,wait_for_health_check,)+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError, raise_startup_failureif TYPE_CHECKING:from products.tasks.backend.temporal.process_task.utils import McpServerConfig@@ -648,6 +649,13 @@ class AgentServerLaunchMixin(SandboxBase):start_and_health_ms = int((time.perf_counter() - start_time) * 1000)if launch_result.exit_code != 0:health_duration_ms = _health_duration_ms(launch_result.stdout)+ try:+ raise_startup_failure(launch_result.stdout, self.id)+ except AgentServerStartupError as error:+ error.context.update(start_and_health_ms=start_and_health_ms)+ if wait_for_health and health_duration_ms is not None:+ error.context.update(health_poll_ms=health_duration_ms)+ raiseif wait_for_health and health_duration_ms is not None:diagnostics = self._diagnose_startup_failure(allowed_domains)credential_error = self._credential_unavailable_error(@@ -719,6 +727,7 @@ class AgentServerLaunchMixin(SandboxBase):codex_model_access: str | None = None,codex_run_token: str | None = None,sandbox_runtime: str | None = None,+ before_launch: Callable[[], None] | None = None,) -> int | None:"""Start the agent-server HTTP server in the sandbox.@@ -735,6 +744,9 @@ class AgentServerLaunchMixin(SandboxBase):if preflight.reused:return 0 if wait_for_health else None+ if before_launch:+ before_launch()+repo_path: str | None = Noneif repository:org, repo = repository.lower().split("/")diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.pyindex 7d7abb0ccbc5..bc9c551a3ed0 100644--- a/products/tasks/backend/logic/services/custom_prompt_internals.py+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py@@ -22,9 +22,13 @@ from posthog.storage import object_storagefrom posthog.storage.object_storage import ObjectStorageErrorfrom posthog.temporal.oauth import PosthogMcpScopes-from products.tasks.backend.exceptions import is_transient_failure_error_typefrom products.tasks.backend.facade.api import upload_task_run_artifacts-from products.tasks.backend.facade.contracts import AgentTaskRunDTO, SessionFile+from products.tasks.backend.facade.contracts import AgentStartupFailure, AgentTaskRunDTO, SessionFile+from products.tasks.backend.logic.services.startup_failure import (+ AgentStartupFailed,+ SandboxRunFailed as SandboxRunFailed,+ read_startup_failure,+)from products.tasks.backend.models import MCPBuiltInAgentKey, Task, TaskRun, stamp_pending_user_message_idif TYPE_CHECKING:@@ -274,16 +278,6 @@ class AgentTurnFailed(RuntimeError):return self.category in UPSTREAM_RETRYABLE_ERROR_CATEGORIES-class SandboxRunFailed(RuntimeError):- def __init__(self, message: str, *, error_type: str | None) -> None:- super().__init__(message)- self.error_type = error_type-- @property- def transient(self) -> bool:- return is_transient_failure_error_type(self.error_type)--SESSION_FILE_ARTIFACT_TYPE = "context"@@ -685,6 +679,7 @@ async def poll_for_turn(refreshed_status=refreshed.status,error_message=refreshed.error_message,failure_error_type=(refreshed.state or {}).get("failure_error_type"),+ startup_failure=read_startup_failure(refreshed.state),printed_lines=printed_lines,original_skip_lines=original_skip_lines,verbose=verbose,@@ -711,6 +706,7 @@ async def poll_for_turn(refreshed_status=refreshed.status,error_message=refreshed.error_message,failure_error_type=(refreshed.state or {}).get("failure_error_type"),+ startup_failure=read_startup_failure(refreshed.state),printed_lines=printed_lines,original_skip_lines=original_skip_lines,verbose=verbose,@@ -894,6 +890,7 @@ async def _drain_final_log(refreshed_status: str,error_message: str | None = None,failure_error_type: str | None = None,+ startup_failure: AgentStartupFailure | None = None,printed_lines: int,original_skip_lines: int,verbose: bool,@@ -910,6 +907,13 @@ async def _drain_final_log(stale earlier-turn response in multi-turn sessions). For single-turn callers `original_skip_lines == 0`, sothe scan covers the full log as before. The walk is idempotent and only runs once at terminal status."""+ if refreshed_status == TaskRun.Status.FAILED and startup_failure is not None:+ raise AgentStartupFailed(+ failure=startup_failure,+ task_id=str(task_run.task_id),+ run_id=str(task_run.id),+ error_type=failure_error_type,+ )final_state = await _read_turn_log_with_retry(task_run, skip_lines=original_skip_lines, context="drain_final_log")printed_lines = _stream_new_lines(final_state.full_log, printed_lines, verbose=verbose, output_fn=output_fn)if final_state.last_message:diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.pyindex 9b9aa680819d..bf0155aec8bd 100644--- a/products/tasks/backend/logic/services/sandbox.py+++ b/products/tasks/backend/logic/services/sandbox.py@@ -47,6 +47,7 @@ from products.tasks.backend.logic.services.sandbox_config import (SANDBOX_TTL_SECONDS,VM_SANDBOX_CPU_CORES,)+from products.tasks.backend.logic.services.startup_failure import raise_startup_failureif TYPE_CHECKING:from products.tasks.backend.temporal.process_task.utils import McpServerConfig@@ -686,6 +687,7 @@ class SandboxBase(ABC):codex_model_access: str | None = None,codex_run_token: str | None = None,sandbox_runtime: str | None = None,+ before_launch: Callable[[], None] | None = None,) -> int | None:"""Start the agent-server HTTP server in the sandbox.@@ -906,6 +908,8 @@ def wait_for_health_check("""health_script = build_health_check_command(port, max_attempts, poll_interval, pid_file)result = execute(health_script, timeout_seconds=health_check_timeout_seconds(max_attempts, poll_interval))+ if result.exit_code != 0:+ raise_startup_failure(result.stdout, sandbox_id)if "claude_credential_unavailable" in result.stdout:from products.tasks.backend.exceptions import ProcessTaskFatalError@@ -954,15 +958,17 @@ def build_health_check_command(f" body=$(curl -q -s --max-time {HEALTH_CURL_MAX_TIME_SECONDS} http://localhost:{port}/health); "" status=$?; "' if [ "$status" = "0" ]; then '- ' case "$body" in *claude_credential_unavailable*) echo "claude_credential_unavailable"; exit 1;; '- '*codex_credential_unavailable*) echo "codex_credential_unavailable"; exit 1;; esac; '" python3 -c '""import json, sys; ""payload = json.loads(sys.argv[1]); "'ready = payload.get("status") == "ok" and payload.get("hasSession") is True; '- 'sys.exit(0 if ready else 2 if payload.get("initializationPhase") == "setup_hooks" else 1)'+ 'sys.exit(0 if ready else 3 if payload.get("readiness") == "failed" '+ 'else 2 if payload.get("initializationPhase") == "setup_hooks" else 1)''\' "$body"; ready=$?; '' if [ "$ready" = "0" ]; then echo "ok:$i"; exit 0; fi; '+ ' if [ "$ready" = "3" ]; then printf "%s\\n" "$body"; exit 1; fi; '+ ' case "$body" in *claude_credential_unavailable*) echo "claude_credential_unavailable"; exit 1;; '+ '*codex_credential_unavailable*) echo "codex_credential_unavailable"; exit 1;; esac; 'f' if [ "$ready" = "2" ]; then max_attempts={hooks_max_attempts}; budget={hooks_budget_seconds}; fi; '" fi; "' if [ "$i" -ge "$max_attempts" ] || [ "$SECONDS" -ge "$budget" ]; then exit 1; fi; 'diff --git a/products/tasks/backend/logic/services/startup_failure.py b/products/tasks/backend/logic/services/startup_failure.pynew file mode 100644index 000000000000..8ca5e083147e--- /dev/null+++ b/products/tasks/backend/logic/services/startup_failure.py@@ -0,0 +1,103 @@+import json+from collections.abc import Mapping+from dataclasses import replace+from typing import Literal++from pydantic import TypeAdapter, ValidationError++from products.tasks.backend.exceptions import SandboxExecutionError, is_transient_failure_error_type+from products.tasks.backend.facade.contracts import AgentStartupFailure++_startup_failure_adapter = TypeAdapter(AgentStartupFailure)++_ACTION_REQUIRED_MESSAGES = {+ "org_pin_api_key_conflict": "The agent's API key conflicts with its organization settings. Use credentials for the configured organization, then retry.",+ "org_pin_mismatch": "The agent's organization settings do not match its credentials. Check the configured organization and credentials, then retry.",+ "managed_settings_invalid": "The agent's managed settings are invalid. Check the provider's managed settings, then retry.",+ "gateway_signin_required": "The agent needs to sign in to its provider. Sign in to the provider, then retry.",+ "gateway_access_denied": "The provider denied access while loading the agent's managed settings. Check the provider account's permissions, then retry.",+ "proxy_invalid": "The agent's proxy configuration is invalid. Check the sandbox proxy URL, then retry.",+ "shell_tool_missing": "A shell required by the agent is unavailable. Update the sandbox image to include a supported shell, then retry.",+ "cli_version_too_old": "The agent's provider runtime is too old. Update the provider runtime in the sandbox image, then retry.",+ "bypass_root": "The provider rejected the agent's permission configuration. Run the agent as a non-root user or update its permission settings, then retry.",+}+_ACTION_REQUIRED_REASONS = frozenset({"credential_unavailable", *_ACTION_REQUIRED_MESSAGES})+_TRANSIENT_REASONS = frozenset({"credential_service_unavailable", "session_held_by_background", "worktree_unverified"})+++def read_startup_failure(state: Mapping[str, object] | None) -> AgentStartupFailure | None:+ try:+ return _startup_failure_adapter.validate_python((state or {}).get("startup_failure"))+ except ValidationError:+ return None+++class AgentServerStartupError(SandboxExecutionError):+ def __init__(self, failure: AgentStartupFailure, run_id: str, sandbox_id: str) -> None:+ self.startup_failure = failure+ self.run_id = run_id+ super().__init__(+ _ACTION_REQUIRED_MESSAGES.get(failure.reason or "", "The agent could not initialize its session."),+ {"sandbox_id": sandbox_id, "run_id": run_id},+ cause=RuntimeError("Agent session initialization failed"),+ capture=False,+ )+++def raise_startup_failure(stdout: str, sandbox_id: str) -> None:+ for line in stdout.splitlines():+ if len(line) > 4096:+ continue+ try:+ payload = json.loads(line)+ except (ValueError, TypeError):+ continue+ if not isinstance(payload, dict) or payload.get("readiness") != "failed" or payload.get("hasSession") is True:+ continue+ run_id = payload.get("runId")+ if not isinstance(run_id, str) or not run_id:+ continue+ try:+ failure = _startup_failure_adapter.validate_python(payload.get("startupFailure"))+ except ValidationError:+ continue+ reason = (+ failure.reason+ if failure.provider == "claude"+ or failure.reason in {"credential_unavailable", "credential_service_unavailable"}+ else None+ )+ retryability: Literal["requires_action", "transient", "unknown"] = (+ "requires_action"+ if reason in _ACTION_REQUIRED_REASONS+ else "transient"+ if reason in _TRANSIENT_REASONS+ else "unknown"+ )+ raise AgentServerStartupError(replace(failure, reason=reason, retryability=retryability), run_id, sandbox_id)+++class SandboxRunFailed(RuntimeError):+ def __init__(self, message: str, *, error_type: str | None) -> None:+ super().__init__(message)+ self.error_type = error_type++ @property+ def transient(self) -> bool:+ return is_transient_failure_error_type(self.error_type)+++class AgentStartupFailed(SandboxRunFailed):+ def __init__(+ self, *, failure: AgentStartupFailure, task_id: str, run_id: str, error_type: str | None = None+ ) -> None:+ super().__init__("The agent failed during startup.", error_type=error_type)+ self.failure = failure+ self.task_id = task_id+ self.run_id = run_id++ @property+ def transient(self) -> bool:+ return self.failure.retryability != "requires_action" and (+ self.failure.retryability == "transient" or super().transient+ )diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.pyindex b3066022d925..d8b589f75d65 100644--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py@@ -104,6 +104,7 @@ from products.tasks.backend.logic.services.sandbox import (SandboxWorkload,health_check_timeout_seconds,)+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupErrordef _agent_server_launch_command(mock_execute: Any) -> str:@@ -1046,15 +1047,35 @@ class TestModalSandboxAgentServer:run_id="run-456",)- def test_start_agent_server_raises_on_health_check_failure(self, mock_sandbox: Any):+ @pytest.mark.parametrize("structured", [False, True])+ def test_start_agent_server_raises_on_health_check_failure(self, mock_sandbox: Any, structured: bool):+ payload = (+ json.dumps(+ {+ "hasSession": False,+ "readiness": "failed",+ "runId": "run-456",+ "startupFailure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": "proxy_invalid",+ "retryability": "requires_action",+ },+ }+ )+ + "\n"+ if structured+ else ""+ )+def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:if "./node_modules/.bin/agent-server" in command:- return ExecutionResult(stdout="__posthog_agent_health_ms=120000", stderr="", exit_code=1)+ return ExecutionResult(stdout=payload + "__posthog_agent_health_ms=120000", stderr="", exit_code=1)return ExecutionResult(stdout="", stderr="", exit_code=0)mock_sandbox.execute = MagicMock(side_effect=execute)with patch.object(mock_sandbox, "_diagnose_startup_failure", return_value={"failure_reason": "not ready"}):- with pytest.raises(SandboxExecutionError, match="Agent-server failed to start") as error:+ with pytest.raises(AgentServerStartupError if structured else SandboxExecutionError) as error:mock_sandbox.start_agent_server(repository="posthog/posthog",task_id="task-123",@@ -1062,6 +1083,7 @@ class TestModalSandboxAgentServer:)assert error.value.context["health_poll_ms"] == 120000+ assert isinstance(error.value.context["start_and_health_ms"], int)assert mock_sandbox.supports_combined_agent_server_start_and_health() is True@pytest.mark.parametrize(diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py b/products/tasks/backend/logic/services/tests/test_sandbox_health_check.pyindex a8f911ed6bc2..dedaed627730 100644--- a/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py+++ b/products/tasks/backend/logic/services/tests/test_sandbox_health_check.py@@ -1,4 +1,5 @@import os+import jsonimport timeimport subprocessfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer@@ -18,6 +19,7 @@ from products.tasks.backend.logic.services.sandbox import (build_health_check_command,health_check_budget_seconds,)+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupError, raise_startup_failuredef _path_with_fake_curl(tmp_path: Path, body: str, delay_seconds: float) -> dict[str, str]:@@ -63,6 +65,11 @@ def test_health_check_ignores_curl_config(tmp_path: Path) -> None:[('{"status":"ok","hasSession":true}', 0, "ok:1"),('{"status":"ok","hasSession":false}', 1, ""),+ (+ '{"status":"ok","hasSession":false,"readiness":"failed"}',+ 1,+ '{"status":"ok","hasSession":false,"readiness":"failed"}',+ ),('{"status":"error","code":"codex_credential_unavailable"}', 1, "codex_credential_unavailable"),],)@@ -183,3 +190,74 @@ def test_preflight_reports_only_the_capabilities_the_binary_carries(if line.startswith(AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX)]assert sorted(reported) == sorted(capabilities)+++ ("provider", "reason", "reported_retryability", "expected_retryability"),+ [+ ("claude", "proxy_invalid", "unknown", "requires_action"),+ ("claude", "worktree_unverified", "requires_action", "transient"),+ ("claude", None, "requires_action", "unknown"),+ ("codex", "credential_service_unavailable", "requires_action", "transient"),+ ],+)+def test_health_check_carries_validated_startup_failure(+ tmp_path: Path, provider, reason, reported_retryability, expected_retryability+) -> None:+ payload = {+ "status": "ok",+ "hasSession": False,+ "readiness": "failed",+ "runId": "example-run",+ "startupFailure": {+ "provider": provider,+ "phase": "sdk_initialization",+ "reason": reason,+ "retryability": reported_retryability,+ },+ }+ result = subprocess.run(+ ["bash", "-c", build_health_check_command(port=1, max_attempts=1, poll_interval=0.0)],+ env=_path_with_fake_curl(tmp_path, json.dumps(payload), 0.0),+ capture_output=True,+ text=True,+ timeout=10,+ )+ assert result.returncode == 1+ with pytest.raises(AgentServerStartupError) as raised:+ raise_startup_failure(result.stdout, "example-sandbox")+ assert raised.value.run_id == "example-run"+ assert raised.value.startup_failure.reason == reason+ assert raised.value.startup_failure.retryability == expected_retryability+++ "change",+ [+ {"runId": None},+ {"hasSession": True},+ {"readiness": "ready"},+ {+ "startupFailure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": "arbitrary-reason",+ "retryability": "requires_action",+ }+ },+ ],+)+def test_health_check_ignores_unconfirmed_startup_failures(change) -> None:+ payload = {+ "hasSession": False,+ "readiness": "failed",+ "runId": "example-run",+ "startupFailure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": "proxy_invalid",+ "retryability": "requires_action",+ },+ **change,+ }+ raise_startup_failure(json.dumps(payload), "example-sandbox")diff --git a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/start_agent_server.pyindex 1abe6384165b..ce67fc4b731f 100644--- a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py+++ b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py@@ -4,11 +4,11 @@ import shleximport threadingfrom collections.abc import Sequencefrom concurrent.futures import Future, ThreadPoolExecutor-from dataclasses import dataclass, field+from dataclasses import asdict, dataclass, fieldfrom datetime import UTC, datetimefrom django.conf import settings-from django.db import connection+from django.db import connection, transactionimport posthoganalyticsfrom temporalio import activity@@ -41,12 +41,15 @@ from products.tasks.backend.logic.services.connection_token import (from products.tasks.backend.logic.services.launch_preparation_metrics import launch_preparation_metric_contextfrom products.tasks.backend.logic.services.modal_sandbox import ModalSandboxfrom products.tasks.backend.logic.services.sandbox import (+ CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE,+ CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE,REPO_READY_FILE,SUBSCRIPTION_CLI_FLAGS,SandboxBase,get_sandbox_class_for_sandbox_id,sandbox_repo_path,)+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupErrorfrom products.tasks.backend.logic.stream.redis_stream import release_task_run_milestone_claimsfrom products.tasks.backend.models import Task, TaskRunfrom products.tasks.backend.redis import run_uses_dedicated_stream@@ -607,6 +610,37 @@ def _prepare_launch(ctx: TaskProcessingContext, scopes: PosthogMcpScopes, sandbo)+def _record_agent_startup_attempt(ctx: TaskProcessingContext, runtime: str) -> None:+ TaskRun.update_state_atomic(+ ctx.run_id,+ updates={+ "agent_startup_attempt": {+ "started_at": datetime.now(UTC).isoformat(),+ "repository": (ctx.repository or "").lower(),+ "provider": "pi" if ctx.task_runtime == "pi" else ctx.runtime_adapter or "claude",+ "runtime": runtime,+ },+ },+ remove_keys=["startup_failure", "startup_completed_at", "startup_provider"],+ )+++def _record_agent_startup_success(ctx: TaskProcessingContext) -> None:+ with transaction.atomic():+ state = TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).values_list("state", flat=True).first()+ attempt = (state or {}).get("agent_startup_attempt")+ TaskRun.update_state_atomic(+ ctx.run_id,+ updates={"agent_startup_success": {**attempt, "completed_at": datetime.now(UTC).isoformat()}}+ if isinstance(attempt, dict)+ else {},+ remove_keys=["startup_failure"],+ )+ TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).exclude(+ status__in=[TaskRun.Status.COMPLETED, TaskRun.Status.FAILED, TaskRun.Status.CANCELLED]+ ).update(error_message=None)++def _invoke_start_agent_server(sandbox: SandboxBase,ctx: TaskProcessingContext,@@ -618,6 +652,9 @@ def _invoke_start_agent_server(try:_enforce_subscription_support(sandbox, ctx)health_duration_ms = sandbox.start_agent_server(+ before_launch=lambda: _record_agent_startup_attempt(+ ctx, sandbox_runtime_label(ctx.use_modal_vm_sandbox) if isinstance(sandbox, ModalSandbox) else "unknown"+ ),repository=ctx.repository if len(ctx.repositories) <= 1 else None,task_id=ctx.task_id,run_id=ctx.run_id,@@ -777,6 +814,26 @@ def _read_agent_boot_metrics(sandbox: SandboxBase, input: StartAgentServerInput)return session_init_ms, boot_phases_ms+def _record_agent_startup_failure(ctx: TaskProcessingContext, sandbox_id: str, error: Exception) -> None:+ if not isinstance(error, AgentServerStartupError) or error.run_id != ctx.run_id:+ return+ TaskRun.update_state_atomic(ctx.run_id, updates={"startup_failure": asdict(error.startup_failure)})+ if error.startup_failure.retryability == "requires_action":+ message = error.message+ if error.startup_failure.reason == "credential_unavailable":+ message = (+ CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE+ if error.startup_failure.provider == "codex"+ else CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE+ )+ raise ProcessTaskFatalError(+ message,+ {"run_id": ctx.run_id, "sandbox_id": sandbox_id},+ error,+ capture=False,+ ) from error++@activity.defn@asyncify@posthoganalytics.scoped(fresh=True, capture_exceptions=False)@@ -843,6 +900,7 @@ def start_agent_server(input: StartAgentServerInput) -> StartAgentServerOutput:)invoke_status = "COMPLETED" if health_poll_ms is not None else "FAILED"health_status = "FAILED"+ _record_agent_startup_failure(ctx, input.sandbox_id, error)raisefinally:if start_and_health_ms is None:@@ -887,14 +945,19 @@ def start_agent_server(input: StartAgentServerInput) -> StartAgentServerOutput:origin_product=ctx.origin_product,runtime=runtime,) as health_timer:- sandbox.wait_for_agent_server_ready(- params.agentsh_domains,- claude_model_access=ctx.model_access.access_for("claude"),- )+ try:+ sandbox.wait_for_agent_server_ready(+ params.agentsh_domains,+ claude_model_access=ctx.model_access.access_for("claude"),+ )+ except Exception as error:+ _record_agent_startup_failure(ctx, input.sandbox_id, error)+ raiseinvoke_ms = invoke_timer.elapsed_mshealth_poll_ms = health_timer.elapsed_ms_record_agent_server_launch(sandbox, ctx, params)+ _record_agent_startup_success(ctx)_record_network_enforcement_observation(ctx)log_with_activity_context(f"Agent server started at {input.sandbox_url}", run_id=ctx.run_id)@@ -1093,6 +1156,7 @@ def await_agent_server_ready(input: StartAgentServerInput) -> StartAgentServerOuif agentsh_domains is not None:_emit_agentsh_log_tail(ctx, sandbox)_emit_agent_server_log_tail(ctx, sandbox)+ _record_agent_startup_failure(ctx, input.sandbox_id, error)raiseif attempt > 1:@@ -1106,6 +1170,7 @@ def await_agent_server_ready(input: StartAgentServerInput) -> StartAgentServerOu_record_network_enforcement_observation(ctx)+ _record_agent_startup_success(ctx)log_with_activity_context(f"Agent server ready at {input.sandbox_url}", run_id=ctx.run_id)activity.logger.info(f"Agent server ready at {input.sandbox_url} for task {ctx.task_id}")diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.pyindex ab705984fffe..c099df0ad183 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py@@ -15,9 +15,11 @@ from products.tasks.backend.exceptions import (SandboxRateLimitedError,SandboxTimeoutError,)+from products.tasks.backend.facade.contracts import AgentStartupFailurefrom products.tasks.backend.logic.services.launch_preparation_metrics import record_launch_preparation_msfrom products.tasks.backend.logic.services.modal_sandbox import ModalSandboxfrom products.tasks.backend.logic.services.sandbox import ExecutionResult, sandbox_repo_path+from products.tasks.backend.logic.services.startup_failure import AgentServerStartupErrorfrom products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContextfrom products.tasks.backend.temporal.process_task.activities.start_agent_server import (CollectAgentShadowResultInput,@@ -50,6 +52,20 @@ def organization_state(mocker):return state[email protected](autouse=True)+def task_run_state(mocker):+ mocker.patch("products.tasks.backend.temporal.process_task.activities.start_agent_server.transaction.atomic")+ mocker.patch(+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.update_state_atomic"+ )+ runs = mocker.patch(+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.objects.filter"+ )+ state = runs.return_value.values_list.return_value.first+ state.return_value = None+ return state++@time_machine.travel("2026-08-06T12:01:30Z", tick=False)def test_record_boot_total_excludes_wizard_time_and_labels_runtime(mocker) -> None:record_metric = mocker.patch(@@ -551,6 +567,71 @@ async def test_await_agent_server_ready_relaunches_on_activity_retries(mocker, arecord_retry.assert_not_called()[email protected]("boot_path", ["classic", "combined", "overlap"])+ "retryability",+ ["requires_action", "transient", "unknown", "other_run", "claude_credential", "codex_credential", None],+)+async def test_startup_failure_controls_readiness_retries(mocker, task_run_state, boot_path, retryability) -> None:+ context = _context()+ credential = retryability in {"claude_credential", "codex_credential"}+ requires_action = retryability == "requires_action" or credential+ task_run_state.return_value = {+ "startup_failure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": "proxy_invalid",+ "retryability": "requires_action",+ }+ }+ sandbox = mocker.Mock(id="sandbox-id")+ sandbox.supports_combined_agent_server_start_and_health.return_value = boot_path == "combined"+ failure = (+ AgentServerStartupError(+ AgentStartupFailure(+ provider="codex" if retryability == "codex_credential" else "claude",+ phase="sdk_initialization",+ reason="credential_unavailable" if credential else "proxy_invalid",+ retryability="requires_action" if retryability == "other_run" or credential else retryability,+ ),+ "another-run" if retryability == "other_run" else context.run_id,+ sandbox.id,+ )+ if retryability is not None+ else SandboxExecutionError(+ "Agent startup failed", {"run_id": context.run_id}, RuntimeError("startup"), capture=False+ )+ )+ sandbox.wait_for_agent_server_ready.side_effect = failure+ prefix = "products.tasks.backend.temporal.process_task.activities.start_agent_server."+ mocker.patch(prefix + "get_sandbox_class_for_sandbox_id", **{"return_value.get_by_id.return_value": sandbox})+ mocker.patch(prefix + "current_activity_attempt", return_value=1)+ mocker.patch(prefix + "_prepare_launch", return_value=mocker.Mock(agentsh_domains=None))+ mocker.patch(prefix + "_launch_agent_shadow")+ mocker.patch(prefix + "_emit_agent_server_log_tail")+ mocker.patch(prefix + "_invoke_start_agent_server", side_effect=failure if boot_path == "combined" else None)+ activity_fn = await_agent_server_ready if boot_path == "overlap" else start_agent_server++ with pytest.raises(ProcessTaskFatalError if requires_action else SandboxExecutionError) as error:+ await activity_fn(+ StartAgentServerInput(+ context=context,+ sandbox_id=sandbox.id,+ sandbox_url="https://sandbox.example",+ boot_path=boot_path,+ )+ )++ assert error.value.non_retryable is requires_action+ if credential:+ assert "Settings > Harness" in str(error.value)+ assert ("credential_request" in str(error.value)) is (retryability == "claude_credential")+ elif requires_action:+ assert "Check the sandbox proxy URL, then retry." in str(error.value)+ if not requires_action:+ assert error.value is failure++@pytest.mark.parametrize("pending_deletion", [False, True])async def test_await_agent_server_ready_records_failed_relaunch(mocker, organization_state, pending_deletion: booldiff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.pyindex 7f8d32ea4cb7..2a73ea11e700 100644--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py@@ -1811,6 +1811,25 @@ class TestPosthogCodeSandboxMint:attributes={"surface": "sandbox", "ai_product": "posthog_code", "reason": counted_reason},)+ @pytest.mark.django_db+ def test_retry_after_credit_top_up_rechecks_credit_availability(self, mint_settings):+ team = self._team(paid=True)+ mint_settings.SANDBOX_AI_GATEWAY_PRODUCTS = "posthog_code"+ with (+ patch(_CREDIT_LOOKUP, side_effect=["posthog_code_credits_exhausted", None]) as credit,+ patch(+ "products.tasks.backend.temporal.process_task.utils.mint_scoped_token", return_value="synthetic-token"+ ) as mint,+ ):+ with pytest.raises(GatewayTokenUnavailableError) as refused:+ ai_gateway_env_vars(team_id=team.id, origin_product="user_created", model="claude-opus-5")+ assert refused.value.non_retryable is True+ mint.assert_not_called()+ env = ai_gateway_env_vars(team_id=team.id, origin_product="user_created", model="claude-opus-5")+ assert env["AI_GATEWAY_TOKEN"] == "synthetic-token"+ assert credit.call_count == 2+ mint.assert_called_once()+@pytest.mark.django_db@pytest.mark.parametrize("setting", ["SANDBOX_AI_GATEWAY_URL", "SANDBOX_AI_GATEWAY_MINT_KEY"])def test_a_posthog_code_run_with_the_gateway_unconfigured_fails_and_logs(self, mint_settings, setting):diff --git a/products/tasks/backend/tests/agent_log_fixtures.py b/products/tasks/backend/tests/agent_log_fixtures.pyindex 5f9bff637de7..1af617854726 100644--- a/products/tasks/backend/tests/agent_log_fixtures.py+++ b/products/tasks/backend/tests/agent_log_fixtures.py@@ -164,6 +164,7 @@ def _cost_less_usage_update_line(used: int = 1000) -> str:@dataclassclass FakeTaskRun:id: str = "run-1"+ task_id: str = "task-1"log_url: str = "s3://fake/log"status: str = "running"error_message: str | None = Nonediff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.pyindex b5e8f9d08882..1638708cd016 100644--- a/products/tasks/backend/tests/test_api.py+++ b/products/tasks/backend/tests/test_api.py@@ -7288,6 +7288,43 @@ class TestTaskSummariesAPI(BaseTaskAPITest):class TestTaskRunAPI(BaseTaskAPITest):+ @parameterized.expand([(False,), (True,)])+ @patch("products.tasks.backend.models.TaskRun.publish_stream_state_event")+ def test_patch_cannot_forge_startup_outcomes(self, sandbox_agent: bool, _mock_publish) -> None:+ task = self.create_task()+ attempt = {+ "started_at": django_timezone.now().isoformat(),+ "repository": "example/research",+ "provider": "claude",+ "runtime": "gvisor",+ }+ state = {"agent_startup_attempt": attempt, "agent_startup_success": attempt}+ run = TaskRun.objects.create(team=self.team, task=task, status=TaskRun.Status.IN_PROGRESS, state=state)+ client = self._sandbox_oauth_client(task.id, internal_scope=True) if sandbox_agent else self.client+ forged = {+ "agent_startup_attempt": {**attempt, "provider": "pi"},+ "agent_startup_success": {**attempt, "provider": "pi"},+ "startup_failure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": "proxy_invalid",+ "retryability": "requires_action",+ },+ "startup_started_at": "forged",+ "startup_completed_at": "forged",+ "startup_provider": "pi",+ "startup_runtime": "vm",+ }+ for payload in [+ {"state": forged},+ {"state_remove_keys": list(forged)},+ {"state_append": forged},+ ]:+ response = client.patch(f"/api/projects/@current/tasks/{task.id}/runs/{run.id}/", payload, format="json")+ assert response.status_code == status.HTTP_200_OK+ run.refresh_from_db()+ assert {key: run.state.get(key) for key in forged} == {key: state.get(key) for key in forged}+def _create_run_for_origin(self, origin_product: Task.OriginProduct) -> tuple[Task, TaskRun]:task = Task.objects.create(team=self.team,diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.pyindex c1cc4f5c9429..6186d0651239 100644--- a/products/tasks/backend/tests/test_multi_turn_session.py+++ b/products/tasks/backend/tests/test_multi_turn_session.py@@ -39,6 +39,7 @@ from products.tasks.backend.logic.services.custom_prompt_multi_turn_runner impor_EMPTY_TURN_RETRY_NUDGE,MultiTurnSession,)+from products.tasks.backend.logic.services.startup_failure import AgentStartupFailedfrom products.tasks.backend.models import Task, TaskRunfrom products.tasks.backend.tests.agent_log_fixtures import (FakeTaskRun,@@ -1130,6 +1131,46 @@ class TestPollForTurnSurfacesAgentError:the exception, so a caller separates a retryable provider outage from a scout defect withoutmatching the message text."""+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "reason,retryability,error_type,transient",+ [+ ("proxy_invalid", "requires_action", "SandboxExecutionError", False),+ (None, "unknown", "SandboxExecutionError", True),+ (None, "unknown", None, False),+ ("session_held_by_background", "transient", None, True),+ ],+ )+ async def test_startup_failure_survives_without_session_logs(self, reason, retryability, error_type, transient):+ fake_task_run = FakeTaskRun(+ status="failed",+ state={+ "failure_error_type": error_type,+ "startup_failure": {+ "provider": "claude",+ "phase": "sdk_initialization",+ "reason": reason,+ "retryability": retryability,+ },+ },+ )+ with (+ patch("posthog.storage.object_storage.read", return_value=None),+ patch("asyncio.sleep", new=AsyncMock()),+ patch("products.tasks.backend.logic.services.custom_prompt_internals.POLL_INTERVAL_SECONDS", 0),+ patch("products.tasks.backend.models.TaskRun.objects.get", return_value=fake_task_run),+ ):+ with pytest.raises(AgentStartupFailed) as error:+ await poll_for_turn(fake_task_run, skip_lines=0)++ assert error.value.failure.reason == reason+ assert error.value.failure.retryability == retryability+ assert error.value.task_id == fake_task_run.task_id+ assert error.value.run_id == fake_task_run.id+ assert isinstance(error.value, SandboxRunFailed)+ assert error.value.error_type == error_type+ assert error.value.transient is transient+@parameterized.expand([("upstream_provider_failure", "API Error: 429 rate_limit_error", True),@@ -1233,6 +1274,7 @@ class TestPollForTurnSurfacesAgentError:"state,expected_error_type,expected_transient",[(None, None, False),+ ({"startup_failure": {"reason": "proxy_invalid", "retryability": "requires_action"}}, None, False),({"failure_error_type": "RepositoryCloneError"}, "RepositoryCloneError", True),({"failure_error_type": "RepositoryAccessError"}, "RepositoryAccessError", False),],diff --git a/products/tasks/backend/tests/test_startup_recovery.py b/products/tasks/backend/tests/test_startup_recovery.pynew file mode 100644index 000000000000..576a2351f0ed--- /dev/null+++ b/products/tasks/backend/tests/test_startup_recovery.py@@ -0,0 +1,68 @@+from datetime import timedelta++import time_machine+from posthog.test.base import BaseTest++from django.utils import timezone++from parameterized import parameterized++from products.tasks.backend.models import Task, TaskRun+from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContext+from products.tasks.backend.temporal.process_task.activities.start_agent_server import (+ _record_agent_startup_attempt,+ _record_agent_startup_success,+)+++class TestAgentStartupRecovery(BaseTest):+ @parameterized.expand(+ [+ ("acp", None, "claude", TaskRun.Status.IN_PROGRESS),+ ("acp", "codex", "codex", TaskRun.Status.IN_PROGRESS),+ ("pi", None, "pi", TaskRun.Status.IN_PROGRESS),+ ("acp", None, "claude", TaskRun.Status.FAILED),+ ("acp", None, "claude", TaskRun.Status.CANCELLED),+ ("acp", None, "claude", TaskRun.Status.COMPLETED),+ ]+ )+ def test_only_a_completed_worker_attempt_is_recovery_evidence(+ self, runtime: str, adapter: str | None, provider: str, final_status: str+ ) -> None:+ task = Task.objects.create(team=self.team, title="Example research", repository="example/research")+ run = TaskRun.objects.create(team=self.team, task=task)+ context = TaskProcessingContext(+ task_id=str(task.id),+ run_id=str(run.id),+ team_id=self.team.id,+ team_uuid=str(self.team.uuid),+ organization_id=str(self.organization.id),+ github_integration_id=None,+ repository=task.repository,+ distinct_id="example",+ task_runtime=runtime,+ state={"runtime_adapter": adapter} if adapter else {},+ )+ started_at = timezone.now()+ with time_machine.travel(started_at, tick=False):+ _record_agent_startup_attempt(context, "gvisor")+ run.refresh_from_db()+ assert "agent_startup_success" not in run.state+ with time_machine.travel(started_at + timedelta(seconds=1), tick=False):+ _record_agent_startup_success(context)+ run.refresh_from_db()+ success = run.state["agent_startup_success"]+ assert success["started_at"] == started_at.isoformat()+ assert success["provider"] == provider+ with time_machine.travel(started_at + timedelta(seconds=2), tick=False):+ _record_agent_startup_attempt(context, "gvisor")+ run.refresh_from_db()+ assert run.state["agent_startup_success"] == success+ TaskRun.update_state_atomic(str(run.id), updates={"startup_failure": {"reason": "proxy_invalid"}})+ TaskRun.objects.filter(id=run.id).update(error_message="Turn failed", status=final_status)+ with time_machine.travel(started_at + timedelta(seconds=3), tick=False):+ _record_agent_startup_success(context)+ run.refresh_from_db()+ assert "startup_failure" not in run.state+ assert run.error_message == ("Turn failed" if run.is_terminal else None)+ assert run.state["agent_startup_success"]["started_at"] == (started_at + timedelta(seconds=2)).isoformat()