Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

make provider failures diagnosable from logs (#313)

kt189bb1466836
diff --git a/packages/ai/src/env-api-keys.ts b/packages/ai/src/env-api-keys.ts
index 477fdface1..c52f92f5de 100644
--- a/packages/ai/src/env-api-keys.ts
+++ b/packages/ai/src/env-api-keys.ts
@@ -227,6 +227,8 @@ export function getPrimeTeamId(): string | undefined {
const teamId = (parsed as Record<string, unknown>).team_id;
if (typeof teamId === "string" && teamId.trim()) return teamId.trim();
}
- } catch {}
+ } catch {
+ // Unreadable/malformed config.json: behave as if no team is configured.
+ }
return undefined;
}
diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts
index 2502008aa2..388dffd6e2 100644
--- a/packages/ai/src/index.ts
+++ b/packages/ai/src/index.ts
@@ -3,6 +3,7 @@ export { Type } from "typebox";
export * from "./api-registry.js";
export * from "./env-api-keys.js";
+export * from "./log.js";
export * from "./models.js";
export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js";
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js";
@@ -38,5 +39,6 @@ export type {
OAuthSelectPrompt,
} from "./utils/oauth/types.js";
export * from "./utils/overflow.js";
+export * from "./utils/stream-failure.js";
export * from "./utils/typebox-helpers.js";
export * from "./utils/validation.js";
diff --git a/packages/ai/src/log.ts b/packages/ai/src/log.ts
new file mode 100644
index 0000000000..c3614fe913
--- /dev/null
+++ b/packages/ai/src/log.ts
@@ -0,0 +1,88 @@
+/**
+ * Minimal structured logger shared by pi-ai and its consumers. The library
+ * never writes files: entries go to an injectable sink (setLogSink). Without
+ * a sink, warn/error fall back to console.error and debug/info are dropped.
+ * Logging must never throw into the caller.
+ */
+
+export type LogLevel = "debug" | "info" | "warn" | "error";
+
+export interface LogEntry {
+ ts: string;
+ level: LogLevel;
+ component: string;
+ msg: string;
+ [field: string]: unknown;
+}
+
+export type LogSink = (entry: LogEntry) => void;
+
+export interface Logger {
+ debug(msg: string, fields?: Record<string, unknown>): void;
+ info(msg: string, fields?: Record<string, unknown>): void;
+ warn(msg: string, fields?: Record<string, unknown>): void;
+ error(msg: string, fields?: Record<string, unknown>): void;
+}
+
+let sink: LogSink | undefined;
+
+/** Install the process-wide log sink. Pass undefined to restore the default. */
+export function setLogSink(next: LogSink | undefined): void {
+ sink = next;
+}
+
+function jsonReplacer(): (key: string, value: unknown) => unknown {
+ const seen = new WeakSet<object>();
+ return (_key, value) => {
+ if (typeof value === "bigint") return value.toString();
+ if (typeof value === "object" && value !== null) {
+ if (seen.has(value)) return "[Circular]";
+ seen.add(value);
+ }
+ return value;
+ };
+}
+
+/** JSON.stringify that never throws: circular refs and BigInts degrade instead of dropping the entry. */
+export function stringifyLogEntry(entry: LogEntry): string {
+ try {
+ return JSON.stringify(entry, jsonReplacer());
+ } catch {
+ return JSON.stringify({
+ ts: entry.ts,
+ level: entry.level,
+ component: entry.component,
+ msg: String(entry.msg),
+ fieldsError: "unserializable fields dropped",
+ });
+ }
+}
+
+function emit(level: LogLevel, component: string, msg: string, fields?: Record<string, unknown>): void {
+ try {
+ // Reserved keys win over caller fields so entries can't be misclassified.
+ const entry: LogEntry = { ...fields, ts: new Date().toISOString(), level, component, msg };
+ try {
+ if (sink) {
+ sink(entry);
+ return;
+ }
+ } catch {
+ // Broken sink: fall through to the console fallback.
+ }
+ if (level === "warn" || level === "error") {
+ console.error(stringifyLogEntry(entry));
+ }
+ } catch {
+ // Diagnostics must never break the operation being logged.
+ }
+}
+
+export function getLogger(component: string): Logger {
+ return {
+ debug: (msg, fields) => emit("debug", component, msg, fields),
+ info: (msg, fields) => emit("info", component, msg, fields),
+ warn: (msg, fields) => emit("warn", component, msg, fields),
+ error: (msg, fields) => emit("error", component, msg, fields),
+ };
+}
diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts
index 9d93a9023d..49d17c53b5 100644
--- a/packages/ai/src/providers/amazon-bedrock.ts
+++ b/packages/ai/src/providers/amazon-bedrock.ts
@@ -43,6 +43,7 @@ import type {
import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { parseStreamingJson } from "../utils/json-parse.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { recordStreamFailure, streamFailureFromStopReason } from "../utils/stream-failure.js";
import { adjustMaxTokensForThinking, buildBaseOptions, clampReasoning } from "./simple-options.js";
import { transformMessages } from "./transform-messages.js";
@@ -216,6 +217,7 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream", BedrockOpt
const command = new ConverseStreamCommand(commandInput);
const response = await client.send(command, { abortSignal: options.signal });
+ const requestId = response.$metadata.requestId;
if (response.$metadata.httpStatusCode !== undefined) {
const responseHeaders: Record<string, string> = {};
if (response.$metadata.requestId) {
@@ -238,6 +240,9 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream", BedrockOpt
handleContentBlockStop(item.contentBlockStop, blocks, output, stream);
} else if (item.messageStop) {
output.stopReason = mapStopReason(item.messageStop.stopReason);
+ if (output.stopReason === "error") {
+ output.stopReasonRaw = item.messageStop.stopReason;
+ }
} else if (item.metadata) {
handleMetadata(item.metadata, model, output);
} else if (item.internalServerException) {
@@ -258,7 +263,7 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream", BedrockOpt
}
if (output.stopReason === "error" || output.stopReason === "aborted") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw, { requestId });
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -271,6 +276,7 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream", BedrockOpt
}
output.stopReason = options.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatBedrockError(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts
index 712536e90c..20c3b416f9 100644
--- a/packages/ai/src/providers/anthropic.ts
+++ b/packages/ai/src/providers/anthropic.ts
@@ -31,6 +31,15 @@ import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { headersToRecord } from "../utils/headers.js";
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import {
+ classifyStreamFailure,
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ StreamFailureError,
+ streamFailureFromStopReason,
+ streamFailureMessage,
+ truncateRawPayload,
+} from "../utils/stream-failure.js";
import { resolveCloudflareBaseUrl } from "./cloudflare.js";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js";
@@ -377,9 +386,32 @@ async function* iterateSseMessages(
}
}
+/** Turn an in-stream `error` SSE event (how Anthropic delivers overloads etc.) into a classified failure. */
+function anthropicSseError(data: string, requestId?: string): StreamFailureError {
+ let errorType: string | undefined;
+ let detail: string | undefined;
+ try {
+ const parsed = parseJsonWithRepair<{ error?: { type?: string; message?: string }; request_id?: string }>(data);
+ errorType = parsed.error?.type;
+ detail = parsed.error?.message;
+ // Proxies may strip the request-id header; the error body carries it too.
+ requestId ??= typeof parsed.request_id === "string" ? parsed.request_id : undefined;
+ } catch {
+ detail = data;
+ }
+ const info = {
+ kind: classifyStreamFailure(errorType),
+ providerErrorType: errorType,
+ requestId,
+ raw: truncateRawPayload(data),
+ };
+ return new StreamFailureError(streamFailureMessage(info, detail), info);
+}
+
async function* iterateAnthropicEvents(
response: Response,
signal?: AbortSignal,
+ requestId?: string,
): AsyncGenerator<RawMessageStreamEvent> {
if (!response.body) {
throw new Error("Attempted to iterate over an Anthropic response with no body");
@@ -390,7 +422,7 @@ async function* iterateAnthropicEvents(
for await (const sse of iterateSseMessages(response.body, signal)) {
if (sse.event === "error") {
- throw new Error(sse.data);
+ throw anthropicSseError(sse.data, requestId);
}
if (!ANTHROPIC_MESSAGE_EVENTS.has(sse.event ?? "")) {
@@ -407,14 +439,18 @@ async function* iterateAnthropicEvents(
yield event;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
- throw new Error(
+ throw new StreamFailureError(
`Could not parse Anthropic SSE event ${sse.event}: ${message}; data=${sse.data}; raw=${sse.raw.join("\\n")}`,
+ { kind: "malformed_response", requestId, raw: truncateRawPayload(sse.data) },
);
}
}
if (sawMessageStart && !sawMessageEnd) {
- throw new Error("Anthropic stream ended before message_stop");
+ throw new StreamFailureError("Anthropic stream ended before message_stop", {
+ kind: "malformed_response",
+ requestId,
+ });
}
}
@@ -486,12 +522,13 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
};
const response = await client.messages.create({ ...params, stream: true }, requestOptions).asResponse();
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
+ const requestId = response.headers.get("request-id") ?? undefined;
stream.push({ type: "start", partial: output });
type Block = (ThinkingContent | TextContent | (ToolCall & { partialJson: string })) & { index: number };
const blocks = output.content as Block[];
- for await (const event of iterateAnthropicEvents(response, options?.signal)) {
+ for await (const event of iterateAnthropicEvents(response, options?.signal, requestId)) {
if (event.type === "message_start") {
output.responseId = event.message.id;
// Capture initial token usage from message_start event
@@ -627,6 +664,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
} else if (event.type === "message_delta") {
if (event.delta.stop_reason) {
output.stopReason = mapStopReason(event.delta.stop_reason);
+ if (output.stopReason === "error") {
+ output.stopReasonRaw = event.delta.stop_reason;
+ }
}
// Only update usage fields if present (not null).
// Preserves input_tokens from message_start when proxies omit it in message_delta.
@@ -654,7 +694,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw, { requestId });
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -666,7 +706,8 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
delete (block as { partialJson?: string }).partialJson;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
- output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
+ output.errorMessage = formatStreamFailureMessage(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/providers/azure-openai-responses.ts b/packages/ai/src/providers/azure-openai-responses.ts
index 8b425fbfc9..6265a72301 100644
--- a/packages/ai/src/providers/azure-openai-responses.ts
+++ b/packages/ai/src/providers/azure-openai-responses.ts
@@ -13,6 +13,11 @@ import type {
} from "../types.js";
import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { headersToRecord } from "../utils/headers.js";
+import {
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ streamFailureFromStopReason,
+} from "../utils/stream-failure.js";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.js";
import { buildBaseOptions } from "./simple-options.js";
@@ -98,6 +103,7 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
+ const requestId = response.headers.get("x-request-id") ?? undefined;
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model);
@@ -107,7 +113,7 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw, { requestId });
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -119,7 +125,8 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
delete (block as { partialJson?: string }).partialJson;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
- output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
+ output.errorMessage = formatStreamFailureMessage(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/providers/google-vertex.ts b/packages/ai/src/providers/google-vertex.ts
index 954fa511d9..9bf70809da 100644
--- a/packages/ai/src/providers/google-vertex.ts
+++ b/packages/ai/src/providers/google-vertex.ts
@@ -24,6 +24,11 @@ import type {
} from "../types.js";
import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import {
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ streamFailureFromStopReason,
+} from "../utils/stream-failure.js";
import type { GoogleThinkingLevel } from "./google-shared.js";
import {
convertMessages,
@@ -224,6 +229,9 @@ export const streamGoogleVertex: StreamFunction<"google-vertex", GoogleVertexOpt
if (output.content.some((b) => b.type === "toolCall")) {
output.stopReason = "toolUse";
}
+ if (output.stopReason === "error") {
+ output.stopReasonRaw = candidate.finishReason;
+ }
}
if (chunk.usageMetadata) {
@@ -270,7 +278,7 @@ export const streamGoogleVertex: StreamFunction<"google-vertex", GoogleVertexOpt
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw);
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -283,7 +291,8 @@ export const streamGoogleVertex: StreamFunction<"google-vertex", GoogleVertexOpt
}
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
- output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
+ output.errorMessage = formatStreamFailureMessage(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/providers/google.ts b/packages/ai/src/providers/google.ts
index 5af34c0a5e..68664e089c 100644
--- a/packages/ai/src/providers/google.ts
+++ b/packages/ai/src/providers/google.ts
@@ -22,6 +22,11 @@ import type {
} from "../types.js";
import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import {
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ streamFailureFromStopReason,
+} from "../utils/stream-failure.js";
import type { GoogleThinkingLevel } from "./google-shared.js";
import {
convertMessages,
@@ -207,6 +212,9 @@ export const streamGoogle: StreamFunction<"google-generative-ai", GoogleOptions>
if (output.content.some((b) => b.type === "toolCall")) {
output.stopReason = "toolUse";
}
+ if (output.stopReason === "error") {
+ output.stopReasonRaw = candidate.finishReason;
+ }
}
if (chunk.usageMetadata) {
@@ -253,7 +261,7 @@ export const streamGoogle: StreamFunction<"google-generative-ai", GoogleOptions>
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw);
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -266,7 +274,8 @@ export const streamGoogle: StreamFunction<"google-generative-ai", GoogleOptions>
}
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
- output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
+ output.errorMessage = formatStreamFailureMessage(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/providers/mistral.ts b/packages/ai/src/providers/mistral.ts
index c1500dbfde..f153e36403 100644
--- a/packages/ai/src/providers/mistral.ts
+++ b/packages/ai/src/providers/mistral.ts
@@ -26,6 +26,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { shortHash } from "../utils/hash.js";
import { parseStreamingJson } from "../utils/json-parse.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { recordStreamFailure, streamFailureFromStopReason } from "../utils/stream-failure.js";
import { buildBaseOptions } from "./simple-options.js";
import { transformMessages } from "./transform-messages.js";
@@ -85,7 +86,7 @@ export const streamMistral: StreamFunction<"mistral-conversations", MistralOptio
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw);
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -97,6 +98,7 @@ export const streamMistral: StreamFunction<"mistral-conversations", MistralOptio
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatMistralError(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
@@ -319,6 +321,9 @@ async function consumeChatStream(
if (choice.finishReason) {
output.stopReason = mapChatStopReason(choice.finishReason);
+ if (output.stopReason === "error") {
+ output.stopReasonRaw = choice.finishReason;
+ }
}
const delta = choice.delta;
diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts
index 669a3ab1d4..2083fcf2d5 100644
--- a/packages/ai/src/providers/openai-codex-responses.ts
+++ b/packages/ai/src/providers/openai-codex-responses.ts
@@ -571,12 +571,17 @@ async function* parseSSE(response: Response): AsyncGenerator<Record<string, unkn
}
}
} finally {
+ // Best-effort stream teardown: the reader may already be closed or errored.
try {
await reader.cancel();
- } catch {}
+ } catch {
+ // Already closed.
+ }
try {
reader.releaseLock();
- } catch {}
+ } catch {
+ // Lock already released.
+ }
}
}
@@ -745,7 +750,9 @@ function isWebSocketReusable(socket: WebSocketLike): boolean {
function closeWebSocketSilently(socket: WebSocketLike, code = 1000, reason = "done"): void {
try {
socket.close(code, reason);
- } catch {}
+ } catch {
+ // Already closed.
+ }
}
function scheduleSessionWebSocketExpiry(sessionId: string, entry: CachedWebSocketConnection): void {
@@ -1237,7 +1244,9 @@ async function parseErrorResponse(response: Response): Promise<{ message: string
}
message = err.message || friendlyMessage || message;
}
- } catch {}
+ } catch {
+ // Unparseable error body: fall back to the raw message.
+ }
return { message, friendlyMessage };
}
diff --git a/packages/ai/src/providers/openai-responses-shared.ts b/packages/ai/src/providers/openai-responses-shared.ts
index 2b99990e6c..8a22f03ff4 100644
--- a/packages/ai/src/providers/openai-responses-shared.ts
+++ b/packages/ai/src/providers/openai-responses-shared.ts
@@ -31,6 +31,7 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.js";
import { shortHash } from "../utils/hash.js";
import { parseStreamingJson } from "../utils/json-parse.js";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { classifyStreamFailure, StreamFailureError } from "../utils/stream-failure.js";
import { transformMessages } from "./transform-messages.js";
// =============================================================================
@@ -514,17 +515,27 @@ export async function processResponsesStream<TApi extends Api>(
if (output.content.some((b) => b.type === "toolCall") && output.stopReason === "stop") {
output.stopReason = "toolUse";
}
+ if (output.stopReason === "error" && response?.status) {
+ output.stopReasonRaw = response.status;
+ }
} else if (event.type === "error") {
- throw new Error(`Error Code ${event.code}: ${event.message}` || "Unknown error");
+ throw new StreamFailureError(`Error Code ${event.code}: ${event.message}`, {
+ kind: classifyStreamFailure(event.code ?? undefined),
+ providerErrorType: event.code ?? undefined,
+ });
} else if (event.type === "response.failed") {
const error = event.response?.error;
const details = event.response?.incomplete_details;
+ const providerErrorType = error?.code ?? details?.reason;
const msg = error
? `${error.code || "unknown"}: ${error.message || "no message"}`
: details?.reason
? `incomplete: ${details.reason}`
: "Unknown error (no error details in response)";
- throw new Error(msg);
+ throw new StreamFailureError(msg, {
+ kind: classifyStreamFailure(providerErrorType),
+ providerErrorType,
+ });
}
}
}
diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts
index 6884d08d87..d152f7d3c2 100644
--- a/packages/ai/src/providers/openai-responses.ts
+++ b/packages/ai/src/providers/openai-responses.ts
@@ -16,6 +16,11 @@ import type {
} from "../types.js";
import { AssistantMessageEventStream } from "../utils/event-stream.js";
import { headersToRecord } from "../utils/headers.js";
+import {
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ streamFailureFromStopReason,
+} from "../utils/stream-failure.js";
import { isCloudflareProvider, resolveCloudflareBaseUrl } from "./cloudflare.js";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.js";
@@ -106,6 +111,7 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIRes
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
+ const requestId = response.headers.get("x-request-id") ?? undefined;
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model, {
@@ -118,7 +124,7 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIRes
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
- throw new Error("An unknown error occurred");
+ throw streamFailureFromStopReason(output.stopReasonRaw, { requestId });
}
stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -130,7 +136,8 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIRes
delete (block as { partialJson?: string }).partialJson;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
- output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
+ output.errorMessage = formatStreamFailureMessage(error);
+ recordStreamFailure(model, output, error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts
index 90f12b7c8d..29e272465d 100644
--- a/packages/ai/src/types.ts
+++ b/packages/ai/src/types.ts
@@ -229,6 +229,7 @@ export interface AssistantMessage {
diagnostics?: AssistantMessageDiagnostic[]; // Redacted provider/runtime diagnostics for failures and recoveries.
usage: Usage;
stopReason: StopReason;
+ stopReasonRaw?: string; // Provider's raw stop/finish reason when it mapped to "error" (e.g. "refusal", "SAFETY")
errorMessage?: string;
timestamp: number; // Unix timestamp in milliseconds
}
diff --git a/packages/ai/src/utils/stream-failure.ts b/packages/ai/src/utils/stream-failure.ts
new file mode 100644
index 0000000000..af8b2a7ea5
--- /dev/null
+++ b/packages/ai/src/utils/stream-failure.ts
@@ -0,0 +1,232 @@
+import { getLogger } from "../log.js";
+import type { AssistantMessage } from "../types.js";
+import { appendAssistantMessageDiagnostic, extractDiagnosticError } from "./diagnostics.js";
+
+/**
+ * Shared classification and reporting for provider stream failures, so no
+ * provider collapses a specific cause (refusal, safety filter, overload, ...)
+ * into a generic string before it is logged and persisted.
+ */
+
+export type StreamFailureKind =
+ | "refusal"
+ | "safety"
+ | "overloaded"
+ | "rate_limit"
+ | "server_error"
+ | "auth"
+ | "invalid_request"
+ | "malformed_response"
+ | "unknown";
+
+export interface StreamFailureInfo {
+ kind: StreamFailureKind;
+ /** Provider's own error/stop identifier, e.g. "overloaded_error" or "SAFETY". */
+ providerErrorType?: string;
+ status?: number;
+ requestId?: string;
+ /** Truncated raw provider payload for post-mortems. */
+ raw?: string;
+}
+
+export class StreamFailureError extends Error {
+ readonly info: StreamFailureInfo;
+
+ constructor(message: string, info: StreamFailureInfo) {
+ super(message);
+ this.name = "StreamFailureError";
+ this.info = info;
+ }
+}
+
+const KIND_MESSAGES: Record<StreamFailureKind, string> = {
+ refusal: "Model refused to respond",
+ safety: "Response blocked by provider safety filters",
+ overloaded: "Provider overloaded",
+ rate_limit: "Provider rate limit exceeded",
+ server_error: "Provider server error",
+ auth: "Provider authentication failed",
+ invalid_request: "Provider rejected the request",
+ malformed_response: "Provider returned a malformed response",
+ unknown: "Provider stream failed",
+};
+
+/** Build a user-facing message like "Provider overloaded (overloaded_error, 529) [request_id: req_abc]". */
+export function streamFailureMessage(info: StreamFailureInfo, detail?: string): string {
+ const qualifiers = [info.providerErrorType, info.status !== undefined ? String(info.status) : undefined]
+ .filter(Boolean)
+ .join(", ");
+ let message = KIND_MESSAGES[info.kind];
+ if (qualifiers) message += ` (${qualifiers})`;
+ if (detail) message += `: ${detail}`;
+ if (info.requestId) message += ` [request_id: ${info.requestId}]`;
+ return message;
+}
+
+export function classifyStreamFailure(providerErrorType?: string, status?: number): StreamFailureKind {
+ const type = providerErrorType?.toLowerCase() ?? "";
+ if (type === "refusal") return "refusal";
+ if (/sensitive|safety|prohibited_content|blocklist|spii|recitation|content.?filter|guardrail|flagged/.test(type)) {
+ return "safety";
+ }
+ if (type.includes("overloaded") || status === 529) return "overloaded";
+ if (type.includes("rate_limit") || type.includes("throttl") || status === 429) return "rate_limit";
+ if (/authentication|permission|unauthorized/.test(type) || status === 401 || status === 403) return "auth";
+ if (type.includes("invalid_request") || type.includes("not_found_error") || status === 400 || status === 404) {
+ return "invalid_request";
+ }
+ if (type.includes("malformed")) return "malformed_response";
+ if (
+ type.includes("api_error") ||
+ type.includes("server_error") ||
+ type.includes("unavailable") ||
+ (status !== undefined && status >= 500)
+ ) {
+ return "server_error";
+ }
+ return "unknown";
+}
+
+/**
+ * Failure for a stream that terminated with a provider stop/finish reason that
+ * maps to "error" (e.g. Anthropic "refusal", Gemini "SAFETY"). Providers call
+ * this instead of throwing a generic error, so the raw reason survives.
+ */
+export function streamFailureFromStopReason(
+ rawStopReason: string | undefined,
+ extra?: Pick<StreamFailureInfo, "requestId">,
+): StreamFailureError {
+ const info: StreamFailureInfo = {
+ kind: rawStopReason ? classifyStreamFailure(rawStopReason) : "unknown",
+ providerErrorType: rawStopReason,
+ requestId: extra?.requestId,
+ };
+ if (info.kind === "unknown" && /malformed/i.test(rawStopReason ?? "")) info.kind = "malformed_response";
+ const message = rawStopReason
+ ? streamFailureMessage(info)
+ : streamFailureMessage(info, "stream ended with an error and no stop reason");
+ return new StreamFailureError(message, info);
+}
+
+const MAX_RAW_LENGTH = 2000;
+
+export function truncateRawPayload(raw: string): string {
+ return raw.length > MAX_RAW_LENGTH ? `${raw.slice(0, MAX_RAW_LENGTH)}…` : raw;
+}
+
+function extractStreamFailureParts(error: unknown): { info: StreamFailureInfo; detail?: string } {
+ if (error instanceof StreamFailureError) return { info: error.info };
+ if (!(error instanceof Error)) return { info: { kind: "unknown" } };
+
+ const err = error as Error & {
+ status?: unknown;
+ statusCode?: unknown;
+ code?: unknown;
+ requestID?: unknown;
+ request_id?: unknown;
+ headers?: unknown;
+ error?: unknown;
+ $metadata?: { requestId?: unknown };
+ };
+
+ const status =
+ typeof err.status === "number" ? err.status : typeof err.statusCode === "number" ? err.statusCode : undefined;
+
+ // Error bodies come nested differently per SDK: Anthropic/OpenAI expose
+ // `error.error = {type|code, message}` (sometimes doubly nested).
+ let body = err.error as { type?: unknown; code?: unknown; message?: unknown; error?: unknown } | undefined;
+ if (body && typeof body === "object" && body.error && typeof body.error === "object") {
+ body = body.error as { type?: unknown; code?: unknown; message?: unknown };
+ }
+ const bodyType = body && typeof body === "object" ? (body.type ?? body.code) : undefined;
+ const bodyMessage = body && typeof body === "object" ? body.message : undefined;
+ const providerErrorType =
+ typeof bodyType === "string"
+ ? bodyType
+ : typeof err.code === "string"
+ ? err.code
+ : err.name !== "Error" && err.name !== "StreamFailureError"
+ ? err.name
+ : undefined;
+
+ const headers = err.headers;
+ const headerRequestId =
+ headers && typeof (headers as Headers).get === "function"
+ ? ((headers as Headers).get("request-id") ?? (headers as Headers).get("x-request-id"))
+ : headers && typeof headers === "object"
+ ? ((headers as Record<string, unknown>)["request-id"] ??
+ (headers as Record<string, unknown>)["x-request-id"])
+ : undefined;
+ const rawRequestId = err.requestID ?? err.request_id ?? err.$metadata?.requestId ?? headerRequestId;
+ const requestId = typeof rawRequestId === "string" ? rawRequestId : undefined;
+
+ return {
+ info: {
+ kind: classifyStreamFailure(providerErrorType ?? error.message, status),
+ providerErrorType,
+ status,
+ requestId,
+ },
+ detail: typeof bodyMessage === "string" ? bodyMessage : undefined,
+ };
+}
+
+/**
+ * Best-effort extraction of structured failure info from any thrown value:
+ * StreamFailureError, provider SDK errors (Anthropic/OpenAI APIError, AWS SDK
+ * exceptions, Google ApiError), or plain errors.
+ */
+export function extractStreamFailureInfo(error: unknown): StreamFailureInfo {
+ return extractStreamFailureParts(error).info;
+}
+
+/**
+ * User-facing message for a thrown stream error: a classified one-liner with
+ * the provider's own short message, never the raw payload/trace. Unrecognized
+ * errors pass through verbatim so their text (which downstream retry matching
+ * may depend on) is preserved.
+ */
+export function formatStreamFailureMessage(error: unknown): string {
+ if (error instanceof StreamFailureError) return error.message;
+ const { info, detail } = extractStreamFailureParts(error);
+ if (info.kind === "unknown") {
+ return error instanceof Error ? error.message : JSON.stringify(error);
+ }
+ return streamFailureMessage(info, detail);
+}
+
+const log = getLogger("ai.provider");
+
+/**
+ * Record a terminal stream failure on the message (structured diagnostic that
+ * persists to session JSONL) and emit one structured log line. Call from the
+ * provider's terminal catch after stopReason/errorMessage are set; no-op for
+ * user-initiated aborts.
+ */
+export function recordStreamFailure(
+ model: { provider: string; id: string; api: string },
+ output: AssistantMessage,
+ error: unknown,
+): void {
+ if (output.stopReason !== "error") return;
+ const info = extractStreamFailureInfo(error);
+ appendAssistantMessageDiagnostic(output, {
+ type: "provider_stream_failure",
+ timestamp: Date.now(),
+ error: extractDiagnosticError(error),
+ details: { ...info },
+ });
+ const rawMessage = error instanceof Error ? error.message : String(error);
+ log.error("provider stream failure", {
+ provider: model.provider,
+ model: model.id,
+ api: model.api,
+ kind: info.kind,
+ providerErrorType: info.providerErrorType,
+ status: info.status,
+ requestId: info.requestId,
+ message: output.errorMessage,
+ // errorMessage is user-facing and concise; keep the raw cause for debugging.
+ cause: rawMessage === output.errorMessage ? undefined : truncateRawPayload(rawMessage),
+ });
+}
diff --git a/packages/ai/test/log.test.ts b/packages/ai/test/log.test.ts
new file mode 100644
index 0000000000..50960822f5
--- /dev/null
+++ b/packages/ai/test/log.test.ts
@@ -0,0 +1,59 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { getLogger, type LogEntry, setLogSink, stringifyLogEntry } from "../src/log.js";
+
+afterEach(() => {
+ setLogSink(undefined);
+ vi.restoreAllMocks();
+});
+
+describe("structured logger", () => {
+ test("routes entries with component, level, and fields to the sink", () => {
+ const entries: LogEntry[] = [];
+ setLogSink((entry) => entries.push(entry));
+
+ const log = getLogger("test.component");
+ log.info("hello", { a: 1 });
+ log.error("boom", { requestId: "req_123" });
+
+ expect(entries).toHaveLength(2);
+ expect(entries[0]).toMatchObject({ level: "info", component: "test.component", msg: "hello", a: 1 });
+ expect(entries[1]).toMatchObject({ level: "error", msg: "boom", requestId: "req_123" });
+ expect(new Date(entries[0].ts).getTime()).not.toBeNaN();
+ });
+
+ test("without a sink, warn/error fall back to console.error and debug/info are dropped", () => {
+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+ const log = getLogger("test");
+ log.debug("quiet");
+ log.info("quiet");
+ log.warn("loud");
+ log.error("loud");
+ expect(consoleError).toHaveBeenCalledTimes(2);
+ });
+
+ test("caller fields cannot overwrite reserved entry keys", () => {
+ const entries: LogEntry[] = [];
+ setLogSink((entry) => entries.push(entry));
+ getLogger("test").error("boom", { level: "info", msg: "spoofed", detail: "kept" });
+ expect(entries[0]).toMatchObject({ level: "error", msg: "boom", detail: "kept" });
+ });
+
+ test("stringifyLogEntry survives circular refs and BigInt fields", () => {
+ const circular: Record<string, unknown> = {};
+ circular.self = circular;
+ const entry: LogEntry = { ts: "t", level: "error", component: "c", msg: "m", circular, big: 1n };
+ const line = stringifyLogEntry(entry);
+ expect(JSON.parse(line)).toMatchObject({ msg: "m", circular: { self: "[Circular]" }, big: "1" });
+ });
+
+ test("a throwing sink never propagates and warn/error fall back to console.error", () => {
+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+ setLogSink(() => {
+ throw new Error("sink is broken");
+ });
+ const log = getLogger("test");
+ expect(() => log.error("boom")).not.toThrow();
+ expect(() => log.debug("quiet")).not.toThrow();
+ expect(consoleError).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/ai/test/stream-failure.test.ts b/packages/ai/test/stream-failure.test.ts
new file mode 100644
index 0000000000..5009de0160
--- /dev/null
+++ b/packages/ai/test/stream-failure.test.ts
@@ -0,0 +1,171 @@
+import { afterEach, describe, expect, test } from "vitest";
+import { setLogSink } from "../src/log.js";
+import type { AssistantMessage } from "../src/types.js";
+import {
+ classifyStreamFailure,
+ extractStreamFailureInfo,
+ formatStreamFailureMessage,
+ recordStreamFailure,
+ StreamFailureError,
+ streamFailureFromStopReason,
+} from "../src/utils/stream-failure.js";
+
+afterEach(() => setLogSink(undefined));
+
+function makeOutput(overrides: Partial<AssistantMessage> = {}): AssistantMessage {
+ return {
+ role: "assistant",
+ content: [],
+ api: "anthropic-messages",
+ provider: "anthropic",
+ model: "test-model",
+ usage: {
+ input: 0,
+ output: 0,
+ cacheRead: 0,
+ cacheWrite: 0,
+ totalTokens: 0,
+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+ },
+ stopReason: "error",
+ timestamp: 0,
+ ...overrides,
+ };
+}
+
+describe("classifyStreamFailure", () => {
+ test.each([
+ ["overloaded_error", undefined, "overloaded"],
+ [undefined, 529, "overloaded"],
+ ["rate_limit_error", undefined, "rate_limit"],
+ [undefined, 429, "rate_limit"],
+ ["refusal", undefined, "refusal"],
+ ["sensitive", undefined, "safety"],
+ ["SAFETY", undefined, "safety"],
+ ["PROHIBITED_CONTENT", undefined, "safety"],
+ ["content_filter", undefined, "safety"],
+ ["guardrail_intervened", undefined, "safety"],
+ ["authentication_error", undefined, "auth"],
+ ["invalid_request_error", undefined, "invalid_request"],
+ ["api_error", undefined, "server_error"],
+ [undefined, 503, "server_error"],
+ ["something_else", undefined, "unknown"],
+ ])("classifies %s / %s as %s", (type, status, expected) => {
+ expect(classifyStreamFailure(type, status)).toBe(expected);
+ });
+});
+
+describe("streamFailureFromStopReason", () => {
+ test("preserves the raw stop reason instead of a generic message", () => {
+ const error = streamFailureFromStopReason("refusal", { requestId: "req_abc" });
+ expect(error.message).toBe("Model refused to respond (refusal) [request_id: req_abc]");
+ expect(error.info).toMatchObject({ kind: "refusal", providerErrorType: "refusal", requestId: "req_abc" });
+ });
+
+ test("maps Gemini safety finish reasons", () => {
+ expect(streamFailureFromStopReason("SAFETY").info.kind).toBe("safety");
+ expect(streamFailureFromStopReason("MALFORMED_FUNCTION_CALL").info.kind).toBe("malformed_response");
+ });
+
+ test("still explains a missing stop reason", () => {
+ const error = streamFailureFromStopReason(undefined);
+ expect(error.info.kind).toBe("unknown");
+ expect(error.message).toContain("no stop reason");
+ });
+});
+
+describe("extractStreamFailureInfo", () => {
+ test("passes through StreamFailureError info", () => {
+ const info = { kind: "overloaded" as const, requestId: "req_1" };
+ expect(extractStreamFailureInfo(new StreamFailureError("x", info))).toBe(info);
+ });
+
+ test("extracts status, nested error type, and request id from SDK-shaped errors", () => {
+ const sdkError = Object.assign(new Error("529 overloaded"), {
+ status: 529,
+ error: { type: "error", error: { type: "overloaded_error", message: "Overloaded" } },
+ requestID: "req_sdk",
+ });
+ expect(extractStreamFailureInfo(sdkError)).toMatchObject({
+ kind: "overloaded",
+ providerErrorType: "overloaded_error",
+ status: 529,
+ requestId: "req_sdk",
+ });
+ });
+
+ test("extracts AWS SDK request id and exception name", () => {
+ const awsError = Object.assign(new Error("throttled"), {
+ name: "ThrottlingException",
+ $metadata: { requestId: "aws_req" },
+ });
+ expect(extractStreamFailureInfo(awsError)).toMatchObject({ requestId: "aws_req" });
+ });
+
+ test("falls back to classifying the message text", () => {
+ expect(extractStreamFailureInfo(new Error("provider overloaded, retry later")).kind).toBe("overloaded");
+ expect(extractStreamFailureInfo("not an error").kind).toBe("unknown");
+ });
+});
+
+describe("formatStreamFailureMessage", () => {
+ test("condenses a classified SDK error to a one-liner instead of the raw payload", () => {
+ const sdkError = Object.assign(
+ new Error('401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}'),
+ {
+ status: 401,
+ error: { type: "error", error: { type: "authentication_error", message: "invalid x-api-key" } },
+ requestID: "req_1",
+ },
+ );
+ expect(formatStreamFailureMessage(sdkError)).toBe(
+ "Provider authentication failed (authentication_error, 401): invalid x-api-key [request_id: req_1]",
+ );
+ });
+
+ test("passes unrecognized errors through verbatim", () => {
+ expect(formatStreamFailureMessage(new Error("fetch failed"))).toBe("fetch failed");
+ expect(formatStreamFailureMessage(new Error("Request was aborted"))).toBe("Request was aborted");
+ });
+
+ test("uses the StreamFailureError message as-is", () => {
+ const error = streamFailureFromStopReason("refusal");
+ expect(formatStreamFailureMessage(error)).toBe(error.message);
+ });
+});
+
+describe("recordStreamFailure", () => {
+ const model = { provider: "anthropic", id: "claude-fable-5", api: "anthropic-messages" };
+
+ test("appends a structured diagnostic and logs one entry", () => {
+ const logged: Record<string, unknown>[] = [];
+ setLogSink((entry) => logged.push(entry));
+
+ const output = makeOutput({ errorMessage: "Provider overloaded (overloaded_error) [request_id: req_9]" });
+ recordStreamFailure(model, output, new StreamFailureError("x", { kind: "overloaded", requestId: "req_9" }));
+
+ expect(output.diagnostics).toHaveLength(1);
+ expect(output.diagnostics?.[0]).toMatchObject({
+ type: "provider_stream_failure",
+ details: { kind: "overloaded", requestId: "req_9" },
+ });
+ expect(logged).toHaveLength(1);
+ expect(logged[0]).toMatchObject({
+ level: "error",
+ component: "ai.provider",
+ provider: "anthropic",
+ model: "claude-fable-5",
+ kind: "overloaded",
+ requestId: "req_9",
+ });
+ });
+
+ test("does nothing for user aborts", () => {
+ const logged: unknown[] = [];
+ setLogSink((entry) => logged.push(entry));
+ const output = makeOutput({ stopReason: "aborted" });
+ recordStreamFailure(model, output, new Error("Request was aborted"));
+ expect(output.diagnostics).toBeUndefined();
+ expect(logged).toEqual([]);
+ });
+});
diff --git a/packages/coding-agent/src/cli/daemon-ps.ts b/packages/coding-agent/src/cli/daemon-ps.ts
index 6959fc3046..c376bfc1b4 100644
--- a/packages/coding-agent/src/cli/daemon-ps.ts
+++ b/packages/coding-agent/src/cli/daemon-ps.ts
@@ -576,7 +576,9 @@ async function forceKillDaemon(pid: number): Promise<void> {
}
try {
process.kill(pid, "SIGKILL");
- } catch {}
+ } catch {
+ // Process already exited between the liveness check and the kill.
+ }
}
function isProcessAlive(pid: number): boolean {
diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts
index 4913b84b81..0ac702afc2 100644
--- a/packages/coding-agent/src/config.ts
+++ b/packages/coding-agent/src/config.ts
@@ -542,6 +542,11 @@ export function getAgentTracesLogPath(): string {
return join(getLogsDir(), "agent-traces.log");
}
+/** Shared structured (JSON lines) log for client, daemon, and provider diagnostics. */
+export function getAgentLogPath(): string {
+ return join(getLogsDir(), "agent.jsonl");
+}
+
/**
* Log file for a daemon. The basename keeps it readable; a hash of the full
* socket path makes it unique so two sockets that share a basename (e.g.
@@ -560,11 +565,11 @@ const MAX_LOG_BYTES = 5 * 1024 * 1024;
* — a long-lived writer rotates on the write that crosses the cap, not only at
* startup. Best-effort: diagnostics must never throw into the caller.
*/
-export function appendRotatingLog(logPath: string, message: string): void {
+export function appendRotatingLog(logPath: string, message: string, maxBytes: number = MAX_LOG_BYTES): void {
try {
mkdirSync(dirname(logPath), { recursive: true });
try {
- if (existsSync(logPath) && statSync(logPath).size > MAX_LOG_BYTES) {
+ if (existsSync(logPath) && statSync(logPath).size > maxBytes) {
// Drop any prior .old first: renameSync fails on Windows if it exists.
rmSync(`${logPath}.old`, { force: true });
renameSync(logPath, `${logPath}.old`);
diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts
index 18f72d9899..ba358c7231 100644
--- a/packages/coding-agent/src/core/agent-session.ts
+++ b/packages/coding-agent/src/core/agent-session.ts
@@ -5311,6 +5311,14 @@ export class AgentSession {
const contextWindow = this.model?.contextWindow ?? 0;
if (isContextOverflow(message, contextWindow)) return false;
+ // Structured classification from the provider (stream-failure.ts) beats
+ // message-text matching. Safety/malformed/unknown kinds fall through to
+ // the regex, which distinguishes transient content_filter cases.
+ const failure = message.diagnostics?.find((d) => d.type === "provider_stream_failure");
+ const kind = failure?.details?.kind;
+ if (kind === "overloaded" || kind === "rate_limit" || kind === "server_error") return true;
+ if (kind === "refusal" || kind === "auth" || kind === "invalid_request") return false;
+
const err = message.errorMessage;
// Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, request ended without sending chunks, HTTP/2 closed before response, terminated, retry delay exceeded, transient content_filter finish_reason, provider policy-flag prose, and prose-form transient 5xx ("an error occurred while processing your request. you can retry")
return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay|content.?filter|flagged .*(cybersecurity|usage polic|violat)|error occurred while processing|you can retry/i.test(
diff --git a/packages/coding-agent/src/core/kernel/fork-server.ts b/packages/coding-agent/src/core/kernel/fork-server.ts
index c917319979..d5b5423ac2 100644
--- a/packages/coding-agent/src/core/kernel/fork-server.ts
+++ b/packages/coding-agent/src/core/kernel/fork-server.ts
@@ -212,7 +212,9 @@ class ForkServer {
if (this.abandoned.delete(msg.id) && typeof msg.pid === "number") {
try {
process.kill(msg.pid, "SIGTERM");
- } catch {}
+ } catch {
+ // Orphan already exited.
+ }
}
continue;
}
@@ -271,19 +273,28 @@ class ForkServer {
// those events), so this early flag flip can't strand them on a timeout.
this.dead = true;
this.rejectPending(reason);
+ // Best-effort teardown: each resource may already be gone.
try {
this.conn?.destroy();
- } catch {}
+ } catch {
+ // Already destroyed.
+ }
try {
this.server?.close();
- } catch {}
+ } catch {
+ // Already closed.
+ }
try {
this.proc?.kill("SIGTERM");
- } catch {}
+ } catch {
+ // Already exited.
+ }
if (this.socketDir) {
try {
rmSync(this.socketDir, { recursive: true, force: true });
- } catch {}
+ } catch {
+ // Leave the socket dir for OS tmp cleanup.
+ }
this.socketDir = undefined;
}
}
diff --git a/packages/coding-agent/src/core/kernel/index.ts b/packages/coding-agent/src/core/kernel/index.ts
index 5a95bbc74e..9ee4cb2e98 100644
--- a/packages/coding-agent/src/core/kernel/index.ts
+++ b/packages/coding-agent/src/core/kernel/index.ts
@@ -521,7 +521,9 @@ export class KernelManager {
// collide with it (write the same file / re-bind the same ports).
try {
rmSync(connection.tempDir, { recursive: true, force: true });
- } catch {}
+ } catch {
+ // Leave the temp dir for OS tmp cleanup.
+ }
connection = makeConnection();
this.tempDir = connection.tempDir;
}
@@ -1043,14 +1045,18 @@ export class KernelManager {
// been recycled by the OS, and SIGTERM would then hit an unrelated process.
process.kill(this.kernelPid, "SIGTERM");
}
- } catch {}
+ } catch {
+ // Kernel already exited.
+ }
this.kernel = undefined;
this.kernelPid = undefined;
this.connection = undefined;
if (this.tempDir) {
try {
rmSync(this.tempDir, { recursive: true, force: true });
- } catch {}
+ } catch {
+ // Leave the temp dir for OS tmp cleanup.
+ }
}
this.tempDir = undefined;
this.startPromise = undefined;
@@ -1096,7 +1102,11 @@ export class KernelManager {
await this.control.send(encode(msg, this.connection.key));
await sleep(200);
}
- } catch {}
+ } catch (error) {
+ this.appendKernelDiagnostic(
+ `shutdown_request send failed (killing instead): ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
this.cleanupResources();
}
diff --git a/packages/coding-agent/src/core/logging.ts b/packages/coding-agent/src/core/logging.ts
new file mode 100644
index 0000000000..44ed72f13e
--- /dev/null
+++ b/packages/coding-agent/src/core/logging.ts
@@ -0,0 +1,23 @@
+import { type LogEntry, setLogSink, stringifyLogEntry } from "@earendil-works/pi-ai";
+import { appendRotatingLog, getAgentLogPath } from "../config.js";
+
+const AGENT_LOG_MAX_BYTES = 20 * 1024 * 1024;
+
+let context: Record<string, unknown> = {};
+
+/** Merge late-bound fields (e.g. mode, sessionId) into every subsequent log entry. */
+export function setLogContext(fields: Record<string, unknown>): void {
+ Object.assign(context, fields);
+}
+
+/**
+ * Route all structured logging (coding-agent and pi-ai) to the shared JSONL
+ * log at ~/.prime/agent/logs/agent.jsonl. One master file, filterable by the
+ * pid/context fields; writes are best-effort and size-bounded.
+ */
+export function installFileLogSink(fields?: Record<string, unknown>): void {
+ context = { pid: process.pid, ...fields };
+ setLogSink((entry: LogEntry) => {
+ appendRotatingLog(getAgentLogPath(), stringifyLogEntry({ ...entry, ...context }), AGENT_LOG_MAX_BYTES);
+ });
+}
diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts
index f7f32e3e24..223918bcfa 100644
--- a/packages/coding-agent/src/core/model-resolver.ts
+++ b/packages/coding-agent/src/core/model-resolver.ts
@@ -3,13 +3,15 @@
*/
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
-import { type Api, type KnownProvider, type Model, modelsAreEqual } from "@earendil-works/pi-ai";
+import { type Api, getLogger, type KnownProvider, type Model, modelsAreEqual } from "@earendil-works/pi-ai";
import chalk from "chalk";
import { minimatch } from "minimatch";
import { isValidThinkingLevel } from "../cli/args.js";
import { DEFAULT_THINKING_LEVEL } from "./defaults.js";
import type { ModelRegistry } from "./model-registry.js";
+const log = getLogger("coding-agent.model-resolver");
+
/** Default model IDs for each known provider */
export const defaultModelPerProvider: Record<KnownProvider, string> = {
"amazon-bedrock": "us.anthropic.claude-opus-4-6-v1",
@@ -279,6 +281,7 @@ export function resolveModelScopeFromModels(patterns: string[], availableModels:
});
if (matchingModels.length === 0) {
+ log.warn("no models match pattern", { pattern });
console.warn(chalk.yellow(`Warning: No models match pattern "${pattern}"`));
continue;
}
@@ -294,10 +297,12 @@ export function resolveModelScopeFromModels(patterns: string[], availableModels:
const { model, thinkingLevel, warning } = parseModelPattern(pattern, availableModels);
if (warning) {
+ log.warn(warning, { pattern });
console.warn(chalk.yellow(`Warning: ${warning}`));
}
if (!model) {
+ log.warn("no models match pattern", { pattern });
console.warn(chalk.yellow(`Warning: No models match pattern "${pattern}"`));
continue;
}
@@ -516,6 +521,7 @@ export async function findInitialModel(options: {
modelRegistry,
});
if (resolved.error) {
+ log.error(resolved.error, { cliProvider, cliModel });
console.error(chalk.red(resolved.error));
process.exit(1);
}
@@ -594,6 +600,7 @@ export async function restoreModelFromSession(
// Model not found or no API key - fall back
const reason = !restoredModel ? "model no longer exists" : "no auth configured";
+ log.warn("could not restore model", { provider: savedProvider, model: savedModelId, reason });
if (shouldPrintMessages) {
console.error(chalk.yellow(`Warning: Could not restore model ${savedProvider}/${savedModelId} (${reason}).`));
diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts
index e31620dc0a..fd818595f3 100644
--- a/packages/coding-agent/src/core/package-manager.ts
+++ b/packages/coding-agent/src/core/package-manager.ts
@@ -254,7 +254,9 @@ function addIgnoreRules(ig: IgnoreMatcher, dir: string, rootDir: string): void {
if (patterns.length > 0) {
ig.add(patterns);
}
- } catch {}
+ } catch {
+ // Unreadable ignore file: skip it rather than failing resource loading.
+ }
}
}
diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts
index 1cd1dd1f27..0872e818aa 100644
--- a/packages/coding-agent/src/core/skills.ts
+++ b/packages/coding-agent/src/core/skills.ts
@@ -1,3 +1,4 @@
+import { getLogger } from "@earendil-works/pi-ai";
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
import ignore from "ignore";
import { homedir } from "os";
@@ -8,6 +9,8 @@ import { canonicalizePath } from "../utils/paths.js";
import type { ResourceDiagnostic } from "./diagnostics.js";
import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.js";
+const log = getLogger("coding-agent.skills");
+
/** Max name length per spec */
const MAX_NAME_LENGTH = 64;
@@ -61,7 +64,9 @@ function addIgnoreRules(ig: IgnoreMatcher, dir: string, rootDir: string): void {
if (patterns.length > 0) {
ig.add(patterns);
}
- } catch {}
+ } catch {
+ // Unreadable ignore file: skip it rather than failing skill discovery.
+ }
}
}
@@ -371,7 +376,12 @@ function loadSkillsFromDirInternal(
}
diagnostics.push(...result.diagnostics);
}
- } catch {}
+ } catch (error) {
+ log.warn("skill directory scan failed", {
+ dir,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
return { skills, diagnostics };
}
diff --git a/packages/coding-agent/src/core/timings.ts b/packages/coding-agent/src/core/timings.ts
index 0b3e5cf485..677f0d8868 100644
--- a/packages/coding-agent/src/core/timings.ts
+++ b/packages/coding-agent/src/core/timings.ts
@@ -3,6 +3,9 @@
* Enable with PI_TIMING=1 environment variable.
*/
+import { getLogger } from "@earendil-works/pi-ai";
+
+const log = getLogger("coding-agent.timings");
const ENABLED = process.env.PI_TIMING === "1";
const timings: Array<{ label: string; ms: number }> = [];
let lastTime = Date.now();
@@ -22,10 +25,12 @@ export function time(label: string): void {
export function printTimings(): void {
if (!ENABLED || timings.length === 0) return;
+ const totalMs = timings.reduce((a, b) => a + b.ms, 0);
+ log.debug("startup timings", { timings: [...timings], totalMs });
console.error("\n--- Startup Timings ---");
for (const t of timings) {
console.error(` ${t.label}: ${t.ms}ms`);
}
- console.error(` TOTAL: ${timings.reduce((a, b) => a + b.ms, 0)}ms`);
+ console.error(` TOTAL: ${totalMs}ms`);
console.error("------------------------\n");
}
diff --git a/packages/coding-agent/src/core/tools/edit.ts b/packages/coding-agent/src/core/tools/edit.ts
index ad88aa60be..413e6f32c6 100644
--- a/packages/coding-agent/src/core/tools/edit.ts
+++ b/packages/coding-agent/src/core/tools/edit.ts
@@ -99,7 +99,9 @@ function prepareEditArguments(input: unknown): EditToolInput {
try {
const parsed = JSON.parse(args.edits);
if (Array.isArray(parsed)) args.edits = parsed;
- } catch {}
+ } catch {
+ // Not JSON: leave as-is for schema validation to reject.
+ }
}
const legacy = args as LegacyEditToolInput;
diff --git a/packages/coding-agent/src/core/tools/ipython.ts b/packages/coding-agent/src/core/tools/ipython.ts
index 9dbd62cbdc..3fea3e53ab 100644
--- a/packages/coding-agent/src/core/tools/ipython.ts
+++ b/packages/coding-agent/src/core/tools/ipython.ts
@@ -381,7 +381,9 @@ export function createIpythonToolDefinition(
const setWorkingMessage = (message?: string) => {
try {
ctx?.ui.setWorkingMessage(message);
- } catch {}
+ } catch {
+ // Stale ctx; cosmetic only.
+ }
};
let reportedStartupProgress = false;
const reportStartupProgress: KernelBootstrapProgressHandler = (message) => {
diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts
index 6b5c401128..5e60d8224a 100644
--- a/packages/coding-agent/src/main.ts
+++ b/packages/coding-agent/src/main.ts
@@ -40,6 +40,7 @@ import { AuthStorage } from "./core/auth-storage.js";
import { exportFromFile } from "./core/export-html/index.js";
import type { ExtensionFactory } from "./core/extensions/types.js";
import { KeybindingsManager } from "./core/keybindings.js";
+import { installFileLogSink, setLogContext } from "./core/logging.js";
import type { ModelRegistry } from "./core/model-registry.js";
import { findInitialModel, resolveCliModel, resolveModelScope, type ScopedModel } from "./core/model-resolver.js";
import { restoreStdout, takeOverStdout } from "./core/output-guard.js";
@@ -1012,6 +1013,7 @@ export interface MainOptions {
export async function main(args: string[], options?: MainOptions) {
resetTimings();
+ installFileLogSink();
// Client and daemon are separate processes; both need these in their registry.
registerBuiltinMcpOAuthProviders();
args = normalizeDaemonStartArgs(args) ?? args;
@@ -1058,6 +1060,7 @@ export async function main(args: string[], options?: MainOptions) {
}
time("parseArgs");
let appMode = resolveAppMode(parsed, process.stdin.isTTY);
+ setLogContext({ mode: appMode });
const shouldTakeOverStdout = appMode !== "interactive";
if (shouldTakeOverStdout) {
takeOverStdout();
diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
index 9d98837a8b..48c1710937 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
@@ -8,6 +8,7 @@
import { createServer, type Server, type Socket } from "node:net";
import { resolve } from "node:path";
+import { getLogger } from "@earendil-works/pi-ai";
import { appendRotatingLog, getCronJobsPath, getDaemonLogPath, VERSION } from "../../config.js";
import {
AGENT_MESSAGE_SOURCE,
@@ -128,6 +129,8 @@ export type { DaemonCommand, DaemonOutbound, DaemonResponse } from "./daemon-pro
export type { SessionActivity, SessionLifecycle, SessionSummary } from "./daemon-session-list.js";
export { defaultDaemonSocketPath } from "./daemon-socket.js";
+const structuredLog = getLogger("coding-agent.daemon");
+
const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set([
"list",
"list_saved_sessions",
@@ -266,15 +269,17 @@ export class AgentDaemon {
this.cronScheduler = new AgentCronScheduler(this.cronStore, {
runJob: (job) => this.runCronJob(job),
onError: (job, error) => {
- console.error(`Cron job ${job.id} failed: ${error instanceof Error ? error.message : String(error)}`);
+ this.log(`Cron job ${job.id} failed: ${error instanceof Error ? error.message : String(error)}`);
},
});
}
- // The daemon runs detached with no terminal, so route its diagnostics to a
- // rotating log file (and stderr too, for when it's run in the foreground).
+ // The daemon runs detached with no terminal, so route its diagnostics to its
+ // rotating log file and the shared structured log (and stderr too, for when
+ // it's run in the foreground).
private log(message: string): void {
console.error(message);
+ structuredLog.warn(message, { socketPath: this.socketPath });
appendRotatingLog(getDaemonLogPath(this.socketPath), `[${new Date().toISOString()}] ${message}`);
}
diff --git a/packages/coding-agent/test/no-silent-catch.test.ts b/packages/coding-agent/test/no-silent-catch.test.ts
new file mode 100644
index 0000000000..8ce2bb43fa
--- /dev/null
+++ b/packages/coding-agent/test/no-silent-catch.test.ts
@@ -0,0 +1,55 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import { describe, expect, test } from "vitest";
+
+/**
+ * Guardrail for ENG-4422: errors must never be swallowed without a trace.
+ * An empty `catch {}` is only acceptable with a comment explaining why the
+ * swallow is intentional (which this scan permits), or better, a log call.
+ */
+
+const REPO_ROOT = resolve(__dirname, "..", "..", "..");
+const EMPTY_CATCH = /catch(\s*\([^)]*\))?\s*\{\s*\}/g;
+
+function collectTsFiles(dir: string, out: string[] = []): string[] {
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
+ const full = join(dir, entry.name);
+ if (entry.isDirectory()) {
+ if (entry.name !== "node_modules") collectTsFiles(full, out);
+ } else if (entry.name.endsWith(".ts")) {
+ out.push(full);
+ }
+ }
+ return out;
+}
+
+describe("no silent catch blocks", () => {
+ test("every empty catch in packages/*/src carries an explanatory comment", () => {
+ const srcDirs = readdirSync(join(REPO_ROOT, "packages"), { withFileTypes: true })
+ .filter((e) => e.isDirectory())
+ .map((e) => join(REPO_ROOT, "packages", e.name, "src"));
+
+ const offenders: string[] = [];
+ for (const srcDir of srcDirs) {
+ let files: string[];
+ try {
+ files = collectTsFiles(srcDir);
+ } catch {
+ // Package without a src dir.
+ continue;
+ }
+ for (const file of files) {
+ const content = readFileSync(file, "utf-8");
+ for (const match of content.matchAll(EMPTY_CATCH)) {
+ const line = content.slice(0, match.index).split("\n").length;
+ offenders.push(`${relative(REPO_ROOT, file)}:${line}`);
+ }
+ }
+ }
+
+ expect(
+ offenders,
+ `Empty catch blocks swallow errors silently. Log the error (getLogger from @earendil-works/pi-ai) or add a comment inside the block explaining why ignoring it is safe:\n${offenders.join("\n")}`,
+ ).toEqual([]);
+ });
+});