A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
make trace uploads observable and resilient (#231)
kt237b016b04b4
diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.tsindex d6fc7a9ede..e487ca3095 100644--- a/packages/coding-agent/src/config.ts+++ b/packages/coding-agent/src/config.ts@@ -529,6 +529,10 @@ export function getClientErrorLogPath(): string {return join(getLogsDir(), "client-errors.log");}+export function getAgentTracesLogPath(): string {+ return join(getLogsDir(), "agent-traces.log");+}+/*** Log file for a daemon. The basename keeps it readable; a hash of the full* socket path makes it unique so two sockets that share a basename (e.g.diff --git a/packages/coding-agent/src/core/agent-traces.ts b/packages/coding-agent/src/core/agent-traces.tsindex a1ebc9dd97..f1227c5187 100644--- a/packages/coding-agent/src/core/agent-traces.ts+++ b/packages/coding-agent/src/core/agent-traces.ts@@ -1,7 +1,7 @@import { Buffer } from "node:buffer";import { readFile, stat } from "node:fs/promises";import { dirname, isAbsolute, resolve } from "node:path";-import { VERSION } from "../config.js";+import { appendRotatingLog, getAgentTracesLogPath, VERSION } from "../config.js";import { readFirstLineSync } from "../utils/file-lines.js";import type { AuthStorage } from "./auth-storage.js";import {@@ -17,6 +17,7 @@ const MAX_TRACE_BYTES = 20 * 1024 * 1024;const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;const TRACE_UPLOAD_DEBOUNCE_MS = 1_000;const TRACE_UPLOAD_MIN_INTERVAL_MS = 60_000;+const TRACE_UPLOAD_RETRY_DELAY_MS = 500;export type AgentTraceCredentialSource = "environment" | "stored" | "prime-inference" | "prime-cli";@@ -76,6 +77,43 @@ function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);}+function describeError(error: unknown): string {+ if (!(error instanceof Error)) {+ return String(error);+ }+ const cause = (error as { cause?: unknown }).cause;+ if (isRecord(cause)) {+ const code = typeof cause.code === "string" ? cause.code : undefined;+ const causeMessage = typeof cause.message === "string" ? cause.message : undefined;+ const detail = code ?? causeMessage;+ if (detail && detail !== error.message) {+ return `${error.message} (${detail})`;+ }+ }+ return error.message;+}++const RETRIABLE_NETWORK_CODES = new Set([+ "ECONNRESET",+ "ECONNREFUSED",+ "ECONNABORTED",+ "EPIPE",+ "ETIMEDOUT",+ "ENETUNREACH",+ "ENETDOWN",+ "EAI_AGAIN",+ "UND_ERR_SOCKET",+ "UND_ERR_CONNECT_TIMEOUT",+]);++function isRetriableNetworkError(error: unknown): boolean {+ if (!(error instanceof Error) || error.name === "AbortError") {+ return false;+ }+ const cause = (error as { cause?: unknown }).cause;+ return isRecord(cause) && typeof cause.code === "string" && RETRIABLE_NETWORK_CODES.has(cause.code);+}+function stringField(data: Record<string, unknown>, key: string): string | undefined {const value = data[key];return typeof value === "string" && value.trim() ? value.trim() : undefined;@@ -234,6 +272,43 @@ async function fetchWithTimeout(}}+function delay(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve) => {+ const timeout = setTimeout(finish, ms);+ const onAbort = () => finish();+ function finish() {+ clearTimeout(timeout);+ signal?.removeEventListener("abort", onAbort);+ resolve();+ }+ signal?.addEventListener("abort", onAbort, { once: true });+ });+}++async function fetchWithRetry(+ fetchFn: typeof fetch,+ url: string,+ init: RequestInit,+ timeoutMs: number,+ signal?: AbortSignal,+): Promise<Response> {+ try {+ return await fetchWithTimeout(fetchFn, url, init, timeoutMs, signal);+ } catch (error) {+ if (signal?.aborted) {+ throw signal.reason ?? error;+ }+ if (!isRetriableNetworkError(error)) {+ throw error;+ }+ await delay(TRACE_UPLOAD_RETRY_DELAY_MS, signal);+ if (signal?.aborted) {+ throw signal.reason ?? error;+ }+ return await fetchWithTimeout(fetchFn, url, init, timeoutMs, signal);+ }+}+export async function getPrimeAgentTraceCredential(authStorage: AuthStorage,options: { reloadAuth?: boolean; configPath?: string } = {},@@ -283,6 +358,37 @@ async function getAgentTracesEnabled(}export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Promise<AgentTraceUploadResult> {+ const result = await performAgentTraceUpload(options);+ logAgentTraceOutcome(options.sessionFile, result);+ return result;+}++function logAgentTraceOutcome(sessionFile: string | undefined, result: AgentTraceUploadResult): void {+ let line: string | undefined;+ switch (result.status) {+ case "uploaded":+ line = `uploaded session ${result.sessionId} (${result.bytesStored} bytes)`;+ break;+ case "failed":+ line = `upload failed${result.statusCode ? ` (HTTP ${result.statusCode})` : ""}: ${result.message}`;+ break;+ case "too_large":+ line = `upload skipped: session is ${result.size} bytes (limit ${result.maxBytes})`;+ break;+ case "invalid_session":+ line = `upload skipped: ${result.message}`;+ break;+ case "missing_credentials":+ line = "upload skipped: no Prime credential configured (run /traces login)";+ break;+ default:+ return;+ }+ const suffix = sessionFile ? ` [${sessionFile}]` : "";+ appendRotatingLog(getAgentTracesLogPath(), `[${new Date().toISOString()}] ${line}${suffix}`);+}++async function performAgentTraceUpload(options: AgentTraceUploadOptions): Promise<AgentTraceUploadResult> {if (!(await getAgentTracesEnabled(options))) {return { status: "disabled" };}@@ -328,7 +434,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Prtry {body = await readFile(options.sessionFile, "utf8");} catch (error) {- return { status: "failed", message: error instanceof Error ? error.message : String(error) };+ return { status: "failed", message: describeError(error) };}if (!body.trim()) {return { status: "empty_session" };@@ -366,7 +472,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Prlet response: Response;try {- response = await fetchWithTimeout(+ response = await fetchWithRetry(fetchFn,url,{@@ -378,7 +484,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Proptions.signal,);} catch (error) {- return { status: "failed", message: error instanceof Error ? error.message : String(error) };+ return { status: "failed", message: describeError(error) };}if (!response.ok) {diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.tsindex 4e1592cbb1..77e6965d68 100644--- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts+++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts@@ -46,7 +46,15 @@ import {visibleWidth,} from "@earendil-works/pi-tui";import { spawn, spawnSync } from "child_process";-import { APP_TITLE, getAgentDir, getDebugLogPath, getLogsDir, getShareViewerUrl, VERSION } from "../../config.js";+import {+ APP_TITLE,+ getAgentDir,+ getAgentTracesLogPath,+ getDebugLogPath,+ getLogsDir,+ getShareViewerUrl,+ VERSION,+} from "../../config.js";import {type AgentTraceUploadResult,getPrimeAgentTraceCredential,@@ -69,6 +77,7 @@ import { emptyGoalState, formatGoalUsage, type GoalState } from "../../core/goalimport { type AppKeybinding, KeybindingsManager } from "../../core/keybindings.js";import { createCompactionSummaryMessage } from "../../core/messages.js";import { findExactModelReferenceMatch, resolveModelScopeFromModels } from "../../core/model-resolver.js";+import { resolvePrimeAgentTracesBaseUrl } from "../../core/prime-inference-auth.js";import { formatMissingSessionCwdPrompt, MissingSessionCwdError } from "../../core/session-cwd.js";import { SessionImportFileNotFoundError } from "../../core/session-import-errors.js";import { parseSkillBlock } from "../../core/skill-blocks.js";@@ -356,7 +365,8 @@ const THINKING_LEVEL_DESCRIPTIONS: Record<ThinkingLevel, string> = {low: "Light reasoning",medium: "Moderate reasoning",high: "Deep reasoning",- xhigh: "Maximum reasoning",+ xhigh: "Very deep reasoning",+ max: "Maximum reasoning",};const DEAD_TERMINAL_ERROR_CODES = new Set(["EIO", "EPIPE", "ENOTCONN"]);@@ -6711,7 +6721,7 @@ export class InteractiveMode {if (result.statusCode === 404) {return "Trace upload endpoint was not found. The platform API may not be deployed yet, or PRIME_AGENT_TRACES_BASE_URL points at the wrong API.";}- return `Trace upload failed: ${result.statusCode ? `HTTP ${result.statusCode}: ` : ""}${result.message}`;+ return `Trace upload failed: ${result.statusCode ? `HTTP ${result.statusCode}: ` : ""}${result.message}. See ${getAgentTracesLogPath()} for details.`;}}@@ -6741,6 +6751,7 @@ export class InteractiveMode {"",`${theme.fg("dim", "Status:")} ${this.settingsManager.getAgentTracesEnabled() ? "Enabled" : "Disabled"}`,`${theme.fg("dim", "Credential:")} ${credential?.label ?? "Not configured"}`,+ `${theme.fg("dim", "Endpoint:")} ${resolvePrimeAgentTracesBaseUrl()}`,`${theme.fg("dim", "Session file:")} ${state.sessionFile ?? "In-memory"}`,"",theme.fg("dim", "Commands: /traces on, /traces off, /traces upload, /traces login"),diff --git a/packages/coding-agent/test/agent-traces.test.ts b/packages/coding-agent/test/agent-traces.test.tsindex f91f4ec416..73d96e7e7c 100644--- a/packages/coding-agent/test/agent-traces.test.ts+++ b/packages/coding-agent/test/agent-traces.test.ts@@ -4,9 +4,10 @@ import { tmpdir } from "node:os";import { join } from "node:path";import type { AssistantMessage } from "@earendil-works/pi-ai";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { ENV_AGENT_DIR, getAgentTracesLogPath } from "../src/config.js";import { flushAgentTraceUpload, installAgentTraceUpload, uploadAgentTraceFile } from "../src/core/agent-traces.js";import { AuthStorage } from "../src/core/auth-storage.js";-import { PRIME_AGENT_TRACES_PROVIDER_ID } from "../src/core/prime-inference-auth.js";+import { PRIME_AGENT_TRACES_PROVIDER_ID, PRIME_INFERENCE_PROVIDER_ID } from "../src/core/prime-inference-auth.js";import { SessionManager } from "../src/core/session-manager.js";import { SettingsManager } from "../src/core/settings-manager.js";@@ -72,9 +73,12 @@ describe("agent trace upload", () => {let originalPrimeApiKey: string | undefined;let originalTraceBaseUrl: string | undefined;let originalPrimeBaseUrl: string | undefined;+ let originalAgentDir: string | undefined;beforeEach(() => {tempDir = mkdtempSync(join(tmpdir(), "agent-traces-test-"));+ originalAgentDir = process.env[ENV_AGENT_DIR];+ process.env[ENV_AGENT_DIR] = tempDir;originalTraceApiKey = process.env.PRIME_AGENT_TRACES_API_KEY;originalPrimeApiKey = process.env.PRIME_API_KEY;originalTraceBaseUrl = process.env.PRIME_AGENT_TRACES_BASE_URL;@@ -87,6 +91,11 @@ describe("agent trace upload", () => {afterEach(() => {vi.useRealTimers();+ if (originalAgentDir === undefined) {+ delete process.env[ENV_AGENT_DIR];+ } else {+ process.env[ENV_AGENT_DIR] = originalAgentDir;+ }if (originalTraceApiKey === undefined) {delete process.env.PRIME_AGENT_TRACES_API_KEY;} else {@@ -371,4 +380,178 @@ describe("agent trace upload", () => {sessionManager.appendMessage(createUserMessage("next"));expect(Number(setTimeoutSpy.mock.calls.at(-1)?.[1])).toBe(60_000);});++ it("surfaces the underlying fetch cause and logs the failure", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "failing-session");+ const sessionFile = session.getSessionFile();++ const failingFetch: typeof fetch = async () => {+ const error = new TypeError("fetch failed");+ (error as { cause?: unknown }).cause = { code: "ENOTFOUND", message: "getaddrinfo ENOTFOUND host" };+ throw error;+ };++ const result = await uploadAgentTraceFile({+ sessionFile,+ authStorage: AuthStorage.inMemory({+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ fetchFn: failingFetch,+ reloadConfig: false,+ });++ expect(result.status).toBe("failed");+ if (result.status === "failed") {+ expect(result.message).toBe("fetch failed (ENOTFOUND)");+ }++ const logContents = readFileSync(getAgentTracesLogPath(), "utf8");+ expect(logContents).toContain("upload failed");+ expect(logContents).toContain("fetch failed (ENOTFOUND)");+ expect(logContents).toContain(sessionFile ?? "");+ });++ it("retries once on a transient connection failure, then succeeds", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "retry-session");+ const calls: FetchCall[] = [];+ let attempts = 0;+ const flakyFetch: typeof fetch = async (input, init) => {+ attempts += 1;+ if (attempts === 1) {+ const error = new TypeError("fetch failed");+ (error as { cause?: unknown }).cause = { code: "ECONNRESET" };+ throw error;+ }+ return createFetchRecorder(calls)(input, init);+ };++ const result = await uploadAgentTraceFile({+ sessionFile: session.getSessionFile(),+ authStorage: AuthStorage.inMemory({+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ fetchFn: flakyFetch,+ reloadConfig: false,+ });++ expect(attempts).toBe(2);+ expect(result.status).toBe("uploaded");+ expect(calls).toHaveLength(1);+ });++ it("surfaces the cancellation reason when aborted during the retry backoff", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "aborted-retry-session");+ const controller = new AbortController();+ const abortReason = new Error("upload cancelled");+ let attempts = 0;+ const flakyFetch: typeof fetch = async () => {+ attempts += 1;+ controller.abort(abortReason);+ const error = new TypeError("fetch failed");+ (error as { cause?: unknown }).cause = { code: "ECONNRESET" };+ throw error;+ };++ const result = await uploadAgentTraceFile({+ sessionFile: session.getSessionFile(),+ authStorage: AuthStorage.inMemory({+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ fetchFn: flakyFetch,+ signal: controller.signal,+ reloadConfig: false,+ });++ expect(attempts).toBe(1);+ expect(result.status).toBe("failed");+ if (result.status === "failed") {+ expect(result.message).toBe("upload cancelled");+ }+ });++ it("does not retry a permanent DNS failure", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "dns-failure-session");+ let attempts = 0;+ const dnsFailFetch: typeof fetch = async () => {+ attempts += 1;+ const error = new TypeError("fetch failed");+ (error as { cause?: unknown }).cause = { code: "ENOTFOUND", message: "getaddrinfo ENOTFOUND host" };+ throw error;+ };++ const result = await uploadAgentTraceFile({+ sessionFile: session.getSessionFile(),+ authStorage: AuthStorage.inMemory({+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ fetchFn: dnsFailFetch,+ reloadConfig: false,+ });++ expect(attempts).toBe(1);+ expect(result.status).toBe("failed");+ if (result.status === "failed") {+ expect(result.message).toBe("fetch failed (ENOTFOUND)");+ }+ });++ it("does not retry on an HTTP error response", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "http-error-session");+ let attempts = 0;+ const erroringFetch: typeof fetch = async () => {+ attempts += 1;+ return new Response(JSON.stringify({ detail: "bad request" }), {+ status: 400,+ headers: { "content-type": "application/json" },+ });+ };++ const result = await uploadAgentTraceFile({+ sessionFile: session.getSessionFile(),+ authStorage: AuthStorage.inMemory({+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ fetchFn: erroringFetch,+ reloadConfig: false,+ });++ expect(attempts).toBe(1);+ expect(result.status).toBe("failed");+ if (result.status === "failed") {+ expect(result.statusCode).toBe(400);+ }+ });++ it("prefers the prime-inference credential over the prime-cli config key", async () => {+ const session = writeSession(tempDir, join(tempDir, "sessions"), "credential-order-session");+ const calls: FetchCall[] = [];+ const configPath = join(tempDir, "prime-config.json");+ writeFileSync(configPath, JSON.stringify({ api_key: "cli-fallback-key" }));++ const result = await uploadAgentTraceFile({+ sessionFile: session.getSessionFile(),+ authStorage: AuthStorage.inMemory({+ [PRIME_INFERENCE_PROVIDER_ID]: { type: "api_key", key: "inference-key" },+ }),+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),+ baseUrl: "https://api.example.test",+ configPath,+ fetchFn: createFetchRecorder(calls),+ reloadConfig: false,+ });++ expect(result.status).toBe("uploaded");+ expect(calls).toHaveLength(1);+ expect(calls[0]?.init.headers).toMatchObject({ Authorization: "Bearer inference-key" });+ });});