Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

make trace uploads observable and resilient (#231)

kt237b016b04b4
diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts
index d6fc7a9ede..e487ca3095 100644
--- a/packages/coding-agent/src/config.ts
+++ b/packages/coding-agent/src/config.ts
@@ -529,6 +529,10 @@ export function getClientErrorLogPath(): string {
return join(getLogsDir(), "client-errors.log");
}
+export function getAgentTracesLogPath(): string {
+ return join(getLogsDir(), "agent-traces.log");
+}
+
/**
* Log file for a daemon. The basename keeps it readable; a hash of the full
* socket path makes it unique so two sockets that share a basename (e.g.
diff --git a/packages/coding-agent/src/core/agent-traces.ts b/packages/coding-agent/src/core/agent-traces.ts
index a1ebc9dd97..f1227c5187 100644
--- a/packages/coding-agent/src/core/agent-traces.ts
+++ b/packages/coding-agent/src/core/agent-traces.ts
@@ -1,7 +1,7 @@
import { Buffer } from "node:buffer";
import { readFile, stat } from "node:fs/promises";
import { dirname, isAbsolute, resolve } from "node:path";
-import { VERSION } from "../config.js";
+import { appendRotatingLog, getAgentTracesLogPath, VERSION } from "../config.js";
import { readFirstLineSync } from "../utils/file-lines.js";
import type { AuthStorage } from "./auth-storage.js";
import {
@@ -17,6 +17,7 @@ const MAX_TRACE_BYTES = 20 * 1024 * 1024;
const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
const TRACE_UPLOAD_DEBOUNCE_MS = 1_000;
const TRACE_UPLOAD_MIN_INTERVAL_MS = 60_000;
+const TRACE_UPLOAD_RETRY_DELAY_MS = 500;
export type AgentTraceCredentialSource = "environment" | "stored" | "prime-inference" | "prime-cli";
@@ -76,6 +77,43 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
+function describeError(error: unknown): string {
+ if (!(error instanceof Error)) {
+ return String(error);
+ }
+ const cause = (error as { cause?: unknown }).cause;
+ if (isRecord(cause)) {
+ const code = typeof cause.code === "string" ? cause.code : undefined;
+ const causeMessage = typeof cause.message === "string" ? cause.message : undefined;
+ const detail = code ?? causeMessage;
+ if (detail && detail !== error.message) {
+ return `${error.message} (${detail})`;
+ }
+ }
+ return error.message;
+}
+
+const RETRIABLE_NETWORK_CODES = new Set([
+ "ECONNRESET",
+ "ECONNREFUSED",
+ "ECONNABORTED",
+ "EPIPE",
+ "ETIMEDOUT",
+ "ENETUNREACH",
+ "ENETDOWN",
+ "EAI_AGAIN",
+ "UND_ERR_SOCKET",
+ "UND_ERR_CONNECT_TIMEOUT",
+]);
+
+function isRetriableNetworkError(error: unknown): boolean {
+ if (!(error instanceof Error) || error.name === "AbortError") {
+ return false;
+ }
+ const cause = (error as { cause?: unknown }).cause;
+ return isRecord(cause) && typeof cause.code === "string" && RETRIABLE_NETWORK_CODES.has(cause.code);
+}
+
function stringField(data: Record<string, unknown>, key: string): string | undefined {
const value = data[key];
return typeof value === "string" && value.trim() ? value.trim() : undefined;
@@ -234,6 +272,43 @@ async function fetchWithTimeout(
}
}
+function delay(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve) => {
+ const timeout = setTimeout(finish, ms);
+ const onAbort = () => finish();
+ function finish() {
+ clearTimeout(timeout);
+ signal?.removeEventListener("abort", onAbort);
+ resolve();
+ }
+ signal?.addEventListener("abort", onAbort, { once: true });
+ });
+}
+
+async function fetchWithRetry(
+ fetchFn: typeof fetch,
+ url: string,
+ init: RequestInit,
+ timeoutMs: number,
+ signal?: AbortSignal,
+): Promise<Response> {
+ try {
+ return await fetchWithTimeout(fetchFn, url, init, timeoutMs, signal);
+ } catch (error) {
+ if (signal?.aborted) {
+ throw signal.reason ?? error;
+ }
+ if (!isRetriableNetworkError(error)) {
+ throw error;
+ }
+ await delay(TRACE_UPLOAD_RETRY_DELAY_MS, signal);
+ if (signal?.aborted) {
+ throw signal.reason ?? error;
+ }
+ return await fetchWithTimeout(fetchFn, url, init, timeoutMs, signal);
+ }
+}
+
export async function getPrimeAgentTraceCredential(
authStorage: AuthStorage,
options: { reloadAuth?: boolean; configPath?: string } = {},
@@ -283,6 +358,37 @@ async function getAgentTracesEnabled(
}
export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Promise<AgentTraceUploadResult> {
+ const result = await performAgentTraceUpload(options);
+ logAgentTraceOutcome(options.sessionFile, result);
+ return result;
+}
+
+function logAgentTraceOutcome(sessionFile: string | undefined, result: AgentTraceUploadResult): void {
+ let line: string | undefined;
+ switch (result.status) {
+ case "uploaded":
+ line = `uploaded session ${result.sessionId} (${result.bytesStored} bytes)`;
+ break;
+ case "failed":
+ line = `upload failed${result.statusCode ? ` (HTTP ${result.statusCode})` : ""}: ${result.message}`;
+ break;
+ case "too_large":
+ line = `upload skipped: session is ${result.size} bytes (limit ${result.maxBytes})`;
+ break;
+ case "invalid_session":
+ line = `upload skipped: ${result.message}`;
+ break;
+ case "missing_credentials":
+ line = "upload skipped: no Prime credential configured (run /traces login)";
+ break;
+ default:
+ return;
+ }
+ const suffix = sessionFile ? ` [${sessionFile}]` : "";
+ appendRotatingLog(getAgentTracesLogPath(), `[${new Date().toISOString()}] ${line}${suffix}`);
+}
+
+async function performAgentTraceUpload(options: AgentTraceUploadOptions): Promise<AgentTraceUploadResult> {
if (!(await getAgentTracesEnabled(options))) {
return { status: "disabled" };
}
@@ -328,7 +434,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Pr
try {
body = await readFile(options.sessionFile, "utf8");
} catch (error) {
- return { status: "failed", message: error instanceof Error ? error.message : String(error) };
+ return { status: "failed", message: describeError(error) };
}
if (!body.trim()) {
return { status: "empty_session" };
@@ -366,7 +472,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Pr
let response: Response;
try {
- response = await fetchWithTimeout(
+ response = await fetchWithRetry(
fetchFn,
url,
{
@@ -378,7 +484,7 @@ export async function uploadAgentTraceFile(options: AgentTraceUploadOptions): Pr
options.signal,
);
} catch (error) {
- return { status: "failed", message: error instanceof Error ? error.message : String(error) };
+ return { status: "failed", message: describeError(error) };
}
if (!response.ok) {
diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
index 4e1592cbb1..77e6965d68 100644
--- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts
+++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
@@ -46,7 +46,15 @@ import {
visibleWidth,
} from "@earendil-works/pi-tui";
import { spawn, spawnSync } from "child_process";
-import { APP_TITLE, getAgentDir, getDebugLogPath, getLogsDir, getShareViewerUrl, VERSION } from "../../config.js";
+import {
+ APP_TITLE,
+ getAgentDir,
+ getAgentTracesLogPath,
+ getDebugLogPath,
+ getLogsDir,
+ getShareViewerUrl,
+ VERSION,
+} from "../../config.js";
import {
type AgentTraceUploadResult,
getPrimeAgentTraceCredential,
@@ -69,6 +77,7 @@ import { emptyGoalState, formatGoalUsage, type GoalState } from "../../core/goal
import { type AppKeybinding, KeybindingsManager } from "../../core/keybindings.js";
import { createCompactionSummaryMessage } from "../../core/messages.js";
import { findExactModelReferenceMatch, resolveModelScopeFromModels } from "../../core/model-resolver.js";
+import { resolvePrimeAgentTracesBaseUrl } from "../../core/prime-inference-auth.js";
import { formatMissingSessionCwdPrompt, MissingSessionCwdError } from "../../core/session-cwd.js";
import { SessionImportFileNotFoundError } from "../../core/session-import-errors.js";
import { parseSkillBlock } from "../../core/skill-blocks.js";
@@ -356,7 +365,8 @@ const THINKING_LEVEL_DESCRIPTIONS: Record<ThinkingLevel, string> = {
low: "Light reasoning",
medium: "Moderate reasoning",
high: "Deep reasoning",
- xhigh: "Maximum reasoning",
+ xhigh: "Very deep reasoning",
+ max: "Maximum reasoning",
};
const DEAD_TERMINAL_ERROR_CODES = new Set(["EIO", "EPIPE", "ENOTCONN"]);
@@ -6711,7 +6721,7 @@ export class InteractiveMode {
if (result.statusCode === 404) {
return "Trace upload endpoint was not found. The platform API may not be deployed yet, or PRIME_AGENT_TRACES_BASE_URL points at the wrong API.";
}
- return `Trace upload failed: ${result.statusCode ? `HTTP ${result.statusCode}: ` : ""}${result.message}`;
+ return `Trace upload failed: ${result.statusCode ? `HTTP ${result.statusCode}: ` : ""}${result.message}. See ${getAgentTracesLogPath()} for details.`;
}
}
@@ -6741,6 +6751,7 @@ export class InteractiveMode {
"",
`${theme.fg("dim", "Status:")} ${this.settingsManager.getAgentTracesEnabled() ? "Enabled" : "Disabled"}`,
`${theme.fg("dim", "Credential:")} ${credential?.label ?? "Not configured"}`,
+ `${theme.fg("dim", "Endpoint:")} ${resolvePrimeAgentTracesBaseUrl()}`,
`${theme.fg("dim", "Session file:")} ${state.sessionFile ?? "In-memory"}`,
"",
theme.fg("dim", "Commands: /traces on, /traces off, /traces upload, /traces login"),
diff --git a/packages/coding-agent/test/agent-traces.test.ts b/packages/coding-agent/test/agent-traces.test.ts
index f91f4ec416..73d96e7e7c 100644
--- a/packages/coding-agent/test/agent-traces.test.ts
+++ b/packages/coding-agent/test/agent-traces.test.ts
@@ -4,9 +4,10 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AssistantMessage } from "@earendil-works/pi-ai";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { ENV_AGENT_DIR, getAgentTracesLogPath } from "../src/config.js";
import { flushAgentTraceUpload, installAgentTraceUpload, uploadAgentTraceFile } from "../src/core/agent-traces.js";
import { AuthStorage } from "../src/core/auth-storage.js";
-import { PRIME_AGENT_TRACES_PROVIDER_ID } from "../src/core/prime-inference-auth.js";
+import { PRIME_AGENT_TRACES_PROVIDER_ID, PRIME_INFERENCE_PROVIDER_ID } from "../src/core/prime-inference-auth.js";
import { SessionManager } from "../src/core/session-manager.js";
import { SettingsManager } from "../src/core/settings-manager.js";
@@ -72,9 +73,12 @@ describe("agent trace upload", () => {
let originalPrimeApiKey: string | undefined;
let originalTraceBaseUrl: string | undefined;
let originalPrimeBaseUrl: string | undefined;
+ let originalAgentDir: string | undefined;
beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), "agent-traces-test-"));
+ originalAgentDir = process.env[ENV_AGENT_DIR];
+ process.env[ENV_AGENT_DIR] = tempDir;
originalTraceApiKey = process.env.PRIME_AGENT_TRACES_API_KEY;
originalPrimeApiKey = process.env.PRIME_API_KEY;
originalTraceBaseUrl = process.env.PRIME_AGENT_TRACES_BASE_URL;
@@ -87,6 +91,11 @@ describe("agent trace upload", () => {
afterEach(() => {
vi.useRealTimers();
+ if (originalAgentDir === undefined) {
+ delete process.env[ENV_AGENT_DIR];
+ } else {
+ process.env[ENV_AGENT_DIR] = originalAgentDir;
+ }
if (originalTraceApiKey === undefined) {
delete process.env.PRIME_AGENT_TRACES_API_KEY;
} else {
@@ -371,4 +380,178 @@ describe("agent trace upload", () => {
sessionManager.appendMessage(createUserMessage("next"));
expect(Number(setTimeoutSpy.mock.calls.at(-1)?.[1])).toBe(60_000);
});
+
+ it("surfaces the underlying fetch cause and logs the failure", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "failing-session");
+ const sessionFile = session.getSessionFile();
+
+ const failingFetch: typeof fetch = async () => {
+ const error = new TypeError("fetch failed");
+ (error as { cause?: unknown }).cause = { code: "ENOTFOUND", message: "getaddrinfo ENOTFOUND host" };
+ throw error;
+ };
+
+ const result = await uploadAgentTraceFile({
+ sessionFile,
+ authStorage: AuthStorage.inMemory({
+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ fetchFn: failingFetch,
+ reloadConfig: false,
+ });
+
+ expect(result.status).toBe("failed");
+ if (result.status === "failed") {
+ expect(result.message).toBe("fetch failed (ENOTFOUND)");
+ }
+
+ const logContents = readFileSync(getAgentTracesLogPath(), "utf8");
+ expect(logContents).toContain("upload failed");
+ expect(logContents).toContain("fetch failed (ENOTFOUND)");
+ expect(logContents).toContain(sessionFile ?? "");
+ });
+
+ it("retries once on a transient connection failure, then succeeds", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "retry-session");
+ const calls: FetchCall[] = [];
+ let attempts = 0;
+ const flakyFetch: typeof fetch = async (input, init) => {
+ attempts += 1;
+ if (attempts === 1) {
+ const error = new TypeError("fetch failed");
+ (error as { cause?: unknown }).cause = { code: "ECONNRESET" };
+ throw error;
+ }
+ return createFetchRecorder(calls)(input, init);
+ };
+
+ const result = await uploadAgentTraceFile({
+ sessionFile: session.getSessionFile(),
+ authStorage: AuthStorage.inMemory({
+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ fetchFn: flakyFetch,
+ reloadConfig: false,
+ });
+
+ expect(attempts).toBe(2);
+ expect(result.status).toBe("uploaded");
+ expect(calls).toHaveLength(1);
+ });
+
+ it("surfaces the cancellation reason when aborted during the retry backoff", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "aborted-retry-session");
+ const controller = new AbortController();
+ const abortReason = new Error("upload cancelled");
+ let attempts = 0;
+ const flakyFetch: typeof fetch = async () => {
+ attempts += 1;
+ controller.abort(abortReason);
+ const error = new TypeError("fetch failed");
+ (error as { cause?: unknown }).cause = { code: "ECONNRESET" };
+ throw error;
+ };
+
+ const result = await uploadAgentTraceFile({
+ sessionFile: session.getSessionFile(),
+ authStorage: AuthStorage.inMemory({
+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ fetchFn: flakyFetch,
+ signal: controller.signal,
+ reloadConfig: false,
+ });
+
+ expect(attempts).toBe(1);
+ expect(result.status).toBe("failed");
+ if (result.status === "failed") {
+ expect(result.message).toBe("upload cancelled");
+ }
+ });
+
+ it("does not retry a permanent DNS failure", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "dns-failure-session");
+ let attempts = 0;
+ const dnsFailFetch: typeof fetch = async () => {
+ attempts += 1;
+ const error = new TypeError("fetch failed");
+ (error as { cause?: unknown }).cause = { code: "ENOTFOUND", message: "getaddrinfo ENOTFOUND host" };
+ throw error;
+ };
+
+ const result = await uploadAgentTraceFile({
+ sessionFile: session.getSessionFile(),
+ authStorage: AuthStorage.inMemory({
+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ fetchFn: dnsFailFetch,
+ reloadConfig: false,
+ });
+
+ expect(attempts).toBe(1);
+ expect(result.status).toBe("failed");
+ if (result.status === "failed") {
+ expect(result.message).toBe("fetch failed (ENOTFOUND)");
+ }
+ });
+
+ it("does not retry on an HTTP error response", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "http-error-session");
+ let attempts = 0;
+ const erroringFetch: typeof fetch = async () => {
+ attempts += 1;
+ return new Response(JSON.stringify({ detail: "bad request" }), {
+ status: 400,
+ headers: { "content-type": "application/json" },
+ });
+ };
+
+ const result = await uploadAgentTraceFile({
+ sessionFile: session.getSessionFile(),
+ authStorage: AuthStorage.inMemory({
+ [PRIME_AGENT_TRACES_PROVIDER_ID]: { type: "api_key", key: "trace-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ fetchFn: erroringFetch,
+ reloadConfig: false,
+ });
+
+ expect(attempts).toBe(1);
+ expect(result.status).toBe("failed");
+ if (result.status === "failed") {
+ expect(result.statusCode).toBe(400);
+ }
+ });
+
+ it("prefers the prime-inference credential over the prime-cli config key", async () => {
+ const session = writeSession(tempDir, join(tempDir, "sessions"), "credential-order-session");
+ const calls: FetchCall[] = [];
+ const configPath = join(tempDir, "prime-config.json");
+ writeFileSync(configPath, JSON.stringify({ api_key: "cli-fallback-key" }));
+
+ const result = await uploadAgentTraceFile({
+ sessionFile: session.getSessionFile(),
+ authStorage: AuthStorage.inMemory({
+ [PRIME_INFERENCE_PROVIDER_ID]: { type: "api_key", key: "inference-key" },
+ }),
+ settingsManager: SettingsManager.inMemory({ agentTraces: { enabled: true } }),
+ baseUrl: "https://api.example.test",
+ configPath,
+ fetchFn: createFetchRecorder(calls),
+ reloadConfig: false,
+ });
+
+ expect(result.status).toBe("uploaded");
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.init.headers).toMatchObject({ Authorization: "Bearer inference-key" });
+ });
});