Skip to content

Build your personal knowledge base with Trilium Notes

feat(llm): describe every Codex web tool call, and mark the ones that failed

Elian Doran32da7ec7954d
diff --git a/apps/server/src/services/llm/providers/acp_agent.ts b/apps/server/src/services/llm/providers/acp_agent.ts
index 1a55945741..1cffeb54c2 100644
--- a/apps/server/src/services/llm/providers/acp_agent.ts
+++ b/apps/server/src/services/llm/providers/acp_agent.ts
@@ -258,6 +258,18 @@ export abstract class AcpAgentProvider implements LlmProvider {
return this.state().pool.turnConfig(sessionId);
}
+ /**
+ * Mark a tool call of the chat turn running in `sessionId` as failed, with
+ * `reason` as its result, for a failure the agent reports done and a hook
+ * learns of only afterwards.
+ */
+ protected reportToolFailure(sessionId: string, toolCallId: string, reason: string): void {
+ this.state().pool.deliver(sessionId, {
+ sessionId,
+ update: { sessionUpdate: "tool_call_update", toolCallId, status: "failed", content: [ { type: "content", content: { type: "text", text: reason } } ] }
+ });
+ }
+
/**
* Whether a tool call is the agent's own bookkeeping rather than work the
* user asked for, and so stays out of the chat. Display only: the call has
@@ -833,6 +845,8 @@ export function createUpdateCollector(
// toolCallId → display name, for labelling results; also the guard that
// only this turn's tool calls produce result chunks.
const toolNamesById = new Map<string, string>();
+ // toolCallId → how the call ended, so each end is reported once.
+ const endedById = new Map<string, "completed" | "failed">();
const collector = {
sessionId: undefined as string | undefined,
@@ -888,7 +902,10 @@ export function createUpdateCollector(
if (input && Object.keys(input).length > 0) {
emit({ type: "tool_use", toolCallId, toolName, toolInput: input });
}
- if (update.status === "completed" || update.status === "failed") {
+ // A call ends once; a later failure still replaces a completion, for an agent
+ // that learns only afterwards that a call it reported done did not work.
+ const ended = endedById.get(toolCallId);
+ if ((update.status === "completed" && !ended) || (update.status === "failed" && ended !== "failed")) {
emit({
type: "tool_result",
toolCallId,
@@ -898,7 +915,7 @@ export function createUpdateCollector(
result: flattenToolContent(update.content, update.rawOutput) || update.title || "",
isError: update.status === "failed"
});
- toolNamesById.delete(toolCallId);
+ endedById.set(toolCallId, update.status);
}
break;
}
diff --git a/apps/server/src/services/llm/providers/acp_client_pool.ts b/apps/server/src/services/llm/providers/acp_client_pool.ts
index f7868c498b..88ee76cd53 100644
--- a/apps/server/src/services/llm/providers/acp_client_pool.ts
+++ b/apps/server/src/services/llm/providers/acp_client_pool.ts
@@ -113,6 +113,11 @@ export class AcpClientPool {
this.pooled?.turns.delete(sessionId);
}
+ /** Hand `params` to the turn attached to `sessionId` as a `session/update` of its own. */
+ deliver(sessionId: string, params: unknown): void {
+ this.pooled?.turns.get(sessionId)?.onUpdate(params);
+ }
+
/** The configuration of the turn attached to `sessionId` on the pooled client. */
turnConfig(sessionId: string): LlmProviderConfig | undefined {
return this.pooled?.turns.get(sessionId)?.config;
diff --git a/apps/server/src/services/llm/providers/codex_agent.spec.ts b/apps/server/src/services/llm/providers/codex_agent.spec.ts
index 60da999109..2b9d2c2a31 100644
--- a/apps/server/src/services/llm/providers/codex_agent.spec.ts
+++ b/apps/server/src/services/llm/providers/codex_agent.spec.ts
@@ -322,11 +322,23 @@ describe("CodexAgentProvider web search", () => {
sessionUpdate: "tool_call_update", toolCallId: "search-2", title: "Open page: https://www.kernel.org/", status: "completed",
rawInput: { type: "webSearch", query: "https://www.kernel.org/", action: { type: "openPage", url: "https://www.kernel.org/" } }
});
+ // A weather lookup webrun could not run, which the adapter reports as a finished search
+ // without a query; only the hooks see what it asked and, after it ends, that it failed.
+ const weather = { session_id: "sess-1", tool_name: "webrun", tool_use_id: "search-3" };
+ await decide?.({ ...weather, hook_event_name: "PreToolUse", tool_input: { weather: [ { location: "Romania, Sibiu", duration: 3 } ], response_length: "short" } });
+ update({ sessionUpdate: "tool_call", toolCallId: "search-3", kind: "search", title: "Web search", status: "in_progress", rawInput: { type: "webSearch", query: "", action: null } });
+ update({ sessionUpdate: "tool_call_update", toolCallId: "search-3", title: "Web search", status: "completed", rawInput: { type: "webSearch", query: "", action: { type: "other" } } });
+ await decide?.({ ...weather, hook_event_name: "PostToolUse", tool_response: [ { type: "input_text", text: "Found no tool response. This likely means the arguments you provided were not valid." } ] });
// The search's results, as the PostToolUse hook posts them; the marker below cites one of them and one it never returned.
await decide?.({
hook_event_name: "PostToolUse", session_id: "sess-1", tool_name: "webrun",
tool_response: [ { type: "input_text", text: "Vremea \u00een Sibiu (https://www.celsium.ro/vremea-sibiu)\n\uE200cite\uE202turn3search2\uE201 [wordlim: 200] Crawled: today" } ]
});
+ // The page of one of those results, opened by its id; the completion naming the page by
+ // that id too must not replace the URL the result gave.
+ await decide?.({ session_id: "sess-1", tool_name: "webrun", tool_use_id: "search-4", hook_event_name: "PreToolUse", tool_input: { open: [ { ref_id: "turn3search2" } ], response_length: "medium" } });
+ update({ sessionUpdate: "tool_call", toolCallId: "search-4", kind: "search", title: "Web search", status: "in_progress", rawInput: { type: "webSearch", query: "", action: null } });
+ update({ sessionUpdate: "tool_call_update", toolCallId: "search-4", title: "Open page", status: "completed", rawInput: { type: "webSearch", query: "turn3search2", action: { type: "openPage", url: "turn3search2" } } });
// A citation marker, split across chunks as a stream can split it.
for (const text of [ "Cloudy, 12\u00b0C. \uE200cite\uE202turn3se", "arch2\uE202turn3search0\uE201", " Low chance of rain." ]) {
update({ sessionUpdate: "agent_message_chunk", content: { type: "text", text } });
@@ -349,7 +361,15 @@ describe("CodexAgentProvider web search", () => {
{ type: "tool_result", toolCallId: "search-1", toolName: "web_search", result: "Web search: weather Sibiu", isError: false },
{ type: "tool_use", toolCallId: "search-2", toolName: "web_search", toolInput: {} },
{ type: "tool_use", toolCallId: "search-2", toolName: "web_search", toolInput: { url: "https://www.kernel.org/" } },
- { type: "tool_result", toolCallId: "search-2", toolName: "web_search", result: "Open page: https://www.kernel.org/", isError: false }
+ { type: "tool_result", toolCallId: "search-2", toolName: "web_search", result: "Open page: https://www.kernel.org/", isError: false },
+ { type: "tool_use", toolCallId: "search-3", toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } },
+ { type: "tool_use", toolCallId: "search-3", toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } },
+ { type: "tool_result", toolCallId: "search-3", toolName: "web_search", result: "Web search", isError: false },
+ // The failure the hook learns of afterwards replaces the result.
+ { type: "tool_result", toolCallId: "search-3", toolName: "web_search", result: "Found no tool response. This likely means the arguments you provided were not valid.", isError: true },
+ { type: "tool_use", toolCallId: "search-4", toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } },
+ { type: "tool_use", toolCallId: "search-4", toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } },
+ { type: "tool_result", toolCallId: "search-4", toolName: "read_web_page", result: "Open page", isError: false }
]);
expect(searching.map(c => (c.type === "text" ? c.content : "")).join("")).toBe("Cloudy, 12\u00b0C. Low chance of rain.");
// The marker becomes a Trilium citation for the result the search returned.
diff --git a/apps/server/src/services/llm/providers/codex_agent.ts b/apps/server/src/services/llm/providers/codex_agent.ts
index 71919fc95e..7b20bc25c7 100644
--- a/apps/server/src/services/llm/providers/codex_agent.ts
+++ b/apps/server/src/services/llm/providers/codex_agent.ts
@@ -25,7 +25,7 @@ import { type AcpClient, AcpError } from "./acp_client.js";
import { getAcpHookEndpointUrl } from "./acp_mcp_endpoint.js";
import { buildHookCommand, resolveCurlPath } from "./antigravity_hook.js";
import { resolveCodexAcpScript, resolveCodexBinaryPath } from "./codex_binary.js";
-import { buildCodexHookCommand, type CodexHookAnswer, codexSearchSources, decideCodexToolCall, writeCodexHooks } from "./codex_hook.js";
+import { buildCodexHookCommand, type CodexHookAnswer, codexSearchSources, decideCodexToolCall, describeWebrunInput, webrunFailure, writeCodexHooks } from "./codex_hook.js";
/** The model id that leaves the session on the model Codex picks. */
const DEFAULT_MODEL_ID = "default";
@@ -73,6 +73,13 @@ let catalog: { efforts: Map<string, LlmReasoningEffort[]>; titleModel?: string }
*/
const sourcesByTurn = new WeakMap<LlmProviderConfig, Map<string, LlmCitation>>();
+/**
+ * How the chat shows each `webrun` call, by its id, which is also the ACP tool
+ * call's. An entry goes once its call finishes; the cap bounds calls that never do.
+ */
+const webrunCalls = new Map<string, BuiltInToolDisplay>();
+const MAX_WEBRUN_CALLS = 200;
+
export class CodexAgentProvider extends AcpAgentProvider {
name = "codex-agent";
protected readonly logLabel = "Codex Agent provider";
@@ -133,20 +140,31 @@ export class CodexAgentProvider extends AcpAgentProvider {
}
/**
- * The web search, under the name the other providers' searches carry, with
- * its query or the page it opens, where the chat looks for a detail. The
- * adapter announces a search with the kind `search` and an empty query, and
- * reports the query only in later updates, which carry no kind.
+ * A `webrun` call, under the names the other providers' web tools carry —
+ * `read_web_page` for a page it opens, `web_search` for the rest — with its
+ * query or page, where the chat looks for a detail. The adapter announces
+ * every call as a search with an empty query and reports the query only in
+ * later updates, which carry no kind, so the hook's record fills the gap.
*/
protected describeBuiltInTool(update: AcpToolCallUpdate): BuiltInToolDisplay | undefined {
const input = update.rawInput as { type?: unknown; query?: unknown; action?: { type?: unknown; url?: unknown } | null } | undefined;
if (update.kind !== "search" && input?.type !== "webSearch") {
return undefined;
}
- if (input?.action?.type === "openPage" && typeof input.action.url === "string") {
- return { toolName: "web_search", toolInput: { url: input.action.url } };
+ // What the hook saw of the call, for the `webrun` operations the adapter reports without a query.
+ const call = update.toolCallId ? webrunCalls.get(update.toolCallId) : undefined;
+ if (update.toolCallId && (update.status === "completed" || update.status === "failed")) {
+ webrunCalls.delete(update.toolCallId);
}
- return { toolName: "web_search", toolInput: typeof input?.query === "string" && input.query ? { query: input.query } : {} };
+ const toolName = call?.toolName ?? "web_search";
+ let toolInput = call?.toolInput ?? {};
+ // The adapter can name a page opened by a result id with that id, so only a URL replaces what the hook resolved.
+ if (input?.action?.type === "openPage" && typeof input.action.url === "string" && /^https?:\/\//i.test(input.action.url)) {
+ toolInput = { url: input.action.url };
+ } else if (toolName === "web_search" && typeof input?.query === "string" && input.query) {
+ toolInput = { query: input.query };
+ }
+ return { toolName, toolInput };
}
/**
@@ -196,9 +214,16 @@ export class CodexAgentProvider extends AcpAgentProvider {
/**
* Answer the hook: decide a tool call before it runs, and after a web search
- * remember its sources for the turn, which the citations in the reply name.
+ * remember its sources for the turn, which the citations in the reply name,
+ * or mark the call failed when it did not work.
*/
private answerHook(payload: unknown): Promise<CodexHookAnswer> | CodexHookAnswer {
+ this.recordWebrunCall(payload);
+ // Codex reports a `webrun` call done before this hook learns that it failed.
+ const failure = webrunFailure(payload);
+ if (failure) {
+ this.reportToolFailure(failure.sessionId, failure.toolCallId, failure.reason);
+ }
const search = codexSearchSources(payload);
if (!search) {
return decideCodexToolCall(payload, sessionId => this.turnConfigOf(sessionId));
@@ -214,6 +239,27 @@ export class CodexAgentProvider extends AcpAgentProvider {
return {};
}
+ /**
+ * Remember how the chat shows a `webrun` call, from its `PreToolUse` event,
+ * which reaches Trilium before the adapter announces the call. A page opened
+ * by a result's id is shown by that result's URL.
+ */
+ private recordWebrunCall(payload: unknown): void {
+ const event = payload as { hook_event_name?: unknown; tool_name?: unknown; tool_use_id?: unknown; tool_input?: unknown; session_id?: unknown } | null;
+ if (event?.hook_event_name !== "PreToolUse" || event.tool_name !== "webrun" || typeof event.tool_use_id !== "string") {
+ return;
+ }
+ const config = typeof event.session_id === "string" ? this.turnConfigOf(event.session_id) : undefined;
+ const sources = config ? sourcesByTurn.get(config) : undefined;
+ webrunCalls.set(event.tool_use_id, describeWebrunInput(event.tool_input, ref => sources?.get(ref)?.url));
+ for (const oldest of webrunCalls.keys()) {
+ if (webrunCalls.size <= MAX_WEBRUN_CALLS) {
+ break;
+ }
+ webrunCalls.delete(oldest);
+ }
+ }
+
protected describeFailure(error: unknown): string {
const text = describeError(error);
if (isSignInRequired(error)) {
@@ -403,7 +449,8 @@ export class CitationStripper {
}
}
-/** For tests: forget the recorded catalog. */
+/** For tests: forget the recorded catalog and web tool calls. */
export function resetCodexCatalogForTests(): void {
catalog = { efforts: new Map() };
+ webrunCalls.clear();
}
diff --git a/apps/server/src/services/llm/providers/codex_hook.spec.ts b/apps/server/src/services/llm/providers/codex_hook.spec.ts
index bda70f1f4d..e3a6bda372 100644
--- a/apps/server/src/services/llm/providers/codex_hook.spec.ts
+++ b/apps/server/src/services/llm/providers/codex_hook.spec.ts
@@ -5,7 +5,7 @@ import path from "path";
import { describe, expect, it } from "vitest";
import { resolveCurlPath } from "./antigravity_hook.js";
-import { buildCodexHookCommand, codexSearchSources, decideCodexToolCall, writeCodexHooks } from "./codex_hook.js";
+import { buildCodexHookCommand, codexSearchSources, decideCodexToolCall, describeWebrunInput, webrunFailure, writeCodexHooks } from "./codex_hook.js";
/** `PreToolUse` events as Codex 0.146.0 and 0.156.1 send them, less the session bookkeeping. */
const EVENTS = {
@@ -104,3 +104,40 @@ describe("codexSearchSources", () => {
expect(codexSearchSources(null)).toBeUndefined();
});
});
+
+describe("webrun calls", () => {
+ it("shows a search and any other operation as a web search, with its query", () => {
+ // Inputs as Codex 0.156.1 sent them.
+ expect(describeWebrunInput({ search_query: [ { q: "Sibiu weather" }, { q: "Sibiu forecast" } ], response_length: "short" }))
+ .toEqual({ toolName: "web_search", toolInput: { query: "Sibiu weather, Sibiu forecast" } });
+ expect(describeWebrunInput({ weather: [ { location: "Romania, Sibiu", duration: 3 } ], response_length: "short" }))
+ .toEqual({ toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } });
+ // Two operations at once are both named, and a call naming none shows no detail.
+ expect(describeWebrunInput({ search_query: [ { q: "a" } ], open: [ { ref_id: "https://b.example/" } ] }))
+ .toEqual({ toolName: "web_search", toolInput: { query: "a; open: https://b.example/" } });
+ expect(describeWebrunInput({ response_length: "short" })).toEqual({ toolName: "web_search", toolInput: {} });
+ expect(describeWebrunInput(undefined)).toEqual({ toolName: "web_search", toolInput: {} });
+ });
+
+ it("shows opening a page as reading it, by its URL or the URL of the result it opens", () => {
+ const urlOf = (ref: string) => (ref === "turn2search0" ? "https://www.celsium.ro/vremea-sibiu" : undefined);
+ expect(describeWebrunInput({ open: [ { ref_id: "https://example.com/" } ], response_length: "short" }, urlOf))
+ .toEqual({ toolName: "read_web_page", toolInput: { url: "https://example.com/" } });
+ expect(describeWebrunInput({ open: [ { ref_id: "turn2search0" } ], response_length: "medium" }, urlOf))
+ .toEqual({ toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } });
+ // A result this turn never saw has no URL to show; its id means nothing to the user.
+ expect(describeWebrunInput({ open: [ { ref_id: "turn9search9" } ] }, urlOf)).toEqual({ toolName: "read_web_page", toolInput: {} });
+ });
+
+ it("finds a call webrun could not run, by the message it answers with", () => {
+ const failure = "Found no tool response. This likely means the arguments you provided were not valid.";
+ const post = (text: string, extra: object = {}) => ({
+ hook_event_name: "PostToolUse", session_id: "sess-1", tool_name: "webrun", tool_use_id: "exec-1",
+ tool_response: [ { type: "input_text", text } ], ...extra
+ });
+ expect(webrunFailure(post(failure))).toEqual({ sessionId: "sess-1", toolCallId: "exec-1", reason: failure });
+ expect(webrunFailure(post("Weather in Sibiu (https://example.com)"))).toBeUndefined();
+ expect(webrunFailure(post(failure, { hook_event_name: "PreToolUse" }))).toBeUndefined();
+ expect(webrunFailure(post(failure, { tool_use_id: undefined }))).toBeUndefined();
+ });
+});
diff --git a/apps/server/src/services/llm/providers/codex_hook.ts b/apps/server/src/services/llm/providers/codex_hook.ts
index 10d7869fd5..41f4fd9691 100644
--- a/apps/server/src/services/llm/providers/codex_hook.ts
+++ b/apps/server/src/services/llm/providers/codex_hook.ts
@@ -23,7 +23,7 @@ import type { LlmProviderConfig } from "@triliumnext/core/src/services/llm/types
import { mkdirSync, writeFileSync } from "fs";
import path from "path";
-import { NOTE_TOOLS_MCP_SERVER_NAME } from "./acp_agent.js";
+import { type BuiltInToolDisplay, NOTE_TOOLS_MCP_SERVER_NAME } from "./acp_agent.js";
import { buildHookCommand, stringsIn } from "./antigravity_hook.js";
import { type HostLookup, isPublicHttpUrl } from "./public_url.js";
@@ -39,6 +39,12 @@ export interface CodexHookAnswer {
/** The name Codex gives its web search in hook events. */
const WEB_TOOL = "webrun";
+/** `webrun` arguments that tune every operation rather than name one. */
+const WEBRUN_OPTIONS = new Set([ "response_length" ]);
+
+/** How `webrun` answers a call it could not run, in place of results. */
+const WEBRUN_NO_RESPONSE = "Found no tool response.";
+
/** `<title> (<url>)` and, on the next line, the result's citation marker, `U+E200 cite U+E202 <id> U+E201`. */
const SEARCH_RESULT_HEADER = /^(.*) \((https?:\/\/\S+)\)\n\uE200cite\uE202([^\uE201\uE202]+)\uE201/gm;
@@ -97,6 +103,45 @@ export function codexSearchSources(payload: unknown): { sessionId: string; sourc
return { sessionId: event.session_id, sources };
}
+/**
+ * How the chat shows a `webrun` call. `webrun` runs several operations —
+ * `search_query` (`[{ q }]`), `open` (`[{ ref_id }]`, a URL or the id of an
+ * earlier result, which `urlOf` resolves), and others such as `weather`
+ * (`[{ location }]`) — which the adapter reports as a web search with an empty
+ * query unless it is a search. A call that only opens a page is the chat's
+ * `read_web_page`; the rest are `web_search`, with the query or operation.
+ */
+export function describeWebrunInput(toolInput: unknown, urlOf: (ref: string) => string | undefined = () => undefined): BuiltInToolDisplay {
+ const operations = Object.entries(toolInput && typeof toolInput === "object" ? toolInput : {})
+ .filter(([ operation ]) => !WEBRUN_OPTIONS.has(operation));
+ if (operations.length === 1 && operations[0][0] === "open") {
+ const ref = stringsIn(operations[0][1])[0];
+ const url = ref && (/^https?:\/\//i.test(ref) ? ref : urlOf(ref));
+ return { toolName: "read_web_page", toolInput: url ? { url } : {} };
+ }
+ const parts = operations.map(([ operation, value ]) => {
+ const values = stringsIn(value).join(", ");
+ return operation === "search_query" ? values : `${operation}: ${values}`;
+ }).filter(Boolean);
+ return { toolName: "web_search", toolInput: parts.length > 0 ? { query: parts.join("; ") } : {} };
+}
+
+/**
+ * The `webrun` call a `PostToolUse` event reports as failed although Codex
+ * reported it done: its response then says so in place of results, as for a
+ * `weather` call whose location it could not read. Undefined for any other
+ * event.
+ */
+export function webrunFailure(payload: unknown): { sessionId: string; toolCallId: string; reason: string } | undefined {
+ const event = payload as { hook_event_name?: unknown; tool_name?: unknown; session_id?: unknown; tool_use_id?: unknown; tool_response?: unknown } | null;
+ if (event?.hook_event_name !== "PostToolUse" || event.tool_name !== WEB_TOOL
+ || typeof event.session_id !== "string" || typeof event.tool_use_id !== "string") {
+ return undefined;
+ }
+ const reason = stringsIn(event.tool_response).find(text => text.startsWith(WEBRUN_NO_RESPONSE));
+ return reason ? { sessionId: event.session_id, toolCallId: event.tool_use_id, reason } : undefined;
+}
+
/**
* Write the hooks to `<home>/hooks.json`, replacing what an earlier run wrote:
* `preCommand` before every tool call, `postCommand` after, which reads the