Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(llm): describe every Codex web tool call, and mark the ones that failed
Elian Doran32da7ec7954d
diff --git a/apps/server/src/services/llm/providers/acp_agent.ts b/apps/server/src/services/llm/providers/acp_agent.tsindex 1a55945741..1cffeb54c2 100644--- a/apps/server/src/services/llm/providers/acp_agent.ts+++ b/apps/server/src/services/llm/providers/acp_agent.ts@@ -258,6 +258,18 @@ export abstract class AcpAgentProvider implements LlmProvider {return this.state().pool.turnConfig(sessionId);}+ /**+ * Mark a tool call of the chat turn running in `sessionId` as failed, with+ * `reason` as its result, for a failure the agent reports done and a hook+ * learns of only afterwards.+ */+ protected reportToolFailure(sessionId: string, toolCallId: string, reason: string): void {+ this.state().pool.deliver(sessionId, {+ sessionId,+ update: { sessionUpdate: "tool_call_update", toolCallId, status: "failed", content: [ { type: "content", content: { type: "text", text: reason } } ] }+ });+ }+/*** Whether a tool call is the agent's own bookkeeping rather than work the* user asked for, and so stays out of the chat. Display only: the call has@@ -833,6 +845,8 @@ export function createUpdateCollector(// toolCallId → display name, for labelling results; also the guard that// only this turn's tool calls produce result chunks.const toolNamesById = new Map<string, string>();+ // toolCallId → how the call ended, so each end is reported once.+ const endedById = new Map<string, "completed" | "failed">();const collector = {sessionId: undefined as string | undefined,@@ -888,7 +902,10 @@ export function createUpdateCollector(if (input && Object.keys(input).length > 0) {emit({ type: "tool_use", toolCallId, toolName, toolInput: input });}- if (update.status === "completed" || update.status === "failed") {+ // A call ends once; a later failure still replaces a completion, for an agent+ // that learns only afterwards that a call it reported done did not work.+ const ended = endedById.get(toolCallId);+ if ((update.status === "completed" && !ended) || (update.status === "failed" && ended !== "failed")) {emit({type: "tool_result",toolCallId,@@ -898,7 +915,7 @@ export function createUpdateCollector(result: flattenToolContent(update.content, update.rawOutput) || update.title || "",isError: update.status === "failed"});- toolNamesById.delete(toolCallId);+ endedById.set(toolCallId, update.status);}break;}diff --git a/apps/server/src/services/llm/providers/acp_client_pool.ts b/apps/server/src/services/llm/providers/acp_client_pool.tsindex f7868c498b..88ee76cd53 100644--- a/apps/server/src/services/llm/providers/acp_client_pool.ts+++ b/apps/server/src/services/llm/providers/acp_client_pool.ts@@ -113,6 +113,11 @@ export class AcpClientPool {this.pooled?.turns.delete(sessionId);}+ /** Hand `params` to the turn attached to `sessionId` as a `session/update` of its own. */+ deliver(sessionId: string, params: unknown): void {+ this.pooled?.turns.get(sessionId)?.onUpdate(params);+ }+/** The configuration of the turn attached to `sessionId` on the pooled client. */turnConfig(sessionId: string): LlmProviderConfig | undefined {return this.pooled?.turns.get(sessionId)?.config;diff --git a/apps/server/src/services/llm/providers/codex_agent.spec.ts b/apps/server/src/services/llm/providers/codex_agent.spec.tsindex 60da999109..2b9d2c2a31 100644--- a/apps/server/src/services/llm/providers/codex_agent.spec.ts+++ b/apps/server/src/services/llm/providers/codex_agent.spec.ts@@ -322,11 +322,23 @@ describe("CodexAgentProvider web search", () => {sessionUpdate: "tool_call_update", toolCallId: "search-2", title: "Open page: https://www.kernel.org/", status: "completed",rawInput: { type: "webSearch", query: "https://www.kernel.org/", action: { type: "openPage", url: "https://www.kernel.org/" } }});+ // A weather lookup webrun could not run, which the adapter reports as a finished search+ // without a query; only the hooks see what it asked and, after it ends, that it failed.+ const weather = { session_id: "sess-1", tool_name: "webrun", tool_use_id: "search-3" };+ await decide?.({ ...weather, hook_event_name: "PreToolUse", tool_input: { weather: [ { location: "Romania, Sibiu", duration: 3 } ], response_length: "short" } });+ update({ sessionUpdate: "tool_call", toolCallId: "search-3", kind: "search", title: "Web search", status: "in_progress", rawInput: { type: "webSearch", query: "", action: null } });+ update({ sessionUpdate: "tool_call_update", toolCallId: "search-3", title: "Web search", status: "completed", rawInput: { type: "webSearch", query: "", action: { type: "other" } } });+ await decide?.({ ...weather, hook_event_name: "PostToolUse", tool_response: [ { type: "input_text", text: "Found no tool response. This likely means the arguments you provided were not valid." } ] });// The search's results, as the PostToolUse hook posts them; the marker below cites one of them and one it never returned.await decide?.({hook_event_name: "PostToolUse", session_id: "sess-1", tool_name: "webrun",tool_response: [ { type: "input_text", text: "Vremea \u00een Sibiu (https://www.celsium.ro/vremea-sibiu)\n\uE200cite\uE202turn3search2\uE201 [wordlim: 200] Crawled: today" } ]});+ // The page of one of those results, opened by its id; the completion naming the page by+ // that id too must not replace the URL the result gave.+ await decide?.({ session_id: "sess-1", tool_name: "webrun", tool_use_id: "search-4", hook_event_name: "PreToolUse", tool_input: { open: [ { ref_id: "turn3search2" } ], response_length: "medium" } });+ update({ sessionUpdate: "tool_call", toolCallId: "search-4", kind: "search", title: "Web search", status: "in_progress", rawInput: { type: "webSearch", query: "", action: null } });+ update({ sessionUpdate: "tool_call_update", toolCallId: "search-4", title: "Open page", status: "completed", rawInput: { type: "webSearch", query: "turn3search2", action: { type: "openPage", url: "turn3search2" } } });// A citation marker, split across chunks as a stream can split it.for (const text of [ "Cloudy, 12\u00b0C. \uE200cite\uE202turn3se", "arch2\uE202turn3search0\uE201", " Low chance of rain." ]) {update({ sessionUpdate: "agent_message_chunk", content: { type: "text", text } });@@ -349,7 +361,15 @@ describe("CodexAgentProvider web search", () => {{ type: "tool_result", toolCallId: "search-1", toolName: "web_search", result: "Web search: weather Sibiu", isError: false },{ type: "tool_use", toolCallId: "search-2", toolName: "web_search", toolInput: {} },{ type: "tool_use", toolCallId: "search-2", toolName: "web_search", toolInput: { url: "https://www.kernel.org/" } },- { type: "tool_result", toolCallId: "search-2", toolName: "web_search", result: "Open page: https://www.kernel.org/", isError: false }+ { type: "tool_result", toolCallId: "search-2", toolName: "web_search", result: "Open page: https://www.kernel.org/", isError: false },+ { type: "tool_use", toolCallId: "search-3", toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } },+ { type: "tool_use", toolCallId: "search-3", toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } },+ { type: "tool_result", toolCallId: "search-3", toolName: "web_search", result: "Web search", isError: false },+ // The failure the hook learns of afterwards replaces the result.+ { type: "tool_result", toolCallId: "search-3", toolName: "web_search", result: "Found no tool response. This likely means the arguments you provided were not valid.", isError: true },+ { type: "tool_use", toolCallId: "search-4", toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } },+ { type: "tool_use", toolCallId: "search-4", toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } },+ { type: "tool_result", toolCallId: "search-4", toolName: "read_web_page", result: "Open page", isError: false }]);expect(searching.map(c => (c.type === "text" ? c.content : "")).join("")).toBe("Cloudy, 12\u00b0C. Low chance of rain.");// The marker becomes a Trilium citation for the result the search returned.diff --git a/apps/server/src/services/llm/providers/codex_agent.ts b/apps/server/src/services/llm/providers/codex_agent.tsindex 71919fc95e..7b20bc25c7 100644--- a/apps/server/src/services/llm/providers/codex_agent.ts+++ b/apps/server/src/services/llm/providers/codex_agent.ts@@ -25,7 +25,7 @@ import { type AcpClient, AcpError } from "./acp_client.js";import { getAcpHookEndpointUrl } from "./acp_mcp_endpoint.js";import { buildHookCommand, resolveCurlPath } from "./antigravity_hook.js";import { resolveCodexAcpScript, resolveCodexBinaryPath } from "./codex_binary.js";-import { buildCodexHookCommand, type CodexHookAnswer, codexSearchSources, decideCodexToolCall, writeCodexHooks } from "./codex_hook.js";+import { buildCodexHookCommand, type CodexHookAnswer, codexSearchSources, decideCodexToolCall, describeWebrunInput, webrunFailure, writeCodexHooks } from "./codex_hook.js";/** The model id that leaves the session on the model Codex picks. */const DEFAULT_MODEL_ID = "default";@@ -73,6 +73,13 @@ let catalog: { efforts: Map<string, LlmReasoningEffort[]>; titleModel?: string }*/const sourcesByTurn = new WeakMap<LlmProviderConfig, Map<string, LlmCitation>>();+/**+ * How the chat shows each `webrun` call, by its id, which is also the ACP tool+ * call's. An entry goes once its call finishes; the cap bounds calls that never do.+ */+const webrunCalls = new Map<string, BuiltInToolDisplay>();+const MAX_WEBRUN_CALLS = 200;+export class CodexAgentProvider extends AcpAgentProvider {name = "codex-agent";protected readonly logLabel = "Codex Agent provider";@@ -133,20 +140,31 @@ export class CodexAgentProvider extends AcpAgentProvider {}/**- * The web search, under the name the other providers' searches carry, with- * its query or the page it opens, where the chat looks for a detail. The- * adapter announces a search with the kind `search` and an empty query, and- * reports the query only in later updates, which carry no kind.+ * A `webrun` call, under the names the other providers' web tools carry —+ * `read_web_page` for a page it opens, `web_search` for the rest — with its+ * query or page, where the chat looks for a detail. The adapter announces+ * every call as a search with an empty query and reports the query only in+ * later updates, which carry no kind, so the hook's record fills the gap.*/protected describeBuiltInTool(update: AcpToolCallUpdate): BuiltInToolDisplay | undefined {const input = update.rawInput as { type?: unknown; query?: unknown; action?: { type?: unknown; url?: unknown } | null } | undefined;if (update.kind !== "search" && input?.type !== "webSearch") {return undefined;}- if (input?.action?.type === "openPage" && typeof input.action.url === "string") {- return { toolName: "web_search", toolInput: { url: input.action.url } };+ // What the hook saw of the call, for the `webrun` operations the adapter reports without a query.+ const call = update.toolCallId ? webrunCalls.get(update.toolCallId) : undefined;+ if (update.toolCallId && (update.status === "completed" || update.status === "failed")) {+ webrunCalls.delete(update.toolCallId);}- return { toolName: "web_search", toolInput: typeof input?.query === "string" && input.query ? { query: input.query } : {} };+ const toolName = call?.toolName ?? "web_search";+ let toolInput = call?.toolInput ?? {};+ // The adapter can name a page opened by a result id with that id, so only a URL replaces what the hook resolved.+ if (input?.action?.type === "openPage" && typeof input.action.url === "string" && /^https?:\/\//i.test(input.action.url)) {+ toolInput = { url: input.action.url };+ } else if (toolName === "web_search" && typeof input?.query === "string" && input.query) {+ toolInput = { query: input.query };+ }+ return { toolName, toolInput };}/**@@ -196,9 +214,16 @@ export class CodexAgentProvider extends AcpAgentProvider {/*** Answer the hook: decide a tool call before it runs, and after a web search- * remember its sources for the turn, which the citations in the reply name.+ * remember its sources for the turn, which the citations in the reply name,+ * or mark the call failed when it did not work.*/private answerHook(payload: unknown): Promise<CodexHookAnswer> | CodexHookAnswer {+ this.recordWebrunCall(payload);+ // Codex reports a `webrun` call done before this hook learns that it failed.+ const failure = webrunFailure(payload);+ if (failure) {+ this.reportToolFailure(failure.sessionId, failure.toolCallId, failure.reason);+ }const search = codexSearchSources(payload);if (!search) {return decideCodexToolCall(payload, sessionId => this.turnConfigOf(sessionId));@@ -214,6 +239,27 @@ export class CodexAgentProvider extends AcpAgentProvider {return {};}+ /**+ * Remember how the chat shows a `webrun` call, from its `PreToolUse` event,+ * which reaches Trilium before the adapter announces the call. A page opened+ * by a result's id is shown by that result's URL.+ */+ private recordWebrunCall(payload: unknown): void {+ const event = payload as { hook_event_name?: unknown; tool_name?: unknown; tool_use_id?: unknown; tool_input?: unknown; session_id?: unknown } | null;+ if (event?.hook_event_name !== "PreToolUse" || event.tool_name !== "webrun" || typeof event.tool_use_id !== "string") {+ return;+ }+ const config = typeof event.session_id === "string" ? this.turnConfigOf(event.session_id) : undefined;+ const sources = config ? sourcesByTurn.get(config) : undefined;+ webrunCalls.set(event.tool_use_id, describeWebrunInput(event.tool_input, ref => sources?.get(ref)?.url));+ for (const oldest of webrunCalls.keys()) {+ if (webrunCalls.size <= MAX_WEBRUN_CALLS) {+ break;+ }+ webrunCalls.delete(oldest);+ }+ }+protected describeFailure(error: unknown): string {const text = describeError(error);if (isSignInRequired(error)) {@@ -403,7 +449,8 @@ export class CitationStripper {}}-/** For tests: forget the recorded catalog. */+/** For tests: forget the recorded catalog and web tool calls. */export function resetCodexCatalogForTests(): void {catalog = { efforts: new Map() };+ webrunCalls.clear();}diff --git a/apps/server/src/services/llm/providers/codex_hook.spec.ts b/apps/server/src/services/llm/providers/codex_hook.spec.tsindex bda70f1f4d..e3a6bda372 100644--- a/apps/server/src/services/llm/providers/codex_hook.spec.ts+++ b/apps/server/src/services/llm/providers/codex_hook.spec.ts@@ -5,7 +5,7 @@ import path from "path";import { describe, expect, it } from "vitest";import { resolveCurlPath } from "./antigravity_hook.js";-import { buildCodexHookCommand, codexSearchSources, decideCodexToolCall, writeCodexHooks } from "./codex_hook.js";+import { buildCodexHookCommand, codexSearchSources, decideCodexToolCall, describeWebrunInput, webrunFailure, writeCodexHooks } from "./codex_hook.js";/** `PreToolUse` events as Codex 0.146.0 and 0.156.1 send them, less the session bookkeeping. */const EVENTS = {@@ -104,3 +104,40 @@ describe("codexSearchSources", () => {expect(codexSearchSources(null)).toBeUndefined();});});++describe("webrun calls", () => {+ it("shows a search and any other operation as a web search, with its query", () => {+ // Inputs as Codex 0.156.1 sent them.+ expect(describeWebrunInput({ search_query: [ { q: "Sibiu weather" }, { q: "Sibiu forecast" } ], response_length: "short" }))+ .toEqual({ toolName: "web_search", toolInput: { query: "Sibiu weather, Sibiu forecast" } });+ expect(describeWebrunInput({ weather: [ { location: "Romania, Sibiu", duration: 3 } ], response_length: "short" }))+ .toEqual({ toolName: "web_search", toolInput: { query: "weather: Romania, Sibiu" } });+ // Two operations at once are both named, and a call naming none shows no detail.+ expect(describeWebrunInput({ search_query: [ { q: "a" } ], open: [ { ref_id: "https://b.example/" } ] }))+ .toEqual({ toolName: "web_search", toolInput: { query: "a; open: https://b.example/" } });+ expect(describeWebrunInput({ response_length: "short" })).toEqual({ toolName: "web_search", toolInput: {} });+ expect(describeWebrunInput(undefined)).toEqual({ toolName: "web_search", toolInput: {} });+ });++ it("shows opening a page as reading it, by its URL or the URL of the result it opens", () => {+ const urlOf = (ref: string) => (ref === "turn2search0" ? "https://www.celsium.ro/vremea-sibiu" : undefined);+ expect(describeWebrunInput({ open: [ { ref_id: "https://example.com/" } ], response_length: "short" }, urlOf))+ .toEqual({ toolName: "read_web_page", toolInput: { url: "https://example.com/" } });+ expect(describeWebrunInput({ open: [ { ref_id: "turn2search0" } ], response_length: "medium" }, urlOf))+ .toEqual({ toolName: "read_web_page", toolInput: { url: "https://www.celsium.ro/vremea-sibiu" } });+ // A result this turn never saw has no URL to show; its id means nothing to the user.+ expect(describeWebrunInput({ open: [ { ref_id: "turn9search9" } ] }, urlOf)).toEqual({ toolName: "read_web_page", toolInput: {} });+ });++ it("finds a call webrun could not run, by the message it answers with", () => {+ const failure = "Found no tool response. This likely means the arguments you provided were not valid.";+ const post = (text: string, extra: object = {}) => ({+ hook_event_name: "PostToolUse", session_id: "sess-1", tool_name: "webrun", tool_use_id: "exec-1",+ tool_response: [ { type: "input_text", text } ], ...extra+ });+ expect(webrunFailure(post(failure))).toEqual({ sessionId: "sess-1", toolCallId: "exec-1", reason: failure });+ expect(webrunFailure(post("Weather in Sibiu (https://example.com)"))).toBeUndefined();+ expect(webrunFailure(post(failure, { hook_event_name: "PreToolUse" }))).toBeUndefined();+ expect(webrunFailure(post(failure, { tool_use_id: undefined }))).toBeUndefined();+ });+});diff --git a/apps/server/src/services/llm/providers/codex_hook.ts b/apps/server/src/services/llm/providers/codex_hook.tsindex 10d7869fd5..41f4fd9691 100644--- a/apps/server/src/services/llm/providers/codex_hook.ts+++ b/apps/server/src/services/llm/providers/codex_hook.ts@@ -23,7 +23,7 @@ import type { LlmProviderConfig } from "@triliumnext/core/src/services/llm/typesimport { mkdirSync, writeFileSync } from "fs";import path from "path";-import { NOTE_TOOLS_MCP_SERVER_NAME } from "./acp_agent.js";+import { type BuiltInToolDisplay, NOTE_TOOLS_MCP_SERVER_NAME } from "./acp_agent.js";import { buildHookCommand, stringsIn } from "./antigravity_hook.js";import { type HostLookup, isPublicHttpUrl } from "./public_url.js";@@ -39,6 +39,12 @@ export interface CodexHookAnswer {/** The name Codex gives its web search in hook events. */const WEB_TOOL = "webrun";+/** `webrun` arguments that tune every operation rather than name one. */+const WEBRUN_OPTIONS = new Set([ "response_length" ]);++/** How `webrun` answers a call it could not run, in place of results. */+const WEBRUN_NO_RESPONSE = "Found no tool response.";+/** `<title> (<url>)` and, on the next line, the result's citation marker, `U+E200 cite U+E202 <id> U+E201`. */const SEARCH_RESULT_HEADER = /^(.*) \((https?:\/\/\S+)\)\n\uE200cite\uE202([^\uE201\uE202]+)\uE201/gm;@@ -97,6 +103,45 @@ export function codexSearchSources(payload: unknown): { sessionId: string; sourcreturn { sessionId: event.session_id, sources };}+/**+ * How the chat shows a `webrun` call. `webrun` runs several operations —+ * `search_query` (`[{ q }]`), `open` (`[{ ref_id }]`, a URL or the id of an+ * earlier result, which `urlOf` resolves), and others such as `weather`+ * (`[{ location }]`) — which the adapter reports as a web search with an empty+ * query unless it is a search. A call that only opens a page is the chat's+ * `read_web_page`; the rest are `web_search`, with the query or operation.+ */+export function describeWebrunInput(toolInput: unknown, urlOf: (ref: string) => string | undefined = () => undefined): BuiltInToolDisplay {+ const operations = Object.entries(toolInput && typeof toolInput === "object" ? toolInput : {})+ .filter(([ operation ]) => !WEBRUN_OPTIONS.has(operation));+ if (operations.length === 1 && operations[0][0] === "open") {+ const ref = stringsIn(operations[0][1])[0];+ const url = ref && (/^https?:\/\//i.test(ref) ? ref : urlOf(ref));+ return { toolName: "read_web_page", toolInput: url ? { url } : {} };+ }+ const parts = operations.map(([ operation, value ]) => {+ const values = stringsIn(value).join(", ");+ return operation === "search_query" ? values : `${operation}: ${values}`;+ }).filter(Boolean);+ return { toolName: "web_search", toolInput: parts.length > 0 ? { query: parts.join("; ") } : {} };+}++/**+ * The `webrun` call a `PostToolUse` event reports as failed although Codex+ * reported it done: its response then says so in place of results, as for a+ * `weather` call whose location it could not read. Undefined for any other+ * event.+ */+export function webrunFailure(payload: unknown): { sessionId: string; toolCallId: string; reason: string } | undefined {+ const event = payload as { hook_event_name?: unknown; tool_name?: unknown; session_id?: unknown; tool_use_id?: unknown; tool_response?: unknown } | null;+ if (event?.hook_event_name !== "PostToolUse" || event.tool_name !== WEB_TOOL+ || typeof event.session_id !== "string" || typeof event.tool_use_id !== "string") {+ return undefined;+ }+ const reason = stringsIn(event.tool_response).find(text => text.startsWith(WEBRUN_NO_RESPONSE));+ return reason ? { sessionId: event.session_id, toolCallId: event.tool_use_id, reason } : undefined;+}+/*** Write the hooks to `<home>/hooks.json`, replacing what an earlier run wrote:* `preCommand` before every tool call, `postCommand` after, which reads the