A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(agent): a second protocol, not a third dialect of the first
esenginec88e8ae15dab
diff --git a/desktop/electron/agent/anthropic.ts b/desktop/electron/agent/anthropic.tsindex 9748f56f2..f757b5a54 100644--- a/desktop/electron/agent/anthropic.ts+++ b/desktop/electron/agent/anthropic.ts@@ -24,10 +24,13 @@ import Anthropic from '@anthropic-ai/sdk';import type {AgentProvider, AgentSession, CatalogTool, StepEvent, ToolOutcome, UserImage,} from './types';-import {- DEFAULT_MODEL, DEFAULT_CONTEXT_WINDOW, DEFAULT_EFFORT, KEEP_WHOLE_RUNS, shouldCompact,-} from '../../src/settings/agentIds';+import { ConversationLog, foldLine, type HistoryShape } from './conversation';+import { describeApiError, isTransientApiError } from './apiError';+import { DEFAULT_MODEL, DEFAULT_CONTEXT_WINDOW, DEFAULT_EFFORT } from '../../src/settings/agentIds';+/** Names this history's wire format on disk, so it is never resumed into+ * another provider's session — see conversation.ts LogMemory. */+export const ANTHROPIC_FORMAT = 'anthropic';/**@@ -149,193 +152,50 @@ export function toolResultContent(}/**- * What went wrong, said to the person who has to decide what to do about it.- *- * The SDK's own message is written for whoever is reading a stack trace — it- * names a status code and a URL. In the transcript the only useful content is- * which of four situations this is, because each has a different answer: wait,- * fix the key, check the address, or nothing (it already retried).- *- * Exported so the phrasing is pinned by a test rather than by whichever error- * happens to occur first in production.- */-/** The SDK hands back a `Headers` on some paths and a plain object on others;- * reading only one shape is how the useful half of a 429 goes missing. */-function headerOf(error: unknown, name: string): string | undefined {- const headers = (error as { headers?: unknown })?.headers;- if (!headers) return undefined;- const get = (headers as Headers).get;- if (typeof get === 'function') return (headers as Headers).get(name) ?? undefined;- return (headers as Record<string, string>)[name];-}--/**- * Whether asking again could plausibly work: the failure was the CONNECTION or- * the far side, not the request.- *- * A stream that dies halfway is the common one, and it is not rare over a long- * turn — a dogfood run lost eighty-seven rounds of work to a socket the gateway- * closed. It arrives with no HTTP status at all, because the request had already- * succeeded and the body stopped arriving.+ * How the shared bookkeeping reads THIS format. Everything else about folding a+ * conversation is in conversation.ts, which is not about a wire format.*/-export function isTransientApiError(error: unknown): boolean {- const status = (error as { status?: number })?.status;- if (status === undefined || status === null) return true;- return status === 408 || status === 429 || status >= 500;-}--export function describeApiError(error: unknown): string {- const status = (error as { status?: number })?.status;- const raw = (error as Error)?.message ?? String(error);- const retryAfter = Number(headerOf(error, 'retry-after'));-- if (status === 429) {- // The SDK has already retried this with backoff by the time it reaches us,- // so "try again" is not advice — how long to wait is.- return Number.isFinite(retryAfter) && retryAfter > 0- ? `Rate limited, and the automatic retries did not clear it. The endpoint asks for ${retryAfter}s.`- : 'Rate limited, and the automatic retries did not clear it. Wait a moment and send again.';- }- if (status === 401 || status === 403) {- return 'The endpoint rejected the API key. Check it in Settings › AI Agents.';- }- if (status === 404) {- return 'The endpoint has no such model, or the address is wrong. Check both in Settings › AI Agents.';- }- if (status === 400) return `The endpoint refused the request: ${raw}`;- if (status && status >= 500) return 'The endpoint is having trouble. Nothing is wrong on this side — try again shortly.';- // Not an HTTP failure at all: DNS, offline, a gateway that is not running.- if (!status) return `Could not reach the endpoint: ${raw}`;- return raw;-}--/**- * Fold the oldest runs into one note, keeping what the PERSON said.- *- * A conversation grows mostly by tool traffic — a scene tree, a diagnostics- * list, the reasoning about them — and almost none of that is worth carrying- * once the edits have landed. The intent is: "make it feel like dusk" governs- * the ninth run as much as the first. So the person's own words survive verbatim- * and everything around them becomes a line.- *- * Structural rather than model-written: asking a model to summarise costs a call- * and a wait at exactly the moment the conversation is already long, and the- * facts worth keeping (what was asked) are ones already held exactly.- *- * Pure and exported for the reason buildStepRequest is: it rewrites the history- * everything else depends on, and the bookkeeping — turn COORDINATES outliving- * the messages they named — is the part worth pinning by test rather than by- * whichever long conversation happens to hit it first.- *- * @returns the rewritten history, or null when there is not enough to fold.- */-export function compactHistory(- messages: readonly Message[],- turnStarts: readonly number[],- dropped: number,- keepRuns: number,- folded: readonly string[] = [],-): { messages: Message[]; turnStarts: number[]; dropped: number; folded: string[] } | null {- const cut = turnStarts.length - keepRuns;- if (cut <= 0) return null;- const at = turnStarts[cut];- // Everything ever folded, not only this pass. The note a fold produces is NOT- // in turnStarts, so the next one splices straight over it — carrying the asks- // in a value of their own is what stops the oldest request disappearing one- // compaction at a time. (Caught against a real gateway: the first fold kept- // "remember the passphrase", the second silently ate it.)- const asked = [- ...folded,- ...turnStarts.slice(0, cut).map((start, i) => `${dropped + i + 1}. ${firstText(messages[start])}`),- ];- // A conversation folded many times would otherwise grow a note that is itself- // the problem. The oldest go first: their edits are furthest downstream.- const shown = asked.length > MAX_FOLDED_ASKS- ? [`(${asked.length - MAX_FOLDED_ASKS} earlier requests omitted)`, ...asked.slice(-MAX_FOLDED_ASKS)]- : asked;- const note = 'Earlier in this conversation you were asked, in order:\n'- + `${shown.join('\n')}\n`- + 'The tool calls and results from those runs were dropped to keep this conversation '- + 'inside its context window. Whatever they changed is in the scene — read it back if '- + 'you need the current state rather than trusting this summary.';- const kept: Message[] = [- { role: 'user', content: note },+export const ANTHROPIC_SHAPE: HistoryShape<Message> = {+ askedIn: (message) => {+ const content = message?.content;+ const text = typeof content === 'string'+ ? content+ : content?.find((b) => b.type === 'text')?.text ?? '';+ return foldLine(text);+ },+ foldNote: (text) => [+ { role: 'user', content: text },{ role: 'assistant', content: 'Understood.' },- ...messages.slice(at),- ];- const shift = at - 2;- return {- messages: kept,- turnStarts: turnStarts.slice(cut).map((s) => s - shift),- // Coordinates count from the start of the CONVERSATION, not of what is left.- dropped: dropped + cut,- folded: asked,- };-}--/** How many past requests the note quotes before it elides the oldest. */-const MAX_FOLDED_ASKS = 20;--/**- * What a person's turn said, as one line for the compaction note.- *- * The FIRST text block only. A turn is pushed as a plain string and then gets- * the editor context appended as a second block (flushContext), which is the- * same paragraph every turn — joining them would quote the state of the editor- * once per folded run and bury the sentence that was actually said.- */-function firstText(message: Message | undefined): string {- const content = message?.content;- const text = typeof content === 'string'- ? content- : content?.find((b) => b.type === 'text')?.text ?? '';- const flat = text.replace(/\s+/g, ' ').trim();- if (!flat) return '(no text)';- return flat.length > 200 ? `${flat.slice(0, 197)}…` : flat;-}--/** What a conversation's memory looks like on disk. Versioned: a shape this- * does not recognise is refused rather than half-read. */-interface SessionMemory {- v: 1;- messages: Message[];- turnStarts: number[];- dropped: number;- folded: string[];- lastInputTokens: number;-}--const isSessionMemory = (value: unknown): value is SessionMemory => {- const m = value as Partial<SessionMemory> | null;- return !!m && m.v === 1 && Array.isArray(m.messages) && Array.isArray(m.turnStarts)- && typeof m.dropped === 'number' && Array.isArray(m.folded);+ ],+ weigh: (message) => (typeof message.content === 'string'+ ? message.content.length+ : JSON.stringify(message.content).length),};class AnthropicSession implements AgentSession {- private messages: Message[] = [];+ private readonly log: ConversationLog<Message>;/** Context waiting for a legal spot — see {@link flushContext}. */private readonly pending: string[] = [];- /** Where each person's turn starts. A tool result is a `user` message too, so- * counting roles would not find them. */- private turnStarts: number[] = [];- /** Runs folded away by {@link compactOldest}. Turn coordinates count from the- * start of the CONVERSATION, so they must survive their messages. */- private dropped = 0;- /** What was asked in every folded run, oldest first — see compactHistory. */- private folded: string[] = [];- /** What the last call was billed for its input. Authoritative WHERE it counts- * the whole request — see {@link contextUsed}, which does not assume it does. */- private lastInputTokens = 0;constructor(private readonly client: Anthropic,private readonly opts: { model: string; effort: string; dialect: Dialect; contextWindow: number },private readonly system: string,private readonly tools: readonly CatalogTool[],- ) {}+ ) {+ let fixedChars = system.length;+ for (const tool of tools) {+ fixedChars += tool.name.length + tool.description.length + JSON.stringify(tool.schema).length;+ }+ this.log = new ConversationLog(ANTHROPIC_SHAPE, { contextWindow: opts.contextWindow, fixedChars });+ }++ private get messages(): Message[] {+ return this.log.messages;+ }get turnIndex(): number {- return this.dropped + this.turnStarts.length;+ return this.log.turnIndex;}/**@@ -346,30 +206,18 @@ class AnthropicSession implements AgentSession {* anyway. Restoring it would replay a stale reading of the scene as if it were* current — the one thing the per-turn context exists to avoid.*/- serialize(): SessionMemory {- return {- v: 1,- messages: this.messages,- turnStarts: this.turnStarts,- dropped: this.dropped,- folded: this.folded,- lastInputTokens: this.lastInputTokens,- };+ serialize(): unknown {+ return this.log.serialize(ANTHROPIC_FORMAT);}/** Put a serialized memory back. Anything else leaves the session empty —* see AgentProvider.createSession on why that beats refusing. */restore(memory: unknown): void {- if (!isSessionMemory(memory)) return;- this.messages = memory.messages;- this.turnStarts = memory.turnStarts;- this.dropped = memory.dropped;- this.folded = memory.folded;- this.lastInputTokens = memory.lastInputTokens ?? 0;+ this.log.restore(memory, ANTHROPIC_FORMAT);}pushUser(text: string, images?: readonly UserImage[]): void {- this.turnStarts.push(this.messages.length);+ this.log.markTurnStart();if (!images?.length) {this.messages.push({ role: 'user', content: text });return;@@ -403,71 +251,12 @@ class AnthropicSession implements AgentSession {}rewindTo(n: number): void {- const at = this.turnStarts[n - this.dropped];- // Rewinding into runs that have already been folded away is not something- // this can do — their messages are gone. Refusing beats half-doing it.- if (at === undefined) return;- this.messages.length = at;- this.turnStarts.length = n - this.dropped;+ // Refuses a run already folded away — its messages are gone.+ if (!this.log.rewindTo(n)) return;// Context buffered for a turn that is no longer going to happen.this.pending.length = 0;}- /**- * How full the context is, in tokens — the larger of what the endpoint billed- * and what this conversation obviously weighs.- *- * Measured against a real gateway, `input_tokens` is NOT always the whole- * request: DeepSeek reported 33 for a first turn whose system prompt and 75- * tool schemas are thousands, so trusting it alone meant compaction would- * never fire and the conversation would hit the wall it exists to prevent.- * Nor can the estimate simply replace it — chars/4 is a rule of thumb that- * under-counts CJK badly, which is most of what this editor's users type.- *- * So: whichever is bigger. Both err by under-reporting, and the cost of- * over-reporting is one compaction that was not needed yet.- */- private contextUsed(): number {- let chars = this.system.length;- for (const tool of this.tools) {- chars += tool.name.length + tool.description.length + JSON.stringify(tool.schema).length;- }- for (const message of this.messages) {- chars += typeof message.content === 'string'- ? message.content.length- : JSON.stringify(message.content).length;- }- return Math.max(this.lastInputTokens, Math.ceil(chars / 4));- }-- /**- * Fold the oldest runs away if this conversation has outgrown its budget.- *- * @returns how many runs went, 0 when none did. The caller SAYS it — a- * conversation losing part of its memory with nothing on screen to- * show for it is the thing this number exists to end.- */- private compactIfNeeded(): number {- if (!shouldCompact(this.contextUsed(), this.opts.contextWindow)) return 0;- const next = compactHistory(- this.messages, this.turnStarts, this.dropped, KEEP_WHOLE_RUNS, this.folded,- );- if (!next) return 0;- const runs = next.dropped - this.dropped;- this.messages.splice(0, this.messages.length, ...next.messages);- this.turnStarts.splice(0, this.turnStarts.length, ...next.turnStarts);- this.dropped = next.dropped;- this.folded = next.folded;- // What the endpoint billed describes a history that no longer exists, and- // it is the LARGER half of contextUsed() on an honest endpoint. Left in- // place it would report the conversation as still full immediately after- // emptying it: no visible drop, and the next step would fold again for a- // reason that had already been dealt with. The estimate carries the reading- // until the next call reports a real one.- this.lastInputTokens = 0;- return runs;- }-pushContext(text: string): void {this.pending.push(text);}@@ -527,10 +316,10 @@ class AnthropicSession implements AgentSession {// conversation. The reading is stated here as well as after the answer// because this is the moment it moves DOWN, and a gauge that only ever// climbed would make the fold invisible in the one place it shows.- const folded = this.compactIfNeeded();+ const folded = this.log.compactIfNeeded();if (folded > 0) {yield { type: 'compacted', runs: folded };- yield { type: 'context', used: this.contextUsed(), window: this.opts.contextWindow };+ yield { type: 'context', used: this.log.contextUsed(), window: this.opts.contextWindow };}const request = buildStepRequest({@@ -631,7 +420,7 @@ class AnthropicSession implements AgentSession {this.messages.push({ role: 'assistant', content: message.content as Message['content'] });if (message.usage) {- this.lastInputTokens = message.usage.input_tokens ?? this.lastInputTokens;+ this.log.lastInputTokens = message.usage.input_tokens ?? this.log.lastInputTokens;// Whatever the stream did not already account for. Usually nothing — but// an endpoint that reports usage only at the end still gets it all here,// and one that over-reported mid-stream is not charged twice.@@ -642,7 +431,7 @@ class AnthropicSession implements AgentSession {// this one call cost. A level and a cost project differently (the editor// replaces one and sums the other), and an endpoint that reports no usage// at all still has a context this can answer for from the estimate.- yield { type: 'context', used: this.contextUsed(), window: this.opts.contextWindow };+ yield { type: 'context', used: this.log.contextUsed(), window: this.opts.contextWindow };// Again, now with the PARSED arguments — the streamed deltas are display// text, and the kernel needs a real object to dispatch. The editor treats adiff --git a/desktop/electron/agent/apiError.ts b/desktop/electron/agent/apiError.tsnew file mode 100644index 000000000..49c14b761--- /dev/null+++ b/desktop/electron/agent/apiError.ts@@ -0,0 +1,67 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file apiError.ts+ * @brief What went wrong, said to the person who has to decide what to do+ * about it.+ *+ * An SDK's own message is written for whoever is reading a stack trace — it+ * names a status code and a URL. In the transcript the only useful content is+ * which of four situations this is, because each has a different answer: wait,+ * fix the key, check the address, or nothing (it already retried).+ *+ * Shared by every provider because it is about HTTP, not about a wire format —+ * both SDKs raise the same statuses with the same meanings, and a second copy+ * would be a second set of sentences to keep in step with the settings page they+ * point at.+ */++/** An SDK hands back a `Headers` on some paths and a plain object on others;+ * reading only one shape is how the useful half of a 429 goes missing. */+function headerOf(error: unknown, name: string): string | undefined {+ const headers = (error as { headers?: unknown })?.headers;+ if (!headers) return undefined;+ const get = (headers as Headers).get;+ if (typeof get === 'function') return (headers as Headers).get(name) ?? undefined;+ return (headers as Record<string, string>)[name];+}++/**+ * Whether asking again could plausibly work: the failure was the CONNECTION or+ * the far side, not the request.+ *+ * A stream that dies halfway is the common one, and it is not rare over a long+ * turn — a dogfood run lost eighty-seven rounds of work to a socket the gateway+ * closed. It arrives with no HTTP status at all, because the request had already+ * succeeded and the body stopped arriving.+ */+export function isTransientApiError(error: unknown): boolean {+ const status = (error as { status?: number })?.status;+ if (status === undefined || status === null) return true;+ return status === 408 || status === 429 || status >= 500;+}++export function describeApiError(error: unknown): string {+ const status = (error as { status?: number })?.status;+ const raw = (error as Error)?.message ?? String(error);+ const retryAfter = Number(headerOf(error, 'retry-after'));++ if (status === 429) {+ // The SDK has already retried this with backoff by the time it reaches us,+ // so "try again" is not advice — how long to wait is.+ return Number.isFinite(retryAfter) && retryAfter > 0+ ? `Rate limited, and the automatic retries did not clear it. The endpoint asks for ${retryAfter}s.`+ : 'Rate limited, and the automatic retries did not clear it. Wait a moment and send again.';+ }+ if (status === 401 || status === 403) {+ return 'The endpoint rejected the API key. Check it in Settings › AI Agents.';+ }+ if (status === 404) {+ return 'The endpoint has no such model, or the address is wrong. Check both in Settings › AI Agents.';+ }+ if (status === 400) return `The endpoint refused the request: ${raw}`;+ if (status && status >= 500) return 'The endpoint is having trouble. Nothing is wrong on this side — try again shortly.';+ // Not an HTTP failure at all: DNS, offline, a gateway that is not running.+ if (!status) return `Could not reach the endpoint: ${raw}`;+ return raw;+}diff --git a/desktop/electron/agent/conversation.ts b/desktop/electron/agent/conversation.tsnew file mode 100644index 000000000..35b018b7f--- /dev/null+++ b/desktop/electron/agent/conversation.ts@@ -0,0 +1,276 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file conversation.ts+ * @brief The bookkeeping every provider needs and none of them should own:+ * where each run started, which ones have been folded away, and how+ * full the context is.+ *+ * None of it is about a wire format. Turn COORDINATES have to outlive the+ * messages they named (the editor says "re-ask run 7" long after run 7's+ * messages went); the fold has to carry every request it ever swallowed, not+ * just the last batch; and how full a context is has to be answered even by an+ * endpoint that reports nothing. Those are three pieces of hard-won arithmetic,+ * and a second copy of them is a second place for the bugs they already have+ * scars from.+ *+ * What IS about the wire format is parameterised: a {@link HistoryShape} says+ * how to read the person's line out of a message, what pair of messages a fold+ * leaves behind, and roughly what a message weighs. Three functions, and the+ * provider keeps the rest of its format to itself.+ */+import { COMPACT_AT, KEEP_WHOLE_RUNS, shouldCompact } from '../../src/settings/agentIds';++export { COMPACT_AT, KEEP_WHOLE_RUNS };++/** The three things this needs to know about a provider's message type. */+export interface HistoryShape<M> {+ /**+ * What a person's turn said, as one line for the fold note.+ *+ * The FIRST text of it only. A turn is pushed as a plain string and then gets+ * the editor context appended (flushContext), which is the same paragraph+ * every turn — joining them would quote the state of the editor once per+ * folded run and bury the sentence that was actually said.+ */+ askedIn(message: M | undefined): string;+ /** The user/assistant pair a fold leaves in place of what it dropped. */+ foldNote(text: string): readonly [M, M];+ /** Roughly what this message occupies, in characters. */+ weigh(message: M): number;+}++/**+ * A conversation's memory on disk. Versioned, and NAMED by wire format.+ *+ * The name is what stops a Claude conversation being resumed into an OpenAI+ * session. Both formats serialize to `{ v: 1, messages: [...] }` and a shape+ * check cannot tell them apart, so without it a provider would happily load a+ * history it has no way to send — which fails as a 400 several seconds later,+ * with nothing on screen connecting it to the model switch that caused it.+ * Absent means `anthropic`: conversations saved before this field existed were+ * all that one.+ */+export interface LogMemory<M> {+ v: 1;+ format?: string;+ messages: M[];+ turnStarts: number[];+ dropped: number;+ folded: string[];+ lastInputTokens: number;+}++/** How many past requests a fold note quotes before it elides the oldest. */+const MAX_FOLDED_ASKS = 20;++/**+ * Fold the oldest runs into one note, keeping what the PERSON said.+ *+ * A conversation grows mostly by tool traffic — a scene tree, a diagnostics+ * list, the reasoning about them — and almost none of that is worth carrying+ * once the edits have landed. The intent is: "make it feel like dusk" governs+ * the ninth run as much as the first. So the person's own words survive verbatim+ * and everything around them becomes a line.+ *+ * Structural rather than model-written: asking a model to summarise costs a call+ * and a wait at exactly the moment the conversation is already long, and the+ * facts worth keeping (what was asked) are ones already held exactly.+ *+ * Pure and exported because it rewrites the history everything else depends on,+ * and the bookkeeping — turn COORDINATES outliving the messages they named — is+ * the part worth pinning by test rather than by whichever long conversation+ * happens to hit it first.+ *+ * @returns the rewritten history, or null when there is not enough to fold.+ */+export function compactHistory<M>(+ shape: HistoryShape<M>,+ messages: readonly M[],+ turnStarts: readonly number[],+ dropped: number,+ keepRuns: number,+ folded: readonly string[] = [],+): { messages: M[]; turnStarts: number[]; dropped: number; folded: string[] } | null {+ const cut = turnStarts.length - keepRuns;+ if (cut <= 0) return null;+ const at = turnStarts[cut];+ // Everything ever folded, not only this pass. The note a fold produces is NOT+ // in turnStarts, so the next one splices straight over it — carrying the asks+ // in a value of their own is what stops the oldest request disappearing one+ // compaction at a time. (Caught against a real gateway: the first fold kept+ // "remember the passphrase", the second silently ate it.)+ const asked = [+ ...folded,+ ...turnStarts.slice(0, cut).map((start, i) => `${dropped + i + 1}. ${shape.askedIn(messages[start])}`),+ ];+ // A conversation folded many times would otherwise grow a note that is itself+ // the problem. The oldest go first: their edits are furthest downstream.+ const shown = asked.length > MAX_FOLDED_ASKS+ ? [`(${asked.length - MAX_FOLDED_ASKS} earlier requests omitted)`, ...asked.slice(-MAX_FOLDED_ASKS)]+ : asked;+ const note = 'Earlier in this conversation you were asked, in order:\n'+ + `${shown.join('\n')}\n`+ + 'The tool calls and results from those runs were dropped to keep this conversation '+ + 'inside its context window. Whatever they changed is in the scene — read it back if '+ + 'you need the current state rather than trusting this summary.';+ const kept: M[] = [...shape.foldNote(note), ...messages.slice(at)];+ const shift = at - 2;+ return {+ messages: kept,+ turnStarts: turnStarts.slice(cut).map((s) => s - shift),+ // Coordinates count from the start of the CONVERSATION, not of what is left.+ dropped: dropped + cut,+ folded: asked,+ };+}++/** One line of a fold note, from raw text — the shared half of `askedIn`. */+export function foldLine(text: string): string {+ const flat = text.replace(/\s+/g, ' ').trim();+ if (!flat) return '(no text)';+ return flat.length > 200 ? `${flat.slice(0, 197)}…` : flat;+}++/**+ * One conversation's history, and everything true of it that is not its shape.+ *+ * The provider owns `messages` — it appends in its own format and reads them+ * back to build a request. This owns where the runs are, what has been folded,+ * and what the endpoint last billed.+ */+export class ConversationLog<M> {+ /** The provider's own history, in the provider's own format. */+ readonly messages: M[] = [];+ /** Where each person's turn starts. A tool result is a message too, so+ * counting roles would not find them. */+ private turnStarts: number[] = [];+ /** Runs folded away. Turn coordinates count from the start of the+ * CONVERSATION, so they must survive their messages. */+ private dropped = 0;+ /** What was asked in every folded run, oldest first — see compactHistory. */+ private folded: string[] = [];+ /**+ * What the last call was billed for its input. Authoritative WHERE it counts+ * the whole request — see {@link contextUsed}, which does not assume it does.+ */+ lastInputTokens = 0;++ constructor(+ private readonly shape: HistoryShape<M>,+ private readonly opts: {+ contextWindow: number;+ /** Characters the request carries regardless of history: system + tools. */+ fixedChars: number;+ },+ ) {}++ get turnIndex(): number {+ return this.dropped + this.turnStarts.length;+ }++ /** The next message appended opens a person's turn. */+ markTurnStart(): void {+ this.turnStarts.push(this.messages.length);+ }++ /**+ * Drop everything from the `n`-th person's turn onward.+ *+ * @returns false when that run has already been folded away — its messages+ * are gone, and refusing beats half-doing it.+ */+ rewindTo(n: number): boolean {+ const at = this.turnStarts[n - this.dropped];+ if (at === undefined) return false;+ this.messages.length = at;+ this.turnStarts.length = n - this.dropped;+ return true;+ }++ /**+ * How full the context is, in tokens — the larger of what the endpoint billed+ * and what this conversation obviously weighs.+ *+ * Measured against a real gateway, the billed input is NOT always the whole+ * request: DeepSeek reported 33 for a first turn whose system prompt and 75+ * tool schemas are thousands, so trusting it alone meant compaction would+ * never fire and the conversation would hit the wall it exists to prevent.+ * Nor can the estimate simply replace it — chars/4 is a rule of thumb that+ * under-counts CJK badly, which is most of what this editor's users type.+ *+ * So: whichever is bigger. Both err by under-reporting, and the cost of+ * over-reporting is one compaction that was not needed yet.+ */+ contextUsed(): number {+ let chars = this.opts.fixedChars;+ for (const message of this.messages) chars += this.shape.weigh(message);+ return Math.max(this.lastInputTokens, Math.ceil(chars / 4));+ }++ /**+ * Fold the oldest runs away if this conversation has outgrown its budget.+ *+ * @returns how many runs went, 0 when none did. The caller SAYS it — a+ * conversation losing part of its memory with nothing on screen to+ * show for it is the thing this number exists to end.+ */+ compactIfNeeded(): number {+ if (!shouldCompact(this.contextUsed(), this.opts.contextWindow)) return 0;+ const next = compactHistory(+ this.shape, this.messages, this.turnStarts, this.dropped, KEEP_WHOLE_RUNS, this.folded,+ );+ if (!next) return 0;+ const runs = next.dropped - this.dropped;+ this.messages.splice(0, this.messages.length, ...next.messages);+ this.turnStarts = next.turnStarts;+ this.dropped = next.dropped;+ this.folded = next.folded;+ // What the endpoint billed describes a history that no longer exists, and+ // it is the LARGER half of contextUsed() on an honest endpoint. Left in+ // place it would report the conversation as still full immediately after+ // emptying it: no visible drop, and the next step would fold again for a+ // reason that had already been dealt with. The estimate carries the reading+ // until the next call reports a real one.+ this.lastInputTokens = 0;+ return runs;+ }++ get contextWindow(): number {+ return this.opts.contextWindow;+ }++ serialize(format: string): LogMemory<M> {+ return {+ v: 1,+ format,+ messages: this.messages,+ turnStarts: this.turnStarts,+ dropped: this.dropped,+ folded: this.folded,+ lastInputTokens: this.lastInputTokens,+ };+ }++ /**+ * Put a serialized memory back, if it is one of ours.+ *+ * @returns false for anything else — a shape this does not recognise, or a+ * history in another provider's wire format. The caller starts fresh:+ * a conversation you cannot continue is worth less than a new one,+ * and far less than a crash several seconds into the next turn.+ */+ restore(memory: unknown, format: string): boolean {+ const m = memory as Partial<LogMemory<M>> | null;+ if (!m || m.v !== 1 || !Array.isArray(m.messages) || !Array.isArray(m.turnStarts)) return false;+ if (typeof m.dropped !== 'number' || !Array.isArray(m.folded)) return false;+ // Absent means the format that existed before this field did.+ if ((m.format ?? 'anthropic') !== format) return false;+ this.messages.splice(0, this.messages.length, ...m.messages);+ this.turnStarts = m.turnStarts;+ this.dropped = m.dropped;+ this.folded = m.folded;+ this.lastInputTokens = m.lastInputTokens ?? 0;+ return true;+ }+}diff --git a/desktop/electron/agent/openai.ts b/desktop/electron/agent/openai.tsnew file mode 100644index 000000000..c6d4e8e3d--- /dev/null+++ b/desktop/electron/agent/openai.ts@@ -0,0 +1,502 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file openai.ts+ * @brief The Chat Completions AgentProvider — OpenAI's protocol, and every+ * endpoint that implements it.+ *+ * A sibling of anthropic.ts rather than a third dialect of it. Those two are+ * dialects of ONE format; this is a different one, and the differences are not+ * cosmetic: tool calls arrive as a field on the assistant message with their+ * arguments as a STRING, each result goes back as its own `role: "tool"`+ * message, and the stream carries untyped deltas that have to be stitched by+ * index. Folded into the same file, every function would be a two-way branch.+ *+ * What it does share is the bookkeeping — folding old runs away, reading how+ * full the context is, saying what an HTTP failure means — because none of that+ * is about a wire format. See conversation.ts and apiError.ts.+ *+ * Three things about this protocol that the Anthropic one does not have, and+ * that each cost a turn when missed:+ *+ * · An assistant message carrying `tool_calls` MUST be followed by a `tool`+ * message for every one of them, or the next request is refused outright.+ * The kernel abandons a round mid-dispatch when the user hits Stop, so this+ * reconciles before it sends — see {@link answerOrphanedCalls}.+ * · Reasoning comes back (`reasoning_content`, or `reasoning` on some+ * gateways) and must NOT go back: the endpoints that send it reject it as+ * input. The exact opposite of Anthropic's thinking blocks.+ * · Usage is not streamed unless it is asked for, and asking is a request+ * field a strict gateway may not know.+ */+import OpenAI from 'openai';+import type {+ AgentProvider, AgentSession, CatalogTool, StepEvent, ToolOutcome, UserImage,+} from './types';+import { ConversationLog, foldLine, type HistoryShape } from './conversation';+import { describeApiError, isTransientApiError } from './apiError';+import { DEFAULT_CONTEXT_WINDOW, DEFAULT_EFFORT, type AgentEffort } from '../../src/settings/agentIds';++/** Names this history's wire format on disk — see conversation.ts LogMemory. */+export const OPENAI_FORMAT = 'openai';++type Message = OpenAI.Chat.Completions.ChatCompletionMessageParam;+type Tool = OpenAI.Chat.Completions.ChatCompletionTool;++/** Streaming, so the ceiling can be generous — the same reasoning as the+ * Anthropic provider's: a truncated turn wastes the work before it. */+const MAX_TOKENS = 32000;++export interface OpenAIOptions {+ apiKey: string;+ model: string;+ /** Where Chat Completions lives. Omitted = OpenAI's own API. */+ baseURL?: string;+ effort?: AgentEffort;+ contextWindow?: number;+ /**+ * Whether this endpoint's model can see. Declared rather than probed: finding+ * out costs a call spent on a screenshot the model will be told it cannot+ * look at, and the answer is a property of the endpoint, not of the editor.+ */+ vision?: boolean;+ /**+ * Send `reasoning_effort`. Off for an endpoint that would refuse an argument+ * it has never heard of — a rejected request is worse than an ignored field,+ * and which one you get is not knowable from here.+ */+ reasoningEffort?: boolean;+}++/**+ * How hard to think, in this protocol's vocabulary.+ *+ * The editor's five depths do not map one-to-one onto the four this field+ * takes, and the two ends are what matter: `low` must stay cheap, and the+ * agentic default must land on the deepest thing the endpoint offers.+ */+const EFFORT: Record<AgentEffort, string> = {+ low: 'low',+ medium: 'medium',+ high: 'high',+ xhigh: 'high',+ max: 'high',+};++/**+ * The body for one step.+ *+ * Pure and exported for the reason the Anthropic one is: it is where a field+ * that an endpoint silently ignores costs a behaviour the code claims to have.+ */+export function buildChatRequest(opts: {+ model: string;+ effort: AgentEffort;+ reasoningEffort: boolean;+ system: string;+ tools: readonly CatalogTool[];+ messages: readonly Message[];+}): OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming {+ const request = {+ model: opts.model,+ // Not a top-level parameter here — the system prompt is the first message,+ // and it is prepended per request rather than kept in the history so that+ // the history's indices mean the same thing they do in the other provider+ // (the fold in conversation.ts counts messages, and a system message at+ // index 0 would shift every turn start by one).+ messages: [{ role: 'system', content: opts.system }, ...opts.messages] as Message[],+ // The whole point of the catalog being stable and ordered: tools render+ // first, so an unchanging list is what makes prompt caching work at all —+ // where there is caching.+ tools: opts.tools.map((t): Tool => ({+ type: 'function',+ function: { name: t.name, description: t.description, parameters: t.schema },+ })),+ max_completion_tokens: MAX_TOKENS,+ stream: true,+ // Usage is not streamed unless asked for. Without it the context meter runs+ // entirely on the character estimate, which under-counts CJK badly — and+ // CJK is most of what this editor's users type.+ stream_options: { include_usage: true },+ } as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;+ if (opts.reasoningEffort) {+ (request as { reasoning_effort?: string }).reasoning_effort = EFFORT[opts.effort];+ }+ return request;+}++/**+ * What a tool result looks like on the wire: one message per call.+ *+ * Anthropic takes every result of a round as blocks of ONE user message; here+ * they are separate messages, and every id the assistant asked about must get+ * one back.+ */+function toolMessage(outcome: ToolOutcome, vision: boolean): Message {+ const content = outcome.image && !vision+ ? `${outcome.content}\n[the screenshot could not be sent: this endpoint does not accept images]`+ : outcome.content;+ // The image itself cannot ride on a `tool` message in this protocol — it is+ // text-only — so a followed-up user message carries it. See pushToolResults.+ return { role: 'tool', tool_call_id: outcome.id, content: content || '(no output)' };+}++/** A person's attachment, or a tool's rendered frame, as a content part. */+const imagePart = (im: { data: string; mediaType: string }): OpenAI.Chat.Completions.ChatCompletionContentPart => ({+ type: 'image_url',+ image_url: { url: `data:${im.mediaType};base64,${im.data}` },+});++/**+ * Give every unanswered `tool_calls` id a result, so the history is one this+ * protocol will accept.+ *+ * The kernel abandons a round mid-dispatch when the run is aborted — it breaks+ * out of the loop and never pushes the results — which leaves an assistant+ * message asking for tools that nothing answers. Anthropic tolerates that;+ * Chat Completions refuses the whole request, so the NEXT message after a Stop+ * would fail and keep failing, with nothing on screen linking it to the Stop.+ *+ * Exported because it is the one piece of this file that only ever runs after+ * something went wrong, which is the worst way to find out it is broken.+ */+export function answerOrphanedCalls(messages: Message[]): number {+ let added = 0;+ for (let i = messages.length - 1; i >= 0; i--) {+ const message = messages[i];+ if (message.role !== 'assistant') continue;+ const calls = message.tool_calls;+ if (!calls?.length) break; // the last assistant turn asked for nothing+ const answered = new Set<string>();+ // The results already back sit in a run directly after the call, and the+ // made-up ones join the END of it: inserted before, they would put a+ // result ahead of one that actually happened.+ let at = i + 1;+ for (let j = i + 1; j < messages.length; j++) {+ const later = messages[j];+ if (later.role !== 'tool') break;+ answered.add(later.tool_call_id);+ at = j + 1;+ }+ const missing = calls.filter((c) => !answered.has(c.id));+ if (missing.length === 0) break;+ messages.splice(at, 0, ...missing.map((c): Message => ({+ role: 'tool',+ tool_call_id: c.id,+ content: '(the run was interrupted before this tool ran)',+ })));+ added = missing.length;+ break;+ }+ return added;+}++const SHAPE: HistoryShape<Message> = {+ askedIn: (message) => {+ const content = message?.content;+ if (typeof content === 'string') return foldLine(content);+ const text = Array.isArray(content)+ ? content.find((p) => p.type === 'text')?.text ?? ''+ : '';+ return foldLine(text);+ },+ foldNote: (text) => [+ { role: 'user', content: text },+ { role: 'assistant', content: 'Understood.' },+ ],+ weigh: (message) => (typeof message.content === 'string'+ ? message.content.length+ : JSON.stringify(message.content ?? '').length),+};++class OpenAISession implements AgentSession {+ private readonly log: ConversationLog<Message>;+ /** Context waiting for a legal spot — see {@link flushContext}. */+ private readonly pending: string[] = [];++ constructor(+ private readonly client: OpenAI,+ private readonly opts: {+ model: string;+ effort: AgentEffort;+ reasoningEffort: boolean;+ vision: boolean;+ contextWindow: number;+ },+ private readonly system: string,+ private readonly tools: readonly CatalogTool[],+ ) {+ let fixedChars = system.length;+ for (const tool of tools) {+ fixedChars += tool.name.length + tool.description.length + JSON.stringify(tool.schema).length;+ }+ this.log = new ConversationLog(SHAPE, { contextWindow: opts.contextWindow, fixedChars });+ }++ private get messages(): Message[] {+ return this.log.messages;+ }++ get turnIndex(): number {+ return this.log.turnIndex;+ }++ serialize(): unknown {+ return this.log.serialize(OPENAI_FORMAT);+ }++ restore(memory: unknown): void {+ this.log.restore(memory, OPENAI_FORMAT);+ }++ pushUser(text: string, images?: readonly UserImage[]): void {+ this.log.markTurnStart();+ if (!images?.length) {+ this.messages.push({ role: 'user', content: text });+ return;+ }+ if (!this.opts.vision) {+ // Told, in the turn itself, rather than quietly handed the text alone: a+ // model that never learns a picture was attached answers confidently+ // about something it was not shown.+ const note = images.length === 1+ ? '[the user attached an image; this endpoint cannot receive images, so you have not seen it]'+ : `[the user attached ${images.length} images; this endpoint cannot receive images, so you have not seen them]`;+ this.messages.push({ role: 'user', content: `${text}\n\n${note}` });+ return;+ }+ // Images first: the question after them reads as being about them.+ this.messages.push({+ role: 'user',+ content: [...images.map(imagePart), ...(text ? [{ type: 'text' as const, text }] : [])],+ });+ }++ rewindTo(n: number): void {+ if (!this.log.rewindTo(n)) return;+ this.pending.length = 0;+ }++ pushContext(text: string): void {+ this.pending.push(text);+ }++ pushToolResults(outcomes: readonly ToolOutcome[]): void {+ for (const outcome of outcomes) this.messages.push(toolMessage(outcome, this.opts.vision));+ // A rendered frame cannot ride on a `tool` message — this protocol's are+ // text-only — so the frames of the round follow as one user message. Kept+ // to the round rather than attached to each result for the same reason the+ // Anthropic provider batches: splitting a round across messages teaches the+ // model to stop asking for tools in parallel.+ if (!this.opts.vision) return;+ const frames = outcomes.filter((o) => o.image);+ if (frames.length === 0) return;+ this.messages.push({+ role: 'user',+ content: [+ { type: 'text', text: frames.length === 1 ? 'The frame that tool captured:' : 'The frames those tools captured:' },+ ...frames.map((o) => imagePart(o.image!)),+ ],+ });+ }++ /**+ * Land buffered context after the history rather than by rewriting the system+ * prompt. The prompt is the first message of every request, so editing it per+ * turn would re-bill the whole conversation wherever caching is on.+ *+ * A `system` message mid-conversation is legal here — unlike the Anthropic+ * provider's compatible dialect, which has only two roles to work with and has+ * to smuggle the text onto the user turn.+ */+ private flushContext(): void {+ if (this.pending.length === 0) return;+ this.messages.push({ role: 'system', content: this.pending.join('\n\n') });+ this.pending.length = 0;+ }++ async *step(signal: AbortSignal): AsyncIterable<StepEvent> {+ // Before anything else: a history this protocol would refuse is one no+ // amount of retrying fixes.+ answerOrphanedCalls(this.messages);+ this.flushContext();++ const folded = this.log.compactIfNeeded();+ if (folded > 0) {+ yield { type: 'compacted', runs: folded };+ yield { type: 'context', used: this.log.contextUsed(), window: this.opts.contextWindow };+ }++ const request = buildChatRequest({+ model: this.opts.model,+ effort: this.opts.effort,+ reasoningEffort: this.opts.reasoningEffort,+ system: this.system,+ tools: this.tools,+ messages: this.messages,+ });++ try {+ yield* this.consume(await this.client.chat.completions.create(request, { signal }));+ } catch (e) {+ // Rethrown, not swallowed — the kernel still has to end the turn. Safe to+ // mark retryable because the history is appended only AFTER the stream+ // completes: a call that died left this session as it found it.+ const err = new Error(describeApiError(e)) as Error & { retryable?: boolean };+ err.retryable = isTransientApiError(e);+ throw err;+ }+ }++ private async *consume(+ stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>,+ ): AsyncIterable<StepEvent> {+ let text = '';+ // Tool calls arrive in pieces addressed by INDEX, and only the first piece+ // carries the id and the name. Arguments accumulate as a string.+ const calls = new Map<number, { id: string; name: string; args: string }>();+ let finish: string | null = null;+ let inputTokens = 0;+ let outputTokens = 0;++ for await (const chunk of stream) {+ if (chunk.usage) {+ inputTokens = chunk.usage.prompt_tokens ?? inputTokens;+ outputTokens = chunk.usage.completion_tokens ?? outputTokens;+ }+ const choice = chunk.choices?.[0];+ if (!choice) continue;+ if (choice.finish_reason) finish = choice.finish_reason;+ const delta = choice.delta as (typeof choice.delta & { reasoning_content?: string; reasoning?: string }) | undefined;+ if (!delta) continue;++ // No standard field for it: DeepSeek and Qwen send `reasoning_content`,+ // OpenRouter sends `reasoning`. Shown, never sent back — the endpoints+ // that produce it refuse it as input.+ const reasoning = delta.reasoning_content ?? delta.reasoning;+ if (reasoning) yield { type: 'thinking', delta: reasoning };++ if (typeof delta.content === 'string' && delta.content) {+ text += delta.content;+ yield { type: 'text', delta: delta.content };+ }++ for (const piece of delta.tool_calls ?? []) {+ const at = piece.index ?? 0;+ let call = calls.get(at);+ if (!call) {+ call = { id: piece.id ?? `call_${at}`, name: piece.function?.name ?? '', args: '' };+ calls.set(at, call);+ // An announcement, not a call: the kernel executes `tool_call`, and+ // sharing the type here would run every tool twice.+ if (call.name) yield { type: 'tool_pending', id: call.id, name: call.name };+ } else if (piece.id && call.id !== piece.id) {+ call.id = piece.id;+ }+ // A gateway may send the name in a later piece than the id.+ if (!call.name && piece.function?.name) {+ call.name = piece.function.name;+ yield { type: 'tool_pending', id: call.id, name: call.name };+ }+ const args = piece.function?.arguments;+ if (args) {+ call.args += args;+ yield { type: 'tool_args', id: call.id, delta: args };+ }+ }+ }++ const ordered = [...calls.entries()].sort((a, b) => a[0] - b[0]).map(([, c]) => c);++ // Verbatim enough: this protocol has no reasoning to echo back, and the+ // assistant message is exactly what it will accept next time.+ this.messages.push({+ role: 'assistant',+ content: text || null,+ ...(ordered.length+ ? {+ tool_calls: ordered.map((c) => ({+ id: c.id,+ type: 'function' as const,+ function: { name: c.name, arguments: c.args || '{}' },+ })),+ }+ : {}),+ } as Message);++ if (inputTokens || outputTokens) {+ this.log.lastInputTokens = inputTokens || this.log.lastInputTokens;+ yield { type: 'usage', inputTokens, outputTokens };+ }+ yield { type: 'context', used: this.log.contextUsed(), window: this.opts.contextWindow };++ for (const call of ordered) {+ yield { type: 'tool_call', call: { id: call.id, name: call.name, input: parseArgs(call.args) } };+ }++ yield { type: 'stop', reason: stopReason(finish, ordered.length > 0) };+ }+}++/**+ * The arguments, as an object.+ *+ * They arrive as a STRING the model wrote, so a truncated turn produces one that+ * does not parse. An empty object beats throwing: the kernel dispatches the call,+ * the tool reports what it needed, and the model gets a result it can act on —+ * where a throw would end the run with the reason buried in a stack trace.+ */+function parseArgs(raw: string): Record<string, unknown> {+ if (!raw.trim()) return {};+ try {+ const parsed = JSON.parse(raw) as unknown;+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed)+ ? (parsed as Record<string, unknown>)+ : {};+ } catch {+ return {};+ }+}++/**+ * Why the model stopped.+ *+ * `tool_calls` is the honest signal, but not every gateway sets it — several+ * report `stop` on a response that is nothing but tool calls. Having asked for+ * one is the fact that matters to the kernel, so it wins.+ */+function stopReason(raw: string | null, hasCalls: boolean): Extract<StepEvent, { type: 'stop' }>['reason'] {+ if (hasCalls) return 'tool_use';+ switch (raw) {+ case 'length': return 'max_tokens';+ case 'content_filter': return 'refusal';+ default: return 'end_turn';+ }+}++export function createOpenAIProvider(options: OpenAIOptions): AgentProvider {+ const baseURL = options.baseURL || undefined;+ const vision = options.vision ?? !baseURL;+ const client = new OpenAI({ apiKey: options.apiKey, baseURL });+ return {+ id: baseURL ? `openai:${baseURL}` : 'openai',+ model: options.model,+ acceptsImages: vision,+ createSession: ({ system, tools }, memory) => {+ const session = new OpenAISession(+ client,+ {+ model: options.model,+ effort: options.effort ?? DEFAULT_EFFORT,+ reasoningEffort: options.reasoningEffort ?? true,+ vision,+ contextWindow: options.contextWindow || DEFAULT_CONTEXT_WINDOW,+ },+ system,+ tools,+ );+ if (memory !== undefined) session.restore(memory);+ return session;+ },+ };+}diff --git a/desktop/electron/main.ts b/desktop/electron/main.tsindex cf5876970..d02936227 100644--- a/desktop/electron/main.ts+++ b/desktop/electron/main.ts@@ -54,6 +54,7 @@ import {} from './agent/store';import type { ConfirmAnswer, UserImage } from './agent/types';import { createAnthropicProvider } from './agent/anthropic';+import { createOpenAIProvider } from './agent/openai';import {discoverPlugins, compilePlugin, isTrusted, trustPlugin, revokeTrust, isDisabled, setPluginEnabled,@@ -689,7 +690,7 @@ ipcMain.handle('secret:clear', (_e, id: string) => clearSecret(id));// settings live in its localStorage) and merged, since the two rows fire// separately; read at session creation, so a change lands on the next// conversation rather than under the one being read.-let agentEndpoint: { baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string } = {};+let agentEndpoint: { protocol?: string; baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string } = {};/** Contributed agent tools, as the window last reported them. */let agentPluginTools: ContributedTool[] = [];@@ -704,11 +705,26 @@ const agentHost = createAgentHost({// Phrased for the person who will read it in the transcript, and it names// where to fix it — a bare "401" or "missing apiKey" does neither.if (!apiKey) throw new Error('No API key is configured. Add one in Settings › AI Agents.');+ const effort = agentEndpoint.effort as Parameters<typeof createAnthropicProvider>[0]['effort'];+ // Which wire format, decided by the provider the window picked. The two are+ // different protocols rather than dialects of one — see agent/openai.ts.+ if (agentEndpoint.protocol === 'openai') {+ if (!agentEndpoint.model) {+ throw new Error('No model is selected. Pick one next to the composer, or add it in Settings › AI Agents.');+ }+ return createOpenAIProvider({+ apiKey,+ baseURL: agentEndpoint.baseUrl,+ model: agentEndpoint.model,+ effort,+ contextWindow: agentEndpoint.contextWindow,+ });+ }return createAnthropicProvider({apiKey,baseURL: agentEndpoint.baseUrl,model: agentEndpoint.model,- effort: agentEndpoint.effort as Parameters<typeof createAnthropicProvider>[0]['effort'],+ effort,contextWindow: agentEndpoint.contextWindow,});},@@ -745,7 +761,7 @@ ipcMain.handle('agent:resumeConversation', async (_e, id: string) => {ipcMain.handle('agent:deleteConversation', async (_e, id: string) => {if (projectRoot) await deleteConversation(projectRoot, id);});-ipcMain.handle('agent:setEndpoint', (_e, patch: { baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string }) => {+ipcMain.handle('agent:setEndpoint', (_e, patch: { protocol?: string; baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string }) => {agentEndpoint = { ...agentEndpoint, ...patch };// `ready` is derived from which key this points at, so it just changed.agentHost.announce();diff --git a/desktop/electron/preload.ts b/desktop/electron/preload.tsindex 56ebda35e..568a62963 100644--- a/desktop/electron/preload.ts+++ b/desktop/electron/preload.ts@@ -139,7 +139,7 @@ const api = {ipcRenderer.invoke('agent:deleteConversation', id),/** Point the NEXT session at an endpoint/model. Merged, so each settings row* can push its own field. */- setEndpoint: (patch: { baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string }): Promise<void> =>+ setEndpoint: (patch: { protocol?: string; baseUrl?: string; model?: string; keyId?: string; contextWindow?: number; effort?: string }): Promise<void> =>ipcRenderer.invoke('agent:setEndpoint', patch),/** Report what loaded plugins contributed for the agent to call. Metadata* only — the handlers stay in this window, reached through the editordiff --git a/desktop/package.json b/desktop/package.jsonindex ba9f52847..d3f41611c 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -77,6 +77,7 @@"esengine": "workspace:*","ffmpeg-static": "^5.3.0","lucide-react": "^1.21.0",+ "openai": "^7.4.0","pngjs": "^7.0.0","react": "^19.2.0","react-dom": "^19.2.0",diff --git a/desktop/src/agent/providers.ts b/desktop/src/agent/providers.tsindex 6939efa04..b31f515b4 100644--- a/desktop/src/agent/providers.ts+++ b/desktop/src/agent/providers.ts@@ -19,13 +19,27 @@*/import { ContributionRegistry, type Disposable, type Owner } from '@/contrib/ContributionRegistry';+/**+ * Which wire format an endpoint speaks.+ *+ * Not a vendor and not a dialect: `anthropic` covers the Messages API and every+ * gateway implementing it, `openai` covers Chat Completions and the much larger+ * set that implements THAT. A provider that got this wrong would send a+ * perfectly formed request in the wrong shape, which reads as a 400 about a+ * field nobody wrote.+ */+export type AgentProtocol = 'anthropic' | 'openai';+export interface AgentProviderDef {id: string;label: string;+ /** The wire format. Absent means `anthropic` — what every def was before+ * there was a second one. */+ protocol?: AgentProtocol;/**- * Empty means the Anthropic API itself. Anything else is an- * Anthropic-COMPATIBLE gateway, and its presence is what drops the request to- * the core wire format (electron/agent/anthropic.ts).+ * Empty means the vendor's own API for that protocol. Anything else is a+ * compatible gateway; for `anthropic` its presence is also what drops the+ * request to the core wire format (electron/agent/anthropic.ts).*/baseUrl: string;/** What this provider offers, best-first. */@@ -42,7 +56,18 @@ export interface AgentProviderDef {* high is what makes a turn fail outright.*/contextWindow?: number;- /** Filled in from settings rather than shipped — see {@link CUSTOM_PROVIDER}. */+ /**+ * The model list is TYPED, not shipped.+ *+ * A vendor's address and protocol hold still; its model names do not. A list+ * here would be right until the next release and then quietly wrong — and a+ * wrong name is not an error, it is a gateway mapping it to something smaller+ * for the rest of the session. Everything else about the provider is still+ * shipped, including its key row.+ */+ typedModels?: boolean;+ /** Endpoint, protocol AND models all come from settings — see+ * {@link CUSTOM_PROVIDER}, the one provider we cannot ship anything for. */userDefined?: boolean;}@@ -53,6 +78,10 @@ export const agentKeyId = (providerId: string): string => `agents.key.${providerexport const CUSTOM_PROVIDER = 'custom';+/** A provider whose models the user types. Its list is per provider, so two of+ * them do not share one box. */+export const modelsSettingId = (providerId: string): string => `agents.models.${providerId}`;+const registry = new ContributionRegistry<AgentProviderDef>('agent provider');registry.registerAll('core', [@@ -70,8 +99,18 @@ registry.registerAll('core', [models: ['deepseek-v4-pro', 'deepseek-v4-flash'],contextWindow: 1_000_000,},- // The escape hatch. Its endpoint and model list come from settings, because a- // provider we have not heard of is exactly the one we cannot ship a list for.+ {+ id: 'openai',+ label: 'OpenAI',+ protocol: 'openai',+ baseUrl: '',+ models: [],+ typedModels: true,+ contextWindow: 400_000,+ },+ // The escape hatch. Its endpoint, protocol and model list come from settings,+ // because a provider we have not heard of is exactly the one we cannot ship+ // a list for — and Chat Completions is what most of those speak.{id: CUSTOM_PROVIDER,label: '',@@ -92,6 +131,18 @@ export const providersRevision = (): number => registry.getRevision();export const registerAgentProvider = (def: AgentProviderDef, owner: Owner = 'core'): Disposable =>registry.register(owner, def);+export const AGENT_PROTOCOLS: readonly AgentProtocol[] = ['openai', 'anthropic'];++/** What an endpoint speaks, defaulting to the format that predates the field. */+export const protocolOf = (def: AgentProviderDef | undefined): AgentProtocol =>+ def?.protocol ?? 'anthropic';++/** A stored value, narrowed — a settings file is one a person can edit, and an+ * unknown protocol must not reach the wire. Chat Completions is the default+ * for a custom endpoint because it is what most of them speak. */+export const asProtocol = (value: unknown): AgentProtocol =>+ (AGENT_PROTOCOLS as readonly string[]).includes(String(value)) ? (value as AgentProtocol) : 'openai';+/** Split the custom provider's model list — one per line or comma-separated,* because a person pasting three model names should not have to guess which. */export const parseModelList = (raw: string): string[] =>diff --git a/desktop/src/i18n/messages/settings.ts b/desktop/src/i18n/messages/settings.tsindex 4a0231efd..b39cfb24a 100644--- a/desktop/src/i18n/messages/settings.ts+++ b/desktop/src/i18n/messages/settings.ts@@ -125,10 +125,22 @@ export const settingsMessages = defineMessages({en: 'How hard the model is asked to think before it acts. `xhigh` is what agentic work is for and is the default; drop it when a turn costs more or takes longer than the task deserved. Applies to the next conversation.',zh: '模型动手之前被要求思考到什么程度。`xhigh` 是为 agent 类工作准备的,也是默认值;当一次运行的花费或耗时超过这件事本身的价值时,把它调低。对下一次对话生效。',},+ 'set.agents.providerModels': { en: '{provider} models', zh: '{provider} 型号' },+ 'set.agents.providerModels.desc': {+ en: 'Which models to offer in the composer, comma- or newline-separated. Typed rather than shipped: a list here would be right until the vendor\'s next release, and a name that no longer exists is not refused — the endpoint serves something smaller instead, for the rest of the session.',+ zh: '在输入框旁边可选的型号,用逗号或换行分隔。要自己填而不是内置:内置的列表只能正确到厂商下次发布为止,而写了个已经不存在的名字并不会被拒绝——接口会改用一个更小的模型顶上,并且一整个会话都是如此。',+ },+ 'set.agents.customProtocol': { en: 'Protocol', zh: '协议' },+ 'set.agents.customProtocol.desc': {+ en: 'Which API this endpoint speaks. The two formats differ in every message they send, so a wrong pick is refused rather than degraded. Chat Completions is what most gateways and every local runner implement.',+ zh: '这个接口讲哪种 API。两种格式发出的每一条消息都不一样,所以选错会被直接拒绝,而不是降级运行。绝大多数网关和所有本地运行器讲的都是 Chat Completions。',+ },+ 'set.agents.protocol.openai': { en: 'OpenAI (Chat Completions)', zh: 'OpenAI(Chat Completions)' },+ 'set.agents.protocol.anthropic': { en: 'Anthropic (Messages)', zh: 'Anthropic(Messages)' },'set.agents.customBaseUrl': { en: 'Endpoint', zh: 'API 地址' },'set.agents.customBaseUrl.desc': {- en: 'An Anthropic-compatible endpoint. Only needed for a provider not in the list — the built-in ones already know their address. The agent then sends only the core Messages format, without extended thinking, prompt caching or automatic fallbacks.',- zh: '兼容 Anthropic 的接口地址。只有列表里没有的提供方才需要填——内置的那些自己知道地址。此时 Agent 只发送基础 Messages 格式,不带扩展思考、提示缓存和自动回退。',+ en: 'The address of an endpoint speaking the protocol above. Only needed for a provider not in the list — the built-in ones already know theirs. On the Anthropic protocol the agent then sends only the core Messages format, without extended thinking, prompt caching or automatic fallbacks.',+ zh: '讲上面那个协议的接口地址。只有列表里没有的提供方才需要填——内置的那些自己知道地址。走 Anthropic 协议时,Agent 只发送基础 Messages 格式,不带扩展思考、提示缓存和自动回退。',},'set.agents.customModels': { en: 'Models', zh: '模型' },'set.agents.customModels.desc': {diff --git a/desktop/src/settings/editorSettings.ts b/desktop/src/settings/editorSettings.tsindex 3401867c3..db9247e9b 100644--- a/desktop/src/settings/editorSettings.ts+++ b/desktop/src/settings/editorSettings.ts@@ -20,10 +20,12 @@ import { Toasts } from '@/store/Toasts';import { mcpStatus, setMcpEnabled, subscribeMcp } from '@/store/McpStore';import { secretStatusLine, subscribeSecrets } from '@/store/SecretStore';import { syncAgentEndpoint } from '@/store/AgentStore';-import { agentProviders, agentKeyId, CUSTOM_PROVIDER } from '@/agent/providers';+import {+ agentProviders, agentKeyId, modelsSettingId, AGENT_PROTOCOLS, CUSTOM_PROVIDER,+} from '@/agent/providers';import { AGENT_EFFORTS, DEFAULT_EFFORT } from '@/settings/agentIds';import { applyUiZoom, UI_SCALE_SETTING, ZOOM_DEFAULT, ZOOM_MIN, ZOOM_MAX } from '@/layout/uiZoom';-import { t, editorLocale, systemDefaultLocale, EDITOR_LOCALES, LANGUAGE_SETTING_ID } from '@/i18n';+import { t, type MsgKey, editorLocale, systemDefaultLocale, EDITOR_LOCALES, LANGUAGE_SETTING_ID } from '@/i18n';const root = () => document.documentElement.style;@@ -253,6 +255,22 @@ for (const provider of agentProviders()) {default: false,status: { read: () => secretStatusLine(agentKeyId(provider.id)), subscribe: subscribeSecrets },});+ // A vendor whose address holds still but whose model NAMES do not: shipping a+ // list would be right until its next release, and a name we get wrong is not+ // an error — it is a gateway quietly serving something smaller all session.+ if (!provider.typedModels) continue;+ settingsRegistry.register({+ id: modelsSettingId(provider.id),+ type: 'string',+ scope: 'editor',+ section: 'agents',+ group: t('set.group.builtinAgent'),+ label: t('set.agents.providerModels', { provider: provider.label }),+ description: t('set.agents.providerModels.desc', { provider: provider.label }),+ placeholder: 'model-a, model-b',+ default: '',+ effect: () => syncAgentEndpoint(),+ });}// How hard the model is asked to think. Its own setting rather than part of the@@ -274,6 +292,24 @@ settingsRegistry.register({// The escape hatch, for a provider we have not heard of — which is exactly the// one we cannot ship an endpoint or a model list for.+//+// Which PROTOCOL it speaks has to be asked, not guessed: the two formats differ+// in every message they send, so getting it wrong is a 400 about a field the+// person never wrote. Chat Completions is offered first because it is what most+// endpoints — and every local runner — implement.+settingsRegistry.register({+ id: 'agents.customProtocol',+ type: 'enum',+ scope: 'editor',+ section: 'agents',+ group: t('set.group.customProvider'),+ label: t('set.agents.customProtocol'),+ description: t('set.agents.customProtocol.desc'),+ default: 'openai',+ options: AGENT_PROTOCOLS.map((value) => ({ value, label: t(`set.agents.protocol.${value}` as MsgKey) })),+ effect: () => syncAgentEndpoint(),+});+settingsRegistry.register({id: 'agents.customBaseUrl',type: 'string',@@ -282,7 +318,7 @@ settingsRegistry.register({group: t('set.group.customProvider'),label: t('set.agents.customBaseUrl'),description: t('set.agents.customBaseUrl.desc'),- placeholder: 'https://example.com/anthropic',+ placeholder: 'https://example.com/v1',default: '',effect: () => syncAgentEndpoint(),});diff --git a/desktop/src/store/AgentStore.ts b/desktop/src/store/AgentStore.tsindex 091dbc37d..09352e73c 100644--- a/desktop/src/store/AgentStore.ts+++ b/desktop/src/store/AgentStore.ts@@ -20,7 +20,9 @@ import { create } from 'zustand';import type { AgentStatus, AgentMessage } from '../../electron/agent/host';import type { AgentEvent, ConfirmAnswer, ConfirmReason, ConfirmRequest } from '../../electron/agent/types';import {- agentProviders, agentProvider, agentKeyId, parseModelList, CUSTOM_PROVIDER, DEFAULT_CONTEXT_WINDOW,+ agentProviders, agentProvider, agentKeyId, parseModelList, protocolOf, asProtocol,+ modelsSettingId, CUSTOM_PROVIDER,+ DEFAULT_CONTEXT_WINDOW,} from '@/agent/providers';import { refreshSecret, secretStatus, subscribeSecrets } from '@/store/SecretStore';import { useSettings } from '@/store/settingsStore';@@ -279,10 +281,16 @@ function loadSelection(): AgentSelection | null {function resolveProvider(id: string) {const def = agentProvider(id);if (!def) return undefined;- if (id !== CUSTOM_PROVIDER) return def;const settings = useSettings.getState();+ // A shipped provider whose model NAMES are not shippable — everything else+ // about it still is, including its endpoint and its key row.+ if (def.typedModels) {+ return { ...def, models: parseModelList(String(settings.getValue(modelsSettingId(id)) ?? '')) };+ }+ if (id !== CUSTOM_PROVIDER) return def;return {...def,+ protocol: asProtocol(settings.getValue('agents.customProtocol')),baseUrl: String(settings.getValue('agents.customBaseUrl') ?? ''),models: parseModelList(String(settings.getValue('agents.customModels') ?? '')),};@@ -318,6 +326,9 @@ export function syncAgentEndpoint(): void {const selection = effectiveSelection();const def = selection ? resolveProvider(selection.providerId) : undefined;void window.estella?.agent?.setEndpoint({+ // Which wire format to speak. Travels with the endpoint for the same reason+ // the window does: both are the one fact "which provider is this".+ protocol: protocolOf(def),baseUrl: def?.baseUrl ?? '',model: selection?.model ?? '',keyId: selection ? agentKeyId(selection.providerId) : '',diff --git a/desktop/tests/agent-compaction.test.ts b/desktop/tests/agent-compaction.test.tsindex dd691bf40..3d836ff5b 100644--- a/desktop/tests/agent-compaction.test.ts+++ b/desktop/tests/agent-compaction.test.ts@@ -9,10 +9,16 @@* conversation, so they must outlive the messages they pointed at.*/import { describe, it, expect } from 'vitest';-import { compactHistory, createAnthropicProvider } from '../electron/agent/anthropic';+import { compactHistory } from '../electron/agent/conversation';+import { ANTHROPIC_SHAPE, createAnthropicProvider } from '../electron/agent/anthropic';import { KEEP_WHOLE_RUNS, COMPACT_AT } from '../src/settings/agentIds';-type Message = Parameters<typeof compactHistory>[0][number];+type Message = Parameters<typeof ANTHROPIC_SHAPE.weigh>[0];++/** The shape under test is the real one — folding reads Anthropic content blocks. */+const fold = (+ messages: Message[], turnStarts: number[], dropped: number, keep: number, folded?: readonly string[],+) => compactHistory(ANTHROPIC_SHAPE, messages, turnStarts, dropped, keep, folded);/** `runs` turns, each: the person's line, the model's reply, a tool result. */function history(runs: number): { messages: Message[]; turnStarts: number[] } {@@ -30,13 +36,13 @@ function history(runs: number): { messages: Message[]; turnStarts: number[] } {describe('compacting a long conversation', () => {it('leaves a short one alone', () => {const { messages, turnStarts } = history(3);- expect(compactHistory(messages, turnStarts, 0, 3)).toBeNull();- expect(compactHistory(messages, turnStarts, 0, 5)).toBeNull();+ expect(fold(messages, turnStarts, 0, 3)).toBeNull();+ expect(fold(messages, turnStarts, 0, 5)).toBeNull();});it('keeps the last runs whole and folds the rest into one note', () => {const { messages, turnStarts } = history(6);- const out = compactHistory(messages, turnStarts, 0, 2)!;+ const out = fold(messages, turnStarts, 0, 2)!;// Note + acknowledgement, then the last two runs verbatim (3 messages each).expect(out.messages).toHaveLength(2 + 6);@@ -47,7 +53,7 @@ describe('compacting a long conversation', () => {it('carries what was asked, and says the rest was dropped', () => {const { messages, turnStarts } = history(5);- const note = String(compactHistory(messages, turnStarts, 0, 2)!.messages[0].content);+ const note = String(fold(messages, turnStarts, 0, 2)!.messages[0].content);for (const asked of ['ask 0', 'ask 1', 'ask 2']) expect(note).toContain(asked);// Told it is a summary, and told to re-read rather than trust it — a model@@ -59,7 +65,7 @@ describe('compacting a long conversation', () => {// gone, or "re-ask this one" rewinds the session to the wrong place.it('keeps turn coordinates counting from the start of the conversation', () => {const { messages, turnStarts } = history(6);- const out = compactHistory(messages, turnStarts, 0, 2)!;+ const out = fold(messages, turnStarts, 0, 2)!;expect(out.dropped).toBe(4);// Next turn opens at 6 — the same number it would have without compaction.@@ -68,7 +74,7 @@ describe('compacting a long conversation', () => {it('re-bases what is left so each start still points at that person\'s line', () => {const { messages, turnStarts } = history(6);- const out = compactHistory(messages, turnStarts, 0, 2)!;+ const out = fold(messages, turnStarts, 0, 2)!;expect(out.turnStarts.map((s) => out.messages[s])).toEqual([{ role: 'user', content: 'ask 4' },@@ -78,7 +84,7 @@ describe('compacting a long conversation', () => {it('numbers the note from the runs already folded away, not from zero', () => {const { messages, turnStarts } = history(5);- const out = compactHistory(messages, turnStarts, 10, 2)!;+ const out = fold(messages, turnStarts, 10, 2)!;expect(String(out.messages[0].content)).toContain('11. ask 0');expect(out.dropped).toBe(13);@@ -89,14 +95,14 @@ describe('compacting a long conversation', () => {// earliest request vanishing one compaction at a time — which is exactly what// a real gateway showed happening: fold one kept the passphrase, fold two ate it.it('keeps what was asked in EVERY fold, not just the last one', () => {- const first = compactHistory(...Object.values(history(6)) as [Message[], number[]], 0, 2)!;+ const first = fold(...Object.values(history(6)) as [Message[], number[]], 0, 2)!;expect(String(first.messages[0].content)).toContain('ask 0');const grown = {messages: [...first.messages, { role: 'user' as const, content: 'ask 6' }],turnStarts: [...first.turnStarts, first.messages.length],};- const second = compactHistory(grown.messages, grown.turnStarts, first.dropped, 1, first.folded)!;+ const second = fold(grown.messages, grown.turnStarts, first.dropped, 1, first.folded)!;const note = String(second.messages[0].content);expect(note).toContain('ask 0'); // from the first fold@@ -105,7 +111,7 @@ describe('compacting a long conversation', () => {it('elides the oldest once the note would grow unbounded', () => {const many = Array.from({ length: 40 }, (_, i) => `${i}. old ask ${i}`);- const out = compactHistory(...Object.values(history(5)) as [Message[], number[]], 0, 2, many)!;+ const out = fold(...Object.values(history(5)) as [Message[], number[]], 0, 2, many)!;const note = String(out.messages[0].content);expect(note).toContain('earlier requests omitted');expect(note).not.toContain('old ask 0');@@ -115,12 +121,12 @@ describe('compacting a long conversation', () => {// Compacting twice is the ordinary case in a conversation long enough to need// it once — the second pass must fold the note itself away with the rest.it('can be applied again to its own output', () => {- const first = compactHistory(...Object.values(history(6)) as [Message[], number[]], 0, 2)!;+ const first = fold(...Object.values(history(6)) as [Message[], number[]], 0, 2)!;const grown = {messages: [...first.messages, { role: 'user' as const, content: 'ask 6' }],turnStarts: [...first.turnStarts, first.messages.length],};- const second = compactHistory(grown.messages, grown.turnStarts, first.dropped, 1)!;+ const second = fold(grown.messages, grown.turnStarts, first.dropped, 1)!;expect(second.dropped).toBe(6);expect(second.turnStarts.map((s) => second.messages[s]))@@ -146,12 +152,16 @@ describe('a session deciding to compact', () => {session.pushUser(`ask ${i}`);session.pushToolResults([{ id: `c${i}`, content: 'x'.repeat(4000), isError: false }]);}- return session as unknown as {- compactIfNeeded(): number;- contextUsed(): number;- lastInputTokens: number;+ const reached = session as unknown as {+ log: { compactIfNeeded(): number; contextUsed(): number; lastInputTokens: number };turnIndex: number;};+ return {+ compactIfNeeded: () => reached.log.compactIfNeeded(),+ contextUsed: () => reached.log.contextUsed(),+ set lastInputTokens(n: number) { reached.log.lastInputTokens = n; },+ get turnIndex() { return reached.turnIndex; },+ };};it('says how many runs went, because nothing else can', () => {diff --git a/desktop/tests/agent-dialect.test.ts b/desktop/tests/agent-dialect.test.tsindex 539ad7688..c719fa23f 100644--- a/desktop/tests/agent-dialect.test.ts+++ b/desktop/tests/agent-dialect.test.ts@@ -14,8 +14,9 @@*/import { describe, it, expect } from 'vitest';import {- buildStepRequest, createAnthropicProvider, toolResultContent, describeApiError,+ buildStepRequest, createAnthropicProvider, toolResultContent,} from '../electron/agent/anthropic';+import { describeApiError } from '../electron/agent/apiError';import { DEFAULT_MODEL } from '../src/settings/agentIds';import type { CatalogTool } from '../electron/agent/types';diff --git a/desktop/tests/agent-openai-live.test.ts b/desktop/tests/agent-openai-live.test.tsnew file mode 100644index 000000000..69d0485e3--- /dev/null+++ b/desktop/tests/agent-openai-live.test.ts@@ -0,0 +1,146 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The Chat Completions provider against a real socket.+ *+ * The unit tests above hand the parser an array of chunk objects, which+ * skips the two layers most likely to be wrong about a protocol: what the+ * SDK actually puts on the wire, and how server-sent events are framed.+ * A stand-in endpoint is cheap and covers both — it records the request+ * body, so what this provider CLAIMS to send is checked against what+ * arrives rather than against itself.+ */+import { describe, it, expect, beforeAll, afterAll } from 'vitest';+import { createServer, type Server } from 'node:http';+import { createOpenAIProvider } from '../electron/agent/openai';+import type { StepEvent, CatalogTool } from '../electron/agent/types';++const TOOLS: CatalogTool[] = [+ { name: 'select_entity', description: 'select one', schema: { type: 'object', properties: { id: { type: 'number' } } } },+];++let server: Server;+let baseURL = '';+/** What the endpoint was last sent — the half a self-consistent test misses. */+let received: Record<string, unknown> | null = null;+/** The chunks the next request answers with. */+let script: Record<string, unknown>[] = [];++beforeAll(async () => {+ server = createServer((req, res) => {+ let body = '';+ req.on('data', (c) => { body += c; });+ req.on('end', () => {+ received = JSON.parse(body || '{}') as Record<string, unknown>;+ res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' });+ for (const chunk of script) res.write(`data: ${JSON.stringify(chunk)}\n\n`);+ res.write('data: [DONE]\n\n');+ res.end();+ });+ });+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));+ const address = server.address();+ baseURL = `http://127.0.0.1:${typeof address === 'object' && address ? address.port : 0}/v1`;+});++afterAll(() => new Promise<void>((resolve) => { server.close(() => resolve()); }));++const delta = (d: Record<string, unknown>, finish: string | null = null) =>+ ({ id: 'x', object: 'chat.completion.chunk', choices: [{ index: 0, delta: d, finish_reason: finish }] });++function session() {+ return createOpenAIProvider({ apiKey: 'k', model: 'stand-in', baseURL, vision: true })+ .createSession({ system: 'be useful', tools: TOOLS });+}++async function drain(s: ReturnType<typeof session>): Promise<StepEvent[]> {+ const events: StepEvent[] = [];+ for await (const e of s.step(new AbortController().signal)) events.push(e);+ return events;+}++describe('against a stand-in Chat Completions endpoint', () => {+ it('sends the system prompt, the tools and the history, and reads the answer back', async () => {+ script = [delta({ role: 'assistant', content: 'hello' }, 'stop')];+ const s = session();+ s.pushUser('say hello');+ const events = await drain(s);++ const messages = received?.messages as { role: string; content: string }[];+ expect(messages[0]).toEqual({ role: 'system', content: 'be useful' });+ expect(messages[1]).toEqual({ role: 'user', content: 'say hello' });+ expect((received?.tools as { function: { name: string } }[])[0].function.name).toBe('select_entity');+ expect(received?.stream_options).toEqual({ include_usage: true });+ expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['hello']);+ });++ // The round trip the kernel actually makes: ask for a tool, hand back its+ // result, ask again. Each result is its OWN message in this protocol, and the+ // second request is the first place a mistake about that shows.+ it('carries a tool round trip in the shape the endpoint expects', async () => {+ script = [+ delta({ tool_calls: [{ index: 0, id: 'c1', type: 'function', function: { name: 'select_entity', arguments: '{"id":7}' } }] }, 'tool_calls'),+ ];+ const s = session();+ s.pushUser('select entity 7');+ const first = await drain(s);+ const call = first.find((e) => e.type === 'tool_call') as { call: { id: string; input: unknown } };+ expect(call.call.input).toEqual({ id: 7 });++ s.pushToolResults([{ id: call.call.id, content: 'selected', isError: false }]);+ script = [delta({ content: 'done' }, 'stop')];+ await drain(s);++ const messages = received?.messages as { role: string; tool_call_id?: string; content?: unknown; tool_calls?: unknown[] }[];+ expect(messages[2]).toMatchObject({ role: 'assistant', tool_calls: [{ id: 'c1' }] });+ expect(messages[3]).toEqual({ role: 'tool', tool_call_id: 'c1', content: 'selected' });+ });++ /**+ * The Stop case, end to end. The kernel abandons a round mid-dispatch when a+ * run is aborted, leaving an assistant turn whose calls nothing answered —+ * and this protocol refuses that history outright, on this message and every+ * one after it. The repair has to happen before the request leaves.+ */+ it('repairs a history a Stop left half-answered, instead of being refused', async () => {+ script = [+ delta({ tool_calls: [{ index: 0, id: 'c1', type: 'function', function: { name: 'select_entity', arguments: '{}' } }] }, 'tool_calls'),+ ];+ const s = session();+ s.pushUser('do a thing');+ await drain(s);+ // …and nothing pushes the results: this is what an abort leaves behind.++ s.pushUser('never mind, do something else');+ script = [delta({ content: 'ok' }, 'stop')];+ await drain(s);++ const messages = received?.messages as { role: string; tool_call_id?: string }[];+ const answered = messages.filter((m) => m.role === 'tool');+ expect(answered).toHaveLength(1);+ expect(answered[0].tool_call_id).toBe('c1');+ // And it sits between the call and the next question, not after it.+ expect(messages.map((m) => m.role)).toEqual(['system', 'user', 'assistant', 'tool', 'user']);+ });++ it('sends an attached image as a data URL where the endpoint can see', async () => {+ script = [delta({ content: 'a red square' }, 'stop')];+ const s = session();+ s.pushUser('what is this?', [{ mediaType: 'image/png', data: 'AAAA' }]);+ await drain(s);++ const messages = received?.messages as { role: string; content: { type: string; image_url?: { url: string } }[] }[];+ expect(messages[1].content[0]).toEqual({ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } });+ });++ it('reports usage the endpoint only sends at the end', async () => {+ script = [+ delta({ content: 'hi' }, 'stop'),+ { id: 'x', object: 'chat.completion.chunk', choices: [], usage: { prompt_tokens: 2048, completion_tokens: 4 } },+ ];+ const s = session();+ s.pushUser('hi');+ const events = await drain(s);+ expect(events.find((e) => e.type === 'usage')).toEqual({ type: 'usage', inputTokens: 2048, outputTokens: 4 });+ });+});diff --git a/desktop/tests/agent-openai-stream.test.ts b/desktop/tests/agent-openai-stream.test.tsnew file mode 100644index 000000000..04e931b6f--- /dev/null+++ b/desktop/tests/agent-openai-stream.test.ts@@ -0,0 +1,136 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file Reading one Chat Completions response.+ *+ * The stream is untyped where the other protocol's is not: a tool call+ * arrives in pieces addressed by INDEX, only the first of which carries+ * the id and the name, and its arguments accumulate as a string. Every+ * assertion here is about stitching that back together — and about the+ * history it leaves behind, which is what the NEXT request is made of.+ */+import { describe, it, expect, vi } from 'vitest';+import type { StepEvent } from '../electron/agent/types';++const create = vi.fn();+vi.mock('openai', () => ({+ default: class {+ chat = { completions: { create } };+ },+}));++const { createOpenAIProvider } = await import('../electron/agent/openai');++type Chunk = Record<string, unknown>;++/** A response, as the wire delivers it. */+function stream(...chunks: Chunk[]): AsyncIterable<Chunk> {+ return { async *[Symbol.asyncIterator]() { for (const c of chunks) yield c; } };+}++const delta = (d: Record<string, unknown>, finish: string | null = null): Chunk =>+ ({ choices: [{ delta: d, finish_reason: finish }] });++async function run(...chunks: Chunk[]): Promise<{ events: StepEvent[]; history: unknown[] }> {+ create.mockResolvedValueOnce(stream(...chunks));+ const session = createOpenAIProvider({ apiKey: 'k', model: 'm' })+ .createSession({ system: 's', tools: [] });+ session.pushUser('go');+ const events: StepEvent[] = [];+ for await (const e of session.step(new AbortController().signal)) events.push(e);+ const memory = session.serialize() as { messages: unknown[] };+ return { events, history: memory.messages };+}++const only = <T extends StepEvent['type']>(events: StepEvent[], type: T) =>+ events.filter((e): e is Extract<StepEvent, { type: T }> => e.type === type);++describe('reading one response', () => {+ it('streams text and ends the turn', async () => {+ const { events } = await run(delta({ content: 'Hel' }), delta({ content: 'lo' }, 'stop'));+ expect(only(events, 'text').map((e) => e.delta)).toEqual(['Hel', 'lo']);+ expect(only(events, 'stop')[0].reason).toBe('end_turn');+ });++ // Shown, never sent back. There is no standard field for it — DeepSeek and+ // Qwen send `reasoning_content`, OpenRouter sends `reasoning` — and the+ // endpoints that produce it refuse it as INPUT, which is the exact opposite+ // of the other protocol's thinking blocks.+ it('shows reasoning under either name and keeps it out of the history', async () => {+ const a = await run(delta({ reasoning_content: 'weighing' }), delta({ content: 'ok' }, 'stop'));+ expect(only(a.events, 'thinking').map((e) => e.delta)).toEqual(['weighing']);+ expect(JSON.stringify(a.history)).not.toContain('weighing');++ const b = await run(delta({ reasoning: 'weighing' }), delta({ content: 'ok' }, 'stop'));+ expect(only(b.events, 'thinking').map((e) => e.delta)).toEqual(['weighing']);+ });++ it('stitches a tool call out of the pieces it arrives in', async () => {+ const { events, history } = await run(+ delta({ tool_calls: [{ index: 0, id: 'c1', function: { name: 'move', arguments: '{"x"' } }] }),+ delta({ tool_calls: [{ index: 0, function: { arguments: ':1}' } }] }, 'tool_calls'),+ );+ expect(only(events, 'tool_pending')).toEqual([{ type: 'tool_pending', id: 'c1', name: 'move' }]);+ expect(only(events, 'tool_args').map((e) => e.delta)).toEqual(['{"x"', ':1}']);+ expect(only(events, 'tool_call')[0].call).toEqual({ id: 'c1', name: 'move', input: { x: 1 } });+ expect(only(events, 'stop')[0].reason).toBe('tool_use');+ // The assistant turn it leaves behind is what the next request replays.+ expect(history[1]).toMatchObject({+ role: 'assistant',+ tool_calls: [{ id: 'c1', function: { name: 'move', arguments: '{"x":1}' } }],+ });+ });++ it('keeps two parallel calls apart, in the order they were indexed', async () => {+ const { events } = await run(+ delta({ tool_calls: [{ index: 1, id: 'b', function: { name: 'second', arguments: '{}' } }] }),+ delta({ tool_calls: [{ index: 0, id: 'a', function: { name: 'first', arguments: '{}' } }] }, 'tool_calls'),+ );+ expect(only(events, 'tool_call').map((e) => e.call.name)).toEqual(['first', 'second']);+ });++ // Several gateways report `stop` on a response that is nothing but tool calls.+ // Having asked for one is the fact the kernel acts on, so it wins.+ it('reads a response full of tool calls as tool_use however it was labelled', async () => {+ const { events } = await run(+ delta({ tool_calls: [{ index: 0, id: 'c', function: { name: 'x', arguments: '{}' } }] }, 'stop'),+ );+ expect(only(events, 'stop')[0].reason).toBe('tool_use');+ });++ // A truncated turn writes arguments that do not parse. The kernel dispatches+ // anyway: the tool reports what it needed and the model gets something it can+ // act on, where a throw would end the run with the reason in a stack trace.+ it('dispatches a call whose arguments were cut off', async () => {+ const { events } = await run(+ delta({ tool_calls: [{ index: 0, id: 'c', function: { name: 'x', arguments: '{"half' } }] }, 'length'),+ );+ expect(only(events, 'tool_call')[0].call.input).toEqual({});+ });++ it('reports what the call was billed, and where that leaves the context', async () => {+ const { events } = await run(+ delta({ content: 'hi' }, 'stop'),+ { usage: { prompt_tokens: 1200, completion_tokens: 8 }, choices: [] },+ );+ expect(only(events, 'usage')[0]).toEqual({ type: 'usage', inputTokens: 1200, outputTokens: 8 });+ // The window is the endpoint's, and the reading is the larger of what it+ // billed and what the history weighs.+ expect(only(events, 'context')[0].used).toBe(1200);+ });++ it('still answers for the context when the endpoint reports no usage at all', async () => {+ const { events } = await run(delta({ content: 'hi' }, 'stop'));+ expect(only(events, 'usage')).toHaveLength(0);+ expect(only(events, 'context')[0].used).toBeGreaterThan(0);+ });++ it('says what an endpoint failure means, and whether asking again could work', async () => {+ create.mockRejectedValueOnce(Object.assign(new Error('nope'), { status: 401 }));+ const session = createOpenAIProvider({ apiKey: 'k', model: 'm' })+ .createSession({ system: 's', tools: [] });+ session.pushUser('go');+ const step = async () => { for await (const _ of session.step(new AbortController().signal)) { /* drain */ } };+ await expect(step()).rejects.toThrow(/rejected the API key/i);+ });+});diff --git a/desktop/tests/agent-openai.test.ts b/desktop/tests/agent-openai.test.tsnew file mode 100644index 000000000..bf9a347fa--- /dev/null+++ b/desktop/tests/agent-openai.test.ts@@ -0,0 +1,161 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The Chat Completions provider — the half of it that is not the network.+ *+ * Three of these are about differences from the OTHER protocol, and each+ * of the three costs a turn when it is missed: a history the endpoint+ * refuses outright after a Stop, reasoning that must be shown but never+ * sent back, and usage that is not streamed unless asked for.+ */+import { describe, it, expect } from 'vitest';+import {+ buildChatRequest, answerOrphanedCalls, createOpenAIProvider, OPENAI_FORMAT,+} from '../electron/agent/openai';+import { ANTHROPIC_FORMAT } from '../electron/agent/anthropic';+import type { CatalogTool } from '../electron/agent/types';++const TOOLS: CatalogTool[] = [+ { name: 'get_scene_tree', description: 'read it', schema: { type: 'object', properties: {} } },+];++type Msg = Parameters<typeof answerOrphanedCalls>[0][number];++const request = (over: Partial<Parameters<typeof buildChatRequest>[0]> = {}) =>+ buildChatRequest({+ model: 'a-model',+ effort: 'xhigh',+ reasoningEffort: true,+ system: 'be useful',+ tools: TOOLS,+ messages: [{ role: 'user', content: 'hi' }],+ ...over,+ });++describe('the Chat Completions request', () => {+ // The prompt is a MESSAGE here, not a parameter — but it is prepended per+ // request rather than kept in the history, so the history's indices still mean+ // what the shared fold counts them to mean.+ it('puts the system prompt first without keeping it in the history', () => {+ const body = request();+ expect(body.messages[0]).toEqual({ role: 'system', content: 'be useful' });+ expect(body.messages[1]).toEqual({ role: 'user', content: 'hi' });+ });++ it('declares tools as functions, in catalog order', () => {+ const body = request();+ expect(body.tools?.[0]).toEqual({+ type: 'function',+ function: { name: 'get_scene_tree', description: 'read it', parameters: TOOLS[0].schema },+ });+ });++ // Without this the context meter runs entirely on the character estimate,+ // which under-counts CJK badly — and CJK is most of what these users type.+ it('asks for usage, which is not streamed otherwise', () => {+ expect(request().stream_options).toEqual({ include_usage: true });+ });++ it('leaves reasoning_effort off for an endpoint that would refuse it', () => {+ expect((request() as { reasoning_effort?: string }).reasoning_effort).toBe('high');+ expect((request({ reasoningEffort: false }) as { reasoning_effort?: string }).reasoning_effort)+ .toBeUndefined();+ });+});++/**+ * An assistant message carrying tool_calls MUST be followed by a tool message+ * for every one of them. The kernel abandons a round mid-dispatch when the run+ * is aborted, so a Stop leaves exactly the history this protocol refuses — and+ * refuses again on every message after it, with nothing on screen tying the+ * failure to the Stop that caused it.+ */+describe('a history left half-answered by a Stop', () => {+ const asked = (...ids: string[]): Msg => ({+ role: 'assistant',+ content: null,+ tool_calls: ids.map((id) => ({ id, type: 'function', function: { name: 't', arguments: '{}' } })),+ } as Msg);++ it('answers every call nothing came back for', () => {+ const messages: Msg[] = [{ role: 'user', content: 'go' }, asked('a', 'b')];+ expect(answerOrphanedCalls(messages)).toBe(2);+ expect(messages.slice(2).map((m) => (m as { tool_call_id: string }).tool_call_id)).toEqual(['a', 'b']);+ expect(messages[2]).toMatchObject({ role: 'tool', content: expect.stringContaining('interrupted') });+ });++ it('answers only the ones that are missing', () => {+ const messages: Msg[] = [+ { role: 'user', content: 'go' },+ asked('a', 'b'),+ { role: 'tool', tool_call_id: 'a', content: 'done' },+ ];+ expect(answerOrphanedCalls(messages)).toBe(1);+ expect(messages).toHaveLength(4);+ expect(messages[3]).toMatchObject({ role: 'tool', tool_call_id: 'b' });+ });++ it('leaves a complete history alone', () => {+ const messages: Msg[] = [+ { role: 'user', content: 'go' },+ asked('a'),+ { role: 'tool', tool_call_id: 'a', content: 'done' },+ { role: 'assistant', content: 'there' },+ ];+ expect(answerOrphanedCalls(messages)).toBe(0);+ expect(messages).toHaveLength(4);+ });++ it('does nothing to a conversation that never called a tool', () => {+ const messages: Msg[] = [{ role: 'user', content: 'hi' }, { role: 'assistant', content: 'hello' }];+ expect(answerOrphanedCalls(messages)).toBe(0);+ expect(messages).toHaveLength(2);+ });+});++describe('the provider', () => {+ const provider = (over = {}) => createOpenAIProvider({ apiKey: 'k', model: 'm', ...over });++ it('names the endpoint it is pointed at, so a transcript can say which', () => {+ expect(provider().id).toBe('openai');+ expect(provider({ baseURL: 'http://localhost:11434/v1' }).id).toBe('openai:http://localhost:11434/v1');+ });++ // A gateway we have never heard of may not take images, and finding out costs+ // a call spent on a screenshot the model is then told it cannot look at.+ it('assumes a gateway cannot see, and OpenAI itself can', () => {+ expect(provider().acceptsImages).toBe(true);+ expect(provider({ baseURL: 'http://localhost:11434/v1' }).acceptsImages).toBe(false);+ expect(provider({ baseURL: 'http://x/v1', vision: true }).acceptsImages).toBe(true);+ });++ /**+ * Both formats serialize to `{ v: 1, messages: [...] }`, so nothing but the+ * format's NAME can stop a Claude conversation being resumed into this session+ * — where it fails as a 400 several seconds later, with nothing connecting it+ * to the model switch that caused it.+ */+ it('refuses a conversation another protocol wrote, and starts fresh', () => {+ const memory = {+ v: 1,+ format: ANTHROPIC_FORMAT,+ messages: [{ role: 'user', content: 'from Claude' }],+ turnStarts: [0],+ dropped: 4,+ folded: [],+ lastInputTokens: 99,+ };+ const session = provider().createSession({ system: 's', tools: TOOLS }, memory);+ expect(session.turnIndex).toBe(0);+ });++ it('takes back one of its own', () => {+ const first = provider().createSession({ system: 's', tools: TOOLS });+ first.pushUser('remember this');+ const memory = first.serialize() as { format: string };+ expect(memory.format).toBe(OPENAI_FORMAT);++ const second = provider().createSession({ system: 's', tools: TOOLS }, memory);+ expect(second.turnIndex).toBe(1);+ });+});diff --git a/desktop/tests/agent-providers.test.ts b/desktop/tests/agent-providers.test.tsindex 85db0c5ab..eae08986a 100644--- a/desktop/tests/agent-providers.test.ts+++ b/desktop/tests/agent-providers.test.ts@@ -8,7 +8,9 @@* worse answer for the rest of the session.*/import { describe, it, expect } from 'vitest';-import { agentProviders, agentProvider, agentKeyId, parseModelList, CUSTOM_PROVIDER } from '@/agent/providers';+import {+ agentProviders, agentProvider, agentKeyId, parseModelList, protocolOf, asProtocol, CUSTOM_PROVIDER,+} from '@/agent/providers';describe('the shipped providers', () => {it('knows DeepSeek without being told', () => {@@ -30,6 +32,19 @@ describe('the shipped providers', () => {expect(agentKeyId('deepseek')).toBe('agents.key.deepseek');});+ // Its address and protocol are knowable; its model NAMES are not, and shipping+ // a wrong one is not an error — it is a gateway serving something smaller for+ // the rest of the session.+ it('ships OpenAI on its own protocol, and leaves its model names to be typed', () => {+ const openai = agentProvider('openai');+ expect(openai?.baseUrl).toBe('');+ expect(protocolOf(openai)).toBe('openai');+ expect(openai?.typedModels).toBe(true);+ expect(openai?.models).toEqual([]);+ // Still a shipped provider: it keeps its own key row, unlike the custom slot.+ expect(openai?.userDefined).toBeUndefined();+ });+it('ships the custom slot empty — it is the one we cannot know', () => {const custom = agentProvider(CUSTOM_PROVIDER);expect(custom?.userDefined).toBe(true);@@ -37,6 +52,29 @@ describe('the shipped providers', () => {});});+/**+ * Which wire format an endpoint speaks is not a detail: the two differ in every+ * message they send, so a def that got it wrong would send a perfectly formed+ * request in the wrong shape.+ */+describe('the protocol a provider speaks', () => {+ // Every def predates the field, and every one of them was the Messages API.+ it('reads a def without one as Anthropic', () => {+ expect(protocolOf(agentProvider('anthropic'))).toBe('anthropic');+ expect(protocolOf(agentProvider('deepseek'))).toBe('anthropic');+ expect(protocolOf(undefined)).toBe('anthropic');+ });++ // A settings file is one a person can edit, and Chat Completions is what most+ // endpoints — and every local runner — implement.+ it('narrows a stored value, defaulting a custom endpoint to Chat Completions', () => {+ expect(asProtocol('anthropic')).toBe('anthropic');+ expect(asProtocol('openai')).toBe('openai');+ expect(asProtocol('gemini')).toBe('openai');+ expect(asProtocol(undefined)).toBe('openai');+ });+});+// Someone pasting three model names should not have to guess which separator.describe('the custom model list', () => {it('takes commas or newlines, and ignores the gaps', () => {diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 7e888257d..92b7b8574 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -51,6 +51,9 @@ importers:lucide-react:specifier: ^1.21.0version: 1.21.0([email protected])+ openai:+ specifier: ^7.4.0+ version: 7.4.0([email protected])([email protected])pngjs:specifier: ^7.0.0version: 7.0.0@@ -2670,6 +2673,27 @@ packages:resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}engines: {node: '>=18'}+ resolution: {integrity: sha512-+C9Muit5x8j9R8ej8ZzVgKcrVDtqFqTy9gxFdov0EItLgU68zrJtF9ZeT0cyqJQW9S3PCJkdFgADtRGquRBtew==}+ engines: {node: '>=22.0.0'}+ peerDependencies:+ '@aws-sdk/credential-provider-node': '>=3.972.0 <4'+ '@smithy/hash-node': '>=4.3.0 <5'+ '@smithy/signature-v4': '>=5.4.0 <6'+ ws: ^8.18.0+ zod: ^3.25 || ^4.0+ peerDependenciesMeta:+ '@aws-sdk/credential-provider-node':+ optional: true+ '@smithy/hash-node':+ optional: true+ '@smithy/signature-v4':+ optional: true+ ws:+ optional: true+ zod:+ optional: true+resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==}engines: {node: '>=10'}@@ -6094,6 +6118,11 @@ snapshots:dependencies:mimic-function: 5.0.1+ optionalDependencies:+ ws: 8.21.0+ zod: 4.4.3+dependencies:bl: 4.1.0