A local-first multi-agent app development platform, built on PenguinHarness
/**
* GenerativeModel —— the SDK's LLM interface implementation.
*
* Responsibilities (protocol translation + streaming aggregation):
* 1. Merge a group of OmniMessages that **share the same role** into a single AgentHub `UniMessage`;
* 2. Issue the request via `AutoLLMClient.streamingResponseStateful` (stateful — AgentHub
* maintains history internally), translating streamed `UniEvent`s back into OmniMessages:
* - text/thinking/tool-call deltas → `partial_*` (before the first delta of each segment
* `yield` a `start`; after the segment ends `yield` a `stop`);
* - after a segment ends, append the full `model_msg` (thinking / text / tool_call);
* - a `token_usage` event_msg is produced **only on normal completion**
* (observability/Token).
* 3. Interruption/error handling: `finishInterrupted` first closes any open
* streaming segments and backfills the complete message, then the output ends — never
* leaking a malformed structure. This interface **never retries internally** — it only
* classifies, and `context_engine` owns the retry policy: `retryable` rides the
* engine's reconnect ladder, `fatal` stops the run. The split is an allowlist of
* certainty: only failures a retry provably cannot fix are `fatal` — a provider 4xx
* rejection (408/429 excluded, see `isFatalProviderRejection`), a credentials failure
* (`isAuthenticationError`), AgentHub's fast-mode UnsupportedParameterError
* (`isFastModeUnsupportedError`), and input that fails to assemble into a request at
* all. Everything else — network/transport drops, timeouts, 429/5xx, AgentHub parse
* errors and truncated streams, and every unclassifiable error — ends `retryable`,
* with the concrete failure on `errorMessage`. User interruption ends with `aborted`.
*
* `context_engine` only consumes OmniMessage; all Uni* protocol details are encapsulated here.
* Docs: /docs/interfaces § "The built-in implementation: GenerativeModel".
*/
import {
AutoLLMClient,
EmptyResponseError,
ThinkingLevel,
ToolCallArgumentParseError,
UnsupportedParameterError,
} from "@prismshadow/agenthub";
import type {
ContentItem,
FinishReason,
ToolSchema,
UniConfig,
UniEvent,
UniMessage,
UsageMetadata,
} from "@prismshadow/agenthub";
import {
addTokenCounts,
assistantText,
emptyTokenCounts,
partialText,
partialThinking,
partialToolCall,
thinkingMessage,
tokenUsage,
toolCall,
} from "../omnimessage/index.js";
import type {
CompleteModelPayload,
Fidelity,
OmniMessage,
StopReason,
TokenCounts,
} from "../omnimessage/index.js";
import type {
GenerativeModelConfig,
GenerativeModelParameters,
LLMInterface,
LLMOutcome,
ThinkingLevelName,
ToolDefinition,
} from "../interfaces/index.js";
import { attributionHeaders } from "../state/model-catalog.js";
import { ToolCallIdAllocator, stripToolCallIdSuffix } from "./tool-call-ids.js";
import {
approximateMessagesTokens,
approximateTokens,
effectiveMaxOutputTokens,
resolveContextWindow,
} from "./context-limits.js";
// ---------------------------------------------------------------------------
// Pure conversion function: OmniMessage[] → a single UniMessage (unit-testable, no network)
// ---------------------------------------------------------------------------
/**
* Tool arguments JSON string → object. History only ever contains tool_calls from committed
* turns (non-completed turns are discarded on replay); bad JSON already throws during AgentHub
* parsing and reconnects via malformed, so it never enters history — hence we parse directly
* with no fallback tolerance; an empty string is treated as no arguments.
*/
function parseToolArguments(raw: string): Record<string, unknown> {
if (!raw) return {};
const parsed: unknown = JSON.parse(raw);
return parsed !== null && typeof parsed === "object" ? (parsed as Record<string, unknown>) : {};
}
/**
* Whether a fidelity payload carries at least one key (mirrors AgentHub's baseClient helper —
* an absent and an empty fidelity are equivalent).
*/
function hasFidelity(fidelity?: Fidelity): boolean {
return fidelity != null && Object.keys(fidelity).length > 0;
}
/**
* Compare two fidelity payloads by value (mirrors AgentHub's baseClient helper). Fidelity
* objects are built with a stable key order by each AgentHub client, so JSON serialization is
* a faithful equality check.
*/
function fidelityEquals(a?: Fidelity, b?: Fidelity): boolean {
return JSON.stringify(a ?? {}) === JSON.stringify(b ?? {});
}
/** Spread helper: attach `fidelity` only when it carries at least one key. */
function fidelityProp(fidelity?: Fidelity): { fidelity?: Fidelity } {
return hasFidelity(fidelity) ? { fidelity } : {};
}
/**
* Maps a complete OmniMessage payload to an AgentHub `ContentItem`.
* Only complete model_msg payloads are supported; `partial_*` is an output-only protocol.
*/
function payloadToContentItem(payload: CompleteModelPayload): ContentItem {
// The provider-fidelity payload is opaque and restored verbatim — some models require it
// when history is replayed back (e.g. Claude thinking signatures, GPT-5 encrypted reasoning
// and phase segmentation, the OpenAI-compatible reasoning field name); losing it would break
// Session recovery.
switch (payload.type) {
case "text":
return {
type: "text",
text: payload.text,
...fidelityProp(payload.fidelity),
};
case "image_url":
return { type: "image_url", image_url: payload.image_url };
case "video_url":
return { type: "video_url", video_url: payload.video_url };
case "inline_data":
return {
type: "inline_data",
data: Buffer.from(payload.data, "base64"),
mime_type: payload.mime_type,
...fidelityProp(payload.fidelity),
};
case "inline_thinking":
return {
type: "inline_thinking",
data: Buffer.from(payload.data, "base64"),
mime_type: payload.mime_type,
...fidelityProp(payload.fidelity),
};
case "thinking":
return {
type: "thinking",
thinking: payload.thinking,
...fidelityProp(payload.fidelity),
};
case "tool_call":
return {
type: "tool_call",
name: payload.name,
// OmniMessage stores arguments as a JSON string; UniMessage uses an object.
arguments: parseToolArguments(payload.arguments),
// On the way back, strip the uniqueness suffix to restore the provider's original id (see tool-call-ids.ts).
tool_call_id: stripToolCallIdSuffix(payload.tool_call_id),
...fidelityProp(payload.fidelity),
};
case "tool_call_output":
return {
type: "tool_result",
text: payload.output,
// Images carried by the tool output (data URL array) → AgentHub tool_result.images
// (natively supported).
...(payload.images && payload.images.length > 0 ? { images: payload.images } : {}),
// Gemini pairs by using tool_call_id as functionResponse.name, so it must be restored to the original id (the function name).
tool_call_id: stripToolCallIdSuffix(payload.tool_call_id),
};
default: {
// Exhaustiveness check: compile-time error when a new payload type is added.
const _exhaustive: never = payload;
throw new Error(
`streamGenerate: unsupported message type: ${(_exhaustive as { type?: string }).type}`,
);
}
}
}
/**
* Groups a replayed, complete OmniMessage history into a sequence of UniMessages by
* **adjacent same-role** runs (used for the setHistory injection during Session recovery).
* One committed turn = a group of user-side input + a group of
* assistant output, matching exactly the adjacent user / assistant UniMessages in AgentHub history.
*/
export function groupHistoryToUniMessages(history: OmniMessage[]): UniMessage[] {
const groups: OmniMessage[][] = [];
let currentRole: string | null = null;
for (const msg of history) {
const role = (msg.payload as { role?: string }).role;
if (role !== "user" && role !== "assistant") {
throw new Error(`setHistory: unsupported message without role: ${JSON.stringify(msg.type)}`);
}
if (role !== currentRole) {
groups.push([]);
currentRole = role;
}
groups[groups.length - 1]!.push(msg);
}
return groups.map(mergeOmniToUniMessage);
}
/**
* Merges a group of OmniMessages into **a single** UniMessage.
*
* Constraint: all messages in the array must share the same
* role; the role of the first payload is used as the UniMessage's role (a tool_call_output
* group has role "user"). Throws if roles are mixed.
*/
export function mergeOmniToUniMessage(messages: OmniMessage[]): UniMessage {
if (messages.length === 0) {
throw new Error("streamGenerate requires at least one input message");
}
const payloads = messages.map((m) => m.payload as CompleteModelPayload);
// Each payload carries its own role (tool_call_output is fixed to "user"); take the first one's role.
const role = payloads[0]!.role;
const contentItems: ContentItem[] = [];
for (const payload of payloads) {
if (payload.role !== role) {
throw new Error(
"streamGenerate does not accept mixed roles: all messages merged into one UniMessage must share the same role",
);
}
contentItems.push(payloadToContentItem(payload));
}
return { role, content_items: contentItems };
}
// ---------------------------------------------------------------------------
// Image budget (a request may carry only so many images)
// ---------------------------------------------------------------------------
/**
* The stand-in left where an image was dropped from a request: one text item (or one appended
* line, for a tool result's images) in the place the picture occupied, so the model knows the
* conversation held a picture there and cannot mistake the gap for its own omission. Named in
* the seller's own terms — the limit is the request's image count, not the picture's size.
*/
export function imageOmittedNote(limit: number): string {
return `[image omitted: this model accepts at most ${limit} image${limit === 1 ? "" : "s"} per request]`;
}
/** Whether a content item is an image the request counts: a data URL or an inline image blob. */
function isImageItem(item: ContentItem): boolean {
if (item.type === "image_url") return true;
return item.type === "inline_data" && item.mime_type.startsWith("image/");
}
/** How many images one UniMessage carries — image items plus a tool result's image attachments. */
export function countMessageImages(msg: UniMessage): number {
let count = 0;
for (const item of msg.content_items) {
if (isImageItem(item)) count += 1;
else if (item.type === "tool_result") count += item.images?.length ?? 0;
}
return count;
}
/**
* Keeps at most `budget` of a message's images, **the newest ones**, and replaces each older
* one with {@link imageOmittedNote}. A message that fits is returned as-is (same object), so a
* history made entirely of fitting messages costs nothing.
*/
function capMessageImages(msg: UniMessage, budget: number, limit: number): UniMessage {
if (countMessageImages(msg) <= budget) return msg;
const note = imageOmittedNote(limit);
let left = budget;
const kept: ContentItem[] = [];
// Newest first: the images nearest the current request are the ones worth paying for.
for (let i = msg.content_items.length - 1; i >= 0; i -= 1) {
const item = msg.content_items[i]!;
if (isImageItem(item)) {
if (left > 0) {
left -= 1;
kept.push(item);
} else {
kept.push({ type: "text", text: note });
}
continue;
}
if (item.type === "tool_result" && (item.images?.length ?? 0) > 0) {
const images = item.images!;
if (images.length <= left) {
left -= images.length;
kept.push(item);
} else {
const surviving = left > 0 ? images.slice(-left) : [];
left = 0;
kept.push({
...item,
...(surviving.length > 0 ? { images: surviving } : { images: undefined }),
text: item.text ? `${item.text}\n\n${note}` : note,
});
}
continue;
}
kept.push(item);
}
return { ...msg, content_items: kept.reverse() };
}
/**
* Trims a replayed history so it carries at most `budget` images in aggregate, newest kept (see
* {@link capMessageImages}). `limit` is the seller's per-request cap and only names itself in
* the placeholder — a request whose own input already used part of the budget keeps fewer
* replayed images than the cap without misreporting the cap as smaller.
* Returns a new array of the same length whose untouched messages are the very objects it was
* handed.
*/
export function capHistoryImages(
history: UniMessage[],
budget: number,
limit: number = budget,
): UniMessage[] {
const counts = history.map(countMessageImages);
const capped = new Array<UniMessage>(history.length);
let left = budget;
for (let i = history.length - 1; i >= 0; i -= 1) {
const msg = history[i]!;
const count = counts[i]!;
if (count === 0) {
capped[i] = msg;
continue;
}
capped[i] = capMessageImages(msg, left, limit);
left = Math.max(0, left - count);
}
return capped;
}
// ---------------------------------------------------------------------------
// Video budget (a request may carry only so many videos)
// ---------------------------------------------------------------------------
/**
* The stand-in left where a video was dropped from a request — one text item in the place the
* video occupied, so the model can tell a gap from its own omission. The reason is the body
* size rather than a published count: a video is megabytes of base64, and a provider takes at
* most one request body of it, so a conversation that replays two videos is refused outright.
*/
export function videoOmittedNote(limit: number): string {
return `[video omitted: this request carries at most ${limit} video${limit === 1 ? "" : "s"}]`;
}
/** Whether a content item is a video the request counts. */
function isVideoItem(item: ContentItem): boolean {
return item.type === "video_url";
}
/** How many videos one UniMessage carries. */
export function countMessageVideos(msg: UniMessage): number {
let count = 0;
for (const item of msg.content_items) {
if (isVideoItem(item)) count += 1;
}
return count;
}
/**
* Keeps at most `budget` of a message's videos, **the newest ones**, and replaces each older
* one with {@link videoOmittedNote}. A message that fits is returned as-is (same object).
*/
function capMessageVideos(msg: UniMessage, budget: number, limit: number): UniMessage {
if (countMessageVideos(msg) <= budget) return msg;
const note = videoOmittedNote(limit);
let left = budget;
const kept: ContentItem[] = [];
// Newest first, as the image budget does: the video nearest the current request is the one
// worth paying for.
for (let i = msg.content_items.length - 1; i >= 0; i -= 1) {
const item = msg.content_items[i]!;
if (isVideoItem(item)) {
if (left > 0) {
left -= 1;
kept.push(item);
} else {
kept.push({ type: "text", text: note });
}
continue;
}
kept.push(item);
}
return { ...msg, content_items: kept.reverse() };
}
/**
* Trims a replayed history so it carries at most `budget` videos in aggregate, newest kept
* (see {@link capMessageVideos}). Returns a new array of the same length whose untouched
* messages are the very objects it was handed.
*/
export function capHistoryVideos(
history: UniMessage[],
budget: number,
limit: number = budget,
): UniMessage[] {
const counts = history.map(countMessageVideos);
const capped = new Array<UniMessage>(history.length);
let left = budget;
for (let i = history.length - 1; i >= 0; i -= 1) {
const msg = history[i]!;
const count = counts[i]!;
if (count === 0) {
capped[i] = msg;
continue;
}
capped[i] = capMessageVideos(msg, left, limit);
left = Math.max(0, left - count);
}
return capped;
}
// ---------------------------------------------------------------------------
// Token accounting (as defined by SKILL.md)
// ---------------------------------------------------------------------------
/**
* Converts AgentHub `UsageMetadata` into Adelie `TokenCounts`.
*
* Conversion rules (AgentHub UsageMetadata → OmniMessage TokenCounts, null treated as 0):
* - `cache_read = cached_tokens` (input tokens served from cache hits);
* - `cache_write = prompt_tokens` (input tokens on a cache miss);
* - `output = thoughts_tokens + response_tokens`;
* - `total = cache_read + cache_write + output`.
* That is, `input = cache_read + cache_write = cached_tokens + prompt_tokens`,
* and `total = input + output` (consistent with SKILL.md's input/output accounting).
*/
export function usageToTokenCounts(usage: UsageMetadata): TokenCounts {
const cached = usage.cached_tokens ?? 0;
const prompt = usage.prompt_tokens ?? 0;
const thoughts = usage.thoughts_tokens ?? 0;
const response = usage.response_tokens ?? 0;
const cacheRead = cached;
const cacheWrite = prompt;
const output = thoughts + response;
return {
cache_read: cacheRead,
cache_write: cacheWrite,
output,
total: cacheRead + cacheWrite + output,
};
}
// ---------------------------------------------------------------------------
// Pure translator: UniEvent[] → OmniMessage[] (unit-testable, no network)
// ---------------------------------------------------------------------------
interface ToolCallAccumulator {
name: string;
/** Accumulated arguments delta fragments (JSON string); used as a fallback when no complete tool_call arrives. */
argsBuffer: string;
/** If a complete tool_call appears in an event, its JSON.stringify'd arguments are recorded here. */
completeArgs: string | null;
/** Session-unique id emitted to OmniMessage (with a `#n` suffix on provider id collisions). */
toolCallId: string;
/** The original tool_call_id reported by the provider (the attribution key for inbound events). */
providerKey: string;
/** Provider-fidelity payload (kept verbatim, produced alongside the complete tool_call). */
fidelity: Fidelity | undefined;
/** Whether this tool_call's complete message has already been emitted eagerly in `pushEvent` (avoids duplicate emission in finish). */
emitted: boolean;
}
/**
* Streaming translator. Feed `UniEvent`s one at a time into `pushEvent`, which yields
* incremental OmniMessages (`partial_*`); after the stream ends, call `finish` to produce
* `stop`, the complete `model_msg`, and `token_usage`.
*
* Split into its own class to make unit testing easier (feed a constructed array of UniEvents,
* assert on emission order / aggregation / token counts).
* Docs: /docs/omni-message § "The streaming discipline".
*/
export class EventTranslator {
/**
* tool_call_id uniqueness registry. By default each translator creates its own (unit tests /
* one-off translation); in production `GenerativeModel` injects a Session-level shared instance so
* the uniqueness scope spans Requests and survives compaction rebuilds.
*/
constructor(private readonly toolCallIds: ToolCallIdAllocator = new ToolCallIdAllocator()) {}
// Whether each segment type has already yielded its `start`.
private textStarted = false;
private thinkingStarted = false;
// Tool-call partial starts, tracked by the (uniqueness-resolved) tool_call_id.
private toolStarted = new Set<string>();
// Some providers' tool argument deltas don't carry a tool_call_id; attribute them to the most recently opened tool call (by provider id key).
private activeToolCallId: string | null = null;
// Buffers needed for the complete message.
private textBuffer = "";
private thinkingBuffer = "";
// Provider-fidelity payloads of the currently open segments. Segmentation mirrors AgentHub's
// baseClient aggregation: a thinking block is closed by its fidelity payload (a run of equal
// fidelity is one block); a text segment is closed by a `fidelity.signature` and split by a
// differing `fidelity.phase`.
private thinkingFidelity: Fidelity | undefined;
private textFidelity: Fidelity | undefined;
/** Provider id keys saved in order of appearance, so complete tool_calls are emitted in a stable order. */
private toolOrder: string[] = [];
/** provider's original tool_call_id → the accumulator for the **latest** call under that id. */
private tools = new Map<string, ToolCallAccumulator>();
private finishReason: FinishReason | null = null;
/** Token usage for this request (a snapshot from the most recent usage report). */
private requestTokens: TokenCounts = emptyTokenCounts();
/** Consumes one UniEvent, yielding 0..n streaming OmniMessages. */
*pushEvent(event: UniEvent): Generator<OmniMessage> {
if (event.finish_reason != null) {
this.finishReason = event.finish_reason;
}
if (event.usage_metadata) {
// The same request may report usage multiple times, always as a **cumulative snapshot**
// (Gemini reports one per chunk, as do some OpenAI-compatible endpoints; Claude/GPT-5,
// aggregated by AgentHub, report only once at the end). Overwrite with the latest snapshot
// — never accumulate: summing snapshots chunk by chunk would inflate usage by roughly the
// number of chunks.
this.requestTokens = this.usageOnce(event.usage_metadata);
}
for (const item of event.content_items) {
switch (item.type) {
case "text": {
// Mirrors AgentHub baseClient text aggregation. A `fidelity.phase` marker can arrive
// as an increment with **empty text** (e.g. GPT-5's segment markers): a phase
// differing from the current segment's phase starts a new segment — providers split
// by phase when replaying history, so mixing segments would break fidelity. A
// `fidelity.signature` closes the segment: any further content starts a new one.
// On merge, fidelity keys accumulate ({...current, ...incoming}).
const curPhase = (this.textFidelity as { phase?: unknown } | undefined)?.phase ?? null;
const inPhase = (item.fidelity as { phase?: unknown } | undefined)?.phase ?? null;
const signatureClosed =
(this.textFidelity as { signature?: unknown } | undefined)?.signature != null;
if (
(signatureClosed && (item.text || hasFidelity(item.fidelity))) ||
(inPhase != null && inPhase !== curPhase)
) {
yield* this.flushThinking("completed");
yield* this.flushText("completed");
}
if (hasFidelity(item.fidelity)) {
this.textFidelity = { ...this.textFidelity, ...item.fidelity };
}
if (!item.text) break;
// Type boundary: before a text segment starts, flush any unclosed thinking
// segment, so the complete-message order matches generation order (thinking → text).
// The boundary flush uses completed: that thinking segment's stop reason is "switched
// to text", not "Request ended".
yield* this.flushThinking("completed");
if (!this.textStarted) {
this.textStarted = true;
yield partialText("start");
}
this.textBuffer += item.text;
yield partialText("delta", item.text);
break;
}
case "thinking": {
// Mirrors AgentHub baseClient thinking aggregation: a thinking block is closed by its
// fidelity payload (Claude's signature_delta is empty text + fidelity{signature};
// redacted blocks carry sentinel text + fidelity; GPT-5 encrypted reasoning is empty
// text + fidelity{id, encrypted_content}), and **a run of equal fidelity is one
// block** — OpenAI-compatible clients stamp every delta with the same
// fidelity{reasoning_field}, which must not split blocks. Content arriving after a
// different fidelity is set starts a new block, so each block's fidelity stays
// independently faithful when history is replayed.
if (
hasFidelity(this.thinkingFidelity) &&
!fidelityEquals(this.thinkingFidelity, item.fidelity) &&
(item.thinking || hasFidelity(item.fidelity))
) {
yield* this.flushThinking("completed");
}
if (hasFidelity(item.fidelity)) this.thinkingFidelity = item.fidelity;
if (!item.thinking) break;
// Type boundary: before a thinking segment starts, flush any unclosed text
// segment, so the complete-message order matches generation order (text → thinking).
// The boundary flush uses completed: that text segment's stop reason is "switched to
// thinking", not "Request ended".
yield* this.flushText("completed");
if (!this.thinkingStarted) {
this.thinkingStarted = true;
yield partialThinking("start");
}
this.thinkingBuffer += item.thinking;
yield partialThinking("delta", item.thinking);
break;
}
case "partial_tool_call": {
// Some providers' argument deltas don't carry a tool_call_id; attribute them to the most
// recently opened tool call. If there's no open tool call yet, skip — don't fabricate a
// tool_call with an empty id.
const providerKey = item.tool_call_id || this.activeToolCallId;
if (!providerKey) break;
if (item.tool_call_id) this.activeToolCallId = item.tool_call_id;
const acc = this.ensureTool(providerKey, item.name);
if (item.name) acc.name = item.name;
if (hasFidelity(item.fidelity)) acc.fidelity = item.fidelity;
// Externally always use the uniqueness-resolved id (with a `#n` suffix on provider id collisions), matching the complete tool_call.
if (!this.toolStarted.has(acc.toolCallId)) {
// Type boundary: before a new tool_call starts, flush any unclosed thinking/text
// segment. Only triggered on the tool's first delta (!toolStarted.has); continuation
// deltas (including id-less increments attributed via activeToolCallId) don't re-trigger the flush.
yield* this.flushThinking("completed");
yield* this.flushText("completed");
this.toolStarted.add(acc.toolCallId);
yield partialToolCall({
eventType: "start",
name: acc.name,
toolCallId: acc.toolCallId,
});
}
// Only emit a delta when there's an arguments increment (an empty increment carries
// no information, consistent with how empty text/thinking increments are handled).
// The delta doesn't repeat the name — leave it blank; tool identity is established by
// the start segment and tool_call_id.
if (item.arguments) {
acc.argsBuffer += item.arguments;
yield partialToolCall({
eventType: "delta",
name: "",
arguments: item.arguments,
toolCallId: acc.toolCallId,
});
}
break;
}
case "tool_call": {
// Complete tool-call content item: record the authoritative name/arguments and
// **eagerly emit** the tool's partial(stop) and complete tool_call. This lets the
// engine start approval/execution as soon as the first tool arrives, without waiting
// for the whole turn to finish — key for async/incremental tool calls (see comment
// #24). An empty id is invalid; skip it.
if (!item.tool_call_id) break;
// The model may think/output text before calling a tool: flush any buffered
// thinking/text complete messages before emitting the complete tool_call, so the
// complete-message order is thinking → text → tool_call. finish_reason isn't known
// yet here; the boundary flush uses completed, with the stop reason attributed to
// the tool_call itself.
yield* this.flushThinking("completed");
yield* this.flushText("completed");
// The same provider id already emitted a complete call and now another complete tool_call
// arrives: not a duplicate delivery but **another** call from a name-as-id provider (e.g.
// Gemini using the function name as id) — start a fresh accumulator, allocate a new unique id,
// and emit as usual; never drop it (otherwise parallel same-name calls in one turn would lack
// tool execution and paired output).
let acc = this.tools.get(item.tool_call_id);
if (!acc || acc.emitted) acc = this.createTool(item.tool_call_id, item.name);
acc.name = item.name;
acc.completeArgs = JSON.stringify(item.arguments ?? {});
if (hasFidelity(item.fidelity)) acc.fidelity = item.fidelity;
yield* this.emitCompleteTool(acc);
break;
}
// Other content items (image_url / inline_data / inline_thinking /
// tool_result / embedding) are not treated as model streaming output.
default:
break;
}
}
}
/**
* Stream-end finalization: first `yield` the `stop` and complete message for the text/thinking
* segments, then backfill partial(stop) + complete tool_call for tool_calls that **haven't
* been emitted eagerly yet** (i.e. the fallback case — no complete tool_call content item was
* received, only deltas). Tools already emitted eagerly in `pushEvent` are not repeated here.
*/
*finish(): Generator<OmniMessage> {
const stopReason = this.omniStopReason();
// 1. Close out the **last** unflushed thinking / text segment (earlier segments were already
// flushed at their respective type boundaries or before the first tool_call).
// Since boundaries already flush, at most one buffer is non-empty here, so call
// order doesn't matter — the other call is a no-op; the final finish_reason is used as
// the stop_reason here.
yield* this.flushThinking(stopReason);
yield* this.flushText(stopReason);
// 2. Fallback path: for tools that never received a complete tool_call content item,
// backfill using the accumulated deltas.
for (const id of this.toolOrder) {
if (!id) continue; // Defensive: an invalid tool call with an empty id (its result can't be routed).
const acc = this.tools.get(id)!;
if (acc.emitted) continue; // Already emitted eagerly in pushEvent; don't repeat.
yield* this.emitCompleteTool(acc);
}
}
/**
* Interruption finalization: even when interrupted or on error, close the structure
* as `start → delta → stop → complete message`. Closes any unclosed thinking/text segments and
* backfills their complete messages, then backfills partial(stop) + complete tool_call for
* tool_calls that only have deltas and were never emitted eagerly. All backfilled messages are
* uniformly tagged with the interruption `stopReason` (`aborted` / `retryable` / `fatal`), to
* distinguish them from normal completion (`completed`).
*
* Differs from `finish`: doesn't read `finish_reason`, and doesn't produce `token_usage` (an
* interrupted Request has no usage to report); backfilled incomplete tool_calls carry the
* interruption stop_reason, so `context_engine` won't dispatch them for execution.
*/
*finishInterrupted(stopReason: StopReason): Generator<OmniMessage> {
yield* this.flushThinking(stopReason);
yield* this.flushText(stopReason);
for (const id of this.toolOrder) {
if (!id) continue;
const acc = this.tools.get(id)!;
if (acc.emitted) continue; // A tool_call already emitted eagerly keeps its `tool_call` semantics and is left unchanged.
yield* this.emitCompleteTool(acc, stopReason);
}
}
/**
* Eagerly emits the finalization of a tool_call: partial(stop) (without name) + complete
* tool_call. Marks `emitted` to prevent duplicate emission in finish. `stopReason` defaults to
* `completed` (a normal request); when called during interruption finalization
* (`finishInterrupted`), the interruption reason is passed in, letting `context_engine`
* distinguish "a real tool request" from "an incomplete tool_call backfilled to close the
* structure on interruption" by stop_reason, and dispatch only the former.
*/
private *emitCompleteTool(
acc: ToolCallAccumulator,
stopReason: StopReason = "completed",
): Generator<OmniMessage> {
acc.emitted = true;
if (this.toolStarted.has(acc.toolCallId)) {
// stop doesn't carry name (tool identity is established by start and tool_call_id).
yield partialToolCall({
eventType: "stop",
name: "",
toolCallId: acc.toolCallId,
stopReason,
});
}
yield toolCall({
name: acc.name,
arguments: acc.completeArgs ?? acc.argsBuffer,
toolCallId: acc.toolCallId,
stopReason,
...(acc.fidelity !== undefined ? { fidelity: acc.fidelity } : {}),
});
// activeToolCallId holds the provider id key (used to attribute id-less deltas); reset it by providerKey.
if (this.activeToolCallId === acc.providerKey) {
this.activeToolCallId = null;
}
}
/**
* Closes out the currently buffered thinking segment and appends the complete thinking
* message, then clears the buffer and resets the start flag. May be called before eagerly
* emitting the first complete tool_call (`pushEvent`) or at stream end (`finish`), which
* guarantees the complete-message order is thinking → text → tool_call.
*
* "Flush then reset" rather than a one-shot guard: once the buffer is cleared, a repeated call
* is a no-op (each segment is emitted exactly once); if the model outputs new thinking after a
* tool_call (interleaved/multi-segment models), that new segment accumulates again and gets
* correctly flushed at the next tool_call or `finish`, without being lost.
*
* The complete thinking message passes through the given stop_reason just like partial(stop)
* (aligned with flushText — streamed concatenation == complete message): at type boundaries the
* caller passes completed (the stop reason belongs to the following tool_call/text);
* finish/finishInterrupted follow the actual end reason.
*/
private *flushThinking(stopReason: StopReason): Generator<OmniMessage> {
if (this.thinkingStarted) {
yield partialThinking("stop", "", stopReason);
this.thinkingStarted = false;
}
// A thinking block with empty text but a fidelity payload (GPT-5 encrypted reasoning)
// still produces a complete message — the fidelity is required when replaying history.
if (this.thinkingBuffer || hasFidelity(this.thinkingFidelity)) {
yield thinkingMessage(this.thinkingBuffer, stopReason, this.thinkingFidelity);
this.thinkingBuffer = "";
this.thinkingFidelity = undefined;
}
}
/**
* Closes out the currently buffered text segment and appends the complete text message, then
* clears the buffer and resets the start flag. Uses the same "flush then reset" approach as
* `flushThinking` to support new text segments after a tool_call. When emitted before the
* first complete tool_call, finish_reason is unknown and the caller passes completed; when
* emitted in `finish`, the final `omniStopReason()` is passed (consistent with prior behavior).
*/
private *flushText(stopReason: StopReason): Generator<OmniMessage> {
if (this.textStarted) {
yield partialText("stop", "", stopReason);
this.textStarted = false;
}
// A text segment with empty text but a fidelity payload (e.g. Gemini carrying a
// thoughtSignature on a text part, or a GPT-5 phase marker with no text) still produces a
// complete message — aligned with flushThinking, so the fidelity isn't lost or leaked into
// a later segment just because the buffer is empty.
if (this.textBuffer || hasFidelity(this.textFidelity)) {
yield assistantText(this.textBuffer, stopReason, this.textFidelity);
this.textBuffer = "";
this.textFidelity = undefined;
}
}
/** Whether finish_reason (the terminal event) has been received: signals a fully delivered response (see the defensive branch in streamGenerate). */
sawFinishReason(): boolean {
return this.finishReason !== null;
}
/** Token usage for this request (read after finish). */
getRequestTokens(): TokenCounts {
return this.requestTokens;
}
private ensureTool(providerKey: string, name: string): ToolCallAccumulator {
return this.tools.get(providerKey) ?? this.createTool(providerKey, name);
}
/**
* Create an accumulator for a new call: the emitted id is made unique via the Session-level registry
* (kept as-is when the provider id is free, with a `#n` suffix on collision). Creating again under the
* same provider id (another call with a duplicate id) replaces the old Map entry — the old call has
* finished emitting, so later inbound events are attributed to the latest call.
*/
private createTool(providerKey: string, name: string): ToolCallAccumulator {
const acc: ToolCallAccumulator = {
name: name ?? "",
argsBuffer: "",
completeArgs: null,
toolCallId: this.toolCallIds.allocate(providerKey),
providerKey,
fidelity: undefined,
emitted: false,
};
this.tools.set(providerKey, acc);
this.toolOrder.push(providerKey);
return acc;
}
private usageOnce(usage: UsageMetadata): TokenCounts {
return usageToTokenCounts(usage);
}
/**
* Converts an AgentHub finish_reason into an OmniMessage stop_reason. "stop",
* "tool_call", and null are treated as completed; length or unknown reasons map to
* `fatal` — the request itself committed, so nothing retries, but the segment ended
* abnormally in a way only a human can fix (raise the output cap), which is what the
* abnormal terminal label tells the render layers.
*/
private omniStopReason(): StopReason {
if (
this.finishReason == null ||
this.finishReason === "stop" ||
this.finishReason === "tool_call"
) {
return "completed";
}
return "fatal";
}
}
/**
* One-shot translation: folds a batch of UniEvents into an OmniMessage sequence (including
* complete messages and token_usage). A pure function for easy unit testing; the live streaming
* path is wired up by `GenerativeModel.streamGenerate`.
*
* @param events The event sequence
* @param sessionTokensBefore The session's cumulative tokens before this translation (used to produce token_usage.session)
* @returns `{ messages, requestTokens, sessionTokens }`
*/
export function translateEvents(
events: UniEvent[],
sessionTokensBefore: TokenCounts = emptyTokenCounts(),
): {
messages: OmniMessage[];
requestTokens: TokenCounts;
sessionTokens: TokenCounts;
} {
const translator = new EventTranslator();
const out: OmniMessage[] = [];
for (const event of events) {
for (const msg of translator.pushEvent(event)) out.push(msg);
}
for (const msg of translator.finish()) out.push(msg);
const requestTokens = translator.getRequestTokens();
const sessionTokens = addTokenCounts(sessionTokensBefore, requestTokens);
out.push(tokenUsage(sessionTokens, requestTokens));
return { messages: out, requestTokens, sessionTokens };
}
// ---------------------------------------------------------------------------
// Retry policy
// ---------------------------------------------------------------------------
/**
* A fuller error string than `err.message` for the LLM request outcome. Node's `fetch`
* wraps the real transport failure as `TypeError: terminated` and puts the actual reason
* on `err.cause` (a socket close, `ECONNRESET`, a provider stream abort, …); taking only
* `.message` throws that away and leaves a bare, unactionable "terminated". This walks the
* `cause` chain and appends each level's message and error `code`, so it surfaces as e.g.
* "terminated: other side closed (UND_ERR_SOCKET)". Segments are de-duplicated and the
* chain walk guards against cycles; a non-Error cause tail (string/number) is still kept.
*/
export function describeError(error: unknown): string {
if (!(error instanceof Error)) return String(error);
const parts: string[] = [];
const seen = new Set<unknown>();
let cur: unknown = error;
while (cur instanceof Error && !seen.has(cur)) {
seen.add(cur);
const code = (cur as { code?: unknown }).code;
let piece = cur.message || cur.name;
if (typeof code === "string" && code && !piece.includes(code)) piece = `${piece} (${code})`;
if (piece && !parts.includes(piece)) parts.push(piece);
cur = (cur as { cause?: unknown }).cause;
}
if (cur != null && !(cur instanceof Error)) {
const tail = String(cur);
if (tail && !parts.includes(tail)) parts.push(tail);
}
return parts.join(": ") || error.message || String(error);
}
/**
* Determines whether an error is an AgentHub / Provider "response delivered but unusable"
* parse or validation error. Two shapes (@prismshadow/agenthub 0.4.x):
*
* - A raw `SyntaxError` from `JSON.parse` on a response body;
* - AgentHub's own error classes: `ToolCallArgumentParseError` (streamed tool-call arguments
* are not valid JSON — e.g. a stream truncated mid-arguments) and `EmptyResponseError`
* (a completed response carrying thinking only, which cannot be replayed).
*
* In every case the turn was **not committed** to AgentHub history: this is not an
* auth/parameter failure but an incomplete LLM Request, and should end `retryable` and
* be handed to the engine to reconnect and retry. Judged by exception type with a `name`
* fallback (covers cross-realm or deserialization-reconstructed errors), probing down the
* `cause` chain for wrapped errors.
*/
export function isMalformedJsonParseError(error: unknown): boolean {
if (error == null) return false;
if (
error instanceof SyntaxError ||
error instanceof ToolCallArgumentParseError ||
error instanceof EmptyResponseError
) {
return true;
}
const err = error as { name?: string; cause?: unknown };
if (
err.name === "SyntaxError" ||
err.name === "ToolCallArgumentParseError" ||
err.name === "EmptyResponseError"
) {
return true;
}
if (err.cause && err.cause !== error) {
return isMalformedJsonParseError(err.cause);
}
return false;
}
/**
* Determines whether an error is AgentHub's "incomplete stream" validation error: when a
* server/proxy terminates the stream early **cleanly** at an event boundary (no network error
* thrown), AgentHub's `_validateLastEvent` reports the missing or incomplete final event as a
* plain `Error` ("Streaming response yielded no events" / "Last event must carry
* usage_metadata|finish_reason"). This is not an auth/parameter failure but an incomplete LLM
* Request, and should end `retryable` and be handed to the engine to reconnect and retry.
* AgentHub doesn't provide an error type for this, so we match by message prefix
* (verified against @prismshadow/agenthub 0.4.x), probing down the `cause` chain.
*/
export function isIncompleteStreamError(error: unknown): boolean {
if (error == null) return false;
const err = error as { message?: string; cause?: unknown };
const msg = err.message ?? "";
if (
msg.startsWith("Streaming response yielded no events") ||
msg.startsWith("Last event must carry")
) {
return true;
}
if (err.cause && err.cause !== error) {
return isIncompleteStreamError(err.cause);
}
return false;
}
/** Credentials/authentication error codes and types (OpenAI-compatible bodies / SDK errors). */
const AUTH_CODES: ReadonlySet<string> = new Set([
"invalid_api_key",
"authentication_error",
"unauthorized",
]);
/**
* Walks the error's `cause` chain (cycle-safe, same approach as `describeError`) applying
* `probe` to each level; true as soon as one level matches. Higher layers routinely wrap the
* real failure (Node fetch puts the transport error on `cause`), so single-level checks miss it.
*/
function anyInCauseChain(error: unknown, probe: (level: object) => boolean): boolean {
const seen = new Set<unknown>();
let cur: unknown = error;
while (cur != null && typeof cur === "object" && !seen.has(cur)) {
seen.add(cur);
if (probe(cur)) return true;
cur = (cur as { cause?: unknown }).cause;
}
return false;
}
/**
* Provider error-code signals at one level of an error: `code` and `type` on the error
* itself (the OpenAI SDK exposes the parsed body's code directly), plus the parsed body
* under `error` — both the OpenAI shape (`err.error.code`) and the Anthropic SDK shape,
* whose `error` property holds the whole response body (`err.error.error.code`). Auth
* signals ride on `type` in some bodies, so both fields are collected.
*/
function providerSignals(level: object): string[] {
const err = level as {
code?: unknown;
type?: unknown;
error?: { code?: unknown; type?: unknown; error?: { code?: unknown; type?: unknown } };
};
const body = typeof err.error === "object" && err.error !== null ? err.error : undefined;
const inner = typeof body?.error === "object" && body.error !== null ? body.error : undefined;
const signals = [err.code, body?.code, inner?.code, err.type, body?.type, inner?.type];
return signals.filter((v): v is string => typeof v === "string");
}
/**
* Determines whether an error is a credentials/authentication failure — the one class an
* in-run retry can never fix: the request keeps going out with the same dead credential.
* Only the model REFERENCE is fixed at Session creation; the credential is read from the
* current Project config whenever the Session loads, so the fix is updating that model's
* API key (Models page) — after which the Session can continue — not retrying. Signals:
* HTTP 401 (any), a known auth code/type (on the error, its `cause` chain, or the parsed
* provider body), or the SDK error class name `AuthenticationError` (OpenAI / Anthropic
* SDKs). Deliberately narrow and explicit — this detector names the credential class
* precisely, so nothing heuristic belongs in it: a bare 403 carries no credential signal
* and is left to `isFatalProviderRejection` (any other 4xx is fatal there, so it fails
* fast instead of riding the retry ladder).
*/
export function isAuthenticationError(error: unknown): boolean {
return anyInCauseChain(error, (level) => {
const err = level as { status?: unknown; statusCode?: unknown; name?: unknown };
if (err.status === 401 || err.statusCode === 401) return true;
if (providerSignals(level).some((c) => AUTH_CODES.has(c))) return true;
const name = typeof err.name === "string" ? err.name : "";
return name === "AuthenticationError" || level.constructor?.name === "AuthenticationError";
});
}
/**
* Determines whether an error is AgentHub's `UnsupportedParameterError` for `fast_mode`:
* clients without a fast tier (and claude5 on Bedrock / Claude 4.6 ids) reject the
* parameter by throwing **before any network I/O**, so with the same frozen config the
* failure is deterministic — retrying the identical request can never succeed. Deliberately
* scoped to `parameter === "fast_mode"`: other UnsupportedParameterError sources keep the
* default `retryable` classification (and the engine's ladder) unchanged. Judged by exception
* type with a `name` fallback (covers cross-realm or deserialization-reconstructed errors,
* same approach as isMalformedJsonParseError), probing down the `cause` chain.
*/
export function isFastModeUnsupportedError(error: unknown): boolean {
return anyInCauseChain(error, (level) => {
const err = level as { name?: unknown; parameter?: unknown };
const isUnsupportedParameter =
level instanceof UnsupportedParameterError || err.name === "UnsupportedParameterError";
return isUnsupportedParameter && err.parameter === "fast_mode";
});
}
/**
* Actionable tail appended to a fast-mode rejection's error text: the raw AgentHub message
* says what is unsupported ("Kimi does not support fast mode."); this says where the switch
* lives. Exported for the outcome-classification tests.
*/
export const FAST_MODE_UNSUPPORTED_GUIDANCE =
"Fast mode is enabled for this model; turn it off in the model settings to use this model.";
/**
* Whether any level of the error chain is an HTTP **429** — the provider's rate limit, or a
* quota that is exhausted for now. The outcome stays `retryable` (a short burst is exactly
* what the reconnect ladder is for), but it is classified apart from a transport drop: the
* code is what lets a host tell "the wire blipped" from "this model has no quota left", and
* act on the second.
*/
export function isRateLimitError(error: unknown): boolean {
return anyInCauseChain(error, (level) => {
const err = level as { status?: unknown; statusCode?: unknown };
const status = typeof err.status === "number" ? err.status : err.statusCode;
return status === 429;
});
}
/**
* Determines whether an error is a definitive provider rejection of the request itself —
* the `fatal` detector for errored requests. Only an explicit HTTP client-error status
* counts: 4xx minus 408 (request timeout) and 429 (rate limit), which are transient by
* definition. Deliberately an allowlist of certainty, probed down the `cause` chain
* (SDKs wrap the real response error): everything this misses stays `retryable`, because
* retrying a genuinely fatal error costs one ladder and ends with the same message, while
* refusing to retry a transient one destroys the turn. Authentication is checked
* separately (`isAuthenticationError`) and first — a 401 is fatal through that path with
* its more specific message handling.
*/
export function isFatalProviderRejection(error: unknown): boolean {
return anyInCauseChain(error, (level) => {
const err = level as { status?: unknown; statusCode?: unknown };
const status = typeof err.status === "number" ? err.status : err.statusCode;
if (typeof status !== "number") return false;
return status >= 400 && status <= 499 && status !== 408 && status !== 429;
});
}
// ---------------------------------------------------------------------------
// GenerativeModel
// ---------------------------------------------------------------------------
/**
* A stateful LLM object attached to a Session. AgentHub's `streamingResponseStateful` maintains
* conversation history internally; this class is only responsible for protocol translation,
* streaming aggregation, and token accumulation. **It never retries internally** — retries are
* handled by `context_engine`.
*/
export class GenerativeModel implements LLMInterface {
private client: AutoLLMClient;
private readonly clientOptions: Omit<ConstructorParameters<typeof AutoLLMClient>[0], "apiKey">;
private currentApiKey: string | undefined;
private readonly resolveApiKey: (() => Promise<string | undefined>) | undefined;
private readonly uniConfig: UniConfig;
/**
* Construction-time default thinking level. Kept **out of the frozen uniConfig**: the
* effective level is resolved per request (`params.thinkingLevel ?? default`), so a turn can
* override it without rebuilding the model object — the thinking level is a per-turn
* parameter, not a Session invariant.
*/
private readonly defaultThinkingLevel: ThinkingLevelName | undefined;
/** Streaming idle timeout (milliseconds); <= 0 disables it. A timeout is treated as needing reconnection. */
private readonly requestTimeoutMs: number;
/**
* tool_call_id uniqueness registry (see tool-call-ids.ts): when a name-as-id provider (e.g. Gemini)
* calls the same tool repeatedly, it assigns a `#n` suffix to later calls so engine pairing and the
* frontend tool cards don't collide on id. Injected via config so it can be shared across the new
* instance rebuilt on compaction; defaults to a fresh one.
*/
private readonly toolCallIds: ToolCallIdAllocator;
/** Configured output cap (`GenerativeModelConfig.maxTokens`); the per-request clamp derives the effective cap from it (see effectiveMaxTokens). */
private readonly configuredMaxTokens: number | undefined;
/** Per-request image cap (`GenerativeModelConfig.maxImages`); undefined = the seller publishes none (see fitReplayedImages). */
private readonly maxImages: number | undefined;
/** Per-request video cap (`GenerativeModelConfig.maxVideos`); undefined = the session's model takes no video (see fitReplayedVideos). */
private readonly maxVideos: number | undefined;
/** Model context window; `undefined` when unconfigured (or implausibly small, see resolveContextWindow) — the per-request clamp then disables itself rather than clamp against an assumption. */
private readonly contextWindow: number | undefined;
/**
* Construction-time estimate of the fixed request prefix (system prompt + tool schemas):
* the prefix is part of every request but never part of `newMessages`. Seeds
* `lastRequestTotal`, and re-seeds it in `setHistory`.
*/
private readonly baseInputTokens: number;
/**
* The current context's size: the most recent completed request's real
* `token_usage.request.total` once one exists (a measured total always includes the
* prefix, so it can only refine the seed upward from real data; providers stripping
* historical thinking only make it an overestimate, the safe direction), the
* `baseInputTokens` seed before that, plus the replayed-history estimate after
* `setHistory`. The next request's input is this figure plus the newly appended
* messages.
*/
private lastRequestTotal: number;
/** Last hard-clamped cap already warned about on stderr (dedupe: retries reuse the same estimate and would repeat the identical line). */
private lastWarnedCap: number | undefined;
constructor(config: GenerativeModelConfig) {
// Omit apiKey / baseUrl when undefined, letting AgentHub read them from environment
// variables. clientType determines which protocol to speak (`openai-chat` means OpenAI
// Chat Completions compatible; the bare `openai` spelling is a deprecated upstream alias);
// when omitted, AgentHub infers it from model_id, so it only needs to be specified
// explicitly for custom-named models. `defaultHeaders` carries the app attribution the
// configured endpoint reads (see attributionHeaders); AgentHub hands it to every request
// the routed client makes, and endpoints with no attribution scheme get no extra headers
// at all. The Session's id rides along for the schemes that name the conversation rather
// than the app; without one, attributionHeaders mints a fresh id. Computed once and kept
// in clientOptions, so every request of this instance — across a rotated credential too —
// names the same conversation.
const headers = attributionHeaders(config.baseUrl, config.sessionId);
this.clientOptions = {
model: config.modelId,
...(config.baseUrl !== undefined ? { baseUrl: config.baseUrl } : {}),
...(config.clientType !== undefined ? { clientType: config.clientType } : {}),
...(headers ? { defaultHeaders: headers } : {}),
};
this.currentApiKey = config.apiKey;
this.resolveApiKey = config.resolveApiKey;
this.client = new AutoLLMClient({
...this.clientOptions,
...(this.currentApiKey !== undefined ? { apiKey: this.currentApiKey } : {}),
});
this.uniConfig = buildUniConfig(config);
this.defaultThinkingLevel = config.thinkingLevel;
this.requestTimeoutMs = config.requestTimeoutMs ?? 300000;
this.toolCallIds = config.toolCallIds ?? new ToolCallIdAllocator();
this.configuredMaxTokens = config.maxTokens;
this.maxImages =
config.maxImages !== undefined && config.maxImages > 0
? Math.floor(config.maxImages)
: undefined;
this.maxVideos =
config.maxVideos !== undefined && config.maxVideos > 0
? Math.floor(config.maxVideos)
: undefined;
this.contextWindow = resolveContextWindow(config.contextWindow);
this.baseInputTokens =
approximateTokens(config.systemPrompt ?? "") +
approximateTokens(JSON.stringify(this.uniConfig.tools ?? []));
this.lastRequestTotal = this.baseInputTokens;
}
/** Swap a rotated credential into AgentHub without losing its stateful conversation. */
private async refreshApiKey(): Promise<void> {
if (this.resolveApiKey === undefined) return;
const nextApiKey = await this.resolveApiKey();
if (nextApiKey === undefined || nextApiKey === this.currentApiKey) return;
const history = this.client.getHistory();
const next = new AutoLLMClient({ ...this.clientOptions, apiKey: nextApiKey });
if (history.length > 0) next.setHistory(history);
this.client = next;
this.currentApiKey = nextApiKey;
}
/**
* The UniConfig for one request: the shared frozen config plus this request's effective
* thinking level (per-request override, else the construction-time default; neither → the
* key stays off the wire, preserving the provider default) and this request's effective
* output cap (the window-derived clamp below; equal to the configured cap for big-window
* models, so the frozen value goes out unchanged).
*/
private requestConfig(
override: ThinkingLevelName | undefined,
newMessages: OmniMessage[],
): UniConfig {
const thinking = mapThinkingLevel(override ?? this.defaultThinkingLevel);
const maxTokens = this.effectiveMaxTokens(newMessages);
let cfg = this.uniConfig;
if (thinking !== undefined) cfg = { ...cfg, thinking_level: thinking };
if (maxTokens !== undefined && maxTokens !== this.uniConfig.max_tokens) {
cfg = { ...cfg, max_tokens: maxTokens };
}
return cfg;
}
/**
* Per-request output cap: `min(configured max_tokens, context_window − estimated input −
* safety margin)`, floored — recomputed for every request (compaction requests included:
* they run through the same path, exactly when the context is largest) from the freshest
* input knowledge this object has: the last completed request's real `token_usage` total
* plus a character-heuristic estimate of the newly appended messages (no tokenizer;
* see context-limits.ts). Fixes issue #218: a fixed cap (the seeded 32000) that ignores
* the input made every request to a small-window model (e.g. a 32k vLLM) fail provider
* validation with a non-retryable 400.
*
* Interplay with compaction: the engine's compaction threshold is derived at
* `context_window − COMPACTION_HEADROOM` (see effectiveMaxContextLength), so under normal
* operation the context is summarized before the remaining window ever nears the
* MIN_OUTPUT_TOKENS floor; the floor only binds when compaction is disabled or the window
* is misconfigured, where a deterministic small cap beats a provider rejection.
* `undefined` = no positive cap configured: the key stays off the wire and the provider's
* own remaining-window default applies (the existing `-1` contract). Without a configured
* `contextWindow` the clamp is off entirely (see effectiveMaxOutputTokens).
*
* A hard clamp — the derived cap dropping below half the configured one — is announced
* once per distinct value on stderr with the numbers involved, so a shaved `max_tokens`
* is diagnosable from the log instead of surfacing only as an opaque failed/short turn.
*/
private effectiveMaxTokens(newMessages: OmniMessage[]): number | undefined {
const estimatedInput = this.lastRequestTotal + approximateMessagesTokens(newMessages);
const derived = effectiveMaxOutputTokens(
this.configuredMaxTokens,
this.contextWindow,
estimatedInput,
);
if (
derived !== undefined &&
this.configuredMaxTokens !== undefined &&
derived < this.configuredMaxTokens / 2 &&
derived !== this.lastWarnedCap
) {
this.lastWarnedCap = derived;
process.stderr.write(
`[penguin] output cap clamped hard: max_tokens ${this.configuredMaxTokens} -> ${derived} ` +
`(context_window ${this.contextWindow}, estimated input ${estimatedInput})\n`,
);
}
return derived;
}
/**
* Streaming generation (a single attempt, no internal retry). Merges
* `params.newMessages` into one UniMessage to issue a stateful request, translating streamed
* UniEvents into OmniMessages.
*
* **Never throws to `context_engine`**: whether it ends normally or is interrupted/errors out,
* every `partial_*` segment is closed as `start → delta → stop → complete message`,
* and the terminal state is then returned as `LLMOutcome`:
* - **Normal completion**: `finish()` closes out and produces `token_usage` (usage is only
* produced in this case) → `completed`;
* - **User interruption**: `finishInterrupted("aborted")` closes out, produces no usage →
* `aborted`;
* - **Fatal failure** — a definitive provider 4xx rejection (`isFatalProviderRejection`),
* a credentials failure (`isAuthenticationError`), a fast-mode rejection
* (`isFastModeUnsupportedError`, thrown deterministically before any network I/O when
* this config enables `fast_mode` on a model without a fast tier), or input that never
* assembled into a request: `finishInterrupted("fatal")` closes out, produces no usage
* → `fatal` (carrying `errorMessage`), which the engine stops the run on instead of
* retrying — the identical request can never succeed, so the ladder would only delay
* the actionable message;
* - **Every other failure** — idle timeout, network/transport drops, 408/429/5xx,
* AgentHub parse errors and truncated streams, and anything unclassifiable:
* `finishInterrupted("retryable")` closes out, produces no usage → `retryable`
* (carrying `errorMessage` when a concrete error was caught), reconnected by
* `context_engine` within the same run. Unclassifiable errors stay retryable on
* purpose: the fatal detector is an allowlist, and a gateway phrasing a transient
* fault its own way must keep its retries.
*
* Timeout detection: the idle timer resets on every event received; once idle exceeds
* `requestTimeoutMs`, the underlying stream is aborted and handled as needing reconnection
* (merged with user interruption into a single internal AbortController).
*/
async *streamGenerate(
params: GenerativeModelParameters,
): AsyncGenerator<OmniMessage, LLMOutcome> {
const userSignal = params.signal;
// Already interrupted before issuing: no streaming segment has been opened, so nothing to close out.
if (userSignal?.aborted) return { status: "aborted" };
// Input merging is placed inside a guarded block: build failures such as empty input /
// mixed roles / argument JSON collapse to a fatal outcome — the same input can never
// assemble on a retry, so burning the reconnect ladder would only delay the message —
// and never throw to context_engine.
let uniMessage: UniMessage;
try {
uniMessage = mergeOmniToUniMessage(params.newMessages);
} catch (err) {
return { status: "fatal", errorCode: "invalid_input", errorMessage: describeError(err) };
}
const translator = new EventTranslator(this.toolCallIds);
// Merges "user interruption" and "idle timeout" into a single internal AbortController: either triggering aborts the underlying stream.
const ac = new AbortController();
const onUserAbort = (): void => ac.abort();
userSignal?.addEventListener("abort", onUserAbort, { once: true });
let timedOut = false;
let timer: ReturnType<typeof setTimeout> | null = null;
const clearTimer = (): void => {
if (timer) {
clearTimeout(timer);
timer = null;
}
};
const armTimer = (): void => {
if (this.requestTimeoutMs <= 0) return; // Timeout disabled
clearTimer();
timer = setTimeout(() => {
timedOut = true;
ac.abort();
}, this.requestTimeoutMs);
};
/**
* Settles as soon as the run must stop, whatever upstream is doing: the user aborted, or
* the idle timer fired (both go through `ac`). `it.next()` is raced against this because
* an upstream that does not honour its AbortSignal leaves that promise pending **forever**
* — and once `ac` is aborted the idle timer's own `ac.abort()` is a no-op, so nothing is
* left to unwedge the loop. Observed against Kimi: pressing Stop mid-request left the
* Session running with no way to send, compact or interrupt it again, short of a restart.
*
* The pre-loop `userSignal?.aborted` check below covers the *other* half of this (aborting
* while suspended at a `yield`); it cannot help here, since the loop never gets back to it.
*/
const STOPPED = Symbol("stopped");
const stopped: Promise<typeof STOPPED> = new Promise((resolve) => {
if (ac.signal.aborted) {
resolve(STOPPED);
return;
}
ac.signal.addEventListener("abort", () => resolve(STOPPED), { once: true });
});
// Terminal-state classification: retryable (transport/parse failures) / fatal (definitive
// rejections) / aborted (user). null means it ended normally.
let outcome: LLMOutcome | null = null;
// The upstream iterator, kept outside the try so the finally below can close it: a consumer
// that leaves before the stop would otherwise leave generation running.
let it: AsyncIterator<UniEvent> | undefined;
try {
await this.refreshApiKey();
// The seller counts images per request, not per message: a conversation that read five
// pictures would fail every later turn with 400 media_count_exceeded, so the oldest
// replayed ones — and, when the input alone overflows, the input's own oldest ones — are
// dropped from THIS request (the history and Trace keep them).
it = this.openStream(
this.fitReplayedVideos(this.fitReplayedImages(uniMessage)),
ac.signal,
this.requestConfig(params.thinkingLevel, params.newMessages),
)[Symbol.asyncIterator]();
for (;;) {
// The interruption check must happen **before pulling from upstream**: the user may
// interrupt while this generator is suspended at the `yield` below (the typical case —
// the engine is blocked on `await approve(tc)` waiting for human approval). By then,
// onUserAbort has already called `ac.abort()`, cutting off the upstream stream; when the
// consumer pulls again and we come back here to call `it.next()` on an **already-aborted
// stream**, that promise will never settle. The idle timer can't save us either: once it
// fires, it just calls `ac.abort()` again (already aborted, a no-op), and the pending
// `it.next()` still hangs forever. The consequence is that `run` never closes out and the
// Session stays stuck running forever — after interruption, it can neither send messages
// nor compact.
if (userSignal?.aborted) {
outcome = { status: "aborted" };
break;
}
// Timing runs **only while waiting on an upstream event** (excluding consumer/yield
// time), measuring upstream idleness — this avoids a slow consumer (e.g. a slow Trace
// sink) falsely triggering the timeout.
armTimer();
let res: IteratorResult<UniEvent> | typeof STOPPED;
try {
res = await Promise.race([it.next(), stopped]);
} finally {
clearTimer();
}
if (res === STOPPED) {
// Upstream never settled after the abort. Cleanup below is best effort and does
// not wait on a stream that may also ignore return(). Classify by trigger.
outcome = userSignal?.aborted
? { status: "aborted" }
: { status: "retryable", errorCode: "timeout" };
break;
}
if (res.done) break;
if (userSignal?.aborted) {
outcome = { status: "aborted" };
break;
}
for (const msg of translator.pushEvent(res.value)) yield msg;
}
} catch (error) {
// User interruption **takes priority**: even if the idle timer fires at the same time,
// it's classified as aborted (user intent outweighs a coincidental timeout).
if (userSignal?.aborted) {
outcome = { status: "aborted" };
} else if (timedOut) {
outcome = { status: "retryable", errorCode: "timeout" }; // Idle timeout -> needs reconnection
} else if (isMalformedJsonParseError(error) || isIncompleteStreamError(error)) {
// A response JSON parse error, or a cleanly truncated stream (AgentHub's final-event
// validation failed): both are an incomplete LLM Request — the turn was never
// committed, so the engine reconnects and retries. Checked before the fatal
// detectors: these carry no HTTP status, but the explicit branch keeps the message
// and the intent readable.
outcome = {
status: "retryable",
errorCode: "malformed",
errorMessage: describeError(error),
};
} else if (isAuthenticationError(error)) {
// Credentials failure: fatal — no retry can turn a rejected credential into a
// working one. The errorMessage tells the user to update this model's API key
// (only the model reference is fixed at Session creation; the credential is read
// from the current Project config on load), after which the Session continues.
outcome = { status: "fatal", errorCode: "auth", errorMessage: describeError(error) };
} else if (this.uniConfig.fast_mode === true && isFastModeUnsupportedError(error)) {
// Fast mode rejected by a model without a fast tier: AgentHub throws its
// UnsupportedParameterError before any network I/O, so with this object's frozen
// config the failure is deterministic — fatal, so the engine surfaces the message
// immediately instead of burning the reconnect ladder on a request that can never
// succeed. The guard on our own config keeps the special case honest: without
// fast_mode on the wire this error cannot be ours to explain.
outcome = {
status: "fatal",
errorCode: "unsupported",
errorMessage: `${describeError(error)} ${FAST_MODE_UNSUPPORTED_GUIDANCE}`,
};
} else if (isFatalProviderRejection(error)) {
// A definitive provider 4xx rejection (408/429 excluded): the identical request
// fails identically on every retry, so stop now with the provider's own message.
outcome = { status: "fatal", errorCode: "rejected", errorMessage: describeError(error) };
} else if ((error as { name?: string })?.name === "AbortError") {
outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
} else if (isRateLimitError(error)) {
// A 429: the provider's rate limit, or a quota with nothing left in it. Retryable
// like a transport drop — the ladder may well outlast a burst — but classified
// apart, because a host that has somewhere else to run the Session can tell the two
// apart and move it.
outcome = {
status: "retryable",
errorCode: "rate_limited",
errorMessage: describeError(error),
};
} else {
// Everything else — network/transport drops, 5xx, and anything unclassifiable —
// retries on the engine's ladder. The detail rides on the outcome so observability
// (request_end -> the Cost center's errors panel) shows the real reason behind a
// retried request.
outcome = { status: "retryable", errorCode: "network", errorMessage: describeError(error) };
}
} finally {
clearTimer();
userSignal?.removeEventListener("abort", onUserAbort);
// Leaving before the stop (at a yield, on an error, or abandoning an upstream that never
// settled) must cancel generation, even without a caller-owned signal, and close the
// upstream iterator to release it. Once the stop arrived nothing is cancelled: the turn
// is committed and billed, and the transport may still be reading the stream's end
// (MMSP yields the stop before `[DONE]`); closing it here would drop the connection.
if (!translator.sawFinishReason()) {
ac.abort();
// Do not let a broken or unresponsive cleanup replace the outcome or hold it open.
try {
void Promise.resolve(it?.return?.()).catch(() => undefined);
} catch {
// A synchronous cleanup failure is no more authoritative than a rejected promise.
}
}
}
// Defensive: when the underlying stream responds to an abort with a **graceful end (done)**
// rather than throwing, it must still be closed out as interrupted/timed out, and must not be
// misjudged as completed (priority matches the catch classification: user interruption >
// timeout). Exception: if finish_reason was already received before the stream ended (the
// response was fully delivered and AgentHub has already committed this turn into stateful
// history), close out as completed — if the interruption race lands exactly during the wait
// on the final next() and gets misjudged as aborted, the already-committed tool_use turn
// would be cleaned up by context_engine as "incomplete" flatten, losing the tool_result
// pairing; subsequent requests would all be rejected by the provider as an unanswered
// tool_use (400), and the engine has no fix-up path left that touches LLM history.
if (!outcome && !translator.sawFinishReason()) {
if (userSignal?.aborted) outcome = { status: "aborted" };
else if (timedOut) outcome = { status: "retryable", errorCode: "timeout" };
}
if (outcome) {
// Interrupted/errored: close any opened streaming segments and backfill the complete message, producing no token_usage.
const reason: StopReason = outcome.status === "completed" ? "retryable" : outcome.status;
for (const msg of translator.finishInterrupted(reason)) yield msg;
return outcome;
}
// Normal completion: backfill stop + the complete model_msg, and produce token_usage.
for (const msg of translator.finish()) yield msg;
const requestTokens = translator.getRequestTokens();
// The provider-measured context size, feeding the next request's output-cap clamp
// (see effectiveMaxTokens). Only a completed request updates it: an interrupted or
// failed attempt was never committed, so the context did not grow.
this.lastRequestTotal = requestTokens.total;
// The session series is not this object's business (its lifetime is one model context):
// the engine accumulates and stamps token_usage.session on every message it forwards.
// The request counts stand in for consumers running a GenerativeModel without an engine.
yield tokenUsage(requestTokens, requestTokens);
return { status: "completed" };
}
/**
* Injects the replayed history in one shot when resuming a Session: converts the complete
* OmniMessage history, grouped by adjacent same role, into
* AgentHub UniMessages and calls AgentHub's setHistory, so subsequent Requests continue from a
* history exactly matching the original conversation. **Called only once, on a fresh context
* object, during resumption**; not used during normal operation, where the incremental context
* is maintained by AgentHub itself.
* Docs: /docs/sessions-and-traces § "Session recovery".
*/
setHistory(history: OmniMessage[]): void {
if (history.length === 0) return;
// Resume seeding: register tool_call_ids already used in history into the uniqueness registry. A
// name-as-id provider (e.g. Gemini) only gets a new suffix when it calls the same tool again after
// resume, so it won't collide with the history tool cards the frontend already rendered.
for (const msg of history) {
const p = msg.payload as { type?: string; tool_call_id?: string };
if (p.type === "tool_call" && p.tool_call_id) {
this.toolCallIds.markUsed(p.tool_call_id);
}
}
// The injected history is context this object's first request carries without any
// token_usage having measured it: re-seed the input-size tracker (prefix + history
// estimate) so the output-cap clamp (effectiveMaxTokens) doesn't reason from an empty
// context on a resumed session. The first completed request replaces this with the
// real total.
this.lastRequestTotal = this.baseInputTokens + approximateMessagesTokens(history);
this.client.setHistory(groupHistoryToUniMessages(history));
}
/**
* Trims a request so it carries at most `maxImages` images in all, newest kept.
*
* The **input** is trimmed first and counts first: an interrupted turn replayed from Trace is
* resent as carry-over input, and that input may itself hold more images than the seller
* accepts (the exact case that used to fail every turn of a Session forever). The rest of the
* budget goes to the replayed history, newest first — see capHistoryImages for the
* per-message rule.
*
* Nothing happens without a configured cap, or when everything already fits. The input comes
* back trimmed (the very object when it fits) and the caller sends that; the client's history
* is replaced through setHistory rather than mutated, since the objects getHistory hands out
* are the client's own. A conversation's Trace and stored history are untouched — this
* decides what one request carries, not what the conversation is.
*/
private fitReplayedImages(input: UniMessage): UniMessage {
const limit = this.maxImages;
if (limit === undefined) return input;
const carried =
countMessageImages(input) > limit ? capMessageImages(input, limit, limit) : input;
const budget = Math.max(0, limit - countMessageImages(carried));
const history = this.client.getHistory();
if (history.length === 0) return carried;
const total = history.reduce((sum, msg) => sum + countMessageImages(msg), 0);
if (total <= budget) return carried;
this.client.setHistory(capHistoryImages(history, budget, limit));
return carried;
}
/**
* Trims a request so it carries at most `maxVideos` videos in all, newest kept — the video
* counterpart of {@link fitReplayedImages}, and needed for a harder reason than the seller's
* count: a provider takes one request body (Ark caps it at 64 MB) and a video is megabytes of
* base64, so a conversation replaying two videos would be refused for good once its history
* grows past the cap. Input first, then the replayed history, exactly as the image budget
* does it. Nothing happens without a cap, or when everything already fits.
*/
private fitReplayedVideos(input: UniMessage): UniMessage {
const limit = this.maxVideos;
if (limit === undefined) return input;
const carried =
countMessageVideos(input) > limit ? capMessageVideos(input, limit, limit) : input;
const budget = Math.max(0, limit - countMessageVideos(carried));
const history = this.client.getHistory();
if (history.length === 0) return carried;
const total = history.reduce((sum, msg) => sum + countMessageVideos(msg), 0);
if (total <= budget) return carried;
this.client.setHistory(capHistoryVideos(history, budget, limit));
return carried;
}
/**
* Opens the underlying AgentHub stream (a testing seam): defaults to
* `streamingResponseStateful`; unit tests can subclass and override this method, feeding in a
* controlled UniEvent stream to verify the outcome classification for timeout/network
* drop/interruption/error (without a real API). `config` is this request's resolved
* UniConfig (the shared frozen config plus the per-request thinking level).
*/
protected openStream(
uniMessage: UniMessage,
signal: AbortSignal,
config: UniConfig = this.uniConfig,
): AsyncIterable<UniEvent> {
return this.client.streamingResponseStateful({
message: uniMessage,
config,
signal,
});
}
}
// ---------------------------------------------------------------------------
// UniConfig pre-construction
// ---------------------------------------------------------------------------
/** Maps a ThinkingLevelName to the AgentHub ThinkingLevel enum; returns undefined if not found. */
export function mapThinkingLevel(name: ThinkingLevelName | undefined): ThinkingLevel | undefined {
if (name === undefined) return undefined;
const table: Record<ThinkingLevelName, ThinkingLevel> = {
none: ThinkingLevel.NONE,
low: ThinkingLevel.LOW,
medium: ThinkingLevel.MEDIUM,
high: ThinkingLevel.HIGH,
xhigh: ThinkingLevel.XHIGH,
max: ThinkingLevel.MAX,
};
return table[name];
}
/** Maps ToolDefinition[] to AgentHub ToolSchema[]. */
export function toolDefinitionsToSchemas(tools: ToolDefinition[]): ToolSchema[] {
return tools.map((tool) => ({
name: tool.name,
description: tool.description,
...(tool.parameters !== undefined ? { parameters: tool.parameters } : {}),
}));
}
/**
* Pre-builds UniConfig from GenerativeModelConfig (called once at construction time).
*
* When the tool list is empty (connectivity probe, bare/meta LLM, vision describer), `tools`
* is omitted entirely instead of set to `[]`: strict OpenAI-compatible servers (e.g. vLLM)
* reject an empty array with a 400 ("tools must not be an empty array"), and omission is the
* protocol equivalent. `tool_choice` is likewise never set — AgentHub only puts it on the wire
* when UniConfig defines it, and leaving it off preserves the protocol default ("auto" when
* tools are present).
*
* `thinkingLevel` is deliberately **not** baked in here: the effective level is resolved per
* request (`GenerativeModelParameters.thinkingLevel ?? the construction default`, see
* `GenerativeModel.requestConfig`), so a turn can override it on a live session.
*/
export function buildUniConfig(config: GenerativeModelConfig): UniConfig {
const uniConfig: UniConfig = {};
if (config.tools.length > 0) {
uniConfig.tools = toolDefinitionsToSchemas(config.tools);
}
if (config.systemPrompt !== undefined) {
uniConfig.system_prompt = config.systemPrompt;
}
// Non-positive (-1 per the config contract) means "no explicit cap": the key is left off
// the wire so the provider default applies — sent literally, every provider rejects a
// negative max_tokens with a 400 (issue #55's sibling).
if (config.maxTokens !== undefined && config.maxTokens > 0) {
uniConfig.max_tokens = config.maxTokens;
}
// Fast mode (the model entry's `fast_mode` annotation): only ever set when enabled — the
// key stays off the config otherwise, so models without a fast tier are unaffected by
// default (AgentHub rejects the parameter wherever no fast tier exists).
if (config.fastMode === true) {
uniConfig.fast_mode = true;
}
// Thought summaries, always requested. Two things ride on it: the user gets to watch the
// model reason, and — because the request timeout is an idle budget between upstream events
// — a reasoning phase that reaches the wire keeps resetting that timer instead of counting
// as one long silence. No model rejects the flag (unlike fast_mode): AgentHub maps it where
// it exists and drops it where it doesn't, so the only cost of asking is on families that
// answer with a summarized thinking stream (Claude) instead of the raw one.
uniConfig.thinking_summary = true;
return uniConfig;
}