Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { hasChatFullAccess, withChatFullAccess } from "../utils/chat-full-access";
import { attachedMediaUnavailableReason } from "../lib/attached-media-gate";
import { readGenerationProgress } from "../lib/generation-progress";
import { externalModelLabel } from "../lib/external-model-label";
import { mlxRuntimeStateFrom } from "../lib/mlx-runtime-state";
import { minPSamplingPayload } from "../lib/min-p-policy";
import {
createMinPRecoveryGuard,
invalidateMinPRecoveries,
shouldOfferMinPRecovery,
} from "../lib/min-p-recovery";
import {
type ImageDisclosure,
modelVisibleMessage,
toolOnlyImages,
} from "./mcp-image";
import { llamaServerLoadFields } from "../lib/load-request";
import {
clearedServerTuningState,
committedServerTuningState,
} from "../lib/server-tuning-fields";
import { authFetch, getAuthToken } from "@/features/auth";
import { withSandboxAttachmentPaths } from "../sandbox-attachments";
import { prepareHfTokenForUse } from "@/features/hf-auth";
import { DOWNLOAD_KIND } from "@/features/hub/download-manager/constants";
import {
downloadManager,
jobKeyOf,
selectActiveJob,
subscribeJobListeners,
useDownloadManagerStore,
} from "@/features/hub/download-manager/download-manager-controller";
import {
type LocalModelInfo,
listLocalModels,
} from "@/features/hub/inventory/api";
import { isHiddenModelId } from "@/features/hub/lib/hidden-models";
import {
isServedByLlamaCpp,
resumesThought,
isServedByMlx,
loadedContextFields,
resolveInitialConfig,
} from "@/features/model-picker/model-config/per-model-config";
import { isMlxId } from "@/features/model-picker/components/model-selector/recommended-fit";
import { loadManagedLlamaFlags } from "@/features/model-picker/api/llama-flags";
import { fetchLoadExtraArgs } from "@/features/model-picker/api/model-overrides";
import { sanitizeStoredExtraArgs } from "@/features/model-picker/model-config/llama-extra-args";
import { fetchDeviceType, usePlatformStore } from "@/config/env";
import { resolveSpeculativeType } from "@/lib/speculative-modes";
import { getSkillsSnapshot, settleSkillsForText } from "./skills-api";
const skillsForTools = () =>
useSkillsSwitchStore.getState().on === false ? [] : getSkillsSnapshot().skills;
function messageText(content: unknown): string {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
return content
.map((part) =>
part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
? (part as { text: string }).text
: "",
)
.join(" ");
}
function userTexts(
messages: readonly { role?: string; content?: unknown }[],
): string[] {
return messages
.filter((message) => message?.role === "user")
.map((message) => messageText(message.content));
}
import { endBrowserAgentRun } from "@/features/browser/agent/agent-store";
import { browserAgentToolsForTurn } from "@/features/browser/agent/tools";
import { shownBrowserChat } from "@/features/browser/panel-availability";
import { projectHasSources } from "@/features/rag/api/rag-api";
import {
IMAGE_SENTINEL_TOOLS,
SANDBOX_FILE_TOOLS,
SANDBOX_RUN_TOOLS,
extractCreatedFiles,
isSandboxFileList,
isSandboxToolResult,
type SandboxFile,
sandboxSessionIdFor,
} from "@/components/assistant-ui/sandbox-files";
import { apiUrl } from "@/lib/api-base";
import {
type McpUiToolResult,
extractMcpUiEnvelope,
isMcpUiToolResult,
mcpUiReplayImages,
} from "../mcp-apps/mcp-ui";
import { isMcpToolName } from "../utils/mcp-tool-name";
import { userMessageIdRequestField } from "../utils/rewind";
import {
type McpImage,
planMcpImageBound,
mcpImagesEnvelope,
isImageToolName,
splitMcpImages,
} from "./mcp-images";
import {
answerTextFromParts,
extractSearchImages,
isSearchImageEntry,
isSearchImagesToolResult,
missingListSubjects,
SEARCH_IMAGE_TOOL,
searchResultText,
stripSearchImageTokens,
type SearchImageEntry,
type SearchImagesToolResult,
} from "../search-images/search-images";
import { parseParamCountB } from "@/lib/model-size";
import { createLoadingToastIcon, toast } from "@/lib/toast";
import { notifyPromptQueueRunFailed } from "../utils/prompt-queue-boundary";
import {
providerCompactionConnectionKey,
providerCompactionForTarget,
providerCompactionPart,
} from "../utils/provider-compaction";
import {
adoptPreStreamRunReservation,
findPreStreamRunReservation,
preStreamRunThreadIdsForAdapter,
releasePreStreamRunForThreadIds,
releasePreStreamRunReservation,
} from "../utils/pre-stream-run-reservation";
import { readThreadCreationClaim } from "../utils/chat-thread-creation-claim";
import { withRunThreadScopedSettings } from "../utils/thread-scoped-settings";
import { contextUsageMarkers, threadBranchRecords } from "../utils/saved-context-usage";
import { contextUsageReasoningKey } from "../utils/context-usage-reasoning";
import {
apiCompactionRequestFields,
ggufCompactionRequestFields,
} from "../utils/auto-compaction";
import { tabCloseRunFields } from "../utils/keep-running-when-tab-closed";
import { resolveModelCatalogEntry } from "../model-catalog";
import { prepareComparePaneRun } from "../compare-run-ownership";
import {
lengthIncompleteReason,
lengthStopCause,
windowEvidenceCount,
} from "./generation-length";
import {
studioToolHistoryRequestFields,
type ToolHistoryMessage,
} from "../utils/studio-tool-history";
import {
newDeepResearchHandoff,
readDeepResearchToolEvent,
} from "../utils/deep-research-handoff";
import {
consumeQueuedChatRunSettings,
shouldPersistRunModel,
snapshotQueuedChatRunSettings,
} from "../utils/queued-chat-run-settings";
import {
mergeQueuedModelCapabilities,
type QueuedModelCapabilities,
} from "../utils/queued-model-capabilities";
import type { MessageTiming, ToolCallMessagePart } from "@assistant-ui/core";
import {
addPass,
type PriorPassTiming,
withPriorServerTimings,
withPriorTiming,
} from "../utils/continued-timing";
import type { ChatModelAdapter } from "@assistant-ui/react";
import { parsePartialJsonObject } from "assistant-stream/utils";
import {
externalModelTakesVideo,
isExternalModelId,
isPromptCacheTtl,
loadExternalProviders,
parseExternalModelId,
providerModelSupportsStudioTools,
providerModelSupportsVision,
providerModelTakesMcpImages,
supportsProviderPromptCacheTtl,
supportsProviderPromptCaching,
toExternalBackendProviderType,
} from "../external-providers";
import {
localToolExchangeIndexes,
addCodexReasoning,
codexLocalToolRoundId,
codexReasoningForToolCalls,
readCodexReasoning,
readOpenAIResponsesReasoning,
shouldReplayAssistantReasoning,
startsNewCodexToolRound,
type CodexReasoningLedger,
} from "../codex-reasoning";
import {
toolModelNoteTarget,
toolModelSuffixExtraContent,
withToolModelSuffix,
} from "../tool-model-suffix";
import {
type GeminiContinuationReplay,
type GeminiContinuationReplayTurn,
type GeminiAnswerReplayPart,
type GeminiThoughtReplayPart,
type PositionedGeminiAnswerReplayPart,
type PositionedGeminiThoughtReplayPart,
appendGeminiAnswerReplayPart,
appendGeminiThoughtReplayPart,
geminiAnswerReplayParts,
geminiContinuationReplayEntries,
geminiThoughtReplayParts,
pinGeminiAnswerReplayParts,
pinGeminiTextThoughtSignature,
pinGeminiThoughtReplayParts,
readGeminiContinuationReplay,
withGeminiAnswerReplayParts,
withGeminiThoughtReplayParts,
} from "../gemini-thought-replay";
import {
createBoundaryScan,
mergedToolCallArgumentsText,
splitTopLevelJsonObjects,
streamedToolCallArguments,
toolCallArgumentsText,
toolCallReplayArguments,
} from "../tool-call-arguments";
import {
bindStreamedToolCallCard,
findStreamedToolCallPartIndex,
mintStreamedToolCallId,
resolveToolCallPartId,
} from "../tool-call-id";
import { buildResearchInferenceRequest } from "../research-inference-request";
import { customReasoningRequestFields } from "../custom-reasoning";
import { pickFriendlyContainerName } from "../lib/friendly-names";
import {
buildExternalRoutingFields,
type ExternalRoutingUnavailableReason,
resolveExternalRouting,
} from "../utils/chat-title";
import {
reasoningCapsFromLoad,
resolveInferenceCheckpointId,
tryAdoptServerActiveModel,
} from "../lib/apply-inference-status-to-store";
import { isSpeechOnlyStatus } from "../lib/speech-only-status";
import {
beginServerModelWait,
statusPollSignal,
} from "../lib/server-model-wait";
import {
syncInferenceStatusToStore,
syncModelCapabilities,
} from "../hooks/use-chat-model-runtime";
import {
clampReasoningEffortToLevels,
externalMaxOutputTokensNeedsConnectionCap,
externalStopWindow,
getExternalMaxOutputTokens,
getPublishedExternalMaxOutputTokens,
getGroundedExternalMaxOutputTokens,
getExternalMinOutputTokens,
getExternalReasoningCapabilities,
providerModelReplaysReasoning,
providerSupportsPreserveThinking,
getProviderCapabilities,
providerHostsCodeExecution,
providerSupportsBuiltinCodeExecution,
providerSupportsBuiltinImageGeneration,
providerSupportsBuiltinWebFetch,
providerSupportsBuiltinWebSearch,
providerSupportsFastMode,
} from "../provider-capabilities";
import { selectCodeToolNames } from "./code-tool-placement";
import { skillToolNames } from "./skill-tools";
import { useSkillsSwitchStore } from "../stores/skills-switch-store";
import { ragScopeContextLength } from "./rag-context-length";
import {
type PendingImageEditReference,
type RagAutoInject,
GPU_LAYERS_AUTO,
loadedGpuMemoryFields,
reconcilePersistedGpuIds,
resolveLoadedSpeculativeSettings,
resolveSpeculativeSettingsForLoad,
persistGpuMemoryModeOnLoad,
resolvePreserveThinkingOnLoad,
resolveToolsEnabledOnLoad,
saveSpeculativeType,
awaitThreadScopedPairing,
awaitPendingQwenDefaultsMigration,
flushPendingChatSettings,
codeToolsOn,
withChatOwnedSettings,
withLoadedModelOverrides,
localMediaToolsForTurn,
useChatRuntimeStore,
} from "../stores/chat-runtime-store";
import {
loadedContextForParams,
localMaxTokensCeiling,
unreportedWindowMaxTokens,
resolveFitMaxSeqLength,
resolveExplicitCtxPin,
retainedContextPin,
replayMaxTokensCap,
} from "../presets/preset-policy";
import { reconcileTensorSplit } from "@/hooks/gpu-tensor-split";
import { ensureGpuDeviceCache } from "@/hooks/use-gpu-info";
import { useExternalProvidersStore } from "../stores/external-providers-store";
import { takeThreadCompactionRequest } from "../stores/compact-request-store";
import {
shouldPreserveFullOutput,
toolOutputKey,
toolPaneScope,
toolThreadScope,
} from "../tool-output-scope";
import type { ModelType, ThreadRecord } from "../types";
import { isMultimodalResponse } from "../types/api";
import type {
CpuFallbackReason,
MmprojFallbackReason,
GgufVariantDetail,
InferenceStatusResponse,
LoadModelRequest,
OpenAIChatChunk,
OpenAIChatCompletionsRequest,
OpenAIChatMessage,
OpenAIMessageContent,
OpenAIReasoningContentPart,
ProviderCompactionContentPart,
} from "../types/api";
import { modelReadsSamplingSeed, type ChatModelRow } from "../types/runtime";
import { loadFallbackNotice } from "../utils/mmproj-fallback";
import {
getStoredChatThread,
getStoredChatThreadReadResult,
getStoredChatProject,
isThreadIncognito,
listStoredChatThreads,
listStoredChatMessages,
saveStoredChatMessage,
updateStoredChatThread,
} from "../utils/chat-history-storage";
import {
readLastLocalModelLoad,
recordLastLocalModelLoad,
type LastLocalModelKind,
} from "../utils/last-local-model-load";
import { createRetryableSharedRead } from "../utils/retryable-shared-read";
import {
createImageGateRunOwner,
getImageInputUnavailableReason,
} from "../utils/image-input-support";
import {
appendCompactionMarker,
type CompactionMarker,
compactionAnchorToolCallId,
historyCannotHelp,
latestTurnIsTheProblem,
latestTurnOwnTokens,
mergeContextTruncation,
promptWasShortened,
} from "../utils/context-truncation";
import {
createThinkTagTracker,
extractDeltaText,
parseAssistantContent,
} from "../utils/parse-assistant-content";
import { createSegmentedAssistantText } from "../utils/incremental-assistant-content";
import {
createTrailingPlaceholderWatch,
stripTrailingTemplatePlaceholder,
} from "../utils/trailing-template-placeholder";
import {
TRAILING_PUBLISH,
createStreamPublishGate,
withTrailingPublish,
} from "../utils/stream-pacing";
import {
countReasoningGroups,
createReasoningDurationTracker,
lastReasoningGroupTextLength,
} from "../utils/reasoning-duration";
import { resolveLoadMaxSeqLength } from "../presets/preset-policy";
import type { CachedGgufRepo, CachedModelRepo } from "./chat-api";
import {
budgetImpliesTruncation,
CONTINUE_INSTRUCTION,
carriedThought,
continuationSeed,
createContinuationMerger,
hasRenderableContent,
incompleteLabel,
incompleteReasonAfterError,
type IncompleteReason,
isRestart,
noteRunStartedThisSession,
readIncompleteInfo,
resolveIncompleteReason,
readContinuationRequest,
rejectsAssistantPrefill,
resumesExactly,
} from "../utils/continuation";
import {
claimLiveGenerationRun,
forgetServerActiveGenerationRun,
generationChunkCountsTowardTiming,
generationChunkHasSubstantiveDelta,
generationIsSettled,
releaseLiveGenerationRun,
requestParsesThinkTags,
usageCacheWriteTokens,
} from "../utils/chat-generation-recovery";
import {
generateAudio,
GenerationLengthError,
fetchGgufStagedMetadata,
getChatAgentsMd,
getInferenceStatus,
listCachedGguf,
listCachedModels,
listGgufVariants,
loadModel,
streamChatCompletions,
StreamInterruptedError,
validateModel,
} from "./chat-api";
import {
type AgentsMdRecord,
agentsMdEditCount,
composeChatInstructions,
EMPTY_AGENTS_MD,
} from "../utils/agents-md";
import {
createOpenAIContainer,
listOpenAIContainers,
} from "./openai-containers";
import { isProviderKeyRotationError } from "./providers-api";
import {
beginExternalResearchFollow,
ingestResearchUpdate,
terminalResearchStatuses,
useResearchRunStore,
watchResearchRun,
} from "../stores/research-run-store";
import {
documentCitationToSource,
isSafeNavigableSourceUrl,
parseSourcesFromResult,
} from "../utils/document-citation-source";
import { mergeGoogleNativeParts } from "../utils/google-native-parts";
import { cancelResearchRun, createResearchRun } from "./research-api";
import {
cancelChatGenerationRun,
ChatGenerationStalledError,
chatGenerationStopPlan,
isLegacyFallbackChatGenerationAdmissionError,
type ChatGenerationRun,
type ChatGenerationStatus,
createChatGenerationRunUntilAbort,
explicitStopSignal,
followChatGenerationRun,
supportsChatGenerationRuns,
} from "./chat-generation-api";
import { isDurableRunCandidate, turnRequiresLegacyStream } from "./durable-gate";
import {
evictedSlotsFrom,
resyncEvicted,
toastEvicted,
} from "@/lib/evicted-toast";
import {
type OffloadCounts,
expertSpillNote,
offloadCountsFrom,
offloadWarning,
} from "../lib/partial-offload";
/** Say what an automatic chat load unloaded to make room, e.g. the speech model read aloud uses. */
function reportEvicted(loadResp: unknown): void {
const slots = evictedSlotsFrom(loadResp);
toastEvicted(slots);
resyncEvicted(slots, "chat");
}
// Small models (<=9B) answer from memory, so "auto" forces retrieval for them.
const AUTOINJECT_AUTO_MAX_SIZE_B = 9;
const EXTERNAL_ROUTING_REFUSALS: Record<
ExternalRoutingUnavailableReason,
{ title: string; description: string; error: string }
> = {
"connections-disabled": {
title: "Connections are disabled.",
description:
"Turn on Enable connections in Settings → Connections to use hosted models.",
error: "Connections disabled.",
},
"connection-missing": {
title: "Connection not found.",
description: "Open Settings → Connections and add it again.",
error: "Connection not found.",
},
"missing-api-key": {
title: "Missing API key for selected connection.",
description: "Open Settings → Connections and set the API key again.",
error: "Missing connection API key.",
},
};
class ChatGenerationTerminalError extends Error {
readonly generationStatus: "cancelled" | "failed";
constructor(generationStatus: "cancelled" | "failed", message: string) {
super(message);
this.name = "ChatGenerationTerminalError";
this.generationStatus = generationStatus;
}
}
type ThreadRecordReader = () => Promise<ThreadRecord | undefined>;
function resolveAutoInject(mode: RagAutoInject, checkpoint: string): boolean {
if (mode === "on") return true;
if (mode === "off") return false;
const size = parseParamCountB(checkpoint);
return size === null || size <= AUTOINJECT_AUTO_MAX_SIZE_B;
}
interface ServerUsage {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
// Studio tool loops: context after the turn; total_tokens re-counts earlier passes' completions.
context_tokens?: number;
// cache_creation is Anthropic's cache-write count, cache_write_tokens OpenRouter's.
prompt_tokens_details?: {
cached_tokens?: number;
cache_write_tokens?: number;
};
cache_creation_input_tokens?: number;
cache_read_input_tokens?: number;
}
interface ServerTimings {
prompt_n: number;
cache_n: number;
prompt_ms: number;
prompt_per_token_ms: number;
prompt_per_second: number;
predicted_n: number;
predicted_ms: number;
predicted_per_token_ms: number;
predicted_per_second: number;
// DiffusionGemma-only extras (present when serving a diffusion model; ignored otherwise).
diffusion?: boolean;
diffusion_blocks?: number;
diffusion_steps?: number;
diffusion_canvas?: number;
diffusion_prompt_n?: number;
diffusion_prompt_prepare_ms?: number;
diffusion_decode_ms?: number;
diffusion_wall_ms?: number;
// effective = canvas*blocks/wall, parallel = canvas/per_step, output = answer tokens/wall.
diffusion_effective_tok_s?: number;
diffusion_parallel_tok_s?: number;
diffusion_output_tok_s?: number;
diffusion_steps_per_second?: number;
}
interface ResponseDetailsMetadata {
modelId: string;
modelLabel: string;
responseModelId: string;
providerId?: string;
providerName: string;
providerType: string;
startedAt: number;
finishedAt: number;
durationMs: number;
sessionId?: string;
cancelId: string;
toolCalls: string[];
tools: {
search: boolean;
fetch: boolean;
code: boolean;
images: boolean;
mcp: boolean;
docs: boolean;
artifacts: boolean;
confirmToolCalls: boolean;
bypassPermissions: boolean;
permissionMode?: string;
};
}
type RunMessages = Parameters<ChatModelAdapter["run"]>[0]["messages"];
type RunMessage = RunMessages[number];
type OpenAIStreamAdapterOptions = {
modelType?: ModelType;
pairId?: string;
};
export const sentAudioNames = new Map<string, string>();
// Synthetic provider-side tool names; the backend stamps args._server_tool so user functions with
// the same name aren't dropped. Mirror of backend _SERVER_SIDE_BUILTIN_TOOL_NAMES.
const SERVER_SIDE_BUILTIN_TOOL_NAMES = new Set<string>([
"web_search",
"web_fetch",
"code_execution",
"image_generation",
]);
/** Provider-side synthetic tool parts (args._server_tool or Gemini native_part) are stripped. */
function isServerSideBuiltinToolPart(
toolNameLower: string,
_argsObj: Record<string, unknown> | null,
hasServerToolMarker: boolean,
hasNativePart: boolean,
): boolean {
if (!SERVER_SIDE_BUILTIN_TOOL_NAMES.has(toolNameLower)) return false;
if (hasServerToolMarker) return true;
return hasNativePart;
}
const FIRST_THREAD_SAVE_TIMEOUT_MS = 250;
const rollingContextNoticeThreads = new Set<string>();
type ThreadAutosaveHandle = {
registerFirstSave(threadId: string, promise: Promise<void>): Promise<void>;
awaitFirstSave(threadId: string | undefined): Promise<void>;
};
const pendingFirstThreadSaves = new Map<string, Promise<void>>();
function wait(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Best-effort partial parse of streaming tool_args; null until parseable, never throws. */
function parseLiveToolArgs(
raw: string,
): { args: Record<string, unknown>; argsText: string } | null {
let candidate = raw.trimStart();
if (!candidate.startsWith("{")) {
const brace = candidate.indexOf("{");
if (brace < 0) return null;
candidate = candidate.slice(brace);
}
const parsed = parsePartialJsonObject(candidate) as
| Record<string, unknown>
| undefined;
if (!parsed || typeof parsed !== "object") return null;
const inner = parsed.arguments ?? parsed.parameters;
if (typeof parsed.name === "string" && inner !== undefined) {
if (typeof inner === "string") {
const innerParsed = parsePartialJsonObject(inner) as
| Record<string, unknown>
| undefined;
if (innerParsed && typeof innerParsed === "object") {
return { args: innerParsed, argsText: inner };
}
return null;
}
if (inner && typeof inner === "object" && !Array.isArray(inner)) {
return {
args: inner as Record<string, unknown>,
argsText: JSON.stringify(inner),
};
}
return null;
}
return { args: parsed, argsText: candidate };
}
function parseSystemVariablesMap(raw: string): Record<string, unknown> {
if (!raw.trim()) {
return {};
}
try {
const parsed = JSON.parse(raw) as unknown;
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
} catch {
}
return {};
}
function hasOwn(object: object, key: string): boolean {
return Object.prototype.hasOwnProperty.call(object, key);
}
function getNestedValue(
values: Record<string, unknown>,
path: string,
): unknown | undefined {
const parts = path
.split(".")
.map((part) => part.trim())
.filter(Boolean);
if (parts.length === 0) {
return undefined;
}
let current: unknown = values;
for (const part of parts) {
if (!current || typeof current !== "object" || Array.isArray(current)) {
return undefined;
}
if (!hasOwn(current, part)) {
return undefined;
}
current = (current as Record<string, unknown>)[part];
}
return current;
}
function padDatePart(value: number): string {
return String(value).padStart(2, "0");
}
function formatLocalDate(now: Date): string {
return [
now.getFullYear(),
padDatePart(now.getMonth() + 1),
padDatePart(now.getDate()),
].join("-");
}
function formatLocalTime(now: Date): string {
return [
padDatePart(now.getHours()),
padDatePart(now.getMinutes()),
padDatePart(now.getSeconds()),
].join(":");
}
function formatTimezoneOffset(now: Date): string {
const offsetMinutes = -now.getTimezoneOffset();
const sign = offsetMinutes >= 0 ? "+" : "-";
const abs = Math.abs(offsetMinutes);
const hours = Math.floor(abs / 60);
const minutes = abs % 60;
return `${sign}${padDatePart(hours)}:${padDatePart(minutes)}`;
}
function stringifyTemplateValue(value: unknown): string {
if (value == null) {
return "";
}
if (typeof value === "string") {
return value;
}
if (typeof value === "number" || typeof value === "boolean") {
return String(value);
}
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}
function resolveSystemPromptVariables(
prompt: string,
customVariablesRaw: string,
): string {
if (!prompt) {
return prompt;
}
const now = new Date();
const localDate = formatLocalDate(now);
const localTime = formatLocalTime(now);
const systemVariables: Record<string, string> = {
$date: localDate,
$time: localTime,
$now: `${localDate}T${localTime}${formatTimezoneOffset(now)}`,
};
const customVariables = parseSystemVariablesMap(customVariablesRaw);
return prompt.replaceAll(
/{{\s*([a-zA-Z_$][a-zA-Z0-9_$.-]*)\s*}}/g,
(full, keyRaw) => {
const key = String(keyRaw).trim();
if (hasOwn(systemVariables, key)) {
return systemVariables[key] ?? full;
}
const resolved = getNestedValue(customVariables, key);
if (resolved === undefined) {
return full;
}
return stringifyTemplateValue(resolved);
},
);
}
export const ThreadAutosaveHandle: ThreadAutosaveHandle = {
registerFirstSave(threadId, promise) {
const trackedPromise = promise.catch(() => {});
const cleanupPromise = trackedPromise.finally(() => {
if (pendingFirstThreadSaves.get(threadId) === cleanupPromise) {
pendingFirstThreadSaves.delete(threadId);
}
});
pendingFirstThreadSaves.set(threadId, cleanupPromise);
return cleanupPromise;
},
async awaitFirstSave(threadId) {
if (!threadId) {
return;
}
const pending = pendingFirstThreadSaves.get(threadId);
if (!pending) {
return;
}
await Promise.race([pending, wait(FIRST_THREAD_SAVE_TIMEOUT_MS)]);
},
};
export function useThreadAutosaveHandle(): ThreadAutosaveHandle {
return ThreadAutosaveHandle;
}
/** Match errors meaning the KV cache filled: llama-server's "exceeds the available context
* size" and _friendly_error's "Message too long". Substrings, since both have drifted. */
export function isContextLimitError(message: string): boolean {
if (!message) return false;
const m = message.toLowerCase();
return (
m.includes("context size") ||
m.includes("context shift") ||
m.includes("exceeds the available context") ||
m.includes("message too long") ||
m.includes("context window") ||
(m.includes("n_ctx") && (m.includes("exceed") || m.includes("full")))
);
}
async function updateStoredChatThreadEventually(
threadId: string,
patch: Parameters<typeof updateStoredChatThread>[1],
): Promise<void> {
for (let attempt = 0; attempt < 10; attempt++) {
const updated = await updateStoredChatThread(threadId, patch).catch(
() => undefined,
);
if (updated) return;
await wait(50);
}
}
function estimateTokenCount(text: string): number | undefined {
const trimmed = text.trim();
if (!trimmed) {
return undefined;
}
return Math.max(1, Math.round(trimmed.length / 4));
}
function buildTiming(
streamStartTime: number,
totalChunks: number,
firstTokenTime?: number,
totalStreamTime?: number,
tokenCount?: number,
toolCallCount = 0,
tokensPerSecondOverride?: number,
): MessageTiming {
return {
streamStartTime,
firstTokenTime,
totalStreamTime,
tokenCount,
tokensPerSecond:
tokensPerSecondOverride ??
(typeof totalStreamTime === "number" &&
totalStreamTime > 0 &&
typeof tokenCount === "number"
? tokenCount / (totalStreamTime / 1000)
: undefined),
totalChunks,
toolCallCount,
};
}
function collectTextParts(message: RunMessage): string[] {
message = modelVisibleMessage(message);
const textParts = message.content
.filter((part) => part.type === "text")
.map((part) => part.text);
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
if (part.type === "text") {
textParts.push(part.text);
}
}
}
}
return textParts;
}
function collectImageParts(
message: RunMessage,
): Array<{ type: "image_url"; image_url: { url: string } }> {
message = modelVisibleMessage(message);
const parts: Array<{ type: "image_url"; image_url: { url: string } }> = [];
const pushImagePart = (part: { type: string }) => {
if (part.type !== "image" || !("image" in part)) {
return;
}
const src = (part as { image: string }).image;
if (!src) {
return;
}
parts.push({
type: "image_url",
image_url: {
url: /^(?:data:|https?:\/\/)/i.test(src)
? src
: `data:image/png;base64,${src}`,
},
});
};
for (const part of message.content ?? []) {
pushImagePart(part);
}
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
pushImagePart(part);
}
}
}
return parts;
}
function normalizeOpenAIReasoningItem(
value: unknown,
): OpenAIReasoningContentPart | null {
if (!value || typeof value !== "object") {
return null;
}
const item = value as Record<string, unknown>;
if (item.type !== "reasoning" || typeof item.id !== "string" || !item.id) {
return null;
}
const summary = Array.isArray(item.summary)
? item.summary.flatMap((part) => {
if (!part || typeof part !== "object") {
return [];
}
const summaryPart = part as Record<string, unknown>;
return summaryPart.type === "summary_text" &&
typeof summaryPart.text === "string"
? [{ type: "summary_text" as const, text: summaryPart.text }]
: [];
})
: [];
const normalized: OpenAIReasoningContentPart = {
type: "reasoning",
id: item.id,
summary,
};
if (
item.status === "in_progress" ||
item.status === "completed" ||
item.status === "incomplete"
) {
normalized.status = item.status;
}
return normalized;
}
function toOpenAIImageEditReferenceMessage(
reference: PendingImageEditReference,
): OpenAIChatMessage | null {
if (!reference.openaiImageGenerationCallId) {
return null;
}
const content: Exclude<OpenAIMessageContent, string> = [];
const reasoningItem = normalizeOpenAIReasoningItem(
reference.openaiReasoningItem,
);
if (reasoningItem) {
content.push(reasoningItem);
}
content.push({
type: "image_generation_call",
id: reference.openaiImageGenerationCallId,
...(reference.openaiResponseId
? { response_id: reference.openaiResponseId }
: {}),
});
return { role: "assistant", content };
}
// Use metadata, not spoofable text: Anthropic keeps refusing while refusals stay in context.
function isAnthropicRefusalMessage(message: RunMessage): boolean {
if (message.role !== "assistant") return false;
const metadata = (message as { metadata?: unknown }).metadata as
| { custom?: Record<string, unknown> }
| undefined;
return metadata?.custom?.anthropicRefusal === true;
}
type SerializedMessage = {
role: "system" | "user" | "assistant" | "tool";
content: OpenAIMessageContent | null;
reasoning_content?: string;
tool_calls?: Array<{
id: string;
type: "function";
function: { name: string; arguments: string };
extra_content?: unknown;
}>;
tool_call_id?: string;
name?: string;
/** Gemini text-part thoughtSignature; backend reads extra_content.google.thought_signature. */
extra_content?: unknown;
};
type SerializedToolCall = NonNullable<SerializedMessage["tool_calls"]>[number];
type SerializedToolResult = {
role: "tool";
content: string;
tool_call_id: string;
name?: string;
extra_content?: { studio_model_suffix: string };
};
type ToolPartReplayMetadata = {
argsObj: Record<string, unknown> | null;
argsGoogle: Record<string, unknown> | null;
hasNativePart: boolean;
isServerSideBuiltin: boolean;
};
function getToolPartReplayMetadata(
tc: ToolCallMessagePart,
): ToolPartReplayMetadata {
const toolNameLower = (tc.toolName ?? "").toLowerCase();
const argsObj =
tc.args && typeof tc.args === "object"
? (tc.args as Record<string, unknown>)
: null;
const argsGoogle =
argsObj && typeof argsObj.google === "object" && argsObj.google !== null
? (argsObj.google as Record<string, unknown>)
: null;
const hasNativePart = Boolean(
argsGoogle &&
typeof argsGoogle.native_part === "object" &&
argsGoogle.native_part !== null,
);
const hasServerToolMarker = Boolean(
argsObj && (argsObj as Record<string, unknown>)._server_tool === true,
);
return {
argsObj,
argsGoogle,
hasNativePart,
isServerSideBuiltin: isServerSideBuiltinToolPart(
toolNameLower,
argsObj,
hasServerToolMarker,
hasNativePart,
),
};
}
type ToolReplayProvenance = {
source?: string;
[key: string]: unknown;
};
function getToolReplayProvenance(
part: ToolCallMessagePart,
): ToolReplayProvenance | null {
const provenance = (part as { provenance?: unknown }).provenance;
if (
!provenance ||
typeof provenance !== "object" ||
Array.isArray(provenance)
) {
return null;
}
return provenance as ToolReplayProvenance;
}
function hasToolReplayResult(part: ToolCallMessagePart): boolean {
const result = (part as { result?: unknown }).result;
return result !== undefined && result !== null;
}
function shouldFlushCompletedLocalToolPair(part: ToolCallMessagePart): boolean {
const provenance = getToolReplayProvenance(part);
if (provenance?.source !== "local") {
return false;
}
if (getToolPartReplayMetadata(part).isServerSideBuiltin) {
return false;
}
return hasToolReplayResult(part);
}
function serializeAssistantToolCallPart(
part: ToolCallMessagePart,
): SerializedToolCall | null {
const tc = part as ToolCallMessagePart & {
argsText?: string;
extra_content?: unknown;
};
const { argsGoogle, hasNativePart, isServerSideBuiltin } =
getToolPartReplayMetadata(tc);
if (isServerSideBuiltin && !hasNativePart) {
return null;
}
const argumentsStr = toolCallReplayArguments(tc.argsText, tc.args);
const entry: SerializedToolCall = {
id: tc.toolCallId,
type: "function" as const,
function: {
name: tc.toolName ?? "",
arguments: argumentsStr,
},
};
// Promote args.google to extra_content.google: the backend replay only inspects extra_content.
if (tc.extra_content !== undefined) {
entry.extra_content = tc.extra_content;
} else if (argsGoogle) {
entry.extra_content = { google: argsGoogle };
}
return entry;
}
export interface McpImageToolResult {
text: string;
images: { data: string; mimeType: string }[];
}
/** Exports feed fine-tuning, so the wrapper's card metadata must not be serialized. */
export function toolResultModelText(
result: unknown,
toolName?: string,
): unknown {
if (
isMcpUiToolResult(result, toolName) ||
isMcpImageToolResult(result) ||
isSearchImagesToolResult(result) ||
isSandboxWrapper(result, toolName)
) {
return result.text;
}
return result;
}
/** Only unwrap our own wrapper; unwrapping another MCP result drops its other fields. */
function isSandboxWrapper(
result: unknown,
toolName?: string,
): result is { text: string; sessionId: string } {
if (toolName !== undefined && !SANDBOX_FILE_TOOLS.has(toolName)) return false;
return isSandboxToolResult(result);
}
/** Exactly the live parser's wrapper: text and images, no other field. */
export function isBareMcpImageWrapper(val: unknown): boolean {
if (!isMcpImageToolResult(val)) return false;
const keys = Object.keys(val as object).filter((key) => key !== "text" && key !== "images");
return keys.length === 0;
}
export function isMcpImageToolResult(val: unknown): val is McpImageToolResult {
if (typeof val !== "object" || val === null) {
return false;
}
const v = val as {
text?: unknown;
images?: unknown;
sessionId?: unknown;
ui?: unknown;
};
return (
typeof v.text === "string" &&
v.sessionId === undefined &&
v.ui === undefined &&
Array.isArray(v.images) &&
v.images.length > 0 &&
v.images.every(
(img: unknown) =>
typeof img === "object" &&
img !== null &&
typeof (img as { data?: unknown }).data === "string" &&
typeof (img as { mimeType?: unknown }).mimeType === "string",
)
);
}
function serializeToolResultPart(
part: ToolCallMessagePart,
): SerializedToolResult | null {
const tc = part as ToolCallMessagePart;
const result = (tc as { result?: unknown }).result;
const { isServerSideBuiltin } = getToolPartReplayMetadata(tc);
if (isServerSideBuiltin) {
return null;
}
if (result === undefined || result === null) return null;
let content: string;
if (typeof result === "string") {
// Backend ChatMessage rejects role="tool" with empty content; a sentinel JSON round-trips it.
content = result.length > 0 ? result : JSON.stringify({ result: "" });
} else if (
isMcpUiToolResult(result, tc.toolName ?? "") ||
// Unwrap the live parser's exact {text, images} wrapper by shape, or JSON.stringify replays the
// base64 as prompt text; a client tool's result with other fields keeps normal serialization.
(isMcpImageToolResult(result) &&
(isImageToolName(tc.toolName) || isBareMcpImageWrapper(result))) ||
isSearchImagesToolResult(result) ||
isSandboxWrapper(result, tc.toolName ?? "")
) {
// Replay stdout only; image tokens stay since they resolve against the producing message.
const replayText = isSearchImagesToolResult(result)
? stripSearchImageTokens(result.text)
: result.text;
content = replayText.length > 0 ? replayText : JSON.stringify({ result: "" });
// Gated on the image tool name, not shape: a client tool may return {text, images}, and
// appending the envelope would hand its bytes to the model as image input.
if (isMcpImageToolResult(result) && isImageToolName(tc.toolName)) {
content += mcpImagesEnvelope(result.images);
} else if (isMcpToolName(tc.toolName)) {
const uiImages = mcpUiReplayImages(result, tc.toolName ?? "");
if (uiImages.length > 0) content += mcpImagesEnvelope(uiImages);
}
} else {
try {
content = JSON.stringify(result);
} catch {
content = String(result);
}
}
return {
role: "tool" as const,
content,
tool_call_id: tc.toolCallId,
...(tc.toolName ? { name: tc.toolName } : {}),
// Model-only feedback the backend re-appends on replay.
...toolModelSuffixExtraContent(getToolReplayProvenance(tc)),
};
}
function canReplayToolCallWithoutRoleTool(part: ToolCallMessagePart): boolean {
// Provider-native builtin cards replay via extra_content with no role="tool" message.
return getToolPartReplayMetadata(part).isServerSideBuiltin;
}
function toolCallPartSurvivesOpenAIReplay(part: ToolCallMessagePart): boolean {
if (!serializeAssistantToolCallPart(part)) return false;
if (
!serializeToolResultPart(part) &&
!canReplayToolCallWithoutRoleTool(part)
) {
return false;
}
return true;
}
function studioToolHistoryRequestFieldsAfterReplay(
messages: readonly ToolHistoryMessage[],
): { studio_tool_history?: true } {
return studioToolHistoryRequestFields(messages, {
toolCallSurvives: (part) =>
toolCallPartSurvivesOpenAIReplay(part as unknown as ToolCallMessagePart),
});
}
function sanitizeAssistantReplayText(text: string): string {
// The tokens the model wrote last turn are renderer markup scoped to that message, not prose to repeat.
return stripSearchImageTokens(text).replace(
/data:audio\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
"[audio]",
);
}
function buildReplayContent(
textContent: string,
imageParts: Array<{ type: "image_url"; image_url: { url: string } }>,
): OpenAIMessageContent {
if (imageParts.length === 0) return textContent;
// Anthropic rejects whitespace-only text; spread so the caller's array is not reused as content.
return textContent.trim()
? [{ type: "text", text: textContent }, ...imageParts]
: [...imageParts];
}
function collectAssistantTextThoughtSignature(
message: RunMessage,
): string | undefined {
if (!Array.isArray(message.content)) return undefined;
for (let i = message.content.length - 1; i >= 0; i -= 1) {
const part = message.content[i] as { type?: string } & Record<
string,
unknown
>;
if (part?.type !== "text") continue;
const sig = part._google_thought_signature;
if (typeof sig === "string" && sig) return sig;
}
return undefined;
}
function setAssistantCodexReasoning(
message: SerializedMessage,
reasoning: unknown[] | undefined,
): void {
if (!reasoning) return;
const extra =
message.extra_content &&
typeof message.extra_content === "object" &&
!Array.isArray(message.extra_content)
? (message.extra_content as Record<string, unknown>)
: {};
message.extra_content = { ...extra, openai_codex_reasoning: reasoning };
}
function setAssistantOpenAIResponsesReasoning(
message: SerializedMessage,
reasoning: unknown[] | undefined,
): void {
if (!reasoning) return;
const extra =
message.extra_content &&
typeof message.extra_content === "object" &&
!Array.isArray(message.extra_content)
? (message.extra_content as Record<string, unknown>)
: {};
message.extra_content = { ...extra, openai_responses_reasoning: reasoning };
}
function providerCompactionAssistant(
messages: SerializedMessage[],
afterToolCalls: number,
): SerializedMessage | undefined {
const boundary =
Number.isInteger(afterToolCalls) && afterToolCalls >= 0
? afterToolCalls
: 0;
let precedingToolCalls = 0;
let lastAssistant: SerializedMessage | undefined;
for (const message of messages) {
if (message.role !== "assistant") continue;
lastAssistant = message;
if (precedingToolCalls >= boundary) return message;
precedingToolCalls += message.tool_calls?.length ?? 0;
}
return lastAssistant;
}
/** The provider drops what came before its compaction, so it leads the subturn that produced it. */
function withProviderCompaction(
message: RunMessage,
serialized: SerializedMessage[],
target: {
providerType: string | undefined;
modelId: string | undefined;
connectionKey: string | undefined;
},
): SerializedMessage[] {
const custom = (
message as { metadata?: { custom?: Record<string, unknown> } }
).metadata?.custom;
const compaction = providerCompactionForTarget(
custom,
target.providerType,
target.modelId,
target.connectionKey,
);
const assistant = providerCompactionAssistant(
serialized,
typeof custom?.providerCompactionAfterToolCalls === "number"
? custom.providerCompactionAfterToolCalls
: 0,
);
if (!compaction || !assistant) return serialized;
const content = assistant.content;
assistant.content = [
compaction,
...(typeof content === "string"
? content
? [{ type: "text" as const, text: content }]
: []
: (content ?? [])),
];
return serialized;
}
function attachAssistantThoughtSignature(
messages: SerializedMessage[],
thoughtSignature: string | undefined,
): void {
if (!thoughtSignature) return;
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (message.role !== "assistant") continue;
const extra =
message.extra_content &&
typeof message.extra_content === "object" &&
!Array.isArray(message.extra_content)
? (message.extra_content as Record<string, unknown>)
: {};
const google =
extra.google &&
typeof extra.google === "object" &&
!Array.isArray(extra.google)
? (extra.google as Record<string, unknown>)
: {};
message.extra_content = {
...extra,
google: { ...google, thought_signature: thoughtSignature },
};
return;
}
}
function serializeGeminiContinuationTurns(
turns: GeminiContinuationReplayTurn[],
includeTrailingInstruction: boolean,
): SerializedMessage[] {
const messages: SerializedMessage[] = [];
for (const entry of geminiContinuationReplayEntries(
turns,
includeTrailingInstruction,
)) {
if (entry.role === "user") {
messages.push({ role: "user", content: CONTINUE_INSTRUCTION });
continue;
}
const { turn } = entry;
const assistant: SerializedMessage = {
role: "assistant",
content: turn.text,
...(turn.thoughtParts?.length || turn.answerParts?.length
? {
extra_content: withGeminiAnswerReplayParts(
withGeminiThoughtReplayParts(undefined, turn.thoughtParts ?? []),
turn.answerParts ?? [],
),
}
: {}),
};
attachAssistantThoughtSignature([assistant], turn.thoughtSignature);
messages.push(assistant);
}
return messages;
}
function stripGeminiContinuationVisiblePrefix(
messages: SerializedMessage[],
visiblePrefix: string,
): void {
if (!visiblePrefix) return;
const assistant = messages.find((message) => message.role === "assistant");
if (!assistant) return;
if (typeof assistant.content === "string") {
if (assistant.content.startsWith(visiblePrefix)) {
assistant.content = assistant.content.slice(visiblePrefix.length);
}
return;
}
if (!Array.isArray(assistant.content)) return;
const content = assistant.content as Array<Record<string, unknown>>;
const text = content
.filter((part) => part.type === "text" && typeof part.text === "string")
.map((part) => part.text as string)
.join("");
if (!text.startsWith(visiblePrefix)) return;
let remaining = visiblePrefix.length;
assistant.content = content.flatMap((part) => {
if (remaining === 0 || part.type !== "text" || typeof part.text !== "string") {
return [part];
}
const drop = Math.min(remaining, part.text.length);
remaining -= drop;
const trimmed = part.text.slice(drop);
return trimmed ? [{ ...part, text: trimmed }] : [];
}) as OpenAIMessageContent;
}
function expandGeminiContinuationReplay(
message: RunMessage,
serialized: SerializedMessage[],
): SerializedMessage[] {
const replay = readGeminiContinuationReplay(
(message as { metadata?: unknown }).metadata,
);
if (!replay) return serialized;
if (replay.stripVisiblePrefix) {
stripGeminiContinuationVisiblePrefix(serialized, replay.visiblePrefix);
}
return [
...serializeGeminiContinuationTurns(replay.turns, true),
...serialized,
];
}
function serializeAssistantReplayMessages(
message: RunMessage,
includeReasoningContent = false,
): SerializedMessage[] {
if (isAnthropicRefusalMessage(message)) {
// Prune the refused assistant turn from outbound history; the transcript still shows the notice.
return [];
}
const imageParts = collectImageParts(message);
const codexReasoning = readCodexReasoning(
(message as { metadata?: unknown }).metadata,
);
const openAIResponsesReasoning = readOpenAIResponsesReasoning(
(message as { metadata?: unknown }).metadata,
);
const messages: SerializedMessage[] = [];
const pendingTextParts: string[] = [];
const pendingReasoningParts: string[] = [];
let pendingToolCalls: SerializedToolCall[] = [];
let pendingToolResults: SerializedToolResult[] = [];
let pendingGeminiThoughtParts: GeminiThoughtReplayPart[] = [];
let pendingGeminiAnswerParts: GeminiAnswerReplayPart[] = [];
let imagePartsPending = imageParts.length > 0;
let pendingLocalToolRoundId: number | null = null;
const flushAssistantAndToolResults = (force = false): void => {
const textContent = sanitizeAssistantReplayText(
pendingTextParts.join("\n"),
);
const includeImageParts = imagePartsPending ? imageParts : [];
const hasContent = textContent.length > 0 || includeImageParts.length > 0;
const hasToolCalls = pendingToolCalls.length > 0;
const hasGeminiThoughtParts = pendingGeminiThoughtParts.length > 0;
const hasGeminiAnswerParts = pendingGeminiAnswerParts.length > 0;
const reasoningContent = pendingReasoningParts.join("\n");
const incomplete =
message.status?.type === "incomplete" ||
readIncompleteInfo((message as { metadata?: unknown }).metadata) !== null;
const hasReasoningContent = shouldReplayAssistantReasoning({
enabled: includeReasoningContent,
reasoningContent,
hasContent,
hasToolCalls,
incomplete,
});
if (
!force &&
!hasContent &&
!hasToolCalls &&
!hasReasoningContent &&
!hasGeminiThoughtParts &&
!hasGeminiAnswerParts
) {
return;
}
const assistantMessage: SerializedMessage = {
role: "assistant",
content: hasContent
? buildReplayContent(textContent, includeImageParts)
: "",
};
if (hasToolCalls) {
assistantMessage.tool_calls = pendingToolCalls;
// OpenAI requires content === null on assistant turns whose payload is entirely tool_calls.
if (!hasContent) {
assistantMessage.content = null;
}
}
if (hasReasoningContent) {
assistantMessage.reasoning_content = reasoningContent;
}
if (hasGeminiThoughtParts) {
assistantMessage.extra_content = withGeminiThoughtReplayParts(
assistantMessage.extra_content,
pendingGeminiThoughtParts,
);
}
if (hasGeminiAnswerParts) {
assistantMessage.extra_content = withGeminiAnswerReplayParts(
assistantMessage.extra_content,
pendingGeminiAnswerParts,
);
}
if (hasToolCalls) {
setAssistantCodexReasoning(
assistantMessage,
codexReasoningForToolCalls(
codexReasoning,
pendingToolCalls.map((call) => call.id),
),
);
setAssistantOpenAIResponsesReasoning(
assistantMessage,
codexReasoningForToolCalls(
openAIResponsesReasoning,
pendingToolCalls.map((call) => call.id),
),
);
}
messages.push(assistantMessage);
if (pendingToolResults.length > 0) {
messages.push(...pendingToolResults);
}
pendingTextParts.length = 0;
pendingReasoningParts.length = 0;
pendingToolCalls = [];
pendingToolResults = [];
pendingGeminiThoughtParts = [];
pendingGeminiAnswerParts = [];
imagePartsPending = false;
pendingLocalToolRoundId = null;
};
for (const part of message.content ?? []) {
if (part.type === "reasoning") {
if (pendingToolCalls.length > 0) {
flushAssistantAndToolResults();
}
pendingGeminiThoughtParts.push(...geminiThoughtReplayParts(part));
pendingReasoningParts.push(part.text);
continue;
}
if (part.type === "text") {
if (pendingToolCalls.length > 0) {
flushAssistantAndToolResults();
}
pendingGeminiThoughtParts.push(...geminiThoughtReplayParts(part));
pendingGeminiAnswerParts.push(...geminiAnswerReplayParts(part));
pendingTextParts.push(part.text);
continue;
}
if (part.type === "tool-call") {
const toolPart = part as ToolCallMessagePart;
// Backend preload card is UI evidence, not a model function call.
if (toolPart.toolName === "studio_load_skill") continue;
const toolCall = serializeAssistantToolCallPart(toolPart);
if (!toolCall) continue;
const toolResult = serializeToolResultPart(toolPart);
if (!toolResult && !canReplayToolCallWithoutRoleTool(toolPart)) {
continue;
}
const provenance = getToolReplayProvenance(toolPart);
const localRoundId = codexLocalToolRoundId(provenance);
if (
pendingToolCalls.length > 0 &&
startsNewCodexToolRound(pendingLocalToolRoundId, localRoundId)
) {
flushAssistantAndToolResults();
}
if (localRoundId !== null) pendingLocalToolRoundId = localRoundId;
const flushLocalPair =
localRoundId === null && shouldFlushCompletedLocalToolPair(toolPart);
if (flushLocalPair && pendingToolCalls.length > 0) {
flushAssistantAndToolResults();
}
pendingToolCalls.push(toolCall);
if (toolResult) {
pendingToolResults.push(toolResult);
}
if (flushLocalPair) {
flushAssistantAndToolResults();
}
}
}
flushAssistantAndToolResults(messages.length === 0);
attachAssistantThoughtSignature(
messages,
collectAssistantTextThoughtSignature(message),
);
const finalAssistant = [...messages]
.reverse()
.find((candidate) => candidate.role === "assistant");
if (finalAssistant) {
setAssistantCodexReasoning(finalAssistant, codexReasoning?.final);
}
return messages;
}
/** Strict templates (Qwen3.5) reject a second system turn; the backend joins system parts with "\n\n", so merging is byte-identical there. */
function prependSystemPrompt<T extends { role: string; content: unknown }>(
messages: T[],
text: string,
): void {
const first = messages[0];
if (first?.role === "system" && typeof first.content === "string") {
messages[0] = { ...first, content: `${text}\n\n${first.content}` };
return;
}
messages.unshift({ role: "system", content: text } as T);
}
function toOpenAIMessages(
message: RunMessage,
includeReasoningContent = false,
expandGeminiContinuations = false,
): SerializedMessage[] {
message = modelVisibleMessage(message);
if (
message.role !== "system" &&
message.role !== "user" &&
message.role !== "assistant"
) {
return [];
}
if (message.role === "assistant") {
const serialized = fillStoppedAssistantReplay(
message,
serializeAssistantReplayMessages(message, includeReasoningContent),
);
return expandGeminiContinuations
? expandGeminiContinuationReplay(message, serialized)
: serialized;
}
const textContent = collectTextParts(message).join("\n");
const imageParts = collectImageParts(message);
return [
{
role: message.role,
content: buildReplayContent(textContent, imageParts),
},
];
}
/** Tool calls do not count: a dropped uncarryable call would merge two user turns. */
function assistantTurnCarriesPayload(message: RunMessage): boolean {
for (const part of message.content ?? []) {
// sanitizeAssistantReplayText only substitutes, never empties, so the raw part is equivalent here.
if (part.type === "text" && part.text.trim().length > 0) return true;
}
if (collectImageParts(message).length > 0) return true;
return "attachments" in message && (message.attachments?.length ?? 0) > 0;
}
/** Whitespace is not content: _build_external_messages drops any assistant turn whose content trims away. */
function hasReplayContent(content: OpenAIMessageContent | null): boolean {
if (typeof content === "string") return content.trim().length > 0;
return Boolean(content);
}
/** `status` is session-only (reloads come back complete), so read the persisted marker too. */
function assistantTurnEndedEarly(message: RunMessage): boolean {
return (
message.status?.type === "incomplete" ||
readIncompleteInfo((message as { metadata?: unknown }).metadata) !== null
);
}
/** #10428: an empty Stop loses its prompt to the prune. Only `cancelled` in `status` is deliberate. */
function stoppedAssistantReplayText(message: RunMessage): string {
const info = readIncompleteInfo(
(message as { metadata?: unknown }).metadata,
);
const status = message.status;
const fromStatus: IncompleteReason =
status?.type !== "incomplete" || status.reason === "cancelled"
? "cancelled"
: status.reason === "length"
? "length"
: "interrupted";
return incompleteLabel(info?.reason ?? fromStatus);
}
function fillStoppedAssistantReplay(
message: RunMessage,
serialized: SerializedMessage[],
): SerializedMessage[] {
if (!assistantTurnEndedEarly(message)) {
return serialized;
}
if (serialized.length === 0) {
// Only a refusal serialises to nothing; filling it puts a stop label back on a dropped turn.
return serialized;
}
const [only, ...rest] = serialized;
if (rest.length !== 0 || only?.role !== "assistant") {
return serialized;
}
if (
hasReplayContent(only.content) ||
only.tool_calls ||
only.reasoning_content
) {
return serialized;
}
return [{ ...only, content: stoppedAssistantReplayText(message) }];
}
/** A Stop before any output leaves an empty assistant message between two user turns. */
function isAbandonedAssistantTurn(
message: RunMessage,
includeReasoningContent: boolean,
): boolean {
if (message.role !== "assistant") return false;
if (assistantTurnCarriesPayload(message)) return false;
// A reasoning-only turn is a reply even if the provider serialises reasoning empty.
if (
!assistantTurnEndedEarly(message) &&
(message.content ?? []).some((part) => part.type === "reasoning")
) {
return false;
}
const [only, ...rest] = toOpenAIMessages(message, includeReasoningContent);
return (
rest.length === 0 &&
only?.role === "assistant" &&
!hasReplayContent(only.content) &&
!only.tool_calls &&
!only.reasoning_content
);
}
/** A refusal re-triggers the classifier; an abandoned turn breaks role alternation. */
function pruneOutboundHistory(
messages: RunMessages,
includeReasoningContent: boolean,
): RunMessage[] {
const history = [...messages];
const abandoned = history.map((message) =>
isAbandonedAssistantTurn(message, includeReasoningContent),
);
// A prompt is only stranded when something survives after it, so a trailing abandoned turn keeps its prompt.
let lastSurviving = -1;
for (let index = history.length - 1; index >= 0; index -= 1) {
if (!abandoned[index] && !isAnthropicRefusalMessage(history[index])) {
lastSurviving = index;
break;
}
}
const surviving: RunMessage[] = [];
for (let index = 0; index < history.length; index += 1) {
const message = history[index];
const refused = isAnthropicRefusalMessage(message);
if (refused || abandoned[index]) {
if (refused || index < lastSurviving) {
const last = surviving.at(-1);
if (last && last.role === "user") surviving.pop();
}
continue;
}
surviving.push(message);
}
return surviving;
}
function extractImageBase64(input: string): string | undefined {
if (!input) {
return undefined;
}
if (input.startsWith("data:")) {
const commaIndex = input.indexOf(",");
return commaIndex >= 0 ? input.slice(commaIndex + 1) : undefined;
}
return input;
}
function findLatestUserImageBase64(messages: RunMessages): string | undefined {
messages = messages.map(modelVisibleMessage);
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (!message || message.role !== "user") {
continue;
}
for (const part of message.content ?? []) {
if (part.type === "image" && "image" in part) {
const encoded = extractImageBase64(part.image);
if (encoded) return encoded;
}
}
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
if (part.type !== "image") {
continue;
}
const encoded = extractImageBase64(part.image);
if (encoded) {
return encoded;
}
}
}
}
}
return undefined;
}
function extractAudioPartBase64(
part: { type: string } | null | undefined,
): string | undefined {
if (!part || part.type !== "audio" || !("audio" in part)) return undefined;
const audioPart = (
part as unknown as {
type: "audio";
audio: string | { data: string; format: string };
}
).audio;
const raw = typeof audioPart === "string" ? audioPart : audioPart?.data;
if (!raw) return undefined;
return raw.startsWith("data:") ? raw.split(",")[1] : raw;
}
// A predicate rather than collectImageParts: building the parts would copy the base64 this exists to avoid touching.
export function messagesContainImage(messages: RunMessages): boolean {
messages = messages.map(modelVisibleMessage);
const isImage = (part: { type: string }) =>
part.type === "image" &&
"image" in part &&
Boolean((part as { image: string }).image);
for (const message of messages) {
for (const part of message.content ?? []) {
if (isImage(part)) return true;
}
if ("attachments" in message) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
if (isImage(part)) return true;
}
}
}
}
return false;
}
// Matched on part TYPE: an unparsable clip is still user content. Video is a file part with a video mime.
function isPrivateMediaPart(part: { type: string }): boolean {
if (part.type === "audio") return true;
return (
part.type === "file" &&
/^video\//i.test((part as { mimeType?: string }).mimeType ?? "")
);
}
export function messagesUsePrivateContent(messages: RunMessages): boolean {
// Tool-only images are hidden from the model, not public.
if (
messagesContainImage(messages) ||
messages.some((message) => toolOnlyImages(message).length > 0)
) {
return true;
}
return messages.some((message) => {
if (
(message.content ?? []).some(
(part) =>
(part.type === "tool-call" &&
(part as { toolName?: string }).toolName === "search_knowledge_base") ||
isPrivateMediaPart(part),
)
) {
return true;
}
if (!("attachments" in message)) return false;
return (message.attachments ?? []).some((attachment) =>
(attachment.content ?? []).some(
(part) => part.type === "text" || isPrivateMediaPart(part),
),
);
});
}
/** Every clip on the newest user message, in the order it was attached. */
function latestUserAudioClips(messages: RunMessages): string[] {
messages = messages.map(modelVisibleMessage);
const clips: string[] = [];
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (!message || message.role !== "user") continue;
for (const part of message.content ?? []) {
const base64 = extractAudioPartBase64(part);
if (base64) clips.push(base64);
}
if ("attachments" in message) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
const base64 = extractAudioPartBase64(part);
if (base64) clips.push(base64);
}
}
}
// Only the newest user message: audio_base64 switches the backend onto the audio path, so a
// stale clip would hijack text follow-ups.
break;
}
return clips;
}
export function findLatestUserAudioBase64(
messages: RunMessages,
includePendingAudio = true,
): string | undefined {
const [first] = latestUserAudioClips(messages);
if (first) return first;
const pendingAudio = includePendingAudio
? useChatRuntimeStore.getState().pendingAudioBase64
: null;
return pendingAudio ?? undefined;
}
/** Clips after the first, for extra_audio_base64. Undefined when empty so text turns stay durable. */
export function findLatestUserExtraAudioBase64(
messages: RunMessages,
): string[] | undefined {
const extra = latestUserAudioClips(messages).slice(1);
return extra.length > 0 ? extra : undefined;
}
function extractVideoPartBase64(
part: { type: string } | null | undefined,
): string | undefined {
if (!part || part.type !== "file") return undefined;
const filePart = part as unknown as { data?: string; mimeType?: string };
if (!filePart.data || !/^video\//i.test(filePart.mimeType ?? ""))
return undefined;
return filePart.data.startsWith("data:")
? filePart.data.split(",")[1]
: filePart.data;
}
/** Base64 of the clip on the newest user turn only; replaying an older one would re-sample it into frames. */
export function findLatestUserVideoBase64(
messages: RunMessages,
): string | undefined {
messages = messages.map(modelVisibleMessage);
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (!message || message.role !== "user") continue;
for (const part of message.content ?? []) {
const base64 = extractVideoPartBase64(part);
if (base64) return base64;
}
if ("attachments" in message) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
const base64 = extractVideoPartBase64(part);
if (base64) return base64;
}
}
}
break;
}
return undefined;
}
/**
* MCP image bound applied to the run's tool results before serialization; bounding afterwards
* stringified and re-parsed every image array on each send, for pictures the request would not
* carry; a result with no picture becomes plain text.
*/
function boundMcpImageResults(
messages: RunMessages,
{ readsImages, localMarkers }: { readsImages: boolean; localMarkers: boolean },
): RunMessages {
type Carrier = { message: number; part: number; images: McpImage[] };
// One entry per replay exchange, partitioned by the serializer's own rule so they cannot drift.
const perExchange: Carrier[][] = [];
messages.forEach((message, m) => {
const parts = message.content;
if (!Array.isArray(parts)) return;
const toolParts: { index: number; part: ToolCallMessagePart }[] = [];
parts.forEach((part, p) => {
if (part.type === "tool-call") {
toolParts.push({ index: p, part: part as ToolCallMessagePart });
}
});
const exchanges = localToolExchangeIndexes(
toolParts,
({ part }) => codexLocalToolRoundId(getToolReplayProvenance(part)),
({ part }) => shouldFlushCompletedLocalToolPair(part),
);
const byExchange = new Map<number, Carrier[]>();
toolParts.forEach(({ index, part }, k) => {
const tc = part as { toolName?: string; result?: unknown };
if (!isMcpImageToolResult(tc.result) || !isImageToolName(tc.toolName)) return;
const carrier = { message: m, part: index, images: tc.result.images };
const batch = byExchange.get(exchanges[k]) ?? [];
batch.push(carrier);
byExchange.set(exchanges[k], batch);
});
for (const key of [...byExchange.keys()].sort((a, b) => a - b)) {
perExchange.push(byExchange.get(key)!);
}
});
if (perExchange.length === 0) return messages;
const batches = localMarkers
? perExchange
: perExchange.flatMap((batch) => batch.map((carrier) => [carrier]));
const plan = readsImages
? planMcpImageBound(
batches.map((batch) => batch.map((carrier) => carrier.images)),
{ localMarkers },
)
: batches.map((batch) => batch.map((): McpImage[] => []));
const edits = new Map<number, Map<number, McpImage[]>>();
batches.forEach((batch, b) =>
batch.forEach((carrier, r) => {
const kept = plan[b][r];
if (kept.length === carrier.images.length) return;
const forMessage = edits.get(carrier.message) ?? new Map<number, McpImage[]>();
forMessage.set(carrier.part, kept);
edits.set(carrier.message, forMessage);
}),
);
if (edits.size === 0) return messages;
return messages.map((message, m) => {
const forMessage = edits.get(m);
if (!forMessage || !Array.isArray(message.content)) return message;
const content = message.content.map((part, p) => {
const kept = forMessage.get(p);
if (!kept) return part;
const result = (part as { result: McpImageToolResult }).result;
return {
...part,
result: kept.length > 0 ? { ...result, images: kept } : result.text,
};
});
return { ...message, content } as typeof message;
}) as RunMessages;
}
/** The SELECTED model's vision flag wins (a queued send may target another model); loaded state
* only when unknown. Not loadedIsMultimodal alone: audio-only models set it. */
function localTargetReadsImages(
state: Pick<ChatRuntimeState, "models" | "params" | "loadedIsMultimodal">,
): boolean {
const activeModel = state.models.find(
(model) => model.id === state.params.checkpoint,
);
if (typeof activeModel?.isVision === "boolean") return activeModel.isVision;
return state.loadedIsMultimodal !== false;
}
/** Tools are priced server-side since --enable-tools can inject schemas the client cannot see. */
export async function buildLocalTokenCountHistory(
rawMessages: RunMessages,
threadId: string | undefined,
): Promise<{
messages: OpenAIChatMessage[];
studio_tool_history?: true;
}> {
const runtimeState = useChatRuntimeStore.getState();
const { params } = runtimeState;
const activeModel = runtimeState.models.find(
(model) => model.id === runtimeState.params.checkpoint,
);
// Backend limits run after parsing and cannot bound the uploaded body, so limit here.
const messages = boundMcpImageResults(rawMessages, {
readsImages: localTargetReadsImages(runtimeState),
localMarkers: activeModel?.isGguf === false,
});
const survivingMessages = pruneOutboundHistory(messages, true);
const outboundMessages = survivingMessages
.flatMap((message) => toOpenAIMessages(message, true))
.filter((message): message is NonNullable<typeof message> =>
Boolean(message),
);
const safeSystemPrompt =
typeof params.systemPrompt === "string"
? resolveSystemPromptVariables(
params.systemPrompt,
typeof params.systemVariables === "string"
? params.systemVariables
: "",
)
: "";
const projectId = await resolveProjectId(threadId);
const [projectInstructions, agentsMd] = await Promise.all([
resolveProjectInstructions(projectId),
resolveAgentsMd(projectId),
]);
const combinedSystemPrompt = composeChatInstructions({
agentsMd,
projectInstructions,
systemPrompt: safeSystemPrompt,
});
if (combinedSystemPrompt) {
prependSystemPrompt(outboundMessages, combinedSystemPrompt);
}
return {
messages: outboundMessages as OpenAIChatMessage[],
...studioToolHistoryRequestFieldsAfterReplay(
survivingMessages as unknown as ToolHistoryMessage[],
),
};
}
/** The reasoning fields a completion would send: llama-server falls back to load-time
* --chat-template-kwargs only for omitted keys, so sending none misreports the mode. */
export function buildLocalTokenCountReasoning(
state = useChatRuntimeStore.getState(),
): Record<string, unknown> {
const {
supportsReasoning,
reasoningStyle,
reasoningEnabled,
reasoningEffort,
reasoningEffortLevels,
supportsPreserveThinking,
preserveThinking,
} = state;
// Same clamp the request build applies: a level the loaded template does not offer is dropped by the backend.
const localReasoningEffort = clampReasoningEffortToLevels(
reasoningEffort,
reasoningEffortLevels,
);
return {
...(supportsReasoning
? reasoningStyle === "enable_thinking_effort"
? reasoningEnabled
? { enable_thinking: true, reasoning_effort: localReasoningEffort }
: { enable_thinking: false }
: reasoningStyle === "reasoning_effort"
? reasoningEnabled
? { reasoning_effort: localReasoningEffort }
: {}
: { enable_thinking: reasoningEnabled }
: {}),
...(supportsPreserveThinking
? { preserve_thinking: preserveThinking }
: {}),
};
}
/** The tool flags a completion would send, so the count includes the schemas and the action nudge. */
export async function buildLocalTokenCountExtras(
threadId: string | undefined,
messages: readonly { role?: string; content?: unknown }[] = [],
): Promise<Record<string, unknown>> {
const state = useChatRuntimeStore.getState();
const {
supportsTools,
toolsEnabled,
mcpEnabledForChat,
ragEnabled,
ragSource,
ragMode,
ragTopK,
autoHealToolCalls,
bypassPermissions,
deepResearchEnabled,
permissionMode,
sandboxLevel,
maxToolCallsPerMessage,
ragAutoInject,
ragAutoInjectMinScore,
residentCheckpoint,
} = state;
const codeToolsEnabled = codeToolsOn(state);
// Explicit false, as the completion sends: omitting it lets the launcher default tools on.
// The completion always sends thread_id; the server dates a thread's prompt from it.
const threadField = {
...(threadId ? { thread_id: threadId } : {}),
max_tokens: state.params.maxTokens,
...ggufCompactionRequestFields({
isGguf: isServedByLlamaCpp({
loadedIsGguf: state.loadedIsGguf,
activeGgufVariant: state.activeGgufVariant,
activeNativePathToken: state.activeNativePathToken,
checkpoint: state.params.checkpoint,
}),
isMlx: state.loadedIsMlx === true,
isManagedEngine: state.loadedEngine !== "auto",
autoCompactEnabled: state.autoCompactEnabled,
}),
};
if (!supportsTools) {
return {
enable_tools: false,
bypass_permissions: bypassPermissions,
sandbox_level: sandboxLevel,
...threadField,
};
}
const ragProjectId = await resolveProjectId(threadId);
const projectRagEnabled = ragProjectId
? await projectHasSources(ragProjectId)
: false;
const ragOn = ragEnabled || projectRagEnabled;
// Same id the completion sends (resolveSandboxSessionId).
const sessionId = sandboxSessionIdFor(threadId, ragProjectId);
// re-fetch for every counted user turn so skills created since page load match request pricing.
await settleSkillsForText(userTexts(messages).join("\n"));
const skillTools = skillToolNames(
skillsForTools(),
codeToolsEnabled,
userTexts(messages),
);
const browserTools = browserAgentToolsForTurn();
const localMediaTools = localMediaToolsForTurn(state);
if (
!toolsEnabled &&
!codeToolsEnabled &&
!mcpEnabledForChat &&
!ragOn &&
!deepResearchEnabled &&
browserTools.length === 0 &&
localMediaTools.length === 0 &&
skillTools.length === 0
) {
// Explicit false since the server defaults tools on; --enable-tools still outranks it.
return {
enable_tools: false,
bypass_permissions: bypassPermissions,
sandbox_level: sandboxLevel,
...threadField,
};
}
return {
enable_tools: true,
// Auto-Heal off leaves leaked tool markup in the real prompt, so the count keeps it.
auto_heal_tool_calls: autoHealToolCalls,
// Ask mode gates retrieval, so the count prices a pending RAG turn rather than declining it.
permission_mode: permissionMode,
sandbox_level: sandboxLevel,
// Off suppresses the loop, and the relay renders no schemas or nudge: same zero.
max_tool_calls_per_message: maxToolCallsPerMessage,
// Full access changes tool descriptions and adds a nudge, so the count needs the flag.
bypass_permissions: bypassPermissions,
enabled_tools: [
...(ragOn ? ["search_knowledge_base"] : []),
...(toolsEnabled ? ["web_search"] : []),
...(codeToolsEnabled ? ["python", "terminal", "edit_file", "view_image"] : []),
// match the request gate so token pricing includes only sent skill tools.
...skillTools,
...browserTools,
...localMediaTools,
],
mcp_enabled: mcpEnabledForChat,
...(sessionId ? { session_id: sessionId } : {}),
// Top level, not in rag_scope: archived threads add search_conversation regardless of RAG.
...threadField,
// Armed research puts the deep_research schema in the prompt, so the count carries it.
...(deepResearchEnabled ? { deep_research_armed: true } : {}),
// Keeps the RAG tool in the prompt; no retrieval runs but scope decides whether it would.
...(ragOn
? {
rag_scope: {
...(ragEnabled && ragSource.type === "kb"
? { kb_id: ragSource.kbId }
: ragEnabled && threadId
? { thread_id: threadId }
: {}),
...(projectRagEnabled && ragProjectId
? { project_id: ragProjectId }
: {}),
// An unpersisted New Chat has neither id, and {} is falsy in Python, so the count alone would
// drop the tool and its nudge.
default_top_k: ragTopK,
mode: ragMode,
// With retrieval off the count can price exactly these messages.
autoinject: resolveAutoInject(ragAutoInject, residentCheckpoint ?? ""),
autoinject_min_score: ragAutoInjectMinScore,
...(ragAutoInject === "off" ? { whole_doc: false } : {}),
},
}
: {}),
};
}
async function resolveUseAdapter(
threadId: string | undefined,
options: OpenAIStreamAdapterOptions = {},
readThreadRecord?: ThreadRecordReader,
): Promise<boolean | undefined> {
if (options.modelType === "model1" || options.modelType === "model2") {
return undefined;
}
if (
options.pairId &&
(options.modelType === "base" || options.modelType === "lora")
) {
return options.modelType === "lora";
}
if (!threadId) {
return undefined;
}
try {
const thread = await (readThreadRecord?.() ??
getStoredChatThread(threadId));
if (!thread?.pairId) {
return undefined;
}
// compare threads skip the adapter toggle: each side loads its own model via /api/inference/load.
if (thread.modelType === "model1" || thread.modelType === "model2") {
return undefined;
}
return thread.modelType === "lora";
} catch {
return undefined;
}
}
async function resolveProjectInstructions(projectId: string | null): Promise<string> {
if (!projectId) {
return "";
}
const project = await getStoredChatProject(projectId).catch(() => null);
if (!project || project.archived) {
return "";
}
return project.instructions?.trim() ?? "";
}
// Background recounts and the send that follows them ask within moments: one read serves both.
const AGENTS_MD_TTL_MS = 2_000;
const agentsMdByProject = new Map<
string,
{ at: number; edits: number; value: Promise<AgentsMdRecord> }
>();
/** The AGENTS.md text for a request; a failed read sends what it would without the file. */
function resolveAgentsMd(projectId: string | null): Promise<AgentsMdRecord> {
const key = projectId ?? "";
const now = Date.now();
const edits = agentsMdEditCount();
const hit = agentsMdByProject.get(key);
if (hit && hit.edits === edits && now - hit.at < AGENTS_MD_TTL_MS) {
return hit.value;
}
const value = getChatAgentsMd(projectId).catch(() => EMPTY_AGENTS_MD);
agentsMdByProject.set(key, { at: now, edits, value });
return value;
}
export async function resolveChatInstructions(
threadId: string | undefined,
systemPrompt: unknown,
systemVariables: unknown,
readThreadRecord?: ThreadRecordReader,
// Only what goes to the model carries AGENTS.md; copies and exports of a chat keep its own instructions.
opts?: { agentsMd?: boolean },
): Promise<string> {
const safeSystemPrompt =
typeof systemPrompt === "string"
? resolveSystemPromptVariables(
systemPrompt,
typeof systemVariables === "string" ? systemVariables : "",
)
: "";
const projectId = await resolveProjectId(threadId, readThreadRecord);
const [projectInstructions, agentsMd] = await Promise.all([
resolveProjectInstructions(projectId),
opts?.agentsMd ? resolveAgentsMd(projectId) : undefined,
]);
return composeChatInstructions({
agentsMd,
projectInstructions,
systemPrompt: safeSystemPrompt,
});
}
// Answered once per thread and reused, so sandbox, RAG scope and instructions never mix two projects.
const composerProjectByPendingThread = new Map<string, string | null>();
/** The project the run started in, kept for the whole run; only a thread's first send records one. */
function rememberComposerProjectForRun(
threadId: string,
projectId: string | null,
): void {
// An incognito thread never has a row, so resolveProjectId answers before it reads the map.
if (isThreadIncognito(threadId)) return;
if (!composerProjectByPendingThread.has(threadId)) {
composerProjectByPendingThread.set(threadId, projectId);
}
}
export async function resolveProjectId(
threadId: string | undefined,
readThreadRecord?: ThreadRecordReader,
// A caller gating on the answer must tell "no project" from "could not read the row": one sends, the other waits.
opts?: { rethrowReadFailure?: boolean; composerProjectId?: string | null },
): Promise<string | null> {
// Read before the await: a send survives navigation to another project.
const composerProjectId =
opts?.composerProjectId !== undefined
? opts.composerProjectId
: useChatRuntimeStore.getState().activeProjectId;
if (threadId) {
let thread: ThreadRecord | undefined;
try {
thread = await (readThreadRecord?.() ?? getStoredChatThread(threadId));
} catch (error) {
// A failed lookup is not proof the row is missing, so do not adopt the composer's project.
if (opts?.rethrowReadFailure) throw error;
return null;
}
if (thread) {
composerProjectByPendingThread.delete(threadId);
return thread.projectId ?? null;
}
// initialize() does not await the row write; incognito threads are never persisted.
if (isThreadIncognito(threadId)) {
return null;
}
const pending = composerProjectByPendingThread.get(threadId);
if (pending !== undefined) {
return pending;
}
// The send records it, not this: a poll landing mid-navigation would pin the run to whichever project is on screen.
}
return composerProjectId ?? null;
}
async function resolveSandboxSessionId(
threadId: string | undefined,
readThreadRecord?: ThreadRecordReader,
): Promise<string | undefined> {
const projectId = await resolveProjectId(threadId, readThreadRecord);
return sandboxSessionIdFor(threadId, projectId);
}
function waitForModelReady(abortSignal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const check = () => {
if (abortSignal?.aborted) {
// Uncaught in the adapter, and a Stop that is not an AbortError files as a failure (#10428).
reject(abortSignal.reason ?? new DOMException("Aborted", "AbortError"));
return;
}
if (!useChatRuntimeStore.getState().modelLoading) {
resolve();
return;
}
setTimeout(check, 500);
};
check();
});
}
/** Auto-load the smallest downloaded model when none is picked; GGUF first. */
// Cap cascade so broken cached repos can't spam /api/inference/load.
const MAX_AUTO_LOAD_ATTEMPTS = 3;
// A refused or rejected preflight spends no load attempt, so it needs its own cap; passing
// preflights are bounded by MAX_AUTO_LOAD_ATTEMPTS.
const MAX_AUTO_VALIDATE_FAILURES = 12;
const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi;
const GGUF_KNOWN_QUANT_RE =
/(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)|Q[0-9]+_K|BF16|F16|F32)/i;
type AutoLoadCandidate = {
id: string;
loadId?: string | null;
kind: LastLocalModelKind;
ggufVariant: string | null;
maxSeqLength: number;
successLabel: string;
};
// Same case rules as autoLoadSourceKey: folding a POSIX target would conflate /models/Foo and /models/foo.
function autoLoadCandidateKey(
kind: LastLocalModelKind,
id: string,
ggufVariant?: string | null,
): string {
return `${kind}:${normalizeTarget(id)}:${(ggufVariant ?? "").toLowerCase()}`;
}
function hasBigEndianGgufMarker(
filename: string,
quant?: string | null,
): boolean {
const normalized = filename.replace(/\\/g, "/").toLowerCase();
const separatorIndex = normalized.lastIndexOf("/");
const basename =
separatorIndex >= 0 ? normalized.slice(separatorIndex + 1) : normalized;
const parent = separatorIndex >= 0 ? normalized.slice(0, separatorIndex) : "";
const stem = basename.replace(/\.[^.]*$/, "");
const quantKey = quant?.trim().toLowerCase() || "";
const quantIndex = quantKey ? stem.indexOf(quantKey) : -1;
const quantInParentOnly =
!!parent &&
quantIndex < 0 &&
((!!quantKey && parent.includes(quantKey)) ||
(!quantKey && GGUF_KNOWN_QUANT_RE.test(parent)));
for (const match of stem.matchAll(BIG_ENDIAN_GGUF_FILENAME_RE)) {
if (quantIndex >= 0 && quantIndex < (match.index ?? 0)) {
return true;
}
const tail = stem
.slice((match.index ?? 0) + match[0].length)
.replace(/^[._-]+/, "");
if (!tail || !GGUF_KNOWN_QUANT_RE.test(tail)) {
return !quantInParentOnly;
}
}
return false;
}
function isAutoLoadableGgufVariant(variant: GgufVariantDetail | null): boolean {
if (!variant?.filename) {
return false;
}
const filename = variant.filename.trim().toLowerCase();
if (!filename) {
return false;
}
return !hasBigEndianGgufMarker(filename, variant.quant);
}
type QueuedResolvedModelRuntime = {
checkpoint: string;
activeGgufVariant: string | null;
supportsTools: boolean;
supportsReasoning: boolean;
reasoningAlwaysOn: boolean;
reasoningStyle: ReturnType<typeof reasoningCapsFromLoad>["reasoningStyle"];
supportsReasoningOff: boolean;
reasoningEffortLevels: ReturnType<
typeof reasoningCapsFromLoad
>["reasoningEffortLevels"];
supportsPreserveThinking: boolean;
preserveThinking: boolean;
loadedContextLength: number | null;
loadedIsGguf: boolean | null;
loadedIsMlx: boolean | null;
loadedIsMultimodal: boolean;
modelCapabilities: QueuedModelCapabilities | null;
};
type ChatRuntimeState = ReturnType<typeof useChatRuntimeStore.getState>;
// A background auto-load must restore every field describing the visible chat's model, read from
// the per-model config (the live store holds session defaults).
const VISIBLE_MODEL_RUNTIME_KEYS = [
"activeLoadId",
"activeGgufVariant",
"activeModelIsLocal",
"loadedContextLength",
"loadedContextEnforced",
"loadedContextUnboundedWhenBatched",
"loadedParallelSlots",
"maxContextLength",
"nativeContextLength",
"loadedIsGguf",
"loadedIsMlx",
"modelRequiresTrustRemoteCode",
"supportsReasoning",
"reasoningAlwaysOn",
"reasoningEnabled",
"reasoningStyle",
"supportsReasoningOff",
"reasoningEffortLevels",
"supportsPreserveThinking",
"preserveThinking",
"supportsTools",
"toolsEnabled",
"codeToolsEnabled",
"kvCacheDtype",
"loadedKvCacheDtype",
"nParallel",
"loadedNParallel",
"nBatch",
"loadedNBatch",
"nUbatch",
"loadedNUbatch",
"specDraftCacheDtype",
"loadedSpecDraftCacheDtype",
"loadMode",
"loadedLoadMode",
"ctxCheckpoints",
"loadedCtxCheckpoints",
"cacheRam",
"loadedCacheRam",
"tensorParallel",
"loadedTensorParallel",
"loadedDisableVision",
"disableVision",
"loadedVisionDisabledByUser",
"gpuMemoryMode",
"loadedGpuMemoryMode",
"loadedCpuFallback",
"gpuLayers",
"loadedGpuLayers",
"nCpuMoe",
"loadedNCpuMoe",
"splitRatio",
"loadedSplitRatio",
"ggufLayerCount",
"moeLayerCount",
"selectedGpuIds",
"selectedGpuIndexKind",
"loadedGpuIds",
"loadedGpuIndexKind",
"customContextLength",
"loadedCustomContextLength",
"defaultChatTemplate",
"chatTemplateOverride",
"loadedChatTemplateOverride",
"chatTemplateOverrideReason",
// Or a background autoload leaves its width and verdict on the restored model.
"mlxKvQuant",
"loadedMlxKvQuantRequested",
"mlxKvQuantReason",
"mlxKvQuantNote",
"mlxInt8Prefill",
"loadedMlxInt8PrefillRequested",
"mlxAnePrefill",
"loadedMlxAnePrefillRequested",
"mlxFastVerify",
"loadedMlxFastVerifyRequested",
"mlxPleOffload",
"mlxExpertOffload",
"loadedMlxPleOffloadRequested",
"loadedMlxExpertOffloadRequested",
"loadedMlxPleOffload",
"loadedMlxExpertOffload",
"mlxPleOffloadReason",
"mlxExpertOffloadReason",
"loadedMlxOffloadDiskBytes",
"loadedContextBudget",
"loadedIsMultimodal",
"loadedIsDiffusion",
"speculativeType",
"loadedSpeculativeType",
"specFallbackReason",
"loadedPlacement",
"specDraftNMax",
"loadedSpecDraftNMax",
"specDraftModel",
"loadedSpecDraftModel",
] as const satisfies readonly (keyof ChatRuntimeState)[];
type VisibleModelRuntimeState = Pick<
ChatRuntimeState,
(typeof VISIBLE_MODEL_RUNTIME_KEYS)[number]
>;
type VisibleModelStateSnapshot = {
settings: ReturnType<typeof snapshotQueuedChatRunSettings>;
runtime: VisibleModelRuntimeState;
};
function snapshotVisibleModelState(
state: ChatRuntimeState,
): VisibleModelStateSnapshot {
const runtime = {} as VisibleModelRuntimeState;
for (const key of VISIBLE_MODEL_RUNTIME_KEYS) {
Object.assign(runtime, { [key]: state[key] });
}
return {
settings: snapshotQueuedChatRunSettings(state),
runtime,
};
}
function restoreVisibleModelState(snapshot: VisibleModelStateSnapshot): void {
const liveUsage = useChatRuntimeStore.getState();
liveUsage.setCheckpoint(snapshot.settings.params.checkpoint, undefined, {
trackQueuedSettings: false,
// The model being stepped off is the one the background load put there.
persist: false,
});
useChatRuntimeStore.setState({
...snapshot.runtime,
...snapshot.settings,
params: { ...snapshot.settings.params },
// A visible provider run can finish while the background model loads; keep the newer per-thread totals.
contextUsage: liveUsage.contextUsage,
contextUsageByThreadId: liveUsage.contextUsageByThreadId,
});
}
function queuedResolvedModelFromStore(
state: ChatRuntimeState,
): QueuedResolvedModelRuntime {
const activeModel = state.models.find(
(model) => model.id === state.params.checkpoint,
);
return {
checkpoint: state.params.checkpoint,
activeGgufVariant: state.activeGgufVariant,
supportsTools: state.supportsTools,
supportsReasoning: state.supportsReasoning,
reasoningAlwaysOn: state.reasoningAlwaysOn,
reasoningStyle: state.reasoningStyle,
supportsReasoningOff: state.supportsReasoningOff,
reasoningEffortLevels: state.reasoningEffortLevels,
supportsPreserveThinking: state.supportsPreserveThinking,
preserveThinking: state.preserveThinking,
loadedContextLength: state.loadedContextLength,
loadedIsGguf: state.loadedIsGguf,
loadedIsMlx: state.loadedIsMlx,
loadedIsMultimodal: state.loadedIsMultimodal,
modelCapabilities: activeModel
? {
isVision: activeModel.isVision,
isGguf: activeModel.isGguf,
// The queued run mints its own summary for uncatalogued models, and the seed is gated on this.
isMlx: activeModel.isMlx,
isAudio: activeModel.isAudio,
audioType: activeModel.audioType,
hasAudioInput: activeModel.hasAudioInput,
hasVideoInput: activeModel.hasVideoInput,
speaks: activeModel.speaks,
}
: null,
};
}
type AutoLoadOptions = {
skipAdoptServerModel?: boolean;
preserveVisibleSettings?: boolean;
captureResolvedRuntime?: (runtime: QueuedResolvedModelRuntime) => void;
abortSignal?: AbortSignal;
};
function applyAutoLoadRuntimeState(
options: AutoLoadOptions | undefined,
apply: () => void,
) {
const visibleState = options?.preserveVisibleSettings
? snapshotVisibleModelState(useChatRuntimeStore.getState())
: null;
try {
apply();
options?.captureResolvedRuntime?.(
queuedResolvedModelFromStore(useChatRuntimeStore.getState()),
);
} finally {
if (visibleState) {
restoreVisibleModelState(visibleState);
}
}
}
/** Whether a cache row is chattable. Every field is optional so an older backend keeps trying. */
function isChattableCachedRepo(repo: {
partial?: boolean;
task?: string | null;
model_format?: string | null;
capabilities?: { can_chat?: boolean } | null;
}): boolean {
return (
repo.partial !== true &&
repo.capabilities?.can_chat !== false &&
// Adapters fetch base weights from the Hub, and can_chat is true on file layout alone.
repo.model_format !== "adapter" &&
// Cached diffusion and speech repos report can_chat true on file format alone.
!NON_CHAT_TASKS.has(repo.task ?? "")
);
}
// Chat models are tagged text-generation or null, hence a denylist; audio tasks reply with speech.
const NON_CHAT_TASKS: ReadonlySet<string> = new Set([
"text-to-image",
"text-to-video",
"image-diffusion-unsupported",
"text-to-speech",
"text-to-audio",
"audio-to-audio",
"automatic-speech-recognition",
"speech-recognition-unsupported",
]);
// ollama is excluded: local_model_resolver.py skips its scanner. hermes is read-only, so included.
const AUTO_LOAD_LOCAL_SOURCES: ReadonlySet<string> = new Set([
"models_dir",
"lmstudio",
"omlx",
"hermes",
"custom",
]);
/** Excludes adapters (base fetched from Hub) and scan-folder checkpoints (pickle, no Hub scan). */
function isAutoLoadableLocalRow(
row: LocalModelInfo,
// Set when a cached lookup failed: the excluded hf_cache rows are then the only evidence of that cache.
admitHfCache = false,
): boolean {
return (
(AUTO_LOAD_LOCAL_SOURCES.has(row.source) ||
(admitHfCache && row.source === "hf_cache")) &&
// Absent capabilities means unclassified, not "not chat": requiring true fell through to downloading the default.
row.capabilities?.can_chat !== false &&
row.partial !== true &&
// Allowlist: backends send "unknown" or omit the field, so a denylist admitted pickles.
(isGgufLocalRow(row) || row.model_format === "safetensors") &&
!NON_CHAT_TASKS.has(row.task ?? "") &&
runsOnThisPlatform(row) &&
!isHiddenModelId(row.model_id, row.id, row.path)
);
}
// model_format is optional on older backends, so also check the .gguf path.
function isGgufLocalRow(row: LocalModelInfo): boolean {
return (
row.model_format === "gguf" || row.path.toLowerCase().endsWith(".gguf")
);
}
/** Chat-only installs run GGUF anywhere and MLX on a Mac; the picker hides every other local format there. */
function runsOnThisPlatform(row: LocalModelInfo): boolean {
const platform = usePlatformStore.getState();
// chatOnly is a guess until the backend measures it (deferred detection included); fail open, since validation refuses an ineligible pick.
if (platform.capabilitiesUnknown() || !platform.isChatOnly()) return true;
if (isGgufLocalRow(row)) {
return true;
}
return (
platform.deviceType === "mac" &&
(isMlxId(row.id) ||
isMlxId(row.display_name ?? "") ||
isMlxId(row.model_id ?? ""))
);
}
/** The picker hides cached non-GGUF rows outright on a chat-only install; same wait-for-the-server rule as above. */
function cachedModelsRunOnThisPlatform(): boolean {
const platform = usePlatformStore.getState();
return platform.capabilitiesUnknown() || !platform.isChatOnly();
}
/** One loadable thing on this device; `listVariants` is lazy, so only the repos the cascade reaches are scanned. */
type AutoLoadSource = {
kind: LastLocalModelKind;
id: string;
/** Sent to /api/inference/load as model_path. */
loadId: string;
sizeBytes: number;
maxSeqLength: number;
/** null when the target is one file already (a local .gguf) or safetensors. */
listVariants: (() => Promise<GgufVariantDetail[]>) | null;
};
// Case-sensitive targets keep their case; repo ids and Windows paths fold, separators included.
// NFC first: macOS returns decomposed names.
function normalizeTarget(value: string): string {
const target = value.trim().normalize("NFC");
if (/^[A-Za-z]:[\\/]/.test(target) || target.startsWith("\\\\")) {
const slashed = target.replace(/\\/g, "/");
return /^\/\/wsl[$.]/i.test(slashed) ? slashed : slashed.toLowerCase();
}
return /^[/~]/.test(target) ? target : target.toLowerCase();
}
// Key by load target only so a repo in both lists does not spend two attempts.
function autoLoadSourceKey(source: AutoLoadSource): string {
return normalizeTarget(source.loadId);
}
function buildAutoLoadSources(
ggufRepos: CachedGgufRepo[],
modelRepos: CachedModelRepo[],
localRows: LocalModelInfo[],
safetensorsMaxSeqLength: number,
// Already timeout-bounded, but a stopped send should not wait one out.
signal?: AbortSignal,
): AutoLoadSource[] {
const sources: AutoLoadSource[] = [];
for (const repo of ggufRepos) {
sources.push({
kind: "gguf",
id: repo.repo_id,
loadId: repo.load_id || repo.repo_id,
sizeBytes: repo.size_bytes,
maxSeqLength: 0,
listVariants: () =>
listGgufVariants(repo.repo_id, undefined, {
preferLocalCache: true,
localPath: repo.load_id || repo.cache_path,
signal,
}).then((response) => response.variants),
});
}
for (const repo of modelRepos) {
sources.push({
kind: "model",
id: repo.repo_id,
loadId: repo.load_id || repo.repo_id,
sizeBytes: repo.size_bytes,
maxSeqLength: safetensorsMaxSeqLength,
listVariants: null,
});
}
for (const row of localRows) {
const isGguf = isGgufLocalRow(row);
// A quant folder needs one picked; a .gguf path is already the file.
const needsVariant = isGguf && row.capabilities?.requires_variant === true;
sources.push({
kind: isGguf ? "gguf" : "model",
id: row.id,
loadId: row.load_id || row.id,
sizeBytes: row.size_bytes ?? 0,
maxSeqLength: isGguf ? 0 : safetensorsMaxSeqLength,
listVariants: needsVariant
? () =>
listGgufVariants(row.path, undefined, {
preferLocalCache: true,
localPath: row.path,
signal,
}).then((response) => response.variants)
: null,
});
}
return sources;
}
function isRememberedSource(
source: AutoLoadSource,
remembered: { id: string; kind: LastLocalModelKind },
): boolean {
if (source.kind !== remembered.kind) return false;
// Same case rules as the dedupe key.
const target = normalizeTarget(remembered.id);
return (
normalizeTarget(source.id) === target ||
normalizeTarget(source.loadId) === target
);
}
/** Last used first, then GGUF before safetensors, then smallest first. */
function orderAutoLoadSources(
sources: AutoLoadSource[],
remembered: { id: string; kind: LastLocalModelKind } | null,
): AutoLoadSource[] {
const rank = (source: AutoLoadSource): number => {
if (remembered && isRememberedSource(source, remembered)) return 0;
return source.kind === "gguf" ? 1 : 2;
};
// Unknown sizes sort last so a sizeless row cannot shadow a real one.
const size = (source: AutoLoadSource): number =>
source.sizeBytes > 0 ? source.sizeBytes : Number.MAX_SAFE_INTEGER;
// Sort twins behind rather than drop: a GGUF twin with no quant would lose its safetensors row.
return [...sources].sort((a, b) => rank(a) - rank(b) || size(a) - size(b));
}
/** Remembered quant first, then smallest, skipping tried quants; null when nothing loads. */
async function resolveAutoLoadCandidate(
source: AutoLoadSource,
rememberedVariant: string | null,
isSkipped: (candidate: AutoLoadCandidate) => boolean,
): Promise<AutoLoadCandidate | null> {
const build = (ggufVariant: string | null): AutoLoadCandidate => ({
id: source.id,
loadId: source.loadId,
kind: source.kind,
ggufVariant,
maxSeqLength: source.maxSeqLength,
successLabel: ggufVariant
? `Loaded ${source.id} (${ggufVariant})`
: `Loaded ${source.id}`,
});
if (!source.listVariants) {
const variants =
source.kind === "model" && rememberedVariant
? [rememberedVariant, null]
: [null];
for (const variant of variants) {
const candidate = build(variant);
if (!isSkipped(candidate)) return candidate;
}
return null;
}
const downloaded = (await source.listVariants())
.filter(
(variant) =>
variant.downloaded &&
variant.partial !== true &&
isAutoLoadableGgufVariant(variant),
)
.sort((a, b) => a.size_bytes - b.size_bytes);
const wanted = rememberedVariant?.trim().toLowerCase();
const ordered = wanted
? [
...downloaded.filter((v) => v.quant?.toLowerCase() === wanted),
...downloaded.filter((v) => v.quant?.toLowerCase() !== wanted),
]
: downloaded;
for (const variant of ordered) {
const candidate = build(variant.quant);
if (!isSkipped(candidate)) return candidate;
}
return null;
}
// Fetched only when the device genuinely has nothing on it.
const DEFAULT_CHAT_MODEL_REPO = "unsloth/gemma-4-E2B-it-GGUF";
const DEFAULT_CHAT_MODEL_VARIANT = "UD-Q4_K_XL";
const DEFAULT_CHAT_MODEL_LABEL = "Gemma 4 E2B";
function formatDownloadBytes(bytes: number): string {
if (!(bytes > 0)) return "";
const gb = bytes / 1000 ** 3;
return gb >= 1
? `${gb.toFixed(1)} GB`
: `${Math.max(1, Math.round(bytes / 1000 ** 2))} MB`;
}
/** Use the download manager instead of /load for a panel entry, progress and Cancel. */
async function ensureDefaultModelDownloaded(
hfToken: string | null,
abortSignal: AbortSignal | undefined,
setToast: (title: string, description: string, cancel?: () => void) => void,
): Promise<"ready" | "cancelled" | "failed"> {
const variantKey = DEFAULT_CHAT_MODEL_VARIANT.toLowerCase();
let expectedBytes = 0;
try {
const listing = await listGgufVariants(DEFAULT_CHAT_MODEL_REPO, undefined, {
signal: abortSignal,
});
const variant = listing.variants.find(
(entry) => entry.quant?.toLowerCase() === variantKey,
);
if (variant?.downloaded && variant.partial !== true) return "ready";
expectedBytes = variant?.download_size_bytes || variant?.size_bytes || 0;
} catch {
// Sizing is cosmetic; the manager learns the real total from the server.
}
abortSignal?.throwIfAborted();
// startJob sends the token raw, so an expired one breaks public downloads; after disk check.
const prepared = await prepareHfTokenForUse(hfToken);
abortSignal?.throwIfAborted();
if (!prepared.proceed) return "cancelled";
const request = {
kind: DOWNLOAD_KIND.MODEL,
repoId: DEFAULT_CHAT_MODEL_REPO,
variant: DEFAULT_CHAT_MODEL_VARIANT,
expectedBytes,
};
const jobKey = jobKeyOf(request.kind, request.repoId, request.variant);
// cancel() no-ops before the job exists, so remember the click and replay it.
let cancelRequested = false;
let cancelInFlight = false;
let cancelEverIssued = false;
// The latch clears when the request settles, since a failed cancel leaves the job running for a retry.
const issueCancel = (): boolean => {
const active = selectActiveJob(
useDownloadManagerStore.getState(),
request.kind,
request.repoId,
request.variant,
);
if (!active) return false;
if (cancelInFlight || active.state === "cancelling") return true;
cancelInFlight = true;
cancelEverIssued = true;
void downloadManager.cancel(active.key).finally(() => {
cancelInFlight = false;
});
return true;
};
const cancelDownload = (): void => {
cancelRequested = true;
issueCancel();
};
const totalLabel = formatDownloadBytes(expectedBytes);
const description =
`Unsloth couldn’t find an existing model. Unsloth is now getting ` +
`${DEFAULT_CHAT_MODEL_LABEL} ready for use. You can stop the download or ` +
`manage models later in the 'Model hub'`;
setToast(
`Getting ${DEFAULT_CHAT_MODEL_LABEL} ready`,
description,
cancelDownload,
);
const isOurVariant = (variant: string | null): boolean =>
(variant ?? "").toLowerCase() === variantKey;
return await new Promise<"ready" | "cancelled" | "failed">((resolve) => {
let settled = false;
const cleanups: Array<() => void> = [];
const finish = (outcome: "ready" | "cancelled" | "failed"): void => {
if (settled) return;
settled = true;
for (const cleanup of cleanups) cleanup();
resolve(outcome);
};
// Before the start request, so a fast completion cannot be missed.
cleanups.push(
subscribeJobListeners(request.kind, request.repoId, {
// A cancel can fail and the transfer finish anyway; the bytes are still not handed to a load.
onComplete: (variant) =>
isOurVariant(variant) &&
finish(cancelRequested ? "cancelled" : "ready"),
onCancelled: (variant) => isOurVariant(variant) && finish("cancelled"),
onError: (variant) => isOurVariant(variant) && finish("failed"),
}),
);
cleanups.push(
useDownloadManagerStore.subscribe((state) => {
const job = state.jobs[jobKey];
if (!job) return;
// Cancelled during the preflights: stop it as soon as it exists, once, not on every store tick.
if (cancelRequested) {
if (!cancelEverIssued) issueCancel();
return;
}
if (job.state === "cancelling") return;
const done = formatDownloadBytes(job.downloadedBytes);
const total = formatDownloadBytes(job.expectedBytes) || totalLabel;
if (!done || !total) return;
setToast(
`Getting ${DEFAULT_CHAT_MODEL_LABEL} ready (${done} of ${total})`,
description,
cancelDownload,
);
}),
);
// Stopping the send stops the wait, not the transfer.
if (abortSignal) {
const onAbort = () => finish("cancelled");
abortSignal.addEventListener("abort", onAbort, { once: true });
cleanups.push(() => abortSignal.removeEventListener("abort", onAbort));
}
void downloadManager.requestStart(request).then(
(outcome) => {
if (outcome === "started") {
// Cancelled while the start was still in flight.
if (cancelRequested && !issueCancel()) finish("cancelled");
return;
}
if (cancelRequested) {
finish("cancelled");
return;
}
// "conflict" and "busy" need the Hub download card; "error" already toasted.
finish("failed");
},
() => finish("failed"),
);
});
}
// The test harness slices helpers from here; keep them below MAX_AUTO_LOAD_ATTEMPTS.
// Slow downloads and llama-server warm-up need a long cap.
const CLI_LOAD_ADOPT_MAX_MS = 600_000;
type ServerLoadBlocked = "still-loading" | "status-unavailable";
type SettledServerStatus =
| { outcome: "settled"; status: InferenceStatusResponse }
| { outcome: "stopped"; status?: undefined }
| { outcome: ServerLoadBlocked; status?: undefined };
/**
* `loading` statuses still name the outgoing model, so never read residency off them.
* `stopEarly` abandons the wait; callers without it never see that outcome.
*/
async function waitForSettledServerStatus(options: {
abortSignal?: AbortSignal;
stopEarly: () => boolean;
}): Promise<SettledServerStatus>;
async function waitForSettledServerStatus(options?: {
abortSignal?: AbortSignal;
}): Promise<Exclude<SettledServerStatus, { outcome: "stopped" }>>;
async function waitForSettledServerStatus(options?: {
abortSignal?: AbortSignal;
stopEarly?: () => boolean;
}): Promise<SettledServerStatus> {
const deadline = Date.now() + CLI_LOAD_ADOPT_MAX_MS;
let failures = 0;
let announced = false;
// Own settlement here so a refresh cannot publish a mid-replacement status as the pick.
const release = beginServerModelWait(options?.abortSignal);
try {
for (;;) {
options?.abortSignal?.throwIfAborted();
if (options?.stopEarly?.()) return { outcome: "stopped" };
let status: InferenceStatusResponse | null = null;
// Capped so a half-open read cannot hold the loading lease past the deadline.
const poll = statusPollSignal(options?.abortSignal);
try {
status = await getInferenceStatus(poll.signal);
failures = 0;
} catch {
// Cancellation, not a server that cannot answer: it must not become a toast.
options?.abortSignal?.throwIfAborted();
// A failed or timed-out read is not proof of idle; two in a row end the wait.
if (++failures >= 2) return { outcome: "status-unavailable" };
} finally {
poll.dispose();
}
options?.abortSignal?.throwIfAborted();
if (options?.stopEarly?.()) return { outcome: "stopped" };
if (status && (status.loading?.length ?? 0) === 0) {
return { outcome: "settled", status };
}
if (status && !announced) {
toast.info("Waiting for model to finish loading…");
announced = true;
}
if (Date.now() >= deadline) return { outcome: "still-loading" };
await new Promise((resolve) => setTimeout(resolve, 500));
}
} finally {
release();
}
}
/** Name the reason nothing loaded, so the caller's generic advice stays suppressed. */
function reportBlockedServerLoad(outcome: ServerLoadBlocked): void {
const stillLoading = outcome === "still-loading";
toast.error(
stillLoading ? "A model is still loading" : "Could not reach the model server",
{
description: stillLoading
? "Send again once it finishes, or pick a model in the top bar."
: "Check that the server is running, then send again.",
},
);
}
type CliLoadAdoption = "adopted" | "server-idle" | ServerLoadBlocked;
async function adoptInFlightServerLoad(
abortSignal?: AbortSignal,
): Promise<CliLoadAdoption> {
const settled = await waitForSettledServerStatus({
abortSignal,
stopEarly: () => Boolean(useChatRuntimeStore.getState().params.checkpoint),
});
if (settled.outcome === "stopped") return "adopted";
if (settled.outcome !== "settled") return settled.outcome;
return (await tryAdoptServerActiveModel({
allowWhileModelLoading: true,
status: settled.status,
}))
? "adopted"
: "server-idle";
}
// Busy-host detection outlasts readHealth's one 5 s wait; past this bound autoload stays fail-open.
const AUTO_LOAD_VERDICT_WAIT_MS = 60_000;
const AUTO_LOAD_VERDICT_POLL_MS = 500;
async function awaitMeasuredVerdict(signal?: AbortSignal): Promise<void> {
const deadline = Date.now() + AUTO_LOAD_VERDICT_WAIT_MS;
// readHealth has no timeout or signal: a stalled read must not outlive the deadline or a stopped send.
const bounded = (read: () => Promise<unknown>) =>
new Promise<"ok" | "failed" | "timeout">((resolve, reject) => {
const timer = setTimeout(() => settle("timeout"), Math.max(0, deadline - Date.now()));
const onAbort = () => {
cleanup();
reject(signal?.reason);
};
function cleanup() {
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
}
function settle(outcome: "ok" | "failed" | "timeout") {
cleanup();
resolve(outcome);
}
signal?.addEventListener("abort", onAbort, { once: true });
read().then(
() => settle("ok"),
() => settle("failed"),
);
});
signal?.throwIfAborted();
if ((await bounded(() => fetchDeviceType({ detectHardware: true }))) !== "ok") return;
while (Date.now() < deadline) {
signal?.throwIfAborted();
const platform = usePlatformStore.getState();
if (!platform.capabilitiesUnknown() || platform.detectionDeferred) return;
await new Promise((resolve) => setTimeout(resolve, AUTO_LOAD_VERDICT_POLL_MS));
signal?.throwIfAborted();
if ((await bounded(() => fetchDeviceType({ force: true }))) !== "ok") return;
}
}
async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
loaded: boolean;
blockedByTrustRemoteCode: boolean;
/** A load failure was already toasted, so callers must not replace it with generic advice. */
loadFailureReported?: boolean;
}> {
options?.abortSignal?.throwIfAborted();
if (!options?.skipAdoptServerModel) {
const adoption = await adoptInFlightServerLoad(options?.abortSignal);
options?.abortSignal?.throwIfAborted();
if (adoption === "adopted") {
return { loaded: true, blockedByTrustRemoteCode: false };
}
if (adoption !== "server-idle") {
// Never auto-load unless status confirmed the server is idle.
reportBlockedServerLoad(adoption);
return {
loaded: false,
blockedByTrustRemoteCode: false,
loadFailureReported: true,
};
}
}
const store = useChatRuntimeStore.getState();
const hfToken = store.hfToken || null;
const trustRemoteCode = store.params.trustRemoteCode ?? false;
const specSettings = resolveSpeculativeSettingsForLoad();
const lastLoaded = await readLastLocalModelLoad(options?.abortSignal);
let autoLoadToastDismissed = false;
const toastId = toast.message("Loading a model…", {
description: lastLoaded
? "Loading last used model."
: "Auto-selecting the smallest downloaded model.",
duration: Number.POSITIVE_INFINITY,
closeButton: true,
icon: createLoadingToastIcon(),
onDismiss: () => {
autoLoadToastDismissed = true;
},
});
const updateAutoLoadToast = (
message: string,
description: string,
onCancel?: () => void,
): void => {
if (autoLoadToastDismissed) return;
toast.message(message, {
id: toastId,
description,
duration: Number.POSITIVE_INFINITY,
action: onCancel
? { label: "Cancel", onClick: () => onCancel() }
: undefined,
});
};
// An auto-load that lost GPU acceleration must not read as a plain success.
const showAutoLoadSuccess = (
message: string,
cpuFallbackReason?: CpuFallbackReason | null,
mmprojFallbackReason?: MmprojFallbackReason | null,
offloadCounts?: OffloadCounts,
): void => {
// Both reasons composed (mmproj and CPU); see loadFallbackNotice.
const notice = loadFallbackNotice(
message,
cpuFallbackReason,
mmprojFallbackReason,
offloadWarning(offloadCounts ?? {}),
expertSpillNote(offloadCounts ?? {}),
);
const options = {
description: notice.description,
duration: 5000,
icon: undefined,
};
const showToast = notice.degraded ? toast.warning : toast.success;
const title = notice.title;
if (autoLoadToastDismissed) {
showToast(title, options);
return;
}
showToast(title, { ...options, id: toastId });
};
let blockedByTrustRemoteCode = false;
let hadNonTrustFailure = false;
let loadAttempts = 0;
// Per cascade; a module-level counter would leave later auto-loads with a spent budget.
let validateFailures = 0;
const skippedAutoLoadCandidates = new Set<string>();
// Why the last load attempt failed. Boxed: a `let` set only in a nested fn narrows to `null`.
const loadFailure: {
current: { label: string; detail: string; blamesModel: boolean } | null;
} = { current: null };
// Set once the user declines the HF token dialog, so nothing asks again.
let autoLoadCancelled = false;
function noteLoadFailure(label: string, error: unknown): void {
const detail =
error instanceof Error && error.message.trim()
? error.message.trim()
: "";
// loadModel can reject before /load is sent; those must not blame the model.
const marker = error as {
unslothTransportFailure?: boolean;
unslothUserCancelled?: boolean;
};
const blamesModel = !(
marker?.unslothTransportFailure === true ||
marker?.unslothUserCancelled === true
);
loadFailure.current = {
label,
// Older backends and non-Error throws carry no detail; still name the model that failed.
detail:
detail || "The server did not report a reason. Check the Unsloth logs.",
blamesModel,
};
}
async function canAutoLoad(request: LoadModelRequest): Promise<boolean> {
// Before the POST, so an abort costs nothing: no request was sent.
options?.abortSignal?.throwIfAborted();
const validation = await validateModel(request).catch((error: unknown) => {
// A rejected /validate never reaches loadAttempts, so count it here or a dead backend POSTs
// once per cached repo.
validateFailures += 1;
throw error;
});
options?.abortSignal?.throwIfAborted();
// Background auto-load never runs custom code or unsafe files; those need explicit consent.
if (
validation.requires_trust_remote_code ||
validation.requires_security_review
) {
blockedByTrustRemoteCode = true;
validateFailures += 1;
return false;
}
// Never install packages from a background load; explicit loads raise the upgrade dialog.
if (validation.requires_transformers_upgrade) {
hadNonTrustFailure = true;
validateFailures += 1;
return false;
}
// Engine choice is the picker dialog's; the default engine would decompress and crawl (#8861).
if (validation.managed_engine_offer) {
hadNonTrustFailure = true;
validateFailures += 1;
return false;
}
return true;
}
function recordCandidateFailure(label: string, error: unknown): void {
// Record every rejection: the sweep's bare catches would read it as nothing cached.
const marker = error as { unslothUserCancelled?: boolean };
noteLoadFailure(label, error);
if (marker?.unslothUserCancelled === true) {
// Stop instead of reopening the dialog per repo; transport failures go on, the backend may return.
autoLoadCancelled = true;
}
}
async function canAutoLoadRecordingFailures(
label: string,
request: LoadModelRequest,
): Promise<boolean> {
try {
return await canAutoLoad(request);
} catch (error) {
recordCandidateFailure(label, error);
throw error;
}
}
async function loadAutoLoadCandidate(
candidate: AutoLoadCandidate,
): Promise<boolean> {
if (
autoLoadCancelled ||
loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS ||
validateFailures >= MAX_AUTO_VALIDATE_FAILURES
) {
return false;
}
const currentStore = useChatRuntimeStore.getState();
const modelPath = candidate.loadId ?? candidate.id;
const failureLabel = candidate.ggufVariant
? `${candidate.id} (${candidate.ggufVariant})`
: candidate.id;
const { config } = resolveInitialConfig(
candidate.id,
candidate.ggufVariant,
);
const platform = usePlatformStore.getState();
const effectiveMaxSeqLength = resolveLoadMaxSeqLength({
modelId: candidate.id,
ggufVariant: candidate.ggufVariant,
isGguf: candidate.kind === "gguf",
customContextLength: config.customContextLength,
loadedContextLength: null,
currentCheckpoint: currentStore.params.checkpoint,
activeGgufVariant: currentStore.activeGgufVariant,
isMlx: isServedByMlx(
candidate.kind === "gguf",
platform.deviceType,
platform.chatOnlyReason,
),
pinnedMaxSeqLength: config.maxSeqLength,
defaultMaxSeqLength: candidate.maxSeqLength,
presetSource: currentStore.activePresetSource,
});
// GPU knobs are per-model; the live store holds session defaults during background loads.
const effectiveGpuMemoryMode =
config.gpuMemoryMode ?? currentStore.gpuMemoryMode;
const effectiveGpuLayers = config.gpuLayers ?? GPU_LAYERS_AUTO;
const effectiveNCpuMoe = config.nCpuMoe ?? 0;
if (config.selectedGpuIds != null) {
// Warm the device cache first: on a cold cache the reconcile passes a stale cross-host pick through unvalidated.
await ensureGpuDeviceCache();
}
let isDiffusion = false;
if (
candidate.kind === "gguf" &&
(config.selectedGpuIds != null || config.tensorParallel === true)
) {
// The Hub 401s an invalid Authorization header even for public repos.
const preparedToken = await prepareHfTokenForUse(hfToken);
if (!preparedToken.proceed) {
// Raised before loadModel, so route it through the same helper or every later candidate reopens the dialog.
const cancelled = Object.assign(new Error("Model load cancelled."), {
unslothUserCancelled: true,
});
recordCandidateFailure(failureLabel, cancelled);
throw cancelled;
}
isDiffusion = (
await fetchGgufStagedMetadata({
model_path: modelPath,
gguf_variant: candidate.ggufVariant,
hf_token: preparedToken.token,
}).catch((error: unknown) => {
// Same authFetch as validate and load, so a dead backend throws here first.
recordCandidateFailure(failureLabel, error);
throw error;
})
).isDiffusion;
}
// The override may live only on the server, and nothing is resident at startup to inherit;
// sanitized into an explicit list /load validates strictly.
let resolvedExtraArgs = config.llamaExtraArgs;
if (candidate.kind === "gguf" && !isDiffusion) {
try {
const managed = await loadManagedLlamaFlags();
const clean = (tokens: readonly string[]) =>
sanitizeStoredExtraArgs(tokens, managed?.managed ?? new Set<string>(), {
maxBytes: managed?.maxBytes,
windowsCommandBudget: managed?.windowsCommandBudget,
});
if (resolvedExtraArgs === undefined) {
const stored = await fetchLoadExtraArgs(
modelPath,
// Cached inventory can resolve to a different loadId than the advertised repo id.
candidate.id,
candidate.ggufVariant ?? null,
);
if (stored.tokens.length > 0) {
const cleaned = clean(stored.tokens);
if (cleaned.length > 0) {
resolvedExtraArgs = cleaned;
}
} else if (stored.explicit) {
// An empty list clears; omitting the field would let /load carry old arguments over.
resolvedExtraArgs = [];
}
} else if (resolvedExtraArgs !== null && resolvedExtraArgs.length > 0) {
// Clean the local copy too: newly managed flags would be sent explicitly and get a 400.
const cleaned = clean(resolvedExtraArgs);
if (cleaned.length !== resolvedExtraArgs.length) {
resolvedExtraArgs = cleaned.length > 0 ? cleaned : [];
}
}
} catch {
// An overrides outage must not stop a startup load.
}
}
const effectiveTensorParallel = isDiffusion ? false : config.tensorParallel;
// The diffusion runner has no projector to skip, so the toggle is inert there.
const effectiveDisableVision = isDiffusion ? false : config.disableVision;
const effectiveGpuIds =
config.selectedGpuIds !== undefined
? reconcilePersistedGpuIds(
config.selectedGpuIds,
config.selectedGpuIndexKind,
isDiffusion,
)
: null;
// Reconciled against the raw pick: a ratio only applies to the GPU set it was saved with.
const effectiveTensorSplit = reconcileTensorSplit(
config.tensorSplit,
config.selectedGpuIds,
effectiveGpuIds,
);
// Manual memory with Auto layers lets llama.cpp --fit size context; send 0 or the pin (GGUF).
const fitMaxSeqLength = resolveFitMaxSeqLength(
candidate.kind === "gguf",
effectiveGpuMemoryMode,
effectiveGpuLayers,
config.customContextLength ?? null,
effectiveMaxSeqLength,
);
const candidateIsMlx = isServedByMlx(
candidate.kind === "gguf",
platform.deviceType,
platform.chatOnlyReason,
);
// MLX falls back to the standing preference: the live store may hold another model's settings.
const standingSpec = candidateIsMlx
? resolveSpeculativeSettingsForLoad({ usePersistedPreference: true })
: specSettings;
const effectiveSpeculativeType = resolveSpeculativeType(
config.speculativeType,
standingSpec.speculativeType ?? "auto",
candidateIsMlx,
);
const effectiveSpecDraftNMax =
config.specDraftNMax ?? standingSpec.specDraftNMax;
const mlxDrafter = candidateIsMlx
? { spec_draft_model: config.specDraftModel ?? null }
: {};
const effectiveChatTemplateOverride = config.chatTemplateOverride?.trim()
? config.chatTemplateOverride
: null;
const request: LoadModelRequest = {
model_path: modelPath,
hf_token: hfToken,
max_seq_length: fitMaxSeqLength,
load_in_4bit: config.loadIn4bit !== false,
is_lora: false,
gguf_variant: candidate.ggufVariant,
trust_remote_code: trustRemoteCode,
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: config.kvCacheDtype,
mlx_kv_quant: config.mlxKvQuant ?? null,
mlx_int8_prefill: config.mlxInt8Prefill ?? false,
mlx_ane_prefill: config.mlxAnePrefill ?? false,
mlx_fast_verify: config.mlxFastVerify ?? false,
mlx_ple_offload: config.mlxPleOffload ?? null,
mlx_expert_offload: config.mlxExpertOffload ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
...mlxDrafter,
tensor_parallel: effectiveTensorParallel,
disable_vision: effectiveDisableVision,
n_parallel: config.nParallel ?? null,
// Remembered pass-through args: nothing is resident at startup to inherit them from.
...llamaServerLoadFields(
{ ...config, llamaExtraArgs: resolvedExtraArgs },
{ isGguf: candidate.kind === "gguf", isDiffusion },
),
// GGUF-only. A remembered manual DiffusionGemma split (0 especially) must not be refused as a
// full-GGUF occupant.
...(candidate.kind === "gguf"
? {
gpu_memory_mode: effectiveGpuMemoryMode,
gpu_layers: effectiveGpuLayers,
n_cpu_moe: effectiveNCpuMoe,
tensor_split: effectiveTensorSplit ?? undefined,
gpu_ids: effectiveGpuIds ?? undefined,
}
: {}),
};
if (!(await canAutoLoadRecordingFailures(failureLabel, request))) {
skippedAutoLoadCandidates.add(
autoLoadCandidateKey(
candidate.kind,
// A cached repo and a local row can alias the same files, so blocking one must block the other.
candidate.loadId ?? candidate.id,
candidate.ggufVariant,
),
);
return false;
}
const autoLoadPin = retainedContextPin({
isMlx: isServedByMlx(
candidate.kind === "gguf",
platform.deviceType,
platform.chatOnlyReason,
),
// What the load asked for, so a pin carried in either field is kept.
requestedContextLength: fitMaxSeqLength,
});
loadAttempts += 1;
options?.abortSignal?.throwIfAborted();
// Keep a speech model loaded for read aloud when Keep multiple models loaded allows it.
const loadResp = await loadModel({ ...request, alongside: true }).catch((error: unknown) => {
// The sweep's parameterless catches discard this error, so record it before rethrowing.
noteLoadFailure(failureLabel, error);
throw error;
});
reportEvicted(loadResp);
// Do not apply this load to the visible runtime once its queue was cancelled; still await
// /load so the lifecycle stays serialized.
options?.abortSignal?.throwIfAborted();
applyAutoLoadRuntimeState(options, () => {
// Persist the global preference only when the value came from global settings, or autoloading
// a remembered model rewrites the default. MLX only reads it.
if (config.speculativeType == null && !candidateIsMlx) {
saveSpeculativeType(effectiveSpeculativeType);
}
// Self-gates on is_gguf (skips diffusion), so persists only for a real GGUF load.
persistGpuMemoryModeOnLoad(loadResp, effectiveGpuMemoryMode);
const loadedModelId = loadResp.model || modelPath;
// The identity stays the backend's; the pin is recorded alongside it so a later reload finds the same directory.
useChatRuntimeStore.setState({
activeLoadId: modelPath === candidate.id ? null : modelPath,
});
useChatRuntimeStore
.getState()
.setCheckpoint(loadedModelId, candidate.ggufVariant ?? undefined, {
trackQueuedSettings: !options?.preserveVisibleSettings,
});
const store = useChatRuntimeStore.getState();
store.setModelRequiresTrustRemoteCode(
loadResp.requires_trust_remote_code ?? false,
);
// Neither backend's request works: when self-sizing it is the auto-size sentinel.
const loadedWindow = loadedContextForParams(
loadedContextFields(loadResp).loadedContextLength,
effectiveMaxSeqLength,
store.params.maxSeqLength,
);
store.setParams(
{
...store.params,
...(candidate.kind === "gguf" ? {} : { maxSeqLength: loadedWindow }),
// Through the ceiling so value and slider agree; the app default could halve Max Tokens.
maxTokens: localMaxTokensCeiling(
loadedContextFields(loadResp).loadedContextLength,
loadedWindow,
),
},
{
persist: !options?.preserveVisibleSettings,
trackQueuedSettings: !options?.preserveVisibleSettings,
fromModelDefaults: true,
// A budget remembered from a larger context may not fit; cap by the window, not the request.
maxTokensCap: replayMaxTokensCap(
candidate.kind === "gguf"
? loadedContextFields(loadResp).loadedContextLength
: loadedWindow,
),
},
);
// Upsert: the load response is the authority on audio capability, which a pre-load catalog entry lacks.
syncModelCapabilities(loadedModelId, {
...loadResp,
display_name: loadResp.display_name ?? candidate.id,
is_gguf: loadResp.is_gguf ?? candidate.kind === "gguf",
});
const committedSlots =
(loadResp.is_diffusion ?? false) ? null : (config.nParallel ?? null);
if (candidate.kind === "gguf") {
// Saved value, not fitMaxSeqLength: the wire value is Auto-resolved and would pin Auto.
const keepCustomCtx = resolveExplicitCtxPin(config.customContextLength);
// same rule for the batch sizes
const committedNBatch =
(loadResp.is_diffusion ?? false) ? null : (config.nBatch ?? null);
const committedNUbatch =
(loadResp.is_diffusion ?? false) ? null : (config.nUbatch ?? null);
useChatRuntimeStore.setState({
...loadedContextFields(loadResp),
supportsReasoning: loadResp.supports_reasoning ?? false,
reasoningAlwaysOn: loadResp.reasoning_always_on ?? false,
reasoningEnabled: loadResp.supports_reasoning ?? false,
...reasoningCapsFromLoad(loadResp),
supportsPreserveThinking:
loadResp.supports_preserve_thinking ?? false,
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
kvCacheDtype: loadResp.cache_type_kv ?? null,
loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
...mlxRuntimeStateFrom(loadResp),
// Click-time value, not the resolved backend echo (see performLoad).
nParallel: committedSlots,
loadedNParallel: committedSlots,
reasoningBudget:
(loadResp.is_diffusion ?? false)
? -1
: (loadResp.reasoning_budget ?? config.reasoningBudget),
loadedReasoningBudget:
(loadResp.is_diffusion ?? false)
? -1
: (loadResp.reasoning_budget ?? config.reasoningBudget),
reasoningBudgetMessage:
(loadResp.is_diffusion ?? false)
? ""
: (loadResp.reasoning_budget_message ??
config.reasoningBudgetMessage),
loadedReasoningBudgetMessage:
(loadResp.is_diffusion ?? false)
? ""
: (loadResp.reasoning_budget_message ??
config.reasoningBudgetMessage),
loadedReasoningBudgetRequested: loadResp.is_diffusion
? -1
: (loadResp.requested_reasoning_budget ?? config.reasoningBudget),
loadedReasoningBudgetMessageRequested: loadResp.is_diffusion
? ""
: (loadResp.requested_reasoning_budget_message ?? config.reasoningBudgetMessage),
nBatch: committedNBatch,
loadedNBatch: committedNBatch,
nUbatch: committedNUbatch,
loadedNUbatch: committedNUbatch,
...committedServerTuningState(config, loadResp.is_diffusion ?? false),
// Seed rollback args here; the status applier cannot while the loading lease is held.
loadedLlamaExtraArgs:
loadResp.requested_llama_extra_args !== undefined
? (loadResp.requested_llama_extra_args ?? [])
: (resolvedExtraArgs ?? null),
tensorParallel: loadResp.tensor_parallel ?? false,
loadedTensorParallel: loadResp.tensor_parallel ?? false,
loadedDisableVision: loadResp.disable_vision ?? false,
// A stale true would hide Vision over a loaded projector and be sent on next Apply.
disableVision: loadResp.disable_vision ?? false,
loadedVisionDisabledByUser: loadResp.vision_disabled_by_user ?? false,
...loadedGpuMemoryFields(loadResp),
loadedCustomContextLength: keepCustomCtx,
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: effectiveChatTemplateOverride,
loadedChatTemplateOverride: effectiveChatTemplateOverride,
// Same value as the baseline above, so the control opens undirtied.
customContextLength: keepCustomCtx,
loadedIsMultimodal: isMultimodalResponse(loadResp),
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
loadedIsDiffusion: loadResp.is_diffusion ?? false,
activeModelIsLocal: loadResp.is_local_model ?? false,
...resolveLoadedSpeculativeSettings(loadResp),
});
} else {
useChatRuntimeStore.setState({
supportsReasoning: loadResp.supports_reasoning ?? false,
reasoningAlwaysOn: loadResp.reasoning_always_on ?? false,
reasoningEnabled: loadResp.supports_reasoning ?? false,
...reasoningCapsFromLoad(loadResp),
supportsPreserveThinking:
loadResp.supports_preserve_thinking ?? false,
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
kvCacheDtype: loadResp.cache_type_kv ?? null,
loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
...mlxRuntimeStateFrom(loadResp),
nParallel: committedSlots,
loadedNParallel: committedSlots,
reasoningBudget: -1,
loadedReasoningBudget: -1,
loadedReasoningBudgetRequested: -1,
reasoningBudgetMessage: "",
loadedReasoningBudgetMessage: "",
loadedReasoningBudgetMessageRequested: "",
nBatch: null,
loadedNBatch: null,
nUbatch: null,
loadedNUbatch: null,
...clearedServerTuningState(),
// Clear the baseline so a rollback to this model does not resend a GGUF's arguments.
loadedLlamaExtraArgs: null,
tensorParallel: loadResp.tensor_parallel ?? false,
loadedTensorParallel: loadResp.tensor_parallel ?? false,
loadedDisableVision: loadResp.disable_vision ?? false,
// Safetensors loads have no projector; a stale true would carry to the next model.
disableVision: false,
loadedVisionDisabledByUser:
loadResp.vision_disabled_by_user ?? false,
// Non-GGUF response: clears any stale GPU baseline a prior manual-GPU GGUF load left.
...loadedGpuMemoryFields(loadResp),
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: effectiveChatTemplateOverride,
loadedChatTemplateOverride: effectiveChatTemplateOverride,
// Clear the previous model's serving state, but keep this model's own pin.
customContextLength: autoLoadPin,
loadedCustomContextLength: autoLoadPin,
...loadedContextFields(loadResp),
activeNativePathToken: null,
activeNativePathExpiresAtMs: null,
...resolveLoadedSpeculativeSettings(loadResp),
loadedIsMultimodal: isMultimodalResponse(loadResp),
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
loadedIsDiffusion: loadResp.is_diffusion ?? false,
activeModelIsLocal: loadResp.is_local_model ?? false,
});
}
if (!(loadResp.is_lora ?? false)) {
recordLastLocalModelLoad({
id: candidate.id,
kind: candidate.kind,
ggufVariant: candidate.ggufVariant,
});
}
showAutoLoadSuccess(
candidate.successLabel,
loadResp.cpu_fallback_reason,
loadResp.mmproj_fallback_reason,
offloadCountsFrom(loadResp),
);
});
return true;
}
try {
// Fail closed: these lists are the only evidence of what is on the device, so a flaky request
// must not read as "nothing here" and trigger a download.
const inventory = await Promise.allSettled([
listCachedGguf(options?.abortSignal),
listCachedModels(hfToken, options?.abortSignal),
listLocalModels(),
]);
options?.abortSignal?.throwIfAborted();
// allSettled: one rejected source must not discard the lists that did arrive.
const [ggufSettled, modelsSettled, localSettled] = inventory;
const allGgufRepos =
ggufSettled.status === "fulfilled" ? ggufSettled.value : [];
const allModelRepos =
modelsSettled.status === "fulfilled" ? modelsSettled.value : [];
const localRows =
localSettled.status === "fulfilled" ? localSettled.value.models : [];
const inventoryIncomplete = inventory.some((r) => r.status === "rejected");
// Only the two managed-cache lookups: /api/hub/local also reports hf_cache rows.
const cachedInventoryFailed =
ggufSettled.status === "rejected" || modelsSettled.status === "rejected";
if (inventoryIncomplete) hadNonTrustFailure = true;
// A deferred verdict admits non-GGUF caches a chat-only host refuses; GGUF-only caches skip the probe.
if (
usePlatformStore.getState().detectionDeferred &&
(allModelRepos.some(isChattableCachedRepo) ||
localRows.some(
(row) =>
!isGgufLocalRow(row) &&
isAutoLoadableLocalRow(row, cachedInventoryFailed),
))
) {
await awaitMeasuredVerdict(options?.abortSignal);
options?.abortSignal?.throwIfAborted();
}
// Include on-disk models the picker indexes, not just the managed cache.
const sources = orderAutoLoadSources(
buildAutoLoadSources(
allGgufRepos.filter(isChattableCachedRepo),
cachedModelsRunOnThisPlatform()
? allModelRepos.filter(isChattableCachedRepo)
: [],
localRows.filter((row) =>
isAutoLoadableLocalRow(row, cachedInventoryFailed),
),
store.params.maxSeqLength,
options?.abortSignal,
),
lastLoaded,
);
// A target can be in several lists; a twin is tried only if the preferred row resolved nothing.
const candidateResolvedFor = new Set<string>();
for (const source of sources) {
if (
autoLoadCancelled ||
loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS ||
validateFailures >= MAX_AUTO_VALIDATE_FAILURES
) {
break;
}
const sourceKey = autoLoadSourceKey(source);
if (candidateResolvedFor.has(sourceKey)) continue;
const isRemembered = lastLoaded
? isRememberedSource(source, lastLoaded)
: false;
const isTried = (entry: AutoLoadCandidate): boolean =>
skippedAutoLoadCandidates.has(
autoLoadCandidateKey(
entry.kind,
entry.loadId ?? entry.id,
entry.ggufVariant,
),
);
try {
// Each failure marks one quant tried, so one corrupt file does not cost the whole repo.
while (
!autoLoadCancelled &&
loadAttempts < MAX_AUTO_LOAD_ATTEMPTS &&
validateFailures < MAX_AUTO_VALIDATE_FAILURES
) {
const candidate = await resolveAutoLoadCandidate(
source,
isRemembered ? (lastLoaded?.ggufVariant ?? null) : null,
isTried,
);
options?.abortSignal?.throwIfAborted();
if (!candidate) break;
candidateResolvedFor.add(sourceKey);
updateAutoLoadToast(
isRemembered ? "Loading last used model…" : "Loading a model…",
candidate.ggufVariant
? `${candidate.id} (${candidate.ggufVariant})`
: candidate.id,
);
try {
if (await loadAutoLoadCandidate(candidate)) {
return { loaded: true, blockedByTrustRemoteCode: false };
}
} catch {
options?.abortSignal?.throwIfAborted();
hadNonTrustFailure = true;
// loadAutoLoadCandidate does not record rejections; mark here or the same quant loops forever.
skippedAutoLoadCandidates.add(
autoLoadCandidateKey(
candidate.kind,
candidate.loadId ?? candidate.id,
candidate.ggufVariant,
),
);
}
}
} catch {
options?.abortSignal?.throwIfAborted();
hadNonTrustFailure = true;
continue;
}
}
// The cap gates the default download too, so the whole /load budget is MAX_AUTO_LOAD_ATTEMPTS.
if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS || loadFailure.current) {
toast.dismiss(toastId);
if (loadFailure.current) {
toast.error(
loadFailure.current.blamesModel
? `Could not load ${loadFailure.current.label}`
: loadFailure.current.detail,
{
description: loadFailure.current.blamesModel
? loadFailure.current.detail
: `Stopped before loading ${loadFailure.current.label}.`,
duration: 10000,
closeButton: true,
},
);
}
return {
loaded: false,
blockedByTrustRemoteCode:
blockedByTrustRemoteCode && !hadNonTrustFailure,
loadFailureReported: loadFailure.current !== null,
};
}
// Only an empty *complete* inventory means the device has nothing.
if (inventoryIncomplete) {
toast.dismiss(toastId);
return { loaded: false, blockedByTrustRemoteCode: false };
}
// Fetch the default as a managed download job and hand it to /api/inference/load only once the bytes are here.
try {
const rt = useChatRuntimeStore.getState();
if (rt.selectedGpuIds != null) {
await ensureGpuDeviceCache();
}
const defaultGpuIds = reconcilePersistedGpuIds(
rt.selectedGpuIds,
rt.selectedGpuIndexKind,
);
const request: LoadModelRequest = {
model_path: DEFAULT_CHAT_MODEL_REPO,
hf_token: hfToken,
// Model default under both modes: Auto layers + no pin means resolveFitMaxSeqLength returns 0.
max_seq_length: 0,
load_in_4bit: true,
is_lora: false,
gguf_variant: DEFAULT_CHAT_MODEL_VARIANT,
trust_remote_code: trustRemoteCode,
speculative_type: specSettings.speculativeType,
spec_draft_n_max: specSettings.specDraftNMax,
reasoning_budget: -1,
reasoning_budget_message: "",
// GPU Memory mode is a standing preference and the GPU pick is the on-screen one; the
// per-model layer/MoE/split knobs and context pin stay at their defaults.
gpu_memory_mode: rt.gpuMemoryMode,
gpu_layers: GPU_LAYERS_AUTO,
n_cpu_moe: 0,
gpu_ids: defaultGpuIds ?? undefined,
};
// Preflight BEFORE the transfer: training or the placement guard can refuse this model, and
// learning that after gigabytes costs a long wait. The load sends this same request.
if (!(await canAutoLoad(request))) {
toast.dismiss(toastId);
return { loaded: false, blockedByTrustRemoteCode };
}
const download = await ensureDefaultModelDownloaded(
hfToken,
options?.abortSignal,
updateAutoLoadToast,
);
options?.abortSignal?.throwIfAborted();
if (download !== "ready") {
toast.dismiss(toastId);
if (download === "cancelled") {
toast.message(`${DEFAULT_CHAT_MODEL_LABEL} download stopped`, {
description:
"No model is loaded. Pick one from the top bar, or download another from the Hub, then send again.",
duration: 8000,
closeButton: true,
});
}
return {
loaded: false,
blockedByTrustRemoteCode,
loadFailureReported: download === "cancelled",
};
}
updateAutoLoadToast(
`Loading ${DEFAULT_CHAT_MODEL_LABEL}…`,
`${DEFAULT_CHAT_MODEL_REPO} (${DEFAULT_CHAT_MODEL_VARIANT})`,
);
loadAttempts += 1;
options?.abortSignal?.throwIfAborted();
const loadResp = await loadModel({ ...request, alongside: true });
reportEvicted(loadResp);
options?.abortSignal?.throwIfAborted();
applyAutoLoadRuntimeState(options, () => {
saveSpeculativeType(specSettings.speculativeType);
persistGpuMemoryModeOnLoad(loadResp, rt.gpuMemoryMode);
useChatRuntimeStore
.getState()
.setCheckpoint(DEFAULT_CHAT_MODEL_REPO, DEFAULT_CHAT_MODEL_VARIANT, {
trackQueuedSettings: !options?.preserveVisibleSettings,
});
const store = useChatRuntimeStore.getState();
store.setModelRequiresTrustRemoteCode(
loadResp.requires_trust_remote_code ?? false,
);
store.setParams(
{
...store.params,
maxTokens: localMaxTokensCeiling(
loadedContextFields(loadResp).loadedContextLength,
unreportedWindowMaxTokens(loadResp.is_gguf ?? false, store.params.maxTokens),
),
},
{
persist: !options?.preserveVisibleSettings,
trackQueuedSettings: !options?.preserveVisibleSettings,
fromModelDefaults: true,
maxTokensCap: replayMaxTokensCap(
loadedContextFields(loadResp).loadedContextLength,
),
},
);
const defaultModel: ChatModelRow = {
id: DEFAULT_CHAT_MODEL_REPO,
name: loadResp.display_name ?? DEFAULT_CHAT_MODEL_LABEL,
isVision: loadResp.is_vision ?? false,
isLora: false,
isGguf: true,
isMlx: false,
isAudio: false,
hasAudioInput: false,
};
if (!store.models.some((m) => m.id === DEFAULT_CHAT_MODEL_REPO)) {
store.setModels([...store.models, defaultModel]);
}
useChatRuntimeStore.setState({
...loadedContextFields(loadResp),
supportsReasoning: loadResp.supports_reasoning ?? false,
reasoningAlwaysOn: loadResp.reasoning_always_on ?? false,
reasoningEnabled: loadResp.supports_reasoning ?? false,
...reasoningCapsFromLoad(loadResp),
supportsPreserveThinking:
loadResp.supports_preserve_thinking ?? false,
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
kvCacheDtype: loadResp.cache_type_kv ?? null,
loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
...mlxRuntimeStateFrom(loadResp),
// n_parallel was omitted, so a staged override would look applied and be re-sent.
nParallel: null,
loadedNParallel: null,
reasoningBudget: -1,
loadedReasoningBudget: -1,
loadedReasoningBudgetRequested: -1,
reasoningBudgetMessage: "",
loadedReasoningBudgetMessage: "",
loadedReasoningBudgetMessageRequested: "",
nBatch: null,
loadedNBatch: null,
nUbatch: null,
loadedNUbatch: null,
...clearedServerTuningState(),
tensorParallel: loadResp.tensor_parallel ?? false,
loadedTensorParallel: loadResp.tensor_parallel ?? false,
loadedDisableVision: loadResp.disable_vision ?? false,
// The request above omits disable_vision, so the echo is what the load ran with.
disableVision: loadResp.disable_vision ?? false,
loadedVisionDisabledByUser:
loadResp.vision_disabled_by_user ?? false,
...loadedGpuMemoryFields(loadResp),
// Drives the GPU Memory controls' diffusion gate; set on every load path so the gate cannot read stale.
loadedIsDiffusion: loadResp.is_diffusion ?? false,
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: null,
loadedIsMultimodal: isMultimodalResponse(loadResp),
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
activeModelIsLocal: loadResp.is_local_model ?? false,
...resolveLoadedSpeculativeSettings(loadResp),
});
recordLastLocalModelLoad({
id: DEFAULT_CHAT_MODEL_REPO,
kind: "gguf",
ggufVariant: DEFAULT_CHAT_MODEL_VARIANT,
});
showAutoLoadSuccess(
`Loaded ${DEFAULT_CHAT_MODEL_LABEL} (${DEFAULT_CHAT_MODEL_VARIANT})`,
loadResp.cpu_fallback_reason,
loadResp.mmproj_fallback_reason,
offloadCountsFrom(loadResp),
);
});
return { loaded: true, blockedByTrustRemoteCode: false };
} catch {
options?.abortSignal?.throwIfAborted();
toast.dismiss(toastId);
hadNonTrustFailure = true;
return {
loaded: false,
blockedByTrustRemoteCode:
blockedByTrustRemoteCode && !hadNonTrustFailure,
};
}
} catch {
options?.abortSignal?.throwIfAborted();
toast.dismiss(toastId);
hadNonTrustFailure = true;
return {
loaded: false,
blockedByTrustRemoteCode: blockedByTrustRemoteCode && !hadNonTrustFailure,
};
}
}
async function resolveQueuedEmptyLocalModel(abortSignal: AbortSignal): Promise<{
loaded: boolean;
blockedByTrustRemoteCode: boolean;
loadFailureReported?: boolean;
modelRuntime: QueuedResolvedModelRuntime | null;
}> {
// Auto-load does not sweep queues, so follow-ups can be accepted immediately.
let lifecycleLease = useChatRuntimeStore.getState().beginModelLoading("loading");
while (lifecycleLease === null) {
await waitForModelReady(abortSignal);
abortSignal.throwIfAborted();
lifecycleLease = useChatRuntimeStore.getState().beginModelLoading("loading");
}
try {
abortSignal.throwIfAborted();
const visibleState = useChatRuntimeStore.getState();
if (isExternalModelId(visibleState.params.checkpoint)) {
// Hold the lifecycle lease across the probe and fail closed: a failed or mid-replacement read is
// not evidence the server is empty.
const settled = await waitForSettledServerStatus({ abortSignal });
abortSignal.throwIfAborted();
if (settled.outcome !== "settled") {
reportBlockedServerLoad(settled.outcome);
return {
loaded: false,
blockedByTrustRemoteCode: false,
loadFailureReported: true,
modelRuntime: null,
};
}
const status = settled.status;
// A speech model is not one chat can queue against, so read the slot as empty.
const checkpoint = isSpeechOnlyStatus(status)
? null
: resolveInferenceCheckpointId(status);
if (checkpoint) {
return {
loaded: true,
blockedByTrustRemoteCode: false,
modelRuntime: {
checkpoint,
activeGgufVariant: status.gguf_variant ?? null,
supportsTools: status.supports_tools ?? false,
supportsReasoning: status.supports_reasoning ?? false,
reasoningAlwaysOn: status.reasoning_always_on ?? false,
...reasoningCapsFromLoad(status),
supportsPreserveThinking:
status.supports_preserve_thinking ?? false,
preserveThinking: resolvePreserveThinkingOnLoad(status),
loadedContextLength: loadedContextFields(status).loadedContextLength,
loadedIsGguf: loadedContextFields(status).loadedIsGguf,
loadedIsMlx: loadedContextFields(status).loadedIsMlx,
loadedIsMultimodal: isMultimodalResponse(status),
modelCapabilities: {
isVision: status.is_vision ?? false,
isGguf: status.is_gguf ?? false,
isMlx: status.is_mlx ?? false,
isAudio: status.is_audio ?? false,
audioType: status.audio_type ?? null,
hasAudioInput: status.has_audio_input ?? false,
hasVideoInput: status.has_video_input ?? false,
speaks: status.speaks ?? false,
},
},
};
}
const visibleExternalState = snapshotVisibleModelState(visibleState);
const visibleThreadEpoch = visibleState.activeThreadEpoch;
const visibleSettingsEpoch = visibleState.queuedSettingsEpoch;
const visibleRoute = window.location.href;
let result: Awaited<ReturnType<typeof autoLoadSmallestModel>>;
let modelRuntime: QueuedResolvedModelRuntime | null = null;
try {
result = await autoLoadSmallestModel({
skipAdoptServerModel: true,
preserveVisibleSettings: true,
captureResolvedRuntime: (runtime) => {
modelRuntime = runtime;
},
abortSignal,
});
if (result.loaded && !modelRuntime) {
modelRuntime = queuedResolvedModelFromStore(
useChatRuntimeStore.getState(),
);
}
} finally {
if (
useChatRuntimeStore.getState().activeThreadEpoch ===
visibleThreadEpoch &&
useChatRuntimeStore.getState().queuedSettingsEpoch ===
visibleSettingsEpoch &&
window.location.href === visibleRoute
) {
restoreVisibleModelState(visibleExternalState);
}
}
return { ...result, modelRuntime };
}
if (visibleState.params.checkpoint) {
return {
loaded: true,
blockedByTrustRemoteCode: false,
modelRuntime: queuedResolvedModelFromStore(visibleState),
};
}
const result = await autoLoadSmallestModel({ abortSignal });
return {
...result,
modelRuntime: result.loaded
? queuedResolvedModelFromStore(useChatRuntimeStore.getState())
: null,
};
} finally {
useChatRuntimeStore.getState().endModelLoading(lifecycleLease);
}
}
export function createOpenAIStreamAdapter(
options: OpenAIStreamAdapterOptions = {},
): ChatModelAdapter {
const adapter = {
async *run({
messages: rawMessages,
runConfig,
abortSignal,
unstable_threadId,
unstable_assistantMessageId,
}) {
// Read before the first await: later awaits make the store unreliable for the project.
const creationClaim = unstable_threadId
? readThreadCreationClaim(unstable_threadId)
: undefined;
const composerProjectIdAtSend = creationClaim
? creationClaim.projectId
: (useChatRuntimeStore.getState().activeProjectId ?? null);
await useChatRuntimeStore.getState().hydratePersistedSettings();
// The backend reads settings from SQLite and the mirror is debounced, so flush first.
await flushPendingChatSettings();
// External selections get no load callback; this is the only join with the migration.
await awaitPendingQwenDefaultsMigration();
// Every run path reaches here, so wait for this chat's own settings (else a chat stored as "ask"
// could run tools on installation defaults). Bound to this run's chat.
const runThreadId =
unstable_threadId ?? useChatRuntimeStore.getState().activeThreadId;
// Refuse rather than send with another chat's tools, which is unrecoverable.
if (!(await awaitThreadScopedPairing(runThreadId))) {
throw new Error(
"This chat's settings could not be loaded, so the message was not sent. Reopen the chat and try again.",
);
}
await prepareComparePaneRun(
options.pairId,
options.modelType,
abortSignal,
() => waitForModelReady(abortSignal),
);
abortSignal.throwIfAborted();
let runtime = useChatRuntimeStore.getState();
// Capture the thread ID once so it stays stable across chat switches during model load.
const resolvedThreadId =
(runThreadId ?? runtime.activeThreadId) || undefined;
// Taken as it sends, from the run's own thread: its browser pages keep that chat's history rule
// after a switch. The shown chat's toggle counts only for the thread on screen (marked as it initializes).
const browserTemporary = resolvedThreadId
? isThreadIncognito(resolvedThreadId) ||
(resolvedThreadId === runtime.activeThreadId && runtime.incognito)
: runtime.incognito;
if (resolvedThreadId) {
rememberComposerProjectForRun(resolvedThreadId, composerProjectIdAtSend);
}
const sharedThreadRecordRead = resolvedThreadId
? createRetryableSharedRead(
() => getStoredChatThreadReadResult(resolvedThreadId),
(result) => result.cacheable,
)
: undefined;
const readThreadRecord = sharedThreadRecordRead
? async () => (await sharedThreadRecordRead()).thread
: undefined;
const releaseCurrentPreStreamRun = () =>
releasePreStreamRunForThreadIds([unstable_threadId, resolvedThreadId]);
const queuedRunSettings = consumeQueuedChatRunSettings(resolvedThreadId);
let queuedEmptyModelRuntime: QueuedResolvedModelRuntime | null = null;
const persistRunModel = async (
modelId: string,
modelGgufVariant: string | null,
) => {
if (!resolvedThreadId || !modelId) {
return;
}
try {
const storedThread = await readThreadRecord?.();
if (!shouldPersistRunModel(storedThread, modelId, modelGgufVariant)) {
return;
}
await updateStoredChatThread(resolvedThreadId, {
modelId,
modelGgufVariant,
});
} catch {
// Only what the chat offers back on reopen.
}
};
if (queuedRunSettings) {
runtime = { ...runtime, ...queuedRunSettings };
}
const threadAlreadyResearched = Boolean(
resolvedThreadId &&
useResearchRunStore.getState().claimedThreadIds[resolvedThreadId],
);
if (runtime.deepResearchEnabled && threadAlreadyResearched) {
if (queuedRunSettings) {
runtime = { ...runtime, deepResearchEnabled: false };
} else {
runtime.setDeepResearchEnabled(false);
runtime = useChatRuntimeStore.getState();
}
}
// Runs only when the model hands off; arming research just offers the tool.
const startDeepResearch = async function* (
researchQuestion: string,
transitionSignal: AbortSignal = abortSignal,
) {
if (runtime.modelLoading) {
toast.info("Waiting for model to finish loading…");
try {
await waitForModelReady(transitionSignal);
} catch (error) {
throw error;
}
}
if (!runtime.params.checkpoint) {
let resolution: Awaited<
ReturnType<typeof resolveQueuedEmptyLocalModel>
>;
try {
resolution = await resolveQueuedEmptyLocalModel(transitionSignal);
} catch (error) {
throw error;
}
queuedEmptyModelRuntime = resolution.modelRuntime;
if (!resolution.loaded) {
// A reported failure already names the model; generic advice buries it.
if (!resolution.loadFailureReported) {
toast.error(
resolution.blockedByTrustRemoteCode
? "This model needs custom code approval"
: "No model loaded",
{
description: resolution.blockedByTrustRemoteCode
? "Select it from the top bar to review and approve its custom code, or pick another model."
: "Pick a model in the top bar, then retry.",
},
);
}
throw new Error("Load a model first.");
}
}
// Use the turn's runtime: the model selector stays usable while a turn streams.
const sendTimeRuntime = runtime;
const withResolvedModel = <T extends typeof sendTimeRuntime>(base: T): T =>
queuedEmptyModelRuntime === null
? base
: {
...base,
params: {
...base.params,
checkpoint: queuedEmptyModelRuntime.checkpoint,
},
activeGgufVariant: queuedEmptyModelRuntime.activeGgufVariant,
supportsTools: queuedEmptyModelRuntime.supportsTools,
supportsReasoning: queuedEmptyModelRuntime.supportsReasoning,
reasoningAlwaysOn: queuedEmptyModelRuntime.reasoningAlwaysOn,
reasoningStyle: queuedEmptyModelRuntime.reasoningStyle,
supportsReasoningOff: queuedEmptyModelRuntime.supportsReasoningOff,
reasoningEffortLevels: queuedEmptyModelRuntime.reasoningEffortLevels,
supportsPreserveThinking:
queuedEmptyModelRuntime.supportsPreserveThinking,
preserveThinking: queuedEmptyModelRuntime.preserveThinking,
loadedContextLength: queuedEmptyModelRuntime.loadedContextLength,
loadedIsGguf: queuedEmptyModelRuntime.loadedIsGguf,
loadedIsMlx: queuedEmptyModelRuntime.loadedIsMlx,
models: mergeQueuedModelCapabilities(
base.models,
queuedEmptyModelRuntime.checkpoint,
queuedEmptyModelRuntime.modelCapabilities,
),
};
runtime = queuedRunSettings
? queuedRunSettings.params.checkpoint
? { ...sendTimeRuntime, ...queuedRunSettings }
: withResolvedModel({
...sendTimeRuntime,
...queuedRunSettings,
// The queued snapshot carries no model of its own.
params: {
...queuedRunSettings.params,
checkpoint: sendTimeRuntime.params.checkpoint,
},
})
: withResolvedModel(sendTimeRuntime);
if (!resolvedThreadId) throw new Error("Research requires a saved chat.");
if (!unstable_assistantMessageId) {
throw new Error(
"Deep research could not bind its assistant message. Please retry the send.",
);
}
const userMessage = [...messages]
.reverse()
.find((m) => m.role === "user");
if (!userMessage) throw new Error("Research requires a user message.");
const userMessageIndex = rawMessages.indexOf(userMessage);
const userMessageParentId =
userMessageIndex > 0 ? rawMessages[userMessageIndex - 1]!.id : null;
const { params } = runtime;
const selectedCheckpoint = params.checkpoint.trim();
const researchExternalSelection =
parseExternalModelId(selectedCheckpoint);
const researchExternalProvider = researchExternalSelection
? loadExternalProviders().find(
(provider) =>
provider.id === researchExternalSelection.providerId,
)
: null;
if (
!selectedCheckpoint ||
(researchExternalSelection &&
providerModelSupportsStudioTools(
researchExternalProvider?.providerType,
researchExternalSelection.modelId,
) !== true)
) {
throw new Error(
"Deep research requires a selected local model or a connection whose provider supports Unsloth tools.",
);
}
const reasoningRequested =
runtime.reasoningAlwaysOn ||
(runtime.reasoningEnabled && runtime.reasoningEffort !== "none");
const inferenceRequest = buildResearchInferenceRequest({
checkpoint: selectedCheckpoint,
external:
researchExternalSelection && researchExternalProvider
? {
providerId: researchExternalProvider.id,
providerType: researchExternalProvider.providerType,
modelId: researchExternalSelection.modelId,
maxOutputTokens: getGroundedExternalMaxOutputTokens(
researchExternalProvider.providerType,
researchExternalSelection.modelId,
researchExternalProvider.maxOutputTokens,
),
maxOutputTokensFromSavedCap: externalMaxOutputTokensNeedsConnectionCap(
researchExternalProvider.providerType,
researchExternalSelection.modelId,
),
maxOutputTokensPublished: getPublishedExternalMaxOutputTokens(
researchExternalProvider.providerType,
researchExternalSelection.modelId,
),
supportsReasoning: runtime.supportsReasoning,
supportsReasoningOff: runtime.supportsReasoningOff,
}
: undefined,
temperature: params.temperature,
topP: params.topP,
maxTokens: params.maxTokens,
reasoningRequested,
reasoningStyle: runtime.reasoningStyle,
reasoningEffort: runtime.reasoningEffort,
reasoningEffortLevels: runtime.reasoningEffortLevels,
clampReasoningEffort: clampReasoningEffortToLevels,
});
const researchProjectId = await resolveProjectId(
resolvedThreadId,
readThreadRecord,
);
const projectRagEnabled = researchProjectId
? await projectHasSources(researchProjectId)
: false;
const researchInstructions = await resolveChatInstructions(
resolvedThreadId,
params.systemPrompt,
params.systemVariables,
readThreadRecord,
{ agentsMd: true },
);
if (transitionSignal.aborted) return;
const ragScope =
runtime.ragEnabled || projectRagEnabled
? {
...(runtime.ragEnabled && runtime.ragSource.type === "kb"
? { kb_id: runtime.ragSource.kbId }
: runtime.ragEnabled
? { thread_id: resolvedThreadId }
: {}),
...(projectRagEnabled && researchProjectId
? { project_id: researchProjectId }
: {}),
default_top_k: runtime.ragTopK,
mode: runtime.ragMode,
autoinject: runtime.ragAutoInject,
autoinject_min_score: runtime.ragAutoInjectMinScore,
}
: undefined;
const threadKey = resolvedThreadId;
// Background stop/archive/delete reach the run only via this map; register before create.
let researchRunId: string | null = null;
let researchStopRequested = false;
const researchServerCancel = () => {
researchStopRequested = true;
if (researchRunId) {
void cancelResearchRun(researchRunId).catch(() => {});
}
};
runtime.registerThreadServerCancel(threadKey, researchServerCancel);
releaseCurrentPreStreamRun();
void persistRunModel(params.checkpoint, runtime.activeGgufVariant);
runtime.setThreadRunning(threadKey, true, {
owner: researchServerCancel,
});
let report = "";
let releaseResearchFollow: (() => void) | null = null;
const researchFollowController = new AbortController();
const detachResearchFollow = () => {
researchFollowController.abort({ detach: true });
};
const forwardAdapterAbort = () => {
researchFollowController.abort(abortSignal.reason);
};
abortSignal.addEventListener("abort", forwardAdapterAbort, {
once: true,
});
try {
// Research validates the user message before starting, unlike the history adapter.
const storedUserMessage = (
await listStoredChatMessages(resolvedThreadId)
).find((message) => message.id === userMessage.id);
await saveStoredChatMessage({
id: userMessage.id,
threadId: resolvedThreadId,
// A stored null is an edited root; `??` would reparent it.
parentId:
storedUserMessage && storedUserMessage.parentId !== undefined
? storedUserMessage.parentId
: userMessageParentId,
role: "user",
content: userMessage.content,
...(userMessage.attachments?.length
? { attachments: userMessage.attachments }
: {}),
createdAt: userMessage.createdAt?.getTime?.() ?? Date.now(),
});
const createdRun = await createResearchRun({
threadId: resolvedThreadId,
userMessageId: userMessage.id,
assistantMessageId: unstable_assistantMessageId,
inferenceRequest,
// omit empty fields: old backends reject unknown fields with 422.
...(researchQuestion ? { question: researchQuestion } : {}),
...(researchInstructions ? { instructions: researchInstructions } : {}),
...(ragScope ? { ragScope } : {}),
...(runtime.researchMcpSources.length
? { mcpSources: runtime.researchMcpSources }
: {}),
budgets: {
modelTimeoutSeconds: runtime.researchModelTimeoutSeconds,
},
websitePolicy: {
allowedDomains: [...runtime.researchWebsitePolicy.allowedDomains],
blockedDomains: [...runtime.researchWebsitePolicy.blockedDomains],
},
...tabCloseRunFields(runtime.keepRunningWhenTabClosed),
});
researchRunId = createdRun.id;
if (researchStopRequested) {
// replay stop once creation supplies the missing run id.
void cancelResearchRun(createdRun.id).catch(() => {});
return;
}
releaseResearchFollow = beginExternalResearchFollow(
createdRun,
detachResearchFollow,
);
// The toggle stays on through the run; the composer takes it away once the chat's research is spent.
if (abortSignal.aborted) {
const detached = Boolean(
(abortSignal.reason as { detach?: boolean } | undefined)?.detach,
);
if (!detached) {
try {
ingestResearchUpdate(await cancelResearchRun(createdRun.id));
} catch {
// The durable run remains visible and can be stopped again after recovery.
}
}
return;
}
// read the store, not the stream: a stalled reader here must not freeze ingestion.
let yieldedStatus: string | null = null;
for await (const run of watchResearchRun(createdRun.id, {
signal: researchFollowController.signal,
})) {
if (typeof run.report === "string") {
report = run.report;
}
const settled = terminalResearchStatuses.has(run.status);
// per-delta yields would rewrite the message and drive an autosave the server rejects.
if (run.status === yieldedStatus && !settled) {
continue;
}
yieldedStatus = run.status;
yield {
content: [{ type: "text" as const, text: report }],
metadata: {
custom: {
researchRunId: run.id,
researchRun: run,
serverManaged: true,
serverRevision: run.lastEventSeq,
},
},
};
}
} catch (error) {
if (
!abortSignal.aborted &&
!researchFollowController.signal.aborted
) {
throw error;
}
} finally {
abortSignal.removeEventListener("abort", forwardAdapterAbort);
releaseResearchFollow?.();
runtime.clearThreadServerCancel(threadKey, researchServerCancel);
runtime.setThreadRunning(threadKey, false, {
owner: researchServerCancel,
});
}
};
const deepResearchHandoff = newDeepResearchHandoff();
const continuation = readContinuationRequest(runConfig);
let deepResearchArmed =
runtime.deepResearchEnabled &&
!options.pairId &&
(options.modelType === undefined || options.modelType === "base");
const browserTools = browserAgentToolsForTurn();
const toolConfirmationIdsByBackendId = new Map<string, string>();
// Local tool ids ("call_0") repeat across turns and panes, so scope by pane and thread
// (unstable_threadId only, matching the reader's remoteId).
const toolOutputPaneScope = toolThreadScope(
toolPaneScope(options.modelType, options.pairId),
unstable_threadId,
);
const scopedToolOutputKey = (id: string) =>
toolOutputKey(toolOutputPaneScope, id);
const runToolLiveOutputKeys = new Set<string>();
const resolvedThreadKey = resolvedThreadId ?? null;
// Which conversation was on screen when this run started; a first turn has no id yet.
const activeThreadIdAtRunStart =
useChatRuntimeStore.getState().activeThreadId ?? null;
const pendingImageEditReferenceForRun = runtime.pendingImageEditReference;
const selectedImageEditReference =
(pendingImageEditReferenceForRun?.threadId ?? null) ===
resolvedThreadKey
? pendingImageEditReferenceForRun
: null;
const clearSelectedImageEditReference = () => {
if (!selectedImageEditReference) {
return;
}
const store = useChatRuntimeStore.getState();
const pending = store.pendingImageEditReference;
if (
pending?.openaiImageGenerationCallId ===
selectedImageEditReference.openaiImageGenerationCallId &&
pending.openaiResponseId ===
selectedImageEditReference.openaiResponseId &&
(pending.threadId ?? null) ===
(selectedImageEditReference.threadId ?? null)
) {
store.clearPendingImageEditReference();
}
};
// The waits below let the user open another chat, whose settings then fill the store.
const runtimeAtSend = runtime;
const chatOwnedAtSend = withChatOwnedSettings(runtime);
if (runtime.modelLoading) {
toast.info("Waiting for model to finish loading…");
try {
await waitForModelReady(abortSignal);
} catch (error) {
clearSelectedImageEditReference();
throw error;
}
}
if (!runtime.params.checkpoint) {
let resolution: Awaited<
ReturnType<typeof resolveQueuedEmptyLocalModel>
>;
try {
resolution = await resolveQueuedEmptyLocalModel(abortSignal);
} catch (error) {
clearSelectedImageEditReference();
throw error;
}
queuedEmptyModelRuntime = resolution.modelRuntime;
if (!resolution.loaded) {
if (!resolution.loadFailureReported) {
toast.error(
resolution.blockedByTrustRemoteCode
? "This model needs custom code approval"
: "No model loaded",
{
description: resolution.blockedByTrustRemoteCode
? "Select it from the top bar to review and approve its custom code, or pick another model."
: "Pick a model in the top bar, then retry.",
},
);
}
clearSelectedImageEditReference();
throw new Error("Load a model first.");
}
}
const liveRuntime = useChatRuntimeStore.getState();
// A pending read means the store holds defaults, not this chat's settings.
const runChatSettingsLive =
!liveRuntime.threadScopedSettingsPending &&
(liveRuntime.activeThreadId === runtimeAtSend.activeThreadId ||
(resolvedThreadId !== undefined &&
liveRuntime.activeThreadId === resolvedThreadId));
runtime = queuedRunSettings
? queuedRunSettings.params.checkpoint
? { ...liveRuntime, ...queuedRunSettings }
: {
...liveRuntime,
...queuedRunSettings,
params: {
...queuedRunSettings.params,
checkpoint:
queuedEmptyModelRuntime?.checkpoint ??
liveRuntime.params.checkpoint,
},
activeGgufVariant:
queuedEmptyModelRuntime !== null
? queuedEmptyModelRuntime.activeGgufVariant
: liveRuntime.activeGgufVariant,
supportsTools:
queuedEmptyModelRuntime?.supportsTools ??
liveRuntime.supportsTools,
supportsReasoning:
queuedEmptyModelRuntime?.supportsReasoning ??
liveRuntime.supportsReasoning,
reasoningAlwaysOn:
queuedEmptyModelRuntime?.reasoningAlwaysOn ??
liveRuntime.reasoningAlwaysOn,
reasoningStyle:
queuedEmptyModelRuntime?.reasoningStyle ??
liveRuntime.reasoningStyle,
supportsReasoningOff:
queuedEmptyModelRuntime?.supportsReasoningOff ??
liveRuntime.supportsReasoningOff,
reasoningEffortLevels:
queuedEmptyModelRuntime?.reasoningEffortLevels ??
liveRuntime.reasoningEffortLevels,
supportsPreserveThinking:
queuedEmptyModelRuntime?.supportsPreserveThinking ??
liveRuntime.supportsPreserveThinking,
preserveThinking:
queuedEmptyModelRuntime?.preserveThinking ??
liveRuntime.preserveThinking,
loadedContextLength:
queuedEmptyModelRuntime !== null
? queuedEmptyModelRuntime.loadedContextLength
: liveRuntime.loadedContextLength,
loadedIsGguf:
queuedEmptyModelRuntime !== null
? queuedEmptyModelRuntime.loadedIsGguf
: liveRuntime.loadedIsGguf,
loadedIsMlx:
queuedEmptyModelRuntime !== null
? queuedEmptyModelRuntime.loadedIsMlx
: liveRuntime.loadedIsMlx,
loadedIsMultimodal:
queuedEmptyModelRuntime?.loadedIsMultimodal ??
liveRuntime.loadedIsMultimodal,
models: mergeQueuedModelCapabilities(
liveRuntime.models,
queuedEmptyModelRuntime?.checkpoint ?? "",
queuedEmptyModelRuntime?.modelCapabilities ?? null,
),
}
: runChatSettingsLive
? liveRuntime
: // The same model keeps the chat's effective values; a new one re-derives what its load sets.
liveRuntime.params.checkpoint === runtimeAtSend.params.checkpoint &&
(liveRuntime.activeGgufVariant ?? null) === (runtimeAtSend.activeGgufVariant ?? null)
? withRunThreadScopedSettings(liveRuntime, runtimeAtSend)
: withLoadedModelOverrides(
withRunThreadScopedSettings(liveRuntime, chatOwnedAtSend),
liveRuntime,
);
runtime = withChatFullAccess(runtime, liveRuntime.chatFullAccessGrants, resolvedThreadId);
const { params } = runtime;
const sandboxSessionId = await resolveSandboxSessionId(
resolvedThreadId,
readThreadRecord,
);
const toolConfirmationScopeId = resolvedThreadId
? `${sandboxSessionId || "_default"}:${resolvedThreadId}`
: sandboxSessionId || "_default";
const {
supportsTools,
toolsEnabled,
imageToolsEnabled,
mcpEnabledForChat,
confirmToolCalls,
bypassPermissions,
permissionMode,
webFetchToolsEnabled,
ragEnabled,
ragSource,
ragMode,
ragTopK,
ragAutoInject,
ragAutoInjectMinScore,
} = runtime;
const codeToolsEnabled = codeToolsOn(runtime);
if (
deepResearchArmed &&
!supportsTools &&
!isExternalModelId(params.checkpoint)
) {
deepResearchArmed = false;
if (!queuedRunSettings) {
runtime.setDeepResearchEnabled(false);
}
runtime = { ...runtime, deepResearchEnabled: false };
toast.info("Deep Research needs a model that supports tools");
}
// A project chat retrieves from its indexed sources even with Docs off; the probe is cached.
const ragProjectId = await resolveProjectId(
resolvedThreadId,
readThreadRecord,
);
const projectRagEnabled = ragProjectId
? await projectHasSources(ragProjectId)
: false;
const externalSelection = parseExternalModelId(params.checkpoint);
const isExternalRequest = externalSelection !== null;
const externalRouting = resolveExternalRouting(params.checkpoint);
if (externalRouting.kind === "unavailable") {
const refusal = EXTERNAL_ROUTING_REFUSALS[externalRouting.reason];
toast.error(refusal.title, { description: refusal.description });
clearSelectedImageEditReference();
throw new Error(refusal.error);
}
const externalProvider =
externalRouting.kind === "external" ? externalRouting.provider : null;
const providerCompactionTargetConnectionKey =
providerCompactionConnectionKey(
externalProvider?.id,
externalProvider?.baseUrl,
externalProvider?.apiType,
);
const externalUsesStudioTools =
providerModelSupportsStudioTools(
externalProvider?.providerType,
externalSelection?.modelId,
) === true;
const supportsStudioToolsForThisTurn = isExternalRequest
? externalUsesStudioTools
: supportsTools;
const selectedModelSummary = runtime.models.find(
(model) => model.id === params.checkpoint,
);
const externalApiKey =
externalRouting.kind === "external" ? externalRouting.apiKey : "";
const externalModelId =
externalRouting.kind === "external" ? externalRouting.modelId : "";
// Image-generation flag; computed first so Gemini image mode can suppress Search/Code.
const hostedImageGenerationForThisTurn = Boolean(
externalProvider &&
externalSelection &&
providerSupportsBuiltinImageGeneration(
externalProvider.providerType,
externalSelection.modelId,
externalProvider.baseUrl,
externalProvider.apiType,
),
);
const imageGenerationEnabledForThisTurn =
imageToolsEnabled && hostedImageGenerationForThisTurn;
// Otherwise the Images pill renders on the image or video model loaded in Unsloth.
const localMediaTools = localMediaToolsForTurn({
imageToolsEnabled,
supportsBuiltinImageGeneration: hostedImageGenerationForThisTurn,
supportsTools: supportsStudioToolsForThisTurn,
localImageModelReady: runtime.localImageModelReady,
localVideoModelReady: runtime.localVideoModelReady,
});
// Per-model Search/Code allowances live in providerSupportsBuiltin*; this flag only signals image-mode.
const geminiImageModeForThisTurn =
externalProvider?.providerType === "gemini" &&
imageGenerationEnabledForThisTurn;
const webSearchEnabledForThisTurn = Boolean(
externalProvider &&
externalSelection &&
toolsEnabled &&
providerSupportsBuiltinWebSearch(
externalProvider.providerType,
externalSelection.modelId,
externalProvider.baseUrl,
),
);
const codeExecEnabledForThisTurn = Boolean(
externalProvider &&
externalSelection &&
codeToolsEnabled &&
!geminiImageModeForThisTurn &&
providerSupportsBuiltinCodeExecution(
externalProvider.providerType,
externalSelection.modelId,
externalProvider.baseUrl,
externalProvider.apiType,
),
);
// Anthropic bills web_fetch separately from search; forced off in chat-page where unsupported.
const webFetchEnabledForThisTurn = Boolean(
externalProvider &&
webFetchToolsEnabled &&
providerSupportsBuiltinWebFetch(externalProvider.providerType),
);
const providerShipsWebFetch = Boolean(
externalProvider &&
providerSupportsBuiltinWebFetch(externalProvider.providerType),
);
// Hosted `code_execution` and local `python`/`terminal` are two trust boundaries, not two
// spellings of one feature. See code-tool-placement.ts.
const {
local: studioLocalCodeTools,
hosted: hostedCodeToolsForThisTurn,
} = selectCodeToolNames({
codeToolsEnabled,
hostedCodeExecutionForThisTurn: codeExecEnabledForThisTurn,
providerHostsCodeExecution: providerHostsCodeExecution(
externalProvider?.providerType,
externalProvider?.baseUrl,
externalProvider?.apiType,
),
});
if (selectedImageEditReference && !imageGenerationEnabledForThisTurn) {
clearSelectedImageEditReference();
toast.error("Image editing is unavailable", {
description:
"Select an OpenAI image-generation model, then retry the edit.",
});
throw new Error("Image generation edit unavailable.");
}
// Match backend gates and local vision flags so targets that strip MCP images never upload them.
const targetReadsImages = isExternalRequest
? providerModelTakesMcpImages(
externalProvider?.providerType,
externalSelection?.modelId,
)
: localTargetReadsImages(runtime);
// A non-GGUF local target renders replayed pictures as markers, so bound the upload accordingly.
const mcpImagesLocalMarkers =
!isExternalRequest &&
runtime.models.find((model) => model.id === runtime.params.checkpoint)
?.isGguf === false;
// Bound before building the envelope so dropped content is never stringified or reparsed.
const messages = boundMcpImageResults(rawMessages, {
readsImages: targetReadsImages,
localMarkers: mcpImagesLocalMarkers,
});
const replayReasoning = !isExternalRequest || (
providerSupportsPreserveThinking(externalProvider?.providerType) &&
runtime.preserveThinking
) || providerModelReplaysReasoning(
externalProvider?.providerType,
externalSelection?.modelId,
);
const survivingMessages = pruneOutboundHistory(messages, replayReasoning);
const { messages: renderedMessages, sandboxAttachments } =
supportsStudioToolsForThisTurn &&
studioLocalCodeTools.includes("python")
? withSandboxAttachmentPaths(survivingMessages)
: { messages: survivingMessages, sandboxAttachments: [] };
// The backend Gemini translator rebuilds functionCall/functionResponse parts from these.
let outboundMessages = renderedMessages
.flatMap((message) => {
const serialized = toOpenAIMessages(
message,
replayReasoning,
externalProvider?.providerType === "gemini",
);
return isExternalRequest
? withProviderCompaction(message, serialized, {
providerType: toExternalBackendProviderType(
externalProvider?.providerType,
),
modelId: externalSelection?.modelId,
connectionKey: providerCompactionTargetConnectionKey,
})
: serialized;
})
.filter((message): message is NonNullable<typeof message> =>
Boolean(message),
);
if (selectedImageEditReference) {
const referenceMessage = toOpenAIImageEditReferenceMessage(
selectedImageEditReference,
);
if (!referenceMessage) {
clearSelectedImageEditReference();
toast.error("This generated image cannot be edited", {
description:
"The original image reference is missing. Generate the image again, then retry the edit.",
});
throw new Error("Generated image edit reference missing.");
}
let insertAt = outboundMessages.length;
for (let i = outboundMessages.length - 1; i >= 0; i -= 1) {
if (outboundMessages[i]?.role === "user") {
insertAt = i;
break;
}
}
outboundMessages.splice(
insertAt,
0,
referenceMessage as unknown as SerializedMessage,
);
}
// Carry reasoning only to a backend that can resume it.
const resumedThought = continuation
? carriedThought(continuation, {
resumesThought: resumesThought({
loadedIsGguf: runtime.loadedIsGguf,
loadedIsMlx: runtime.loadedIsMlx,
activeGgufVariant: runtime.activeGgufVariant,
activeNativePathToken: runtime.activeNativePathToken,
checkpoint: params.checkpoint,
}),
managedEngine:
!isExternalModelId(params.checkpoint) &&
runtime.loadedEngine !== "auto",
})
: "";
const hasGeminiReplayContinuation = Boolean(
continuation &&
externalProvider?.providerType === "gemini" &&
(continuation.geminiReplayTurns?.length ||
continuation.thoughtParts?.length ||
continuation.answerParts?.length ||
continuation.thoughtSignature),
);
if (
continuation &&
!continuation.partial &&
!resumedThought &&
!hasGeminiReplayContinuation
) {
toast.error("This response cannot be resumed", {
description:
"It stopped mid-thought, and only GGUF and MLX models can resume a thought. Use Retry instead.",
});
throw new Error("A response that stopped mid-thought cannot be resumed here.");
}
const continuationCompaction = continuation
? providerCompactionForTarget(
continuation,
toExternalBackendProviderType(externalProvider?.providerType),
externalSelection?.modelId,
providerCompactionTargetConnectionKey,
)
: null;
// The run's messages stop at the user turn, so the partial is appended here for the backend to resume.
if (continuation) {
const geminiReplayTurns =
externalProvider?.providerType === "gemini"
? continuation.geminiReplayTurns
: undefined;
if (geminiReplayTurns?.length) {
outboundMessages.push(
...serializeGeminiContinuationTurns(geminiReplayTurns, false),
);
} else {
outboundMessages.push({
role: "assistant",
content: continuationCompaction
? [
continuationCompaction,
...(continuation.partial
? [{ type: "text" as const, text: continuation.partial }]
: []),
]
: continuation.partial,
...(resumedThought ? { reasoning_content: resumedThought } : {}),
...(continuation.thoughtParts && continuation.thoughtParts.length > 0
? {
extra_content: withGeminiThoughtReplayParts(
undefined,
continuation.thoughtParts,
),
}
: {}),
});
attachAssistantThoughtSignature(
outboundMessages,
continuation.thoughtSignature,
);
}
// Anthropic 400s when the last message is an assistant turn, so append an instruction turn
// instead of a prefill.
if (rejectsAssistantPrefill(externalProvider?.providerType)) {
outboundMessages.push({
role: "user",
content: CONTINUE_INSTRUCTION,
});
}
}
const combinedSystemPrompt = await resolveChatInstructions(
resolvedThreadId,
params.systemPrompt,
params.systemVariables,
readThreadRecord,
{ agentsMd: true },
);
if (combinedSystemPrompt) {
prependSystemPrompt(outboundMessages, combinedSystemPrompt);
}
let disabledToolGuard: string | null = null;
const disabledToolGuardProviderType = externalProvider?.providerType;
if (
disabledToolGuardProviderType === "anthropic" ||
disabledToolGuardProviderType === "openai"
) {
const webLabel = providerShipsWebFetch
? "web search or web fetch"
: "web search";
// Treat search and fetch as one "any web tool" axis: checking webSearch alone mis-fired when only Fetch was on.
const anyWebEnabledForThisTurn =
webSearchEnabledForThisTurn || webFetchEnabledForThisTurn;
if (
!anyWebEnabledForThisTurn &&
!codeExecEnabledForThisTurn &&
!imageGenerationEnabledForThisTurn
) {
disabledToolGuard =
`You do not have ${webLabel}, code execution, or image generation tools in this conversation. ` +
"Answer from your own knowledge. " +
"If a request genuinely requires tool use, live data fetch, running code, or image generation, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
} else if (!anyWebEnabledForThisTurn && !codeExecEnabledForThisTurn) {
disabledToolGuard =
`You do not have ${webLabel} or code execution tools in this conversation. ` +
"You may still use image generation tools when they are available and useful. " +
"If a request genuinely requires live data fetch or running code, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
} else if (!anyWebEnabledForThisTurn) {
const availableTools = [
codeExecEnabledForThisTurn ? "code execution" : null,
imageGenerationEnabledForThisTurn ? "image generation" : null,
].filter(Boolean);
disabledToolGuard =
`You do not have ${webLabel} tools in this conversation. ` +
(availableTools.length > 0
? `You may still use ${availableTools.join(" and ")} tools when they are available and useful. `
: "") +
"If a request genuinely requires live data fetch or web search tool use, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
} else if (!codeExecEnabledForThisTurn) {
const availableTools = [
webLabel,
imageGenerationEnabledForThisTurn ? "image generation" : null,
].filter(Boolean);
disabledToolGuard =
"You do not have code execution tools in this conversation. " +
`You may still use ${availableTools.join(" and ")} tools when they are available and useful. ` +
"If a request genuinely requires running code or code execution tool use, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
}
}
type OutboundMessage = (typeof outboundMessages)[number];
function addSystemInstruction(
targetMessages: OutboundMessage[],
text: string | null,
): void {
if (!text) return;
const firstMessage = targetMessages[0];
if (firstMessage?.role === "system") {
if (typeof firstMessage.content === "string") {
targetMessages[0] = {
...firstMessage,
content: `${firstMessage.content}\n\n${text}`,
};
} else {
targetMessages[0] = {
...firstMessage,
content: [
...(Array.isArray(firstMessage.content)
? firstMessage.content
: []),
{ type: "text", text: `\n\n${text}` },
],
};
}
return;
}
targetMessages.unshift({ role: "system", content: text });
}
// Scan post-prune history so a refused user turn's image/audio does not gate or mis-attribute the next turn.
const imageBase64 = findLatestUserImageBase64(survivingMessages);
// A clip staged since would switch a continuation onto the uncontinuable audio path.
const audioBase64 = findLatestUserAudioBase64(
survivingMessages,
!queuedRunSettings && !continuation,
);
const videoBase64 = findLatestUserVideoBase64(survivingMessages);
addSystemInstruction(outboundMessages, disabledToolGuard);
const blockAttachmentRun = (reason: string): never => {
toast.error(reason);
// Flip on->off so compare-mode waitForRunEnd resolves: this gate fires before setThreadRunning(true).
const gatedThreadKey = resolvedThreadId || "__default";
// Own token: siblings share "__default", so an ownerless clear would drop entries that are still generating.
const gateOwner = createImageGateRunOwner();
runtime.setThreadRunning(gatedThreadKey, true, { owner: gateOwner });
runtime.setThreadRunning(gatedThreadKey, false, { owner: gateOwner });
clearSelectedImageEditReference();
throw new Error(reason);
};
// Block any image for a non-vision model; switching models requires a new chat.
if (imageBase64) {
const activeModel = runtime.models.find(
(m) => m.id === params.checkpoint,
);
const imageGateReason = getImageInputUnavailableReason({
activeModel,
isExternalModel: isExternalRequest,
externalSupportsVision: providerModelSupportsVision(
externalProvider?.providerType,
externalSelection?.modelId,
),
externalModelLabel: externalSelection?.modelId ?? null,
loadedIsMultimodal: runtime.loadedIsMultimodal,
modelLoaded: !!params.checkpoint && !runtime.modelLoading,
loadError: runtime.lastModelLoadError,
visionDisabledByUser: runtime.loadedVisionDisabledByUser,
mmprojFallbackReason: runtime.mmprojFallbackReason,
});
if (imageGateReason) {
blockAttachmentRun(imageGateReason);
}
}
// Media attached before a model loaded skipped the add-time check; recorded audio is not counted.
const answeringModel = runtime.models.find(
(m) => m.id === params.checkpoint,
);
const attachedMediaReason = attachedMediaUnavailableReason({
activeModel: answeringModel,
checkpoint: params.checkpoint,
modelLabel:
answeringModel?.name ||
externalModelLabel(params.checkpoint) ||
params.checkpoint,
audio: Boolean(findLatestUserAudioBase64(survivingMessages, false)),
audioCount: latestUserAudioClips(survivingMessages).length,
video: Boolean(videoBase64),
externalTakesVideo: externalModelTakesVideo(params.checkpoint),
});
if (attachedMediaReason) {
blockAttachmentRun(attachedMediaReason);
}
if (audioBase64 && !queuedRunSettings) {
const audioName = runtime.pendingAudioName;
if (audioName) {
const lastUserMsg = [...survivingMessages]
.reverse()
.find((m) => m.role === "user");
if (lastUserMsg) sentAudioNames.set(lastUserMsg.id, audioName);
}
runtime.clearPendingAudio();
}
const useAdapter = await resolveUseAdapter(
resolvedThreadId,
options,
readThreadRecord,
);
const threadKey = resolvedThreadId || "__default";
// A new chat's browser is the one on screen as it sends; a switch mid-run must not move it.
const browserChat = options.pairId
? `compare:${options.pairId}`
: threadKey === "__default"
? shownBrowserChat()
: undefined;
// A first turn files its handles under "__default" and autosave re-keys them mid-run, so resolve per use.
const liveThreadKey = (owner: () => void) => {
const key = threadKey === "__default" ? useChatRuntimeStore.getState().runKeyForOwner(threadKey, owner) : threadKey;
if (key !== "__default") useChatRuntimeStore.getState().adoptChatFullAccessRun(cancelId, key);
return key;
};
// Per-run token so a delayed stop POST can't match the next run.
const cancelId =
typeof crypto !== "undefined" && "randomUUID" in crypto
? crypto.randomUUID()
: `${Date.now()}-${Math.random().toString(36).slice(2)}`;
// Per-run abort chained to assistant-ui's signal: cancelByThreadId only holds the visible thread's cancelRun().
const runAbort = new AbortController();
const runSignal = runAbort.signal;
const forwardAbort = () => runAbort.abort(abortSignal.reason);
// Declared here because it doubles as this run's identity token on the per-thread maps.
const serverCancel = () => runAbort.abort();
if (abortSignal.aborted) {
forwardAbort();
} else {
abortSignal.addEventListener("abort", forwardAbort, { once: true });
}
const activeModel = runtime.models.find(
(m) => m.id === params.checkpoint,
);
// Same owner the settings panel asks, so body and panel agree; a catalog row can be replaced.
const isGgufForCompaction = isServedByLlamaCpp({
loadedIsGguf: runtime.loadedIsGguf,
activeGgufVariant: runtime.activeGgufVariant,
activeNativePathToken: runtime.activeNativePathToken,
checkpoint: params.checkpoint,
});
// Use the backend's report: an external id leaves it describing the resident local model.
const isMlxForCompaction =
!isExternalModelId(params.checkpoint) && runtime.loadedIsMlx === true;
const isManagedEngineForCompaction =
!isExternalModelId(params.checkpoint) && runtime.loadedEngine !== "auto";
// Taken once, so a retried request still carries it and the next turn does not.
const compactNow =
takeThreadCompactionRequest(resolvedThreadId) &&
(isGgufForCompaction || isMlxForCompaction);
const generationUserMessage = [...survivingMessages]
.reverse()
.find((message) => message.role === "user");
// Names the turn whose file changes /rewind can undo; a continuation reuses its turn's prompt.
// A temporary chat keeps no file history.
const userMessageIdField = userMessageIdRequestField(
resolvedThreadId && isThreadIncognito(resolvedThreadId)
? undefined
: generationUserMessage?.id,
);
// The durability gate keys on THIS turn's attachments only, so an old screenshot does not keep
// every later text turn off the durable path.
const currentTurnMessages = [generationUserMessage] as unknown as Parameters<
typeof findLatestUserImageBase64
>[0];
const [mcpImage, ...extraToolImages] = toolOnlyImages(
generationUserMessage,
);
if (extraToolImages.length > 0) {
throw new Error("Attach only one image for MCP tools per message.");
}
const currentTurnCarriesMedia = Boolean(
// A tool-only image needs the live stream to ask before it is sent.
mcpImage ||
findLatestUserImageBase64(currentTurnMessages) ||
findLatestUserAudioBase64(currentTurnMessages, !queuedRunSettings && !continuation) ||
findLatestUserVideoBase64(currentTurnMessages),
);
const generationCandidate = isDurableRunCandidate({
externalProvider: isExternalRequest,
modelIsAudio: activeModel?.isAudio,
loadedIsDiffusion: runtime.loadedIsDiffusion,
// Turn-scoped, not thread-scoped: see currentTurnCarriesMedia above.
turnCarriesMedia: currentTurnCarriesMedia,
// Yield the partial first so autosave lands before admission, which 409s substantive placeholders.
continuation,
threadId: resolvedThreadId,
incognito: resolvedThreadId ? isThreadIncognito(resolvedThreadId) : false,
assistantMessageId: unstable_assistantMessageId,
hasUserMessage: Boolean(generationUserMessage),
});
let generationDecision: "pending" | "durable" | "legacy" =
generationCandidate ? "pending" : "legacy";
let generationRun: ChatGenerationRun | null = null;
let generationRunId: string | null = null;
let generationSeq = 0;
let generationStatus: ChatGenerationStatus | null = null;
let generationFirstChunkAt: number | undefined;
let generationChunkCount = 0;
let generationStopRequested = false;
// Both this stream and recovery followers persist it, or reload blocks the composer again.
let generationStalled = false;
let parseThink = true;
const generationCustom = () =>
generationRunId
? {
generationRunId,
generationSeq,
generationStatus,
generationFirstChunkAt,
generationChunkCount,
generationSettled: generationIsSettled(
generationStatus,
generationSeq,
generationRun?.lastEventSeq ?? Number.POSITIVE_INFINITY,
),
generationLocallyInterrupted: generationStalled,
serverManaged: true,
parseThinkTags: parseThink,
}
: {};
if (activeModel?.isAudio && !activeModel?.hasAudioInput) {
const audioCancel = () => runAbort.abort();
runtime.registerThreadServerCancel(threadKey, audioCancel);
releaseCurrentPreStreamRun();
void persistRunModel(params.checkpoint, runtime.activeGgufVariant);
runtime.setThreadRunning(threadKey, true, { owner: audioCancel });
try {
yield {
content: [{ type: "text" as const, text: "Generating audio..." }],
};
const result = await generateAudio(
{
model: params.checkpoint,
messages: outboundMessages,
// Same run in both registries, or the stop-chats prompt counts it twice.
...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}),
stream: false,
temperature: params.temperature,
top_p: params.topP,
max_tokens: params.maxTokens,
top_k: params.topK,
min_p: params.minP,
repetition_penalty: params.repetitionPenalty,
presence_penalty: params.presencePenalty,
...(useAdapter === undefined ? {} : { use_adapter: useAdapter }),
},
runSignal,
);
const audioUrl = `data:audio/wav;base64,${result.audio.data}`;
yield {
content: [
{
type: "text" as const,
text: `<audio-player src="${audioUrl}" />`,
},
],
};
} catch (err) {
if (!runSignal.aborted) {
toast.error("Audio generation failed", {
description: err instanceof Error ? err.message : "Unknown error",
});
}
throw err;
} finally {
abortSignal.removeEventListener("abort", forwardAbort);
const audioKey = liveThreadKey(audioCancel);
runtime.setThreadRunning(audioKey, false, { owner: audioCancel });
runtime.clearThreadServerCancel(audioKey, audioCancel);
}
return;
}
let waitingFirstChunk = true;
let firstTokenSettled = false;
const streamStartTime = Date.now();
let responseModelId = externalSelection?.modelId ?? params.checkpoint;
let firstTokenTime: number | undefined;
let totalChunks = 0;
let lastFinishReason: string | null = null;
let resolveFirstToken: (() => void) | null = null;
let rejectFirstToken: ((err: unknown) => void) | null = null;
const firstTokenPromise = new Promise<void>((resolve, reject) => {
resolveFirstToken = resolve;
rejectFirstToken = reject;
});
// Avoid unhandled rejections if toast.promise never attached.
void firstTokenPromise.catch(() => {});
function settleFirstTokenOk(): void {
if (firstTokenSettled) return;
firstTokenSettled = true;
resolveFirstToken?.();
}
function settleFirstTokenErr(err: unknown): void {
if (firstTokenSettled) return;
firstTokenSettled = true;
rejectFirstToken?.(err);
}
const warmupDelayMs = 450;
const warmupTimer = setTimeout(() => {
if (!waitingFirstChunk) return;
if (runSignal.aborted) return;
runtime.setGeneratingStatus("waiting");
}, warmupDelayMs);
// The swap gate counts only local runs; backend excludes external ones likewise.
releaseCurrentPreStreamRun();
// Past every gate that refuses the send, and not awaited: only what reopening the chat offers.
void persistRunModel(params.checkpoint, runtime.activeGgufVariant);
runtime.setThreadRunning(threadKey, true, {
local: !isExternalRequest,
owner: serverCancel,
});
const continuationPartial = continuation
? continuationSeed(continuation.partial, resumedThought)
: "";
const geminiContinuationReplayTurns =
continuation?.geminiReplayTurns?.length
? continuation.geminiReplayTurns
: undefined;
// A seed that ends inside the thought: the next reasoning delta extends it.
const resumesInsideThought = Boolean(resumedThought) && !continuation?.partial;
let cumulativeText = continuationPartial;
const passText = () => cumulativeText.slice(continuationPartial.length);
const priorTiming = continuation?.priorTiming;
let passSoFar: (() => PriorPassTiming) | null = null;
const geminiContinuationReplay = (
final: boolean,
): GeminiContinuationReplay | undefined =>
geminiContinuationReplayTurns
? {
turns: geminiContinuationReplayTurns,
visiblePrefix: continuation?.partial ?? "",
stripVisiblePrefix:
!final ||
!isRestart(
continuation?.partial ?? "",
cumulativeText.slice(continuationPartial.length),
),
}
: undefined;
// Reading `cumulativeText` costs O(reply): each `+=` builds a cons string that the first read
// flattens, so one charCodeAt per arrival is as expensive as a scan. Everything below is fed
// the delta through `appendCumulative` and the buffer is read only where the reply is
// published. This tracks "does the text end inside <think>".
const thinkTags = createThinkTagTracker();
// Only run the ${...} strip when an arrival ends in a fragment.
const placeholderWatch = createTrailingPlaceholderWatch();
// What the cap is measured against: only grows, and counts tool-argument deltas.
let streamedChars = 0;
// Continuations are seeded with the prior partial; a run adding nothing must not be trimmed.
let producedReplyText = false;
// Local backends resume at the exact token, so only providers that repeat or restart need repair.
const repairContinuation =
isExternalRequest && !resumesExactly(externalProvider?.providerType);
// Repair once on the finished turn; per-arrival runs could shrink published text.
const mergeContinuation = createContinuationMerger(
continuationPartial,
repairContinuation,
);
// The final merge can rewrite the prefix, which an extend-only parse cannot follow.
const trustAppends = !(continuationPartial && repairContinuation);
let segmentedText = createSegmentedAssistantText({ trustAppends });
// Thinking off leaves <think> as text unless reasoning arrives; then reparse with the tags.
const setParseThink = (next: boolean): void => {
if (next === parseThink) {
return;
}
parseThink = next;
segmentedText = createSegmentedAssistantText({
trustAppends,
parseThink,
});
};
// The only place cumulativeText grows, so all derived state sees the same order.
const appendCumulative = (text: string): void => {
if (!text) {
return;
}
cumulativeText += text;
segmentedText.appendText(text);
thinkTags.append(text);
placeholderWatch.append(text);
};
// A resumed turn starts with the partial already in the buffer; one read of it puts the trackers in step.
if (cumulativeText) {
thinkTags.append(cumulativeText);
placeholderWatch.append(cumulativeText);
}
// assistant-ui drops yields after an abort, so every streamed yield carries repaired text.
let codexReasoningLedger: CodexReasoningLedger = { byToolCall: {} };
let openAIResponsesReasoningLedger: CodexReasoningLedger = {
byToolCall: {},
};
let codexRoundToolCallIds: string[] = [];
let contextTruncation: OpenAIChatChunk["context_truncated"];
let compactionMarkers: CompactionMarker[] | undefined;
let providerCompaction: ProviderCompactionContentPart | undefined =
continuationCompaction ?? undefined;
let providerCompactionAfterToolCalls: number | undefined =
continuationCompaction
? continuation?.providerCompactionAfterToolCalls
: undefined;
let providerCompactionProviderType: string | undefined =
continuationCompaction
? continuation?.providerCompactionProviderType
: undefined;
let providerCompactionModelId: string | undefined =
continuationCompaction
? continuation?.providerCompactionModelId
: undefined;
let providerCompactionOriginConnectionKey: string | undefined =
continuationCompaction
? continuation?.providerCompactionConnectionKey
: undefined;
const liveAssistantContent = () =>
buildAssistantContent(mergeContinuation(cumulativeText));
// Declared above the live metadata that reads it, or it is in its temporal dead zone.
let contextWindowExceeded = false;
let quoteCut = false;
// Provisional reason per yield since aborts skip terminal yields; a reported window outranks it.
const liveCustom = () => ({
...reasoningDurationTracker.metadata(),
resumeTiming: passSoFar ? addPass(priorTiming, passSoFar()) : priorTiming,
openaiCodexReasoning: codexReasoningLedger,
openaiResponsesReasoning: openAIResponsesReasoningLedger,
contextTruncation,
compactionMarkers,
providerCompaction,
providerCompactionAfterToolCalls,
providerCompactionProviderType,
providerCompactionModelId,
providerCompactionConnectionKey: providerCompactionOriginConnectionKey,
geminiContinuationReplay: geminiContinuationReplay(false),
// A legacy (browser-tool / attachment / incognito) run that ends because you closed the tab has no
// server-side run to resume from, so its last streamed yield is what persists. Mark it an interruption
// — partial kept + Resume — instead of a silent blank/ambiguous state. Durable runs keep "cancelled":
// their reconnect path marks state via recovery, and an explicit Stop still reads as cancelled below.
// A window the provider reported as full outranks either guess (utils/continuation.ts).
incomplete: {
reason: resolveIncompleteReason(
// After an explicit Stop, yields carry it so the turn persists as cancelled.
incompleteReason ??
(generationDecision === "durable" ? "cancelled" : "interrupted"),
contextWindowExceeded,
),
},
...generationCustom(),
});
let incompleteReason: IncompleteReason | null = null;
// MLX reports finish_reason "stop" even at the cap, so an exhausted budget is its only truncation signal.
let requestedMaxTokens: number | undefined;
// Served window: null if unknown, Infinity if only the output cap can bind.
let servedContextLength: number | null = null;
const isMlxRequest = !isExternalRequest && activeModel?.isMlx === true;
const reasoningDurationTracker = createReasoningDurationTracker();
if (resumedThought) {
reasoningDurationTracker.seedThought({
duration: continuation?.reasoningDuration,
open: resumesInsideThought,
textLength: resumedThought.length,
});
}
// Keep the <think> block open across reasoning deltas; answer content closes it.
let reasoningContentOpen = resumesInsideThought;
type ToolCallProvenance = {
source?: string;
healed?: boolean;
forced?: boolean;
provisional?: boolean;
duplicate?: boolean;
reason?: string;
mcp_server?: string;
[key: string]: unknown;
};
type PositionedToolCallPart = ToolCallMessagePart & {
backendToolCallId?: string;
generationToolCallId?: string;
toolApprovalId?: string;
textCursor?: number;
_delta_index?: number;
_has_stable_id?: boolean;
// No `arguments` field at all; a zero-parameter tool sends "".
_announced_only?: boolean;
_resend_suspect?: boolean;
_superseded?: boolean;
_from_fork?: boolean;
extra_content?: unknown;
provenance?: ToolCallProvenance;
};
const toolCallParts: PositionedToolCallPart[] = [];
// Ids reserved for id-less calls; cards mint before joining toolCallParts, risking duplicates.
const reservedToolCallIds = new Set<string>();
// Provider-sent ids; a split marks all but the last card to keep late ids off claimed calls.
const providerSentToolCallIds = new Set<string>();
const boundaryScans = new Map<
string,
ReturnType<typeof createBoundaryScan>
>();
const isPlainRecord = (v: unknown): v is Record<string, unknown> =>
typeof v === "object" && v !== null && !Array.isArray(v);
// Unclosed tail forks (`{"a":1}{` is not truncated) would otherwise persist as dead cards.
const openTailIds = new Set<string>();
// Parked until the next object shows whether a repeated name is a resend or a new call.
const pendingExtraByPartId = new Map<string, Record<string, unknown>>();
const endProviderTurn = (): boolean => {
let changed = false;
// _call_is_finished, on the arguments alone: the backend holds the fork back whatever id reached it.
for (const tailId of openTailIds) {
const at = toolCallParts.findIndex((p) => p.toolCallId === tailId);
if (at === -1) continue;
const part = toolCallParts[at];
const scanned = splitTopLevelJsonObjects(part.argsText ?? "");
if (scanned.complete.length > 0 && !scanned.tail) continue;
toolCallParts.splice(at, 1);
releaseStreamedCard(part.toolCallId);
changed = true;
}
// Drop announcements another call took over; keep lone ones (zero-parameter tools).
for (let at = toolCallParts.length - 1; at >= 0; at -= 1) {
const part = toolCallParts[at] as PositionedToolCallPart;
if (
(!part._superseded &&
!part._resend_suspect &&
!(part._announced_only && part._from_fork)) ||
part.argsText
)
continue;
// Move metadata to that call: Gemini stores a thought signature there and requires it on replay.
const previous = toolCallParts[at - 1] as
| PositionedToolCallPart
| undefined;
if (part.extra_content !== undefined && previous) {
previous.extra_content = {
...(previous.extra_content ?? {}),
...part.extra_content,
};
}
toolCallParts.splice(at, 1);
releaseStreamedCard(part.toolCallId);
changed = true;
}
// The backend drops nameless calls before reserving ids, so a kept card would steal a number.
for (let at = toolCallParts.length - 1; at >= 0; at -= 1) {
const part = toolCallParts[at] as PositionedToolCallPart;
if (part.toolName || part._delta_index === undefined) continue;
toolCallParts.splice(at, 1);
releaseStreamedCard(part.toolCallId);
changed = true;
}
for (const [partId, waiting] of pendingExtraByPartId) {
// No object ever came, so the repeated name was that call's after all, and so is its metadata.
const at = toolCallParts.findIndex((part) => part.toolCallId === partId);
if (at === -1) continue;
const part = toolCallParts[at] as PositionedToolCallPart;
toolCallParts[at] = {
...part,
extra_content: {
...(isPlainRecord(part.extra_content) ? part.extra_content : {}),
...waiting,
},
};
changed = true;
}
pendingExtraByPartId.clear();
// The backend numbers surviving calls, so close the gaps here the same way.
if (changed) renumberMintedCards();
openTailIds.clear();
// The next round opens at index 0 again, and without this "B" then "C" across the boundary named one call "BC".
for (let at = 0; at < toolCallParts.length; at += 1) {
const part = toolCallParts[at] as PositionedToolCallPart;
if (part._delta_index === undefined) continue;
const closed = { ...part };
delete closed._delta_index;
toolCallParts[at] = closed;
}
return changed;
};
const scanArgsText = (partId: string, text: string) => {
let scan = boundaryScans.get(partId);
if (!scan) {
scan = createBoundaryScan();
boundaryScans.set(partId, scan);
}
return scan.feed(text);
};
const mintStreamedCardId = (deltaIndex: number | undefined): string =>
mintStreamedToolCallId(toolCallParts, deltaIndex, reservedToolCallIds);
const paintStreamedCard = (partId: string): void => {
reservedToolCallIds.add(partId);
bindStreamedToolCallCard(toolPartIdByBackendId, partId);
};
// The backend streams partial arguments; this accumulates them for the live parse.
const liveArgsTextById = new Map<string, string>();
// A dropped card gives its id back: holding it makes the next round's mint skip a number it then reuses.
const releaseStreamedCard = (partId: string): void => {
reservedToolCallIds.delete(partId);
toolPartIdByBackendId.delete(partId);
// A stale provider-id alias makes the mint skip a number, so remove it too.
for (const [backendId, mapped] of [...toolPartIdByBackendId]) {
if (mapped !== partId) continue;
toolPartIdByBackendId.delete(backendId);
reservedToolCallIds.delete(backendId);
providerSentToolCallIds.delete(backendId);
}
boundaryScans.delete(partId);
openTailIds.delete(partId);
liveArgsTextById.delete(partId);
pendingExtraByPartId.delete(partId);
};
// The backend reserves provider ids before minting; renumber in its walk order or tool_start
// reaches the wrong card.
const renameStreamedCard = (from: string, to: string): void => {
reservedToolCallIds.delete(from);
toolPartIdByBackendId.delete(from);
const scan = boundaryScans.get(from);
if (scan) boundaryScans.set(to, scan);
boundaryScans.delete(from);
if (openTailIds.delete(from)) openTailIds.add(to);
const live = liveArgsTextById.get(from);
if (live !== undefined) liveArgsTextById.set(to, live);
liveArgsTextById.delete(from);
// The turn-end sweep looks up parked metadata by current part id, so move it with the rename.
const pending = pendingExtraByPartId.get(from);
if (pending) pendingExtraByPartId.set(to, pending);
pendingExtraByPartId.delete(from);
const at = codexRoundToolCallIds.indexOf(from);
if (at !== -1) codexRoundToolCallIds[at] = to;
paintStreamedCard(to);
};
const renumberMintedCards = (claimed?: string): void => {
const minted = toolCallParts.filter(
(part) =>
!providerSentToolCallIds.has(part.toolCallId) &&
// This round's cards only: the backend's ledger is append-only and never renumbers, so neither may this.
(part as PositionedToolCallPart)._delta_index !== undefined,
) as PositionedToolCallPart[];
const carried = minted.map((part) => ({
part,
from: part.toolCallId,
scan: boundaryScans.get(part.toolCallId),
live: liveArgsTextById.get(part.toolCallId),
openTail: openTailIds.has(part.toolCallId),
pending: pendingExtraByPartId.get(part.toolCallId),
}));
for (const held of carried) {
reservedToolCallIds.delete(held.from);
toolPartIdByBackendId.delete(held.from);
boundaryScans.delete(held.from);
liveArgsTextById.delete(held.from);
openTailIds.delete(held.from);
pendingExtraByPartId.delete(held.from);
// Clear before minting: the mint reads held ids, and a stale one skips a number.
const at = toolCallParts.indexOf(held.part);
if (at !== -1) toolCallParts[at] = { ...held.part, toolCallId: "" };
}
// Held back only now: the loop above frees every minted id, and one of them is the spelling being claimed.
if (claimed !== undefined) reservedToolCallIds.add(claimed);
for (const held of carried) {
const to = mintStreamedCardId(held.part._delta_index);
paintStreamedCard(to);
if (held.scan) boundaryScans.set(to, held.scan);
if (held.live !== undefined) liveArgsTextById.set(to, held.live);
if (held.openTail) openTailIds.add(to);
if (held.pending) pendingExtraByPartId.set(to, held.pending);
const at = codexRoundToolCallIds.indexOf(held.from);
if (at !== -1) codexRoundToolCallIds[at] = to;
const index = toolCallParts.findIndex((part) => part.toolCallId === "");
if (index !== -1) {
toolCallParts[index] = { ...held.part, toolCallId: to };
}
}
};
/** Nothing per-call is copied; the thought signature goes only to the last call. */
const bornSplitToolCalls = (
extraSegments: string[],
toolName: string,
deltaIndex: number | undefined,
extraContent: unknown,
): PositionedToolCallPart[] =>
extraSegments.map((segment, n) => {
const isLast = n === extraSegments.length - 1;
let segmentArgs: ToolCallMessagePart["args"] = {};
try {
segmentArgs = JSON.parse(segment) as ToolCallMessagePart["args"];
} catch {
segmentArgs = { _raw: segment } as ToolCallMessagePart["args"];
}
const bornId = mintStreamedCardId(deltaIndex);
paintStreamedCard(bornId);
if (!codexRoundToolCallIds.includes(bornId)) {
codexRoundToolCallIds.push(bornId);
}
return {
type: "tool-call" as const,
toolCallId: bornId,
toolName,
argsText: segment,
args: segmentArgs,
textCursor: cumulativeText.length,
...(isLast && extraContent !== undefined
? { extra_content: extraContent }
: {}),
// All but the last are spoken for; the last stays claimable.
...(isLast ? {} : { _has_stable_id: true }),
...(deltaIndex !== undefined ? { _delta_index: deltaIndex } : {}),
};
});
// Backend tool ids restart every response, so mint one run-unique part id per backend id;
// dropped at tool_end.
const toolPartIdByBackendId = new Map<string, string>();
const resolveToolPartId = (backendToolCallId: string): string =>
resolveToolCallPartId(
toolPartIdByBackendId,
backendToolCallId,
toolConfirmationIdsByBackendId.get(backendToolCallId),
toolCallParts[toolCallParts.length - 1]?.toolCallId ?? "",
() => `${backendToolCallId}:${crypto.randomUUID()}`,
);
let legacyGeminiTextSignature: string | undefined;
const geminiThoughtParts: PositionedGeminiThoughtReplayPart[] =
geminiContinuationReplayTurns
? []
: (continuation?.thoughtParts?.map((part) => ({
...part,
afterToolCalls: 0,
})) ?? []);
const geminiAnswerParts: PositionedGeminiAnswerReplayPart[] =
geminiContinuationReplayTurns
? []
: (continuation?.answerParts?.map((part) => ({
...part,
afterToolCalls: 0,
})) ?? []);
// A spoken reply's audio, kept apart from the text so the player renders on its own.
let spokenReplyPart: { type: "text"; text: string } | null = null;
const buildAssistantContent = (rawText: string) => {
const positionedTools = toolCallParts
.map((part, index) => {
const cursor = (part as PositionedToolCallPart).textCursor;
return {
part,
index,
cursor:
typeof cursor === "number" && Number.isFinite(cursor)
? Math.min(Math.max(cursor, 0), rawText.length)
: 0,
};
})
.sort((a, b) => a.cursor - b.cursor || a.index - b.index);
// The distinct cursors, where the text is cut into runs: only the last one is still growing.
const boundaries: number[] = [];
for (const positioned of positionedTools) {
if (boundaries[boundaries.length - 1] !== positioned.cursor) {
boundaries.push(positioned.cursor);
}
}
const runs = segmentedText.runs(rawText, boundaries);
const assembled: Array<
ReturnType<typeof parseAssistantContent>[number] | ToolCallMessagePart
> = [];
let toolIndex = 0;
for (let index = 0; index < boundaries.length; index += 1) {
assembled.push(...runs[index]);
const cursor = boundaries[index];
while (
toolIndex < positionedTools.length &&
positionedTools[toolIndex].cursor === cursor
) {
assembled.push(positionedTools[toolIndex].part);
toolIndex += 1;
}
}
assembled.push(...runs[boundaries.length]);
if (spokenReplyPart) assembled.push(spokenReplyPart);
pinGeminiThoughtReplayParts(assembled, geminiThoughtParts);
pinGeminiAnswerReplayParts(assembled, geminiAnswerParts);
return pinGeminiTextThoughtSignature(
assembled,
geminiAnswerParts.length > 0
? undefined
: legacyGeminiTextSignature,
);
};
const {
supportsReasoning,
reasoningEnabled,
reasoningAlwaysOn,
reasoningStyle,
reasoningEffort,
reasoningEffortLevels,
} = runtime;
const externalReasoningCaps: ReturnType<
typeof getExternalReasoningCapabilities
> =
externalSelection && externalProvider
? getExternalReasoningCapabilities(
externalProvider.providerType,
externalSelection.modelId,
{
isReasoningProvider: externalProvider.isReasoningModel === true,
baseUrl: externalProvider.baseUrl ?? null,
apiType: externalProvider.apiType,
reasoningConfig: externalProvider.reasoningConfig,
},
)
: {
supportsReasoning,
reasoningStyle,
reasoningAlwaysOn: false,
supportsReasoningOff: false,
reasoningEffortLevels: ["low", "medium", "high"] as const,
};
const externalReasoningEnabled =
!externalReasoningCaps.supportsReasoningOff ? true : reasoningEnabled;
type RequestReasoningEffort = Extract<
NonNullable<OpenAIChatCompletionsRequest["reasoning_effort"]>,
"none" | "minimal" | "low" | "medium" | "high" | "max" | "xhigh"
>;
type ReasoningRequestFields = Pick<
OpenAIChatCompletionsRequest,
"enable_thinking" | "reasoning_effort" | "thinking"
>;
const fallbackExternalEffort = (externalReasoningCaps
.reasoningEffortLevels[0] ?? "low") as RequestReasoningEffort;
const selectedExternalEffort: RequestReasoningEffort =
clampReasoningEffortToLevels(
reasoningEffort,
externalReasoningCaps.reasoningEffortLevels,
) as RequestReasoningEffort;
// Clamp stale values to supported levels (gpt-oss low|medium|high, GLM high|max).
const localReasoningEffort = clampReasoningEffortToLevels(
reasoningEffort,
reasoningEffortLevels,
);
const externalReasoningFields: ReasoningRequestFields =
externalProvider?.providerType === "custom" &&
externalProvider.apiType !== "responses"
? customReasoningRequestFields(
externalProvider.reasoningConfig,
externalReasoningEnabled,
selectedExternalEffort,
)
: externalReasoningCaps.supportsReasoning
? externalReasoningCaps.reasoningStyle === "reasoning_effort"
? externalReasoningEnabled
? { reasoning_effort: selectedExternalEffort }
: externalReasoningCaps.supportsReasoningOff
? { reasoning_effort: "none" }
: { reasoning_effort: fallbackExternalEffort }
: {
thinking: {
type: externalReasoningEnabled ? "enabled" : "disabled",
},
}
: {};
const localReasoningFields: ReasoningRequestFields = supportsReasoning
? reasoningStyle === "enable_thinking_effort"
? // GLM-5.2-style gate plus level, e.g. high|max.
reasoningEnabled
? { enable_thinking: true, reasoning_effort: localReasoningEffort }
: { enable_thinking: false }
: reasoningStyle === "reasoning_effort"
? reasoningEnabled
? { reasoning_effort: localReasoningEffort }
: {}
: { thinking: { type: reasoningEnabled ? "enabled" : "disabled" } }
: {};
const contextReasoningKey = isExternalRequest
? undefined
: contextUsageReasoningKey({
...localReasoningFields,
...(runtime.supportsPreserveThinking
? { preserve_thinking: runtime.preserveThinking }
: {}),
});
// Decided before the continuation yield below, which an abort during load saves as is.
// A carried thought is reasoning whatever this request's thinking setting says.
setParseThink(
isExternalRequest
? requestParsesThinkTags({
...externalReasoningFields,
provider_type: externalProvider?.providerType,
external_model: externalSelection?.modelId,
})
: reasoningAlwaysOn ||
Boolean(resumedThought) ||
requestParsesThinkTags(localReasoningFields),
);
// Yielded before the request starts: an abort during load skips the partial-content yield below.
if (continuation) {
yield {
content: buildAssistantContent(cumulativeText),
metadata: { custom: liveCustom() },
};
}
const parseToolProvenance = (
value: unknown,
): ToolCallProvenance | undefined => {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
return { ...(value as Record<string, unknown>) } as ToolCallProvenance;
};
const mergeToolProvenance = (
existing: ToolCallProvenance | undefined,
incoming: ToolCallProvenance | undefined,
): ToolCallProvenance | undefined => {
if (!incoming) return existing;
if (!existing) return incoming;
const merged: ToolCallProvenance = { ...existing, ...incoming };
for (const key of [
"healed",
"forced",
"provisional",
"duplicate",
] as const) {
if (existing[key] === true || incoming[key] === true) {
merged[key] = true;
}
}
return merged;
};
const closeReasoningContent = () => {
if (reasoningContentOpen) {
appendCumulative("</think>");
reasoningContentOpen = false;
}
reasoningDurationTracker.finishGroup();
};
// Anthropic document_citations, converted at end-of-stream so inline [N] markers have matching entries.
const documentCitationParts: Array<{
type: "source";
sourceType: "url";
id: string;
url: string;
title: string;
metadata?: { description: string };
}> = [];
// Stamped as custom.anthropicRefusal to drive the history prune.
let anthropicRefusalSeen = false;
let serverMetadata: {
usage?: ServerUsage;
timings?: ServerTimings;
} | null = null;
// Gate on the captured checkpoint and thread so a late write from provider A cannot repaint
// the bar after a switch to B. A first turn is adopted onto an id mid-run and autosave moves
// activeThreadId with it, so read the adopted key, or the bar stays blank for life.
const publishContextUsage = (
reported: NonNullable<
ReturnType<typeof useChatRuntimeStore.getState>["contextUsage"]
>,
) => {
const usage = {
...reported,
modelId: params.checkpoint,
...(contextReasoningKey ? { reasoningKey: contextReasoningKey } : {}),
};
const usageKey = liveThreadKey(serverCancel);
const usageThreadKey = usageKey === "__default" ? null : usageKey;
const store = useChatRuntimeStore.getState();
// File it under this run's own thread even when the gate blocks the visible write, so
// switching back re-applies it.
if (usageThreadKey !== null) {
store.setThreadContextUsage(usageThreadKey, usage);
}
// The realtime meter shows this exact count from here until the next pass starts.
store.settleLiveGeneration(usageKey);
if (
store.activeThreadId === (usageThreadKey ?? activeThreadIdAtRunStart) &&
store.params.checkpoint === params.checkpoint
) {
store.setContextUsage(usage);
}
};
// llama.cpp output count, including reasoning.
let windowCount: number | null = null;
let liveServerTimings: ServerTimings | undefined;
passSoFar = () => {
const now = Date.now();
return {
totalStreamTime: now - streamStartTime,
totalChunks,
toolCallCount: toolCallParts.length,
...(windowCount !== null
? { tokenCount: windowCount }
: cumulativeText.length > continuationPartial.length
? { tokenCount: Math.max(1, Math.round((cumulativeText.length - continuationPartial.length) / 4)) }
: {}),
...(firstTokenTime !== undefined ? { firstTokenTime } : {}),
startedAt: streamStartTime,
durationMs: now - streamStartTime,
...(liveServerTimings ? { serverTimings: { ...liveServerTimings } as unknown as Record<string, number> } : {}),
};
};
// Colab-style proxies can swallow fetch aborts, so also POST /inference/cancel explicitly.
const onAbortCancel = () => {
// detach=true means unmount; only an explicit Stop cancels the backend run.
if ((runSignal.reason as { detach?: boolean } | undefined)?.detach) {
return;
}
generationStopRequested = true;
// Override the provisional reason so a deliberate Stop persists as cancelled.
incompleteReason = "cancelled";
const stopPlan = chatGenerationStopPlan(
generationDecision,
generationRunId,
);
if (stopPlan.cancelRunId) {
void cancelChatGenerationRun(stopPlan.cancelRunId).catch(() => {});
}
if (!stopPlan.postLegacyCancel) return;
const body: Record<string, string> = { cancel_id: cancelId };
if (sandboxSessionId) body.session_id = sandboxSessionId;
// Plain fetch, not authFetch: authFetch redirects to login on 401, kicking the user out mid-stop.
const token = getAuthToken();
// apiUrl: in Tauri the webview origin is not the backend; browser builds get the empty base.
void fetch(apiUrl("/api/inference/cancel"), {
method: "POST",
headers: {
"Content-Type": "application/json",
...(token ? { Authorization: `Bearer ${token}` } : {}),
},
body: JSON.stringify(body),
keepalive: true,
}).catch(() => {});
};
// Reaches non-visible threads; for external providers the abort is the stop (no cancel_id).
runtime.registerThreadServerCancel(threadKey, serverCancel);
const recoveryState = useChatRuntimeStore.getState();
const minPRecoveryGuard =
externalProvider?.providerType === "vllm" &&
recoveryState.activeThreadId === (resolvedThreadId ?? null) &&
recoveryState.params.checkpoint === params.checkpoint &&
recoveryState.params.minP === params.minP &&
recoveryState.params.minPMode === params.minPMode
? createMinPRecoveryGuard(
() => {
const state = useChatRuntimeStore.getState();
const connections = useExternalProvidersStore.getState();
return [
state.activeThreadId,
state.params,
connections.providers,
connections.connectionsEnabled,
];
},
(check) => {
const stopSettings = useChatRuntimeStore.subscribe(check);
const stopConnections = useExternalProvidersStore.subscribe(check);
return () => {
stopSettings();
stopConnections();
};
},
)
: null;
let keepMinPRecovery = false;
try {
if (runSignal.aborted) {
onAbortCancel();
} else {
runSignal.addEventListener("abort", onAbortCancel, { once: true });
}
const { supportsPreserveThinking, preserveThinking } = runtime;
const buildResponseDetails = (
finishedAt: number,
): ResponseDetailsMetadata => ({
modelId: params.checkpoint,
modelLabel:
(isExternalRequest || responseModelId !== params.checkpoint
? responseModelId
: selectedModelSummary?.name || responseModelId) ||
params.checkpoint ||
"Unknown model",
responseModelId:
responseModelId || externalSelection?.modelId || params.checkpoint,
...(externalProvider?.id ? { providerId: externalProvider.id } : {}),
providerName:
externalProvider?.name ??
(isExternalRequest ? "External provider" : "Local model"),
providerType: externalProvider?.providerType ?? "local",
startedAt: priorTiming?.startedAt ?? streamStartTime,
finishedAt,
durationMs:
(priorTiming ? (priorTiming.durationMs ?? priorTiming.totalStreamTime) : 0) +
finishedAt -
streamStartTime,
...(sandboxSessionId ? { sessionId: sandboxSessionId } : {}),
cancelId,
toolCalls: Array.from(
new Set(
toolCallParts
.map((part) => part.toolName)
.filter(
(toolName): toolName is string =>
typeof toolName === "string" && toolName.length > 0,
),
),
),
tools: {
search:
webSearchEnabledForThisTurn ||
(!isExternalRequest && supportsTools && toolsEnabled),
fetch: webFetchEnabledForThisTurn,
code:
hostedCodeToolsForThisTurn.length > 0 ||
(supportsStudioToolsForThisTurn &&
studioLocalCodeTools.length > 0),
images:
imageGenerationEnabledForThisTurn || localMediaTools.length > 0,
mcp: supportsStudioToolsForThisTurn && mcpEnabledForChat,
docs:
supportsStudioToolsForThisTurn &&
(ragEnabled || projectRagEnabled),
artifacts: false,
confirmToolCalls,
bypassPermissions,
permissionMode,
},
});
const externalCapabilities = getProviderCapabilities(
externalProvider?.providerType,
externalProvider?.apiType,
externalSelection?.modelId,
externalProvider?.baseUrl,
);
const buildRequestPayload = async (
forceRefreshPublicKey = false,
): Promise<OpenAIChatCompletionsRequest> => {
if (supportsStudioToolsForThisTurn) {
// include every user turn because follow-ups still need skills mentioned earlier.
await settleSkillsForText(userTexts(outboundMessages).join("\n"));
}
const skillTools = skillToolNames(
skillsForTools(),
codeToolsEnabled,
userTexts(outboundMessages),
);
if (externalSelection && externalProvider) {
// Per-thread container reuse; empty falls back to container_auto. Anthropic has its own key.
let openaiCodeExecContainerId: string | null = null;
let anthropicCodeExecContainerId: string | null = null;
if (codeExecEnabledForThisTurn && resolvedThreadId) {
try {
// Read at build time; container selection can change while waiting for model load.
const thread = await getStoredChatThread(resolvedThreadId);
openaiCodeExecContainerId =
thread?.openaiCodeExecContainerId ?? null;
anthropicCodeExecContainerId =
thread?.anthropicCodeExecContainerId ?? null;
} catch {
openaiCodeExecContainerId = null;
anthropicCodeExecContainerId = null;
}
// Stale ids fall through to lazy-create; a failed list call relies on backend retry.
let activeContainerIds: Set<string> | null = null;
if (externalProvider.providerType === "openai") {
try {
const list = await listOpenAIContainers({
providerId: externalProvider.id,
apiKey: externalApiKey,
baseUrl: externalProvider.baseUrl || null,
});
activeContainerIds = new Set(list.map((c) => c.id));
} catch {
activeContainerIds = null;
}
if (
activeContainerIds &&
openaiCodeExecContainerId &&
!activeContainerIds.has(openaiCodeExecContainerId)
) {
void updateStoredChatThreadEventually(resolvedThreadId, {
openaiCodeExecContainerId: null,
}).catch(() => {});
openaiCodeExecContainerId = null;
}
}
// Reuse the newest container from any other thread unless opted out in the picker.
if (
!openaiCodeExecContainerId &&
externalProvider.providerType === "openai"
) {
try {
const others = await listStoredChatThreads({
includeArchived: true,
});
for (const t of others) {
if (t.id === resolvedThreadId) continue;
if (!t.openaiCodeExecContainerId) continue;
// Skip ids not in the active set; null the source thread so the next pass does not re-pick it.
if (
activeContainerIds &&
!activeContainerIds.has(t.openaiCodeExecContainerId)
) {
void updateStoredChatThreadEventually(t.id, {
openaiCodeExecContainerId: null,
}).catch(() => {});
continue;
}
openaiCodeExecContainerId = t.openaiCodeExecContainerId;
void updateStoredChatThreadEventually(resolvedThreadId, {
openaiCodeExecContainerId,
}).catch(() => {});
break;
}
} catch {
/* fall through to lazy-create below */
}
}
// Pre-create rather than container_auto for a named, TTL-configured picker entry.
if (
!openaiCodeExecContainerId &&
externalProvider.providerType === "openai"
) {
const ttl = externalProvider.openaiContainerTtlMinutes;
const ttlToUse = typeof ttl === "number" && ttl >= 1 ? ttl : 20;
try {
const created = await createOpenAIContainer(
{
providerId: externalProvider.id,
apiKey: externalApiKey,
baseUrl: externalProvider.baseUrl || null,
},
{
name: pickFriendlyContainerName(),
ttlMinutes: ttlToUse,
},
);
openaiCodeExecContainerId = created.id;
void updateStoredChatThreadEventually(resolvedThreadId, {
openaiCodeExecContainerId: created.id,
}).catch(() => {});
} catch {
// Fall back to the backend's container_auto path on failure; the next turn can retry.
openaiCodeExecContainerId = null;
}
}
}
return {
model: externalSelection.modelId,
messages: outboundMessages,
...(providerSupportsPreserveThinking(externalProvider?.providerType)
? { preserve_thinking: runtime.preserveThinking }
: {}),
stream: true,
stream_options: { include_usage: true },
video_base64: findLatestUserVideoBase64(currentTurnMessages),
// Never forwarded upstream; the trailing assistant turn is what requests continuation.
...(continuation ? { continue_final_message: true } : {}),
// Reasoning-class models (OpenAI gpt-5.x / o3) reject temperature and top_p; forward only when supported.
...(externalCapabilities?.temperature !== false
? { temperature: params.temperature }
: {}),
...(externalCapabilities?.topP !== false && params.topP < 1
? { top_p: params.topP }
: {}),
// Floor at the provider's documented min (Kimi thinking needs >=16k); clamp at the per-model max.
max_tokens: Math.min(
Math.max(
params.maxTokens,
getExternalMinOutputTokens(externalProvider?.providerType),
),
getExternalMaxOutputTokens(
externalProvider?.providerType,
externalSelection?.modelId,
externalProvider?.maxOutputTokens,
),
),
...(externalCapabilities?.topK ? { top_k: params.topK } : {}),
...(externalCapabilities?.minP
? minPSamplingPayload(externalProvider?.providerType, params)
: {}),
...(externalCapabilities?.repetitionPenalty
? { repetition_penalty: params.repetitionPenalty }
: {}),
...(externalCapabilities?.presencePenalty
? { presence_penalty: params.presencePenalty }
: {}),
// studioLocalCodeTools, not codeToolsEnabled: a Tools pill that resolved to the provider's
// sandbox is a hosted request and belongs below, where this body would 400 on permission_mode.
...(supportsStudioToolsForThisTurn &&
(toolsEnabled ||
studioLocalCodeTools.length > 0 ||
mcpEnabledForChat ||
ragEnabled ||
projectRagEnabled ||
skillTools.length > 0 ||
browserTools.length > 0 ||
localMediaTools.length > 0 ||
// Armed research needs Studio's loop: deep_research is appended past every tool filter, but
// only for a request that asked for the loop at all.
deepResearchArmed)
? {
enable_tools: true,
enabled_tools: [
...(ragEnabled || projectRagEnabled
? ["search_knowledge_base"]
: []),
...(toolsEnabled ? ["web_search"] : []),
...skillTools,
...studioLocalCodeTools,
...browserTools,
...localMediaTools,
// Hosted tools with no local stand-in; their pills stay lit regardless, so listing only local
// names dropped Images/Fetch whenever another tool selected this branch. Search is excluded
// (Unsloth runs it above); Code rides along only when it resolved to the provider's sandbox.
...(imageGenerationEnabledForThisTurn
? ["image_generation"]
: []),
...(webFetchEnabledForThisTurn ? ["web_fetch"] : []),
...hostedCodeToolsForThisTurn,
],
mcp_enabled: mcpEnabledForChat,
permission_mode: permissionMode,
sandbox_level: runtime.sandboxLevel,
...(permissionMode === "auto" || permissionMode === "off"
? {}
: { confirm_tool_calls: permissionMode === "ask" }),
bypass_permissions: bypassPermissions,
max_tool_calls_per_message: runtime.maxToolCallsPerMessage,
tool_call_timeout:
runtime.toolCallTimeout >= 9999
? 9999
: runtime.toolCallTimeout * 60,
// Self-hosted models often emit calls as text, so heal here too instead of the process default.
auto_heal_tool_calls: runtime.autoHealToolCalls,
// false, not omitted: omission follows UNSLOTH_TOOL_CALL_NUDGE, which the
// launchers set to 1 when unset, so this loop would keep nudging (#9686, #9125).
// Browser use is the exception: a step announced and not taken ends the task.
nudge_tool_calls: browserTools.length > 0 ? runtime.nudgeToolCalls : false,
deduplicate_tool_calls: runtime.deduplicateToolCalls,
// This branch runs the tools here, so say so by name: enabled_tools ["web_search"] is
// byte-identical to an older bundle's hosted search.
run_tools_locally: true,
...(sandboxSessionId
? { session_id: sandboxSessionId }
: {}),
...userMessageIdField,
...(sandboxAttachments.length > 0
? { sandbox_attachments: sandboxAttachments }
: {}),
...(ragEnabled || projectRagEnabled
? {
rag_scope: {
...(ragEnabled && ragSource.type === "kb"
? { kb_id: ragSource.kbId }
: ragEnabled && resolvedThreadId
? { thread_id: resolvedThreadId }
: {}),
...(projectRagEnabled && ragProjectId
? { project_id: ragProjectId }
: {}),
default_top_k: ragTopK,
mode: ragMode,
autoinject: resolveAutoInject(
ragAutoInject,
params.checkpoint,
),
autoinject_min_score: ragAutoInjectMinScore,
...(ragAutoInject === "off"
? { whole_doc: false }
: {}),
context_length: ragScopeContextLength({
isExternalRequest,
loadedContextLength: runtime.loadedContextLength,
maxSeqLength: params.maxSeqLength,
}),
},
}
: {}),
}
: webSearchEnabledForThisTurn ||
webFetchEnabledForThisTurn ||
codeExecEnabledForThisTurn ||
imageGenerationEnabledForThisTurn
? {
enable_tools: true,
enabled_tools: [
...(webSearchEnabledForThisTurn ? ["web_search"] : []),
...(webFetchEnabledForThisTurn ? ["web_fetch"] : []),
...(codeExecEnabledForThisTurn
? ["code_execution"]
: []),
...(imageGenerationEnabledForThisTurn
? ["image_generation"]
: []),
],
}
// Explicit false: omitting it uses the server's tools-on default and bills provider tools.
:
{ enable_tools: false }),
// Needed here too, or arming research is a no-op for providers running Studio tools.
...(deepResearchArmed ? { deep_research_armed: true } : {}),
...(await buildExternalRoutingFields(
{
provider: externalProvider,
modelId: externalModelId,
apiKey: externalApiKey,
},
{ forceRefreshPublicKey },
)),
...apiCompactionRequestFields({
autoCompactEnabled: runtime.autoCompactEnabled,
contextLength: resolveModelCatalogEntry(
externalProvider.providerType,
externalModelId,
)?.contextLength,
}),
// On every external request: OpenRouter's cache routing, and the prompt date unless Claude caches.
...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}),
...(openaiCodeExecContainerId
? {
openai_code_exec_container_id: openaiCodeExecContainerId,
}
: {}),
...(anthropicCodeExecContainerId
? {
anthropic_code_exec_container_id:
anthropicCodeExecContainerId,
}
: {}),
...(supportsProviderPromptCaching(externalProvider.providerType)
? {
enable_prompt_caching:
externalProvider.enablePromptCaching ?? true,
}
: {}),
// Anthropic prompt-cache TTL; unknown values no-op on backend.
...(supportsProviderPromptCacheTtl(
externalProvider.providerType,
) &&
(externalProvider.enablePromptCaching ?? true) &&
isPromptCacheTtl(externalProvider.promptCacheTtl)
? { prompt_cache_ttl: externalProvider.promptCacheTtl }
: {}),
// Anthropic fast mode (Opus 4.6 / 4.7 only); the backend drops it on unsupported models as a backstop.
...(params.fastMode &&
providerSupportsFastMode(
externalProvider.providerType,
externalSelection.modelId,
)
? { fast_mode: true }
: {}),
...externalReasoningFields,
};
}
const turnAudioBase64 = findLatestUserAudioBase64(
currentTurnMessages,
!queuedRunSettings && !continuation,
);
return {
model: params.checkpoint,
messages: outboundMessages,
stream: true,
...(continuation ? { continue_final_message: true } : {}),
...studioToolHistoryRequestFieldsAfterReplay(
survivingMessages as unknown as ToolHistoryMessage[],
),
// Opt into the trailing usage chunk so the context bar and tok/s populate (backend gates it).
stream_options: { include_usage: true },
...ggufCompactionRequestFields({
isGguf: isGgufForCompaction,
isMlx: isMlxForCompaction,
isManagedEngine: isManagedEngineForCompaction,
autoCompactEnabled: runtime.autoCompactEnabled,
compactNow,
}),
temperature: params.temperature,
top_p: params.topP,
max_tokens: params.maxTokens,
top_k: params.topK,
min_p: params.minP,
repetition_penalty: params.repetitionPenalty,
presence_penalty: params.presencePenalty,
// Omit when unset (server draws a seed) or unread, so a stale GGUF pin is never sent.
...(params.seed == null ||
!modelReadsSamplingSeed(activeModel, runtime.loadedEngine)
? {}
: { seed: params.seed }),
// Turn-scoped: a stale screenshot here made the backend refuse every later text turn's durable run.
image_base64: findLatestUserImageBase64(currentTurnMessages),
audio_base64: turnAudioBase64,
...(activeModel?.speaks && turnAudioBase64
? { modalities: ["text", "audio"] }
: {}),
extra_audio_base64:
findLatestUserExtraAudioBase64(currentTurnMessages),
video_base64: findLatestUserVideoBase64(currentTurnMessages),
cancel_id: cancelId,
...(sandboxSessionId ? { session_id: sandboxSessionId } : {}),
...(sandboxAttachments.length > 0
? { sandbox_attachments: sandboxAttachments }
: {}),
...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}),
...userMessageIdField,
...(useAdapter === undefined ? {} : { use_adapter: useAdapter }),
...localReasoningFields,
...(supportsPreserveThinking
? { preserve_thinking: preserveThinking }
: {}),
// Sent for every local chat, since `unsloth run --enable-tools` can open the tool loop with
// no pill lit. "auto" OMITS confirm_tool_calls (an explicit true would force a stream and
// defeat the safe-only exception); "off" omits it too, since an explicit false opts out of
// its risky-call prompt without an OS sandbox; "ask" sends true, full sends false.
// sandbox_level "low" runs Python/Terminal on software safeguards only; Full access overrides it.
permission_mode: permissionMode,
sandbox_level: runtime.sandboxLevel,
...(permissionMode === "auto" || permissionMode === "off"
? {}
: { confirm_tool_calls: permissionMode === "ask" }),
bypass_permissions: bypassPermissions,
...(deepResearchArmed ? { deep_research_armed: true } : {}),
...(supportsTools &&
(toolsEnabled ||
codeToolsEnabled ||
mcpEnabledForChat ||
ragEnabled ||
projectRagEnabled ||
skillTools.length > 0 ||
browserTools.length > 0 ||
localMediaTools.length > 0 ||
deepResearchArmed)
? {
enable_tools: true,
enabled_tools: [
// First so retrieval is the primary tool when Docs is on.
...(ragEnabled || projectRagEnabled
? ["search_knowledge_base"]
: []),
...(toolsEnabled ? ["web_search"] : []),
...skillTools,
...(codeToolsEnabled
? ["python", "terminal", "edit_file", "view_image"]
: []),
...browserTools,
...localMediaTools,
],
mcp_enabled: mcpEnabledForChat,
...(ragEnabled || projectRagEnabled
? {
rag_scope: {
...(ragEnabled && ragSource.type === "kb"
? { kb_id: ragSource.kbId }
: ragEnabled && resolvedThreadId
? { thread_id: resolvedThreadId }
: {}),
...(projectRagEnabled && ragProjectId
? { project_id: ragProjectId }
: {}),
default_top_k: ragTopK,
mode: ragMode,
autoinject: resolveAutoInject(
ragAutoInject,
params.checkpoint,
),
autoinject_min_score: ragAutoInjectMinScore,
...(ragAutoInject === "off"
? { whole_doc: false }
: {}),
context_length: ragScopeContextLength({
isExternalRequest,
loadedContextLength: runtime.loadedContextLength,
maxSeqLength: params.maxSeqLength,
}),
},
}
: {}),
auto_heal_tool_calls: runtime.autoHealToolCalls,
nudge_tool_calls: runtime.nudgeToolCalls,
deduplicate_tool_calls: runtime.deduplicateToolCalls,
max_tool_calls_per_message: runtime.maxToolCallsPerMessage,
tool_call_timeout: (() => {
const mins = runtime.toolCallTimeout;
return mins >= 9999 ? 9999 : mins * 60;
})(),
}
: // Explicit false keeps UI-off tools disabled; --enable-tools still overrides it
// and sees no exhaustive enabled_tools list, so it can supply the default catalog.
{ enable_tools: false }),
};
};
let retriedWithRefreshedKey = false;
while (true) {
try {
let requestPayload: OpenAIChatCompletionsRequest;
try {
requestPayload = await buildRequestPayload(
retriedWithRefreshedKey,
);
if (mcpImage) requestPayload = { ...requestPayload, mcp_image: mcpImage };
} catch (error) {
clearSelectedImageEditReference();
throw error;
}
clearSelectedImageEditReference();
requestedMaxTokens = requestPayload.max_tokens;
// External requests ignore local settings; maxSeqLength 0 means unknown, matching RAG.
servedContextLength = isExternalRequest
? externalStopWindow(
externalProvider?.providerType,
externalProvider?.baseUrl,
)
: (runtime.loadedCustomContextLength ??
runtime.loadedContextLength ??
(params.maxSeqLength || null));
await ThreadAutosaveHandle.awaitFirstSave(resolvedThreadId);
if (generationDecision === "pending") {
// Keyed on `enabled_tools`, never `requestPayload.tools`: see durable-gate.ts.
if (turnRequiresLegacyStream(requestPayload)) {
// Only a tool chain the BROWSER must execute needs the live tab; everything else is durable.
generationDecision = "legacy";
} else {
const admission = explicitStopSignal(runSignal);
const supported = await supportsChatGenerationRuns(
resolvedThreadId!,
admission.signal,
).finally(admission.dispose);
if (!supported) {
generationDecision = "legacy";
} else {
generationDecision = "durable";
// Claim before admission: the run is visible through /active as soon as the POST lands.
claimLiveGenerationRun(cancelId, resolvedThreadId!, {
provisional: true,
});
try {
generationRun = await createChatGenerationRunUntilAbort(
{
runId: cancelId,
threadId: resolvedThreadId!,
userMessageId: generationUserMessage!.id,
assistantMessageId: unstable_assistantMessageId!,
requestPayload,
...tabCloseRunFields(
runtime.keepRunningWhenTabClosed,
),
},
runSignal,
);
} catch (error) {
if (!isLegacyFallbackChatGenerationAdmissionError(error)) {
throw error;
}
// Durable recovery cannot replay server-side tool events yet, so use the legacy stream.
generationDecision = "legacy";
// Release now: a lingering claim would let the cap fire on a checkpoint-only stream.
releaseLiveGenerationRun(cancelId);
}
if (!generationRun) {
// Null on a Stop race or an unparsable 2xx body; a bare return settles both as complete.
if (generationDecision === "durable") {
if (runSignal.aborted) {
throw runSignal.reason ??
new DOMException("Aborted", "AbortError");
}
throw new Error(
"The server accepted the request without starting a generation run",
);
}
} else {
generationRunId = generationRun.id;
// Re-claimed in case the server echoes a different id; both are released in the finally.
claimLiveGenerationRun(generationRunId, resolvedThreadId!);
generationStatus = generationRun.status;
if (generationStopRequested) {
void cancelChatGenerationRun(generationRun.id).catch(
() => {},
);
}
}
}
}
}
const durableStream = async function* () {
try {
for await (const update of followChatGenerationRun(
generationRunId!,
{
initialRun: generationRun!,
replayFrom: 0,
signal: runSignal,
},
)) {
generationRun = update.run;
generationStatus = update.run.status;
if (update.event) {
generationSeq = Math.max(generationSeq, update.event.seq);
if (update.event.type === "chunk") {
const chunk = update.event.payload as OpenAIChatChunk;
if (generationChunkCountsTowardTiming(chunk)) {
generationChunkCount += 1;
if (generationChunkHasSubstantiveDelta(chunk)) {
generationFirstChunkAt ??= update.event.createdAt;
}
}
yield chunk;
}
}
}
} catch (error) {
if (!(error instanceof ChatGenerationStalledError)) throw error;
// End rather than rethrow to keep replayed output; a stalled run is non-terminal.
generationStalled = true;
// Without `incomplete`, assistant-ui treats the partial reply as finished with no Continue.
incompleteReason = "interrupted";
}
if (generationStatus === "failed") {
throw new ChatGenerationTerminalError(
"failed",
generationRun?.error ||
"The Unsloth backend restarted during generation.",
);
}
if (generationStatus === "cancelled" && !runSignal.aborted) {
throw new ChatGenerationTerminalError(
"cancelled",
"Generation was cancelled.",
);
}
};
// The window lets a length-stop tell user Max Tokens from the backend's stand-in for Max.
const sourceStream =
generationDecision === "durable"
? durableStream()
: streamChatCompletions(
requestPayload,
runSignal,
servedContextLength,
);
// Per run, not per module: two turns must not share a cycle.
const canPublish = createStreamPublishGate();
const stream = withTrailingPublish(sourceStream, canPublish);
for await (const chunk of stream) {
if (chunk === TRAILING_PUBLISH) {
if (canPublish(streamedChars)) {
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
continue;
}
const chunkModel = (chunk as { model?: unknown }).model;
if (typeof chunkModel === "string" && chunkModel.length > 0) {
responseModelId = chunkModel;
}
const toolStatusText = (
chunk as unknown as { _toolStatus?: string }
)._toolStatus;
if (toolStatusText !== undefined) {
// Only-disabled rounds emit no card and upstream [DONE] has no finish_reason.
if (!toolStatusText) {
endProviderTurn();
}
runtime.setToolStatus(
liveThreadKey(serverCancel),
toolStatusText || null,
serverCancel,
);
continue;
}
if (chunk.quote_cut) {
quoteCut = true;
continue;
}
if (chunk.context_truncated) {
runtime.setLiveCompacting(liveThreadKey(serverCancel), false);
contextTruncation = mergeContextTruncation(
contextTruncation,
chunk.context_truncated,
);
compactionMarkers = appendCompactionMarker(
compactionMarkers,
chunk.context_truncated,
compactionAnchorToolCallId(
toolCallParts,
chunk.context_truncated,
),
);
const activeThreadId = useChatRuntimeStore.getState().activeThreadId;
// Stay silent when the fit returned the ORIGINAL messages; not `fits`, which is also false for
// a shortened prompt.
const reallyCompacted = promptWasShortened(chunk.context_truncated);
if (
reallyCompacted &&
resolvedThreadId &&
activeThreadId === resolvedThreadId &&
!rollingContextNoticeThreads.has(resolvedThreadId)
) {
rollingContextNoticeThreads.add(resolvedThreadId);
// Once per thread per page load; the durable record is the notice on the turn that compacted.
const archived = contextTruncation?.archived_messages ?? 0;
toast.info("This conversation was compacted", {
description: archived
? "It got long, so older turns were removed from the model's " +
"context. They are saved and searchable, and relevant parts are " +
"brought back automatically."
: contextTruncation?.summarized
? "The full conversation is still visible and saved. " +
"The provider summarized older turns so the chat can continue."
: "The full conversation is still visible and saved. " +
"Unsloth removed complete older turns from this request so the chat can continue.",
duration: 8000,
});
}
continue;
}
// Guard the type so a malformed or proxied chunk (string/null/NaN duration) cannot make NaN.
const reasoningMs = (
chunk as { _reasoningDurationMs?: number } | null | undefined
)?._reasoningDurationMs;
if (reasoningDurationTracker.recordServerDuration(reasoningMs)) {
continue;
}
// A tool loop handing off to a tool: show its context now rather than at turn end, which
// still writes the final figure.
const liveContext = (
chunk as unknown as {
_contextUsage?: {
prompt_tokens?: unknown;
completion_tokens?: unknown;
context_tokens?: unknown;
cached_tokens?: unknown;
};
}
)._contextUsage;
if (liveContext !== undefined) {
const { prompt_tokens, completion_tokens, context_tokens, cached_tokens } =
liveContext ?? {};
if (
typeof prompt_tokens === "number" &&
typeof completion_tokens === "number" &&
typeof context_tokens === "number"
) {
publishContextUsage({
promptTokens: prompt_tokens,
completionTokens: completion_tokens,
totalTokens: context_tokens,
contextTokens: context_tokens,
cachedTokens: typeof cached_tokens === "number" ? cached_tokens : 0,
});
}
continue;
}
// Transient like the diffusion canvas: never transcript text, never a first token.
const generationProgress = (
chunk as unknown as { _generationProgress?: unknown }
)._generationProgress;
if (generationProgress !== undefined) {
const update = readGenerationProgress(generationProgress);
if (update?.phase === "prefill") {
runtime.setLivePrefill(liveThreadKey(serverCancel), update.prefill);
} else if (update?.phase === "decode") {
runtime.setLiveDecode(liveThreadKey(serverCancel), update.decode);
} else if (update?.phase === "compacting") {
runtime.setLiveCompacting(liveThreadKey(serverCancel), true);
}
continue;
}
// Diffusion frame: a transient canvas snapshot routed to the transient store and skipped; it
// has no assistant text, so it never enters the transcript.
const diffusionFrame = (
chunk as unknown as {
_diffusionFrame?: {
block?: number;
step?: number;
total?: number;
text?: string;
};
}
)._diffusionFrame;
if (diffusionFrame !== undefined) {
// Keyed by thread so a background run's frames stay out of the visible chat.
runtime.setActiveDiffusionCanvas(liveThreadKey(serverCancel), {
block: diffusionFrame.block ?? 0,
step: diffusionFrame.step ?? 0,
total: diffusionFrame.total ?? 0,
text: diffusionFrame.text ?? "",
});
continue;
}
// tool_start adds a part (renders "running"), tool_end sets its result (transitions to "complete").
const toolEvent = (
chunk as unknown as { _toolEvent?: Record<string, unknown> }
)._toolEvent;
if (toolEvent !== undefined) {
// Unsloth tool events end the turn; finish_reason is not enough since hosted
// tools run inside the turn.
if (!chunk.choices && toolEvent.tool_name !== "studio_load_skill") {
endProviderTurn();
}
// Deep Research is an ordinary tool to every loop that runs it, so the handoff is read off the
// shared events. An ungated pair is not rendered: the research card is the reply.
if (
toolEvent.tool_name === "deep_research" &&
readDeepResearchToolEvent(deepResearchHandoff, toolEvent)
) {
if (deepResearchHandoff.question !== null) break;
continue;
}
// Answered in another tab or device: clear the prompt now, not at tool_end.
if (toolEvent.type === "tool_approved") {
const approvedBackendId =
(toolEvent.tool_call_id as string) || "";
const approvedId = toolEvent.approval_id;
const confirmationId =
toolConfirmationIdsByBackendId.get(approvedBackendId) ??
(typeof approvedId === "string" && approvedId
? toolCallParts.find(
(p) => p.toolApprovalId === approvedId,
)?.toolCallId
: undefined);
if (confirmationId) {
useChatRuntimeStore
.getState()
.clearToolConfirmation(confirmationId);
}
continue;
}
// Persist container_id onto the thread (OpenAI / Anthropic).
if (toolEvent.type === "container_ready") {
const newContainerId = toolEvent.container_id as
| string
| undefined;
if (newContainerId && resolvedThreadId) {
const field =
externalProvider?.providerType === "anthropic"
? "anthropicCodeExecContainerId"
: "openaiCodeExecContainerId";
void updateStoredChatThreadEventually(resolvedThreadId, {
[field]: newContainerId,
}).catch(() => {});
}
continue;
}
if (toolEvent.type === "compaction_block") {
const nextCompaction = providerCompactionPart(toolEvent);
if (nextCompaction) {
providerCompaction = nextCompaction;
providerCompactionProviderType =
toExternalBackendProviderType(
externalProvider?.providerType,
);
providerCompactionModelId = externalSelection?.modelId;
providerCompactionOriginConnectionKey =
providerCompactionTargetConnectionKey;
// Compaction items arrive before the provider content they introduce. Remember
// how much of this stored run preceded it so replay does not put a later marker
// ahead of earlier Studio tool rounds that the marker already summarizes.
providerCompactionAfterToolCalls = toolCallParts.filter(
(part) =>
part.toolName !== "studio_load_skill" &&
toolCallPartSurvivesOpenAIReplay(part),
).length;
}
continue;
}
if (toolEvent.type === "document_citations") {
// Convert citations_delta footnotes into Sources entries matching the inline [N] markers.
const cits = toolEvent.citations;
if (Array.isArray(cits)) {
cits.forEach((entry, idx) => {
if (!entry || typeof entry !== "object") return;
const part = documentCitationToSource(
entry as Record<string, unknown>,
idx,
);
if (
part &&
!documentCitationParts.some((p) => p.id === part.id)
) {
documentCitationParts.push(part);
}
});
}
continue;
}
if (toolEvent.type === "container_invalidated") {
if (resolvedThreadId) {
const field =
externalProvider?.providerType === "anthropic"
? "anthropicCodeExecContainerId"
: "openaiCodeExecContainerId";
void updateStoredChatThreadEventually(resolvedThreadId, {
[field]: null,
}).catch(() => {});
}
continue;
}
if (toolEvent.type === "anthropic_refusal") {
// Latch the backend refusal signal so final metadata can drive the prune.
anthropicRefusalSeen = true;
continue;
}
if (toolEvent.type === "context_window_exceeded") {
contextWindowExceeded = true;
// assistant-ui saves the last STREAMED yield, so publish why the turn ended now.
canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
continue;
}
if (toolEvent.type === "tool_output" && toolEvent.browser_action) {
// A browser tool asking this tab to act; it answers on its own request, so the stream never waits.
const request = toolEvent.browser_action;
const userText = userTexts(survivingMessages).join("\n");
void import("@/features/browser/agent/executor").then(
({ parseBrowserAction, runBrowserAction }) => {
const action = parseBrowserAction(request);
if (action) {
void runBrowserAction(action, {
runKey: cancelId,
chat: browserChat,
// Resolved per action: a first turn is re-keyed from "__default" to its id.
scope: liveThreadKey(serverCancel),
userText,
fullAccess: (bypassPermissions && runtime.chatFullAccessPreviousMode === null) || hasChatFullAccess(
useChatRuntimeStore.getState().chatFullAccessGrants, liveThreadKey(serverCancel), cancelId,
),
temporary: browserTemporary,
signal: runSignal,
});
}
},
// A stale chunk after an update: answer now rather than leave the tool waiting.
() => {
const raw = (request ?? {}) as Record<string, unknown>;
if (typeof raw.id !== "string") return;
void authFetch("/api/inference/browser-action-result", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
action_id: raw.id,
session_id: typeof raw.session_id === "string" ? raw.session_id : "",
result: { ok: false, error: "the browser panel could not load; ask the user to reload Unsloth." },
}),
}).catch(() => undefined);
},
);
continue;
}
if (toolEvent.type === "tool_model_note") {
const at = toolModelNoteTarget(
toolCallParts as Parameters<typeof toolModelNoteTarget>[0],
(toolEvent.tool_call_id as string) || "",
);
const suffix =
typeof toolEvent.suffix === "string" ? toolEvent.suffix : "";
if (at !== -1 && suffix) {
const existing = toolCallParts[at] as PositionedToolCallPart;
toolCallParts[at] = {
...existing,
provenance: withToolModelSuffix(
existing.provenance as Record<string, unknown> | undefined,
suffix,
),
} as PositionedToolCallPart;
// A stop before the next chunk saves only what was yielded.
canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
continue;
}
if (toolEvent.type === "tool_output") {
const backendToolCallId =
(toolEvent.tool_call_id as string) || "";
const liveId = resolveToolPartId(backendToolCallId);
const liveText =
typeof toolEvent.text === "string" ? toolEvent.text : "";
if (liveId && liveText) {
const liveKey = scopedToolOutputKey(liveId);
runToolLiveOutputKeys.add(liveKey);
useChatRuntimeStore
.getState()
.appendToolLiveOutput(liveKey, liveText);
}
continue;
}
if (toolEvent.type === "tool_args") {
// Arguments still streaming: show a partial parse live; tool_start replaces it.
const backendToolCallId =
(toolEvent.tool_call_id as string) || "";
const liveId = resolveToolPartId(backendToolCallId);
const fragment =
typeof toolEvent.text === "string" ? toolEvent.text : "";
if (liveId && fragment) {
const accum =
(liveArgsTextById.get(liveId) ?? "") + fragment;
liveArgsTextById.set(liveId, accum);
streamedChars += fragment.length;
const partial = parseLiveToolArgs(accum);
const idx = toolCallParts.findIndex(
(p) => p.toolCallId === liveId,
);
if (partial && idx !== -1) {
const existing = toolCallParts[
idx
] as PositionedToolCallPart;
toolCallParts[idx] = {
...existing,
args: partial.args as ToolCallMessagePart["args"],
argsText: partial.argsText,
};
// Pace previews only; tool events carry state, so they stay ungated.
if (canPublish(streamedChars)) {
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
}
}
continue;
}
closeReasoningContent();
const toolProvenance = parseToolProvenance(
toolEvent.provenance,
);
if (toolEvent.type === "tool_start") {
const backendToolCallId =
(toolEvent.tool_call_id as string) || "";
const approvalId = (toolEvent.approval_id as string) || "";
const awaitingConfirmation =
toolEvent.awaiting_confirmation === true;
// Reuse the provisional card's id, else a second card opens and the first spins forever.
const openPartId = backendToolCallId
? toolPartIdByBackendId.get(backendToolCallId)
: undefined;
const reuseOpenPart =
!!openPartId &&
toolCallParts.some((p) => p.toolCallId === openPartId);
const id =
awaitingConfirmation && approvalId && !reuseOpenPart
? `${toolConfirmationScopeId}:${approvalId}`
: backendToolCallId
? resolveToolPartId(backendToolCallId)
: approvalId || `${toolEvent.tool_name}_${Date.now()}`;
if (awaitingConfirmation && backendToolCallId) {
toolConfirmationIdsByBackendId.set(backendToolCallId, id);
}
// Call ids like call_0 restart per response, so drop stale output under this key.
const staleKey = scopedToolOutputKey(id);
useChatRuntimeStore.getState().clearToolLiveOutput(staleKey);
useChatRuntimeStore.getState().clearToolFullOutput(staleKey);
const toolArgs = (toolEvent.arguments ??
{}) as ToolCallMessagePart["args"];
const toolArgsText = toolCallArgumentsText(
toolEvent.arguments_text,
toolArgs,
);
const toolIdentity = {
backendToolCallId,
...(generationRunId
? {
generationToolCallId: `${generationRunId}:${generationSeq}`,
}
: {}),
...(approvalId ? { toolApprovalId: approvalId } : {}),
};
const idx = toolCallParts.findIndex(
(p) => p.toolCallId === id,
);
if (idx !== -1) {
const existing = toolCallParts[
idx
] as PositionedToolCallPart;
toolCallParts[idx] = {
...existing,
...toolIdentity,
toolName: toolEvent.tool_name as string,
argsText: toolArgsText,
args: toolArgs,
provenance: mergeToolProvenance(
existing.provenance,
toolProvenance,
),
};
} else {
toolCallParts.push({
type: "tool-call" as const,
toolCallId: id,
...toolIdentity,
toolName: toolEvent.tool_name as string,
argsText: toolArgsText,
args: toolArgs,
textCursor: cumulativeText.length,
...(toolProvenance ? { provenance: toolProvenance } : {}),
} as PositionedToolCallPart);
}
if (awaitingConfirmation) {
// The run's key now, not at send: a first turn adopted mid-run files its calls under the chat.
const confirmationThreadKey = liveThreadKey(serverCancel);
useChatRuntimeStore
.getState()
.setToolConfirmation(
id,
approvalId,
sandboxSessionId ?? "",
toolConfirmationScopeId,
confirmationThreadKey === "__default" ? "" : confirmationThreadKey,
toolEvent.image_disclosure as ImageDisclosure | undefined,
cancelId,
);
}
} else if (toolEvent.type === "tool_end") {
// The round boundary: the last pass's prompt and decode figures must not stand in for the next one's.
runtime.clearLiveGenerationForThread(liveThreadKey(serverCancel));
const backendToolCallId =
(toolEvent.tool_call_id as string) || "";
const id = resolveToolPartId(backendToolCallId);
if (backendToolCallId) {
toolConfirmationIdsByBackendId.delete(backendToolCallId);
toolPartIdByBackendId.delete(backendToolCallId);
}
useChatRuntimeStore.getState().clearToolConfirmation(id);
// Use the shared predicate, not length: a footer or __IMAGES__ tail can lengthen the result.
const liveKey = scopedToolOutputKey(id);
const liveOutput =
useChatRuntimeStore.getState().toolLiveOutput[liveKey] ??
"";
if (
id &&
shouldPreserveFullOutput(
liveOutput,
(toolEvent.result as string) ?? "",
)
) {
useChatRuntimeStore
.getState()
.setToolFullOutput(liveKey, liveOutput);
}
useChatRuntimeStore.getState().clearToolLiveOutput(liveKey);
runToolLiveOutputKeys.delete(liveKey);
liveArgsTextById.delete(id);
const idx = toolCallParts.findIndex(
(p) => p.toolCallId === id,
);
if (idx !== -1) {
const rawEvent = (toolEvent.result as string) ?? "";
// Extracted before __IMAGES__, and only for tools that emit it; elsewhere the line is content.
const { text: withUi, files: createdFiles } =
SANDBOX_FILE_TOOLS.has(toolCallParts[idx].toolName ?? "")
? extractCreatedFiles(rawEvent)
: { text: rawEvent, files: [] as SandboxFile[] };
// Ahead of the image slice, which parses to end of string.
const { text: rawResult, ui: mcpUi } =
extractMcpUiEnvelope(
withUi,
toolCallParts[idx].toolName ?? "",
);
// Same rule: only from the tool that emits it.
const { text: searchText, images: webImages } =
toolCallParts[idx].toolName === SEARCH_IMAGE_TOOL
? extractSearchImages(rawResult)
: { text: rawResult, images: [] as SearchImageEntry[] };
const imgMarker = "\n__IMAGES__:";
// Only from tools that emit the envelope; elsewhere the backend keeps the line for the model.
const imgIdx = IMAGE_SENTINEL_TOOLS.has(
toolCallParts[idx].toolName ?? "",
)
? rawResult.lastIndexOf(imgMarker)
: -1;
let parsedResult:
| string
| {
text: string;
images: string[];
sessionId: string;
files?: SandboxFile[];
}
| McpImageToolResult
| McpUiToolResult
| SearchImagesToolResult
| {
image_b64: string;
image_mime: string;
size?: string;
quality?: string;
background?: string;
prompt?: string;
};
const imageB64 = toolEvent.image_b64 as string | undefined;
// A valid MCP envelope wins; an invalid one falls through so a sandbox __IMAGES__ suffix renders.
const mcpCandidate = splitMcpImages(rawResult);
const mcpImages: McpImageToolResult | null =
isMcpImageToolResult(mcpCandidate) ? mcpCandidate : null;
if (
toolCallParts[idx].toolName === "image_generation" &&
typeof imageB64 === "string" &&
imageB64
) {
// The backend keeps base64 on separate image_b64 / image_mime fields to keep logs small.
parsedResult = {
image_b64: imageB64,
image_mime:
(toolEvent.image_mime as string | undefined) ??
"image/png",
size: toolEvent.size as string | undefined,
quality: toolEvent.quality as string | undefined,
background: toolEvent.background as string | undefined,
prompt: toolEvent.prompt as string | undefined,
};
} else if (mcpImages !== null) {
parsedResult = mcpImages;
} else if (imgIdx !== -1) {
const text = rawResult.slice(0, imgIdx);
// "_default" is the backend sandbox dir when there is no session_id (tools.py _get_workdir).
const sessionId = sandboxSessionId || "_default";
try {
const images = JSON.parse(
rawResult.slice(imgIdx + imgMarker.length),
) as string[];
parsedResult = {
text,
images,
sessionId,
files: createdFiles,
};
} catch {
parsedResult = rawResult;
}
} else if (
createdFiles.length > 0 ||
SANDBOX_RUN_TOOLS.has(toolCallParts[idx].toolName ?? "")
) {
// Structured even without files or images: the session is the only record of WHERE it ran.
parsedResult = {
text: rawResult,
images: [],
sessionId: sandboxSessionId || "_default",
files: createdFiles,
};
} else if (webImages.length > 0) {
parsedResult = { text: searchText, webImages };
} else {
parsedResult = rawResult;
}
if (mcpUi) {
parsedResult = isMcpImageToolResult(parsedResult)
? { ...parsedResult, ui: mcpUi }
: {
text:
typeof parsedResult === "string"
? parsedResult
: rawResult,
ui: mcpUi,
};
}
const nextArgs =
toolEvent.arguments &&
typeof toolEvent.arguments === "object"
? (toolEvent.arguments as ToolCallMessagePart["args"])
: undefined;
const mergedArgs = mergeGoogleNativeParts(
{ ...(toolCallParts[idx].args ?? {}), ...(nextArgs ?? {}) },
toolEvent.google,
) as ToolCallMessagePart["args"];
const overwrittenArgumentKeys =
nextArgs !== undefined ? Object.keys(nextArgs) : [];
const existing = toolCallParts[
idx
] as PositionedToolCallPart;
toolCallParts[idx] = {
...existing,
args: mergedArgs,
argsText: mergedToolCallArgumentsText(
existing.argsText,
mergedArgs,
overwrittenArgumentKeys,
),
result: parsedResult,
provenance: mergeToolProvenance(
existing.provenance,
toolProvenance,
),
};
}
}
canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
continue;
}
const chunkTimings = (chunk as Record<string, unknown>).timings as
| ServerTimings
| undefined;
// Timings can arrive without usage.
windowCount = windowEvidenceCount(chunkTimings) ?? windowCount;
if (chunkTimings && chunkTimings.diffusion !== true) liveServerTimings = chunkTimings;
// OpenAI usage may arrive in an empty trailing chunk or on the terminal Codex chunk.
if (chunk.usage) {
serverMetadata = {
usage: chunk.usage,
timings: chunkTimings,
};
if (chunk.choices?.length === 0) continue;
}
totalChunks += 1;
// Latched, not read off the last chunk: a provider can send the terminal reason ahead of its usage chunk.
if (chunk.choices?.[0]?.finish_reason === "length") {
incompleteReason = "length";
} else if (chunk.choices?.[0]?.finish_reason) {
incompleteReason = null;
}
const chunkFinishReason = chunk.choices?.[0]?.finish_reason;
if (typeof chunkFinishReason === "string" && chunkFinishReason) {
lastFinishReason = chunkFinishReason;
}
// Latch the chunk's `model` field so the openrouter/free chip shows the underlying model.
if (
isExternalRequest &&
externalProvider?.providerType === "openrouter" &&
externalSelection?.modelId === "openrouter/free"
) {
const chunkModel = (chunk as { model?: unknown }).model;
if (
typeof chunkModel === "string" &&
chunkModel.length > 0 &&
chunkModel !== externalSelection.modelId
) {
const storeState = useChatRuntimeStore.getState();
if (storeState.lastOpenRouterChosenModel !== chunkModel) {
storeState.setLastOpenRouterChosenModel(chunkModel);
}
}
}
const rawDelta = chunk.choices?.[0]?.delta?.content;
// Normalize structured delta.content (mistral magistral).
const {
text: delta,
structuredReasoningContinues,
hasStructuredReasoning,
} = extractDeltaText(rawDelta);
const deltaExtraContent = (
chunk.choices?.[0]?.delta as
| { extra_content?: unknown }
| undefined
)?.extra_content;
// State reaches the message only via a yield, so pace previews but never state changes.
let replayStateChanged = false;
const spokenAudio = (
chunk.choices?.[0]?.delta as
| { audio?: { data?: unknown } }
| undefined
)?.audio?.data;
if (typeof spokenAudio === "string" && spokenAudio) {
spokenReplyPart = {
type: "text",
text: `<audio-player src="data:audio/wav;base64,${spokenAudio}" />`,
};
replayStateChanged = true;
}
let geminiReplayStateChanged = false;
let geminiThoughtSignature: string | undefined;
let capturedGeminiThoughtPart = false;
if (deltaExtraContent && typeof deltaExtraContent === "object") {
const extraRecord = deltaExtraContent as Record<
string,
unknown
>;
const eGoogle = extraRecord.google;
if (eGoogle && typeof eGoogle === "object") {
const googleRecord = eGoogle as Record<string, unknown>;
const sig = googleRecord.thought_signature;
const thoughtPart = googleRecord.thought_part;
const answerPart = googleRecord.answer_part;
if (
thoughtPart &&
typeof thoughtPart === "object" &&
!Array.isArray(thoughtPart)
) {
const thoughtRecord = thoughtPart as Record<string, unknown>;
const thoughtText = thoughtRecord.text;
const thoughtSignature = thoughtRecord.thought_signature;
if (
typeof thoughtText === "string" &&
(thoughtSignature === undefined ||
(typeof thoughtSignature === "string" && thoughtSignature))
) {
appendGeminiThoughtReplayPart(
geminiThoughtParts,
thoughtText,
typeof thoughtSignature === "string"
? thoughtSignature
: undefined,
toolCallParts.length,
);
capturedGeminiThoughtPart = true;
geminiReplayStateChanged = true;
}
}
let capturedAnswerPart = false;
if (
answerPart &&
typeof answerPart === "object" &&
!Array.isArray(answerPart)
) {
const answerRecord = answerPart as Record<string, unknown>;
const answerText = answerRecord.text;
const answerSignature = answerRecord.thought_signature;
if (
typeof answerText === "string" &&
(answerSignature === undefined ||
(typeof answerSignature === "string" && answerSignature))
) {
appendGeminiAnswerReplayPart(
geminiAnswerParts,
{
text: answerText,
...(typeof answerSignature === "string"
? { thoughtSignature: answerSignature }
: {}),
},
toolCallParts.length,
);
capturedAnswerPart = true;
geminiReplayStateChanged = true;
}
}
if (typeof sig === "string" && sig) {
const belongsToThought = googleRecord.thought === true;
if (belongsToThought) {
geminiThoughtSignature = sig;
} else if (!capturedAnswerPart) {
geminiReplayStateChanged ||= sig !== legacyGeminiTextSignature;
legacyGeminiTextSignature = sig;
}
}
}
const codexReasoning = extraRecord.openai_codex_reasoning;
if (
Array.isArray(codexReasoning) &&
codexReasoning.length > 0
) {
codexReasoningLedger = addCodexReasoning(
codexReasoningLedger,
codexReasoning,
codexRoundToolCallIds,
);
replayStateChanged = true;
}
const openAIResponsesReasoning =
extraRecord.openai_responses_reasoning;
if (
Array.isArray(openAIResponsesReasoning) &&
openAIResponsesReasoning.length > 0
) {
openAIResponsesReasoningLedger = addCodexReasoning(
openAIResponsesReasoningLedger,
openAIResponsesReasoning,
codexRoundToolCallIds,
);
replayStateChanged = true;
}
}
if (chunk.choices?.[0]?.finish_reason) {
codexRoundToolCallIds = [];
}
// Kimi / DeepSeek stream thinking via delta.reasoning_content; wrap it inline as <think>.
const rawReasoning = (
chunk.choices?.[0]?.delta as
| { reasoning_content?: unknown }
| undefined
)?.reasoning_content;
// OpenRouter ships reasoning as delta.reasoning_details[] whatever the provider; same wrap.
const rawReasoningDetails = (
chunk.choices?.[0]?.delta as
| { reasoning_details?: unknown }
| undefined
)?.reasoning_details;
const reasoningFromDetails = Array.isArray(rawReasoningDetails)
? rawReasoningDetails
.map((part) => {
if (!part || typeof part !== "object") return "";
const text = (part as { text?: unknown }).text;
return typeof text === "string" ? text : "";
})
.join("")
: "";
const reasoning =
(typeof rawReasoning === "string" ? rawReasoning : "") +
reasoningFromDetails;
if (
!capturedGeminiThoughtPart &&
externalProvider?.providerType === "gemini" &&
(reasoning || geminiThoughtSignature)
) {
appendGeminiThoughtReplayPart(
geminiThoughtParts,
reasoning,
geminiThoughtSignature,
toolCallParts.length,
);
geminiReplayStateChanged = true;
}
// Text-bearing replay metadata is published with the same paced preview as its visible delta.
// Metadata-only signature parts still need an immediate yield or Stop could lose them.
replayStateChanged ||=
geminiReplayStateChanged && !delta && !reasoning;
// OpenAI delta.tool_calls streams fragments by index; accumulate into one part. extra_content
// carries the Gemini 3 thoughtSignature.
const rawDeltaToolCalls = (
chunk.choices?.[0]?.delta as
| { tool_calls?: unknown }
| undefined
)?.tool_calls;
if (
Array.isArray(rawDeltaToolCalls) &&
rawDeltaToolCalls.length > 0
) {
closeReasoningContent();
// Extending a call's arguments is a preview; introducing one is state, so it always publishes.
let addedToolCall = false;
for (const tc of rawDeltaToolCalls) {
if (!tc || typeof tc !== "object") continue;
const call = tc as {
id?: string;
index?: number;
function?: { name?: string; arguments?: string };
extra_content?: unknown;
};
const idx =
typeof call.index === "number" ? call.index : undefined;
const stableId = call.id;
// The chunk is cast, not validated, and llama-server has shipped `arguments` as a decoded object.
const deltaArgs = streamedToolCallArguments(
call.function?.arguments,
);
// The local Codex loop follows the delta with tool_start/tool_end; resolve the backend id now
// to keep all three on one card.
if (
stableId &&
!providerSentToolCallIds.has(stableId) &&
toolCallParts.some((part) => part.toolCallId === stableId)
) {
// Renumber first: marking a minted card provider-sent earlier would exempt it from the move.
renumberMintedCards(stableId);
providerSentToolCallIds.add(stableId);
reservedToolCallIds.add(stableId);
}
const stablePartId = stableId
? resolveToolPartId(stableId)
: undefined;
// Reserved before any card id is minted, so a provider spelling an id `tool_call_0` keeps it.
if (stableId) {
reservedToolCallIds.add(stableId);
providerSentToolCallIds.add(stableId);
}
if (stablePartId) providerSentToolCallIds.add(stablePartId);
const existingIndex = findStreamedToolCallPartIndex(
toolCallParts,
stablePartId,
idx,
);
const matched =
existingIndex === -1
? undefined
: toolCallParts[existingIndex];
// A name or arguments reaching a closed object open the next call.
const slotIsClosed = (() => {
if (!matched?.argsText) return false;
const held = scanArgsText(
matched.toolCallId,
matched.argsText,
);
return held.complete.length > 0 && !held.tail;
})();
// A fragment repeating the id this part holds continues it however complete the arguments look.
const namesThisCall =
!!stablePartId && matched?.toolCallId === stablePartId;
// Only a "{" opens a next call; splitting on a scalar suffix runs the tool twice.
const bringsArgs = deltaArgs.trim().startsWith("{");
const closedSlot = slotIsClosed && !namesThisCall;
// An id naming a DIFFERENT call opens the next even before its arguments arrive.
const idNamesAnotherCall =
!!stablePartId &&
!!call.function?.name &&
!!matched?.toolName &&
// Not a prefix test: a catalog holds both "web" and "web_search".
call.function.name !== matched.toolName;
// Snapshot providers repeat a finished call once it has an id; match exact repeats only.
const resendsThisCall =
!!stablePartId &&
!!matched &&
!matched._has_stable_id &&
call.function?.name === matched.toolName &&
deltaArgs === matched.argsText;
// A name at a closed slot announces the next call, but the SAME name is a
// resend (llama-server, vLLM).
const namesNextCall =
!!call.function?.name &&
!!matched?.toolName &&
call.function.name !== matched.toolName;
// A different name bringing an object after an announcement is a new call, not a name suffix.
const announcesOverAnnouncement =
((matched as PositionedToolCallPart | undefined)
?._announced_only === true ||
(matched as PositionedToolCallPart | undefined)
?._resend_suspect === true) &&
!matched?.argsText &&
!!call.function?.name &&
!!matched?.toolName &&
call.function.name !== matched.toolName &&
bringsArgs;
if (announcesOverAnnouncement && matched) {
(matched as PositionedToolCallPart)._superseded = true;
}
// Metadata on a repeated closed-card name belongs to the next call; merging overwrites its signature.
const extraIsAmbiguous =
closedSlot &&
!!call.function?.name &&
!!matched?.toolName &&
call.function.name === matched.toolName &&
isPlainRecord(call.extra_content);
const opensNextCall =
(closedSlot &&
(bringsArgs || idNamesAnotherCall || namesNextCall) &&
!resendsThisCall) ||
announcesOverAnnouncement;
const existing = opensNextCall ? undefined : matched;
if (
stablePartId &&
!codexRoundToolCallIds.includes(stablePartId)
) {
codexRoundToolCallIds.push(stablePartId);
}
const argsFragment = deltaArgs;
streamedChars +=
argsFragment.length + (call.function?.name?.length ?? 0);
if (existing) {
const prevName = existing.toolName ?? "";
// llama-server resends the growing name, OpenAI streams fragments; same rule as the backend.
const nameFragment = call.function?.name ?? "";
// Never rename once the object closed with a name set; naming an unnamed card is fine.
const nextName =
!nameFragment || (closedSlot && prevName)
? prevName
: nameFragment.startsWith(prevName)
? nameFragment
: prevName + nameFragment;
// A snapshot repeated to carry the id adds nothing; appending gives `{"a":1}{"a":1}`.
const merged = resendsThisCall
? (existing.argsText ?? "")
: (existing.argsText ?? "") + argsFragment;
// Two objects in one slot are two calls (vLLM id-less deltas, #9807); cut on the object boundary.
const split = stablePartId
? { complete: [], tail: "" }
: scanArgsText(existing.toolCallId, merged);
// Whether the last segment is still open decides who may go on writing to it.
const splitTailIsOpen = split.tail.length > 0;
const segments = splitTailIsOpen
? [...split.complete, split.tail]
: split.complete;
const isSplit = segments.length > 1;
const slotText = isSplit ? segments[0] : merged;
let parsedArgs: ToolCallMessagePart["args"] =
existing.args ?? {};
if (slotText) {
try {
parsedArgs = JSON.parse(
slotText,
) as ToolCallMessagePart["args"];
} catch {
parsedArgs = {
_raw: slotText,
} as ToolCallMessagePart["args"];
}
}
const prevExtra = (existing as PositionedToolCallPart)
.extra_content;
// Merge, not replace: signatures with the name and with the arguments are different fields.
if (extraIsAmbiguous && isPlainRecord(call.extra_content)) {
pendingExtraByPartId.set(existing.toolCallId, {
...(pendingExtraByPartId.get(existing.toolCallId) ?? {}),
...call.extra_content,
});
}
const incomingExtra = extraIsAmbiguous
? prevExtra
: isPlainRecord(prevExtra) && isPlainRecord(call.extra_content)
? { ...prevExtra, ...call.extra_content }
: call.extra_content;
if (
incomingExtra !== undefined &&
JSON.stringify(incomingExtra) !== JSON.stringify(prevExtra)
) {
// Gemini puts the thought signature on the call, and the next turn is rejected without it.
replayStateChanged = true;
}
const updated: PositionedToolCallPart = {
...(existing as PositionedToolCallPart),
// A late id claims its id-less slot so tool_start and tool_end find the same card.
...(stablePartId
? { toolCallId: stablePartId, _has_stable_id: true }
: {}),
toolName: isSplit ? prevName || nextName : nextName,
argsText: slotText,
args: parsedArgs,
...(incomingExtra !== undefined && !isSplit
? { extra_content: incomingExtra }
: prevExtra !== undefined
? { extra_content: prevExtra }
: {}),
...(idx !== undefined ? { _delta_index: idx } : {}),
// Any arguments field, "" included, means the call started, so the card survives the sweep.
_announced_only:
(existing as PositionedToolCallPart)._announced_only ===
true && call.function?.arguments === undefined,
// A name that brought its own arguments was no resend.
_resend_suspect:
(existing as PositionedToolCallPart)._resend_suspect ===
true && !slotText,
};
toolCallParts[existingIndex] = updated;
// Move keys to the late id and return the provisional id so the next mint does not skip.
if (stablePartId && stablePartId !== existing.toolCallId) {
renameStreamedCard(existing.toolCallId, stablePartId);
}
if (isSplit) {
// The slot keeps one segment, not the whole string, so the resumable scan no longer describes it.
boundaryScans.delete(existing.toolCallId);
boundaryScans.delete(updated.toolCallId);
// Appended so call order holds; this delta's own metadata, since Gemini validates a signature
// against the part it was returned on.
const born = bornSplitToolCalls(
segments.slice(1),
nextName,
idx,
call.extra_content,
);
// The last segment may still be open: keep it for later fragments, drop it if it never closes.
if (splitTailIsOpen && born.length > 0) {
openTailIds.add(born[born.length - 1].toolCallId);
}
toolCallParts.push(...born);
// New calls are state, so they never wait on the gate.
addedToolCall = true;
}
} else {
const callId =
stablePartId ||
mintStreamedCardId(idx ?? toolCallParts.length);
if (!stablePartId) paintStreamedCard(callId);
if (!codexRoundToolCallIds.includes(callId)) {
codexRoundToolCallIds.push(callId);
}
// vLLM bundles several calls into one delta when the model writes them in one pass.
const freshSplit = stablePartId
? { complete: [], tail: "" }
: splitTopLevelJsonObjects(argsFragment);
const freshTailIsOpen = freshSplit.tail.length > 0;
const freshSegments = freshTailIsOpen
? [...freshSplit.complete, freshSplit.tail]
: freshSplit.complete;
const freshIsSplit = freshSegments.length > 1;
const nameFragment = call.function?.name ?? "";
const heldName = matched?.toolName ?? "";
// A second call to the same tool can arrive with no name, the first delta having given it.
const freshName = nameFragment || heldName;
// With several calls, metadata belongs to the last one this delta closes, as in the backend.
const waiting = matched
? pendingExtraByPartId.get(matched.toolCallId)
: undefined;
if (waiting && matched) {
pendingExtraByPartId.delete(matched.toolCallId);
}
const withWaiting =
waiting === undefined
? call.extra_content
: {
...waiting,
...(isPlainRecord(call.extra_content)
? call.extra_content
: {}),
};
const freshOwnExtra = freshIsSplit ? undefined : withWaiting;
const bornExtra = freshIsSplit ? withWaiting : undefined;
const argsText = freshIsSplit
? freshSegments[0]
: argsFragment;
let parsedArgs: ToolCallMessagePart["args"] = {};
if (argsText) {
try {
parsedArgs = JSON.parse(
argsText,
) as ToolCallMessagePart["args"];
} catch {
parsedArgs = {
_raw: argsText,
} as ToolCallMessagePart["args"];
}
}
const fresh: PositionedToolCallPart = {
type: "tool-call" as const,
toolCallId: callId,
toolName: freshName,
argsText,
args: parsedArgs,
textCursor: cumulativeText.length,
...(freshOwnExtra !== undefined
? { extra_content: freshOwnExtra }
: {}),
...(stablePartId ? { _has_stable_id: true } : {}),
...(idx !== undefined ? { _delta_index: idx } : {}),
...(call.function?.arguments === undefined && freshName
? { _announced_only: true }
: {}),
// Only the provider's own announcement may run unfilled.
...(matched ? { _from_fork: true } : {}),
// A name extending this fork's leftover is likely a resend: open a card but yield, not glue.
...(freshName &&
heldName &&
freshName !== heldName &&
freshName.startsWith(heldName)
? { _resend_suspect: true }
: {}),
};
const born = freshIsSplit
? bornSplitToolCalls(
freshSegments.slice(1),
freshName,
idx,
bornExtra,
)
: [];
// As above: the fork whose object is still open is held only for the rest of the turn.
if (freshTailIsOpen && born.length > 0) {
openTailIds.add(born[born.length - 1].toolCallId);
}
toolCallParts.push(fresh, ...born);
addedToolCall = true;
}
}
const mcpStamps = (
chunk as { _mcp_provenance?: Record<string, unknown> }
)._mcp_provenance;
let stampedProvenance = false;
if (mcpStamps && typeof mcpStamps === "object") {
for (const [backendId, raw] of Object.entries(mcpStamps)) {
const partId = resolveToolPartId(backendId);
const at = toolCallParts.findIndex(
(p) => p.toolCallId === partId,
);
if (at === -1) continue;
const existing = toolCallParts[at] as PositionedToolCallPart;
toolCallParts[at] = {
...existing,
provenance: mergeToolProvenance(
existing.provenance,
parseToolProvenance(raw),
),
};
stampedProvenance = true;
}
}
// After the deltas: finish_reason can share a chunk with the last name-only delta.
if (chunk.choices?.[0]?.finish_reason) {
// Ending the turn drops cards, so the publish below must see it rather than wait for the pacing gate.
replayStateChanged ||= endProviderTurn();
}
// A relabel adds no characters, so the pacing gate alone would hold it back.
const forcePublish =
addedToolCall || replayStateChanged || stampedProvenance;
if (forcePublish || canPublish(streamedChars)) {
canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
continue;
}
if (chunk.choices?.[0]?.finish_reason) {
replayStateChanged ||= endProviderTurn();
}
// extra_content can arrive with no content (Gemini signature, codex ledger); do not skip it.
if (replayStateChanged && !delta && !reasoning) {
const replayContent = liveAssistantContent();
if (replayContent.length > 0) {
canPublish.published();
yield {
content: replayContent,
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
continue;
}
if (!delta && !reasoning) {
continue;
}
// So a chunk that added nothing can be told from one that did.
const textLenBeforeChunk = cumulativeText.length;
if (waitingFirstChunk) {
waitingFirstChunk = false;
firstTokenTime = Date.now() - streamStartTime;
settleFirstTokenOk();
runtime.setGeneratingStatus(null);
}
if (reasoning || hasStructuredReasoning) {
setParseThink(true);
}
if (reasoning) {
if (!reasoningContentOpen) {
reasoningDurationTracker.startGroup();
appendCumulative(`<think>${reasoning}`);
reasoningContentOpen = true;
} else {
appendCumulative(reasoning);
}
}
if (delta) {
if (reasoningContentOpen) {
closeReasoningContent();
}
appendCumulative(delta);
}
streamedChars += reasoning.length + delta.length;
producedReplyText = true;
const textEndsInsideThink =
parseThink && thinkTags.endsInsideThink();
const assistantContent = liveAssistantContent();
// Fallback when no server-side reasoning_summary arrives.
const parsedReasoningGroupCount =
countReasoningGroups(assistantContent);
if (
parsedReasoningGroupCount > reasoningDurationTracker.groupCount
) {
reasoningDurationTracker.startGroup(
parsedReasoningGroupCount - 1,
);
}
if (parsedReasoningGroupCount > 0) {
// Some providers close reasoning per chunk; reopen while it grows so the timer spans the pass.
reasoningDurationTracker.resumeGroup(
parsedReasoningGroupCount - 1,
lastReasoningGroupTextLength(assistantContent),
);
}
if (
reasoningDurationTracker.hasActiveGroup &&
!reasoningContentOpen &&
!structuredReasoningContinues &&
!textEndsInsideThink
) {
reasoningDurationTracker.finishGroup();
}
// Only the publish is coalesced (the cost is downstream of the yield); a chunk
// with nothing new is skipped.
if (
!replayStateChanged &&
(assistantContent.length === 0 ||
cumulativeText.length === textLenBeforeChunk)
) {
continue;
}
if (!replayStateChanged && !canPublish(streamedChars)) {
continue;
}
if (assistantContent.length > 0) {
canPublish.published();
yield {
content: assistantContent,
metadata: {
timing: buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
),
custom: liveCustom(),
},
};
}
}
break;
} catch (streamError) {
if (
isExternalRequest &&
!retriedWithRefreshedKey &&
isProviderKeyRotationError(streamError)
) {
retriedWithRefreshedKey = true;
continue;
}
// The tool ran, so its research run is owed regardless of the acknowledgement; not after Stop.
if (deepResearchHandoff.question !== null && !runSignal.aborted) {
try {
yield* startDeepResearch(deepResearchHandoff.question, runSignal);
return;
} catch (researchError) {
toast.error("Deep research could not start", {
description:
researchError instanceof Error
? researchError.message
: String(researchError),
});
}
}
throw streamError;
}
}
// Deep Research card replaces this reply, unless the user already pressed Stop.
if (deepResearchHandoff.question !== null && !runSignal.aborted) {
try {
yield* startDeepResearch(deepResearchHandoff.question, runSignal);
} catch (error) {
// The reply the model already wrote stays; the user can send again.
toast.error("Deep research could not start", {
description:
error instanceof Error ? error.message : String(error),
});
}
return;
}
// Strip a trailing ${...} fragment (mistral magistral) once, on the finished reply only: per
// arrival it ate real text (#9098). Before the <think> close.
if (
isExternalRequest &&
producedReplyText &&
placeholderWatch.isCandidate()
) {
const stripped = stripTrailingTemplatePlaceholder(cumulativeText);
if (stripped.length !== cumulativeText.length) {
cumulativeText = stripped;
// A suffix went away, so both trackers have to be told; the parse notices by itself, from the length.
thinkTags.retract(cumulativeText);
placeholderWatch.retract(cumulativeText);
}
}
// If the stream ended inside a delta.reasoning_content block, close the open <think> tag.
closeReasoningContent();
settleFirstTokenOk();
// web_search and web_fetch share the Title/URL/Snippet shape, so no per-tool branching.
const sourceParts = toolCallParts.flatMap((tc) => {
if (
(tc.toolName !== "web_search" && tc.toolName !== "web_fetch") ||
!tc.result
) {
return [];
}
// An image-bearing result is an object; citations live on `.text`.
return parseSourcesFromResult(searchResultText(tc.result));
});
const meta = serverMetadata;
const finalTokenCount =
meta?.usage?.completion_tokens ?? estimateTokenCount(passText());
const finalServerTimings = withPriorServerTimings(
meta?.timings,
priorTiming?.serverTimings,
);
const finalTokPerSec = finalServerTimings?.predicted_per_second;
const serverPromptEvalTime = meta?.timings?.prompt_ms;
// Prefer llama-server timings; fall back to provider usage envelope.
const cachedTokens =
meta?.timings?.cache_n ??
meta?.usage?.prompt_tokens_details?.cached_tokens ??
meta?.usage?.cache_read_input_tokens ??
0;
const cacheWriteTokens = usageCacheWriteTokens(meta?.usage);
if (
meta?.usage &&
typeof meta.usage.prompt_tokens === "number" &&
typeof meta.usage.completion_tokens === "number" &&
typeof meta.usage.total_tokens === "number"
) {
publishContextUsage({
promptTokens: meta.usage.prompt_tokens,
completionTokens: meta.usage.completion_tokens,
totalTokens: meta.usage.total_tokens,
...(typeof meta.usage.context_tokens === "number"
? { contextTokens: meta.usage.context_tokens }
: {}),
cachedTokens,
cacheWriteTokens,
});
}
if (
incompleteReason === null &&
budgetImpliesTruncation({
isMlx: isMlxRequest,
maxTokens: requestedMaxTokens,
completionTokens: meta?.usage?.completion_tokens,
})
) {
incompleteReason = "length";
}
if (incompleteReason === "length") {
incompleteReason = lengthIncompleteReason(
lengthStopCause({
cap: requestedMaxTokens ?? null,
contextLength: servedContextLength,
promptTokens: meta?.usage?.prompt_tokens ?? null,
completionTokens: windowCount,
}),
);
}
// Before the lookup below: its network time is not generation time.
const finishedAt = Date.now();
// Fetch pictures small models forgot to request: local tool loop only, never behind approval,
// and never once the THREAD has retrieved private documents.
const answerUsedPrivateDocs =
ragEnabled ||
projectRagEnabled ||
messagesUsePrivateContent(messages) ||
toolCallParts.some((part) => part.toolName === "search_knowledge_base");
// This run's own values: a run finishing after a chat switch must not read that chat's level.
const toolCallsNeedApproval = confirmToolCalls && permissionMode === "ask";
if (
incompleteReason === null &&
!isExternalRequest &&
supportsTools &&
toolsEnabled &&
!answerUsedPrivateDocs &&
!toolCallsNeedApproval &&
useChatRuntimeStore.getState().searchImages
) {
// Rendered parts, not the raw stream: `cumulativeText` carries <think>.
const answerText = answerTextFromParts(
buildAssistantContent(mergeContinuation(cumulativeText, { final: true })),
);
const subjects = missingListSubjects(answerText, toolCallParts);
if (subjects.length > 0) {
// Bounded so a slow engine cannot hold a finished answer; linked by hand since
// AbortSignal.any is newer than the Safari floor.
const lookupAbort = new AbortController();
const onRunAbort = () => lookupAbort.abort();
runAbort.signal.addEventListener("abort", onRunAbort, { once: true });
const lookupTimer = setTimeout(() => lookupAbort.abort(), 15_000);
try {
// authFetch refreshes a token that expired mid-generation; a login redirect is harmless here.
const response = await authFetch(
"/api/inference/search-images/lookup",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ subjects }),
signal: lookupAbort.signal,
},
);
if (response.ok) {
const data = (await response.json()) as {
text?: unknown;
images?: unknown;
};
const webImages = Array.isArray(data.images)
? data.images.filter(isSearchImageEntry)
: [];
if (webImages.length > 0) {
const args = { image_queries: subjects };
toolCallParts.push({
type: "tool-call" as const,
// Ids must match ^[a-zA-Z0-9_-]+$; randomUUID is undefined over plain HTTP on a LAN.
toolCallId: `auto-images-${
typeof crypto !== "undefined" && "randomUUID" in crypto
? crypto.randomUUID()
: `${Date.now()}-${Math.random().toString(36).slice(2)}`
}`,
toolName: "web_search",
argsText: JSON.stringify(args),
args,
result: {
text: typeof data.text === "string" ? data.text : "",
webImages,
},
textCursor: cumulativeText.length,
} as PositionedToolCallPart);
}
}
} catch {
// Pictures are a garnish: a failed lookup leaves the answer as written.
} finally {
clearTimeout(lookupTimer);
runAbort.signal.removeEventListener("abort", onRunAbort);
}
}
}
if (
compactNow &&
!compactionMarkers?.some(
(marker) =>
marker.truncation.manual === true &&
marker.truncation.checkpoint_started === true,
)
) {
toast.info("This chat was not compacted", {
description:
"Compaction needs the conversation archive and the default checkpoint policy.",
});
}
const finalTiming = withPriorTiming(
buildTiming(
streamStartTime,
totalChunks,
// What the user waited, as a resumed run records it. Server prompt eval has its own row.
firstTokenTime ?? serverPromptEvalTime,
finishedAt - streamStartTime,
finalTokenCount,
toolCallParts.length,
finalTokPerSec,
),
priorTiming,
finalTokPerSec,
);
reasoningDurationTracker.finishGroup();
const finalContent = [
...buildAssistantContent(mergeContinuation(cumulativeText, { final: true })),
...sourceParts,
...documentCitationParts,
];
const finalIncompleteReason =
resolveIncompleteReason(incompleteReason, contextWindowExceeded) ??
(quoteCut ? "quote_cut" : null) ??
// An empty clean stop would otherwise save as a blank complete bubble with no way out.
(hasRenderableContent(finalContent) ? null : "empty");
if (continuation && !producedReplyText && !finalIncompleteReason) {
toast("The model had nothing to add", {
description: "It ended the reply where it already stopped.",
});
}
yield {
content: finalContent,
metadata: {
timing: finalTiming,
custom: {
...reasoningDurationTracker.metadata(),
// Persisted so Continue survives a reload; cleared on a normal end.
openaiCodexReasoning: codexReasoningLedger,
openaiResponsesReasoning: openAIResponsesReasoningLedger,
contextTruncation,
compactionMarkers,
providerCompaction,
providerCompactionAfterToolCalls,
providerCompactionProviderType,
providerCompactionModelId,
providerCompactionConnectionKey:
providerCompactionOriginConnectionKey,
geminiContinuationReplay: geminiContinuationReplay(true),
incomplete: finalIncompleteReason
? { reason: finalIncompleteReason }
: undefined,
// Persisted refusal flag driving the two-pass prune.
anthropicRefusal: anthropicRefusalSeen || undefined,
serverTimings: finalServerTimings ?? undefined,
stopReason: lastFinishReason ?? undefined,
contextUsage: meta?.usage
? {
promptTokens: meta.usage.prompt_tokens,
completionTokens: meta.usage.completion_tokens,
totalTokens: meta.usage.total_tokens,
...(typeof meta.usage.context_tokens === "number"
? { contextTokens: meta.usage.context_tokens }
: {}),
cachedTokens,
cacheWriteTokens,
modelId: params.checkpoint,
...(contextReasoningKey ? { reasoningKey: contextReasoningKey } : {}),
...contextUsageMarkers([
...threadBranchRecords(rawMessages),
{ id: "", role: "assistant", content: finalContent },
]),
}
: undefined,
responseDetails: buildResponseDetails(finishedAt),
timing: finalTiming,
...generationCustom(),
},
},
};
} catch (err) {
settleFirstTokenErr(
err instanceof Error ? err : new Error("Generation failed"),
);
if (!runSignal.aborted) {
const msg = err instanceof Error ? err.message : String(err);
if (
minPRecoveryGuard?.isValid() &&
shouldOfferMinPRecovery(msg, externalProvider?.providerType, params)
) {
keepMinPRecovery = true;
const cleanupTimer = setTimeout(
() => minPRecoveryGuard.dispose(),
8000,
);
const disposeRecovery = () => {
clearTimeout(cleanupTimer);
minPRecoveryGuard.dispose();
};
toast.error("Min P is incompatible with speculative decoding", {
description: `This server requires Min P to be disabled. Set it to 0, then retry. ${msg}`,
duration: 8000,
action: {
label: "Set Min P to 0",
onClick: () => {
const valid = minPRecoveryGuard.isValid();
disposeRecovery();
if (valid) {
const state = useChatRuntimeStore.getState();
state.setParams(
{ ...state.params, minPMode: "custom", minP: 0 },
{ minPChoiceEdited: true },
);
}
},
},
onDismiss: disposeRecovery,
onAutoClose: disposeRecovery,
});
} else if (err instanceof GenerationLengthError) {
toast.error("Response ran out of tokens", {
// The error already picked Max Tokens vs Context Length advice; do not override it here.
description:
msg ||
"The model used the full Max Tokens budget while thinking " +
"and did not produce a final answer. Increase Max Tokens in " +
"chat Settings or turn off thinking, then retry.",
duration: 8000,
});
} else if (err instanceof ChatGenerationTerminalError) {
if (err.generationStatus === "failed") {
toast.error("Response interrupted", {
description:
err.message ||
"The Unsloth backend stopped during generation.",
duration: 8000,
});
}
} else if (err instanceof StreamInterruptedError) {
// Connection dropped mid-turn: surface it explicitly (the rethrow also marks the message).
toast.error("Response interrupted", {
description:
"The connection dropped before the model finished. " +
"The partial answer is kept. Use Retry to regenerate.",
duration: 8000,
});
} else if (isContextLimitError(msg)) {
// fits false: eviction was exhausted, so the new message is the problem, not the chat length.
const irreducible =
contextTruncation?.fits === false ? contextTruncation : null;
// Use prompt_target: the fit reserves up to a quarter of the window for the reply.
const budget =
irreducible?.prompt_target ?? irreducible?.context_length ?? 0;
// latest_turn_tokens prices a whole prompt, so subtract the shared prompt floor first.
const oneTurnIsTheProblem = latestTurnIsTheProblem(
irreducible,
budget,
);
// In a tool loop the offending turn is often output the user cannot edit.
const userCanShortenIt =
(irreducible?.latest_turn_role ?? "user") === "user";
const tooLong =
`${latestTurnOwnTokens(irreducible).toLocaleString()} tokens on its own, ` +
`against the ${budget.toLocaleString()} tokens this ` +
`${irreducible?.context_length?.toLocaleString()}-token window leaves for the prompt. ` +
"The earlier turns were already removed and it still does not fit, so " +
"shortening the conversation will not help. ";
toast.error("Context limit reached", {
description: oneTurnIsTheProblem
? userCanShortenIt
? `This message is ${tooLong}` +
'Shorten this message, or raise "Context Length" ' +
"in the chat Settings panel (⚙ in the top-right)."
: `The last tool result is ${tooLong}` +
'Raise "Context Length" in the chat Settings panel (⚙ in the top-right), ' +
"or ask for less output from that tool."
: historyCannotHelp(irreducible)
?
"Even with every earlier turn dropped, this prompt would still be " +
"too long, so shortening the conversation will not help. " +
'Raise "Context Length" in the chat Settings panel (⚙ in the top-right), ' +
"or reduce what every request carries: the system prompt and any " +
"tools that are enabled."
// llama-server runs with --no-context-shift, so point at the control that raises the ceiling.
:
"The conversation has filled the model's context window. " +
'Increase "Context Length" in the chat Settings panel (⚙ in the top-right), ' +
"or start a new chat.",
duration: 8000,
});
} else {
toast.error("Generation failed", {
description: msg || "Unknown error",
});
}
// Only a status read shows a killed worker's model is gone; quiet, the failure has its toast.
if (!isExternalRequest) {
void syncInferenceStatusToStore({ includeLoras: false, quiet: true });
}
}
// An explicit Stop is an abort too, but must persist its latched "cancelled" reason via the replacement yield below.
if (!abortSignal.aborted || generationStopRequested) {
closeReasoningContent();
const partialText = mergeContinuation(cumulativeText, { final: true });
const partialContent = buildAssistantContent(partialText);
const partialTiming = withPriorTiming(
buildTiming(
streamStartTime,
totalChunks,
firstTokenTime,
Date.now() - streamStartTime,
estimateTokenCount(continuation ? passText() : partialText),
toolCallParts.length,
),
priorTiming,
);
yield {
content: partialContent,
metadata: {
timing: partialTiming,
custom: {
...reasoningDurationTracker.metadata(),
contextTruncation,
compactionMarkers,
providerCompaction,
providerCompactionAfterToolCalls,
providerCompactionProviderType,
providerCompactionModelId,
providerCompactionConnectionKey:
providerCompactionOriginConnectionKey,
geminiContinuationReplay: geminiContinuationReplay(true),
// Unfinished too, so it also offers Continue -- unless the provider already
// said why the model stopped.
incomplete: {
reason: resolveIncompleteReason(
incompleteReasonAfterError(
incompleteReason,
err instanceof GenerationLengthError
? lengthIncompleteReason(err.stopCause)
: err instanceof ChatGenerationTerminalError &&
err.generationStatus === "cancelled"
? "cancelled"
: "interrupted",
),
contextWindowExceeded,
),
},
timing: partialTiming,
resumeTiming: passSoFar ? addPass(priorTiming, passSoFar()) : priorTiming,
...generationCustom(),
},
},
};
}
throw err;
} finally {
if (!keepMinPRecovery) minPRecoveryGuard?.dispose();
// Release both ids unconditionally; a run left claimed after its stream died is never recovered.
releaseLiveGenerationRun(cancelId);
if (generationRunId) releaseLiveGenerationRun(generationRunId);
// Otherwise the thread reads as durable until a history load, capping the next legacy stream.
if (
generationRunId &&
(generationStatus === "completed" ||
generationStatus === "failed" ||
generationStatus === "cancelled")
) {
forgetServerActiveGenerationRun(generationRunId);
}
runSignal.removeEventListener("abort", onAbortCancel);
abortSignal.removeEventListener("abort", forwardAbort);
// Resolve once: the clears below drop the owner the lookup keys on.
const cleanupKey = liveThreadKey(serverCancel);
useChatRuntimeStore.getState().clearUnadoptedChatFullAccessRun(cancelId);
const confirmStore = useChatRuntimeStore.getState();
for (const part of toolCallParts) {
confirmStore.clearToolConfirmation(part.toolCallId);
}
runtime.setGeneratingStatus(null);
// Scoped by thread AND by run: a global clear wiped every other running chat's badge.
runtime.setToolStatus(cleanupKey, null, serverCancel);
// Promote stdout of keys that never reached tool_end before clearing, or diagnostics vanish.
for (const liveKey of runToolLiveOutputKeys) {
const store = useChatRuntimeStore.getState();
const liveOutput = store.toolLiveOutput[liveKey] ?? "";
if (liveOutput) {
store.setToolFullOutput(liveKey, liveOutput);
}
store.clearToolLiveOutput(liveKey);
}
runToolLiveOutputKeys.clear();
// Scoped clear: a global one wiped another chat's diffusion frame.
runtime.clearActiveDiffusionCanvasForThread(cleanupKey);
runtime.clearLiveGenerationForThread(cleanupKey);
clearTimeout(warmupTimer);
if (waitingFirstChunk) {
if (firstTokenSettled) {
settleFirstTokenOk();
} else if (runSignal.aborted) {
settleFirstTokenErr(new Error("Cancelled"));
} else {
settleFirstTokenErr(new Error("No tokens received"));
}
}
// serverCancel narrows both clears: runs with no resolved thread id share the "__default" key.
runtime.setThreadRunning(cleanupKey, false, { owner: serverCancel });
runtime.clearThreadServerCancel(cleanupKey, serverCancel);
if (browserTools.length > 0) endBrowserAgentRun(cancelId);
}
},
} satisfies ChatModelAdapter;
return {
async *run(args) {
// Only runs started here may auto-continue a Max Tokens cut.
noteRunStartedThisSession(args.unstable_assistantMessageId);
invalidateMinPRecoveries();
const preStreamThreadIds = preStreamRunThreadIdsForAdapter(
args.unstable_threadId,
useChatRuntimeStore.getState().activeThreadId,
);
const reservationToken = findPreStreamRunReservation(preStreamThreadIds);
if (reservationToken) {
adoptPreStreamRunReservation(reservationToken, preStreamThreadIds);
}
try {
yield* adapter.run(args);
} catch (error) {
if (!args.abortSignal.aborted) {
notifyPromptQueueRunFailed(
args.unstable_threadId ?? preStreamThreadIds[0] ?? null,
);
}
throw error;
} finally {
if (
reservationToken &&
releasePreStreamRunReservation(reservationToken)
) {
notifyPromptQueueRunFailed(args.unstable_threadId ?? null);
}
}
},
};
}