Skip to content

Build your personal knowledge base with Trilium Notes

13.3 KB
Raw Blame
import type {
    LlmChatConfig,
    LlmCitation,
    LlmErrorDetails,
    LlmMessage,
    LlmModelInfo,
    LlmStreamChunk,
    LlmStreamStatus,
    LlmToolApprovalAnswer,
    LlmToolApprovalRequest,
    LlmUsage,
    WebSocketMessage
} from "@triliumnext/commons";

import server from "./server.js";
import { isStandalone, randomString } from "./utils.js";
import { subscribeToMessages, unsubscribeToMessage } from "./ws.js";

/** Credentials describing a provider whose live model list should be fetched. */
export interface ProviderModelsQuery {
    provider: string;
    apiKey?: string;
    baseURL?: string;
}

/** How long the model list can take, unless the provider states its own limit. */
const PROVIDER_MODELS_TIMEOUT_MS = 60_000;

/**
 * Fetch the live model list for a provider from its credentials. Used by the
 * model-selection screen while adding or editing a provider — the config need
 * not be saved yet. A server-side failure (e.g. a bad API key) rejects with a
 * clean message the screen can display, and shows no toast.
 */
export async function fetchProviderModels(query: ProviderModelsQuery, timeoutMs = PROVIDER_MODELS_TIMEOUT_MS): Promise<LlmModelInfo[]> {
    try {
        const response = await server.postWithTimeout<{ models?: LlmModelInfo[] }>(
            "llm-chat/provider-models", timeoutMs, query, undefined, { silentBadRequest: true });
        return response.models ?? [];
    } catch (error) {
        throw new Error(serverErrorMessage(error));
    }
}

/**
 * Extract a human-readable message from a rejected `server.post`, which surfaces
 * the raw response body (a `{ "message": … }` JSON string) rather than an Error.
 */
function serverErrorMessage(error: unknown): string {
    if (typeof error === "string") {
        try {
            const parsed = JSON.parse(error);
            if (parsed && typeof parsed.message === "string") {
                return parsed.message;
            }
        } catch {
            // Not JSON — the raw string is the best message we have.
        }
        return error;
    }
    return error instanceof Error ? error.message : String(error);
}

export interface StreamCallbacks {
    onChunk: (text: string) => void;
    onThinking?: (text: string) => void;
    onToolInputStart?: (toolCallId: string, toolName: string) => void;
    onToolInputDelta?: (toolCallId: string, delta: string) => void;
    onToolUse?: (toolCallId: string, toolName: string, input: Record<string, unknown>) => void;
    /**
     * A tool call waits for the user to allow or deny it. `answer` sends the decision, and the call
     * then runs or reports itself declined in a `tool_result`. With the `"chat"` scope, the turn
     * also runs its tool's other calls without asking.
     */
    onToolApprovalRequest?: (
        request: LlmToolApprovalRequest,
        answer: (approved: boolean, scope?: LlmToolApprovalAnswer["scope"]) => Promise<void>
    ) => void;
    onToolResult?: (toolCallId: string, toolName: string, result: string, isError?: boolean) => void;
    onCitation?: (citation: LlmCitation) => void;
    onUsage?: (usage: LlmUsage) => void;
    /** What the turn waits on before its reply starts. */
    onStatus?: (status: LlmStreamStatus) => void;
    /**
     * @param error human-readable message.
     * @param details provider-call context (status, URL, response body), present only
     *   for failures that reached a provider — never for connection/transport errors raised here.
     */
    onError: (error: string, details?: LlmErrorDetails) => void;
    onDone: () => void;
}

/**
 * Stream a chat completion from the LLM API.
 *
 * Two transports, one contract: the completion is delivered chunk by chunk to
 * {@link StreamCallbacks} and the promise settles when it is over, rejecting with
 * an `AbortError` if `abortSignal` fired. Which transport is used depends on what
 * the backend can be reached over — see {@link streamChatOverMessages}.
 *
 * The stream id is minted here for either transport: the backend finds the turn by
 * it when the user answers one of its tool calls.
 */
export function streamChatCompletion(
    messages: LlmMessage[],
    config: LlmChatConfig,
    callbacks: StreamCallbacks,
    abortSignal?: AbortSignal
): Promise<void> {
    const streamId = randomString(12);
    return isStandalone
        ? streamChatOverMessages(streamId, messages, config, callbacks, abortSignal)
        : streamChatOverSse(streamId, messages, config, callbacks, abortSignal);
}

/** Tell a running turn whether the user allows one of its tool calls. */
async function answerToolApproval(answer: LlmToolApprovalAnswer): Promise<void> {
    await server.post("llm-chat/tool-approval", answer);
}

/**
 * Stream a chat completion over Server-Sent Events, the response of a request
 * that stays open for the whole completion.
 */
async function streamChatOverSse(
    streamId: string,
    messages: LlmMessage[],
    config: LlmChatConfig,
    callbacks: StreamCallbacks,
    abortSignal?: AbortSignal
): Promise<void> {
    let response: Response;
    try {
        const headers = await server.getHeaders();
        response = await fetch(`${window.glob.baseApiUrl}llm-chat/stream`, {
            method: "POST",
            headers: {
                ...headers,
                "Content-Type": "application/json"
            } as HeadersInit,
            body: JSON.stringify({ streamId, messages, config }),
            signal: abortSignal
        });
    } catch (e) {
        // AbortError is the user stopping generation — let the caller handle it.
        // Everything else (network failure, custom-protocol/CORS issues, DNS, etc.)
        // is reported via onError so the chat UI shows it instead of hanging.
        if (e instanceof DOMException && e.name === "AbortError") {
            throw e;
        }
        const message = e instanceof Error ? e.message : String(e);
        callbacks.onError(`Failed to connect to LLM stream: ${message}`);
        return;
    }

    if (!response.ok) {
        callbacks.onError(`HTTP ${response.status}: ${response.statusText}`);
        return;
    }

    const reader = response.body?.getReader();
    if (!reader) {
        callbacks.onError("No response body");
        return;
    }

    const decoder = new TextDecoder();
    let buffer = "";
    // Only a `done` or `error` chunk ends a turn: a proxy can close the body cleanly
    // before the server sends either.
    let completed = false;

    try {
        while (true) {
            const { done, value } = await reader.read();
            if (done) break;

            buffer += decoder.decode(value, { stream: true });
            const lines = buffer.split("\n");
            buffer = lines.pop() || "";

            for (const line of lines) {
                if (line.startsWith("data: ")) {
                    try {
                        const chunk: LlmStreamChunk = JSON.parse(line.slice(6));
                        if (chunk.type === "done" || chunk.type === "error") {
                            completed = true;
                        }
                        await handleChunk(streamId, chunk, callbacks);
                    } catch (e) {
                        console.error("Failed to parse SSE data line:", line, e);
                    }
                }
            }
        }

        if (!completed) {
            callbacks.onError(
                "LLM stream ended without completing — the connection closed before the response "
                + "finished (network or proxy interruption)."
            );
        }
    } catch (e) {
        if (e instanceof DOMException && e.name === "AbortError") {
            throw e;
        }
        const message = e instanceof Error ? e.message : String(e);
        callbacks.onError(`LLM stream interrupted: ${message}`);
    } finally {
        reader.releaseLock();
    }
}

/**
 * Stream a chat completion over the WebSocket-style message channel, for the
 * standalone build: its backend runs in a worker reached through a request bridge
 * that answers with one buffered body, so a response cannot stay open. The
 * request here only starts the completion; the chunks arrive as `llm-stream`
 * messages and an `llm-stream-end` says when there are no more.
 *
 * The stream id is minted before the completion is asked for, so the
 * subscription is already in place when the first chunk lands. Every tab receives
 * the messages — only the leader holds the worker, and the rest hear it through a
 * relay — so the id is also what makes a tab keep just its own.
 */
async function streamChatOverMessages(
    streamId: string,
    messages: LlmMessage[],
    config: LlmChatConfig,
    callbacks: StreamCallbacks,
    abortSignal?: AbortSignal
): Promise<void> {
    let resolveEnd: () => void = () => undefined;
    const ended = new Promise<void>((resolve) => { resolveEnd = resolve; });
    // Handlers are dispatched without being awaited, so nothing else keeps the
    // chunks in order: each is chained onto the one before it.
    let handled: Promise<void> = Promise.resolve();

    const onMessage = (message: WebSocketMessage) => {
        if (message.type === "llm-stream" && message.streamId === streamId) {
            handled = handled.then(() => handleChunk(streamId, message.chunk, callbacks));
            // Marks the link handled the moment it is made. `await handled` below
            // still rethrows, but without this a chunk handler that threw would sit
            // as an unhandled rejection until the stream ends — and in the
            // standalone build a stray rejection is reported, not ignored.
            handled.catch(() => undefined);
        } else if (message.type === "llm-stream-end" && message.streamId === streamId) {
            resolveEnd();
        }
    };
    subscribeToMessages(onMessage);

    try {
        try {
            await server.post("llm-chat/stream-start", { streamId, messages, config });
        } catch (e) {
            callbacks.onError(`Failed to start the LLM stream: ${serverErrorMessage(e)}`);
            return;
        }

        try {
            await Promise.race([ended, rejectWhenAborted(abortSignal)]);
        } catch (e) {
            // Tell the backend to stop the completion; it has no disconnect to notice.
            void server.post("llm-chat/stream-abort", { streamId }).catch(() => undefined);
            throw e;
        }

        // Chunks delivered right before the end marker may still be in the chain.
        await handled;
    } finally {
        unsubscribeToMessage(onMessage);
    }
}

/**
 * A promise that rejects the way an aborted `fetch` does, so both transports fail
 * a stopped generation identically. Never settles when there is no signal.
 */
function rejectWhenAborted(signal?: AbortSignal): Promise<never> {
    return new Promise((_resolve, reject) => {
        if (!signal) {
            return;
        }
        const fail = () => reject(new DOMException("The LLM stream was aborted.", "AbortError"));
        if (signal.aborted) {
            fail();
        } else {
            signal.addEventListener("abort", fail, { once: true });
        }
    });
}

/**
 * Dispatch one chunk to the callbacks. Shared by both transports, so a chunk means
 * the same thing however it arrived.
 */
async function handleChunk(
    streamId: string,
    chunk: LlmStreamChunk,
    callbacks: StreamCallbacks
): Promise<void> {
    switch (chunk.type) {
        case "status":
            callbacks.onStatus?.(chunk.status);
            break;
        case "text":
            callbacks.onChunk(chunk.content);
            break;
        case "thinking":
            callbacks.onThinking?.(chunk.content);
            break;
        case "tool_input_start":
            callbacks.onToolInputStart?.(chunk.toolCallId, chunk.toolName);
            // Yield to force Preact to commit the pending tool call
            // state before any deltas arrive.
            await new Promise((r) => setTimeout(r, 1));
            break;
        case "tool_input_delta":
            callbacks.onToolInputDelta?.(chunk.toolCallId, chunk.delta);
            break;
        case "tool_use":
            callbacks.onToolUse?.(chunk.toolCallId, chunk.toolName, chunk.toolInput);
            // Yield to force Preact to commit the pending tool call
            // state before we process the result.
            await new Promise((r) => setTimeout(r, 1));
            break;
        case "tool_approval_request": {
            const { toolCallId, toolName, toolInput } = chunk;
            callbacks.onToolApprovalRequest?.({ toolCallId, toolName, toolInput }, (approved, scope) => {
                const answer: LlmToolApprovalAnswer = { streamId, toolCallId, approved };
                if (scope) answer.scope = scope;
                return answerToolApproval(answer);
            });
            break;
        }
        case "tool_result":
            callbacks.onToolResult?.(chunk.toolCallId, chunk.toolName, chunk.result, chunk.isError);
            await new Promise((r) => setTimeout(r, 1));
            break;
        case "citation":
            if (chunk.citation) {
                callbacks.onCitation?.(chunk.citation);
            }
            break;
        case "usage":
            if (chunk.usage) {
                callbacks.onUsage?.(chunk.usage);
            }
            break;
        case "error":
            callbacks.onError(chunk.error, chunk.errorDetails);
            break;
        case "done":
            callbacks.onDone();
            break;
    }
}