Skip to content

LLM inference in C/C++

C++ ggml
20.3 KB
Raw Blame
/**
 * modelsStore - Model management for MODEL and ROUTER modes
 *
 * Owns model lists, selection, favorites and load/unload state. Composes the
 * per-model props cache (modalities, thinking detection) as
 * {@link ModelsStore.props} and the /models/sse status feed as
 * {@link ModelsStore.status}; tracks which conversations use which models.
 */

import { browser } from '$app/environment';
import {
	FAVORITE_MODELS_LOCALSTORAGE_KEY,
	HIDDEN_MODELS_LOCALSTORAGE_KEY,
	MODEL_GROUP_OPEN_LOCALSTORAGE_KEY,
	RECENT_MODEL_LIMIT,
	RECENT_MODELS_LOCALSTORAGE_KEY
} from '$lib/constants';
import { ServerModelStatus } from '$lib/enums';
import { HuggingFaceService } from '$lib/services/huggingface.service';
import { ModelsService } from '$lib/services/models.service';
// direct imports between stores, not via the barrel, to avoid circular deps
import { conversationsStore } from '$lib/stores/conversations/index.svelte';
import { type ModelPropsHost, ModelPropsManager } from '$lib/stores/models/props.svelte';
import { type ModelStatusHost, ModelStatusManager } from '$lib/stores/models/status.svelte';
import { serverStore } from '$lib/stores/server.svelte';
import type { ParsedModelId } from '$lib/types/models';
import { getConversationModel } from '$lib/utils/conversation-utils';
import { repoOf } from '$lib/utils/model-names';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
import { toast } from 'svelte-sonner';

/** Group open states kept before the oldest ones fall off; the map only ever grows. */
const MAX_GROUP_OPEN_ENTRIES = 200;
/**
 * Repos whose Hub details are prefetched on load. The Hub rate limits an
 * unauthenticated client aggressively, so the warm slice stays far below the
 * list window and the rest of the rows fetch as they scroll near the viewport.
 */
const HUB_WARM_LIMIT = 12;

/** Models kept out of the selector. */
function loadHiddenModels(): Set<string> {
	if (!browser) return new SvelteSet<string>();

	try {
		const raw = localStorage.getItem(HIDDEN_MODELS_LOCALSTORAGE_KEY);

		return raw ? new SvelteSet(JSON.parse(raw) as string[]) : new SvelteSet<string>();
	} catch {
		return new SvelteSet<string>();
	}
}

/** Open state the user set for a section or family of the model lists. */
function loadGroupOpenState(): SvelteMap<string, boolean> {
	if (!browser) return new SvelteMap<string, boolean>();

	try {
		const raw = localStorage.getItem(MODEL_GROUP_OPEN_LOCALSTORAGE_KEY);
		const parsed = raw ? (JSON.parse(raw) as unknown) : null;
		const entries =
			parsed && typeof parsed === 'object' ? Object.entries(parsed as Record<string, unknown>) : [];

		return new SvelteMap(
			entries.filter((entry): entry is [string, boolean] => typeof entry[1] === 'boolean')
		);
	} catch {
		return new SvelteMap<string, boolean>();
	}
}

/** Recently used backend-qualified ids, most recent first. */
function loadRecentModels(): string[] {
	if (!browser) return [];

	try {
		const raw = localStorage.getItem(RECENT_MODELS_LOCALSTORAGE_KEY);

		if (!raw) return [];

		const parsed = JSON.parse(raw) as unknown;

		return Array.isArray(parsed)
			? parsed.filter((id): id is string => typeof id === 'string').slice(0, RECENT_MODEL_LIMIT)
			: [];
	} catch {
		return [];
	}
}

class ModelsStore implements ModelPropsHost, ModelStatusHost {
	error = $state<string | null>(null);
	favoriteModelIds = $state<Set<string>>(this.loadFavoritesFromStorage());
	groupOpenState = $state<SvelteMap<string, boolean>>(loadGroupOpenState());
	hiddenModelIds = $state<Set<string>>(loadHiddenModels());
	loading = $state(false);
	models = $state<ModelOption[]>([]);
	recentModelIds = $state<string[]>(loadRecentModels());
	routerModels = $state<ApiModelDataEntry[]>([]);
	selectedModelId = $state<string | null>(null);
	selectedModelName = $state<string | null>(null);

	updating = $state(false);

	/** Per-model props cache, modalities and thinking detection, composed here. */
	private _props = new ModelPropsManager(this);

	/** Load/unload operations and the /models/sse status feed, composed here. */
	private _status = new ModelStatusManager(this);

	// every caller awaits the same inflight promise
	// Without this, ?model=<name> URL handler races an in-progress fetch and sees an empty list.
	private inflightFetch: Promise<void> | null = null;

	/**
	 * Model the active conversation view resolves to. Router mode: the user's
	 * selection first, then the conversation's own model. Otherwise the single
	 * served model, from the models list or the server props as a fallback.
	 */
	get activeModelId(): string | null {
		if (!serverStore.isRouterMode) {
			return this.models.length > 0 ? this.models[0].model : this.singleModelName;
		}

		if (this.selectedModelId) {
			const selected = this.models.find((m) => m.id === this.selectedModelId);

			if (selected) return selected.model;
		}

		const conversationModel = getConversationModel(conversationsStore.activeMessages);

		if (conversationModel) {
			const model = this.models.find((m) => m.model === conversationModel);

			if (model) return model.model;
		}

		return null;
	}

	get loadedModelIds(): string[] {
		return this.routerModels
			.filter(
				(m) =>
					m.status.value === ServerModelStatus.LOADED ||
					m.status.value === ServerModelStatus.SLEEPING
			)
			.map((m) => m.id);
	}

	get props() {
		return this._props;
	}

	get selectedModel(): ModelOption | null {
		if (!this.selectedModelId) return null;

		return this.models.find((m) => m.id === this.selectedModelId) ?? null;
	}

	get selectedModelContextSize(): number | null {
		if (!this.selectedModelName) return null;

		return this.props.getModelContextSize(this.selectedModelName);
	}

	/**
	 * Get model name in MODEL mode (single model).
	 * Extracts from model_path or model_alias from server props.
	 * In ROUTER mode, returns null (model is per-conversation).
	 */
	get singleModelName(): string | null {
		if (serverStore.isRouterMode) return null;

		const props = serverStore.props;

		if (props?.model_alias) return props.model_alias;

		if (!props?.model_path) return null;

		return props.model_path.split(/(\\|\/)/).pop() || null;
	}

	get status() {
		return this._status;
	}

	clearSelection(): void {
		this.selectedModelId = null;
		this.selectedModelName = null;
	}

	/** Family keys folded away under one section, for a list that restores them. */
	collapsedGroupsUnder(prefix: string): string[] {
		const head = `${prefix}-`;

		return [...this.groupOpenState]
			.filter(([id, open]) => !open && id.startsWith(head))
			.map(([id]) => id.slice(head.length));
	}

	/**
	 * Auto-selects the first available model if none is selected.
	 * Prioritizes:
	 * 1. Model from active conversation's last assistant response (if loaded)
	 * 2. Model from active conversation's last assistant response (if not loaded)
	 * 3. First loaded model (not from active conversation)
	 * 4. A favorite model
	 * 5. First available model
	 */
	async ensureFirstModelSelected(): Promise<void> {
		if (this.selectedModelName) return;

		const availableModels = this.getVisibleModels();

		if (availableModels.length === 0) return;

		// Try to select model from last assistant response first
		const lastModel = this.getModelFromLastAssistantResponse();

		if (lastModel) {
			const lastModelOption = availableModels.find((m) => m.model === lastModel);

			if (lastModelOption) {
				await this.selectModelById(lastModelOption.id);

				if (this.isModelLoaded(lastModel)) {
					await this.props.fetchModelProps(lastModel);
				}

				return;
			}
		}

		// Try a loaded model first
		const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model));

		if (loadedModel) {
			await this.selectModelById(loadedModel.id);
			await this.props.fetchModelProps(loadedModel.model);

			return;
		}

		// favorites are shared across backends, so a stored id may belong to another one
		const favorite = this.favoriteModelIds.values().next()?.value;
		const favoriteOption = favorite
			? availableModels.find((m) => m.id === favorite || m.model === favorite)
			: undefined;

		if (favoriteOption) {
			await this.selectModelById(favoriteOption.id);

			return;
		}

		// Fall back to the first available model
		await this.selectModelById(availableModels[0].id);
	}

	/**
	 * Fetch list of models from server and detect server role.
	 * Also fetches modalities for MODEL mode (single model).
	 */
	async fetch(force = false): Promise<void> {
		if (this.inflightFetch) return this.inflightFetch;

		if (this.models.length > 0 && !force) return;

		this.inflightFetch = this.runFetch();
		try {
			await this.inflightFetch;
		} finally {
			this.inflightFetch = null;
		}
	}

	/**
	 * Fetch models with full metadata (ROUTER mode only).
	 * No-op in MODEL mode - fetch() already calls list() internally.
	 * Kept for API compatibility (e.g. handleOpenChange dropdown open handler).
	 */
	async fetchRouterModels(): Promise<void> {
		if (!serverStore.isRouterMode) return;

		try {
			const response = await ModelsService.list();

			this.routerModels = response.data;
			// a download an earlier run left unfinished is listed as incomplete, and
			// is offered as paused until it is resumed
			this._status.syncPausedDownloads();
			// keep the selector options in sync: a downloaded / deleted model shows
			// up here too, not only in the router model rows
			this.models = this.buildModelOptions(response);
			this.warmHubDetails();
			await this.props.fetchModalitiesForLoadedModels();

			const visible = this.getVisibleModels();

			if (visible.length === 1 && this.isModelLoaded(visible[0].model)) {
				this.selectModelById(visible[0].id);
			}
		} catch (error) {
			console.warn('Failed to fetch router models:', error);
			this.routerModels = [];
		}
	}

	findModelById(modelId: string): ModelOption | null {
		return this.models.find((model) => model.id === modelId) ?? null;
	}

	findModelByName(modelName: string): ModelOption | null {
		return (
			this.models.find(
				(model) =>
					model.model === modelName || model.id === modelName || model.aliases?.includes(modelName)
			) ?? null
		);
	}

	/**
	 * Gets the model name from the last assistant message in the active conversation.
	 * Used by both the chat page and settings page to maintain model consistency.
	 */
	getModelFromLastAssistantResponse(): string | null {
		const messages = conversationsStore.activeMessages;

		if (!messages || messages.length === 0) return null;

		for (let i = messages.length - 1; i >= 0; i--) {
			if (messages[i].model) {
				return messages[i].model;
			}
		}

		return null;
	}

	getModelStatus(modelId: string): ServerModelStatus | null {
		const model = this.routerModels.find((m) => m.id === modelId);

		return (model?.status?.value as ServerModelStatus) ?? null;
	}

	hasModel(modelName: string): boolean {
		return this.models.some((model) => model.model === modelName);
	}

	isFavorite(modelId: string): boolean {
		return this.favoriteModelIds.has(modelId);
	}

	isGroupOpen(id: string, fallbackOpen: boolean): boolean {
		return this.groupOpenState.get(id) ?? fallbackOpen;
	}

	isHidden(modelId: string): boolean {
		return this.hiddenModelIds.has(modelId);
	}

	isModelLoaded(modelId: string): boolean {
		const status = this.getModelStatus(modelId);

		return status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING;
	}

	/** Loaded, or sleeping, and not mid-operation: what a row shows as running. */
	isModelRunning(modelId: string): boolean {
		return this.isModelLoaded(modelId) && !this._status.isOperationInProgress(modelId);
	}

	/**
	 * Select a model. `recordRecent` keeps automatic picks out of the recency list.
	 */
	async selectModelById(modelId: string, options?: { recordRecent?: boolean }): Promise<void> {
		if (!modelId || this.updating) return;

		if (this.selectedModelId === modelId) {
			if (options?.recordRecent) this.recordRecentModel(modelId);

			return;
		}

		const option = this.models.find((model) => model.id === modelId);

		if (!option) throw new Error('Selected model is not available');

		this.updating = true;
		this.error = null;

		try {
			this.selectedModelId = option.id;
			this.selectedModelName = option.model;

			if (options?.recordRecent) this.recordRecentModel(modelId);
		} finally {
			this.updating = false;
		}
	}

	/**
	 * Select a model by its model name (used for syncing with conversation model).
	 */
	selectModelByName(modelName: string): void {
		const option = this.models.find((model) => model.model === modelName);

		if (option) {
			// the updating guard inside selectModelById cannot refuse this call: that
			// call is the only one that raises the flag, and it clears it before it returns
			void this.selectModelById(option.id);
		}
	}

	/**
	 * Auto-selects the model from the last assistant response if available and loaded.
	 * Returns true if a model was selected, false otherwise.
	 */
	async selectModelFromLastAssistantResponse(): Promise<boolean> {
		const lastModel = this.getModelFromLastAssistantResponse();

		if (!lastModel || this.selectedModelName === lastModel) return false;

		const matchingModel = this.models.find((option) => option.model === lastModel);

		if (!matchingModel || !this.isModelLoaded(lastModel)) return false;

		try {
			await this.selectModelById(matchingModel.id);
			console.log(`[modelsStore] Automatically selected model: ${lastModel} from last message`);

			return true;
		} catch (error) {
			console.warn('[modelsStore] Failed to automatically select model from last message:', error);

			return false;
		}
	}

	/** Add or remove several models at once, e.g. every quant of a family. */
	setFavorites(modelIds: string[], favorite: boolean): void {
		const next = new SvelteSet(this.favoriteModelIds);

		for (const modelId of modelIds) {
			if (favorite) {
				next.add(modelId);
			} else {
				next.delete(modelId);
			}
		}

		this.favoriteModelIds = next;

		try {
			localStorage.setItem(FAVORITE_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next]));
		} catch {
			toast.error('Failed to save favorite models to local storage');
		}
	}

	/** Persist one family of a section, e.g. a folded `Qwen` under the local models. */
	setGroupCollapsed(prefix: string, key: string, collapsed: boolean): void {
		this.setGroupOpen(`${prefix}-${key}`, !collapsed);
	}

	setGroupOpen(id: string, open: boolean): void {
		const next = new SvelteMap(this.groupOpenState);

		// re-set moves the id to the end, so the oldest states fall off first
		next.delete(id);
		next.set(id, open);

		while (next.size > MAX_GROUP_OPEN_ENTRIES) {
			const oldest = next.keys().next();

			if (oldest.done) break;

			next.delete(oldest.value);
		}

		this.groupOpenState = next;

		try {
			localStorage.setItem(
				MODEL_GROUP_OPEN_LOCALSTORAGE_KEY,
				JSON.stringify(Object.fromEntries(next))
			);
		} catch {
			toast.error('Failed to save the model list state to local storage');
		}
	}

	toDisplayName(id: string): string {
		const segments = id.split(/\\|\//);
		const candidate = segments.pop();

		return candidate && candidate.trim().length > 0 ? candidate : id;
	}

	toggleFavorite(modelId: string): void {
		this.setFavorites([modelId], !this.favoriteModelIds.has(modelId));
	}

	/** Models hidden from the selector stay in the manager, flagged and unhideable. */
	toggleHidden(modelId: string): void {
		const next = new SvelteSet(this.hiddenModelIds);

		if (next.has(modelId)) {
			next.delete(modelId);
		} else {
			next.add(modelId);
		}

		this.hiddenModelIds = next;

		try {
			localStorage.setItem(HIDDEN_MODELS_LOCALSTORAGE_KEY, JSON.stringify([...next]));
		} catch {
			toast.error('Failed to save hidden models to local storage');
		}
	}

	/**
	 * Warm the Hub record of the local repos, so a list can read the context and size
	 * the server reports only once a model is loaded.
	 */
	warmHubDetails(): void {
		if (!HuggingFaceService.isEnabled()) return;

		const repos: string[] = [];

		for (const option of this.models) {
			const repo = repoOf(option.model);

			if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo);
		}

		// the Hub rate limits aggressively and an unauthenticated burst of requests
		// leaves every detail uncached for a retry on the next mount, so warm only the
		// rows the lists show at rest and let the rest arrive near the viewport
		for (const repo of repos.slice(0, HUB_WARM_LIMIT)) {
			void HuggingFaceService.getDetails(repo).catch(() => {});
		}
	}

	/**
	 * Build ModelOption[] from an API response.
	 * Both MODEL and ROUTER modes share the same mapping logic;
	 * they differ only in which endpoint is called.
	 */
	private buildModelOptions(response: ApiModelsListResponse): ModelOption[] {
		const entries: {
			details?: ApiModelsListResponse['models'][number];
			item: ApiModelDataEntry;
			parsed: ParsedModelId;
		}[] = response.data.map((item: ApiModelDataEntry, index: number) => ({
			details: response.models?.[index],
			item,
			parsed: ModelsService.parseModelId(item.id)
		}));

		return (
			entries
				// a models dir can still name an entry after a projector file, which is
				// no loadable model
				.filter(({ parsed }) => !parsed.sidecar)
				// in-flight and unfinished downloads are not usable models yet; the
				// selector tracks them in its "Download in progress" section instead
				.filter(
					({ item }) =>
						item.status?.value !== ServerModelStatus.DOWNLOADING &&
						item.status?.value !== ServerModelStatus.INCOMPLETE
				)
				.map(({ details, item }) => {
					const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : [];
					const displayNameSource =
						details?.name && details.name.trim().length > 0 ? details.name : item.id;
					const modelId = details?.model || item.id;

					return {
						aliases: item.aliases ?? [],
						capabilities: rawCapabilities.filter((value: unknown): value is string =>
							Boolean(value)
						),
						// 0 is the server's way of leaving the trained context unknown; a
						// model-mode listing reports it as meta.n_ctx_train instead
						contextLength:
							item.context_length ||
							(typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0
								? item.meta.n_ctx_train
								: undefined),
						description: details?.description,
						details: details?.details,
						draftSidecars: this._status.getDraftSidecars(modelId),
						id: item.id,
						meta: item.meta ?? null,
						modalities: this.props.buildArchitectureModalities(item.architecture),
						model: modelId,
						name: this.toDisplayName(displayNameSource),
						parsedId: ModelsService.parseModelId(modelId),
						tags: item.tags ?? []
					};
				})
		);
	}

	/** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */
	private async fetchModelModeInternal(): Promise<ModelOption[]> {
		const response = await ModelsService.list();

		return this.buildModelOptions(response);
	}
	/**
	 * Filter to models visible in the UI (ui !== false).
	 */
	private getVisibleModels(): ModelOption[] {
		return this.models.filter((option) => this.props.getModelProps(option.model)?.ui !== false);
	}

	private loadFavoritesFromStorage(): Set<string> {
		try {
			const raw = localStorage.getItem(FAVORITE_MODELS_LOCALSTORAGE_KEY);

			return raw ? new Set(JSON.parse(raw) as string[]) : new Set();
		} catch {
			toast.error('Failed to load favorite models from local storage');

			return new Set();
		}
	}

	/** Move a model to the front of the recently used list. */
	private recordRecentModel(qualifiedId: string): void {
		this.recentModelIds = [
			qualifiedId,
			...this.recentModelIds.filter((id) => id !== qualifiedId)
		].slice(0, RECENT_MODEL_LIMIT);

		if (!browser) return;

		try {
			localStorage.setItem(RECENT_MODELS_LOCALSTORAGE_KEY, JSON.stringify(this.recentModelIds));
		} catch {
			console.warn('[ModelsStore] Failed to persist the recently used models');
		}
	}

	private async runFetch(): Promise<void> {
		this.loading = true;
		this.error = null;

		try {
			if (!serverStore.props) {
				await serverStore.fetch();
			}

			const router = serverStore.isRouterMode;

			if (router) {
				const response = await ModelsService.list();

				this.routerModels = response.data;
				this.models = this.buildModelOptions(response);

				await this.props.fetchModalitiesForLoadedModels();

				const visible = this.getVisibleModels();

				if (visible.length === 1 && this.isModelLoaded(visible[0].model)) {
					this.selectModelById(visible[0].id);
				}
			} else {
				this.models = await this.fetchModelModeInternal();
			}
		} catch (error) {
			this.models = [];
			this.error = error instanceof Error ? error.message : 'Failed to load models';

			throw error;
		} finally {
			this.loading = false;
		}
	}
}

export const modelsStore = new ModelsStore();