Skip to content

LLM inference in C/C++

C++ ggml
6.2 KB
Raw Blame
import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants';
import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';
import { HuggingFaceService, ModelsService } from '$lib/services';
import { modelsStore } from '$lib/stores';
import type {
	ModelDownloadEntry,
	ModelDownloadProgress,
	ModelOption,
	ModelSidecarBadge
} from '$lib/types/models';
import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils';

/** One repo of the table, with the rows it ships as. */
export interface ModelQuantGroup {
	base: ModelOption;
	key: string;
	kind: ModelGroupKind;
	quants: ModelOption[];
	/** Draft sidecars already on disk, shown as checked sub-lines. */
	sidecarBadges?: ModelSidecarBadge[];
}

/** True when any of the manager's filters narrows the table. */
export function hasActiveFilters(
	contextLimit: number,
	modalities: ModalityKey[],
	capabilities: ModelCapability[]
): boolean {
	return contextLimit > 0 || modalities.length > 0 || capabilities.length > 0;
}

/** Byte counts of a tracked download: live while it runs, frozen while paused. */
export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress | null {
	return (
		modelsStore.status.getDownloadProgress(repoWithTag) ??
		modelsStore.status.getPausedDownloadProgress(repoWithTag)
	);
}

/**
 * One table row per tracked download, listed like any other model. A paused download
 * is already in the router's listing, so its own option is reused; a fresh one is
 * synthesized for its tag alone.
 */
export function downloadGroups(
	entries: ModelDownloadEntry[],
	models: ModelOption[],
	sidecarsFor: (repo: string) => ModelSidecarBadge[]
): ModelQuantGroup[] {
	return entries.map((entry) => {
		const listed = models.find((candidate) => candidate.model === entry.repoWithTag);
		// the synthesized option carries backendId, which ModelOption drops on
		// assignment, so the callers spread it instead of typing this as one
		const option = listed ?? {
			backendId: LOCAL_BACKEND_ID,
			capabilities: [],
			id: entry.repoWithTag,
			model: entry.repoWithTag,
			name: entry.repoWithTag
		};
		// a draft still coming down badges the id, an on-disk one keeps its sub-line
		const sidecars = sidecarsFor(repoOf(entry.repoWithTag));
		const pending = sidecars.filter((badge) => badge.pending);

		return {
			base: { ...option, draftSidecars: pending },
			key: `download::${entry.repoWithTag}`,
			kind: ModelGroupKind.QUANTS,
			quants: [{ ...option, draftSidecars: pending }],
			sidecarBadges: sidecars.filter((badge) => !badge.pending)
		};
	});
}

/** One collapsible block of the manager's table. */
export interface ModelsTableGroup {
	/** One entry per repo, its quants hanging off it. */
	items: ModelQuantGroup[];
	key: string;
	kind: ModelsTableGroupKind;
	label: string;
}

/** Context the model runs with: what a loaded model reports. */
export function configuredContext(option: ModelOption): number | null {
	return modelsStore.isModelRunning(option.model)
		? modelsStore.props.getModelContextSize(option.model)
		: null;
}

/**
 * Capability a model reports: true or false once its listing or its chat template
 * answers, null while the Hub record that carries the template is not read yet.
 */
export function modelCapability(option: ModelOption, capability: ModelCapability): boolean | null {
	if (option.capabilities.includes(capability)) return true;

	const details = HuggingFaceService.cachedDetails(option.model);

	// only the chat template answers this, and it arrives with the Hub record
	if (details === undefined) return null;

	const template = details?.gguf?.chat_template ?? '';

	return capability === ModelCapability.TOOL_USE
		? detectToolUseSupport(template)
		: detectThinkingSupport(template);
}

/**
 * Context a model reports: the provider listing first, then the cached Hub record.
 * Null while neither of them has answered.
 */
export function modelContextLength(option: ModelOption): number | null {
	return (
		option.contextLength ??
		HuggingFaceService.cachedDetails(option.model)?.gguf?.context_length ??
		null
	);
}

/** Fold the quants of one repo into a single entry, so the table shows one row per model. */
export function groupModelQuants(models: ModelOption[]): ModelQuantGroup[] {
	const groups = new Map<string, ModelQuantGroup>();

	for (const option of models) {
		const key = repoOf(option.model);
		const group = groups.get(key);

		if (group) {
			group.quants.push(option);

			continue;
		}

		groups.set(key, { base: option, key, kind: ModelGroupKind.QUANTS, quants: [option] });
	}

	return Array.from(groups.values()).flatMap((group) => {
		const kind = groupKind(group.quants);
		const modelIds = group.quants.map((option) => option.model);

		// the very same id twice is not a quant set; keep those rows apart
		if (modelIds.length > 1 && modelIds.every((model) => model === modelIds[0])) {
			return group.quants.map((option) => ({
				...group,
				base: option,
				key: `${group.key}::${option.id}`,
				kind: ModelGroupKind.VARIANTS,
				quants: [option]
			}));
		}

		return [{ ...group, kind }];
	});
}

/** What a group folds: quants or variants of one repo. */
function groupKind(quants: ModelOption[]): ModelGroupKind {
	const isQuant = quants.every(
		(option) => (option.parsedId ?? ModelsService.parseModelId(option.model)).quantization
	);

	return isQuant ? ModelGroupKind.QUANTS : ModelGroupKind.VARIANTS;
}

/**
 * Split the repos of a section into the quants that are hidden and the ones that
 * are not, so a partly hidden repo lists in both blocks instead of dragging its
 * visible quants into the hidden one.
 */
export function splitHiddenQuants(
	entries: ModelQuantGroup[],
	isHidden: (option: ModelOption) => boolean
): { hidden: ModelQuantGroup[]; local: ModelQuantGroup[] } {
	const hidden: ModelQuantGroup[] = [];
	const local: ModelQuantGroup[] = [];

	for (const entry of entries) {
		const hiddenQuants = entry.quants.filter((quant) => isHidden(quant));
		const localQuants = entry.quants.filter((quant) => !isHidden(quant));

		if (hiddenQuants.length > 0) {
			hidden.push({
				...entry,
				base: hiddenQuants[0],
				key: `${entry.key}::hidden`,
				quants: hiddenQuants
			});
		}

		if (localQuants.length > 0) {
			local.push({ ...entry, base: localQuants[0], quants: localQuants });
		}
	}

	return { hidden, local };
}