LLM inference in C/C++
C++ ggml
ui : model download pipeline (#27959)
Aleksander Grygier8664eaea3038
diff --git a/tools/ui/src/app.d.ts b/tools/ui/src/app.d.tsindex 0a206381f8..5c039063ad 100644--- a/tools/ui/src/app.d.ts+++ b/tools/ui/src/app.d.ts@@ -19,6 +19,8 @@ import type {ApiLlamaCppServerProps,ApiModelDataEntry,ApiModelLoadStage,+ ApiModelsDownloadRequest,+ ApiModelsDownloadResponse,ApiModelsListResponse,ApiModelsLoadResponse,ApiModelsSseData,@@ -80,11 +82,15 @@ declare global {ApiLlamaCppServerProps,ApiModelDataEntry,ApiModelLoadStage,+ ModelDownloadProgress,ApiModelsSseProgress,ApiModelsSseData,+ ApiModelsSseDownloadProgressData,ApiModelsSseEvent,ApiModelsListResponse,ApiModelsLoadResponse,+ ApiModelsDownloadRequest,+ ApiModelsDownloadResponse,ApiModelsUnloadResponse,ApiProcessingState,// Chat typesdiff --git a/tools/ui/src/lib/constants/api-endpoints.constants.ts b/tools/ui/src/lib/constants/api-endpoints.constants.tsindex 8611d49fbb..2f6efb94cb 100644--- a/tools/ui/src/lib/constants/api-endpoints.constants.ts+++ b/tools/ui/src/lib/constants/api-endpoints.constants.ts@@ -1,4 +1,7 @@export const API_MODELS = {+ /** Download a model from HuggingFace (ROUTER mode, POST) or cancel/remove it (DELETE) */+ DELETE: '/models',+ DOWNLOAD: '/models',LIST: '/v1/models',LOAD: '/models/load',SSE: '/models/sse',diff --git a/tools/ui/src/lib/constants/cli-flags.constants.ts b/tools/ui/src/lib/constants/cli-flags.constants.tsindex c4af2b6f46..0b6b2af86e 100644--- a/tools/ui/src/lib/constants/cli-flags.constants.ts+++ b/tools/ui/src/lib/constants/cli-flags.constants.ts@@ -2,6 +2,12 @@ export const CLI_FLAGS = {AGENT: '--agent',API_KEY: '--api-key',MCP_PROXY: '--ui-mcp-proxy',+ /** Multimodal projector path; unlocks vision/audio for the model. */+ MMPROJ: '--mmproj',+ /** Draft model weights path (long form); the router records it per model. */+ MODEL_DRAFT: '--model-draft',+ /** Draft model weights path (short form). */+ MODEL_DRAFT_SHORT: '-md',SLOTS: '--slots',TOOLS: '--tools'} as const;diff --git a/tools/ui/src/lib/constants/storage.constants.ts b/tools/ui/src/lib/constants/storage.constants.tsindex 0aad7c7706..5e6610d859 100644--- a/tools/ui/src/lib/constants/storage.constants.ts+++ b/tools/ui/src/lib/constants/storage.constants.ts@@ -15,6 +15,9 @@ export const STORAGE_APP_NAME_DEPRECATED = 'LlamaCppWebui';export const DB_APP_NAME_DEPRECATED = 'LlamacppWebui';export const ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.alwaysAllowedTools`;++/** Paused model download ids (`<repo>:<tag>`), restored on the next page load. */+export const PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.pausedModelDownloads`;export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`;export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`;diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.tsindex db4f5b68cb..947d1c757e 100644--- a/tools/ui/src/lib/enums/index.ts+++ b/tools/ui/src/lib/enums/index.ts@@ -71,6 +71,8 @@ export {export { ModelAuxSidecar, ModelCapability, ModelDraftSidecar, ModelModality } from './model.enums';+export { ModelDownloadStopRequest } from './model.enums';+export { ServerRole, ServerModelStatus, ServerModelsSseEventType } from './server.enums';export { ParameterSource, SyncableParameterType, SettingsFieldType } from './settings.enums';diff --git a/tools/ui/src/lib/enums/model.enums.ts b/tools/ui/src/lib/enums/model.enums.tsindex 177fffe19a..2440dc007b 100644--- a/tools/ui/src/lib/enums/model.enums.ts+++ b/tools/ui/src/lib/enums/model.enums.ts@@ -35,3 +35,13 @@ export enum ModelAuxSidecar {/** Multimodal projector: unlocks vision and/or audio input modalities. */MMPROJ = 'mmproj'}++/**+ * Why an in-flight download is being stopped, so the terminal `download_failed`+ * feed event can be attributed: a user pause (resumable) or a user cancel+ * (discard). Distinguishes these from a genuine download failure.+ */+export enum ModelDownloadStopRequest {+ CANCEL = 'cancel',+ PAUSE = 'pause'+}diff --git a/tools/ui/src/lib/enums/server.enums.ts b/tools/ui/src/lib/enums/server.enums.tsindex b7e80433c6..b558b51236 100644--- a/tools/ui/src/lib/enums/server.enums.ts+++ b/tools/ui/src/lib/enums/server.enums.ts@@ -13,6 +13,10 @@ export enum ServerRole {* Used as the `value` field in the status object from /models endpoint*/export enum ServerModelStatus {+ DOWNLOAD_FAILED = 'download_failed',+ DOWNLOAD_FINISHED = 'download_finished',+ DOWNLOADED = 'downloaded',+ DOWNLOADING = 'downloading',FAILED = 'failed',LOADED = 'loaded',LOADING = 'loading',@@ -26,6 +30,8 @@ export enum ServerModelStatus {* tools/server/server-models.cpp from the C++ server.*/export enum ServerModelsSseEventType {+ DOWNLOAD_FAILED = 'download_failed',+ DOWNLOAD_FINISHED = 'download_finished',DOWNLOAD_PROGRESS = 'download_progress',MODEL_REMOVE = 'model_remove',MODEL_STATUS = 'model_status',diff --git a/tools/ui/src/lib/services/models.service.ts b/tools/ui/src/lib/services/models.service.tsindex 21a9b22a16..96902a0b9d 100644--- a/tools/ui/src/lib/services/models.service.ts+++ b/tools/ui/src/lib/services/models.service.ts@@ -11,11 +11,13 @@ import { API_MODELS, MODEL_ID, type ModelSidecar } from '$lib/constants';import { ServerModelStatus } from '$lib/enums';import type { ParsedModelId } from '$lib/types/models';import {+ apiDelete,apiFetch,apiPost,extractSseDataPayload,normalizeModelName,sidecarFromFileToken,+ sidecarFromTag,splitSseRecords} from '$lib/utils';import { getAuthHeaders } from '$lib/utils/api-headers';@@ -24,14 +26,8 @@ export class ModelsService {private static readonly SSE_RECONNECT_MS = 1000;/**- * Build the `<repo>:<tag>` string expected by POST /models from a parsed- * filename quant + optional sidecar type. Used by the model download- * dialog so callers don't have to know about the tag conventions.- *- * @param repoId - HuggingFace repo id (e.g. `ggml-org/gemma-3-4b-it-GGUF`)- * @param quant - Quantization token, e.g. `Q4_K_M`- * @param sidecar - Sidecar type, as its lowercase filename token (e.g. `mtp`)- * @returns Repo id possibly suffixed with `:tag`+ * Build the `<repo>:<tag>` string POST /models expects, so callers don't need+ * to know the tag conventions.*/static buildDownloadTag(repoId: string,@@ -47,6 +43,27 @@ export class ModelsService {return `${repoId}:${tag}`;}+ /**+ * Cancel an in-flight download, or remove a downloaded/failed entry from the+ * model cache (ROUTER mode only): DELETE /models?model=<repo:tag>.+ */+ static async cancelDownload(hfRepoWithTag: string): Promise<ApiModelsDownloadResponse> {+ return apiDelete<ApiModelsDownloadResponse>(API_MODELS.DELETE, {+ model: hfRepoWithTag+ });+ }++ /**+ * Start a model download from HuggingFace (ROUTER mode only). The response+ * returns immediately; progress arrives over /models/sse. The server picks+ * the file matching the tag and also pulls the model's mmproj/draft sidecars.+ */+ static async downloadModel(hfRepoWithTag: string): Promise<ApiModelsDownloadResponse> {+ const payload: ApiModelsDownloadRequest = { model: hfRepoWithTag };++ return apiPost<ApiModelsDownloadResponse>(API_MODELS.DOWNLOAD, payload);+ }+/*** Check if a model is loaded based on its metadata.*@@ -57,6 +74,10 @@ export class ModelsService {return model.status.value === ServerModelStatus.LOADED;}+ static isModelLoading(model: ApiModelDataEntry): boolean {+ return model.status.value === ServerModelStatus.LOADING;+ }+/****@@ -66,13 +87,15 @@ export class ModelsService {*//**- * Check if a model is currently loading.- *- * @param model - Model data entry from the API response- * @returns True if the model status is LOADING+ * True when a router entry id marks a downloaded sidecar file, e.g.+ * `org/model:Q4_0-mtp` or `org/model:mmproj`, not a loadable model.*/- static isModelLoading(model: ApiModelDataEntry): boolean {- return model.status.value === ServerModelStatus.LOADING;+ static isSidecarEntry(modelId: string): boolean {+ const idx = modelId.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR);++ if (idx === MODEL_ID.NOT_FOUND) return false;++ return sidecarFromTag(modelId.slice(idx + 1)) !== null;}/**diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.tsindex d25b59cfc2..d367ad9874 100644--- a/tools/ui/src/lib/stores/models/index.svelte.ts+++ b/tools/ui/src/lib/stores/models/index.svelte.ts@@ -204,6 +204,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {const response = await ModelsService.list();this.routerModels = response.data;+ // keep the selector options in sync: a downloaded / deleted model shows+ // up here too, not only in the router model rows+ this.models = this.buildModelOptions(response);await this.props.fetchModalitiesForLoadedModels();const visible = this.getVisibleModels();@@ -359,27 +362,44 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {* they differ only in which endpoint is called.*/private buildModelOptions(response: ApiModelsListResponse): ModelOption[] {- return response.data.map((item: ApiModelDataEntry, index: number) => {- const details = response.models?.[index];- const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : [];- const displayNameSource =- details?.name && details.name.trim().length > 0 ? details.name : item.id;- const modelId = details?.model || item.id;-- return {- aliases: item.aliases ?? [],- capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)),- description: details?.description,- details: details?.details,- id: item.id,- meta: item.meta ?? null,- modalities: this.props.buildArchitectureModalities(item.architecture),- model: modelId,- name: this.toDisplayName(displayNameSource),- parsedId: ModelsService.parseModelId(modelId),- tags: item.tags ?? []- };- });+ const entries: {+ details?: ApiModelsListResponse['models'][number];+ item: ApiModelDataEntry;+ }[] = response.data.map((item: ApiModelDataEntry, index: number) => ({+ details: response.models?.[index],+ item+ }));++ return (+ entries+ // sidecar entries mark downloaded sidecar files, not loadable models+ .filter(({ item }) => !ModelsService.isSidecarEntry(item.id))+ // in-flight downloads are not usable models yet; the selector tracks+ // them in its "Download in progress" section instead+ .filter(({ item }) => item.status?.value !== ServerModelStatus.DOWNLOADING)+ .map(({ details, item }) => {+ const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : [];+ const displayNameSource =+ details?.name && details.name.trim().length > 0 ? details.name : item.id;+ const modelId = details?.model || item.id;++ return {+ aliases: item.aliases ?? [],+ capabilities: rawCapabilities.filter((value: unknown): value is string =>+ Boolean(value)+ ),+ description: details?.description,+ details: details?.details,+ id: item.id,+ meta: item.meta ?? null,+ modalities: this.props.buildArchitectureModalities(item.architecture),+ model: modelId,+ name: this.toDisplayName(displayNameSource),+ parsedId: ModelsService.parseModelId(modelId),+ tags: item.tags ?? []+ };+ })+ );}/** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */diff --git a/tools/ui/src/lib/stores/models/status.svelte.ts b/tools/ui/src/lib/stores/models/status.svelte.tsindex d0160aa4da..b0ac083664 100644--- a/tools/ui/src/lib/stores/models/status.svelte.ts+++ b/tools/ui/src/lib/stores/models/status.svelte.ts@@ -1,18 +1,26 @@/**- * ModelStatusManager - Model load/unload operations and the /models/sse feed+ * ModelStatusManager - model load/unload operations and the /models/sse feed*- * Owns the status feed subscription, load progress tracking, and the- * awaiters that settle load/unload operations. The feed drives status and- * progress, so it replaces any post-operation polling. Created and owned by- * modelsStore; the host owns the router model rows the feed updates.+ * The feed drives status and progress, replacing any post-operation polling.+ * Created and owned by modelsStore, which also owns the router rows it updates.*/-import { ServerModelsSseEventType, ServerModelStatus } from '$lib/enums';+import {+ CLI_FLAGS,+ HF_UD_QUANT_PREFIX_REGEX,+ MODEL_ID,+ PATH_SEPARATOR,+ PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY+} from '$lib/constants';+import { ModelDownloadStopRequest, ServerModelsSseEventType, ServerModelStatus } from '$lib/enums';+import { HuggingFaceService } from '$lib/services/huggingface.service';import { ModelsService } from '$lib/services/models.service';import type { ModelPropsManager } from '$lib/stores/models/props.svelte';// direct imports between stores, not via the barrel, to avoid circular depsimport { serverStore } from '$lib/stores/server.svelte';-import { SvelteMap } from 'svelte/reactivity';+// explicit type imports: the app.d.ts globals resolve to `any`, so import the real types+import type { ApiModelsSseDownloadProgressData, ModelDownloadProgress } from '$lib/types';+import { SvelteMap, SvelteSet } from 'svelte/reactivity';import { toast } from 'svelte-sonner';/**@@ -30,9 +38,59 @@ export interface ModelStatusHost {toDisplayName(id: string): string;}+/**+ * Comparison key of a `<repo>:<tag>` download identifier: uppercased, with the+ * `UD-` quant prefix stripped. The router derives cached model names from the+ * actual file, which drops the prefix, so `repo:UD-Q4_K_XL` and `repo:Q4_K_XL`+ * must compare equal.+ */+function downloadIdKey(repoWithTag: string): string {+ const idx = repoWithTag.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR);+ const repo = idx === -1 ? repoWithTag : repoWithTag.slice(0, idx);+ const tag = idx === -1 ? '' : repoWithTag.slice(idx + 1);++ return `${repo.toUpperCase()}:${tag.toUpperCase().replace(HF_UD_QUANT_PREFIX_REGEX, '')}`;+}+export class ModelStatusManager {+ /**+ * Sidecar files pulled by registered models, as `<repo>/<file>` keys.+ * Sidecars are not separate /v1/models entries - the router pulls them as+ * sidecars of a main model and records them in its `--model-draft` /+ * `--mmproj` args.+ */+ private downloadedSidecars = $derived.by(() => {+ const result = new SvelteSet<string>();++ for (const m of this.host.routerModels) {+ const args = m.status?.args;++ if (!args) continue;++ for (let i = 0; i < args.length - 1; i++) {+ if (+ args[i] !== CLI_FLAGS.MODEL_DRAFT &&+ args[i] !== CLI_FLAGS.MODEL_DRAFT_SHORT &&+ args[i] !== CLI_FLAGS.MMPROJ+ ) {+ continue;+ }++ const parsed = HuggingFaceService.parseCachePath(args[i + 1]);++ if (parsed) result.add(`${parsed.repo}${PATH_SEPARATOR}${parsed.file}`);+ }+ }++ return result;+ });+ private downloadProgress = new SvelteMap<string, ModelDownloadProgress>();+ /** `<repo>:<tag>` strings whose most recent download attempt failed (download_failed). */+ private failedDownloads = new SvelteSet<string>();private loadingStates = new SvelteMap<string, boolean>();private loadProgress = new SvelteMap<string, ModelLoadProgress>();+ /** Paused downloads with their last reported progress, or null when none arrived before the pause. */+ private pausedDownloads = new SvelteMap<string, ModelDownloadProgress | null>();// /models/sse feed state, the single source of truth for status and load progressprivate statusAbort: AbortController | null = null;private statusReaderActive = false;@@ -40,8 +98,127 @@ export class ModelStatusManager {string,{ target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void }>();+ /** Tags the user asked to stop (pause or cancel); the download_failed the stop triggers is intentional, not a failure. */+ private stopRequests = new SvelteMap<string, ModelDownloadStopRequest>();++ /**+ * Cancel an in-flight download or remove a downloaded/failed entry from+ * the cache (ROUTER mode only).+ */+ async cancelDownload(repoWithTag: string): Promise<boolean> {+ if (!serverStore.isRouterMode) {+ toast.error('Model downloads are only available in router mode');++ return false;+ }++ this.subscribe();- constructor(private host: ModelStatusHost) {}+ // in-flight: the kill triggers download_failed over the feed; mark it as a+ // user cancel so it settles silently instead of toasting a failure+ if (this.downloadProgress.has(repoWithTag)) {+ this.stopRequests.set(repoWithTag, ModelDownloadStopRequest.CANCEL);+ }++ // a downloaded model registers under the name the router derived from the+ // cached file (e.g. the UD- quant prefix is dropped), so resolve the tag to+ // the registered id before asking the server to remove it+ const registeredId =+ this.host.routerModels.find((m) => downloadIdKey(m.id) === downloadIdKey(repoWithTag))?.id ??+ repoWithTag;++ try {+ const res = await ModelsService.cancelDownload(registeredId);+ const ok = res.success === true;++ if (ok) {+ this.downloadProgress.delete(repoWithTag);+ this.failedDownloads.delete(repoWithTag);+ this.deletePausedDownload(repoWithTag);+ }++ return ok;+ } catch (error) {+ toast.error(`Failed to cancel: ${error instanceof Error ? error.message : 'unknown error'}`);++ return false;+ }+ }++ /**+ * The server force-kills a LOADING model on unload, and the feed reports the+ * settled status, so no waiter is registered here.+ */+ async cancelLoad(modelId: string): Promise<void> {+ if (!serverStore.isRouterMode) return;++ this.subscribe();++ try {+ await ModelsService.unload(modelId);+ toast.info(`Load cancelled: ${this.host.toDisplayName(modelId)}`);+ } catch (error) {+ toast.error(`Failed to cancel load: ${this.host.toDisplayName(modelId)}`);++ throw error;+ }+ }++ constructor(private host: ModelStatusHost) {+ // the server has no notion of a paused download, so the ids survive in+ // localStorage; the progress snapshot is stale after a reload and stays null+ try {+ const raw = localStorage.getItem(PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY);+ const parsed: unknown = JSON.parse(raw ?? '[]');++ if (!Array.isArray(parsed)) return;++ for (const repoWithTag of parsed.filter((id): id is string => typeof id === 'string')) {+ this.pausedDownloads.set(repoWithTag, null);+ }+ } catch {+ // unreadable or corrupt: start without the paused set+ }+ }++ /**+ * POST /models starts the download in the background; the feed reports+ * progress, and models_reload refreshes the list once it finishes.+ * Re-posting a paused tag resumes from the partial files kept on disk.+ */+ async downloadModel(repoWithTag: string): Promise<void> {+ if (!serverStore.isRouterMode) {+ toast.error('Model downloads are only available in router mode');++ return;+ }++ // the feed must be live so the resulting models_reload event refreshes the list+ this.subscribe();++ // resuming a paused download: drop the paused state, and let the server+ // discard its stale DOWNLOADED entry (via the list fetch) before re-posting+ if (this.deletePausedDownload(repoWithTag) || this.stopRequests.delete(repoWithTag)) {+ await this.host.fetchRouterModels();+ }++ try {+ const res = await ModelsService.downloadModel(repoWithTag);++ if (!res.success) {+ throw new Error(res.error?.message ?? 'Server rejected the download request');+ }++ // flip the chip to "downloading" right away; the feed refines it with real progress+ this.downloadProgress.set(repoWithTag, { downloadedBytes: 0, files: {}, totalBytes: 0 });++ toast.success(`Download started: ${this.host.toDisplayName(repoWithTag)}`);+ } catch (error) {+ toast.error(`Download failed: ${repoWithTag}`);++ throw error;+ }+ }async ensureLoaded(modelId: string): Promise<void> {if (this.host.isModelLoaded(modelId)) return;@@ -50,16 +227,71 @@ export class ModelStatusManager {}/**- * Current load progress for a model, or null when not loading.+ * Tracked downloads (in flight or paused) for the selector's+ * "Download in progress" section.*/+ getDownloadEntries(): {+ isPaused: boolean;+ progress: ModelDownloadProgress | null;+ repoWithTag: string;+ }[] {+ const inFlight = Array.from(this.downloadProgress, ([repoWithTag, progress]) => ({+ isPaused: false,+ progress,+ repoWithTag+ }));+ const paused = Array.from(this.pausedDownloads, ([repoWithTag, progress]) => ({+ isPaused: true,+ progress,+ repoWithTag+ }));++ return [...inFlight, ...paused];+ }++ getDownloadProgress(repoWithTag: string): ModelDownloadProgress | null {+ return this.downloadProgress.get(repoWithTag) ?? null;+ }+getLoadProgress(modelId: string): ModelLoadProgress | null {return this.loadProgress.get(modelId) ?? null;}+ getPausedDownloadProgress(repoWithTag: string): ModelDownloadProgress | null {+ return this.pausedDownloads.get(repoWithTag) ?? null;+ }++ hasFailedDownload(repoWithTag: string): boolean {+ return this.failedDownloads.has(repoWithTag);+ }++ /** Active while the feed reports download_progress for the tag. */+ isDownloadInProgress(repoWithTag: string): boolean {+ return this.downloadProgress.has(repoWithTag);+ }++ isDownloadPaused(repoWithTag: string): boolean {+ return this.pausedDownloads.has(repoWithTag);+ }++ /**+ * True when the tag is already registered in the /v1/models list; both ids+ * are normalized, see downloadIdKey().+ */+ isModelDownloaded(repoWithTag: string): boolean {+ const key = downloadIdKey(repoWithTag);++ return this.host.routerModels.some((m) => downloadIdKey(m.id) === key);+ }+isOperationInProgress(modelId: string): boolean {return this.loadingStates.get(modelId) ?? false;}+ isSidecarDownloaded(repoId: string, filePath: string): boolean {+ return this.downloadedSidecars.has(`${repoId}/${filePath}`);+ }+async load(modelId: string): Promise<void> {if (this.host.isModelLoaded(modelId)) return;@@ -91,9 +323,30 @@ export class ModelStatusManager {}/**- * Open the /models/sse feed and keep it live with auto reconnect.- * Idempotent and router mode only.+ * The server stops the download child but keeps the partial files, so+ * re-posting the tag resumes where it stopped. The feed reports the stop+ * as download_failed; the 'pause' stop request marks it as intentional.*/+ async pauseDownload(repoWithTag: string): Promise<void> {+ if (!serverStore.isRouterMode) {+ toast.error('Model downloads are only available in router mode');++ return;+ }++ this.subscribe();++ this.stopRequests.set(repoWithTag, ModelDownloadStopRequest.PAUSE);++ try {+ await ModelsService.unload(repoWithTag);+ } catch {+ this.stopRequests.delete(repoWithTag);+ toast.error(`Failed to pause: ${repoWithTag}`);+ }+ }++ /** Open the /models/sse feed with auto reconnect; idempotent, router mode only. */subscribe(): void {if (this.statusReaderActive) return;@@ -133,25 +386,94 @@ export class ModelStatusManager {}}- /**- * Close the /models/sse feed and drop transient progress.- */unsubscribe(): void {this.statusReaderActive = false;this.statusAbort?.abort();this.statusAbort = null;this.loadProgress.clear();+ this.downloadProgress.clear();+ this.failedDownloads.clear();+ this.stopRequests.clear();}/**- * Apply a status envelope: update the model row, track or clear progress,- * settle any pending load or unload awaiter.+ * A user pause keeps the last progress and stays resumable, a user cancel+ * settles silently; genuine failures are marked so the UI can offer a retry.*/+ private applyDownloadFinished(event: ApiModelsSseEvent): void {+ let request: ModelDownloadStopRequest | undefined;++ if (event.event === ServerModelsSseEventType.DOWNLOAD_FAILED) {+ request = this.stopRequests.get(event.model);+ this.stopRequests.delete(event.model);+ }++ const progress = this.downloadProgress.get(event.model) ?? null;++ this.downloadProgress.delete(event.model);++ if (request === ModelDownloadStopRequest.CANCEL) {+ // user cancel: settle silently, the feed's model_remove cleans up the entry+ this.failedDownloads.delete(event.model);+ this.deletePausedDownload(event.model);++ return;+ }++ if (request === ModelDownloadStopRequest.PAUSE) {+ this.setPausedDownload(event.model, progress);+ this.failedDownloads.delete(event.model);++ return;+ }++ this.deletePausedDownload(event.model);++ const ok = event.event === ServerModelsSseEventType.DOWNLOAD_FINISHED;++ if (ok) {+ this.failedDownloads.delete(event.model);++ // the finished download only registers in /v1/models on the next list+ // fetch (the server reloads its model table then), so refetch to flip+ // the quant chips to "downloaded" without waiting for a dialog reopen+ void this.host.fetchRouterModels();++ toast.success(`Download finished: ${this.host.toDisplayName(event.model)}`);+ } else {+ this.failedDownloads.add(event.model);+ toast.error(`Download failed: ${this.host.toDisplayName(event.model)}`);+ }+ }++ /** Aggregate per-file progress into downloaded/total byte counts. */+ private applyDownloadProgress(event: ApiModelsSseEvent): void {+ const data = event.data;++ if (!data || !('progress' in data)) return;++ const progress = (data as ApiModelsSseDownloadProgressData).progress;++ let downloaded = 0;+ let total = 0;++ for (const file of Object.values(progress)) {+ downloaded += file?.done ?? 0;+ total += file?.total ?? 0;+ }++ this.downloadProgress.set(event.model, {+ downloadedBytes: downloaded,+ files: progress,+ totalBytes: total+ });+ }+private applyModelStatus(event: ApiModelsSseEvent): void {const model = event.model;const data = event.data;- if (!model || !data?.status) return;+ if (!model || !data || !('status' in data) || !data.status) return;const status = data.status;@@ -180,11 +502,7 @@ export class ModelStatusManager {this.settleStatus(model, status);}- /**- * Route one feed record by event kind. Only the status_* events carry a- * status payload, models_reload triggers a list refresh, model_remove drops- * the row, download_* belong to the download surface, not here.- */+ /** Route one feed record by event kind. */private applyStatusEvent(event: ApiModelsSseEvent): void {switch (event.event) {case ServerModelsSseEventType.STATUS_CHANGE:@@ -202,13 +520,36 @@ export class ModelStatusManager {break;case ServerModelsSseEventType.DOWNLOAD_PROGRESS:+ this.applyDownloadProgress(event);++ break;+ case ServerModelsSseEventType.DOWNLOAD_FINISHED:+ case ServerModelsSseEventType.DOWNLOAD_FAILED:+ this.applyDownloadFinished(event);+break;}}- /**- * Reject and drop the awaiter for a model.- */+ private deletePausedDownload(repoWithTag: string): boolean {+ if (!this.pausedDownloads.delete(repoWithTag)) return false;++ this.persistPausedDownloads();++ return true;+ }++ private persistPausedDownloads(): void {+ try {+ localStorage.setItem(+ PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY,+ JSON.stringify(Array.from(this.pausedDownloads.keys()))+ );+ } catch {+ // storage unavailable: the pauses just do not survive a reload+ }+ }+private rejectStatus(modelId: string, error: Error): void {const waiter = this.statusWaiters.get(modelId);@@ -218,27 +559,32 @@ export class ModelStatusManager {}}- /**- * Drop a model row reported gone by the feed and settle its awaiters.- */private removeRouterModel(modelId: string): void {if (this.host.routerModels.findIndex((m) => m.id === modelId) === -1) return;this.host.routerModels = this.host.routerModels.filter((m) => m.id !== modelId);this.loadProgress.delete(modelId);+ this.downloadProgress.delete(modelId);+ this.failedDownloads.delete(modelId);+ this.deletePausedDownload(modelId);+ this.stopRequests.delete(modelId);this.rejectStatus(modelId, new Error(`Model removed: ${this.host.toDisplayName(modelId)}`));++ // drop the row from the selector options too; they rebuild from the list+ // response, which only a refetch provides+ void this.host.fetchRouterModels();}- /**- * Read the feed and reconnect until unsubscribed.- */private async runStatusReader(signal: AbortSignal): Promise<void> {await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event));}- /**- * Update one model row status in place, reassigning to trigger reactivity.- */+ private setPausedDownload(repoWithTag: string, progress: ModelDownloadProgress | null): void {+ this.pausedDownloads.set(repoWithTag, progress);+ this.persistPausedDownloads();+ }++ // reassign the array: mutating an entry in place would not trigger reactivityprivate setRouterModelStatus(modelId: string, status: ServerModelStatus): void {const idx = this.host.routerModels.findIndex((m) => m.id === modelId);@@ -254,9 +600,6 @@ export class ModelStatusManager {this.host.routerModels = next;}- /**- * Resolve and drop the awaiter when the model reaches its target status.- */private settleStatus(modelId: string, status: ServerModelStatus): void {const waiter = this.statusWaiters.get(modelId);@@ -266,10 +609,7 @@ export class ModelStatusManager {}}- /**- * Register an awaiter that resolves when the feed reports target status.- * One operation runs per model at a time, so one awaiter per model is kept.- */+ // one operation runs per model at a time, so one waiter per model sufficesprivate waitForStatus(modelId: string, target: ServerModelStatus): Promise<void> {return new Promise((resolve, reject) => {this.statusWaiters.set(modelId, { reject, resolve, target });diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.tsindex 5191b03bd8..b52485ade8 100644--- a/tools/ui/src/lib/types/api.d.ts+++ b/tools/ui/src/lib/types/api.d.ts@@ -138,6 +138,14 @@ export interface ApiModelsSseData {exit_code?: number;}+/**+ * Per-file size snapshot reported by the download_progress SSE envelope.+ * Keys are file URLs, values are byte counters (done <= total).+ */+export interface ApiModelsSseDownloadProgressData {+ progress: Record<string, { done: number; total: number }>;+}+/*** Event kind multiplexed on the /models/sse feed.* Only the status_* events carry a status payload, models_reload signals a@@ -150,7 +158,26 @@ export interface ApiModelsSseData {export interface ApiModelsSseEvent {model: string;event: ServerModelsSseEventType;- data: ApiModelsSseData;+ data?: ApiModelsSseData | ApiModelsSseDownloadProgressData;+}++/**+ * Request body for POST /models (model download).+ * `model` is a HuggingFace repo id, optionally suffixed with `:<tag>` to+ * pin a quantization or sidecar file (e.g. `ggml-org/gemma-3-4b-it-GGUF:Q4_K_M`).+ */+export interface ApiModelsDownloadRequest {+ model: string;+}++/**+ * Response from POST /models and DELETE /models. The POST endpoint returns+ * immediately; the download itself runs in the background and emits events+ * on /models/sse.+ */+export interface ApiModelsDownloadResponse {+ success: boolean;+ error?: { code: number; message: string; type: string };}export interface ApiModelDetails {diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.tsindex 28b7cf6014..453af9bb80 100644--- a/tools/ui/src/lib/types/index.ts+++ b/tools/ui/src/lib/types/index.ts@@ -14,7 +14,10 @@ export type {ApiModelLoadStage,ApiModelsSseProgress,ApiModelsSseData,+ ApiModelsSseDownloadProgressData,ApiModelsSseEvent,+ ApiModelsDownloadRequest,+ ApiModelsDownloadResponse,ApiModelDetails,ApiLlamaCppServerProps,ApiChatCompletionRequest,@@ -102,6 +105,8 @@ export type {ModelCapabilities,ModelModalities,ModelOption,+ ModelDownloadFileProgress,+ ModelDownloadProgress,ModelLoadProgress,ModalityCapabilities} from './models';diff --git a/tools/ui/src/lib/types/models.d.ts b/tools/ui/src/lib/types/models.d.tsindex 03c723e1ae..ca53ca8805 100644--- a/tools/ui/src/lib/types/models.d.ts+++ b/tools/ui/src/lib/types/models.d.ts@@ -26,17 +26,27 @@ export interface ModelOption {tags?: string[];}-/**- * Ephemeral UI-only load progress for one model instance.- * Lives only while a load runs, driven by the /models/sse feed.- * stage is absent until the feed reports its first stage.- */+/** UI-only load progress for one model, driven by the /models/sse feed. */export interface ModelLoadProgress {stages: ApiModelLoadStage[];current: ApiModelLoadStage;value: number;}+/** Per-file bytes of an in-flight download. */+export interface ModelDownloadFileProgress {+ done: number;+ total: number;+}++/** Progress of an in-flight download, summed across its files. */+export interface ModelDownloadProgress {+ downloadedBytes: number;+ totalBytes: number;+ /** Per-file progress keyed by file URL. */+ files: Record<string, ModelDownloadFileProgress>;+}+export interface ParsedModelId {raw: string;orgName: string | null;@@ -48,9 +58,7 @@ export interface ParsedModelId {tags: string[];}-/**- * Modality capabilities for file validation- */+/** Modality capabilities for file validation. */export interface ModalityCapabilities {hasVision: boolean;hasAudio: boolean;diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.tsindex 9aa3a85799..fefea2fe1f 100644--- a/tools/ui/src/lib/utils/api-fetch.ts+++ b/tools/ui/src/lib/utils/api-fetch.ts@@ -137,6 +137,40 @@ export async function apiPost<T, B = unknown>(});}+/**+ * Send a DELETE request to an API endpoint, optionally with query parameters.+ *+ * @param path - API path (query string is appended if `params` is provided)+ * @param params - Optional record of query parameters+ * @param options - Additional fetch options+ * @returns Parsed JSON response+ */+export async function apiDelete<T>(+ path: string,+ params?: Record<string, string>,+ options: ApiFetchOptions = {}+): Promise<T> {+ // the query is appended to the path so `apiFetch` applies its base-path prefix;+ // `apiFetchWithParams` resolves an absolute URL and would bypass it+ let query = '';++ if (params) {+ const search = new URLSearchParams();++ for (const [key, value] of Object.entries(params)) {+ if (value !== undefined && value !== null) {+ search.set(key, value);+ }+ }++ const qs = search.toString();++ if (qs) query = `?${qs}`;+ }++ return apiFetch<T>(`${path}${query}`, { ...options, method: 'DELETE' });+}+/*** Parse error message from a failed response.* Tries to extract error message from JSON body, falls back to status text.diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.tsindex 06ab5d7b6a..21d84dc8d6 100644--- a/tools/ui/src/lib/utils/index.ts+++ b/tools/ui/src/lib/utils/index.ts@@ -9,7 +9,7 @@// API utilitiesexport { getAuthHeaders, getJsonHeaders, sanitizeHeaders } from './api-headers';-export { ApiError, apiFetch, apiFetchWithParams, apiPost } from './api-fetch';+export { ApiError, apiDelete, apiFetch, apiFetchWithParams, apiPost } from './api-fetch';export { validateApiKey } from './api-key-validation';// Attachment utilities@@ -108,7 +108,7 @@ export {export { normalizeModelName, isValidModelName } from './model-names';// Sidecar token utilities-export { isAuxSidecar, isDraftSidecar, sidecarFromFileToken } from './sidecars';+export { isAuxSidecar, isDraftSidecar, sidecarFromFileToken, sidecarFromTag } from './sidecars';// Portal utilitiesexport { portalToBody } from './portal-to-body';diff --git a/tools/ui/src/lib/utils/sidecars.ts b/tools/ui/src/lib/utils/sidecars.tsindex a3eb3841fd..52532b135a 100644--- a/tools/ui/src/lib/utils/sidecars.ts+++ b/tools/ui/src/lib/utils/sidecars.ts@@ -1,4 +1,4 @@-import { type ModelSidecar, SIDECAR_TOKENS } from '$lib/constants';+import { MODEL_ID, type ModelSidecar, SIDECAR_TOKENS } from '$lib/constants';import { ModelAuxSidecar, ModelDraftSidecar } from '$lib/enums';const SIDECAR_TOKEN_SET = new Set<string>(SIDECAR_TOKENS);@@ -10,6 +10,17 @@ export function sidecarFromFileToken(token: string): ModelSidecar | null {return SIDECAR_TOKEN_SET.has(token) ? (token as ModelSidecar) : null;}+/**+ * Sidecar a download tag points at: the segment after the last dash,+ * e.g. `q4_0-mtp` -> `mtp`, `mmproj` -> `mmproj`. Returns null for quant-only+ * tags and tags whose tail is not a sidecar token.+ */+export function sidecarFromTag(tag: string): ModelSidecar | null {+ const token = tag.toLowerCase().split(MODEL_ID.SEGMENT_SEPARATOR).pop() ?? '';++ return sidecarFromFileToken(token);+}+export function isDraftSidecar(sidecar: ModelSidecar): sidecar is ModelDraftSidecar {return DRAFT_SIDECAR_SET.has(sidecar);}diff --git a/tools/ui/tests/unit/model-sidecar-grammar.test.ts b/tools/ui/tests/unit/model-sidecar-grammar.test.tsnew file mode 100644index 0000000000..1ad59171a9--- /dev/null+++ b/tools/ui/tests/unit/model-sidecar-grammar.test.ts@@ -0,0 +1,140 @@+import { ModelAuxSidecar, ModelDraftSidecar, SidecarForm } from '$lib/enums';+import { HuggingFaceService } from '$lib/services/huggingface.service';+import { ModelsService } from '$lib/services/models.service';+import { sidecarFromTag } from '$lib/utils';+import { describe, expect, it } from 'vitest';++const { buildDownloadTag, isSidecarEntry } = ModelsService;+const { extractQuantMeta } = HuggingFaceService;++// the sidecar filename grammar mirrors the server (common/download.cpp):+// the token must be lowercase, and it can sit at the start, between name+// segments, or at the end of the file name+describe('extractQuantMeta', () => {+ it('parses the prefix form', () => {+ expect(extractQuantMeta('mtp-Model-Q4_0.gguf')).toStrictEqual({+ quant: 'Q4_0',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.PREFIX+ });+ });++ it('parses the infix form', () => {+ expect(extractQuantMeta('Model-mtp-Q4_0.gguf')).toStrictEqual({+ quant: 'Q4_0',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.INFIX+ });+ });++ it('parses the suffix form', () => {+ expect(extractQuantMeta('gemma-4-E2B-it-BF16-mtp.gguf')).toStrictEqual({+ quant: 'BF16',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.SUFFIX+ });+ });++ it('parses an uppercase infix token', () => {+ expect(extractQuantMeta('gemma-4-31B-it-MTP-BF16.gguf')).toStrictEqual({+ quant: 'BF16',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.INFIX+ });+ });++ it('parses an uppercase trailing token', () => {+ expect(extractQuantMeta('gemma-4-E2B-it-BF16-MTP.gguf')).toStrictEqual({+ quant: 'BF16',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.SUFFIX+ });+ });++ it('parses a short-form sidecar with a bare quant', () => {+ expect(extractQuantMeta('mmproj-F16.gguf')).toStrictEqual({+ quant: 'F16',+ shared: false,+ sidecar: ModelAuxSidecar.MMPROJ,+ sidecarForm: SidecarForm.PREFIX+ });+ });++ it('parses a bare sidecar file', () => {+ expect(extractQuantMeta('imatrix.gguf')).toStrictEqual({+ quant: null,+ shared: false,+ sidecar: ModelAuxSidecar.IMATRIX,+ sidecarForm: SidecarForm.PREFIX+ });+ });++ it('parses a standalone sidecar with a draft tail', () => {+ expect(extractQuantMeta('Model-mtp-draft.gguf')).toStrictEqual({+ quant: null,+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.SUFFIX+ });+ });++ it('parses a nested sidecar path by its file name', () => {+ expect(extractQuantMeta('MTP/mtp-Model-Q4_0.gguf')).toStrictEqual({+ quant: 'Q4_0',+ shared: false,+ sidecar: ModelDraftSidecar.MTP,+ sidecarForm: SidecarForm.PREFIX+ });+ });++ it('returns null for non-weight files', () => {+ expect(extractQuantMeta('README.md')).toBeNull();+ });+});++describe('buildDownloadTag', () => {+ it('appends the quantization', () => {+ expect(buildDownloadTag('org/repo', 'Q4_0', null)).toBe('org/repo:Q4_0');+ });++ it('appends the quantization and sidecar', () => {+ expect(buildDownloadTag('org/repo', 'Q4_0', ModelDraftSidecar.MTP)).toBe('org/repo:Q4_0-mtp');+ });++ it('uses the sidecar alone when there is no quant', () => {+ expect(buildDownloadTag('org/repo', null, ModelAuxSidecar.MMPROJ)).toBe('org/repo:mmproj');+ });++ it('returns the repo id untouched without a tag', () => {+ expect(buildDownloadTag('org/repo', null, null)).toBe('org/repo');+ });+});++describe('isSidecarEntry', () => {+ it('detects sidecar entries by their tag', () => {+ expect(isSidecarEntry('org/repo:Q4_0-mtp')).toBe(true);+ expect(isSidecarEntry('org/repo:mmproj')).toBe(true);+ });++ it('leaves plain model entries loadable', () => {+ expect(isSidecarEntry('org/repo:Q4_0')).toBe(false);+ expect(isSidecarEntry('org/repo')).toBe(false);+ });+});++describe('sidecarFromTag', () => {+ it('reads the token after the last dash', () => {+ expect(sidecarFromTag('Q4_0-mtp')).toBe(ModelDraftSidecar.MTP);+ expect(sidecarFromTag('mmproj')).toBe(ModelAuxSidecar.MMPROJ);+ });++ it('returns null for quant-only and unrelated tags', () => {+ expect(sidecarFromTag('Q4_K_XL')).toBeNull();+ expect(sidecarFromTag('UD-Q4_K_XL')).toBeNull();+ });+});