LLM inference in C/C++
C++ ggml
ui : type-safe API types, fetch helpers and download-ready models store plumbing (#29582)
Aleksander Grygierfa2bde5543f8
diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.jsonindex f9b793b29f..86d599e671 100644--- a/tools/ui/package-lock.json+++ b/tools/ui/package-lock.json@@ -71,7 +71,7 @@"svelte-check": "4.6.0","svelte-sonner": "1.1.1","tailwind-merge": "3.6.0",- "tailwind-variants": "3.2.2",+ "tailwind-variants": "3.3.1","tailwindcss": "4.3.0","tw-animate-css": "1.4.0","typescript": "5.9.3",@@ -16048,13 +16048,13 @@}},"node_modules/tailwind-variants": {- "version": "3.2.2",- "resolved": "https://registry.npmjs.org/tailwind-variants/-/tailwind-variants-3.2.2.tgz",- "integrity": "sha512-Mi4kHeMTLvKlM98XPnK+7HoBPmf4gygdFmqQPaDivc3DpYS6aIY6KiG/PgThrGvii5YZJqRsPz0aPyhoFzmZgg==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/tailwind-variants/-/tailwind-variants-3.3.1.tgz",+ "integrity": "sha512-4pAvwUtM4HKBiRZftncAbpn6V9Hhwoa5Fl7O2u5zbp7Z5Cvu+/o/6+176WY3WCEES209543quG8zFIcXCsc5Jw==","dev": true,"license": "MIT","engines": {- "node": ">=16.x",+ "node": ">=16.9.x","pnpm": ">=7.x"},"peerDependencies": {@@ -16064,6 +16064,9 @@"peerDependenciesMeta": {"tailwind-merge": {"optional": true+ },+ "tailwindcss": {+ "optional": true}}},diff --git a/tools/ui/package.json b/tools/ui/package.jsonindex 3c15289973..4017604741 100644--- a/tools/ui/package.json+++ b/tools/ui/package.json@@ -13,6 +13,7 @@"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch","reset": "rm -rf .svelte-kit node_modules","format": "eslint --fix . && prettier --write .",+ "format:files": "sh -c 'eslint --fix \"$@\" && prettier --write \"$@\"' sh","lint": "prettier --check . && eslint .","test": "npm run test:ui -- --run && npm run test:client -- --run && npm run test:unit -- --run && npm run test:e2e","test:e2e": "playwright test",@@ -90,7 +91,7 @@"svelte-check": "4.6.0","svelte-sonner": "1.1.1","tailwind-merge": "3.6.0",- "tailwind-variants": "3.2.2",+ "tailwind-variants": "3.3.1","tailwindcss": "4.3.0","tw-animate-css": "1.4.0","typescript": "5.9.3",diff --git a/tools/ui/src/app.d.ts b/tools/ui/src/app.d.tsindex 639a16df21..0a206381f8 100644--- a/tools/ui/src/app.d.ts+++ b/tools/ui/src/app.d.ts@@ -18,20 +18,14 @@ import type {ApiErrorResponse,ApiLlamaCppServerProps,ApiModelDataEntry,- ApiModelListResponse,ApiModelLoadStage,+ ApiModelsListResponse,+ ApiModelsLoadResponse,ApiModelsSseData,ApiModelsSseEvent,ApiModelsSseProgress,+ ApiModelsUnloadResponse,ApiProcessingState,- ApiRouterModelMeta,- ApiRouterModelsListResponse,- ApiRouterModelsLoadRequest,- ApiRouterModelsLoadResponse,- ApiRouterModelsStatusRequest,- ApiRouterModelsStatusResponse,- ApiRouterModelsUnloadRequest,- ApiRouterModelsUnloadResponse,ChatAttachmentDisplayItem,// Chat typesChatMessagePromptProgress,@@ -89,16 +83,10 @@ declare global {ApiModelsSseProgress,ApiModelsSseData,ApiModelsSseEvent,- ApiModelListResponse,+ ApiModelsListResponse,+ ApiModelsLoadResponse,+ ApiModelsUnloadResponse,ApiProcessingState,- ApiRouterModelMeta,- ApiRouterModelsLoadRequest,- ApiRouterModelsLoadResponse,- ApiRouterModelsStatusRequest,- ApiRouterModelsStatusResponse,- ApiRouterModelsListResponse,- ApiRouterModelsUnloadRequest,- ApiRouterModelsUnloadResponse,// Chat typesChatAttachmentDisplayItem,ChatMessagePromptProgress,diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.tsindex cfd0077acc..be6261bd62 100644--- a/tools/ui/src/lib/services/index.ts+++ b/tools/ui/src/lib/services/index.ts@@ -136,7 +136,7 @@ export { ConversationTransferService } from './conversation-transfer.service';** **Server Mode Behavior:*** - **MODEL mode**: Only `list()` is relevant — single model always loaded- * - **ROUTER mode**: Full lifecycle — `list()`, `listRouter()`, `load()`, `unload()`+ * - **ROUTER mode**: Full lifecycle - `list()`, `load()`, `unload()`** **Endpoints:*** - `GET /v1/models` — OpenAI-compatible model list (both modes)diff --git a/tools/ui/src/lib/services/models.service.ts b/tools/ui/src/lib/services/models.service.tsindex b0fcca0eec..070c935c65 100644--- a/tools/ui/src/lib/services/models.service.ts+++ b/tools/ui/src/lib/services/models.service.ts@@ -56,19 +56,8 @@ export class ModelsService {** @returns List of available models with basic metadata*/- static async list(): Promise<ApiModelListResponse> {- return apiFetch<ApiModelListResponse>(API_MODELS.LIST);- }-- /**- * Fetch list of all models with detailed metadata (ROUTER mode).- * Returns models with load status, paths, and other metadata- * beyond what the OpenAI-compatible endpoint provides.- *- * @returns List of models with detailed status and configuration info- */- static async listRouter(): Promise<ApiRouterModelsListResponse> {- return apiFetch<ApiRouterModelsListResponse>(API_MODELS.LIST);+ static async list(): Promise<ApiModelsListResponse> {+ return apiFetch<ApiModelsListResponse>(API_MODELS.LIST);}/**@@ -80,14 +69,14 @@ export class ModelsService {* @param extraArgs - Optional additional arguments to pass to the model instance* @returns Load response from the server*/- static async load(modelId: string, extraArgs?: string[]): Promise<ApiRouterModelsLoadResponse> {+ static async load(modelId: string, extraArgs?: string[]): Promise<ApiModelsLoadResponse> {const payload: { model: string; extra_args?: string[] } = { model: modelId };if (extraArgs && extraArgs.length > 0) {payload.extra_args = extraArgs;}- return apiPost<ApiRouterModelsLoadResponse>(API_MODELS.LOAD, payload);+ return apiPost<ApiModelsLoadResponse>(API_MODELS.LOAD, payload);}/**@@ -220,8 +209,8 @@ export class ModelsService {* @param modelId - Model identifier to unload* @returns Unload response from the server*/- static async unload(modelId: string): Promise<ApiRouterModelsUnloadResponse> {- return apiPost<ApiRouterModelsUnloadResponse>(API_MODELS.UNLOAD, { model: modelId });+ static async unload(modelId: string): Promise<ApiModelsUnloadResponse> {+ return apiPost<ApiModelsUnloadResponse>(API_MODELS.UNLOAD, { model: modelId });}/**diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.tsindex 90d6fe76b7..d25b59cfc2 100644--- a/tools/ui/src/lib/stores/models/index.svelte.ts+++ b/tools/ui/src/lib/stores/models/index.svelte.ts@@ -193,15 +193,15 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {}/**- * Fetch router models with full metadata (ROUTER mode only).- * No-op in router mode — fetch() already calls listRouter() internally.+ * Fetch models with full metadata (ROUTER mode only).+ * No-op in MODEL mode - fetch() already calls list() internally.* Kept for API compatibility (e.g. handleOpenChange dropdown open handler).*/async fetchRouterModels(): Promise<void> {if (!serverStore.isRouterMode) return;try {- const response = await ModelsService.listRouter();+ const response = await ModelsService.list();this.routerModels = response.data;await this.props.fetchModalitiesForLoadedModels();@@ -358,9 +358,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {* Both MODEL and ROUTER modes share the same mapping logic;* they differ only in which endpoint is called.*/- private buildModelOptions(- response: ApiModelListResponse | ApiRouterModelsListResponse- ): ModelOption[] {+ private buildModelOptions(response: ApiModelsListResponse): ModelOption[] {return response.data.map((item: ApiModelDataEntry, index: number) => {const details = response.models?.[index];const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : [];@@ -390,7 +388,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {return this.buildModelOptions(response);}-/*** Filter to models visible in the UI (ui !== false).*/@@ -422,7 +419,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {const router = serverStore.isRouterMode;if (router) {- const response = await ModelsService.listRouter();+ const response = await ModelsService.list();this.routerModels = response.data;this.models = this.buildModelOptions(response);diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.tsindex ebf0a2a48b..5191b03bd8 100644--- a/tools/ui/src/lib/types/api.d.ts+++ b/tools/ui/src/lib/types/api.d.ts@@ -174,12 +174,6 @@ export interface ApiModelDetails {};}-export interface ApiModelListResponse {- object: string;- data: ApiModelDataEntry[];- models?: ApiModelDetails[];-}-export interface ApiLlamaCppServerProps {default_generation_settings: {id: number;@@ -448,79 +442,32 @@ export interface ApiProcessingState {}/**- * Router model metadata - extended from ApiModelDataEntry with additional router-specific fields- * @deprecated Use ApiModelDataEntry instead - the /models endpoint returns this structure directly- */-export interface ApiRouterModelMeta {- /** Model identifier (e.g., "ggml-org/Qwen2.5-Omni-7B-GGUF:latest") */- name: string;- /** Path to model file or manifest */- path: string;- /** Optional path to multimodal projector */- path_mmproj?: string;- /** Whether model is in HuggingFace cache */- in_cache: boolean;- /** Port where model instance is running (0 if not loaded) */- port?: number;- /** Current status of the model */- status: ApiModelStatus;- /** Error message if status is FAILED */- error?: string;-}--/**- * Request to load a model- */-export interface ApiRouterModelsLoadRequest {- model: string;-}--/**- * Response from loading a model+ * Response from POST /models/load*/-export interface ApiRouterModelsLoadResponse {+export interface ApiModelsLoadResponse {success: boolean;error?: string;}/**- * Request to check model status- */-export interface ApiRouterModelsStatusRequest {- model: string;-}--/**- * Response with model status+ * Response with list of all models from /v1/models and /models endpoints+ * (same structure regardless of server mode)*/-export interface ApiRouterModelsStatusResponse {- model: string;- status: ModelStatus;- port?: number;- error?: string;-}--/**- * Response with list of all models from /models endpoint- * Note: This is the same as ApiModelListResponse - the endpoint returns the same structure- * regardless of server mode (MODEL or ROUTER)- */-export interface ApiRouterModelsListResponse {+export interface ApiModelsListResponse {object: string;data: ApiModelDataEntry[];+ /**+ * Optional details aligned by index with `data`: `models[i]` describes+ * `data[i]`. The server fills both arrays in one pass, so the pairing only+ * holds while neither array is filtered independently.+ */+ models?: ApiModelDetails[];}/**- * Request to unload a model- */-export interface ApiRouterModelsUnloadRequest {- model: string;-}--/**- * Response from unloading a model+ * Response from POST /models/unload*/-export interface ApiRouterModelsUnloadResponse {+export interface ApiModelsUnloadResponse {success: boolean;error?: string;}diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.tsindex 333c1bd3cd..a16b5886e0 100644--- a/tools/ui/src/lib/types/index.ts+++ b/tools/ui/src/lib/types/index.ts@@ -16,7 +16,6 @@ export type {ApiModelsSseData,ApiModelsSseEvent,ApiModelDetails,- ApiModelListResponse,ApiLlamaCppServerProps,ApiChatCompletionRequest,ApiChatCompletionToolCallFunctionDelta,@@ -26,14 +25,9 @@ export type {ApiChatCompletionResponse,ApiSlotData,ApiProcessingState,- ApiRouterModelMeta,- ApiRouterModelsLoadRequest,- ApiRouterModelsLoadResponse,- ApiRouterModelsStatusRequest,- ApiRouterModelsStatusResponse,- ApiRouterModelsListResponse,- ApiRouterModelsUnloadRequest,- ApiRouterModelsUnloadResponse,+ ApiModelsLoadResponse,+ ApiModelsListResponse,+ ApiModelsUnloadResponse,AudioInputFormat,ApiStreamSession} from './api';diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.tsindex 2059200049..9aa3a85799 100644--- a/tools/ui/src/lib/utils/api-fetch.ts+++ b/tools/ui/src/lib/utils/api-fetch.ts@@ -49,7 +49,7 @@ export interface ApiFetchOptions extends Omit<RequestInit, 'headers'> {* @example* ```typescript* // GET request- * const models = await apiFetch<ApiModelListResponse>('/v1/models');+ * const models = await apiFetch<ApiModelsListResponse>('/v1/models');** // POST request* const result = await apiFetch<ApiResponse>('/models/load', {