LLM inference in C/C++
C++ ggml
ui: Models Manager Follow-up Improvements (#30228)
Aleksander Grygierbaef3ed9a1e8
diff --git a/common/common.cpp b/common/common.cppindex 48caa80c19..28ab9ac6c8 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1195,7 +1195,9 @@ common_decision_type common_get_decision_type(const struct llama_model * model)return common_decision_type_from_string(buf);}-common_decision_type common_get_decision_type(const std::string & fname) {+common_gguf_info common_get_gguf_info(const std::string & fname) {+ common_gguf_info info;+struct gguf_init_params gguf_params = {/* .no_alloc = */ true,/* .ctx = */ nullptr,@@ -1203,31 +1205,32 @@ common_decision_type common_get_decision_type(const std::string & fname) {gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));if (!gguf_ctx) {- return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file+ return info; // missing or unreadable file}- std::string arch;const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");- if (arch_id < 0) {- return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata- }- if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {- return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata+ if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {+ return info; // no architecture in the metadata}- arch = gguf_get_val_str(gguf_ctx.get(), arch_id);+ const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);if (arch.empty()) {- return COMMON_DECISION_TYPE_UNKNOWN;+ return info;}- const std::string key = arch + ".decision.type";- const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str());+ const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str());if (type_id < 0) {- return COMMON_DECISION_TYPE_NONE;+ info.decision_type = COMMON_DECISION_TYPE_NONE;+ } else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) {+ info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));}- if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) {- return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata++ // same key and type as the model loader+ const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str());+ if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) {+ info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);}- return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));++ return info;}common_init_result::common_init_result(common_params & params, bool model_only) :diff --git a/common/common.h b/common/common.hindex 557518e365..d2fe10fd79 100644--- a/common/common.h+++ b/common/common.h@@ -970,9 +970,13 @@ enum common_decision_type {common_decision_type common_get_decision_type(const struct llama_model * model);-// same as above, but reads a GGUF file; it does not load the model-// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid-common_decision_type common_get_decision_type(const std::string & fname);+// metadata of a GGUF file, read without loading the model+struct common_gguf_info {+ common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid+ uint32_t n_ctx_train = 0; // 0 if unknown+};++common_gguf_info common_get_gguf_info(const std::string & fname);// note: defines the model, context, samplers, ets. lifetimesstruct common_init_result {diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 35d21c8767..60353240ac 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -540,6 +540,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::strvoid server_model_meta::update_caps(const common_params & base) {// reset to the default so a failed refresh cannot keep old valuesarchitecture = server_model_architecture_json(false, false, false, {"text"});+ n_ctx_train = 0;// resolve the model file offline; do not downloadcommon_params params;@@ -564,10 +565,12 @@ void server_model_meta::update_caps(const common_params & base) {return;}- // read the output modalities from the GGUF metadata+ // read the output modalities and the trained context from the GGUF metadatastd::vector<std::string> output_modalities = {"text"};if (!params.model.path.empty()) {- output_modalities = server_model_output_modalities(common_get_decision_type(params.model.path));+ const common_gguf_info info = common_get_gguf_info(params.model.path);+ output_modalities = server_model_output_modalities(info.decision_type);+ n_ctx_train = info.n_ctx_train;}bool inp_image = false;@@ -2122,9 +2125,13 @@ void server_models_routes::init_routes() {{"source", server_model_source_to_string(meta.source)},{"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE},// {"need_download", meta.need_download},- // TODO: add other fields, may require reading GGUF metadata+ // TODO: add other fields from the GGUF metadata};+ if (meta.n_ctx_train > 0) {+ model_info["context_length"] = meta.n_ctx_train;+ }+// merge with loaded_info from the child process if availableif (meta.is_running()) {for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) {diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 238955e7c4..f415b7c5fd 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -86,6 +86,7 @@ struct server_model_meta {int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdownbool hidden = false; // hidden from GET /models, but still accept if requestedjson architecture = server_model_architecture_json(false, false, false, {"text"});+ uint32_t n_ctx_train = 0; // trained context, read from the GGUF metadata; 0 when unknownbool is_ready() const {return status == SERVER_MODEL_STATUS_LOADED;diff --git a/tools/ui/src/lib/components/app/models/ModelRowActions.svelte b/tools/ui/src/lib/components/app/models/ModelRowActions.svelteindex 8f173174aa..b40f88d836 100644--- a/tools/ui/src/lib/components/app/models/ModelRowActions.svelte+++ b/tools/ui/src/lib/components/app/models/ModelRowActions.svelte@@ -12,37 +12,38 @@}let { isFav, option, revealOnHover = true }: Props = $props();++ // the favorite heart stays visible, so the reveal applies to the other icons+ const revealClass =+ 'pointer-events-none opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100';</script><div- class={[- 'flex items-center justify-center gap-1 max-md:gap-2.5',- revealOnHover- ? 'pointer-events-none opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100'- : ''- ]}+ class="flex items-center justify-center gap-1 max-md:gap-2.5"onclick={(event) => event.stopPropagation()}onkeydown={(event) => event.stopPropagation()}role="presentation">- <ActionIcon- class="h-5 w-5 hover:text-foreground"- icon={Info}- iconSize="h-4 w-4"- onclick={() =>- // a phone has no manager: its information dialog takes the icon- deviceStore.isMobile- ? uiStore.openModelInformation(option)- : uiStore.openModelsManager(option.id)}- tooltip="Manage model"- tooltipAsTitle- />+ <span class={revealOnHover ? revealClass : ''}>+ <ActionIcon+ class="h-5 w-5 hover:text-foreground"+ icon={Info}+ iconSize="h-4 w-4"+ onclick={() =>+ // a phone has no manager: its information dialog takes the icon+ deviceStore.isMobile+ ? uiStore.openModelInformation(option)+ : uiStore.openModelsManager(option.id)}+ tooltip="Manage model"+ tooltipAsTitle+ />+ </span>{#if isFav}<span class="flex h-5 w-5 items-center justify-center"><span class="flex group-hover:hidden [@media(pointer:coarse)]:hidden"><ActionIcon- class="h-5 w-5 text-rose-500 hover:text-foreground"+ class="h-5 w-5 text-rose-500"icon={Heart}iconSize="h-4 w-4"onclick={() => modelsStore.toggleFavorite(option.model)}@@ -63,13 +64,15 @@</span></span>{:else}- <ActionIcon- class="h-5 w-5 hover:text-foreground"- icon={Heart}- iconSize="h-4 w-4"- onclick={() => modelsStore.toggleFavorite(option.model)}- tooltip="Add to favorites"- tooltipAsTitle- />+ <span class={revealOnHover ? revealClass : ''}>+ <ActionIcon+ class="h-5 w-5 hover:text-foreground"+ icon={Heart}+ iconSize="h-4 w-4"+ onclick={() => modelsStore.toggleFavorite(option.model)}+ tooltip="Add to favorites"+ tooltipAsTitle+ />+ </span>{/if}</div>diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelteindex e6020e9172..36fd3e22ae 100644--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte@@ -16,7 +16,7 @@import type { ModelOption } from '$lib/types/models';import { filterModelOptions } from '$lib/utils';import { type Snippet, untrack } from 'svelte';- import { SvelteMap, SvelteSet } from 'svelte/reactivity';+ import { SvelteMap } from 'svelte/reactivity';interface Props {class?: string;@@ -170,16 +170,29 @@if (remaining.length > 0) rest.push({ ...entry, base: remaining[0], quants: remaining });}- const claimed = new SvelteSet<string>();- const favorites = rest.filter((entry) =>- entry.quants.some((q) => modelsStore.favoriteModelIds.has(q.model))- );+ // a favorite quant stands on its own, listed flat like a loaded one, and the+ // quants left behind stay with their repo in the local block+ const favorites: ModelQuantGroup[] = [];+ const localRest: ModelQuantGroup[] = [];++ for (const entry of rest) {+ for (const quant of entry.quants) {+ if (modelsStore.favoriteModelIds.has(quant.model)) {+ favorites.push({ ...entry, base: quant, key: quant.id, quants: [quant] });+ }+ }- for (const entry of favorites) claimed.add(entry.key);+ const remaining = entry.quants.filter(+ (quant) => !modelsStore.favoriteModelIds.has(quant.model)+ );++ if (remaining.length > 0) {+ localRest.push({ ...entry, base: remaining[0], quants: remaining });+ }+ }- const { hidden, local } = splitHiddenQuants(- rest.filter((entry) => !claimed.has(entry.key)),- (option) => modelsStore.isHidden(option.id)+ const { hidden, local } = splitHiddenQuants(localRest, (option) =>+ modelsStore.isHidden(option.id));const ordered: ModelsTableGroup[] = [];// loaded models lead the table, then favorites, then the local blockdiff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelteindex f4d86b7293..2ded1926c7 100644--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte@@ -7,7 +7,7 @@import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte';import { modelRowActions } from './row-actions';import { configuredContext, downloadProgressFor } from './utils';- import { MoreHorizontal } from '@lucide/svelte';+ import { Heart, MoreHorizontal } from '@lucide/svelte';import { DropdownMenuActions } from '$lib/components/app';import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '$lib/constants';import { ModelRowDownloadState } from '$lib/enums';@@ -74,6 +74,13 @@<!-- a phone has no width for the modality icons, the id needs it more --><ModelCapabilities hideModalities={deviceStore.isMobile} {option} />++ {#if favorite}+ <!-- the heart is decorative, the button's text carries the state -->+ <Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />++ <span class="sr-only">favorited</span>+ {/if}</span></button>diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelteindex 2a3c2206b2..ae4d7f1841 100644--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte@@ -7,8 +7,7 @@hasActiveFilters,modelContextLength,type ModelQuantGroup,- type ModelsTableGroup,- statusRank+ type ModelsTableGroup} from './utils';import {ArrowDown,@@ -155,10 +154,6 @@return (modelContextLength(a) ?? 0) - (modelContextLength(b) ?? 0);case ModelsTableSortKey.NAME:return a.model.localeCompare(b.model);- case ModelsTableSortKey.STATUS:- // a running model leads, then one that is being worked on (loading,- // sleeping), then the rest; the reported status sorts the row's own cell- return statusRank(b) - statusRank(a);default:return 0;}@@ -357,9 +352,7 @@{@render sortHeader(ModelsTableSortKey.CONTEXT, 'Context')}</span>- <span class="justify-self-center max-md:hidden">- {@render sortHeader(ModelsTableSortKey.STATUS, 'Status')}- </span>+ <span class="justify-self-center max-md:hidden">Status</span><span class="text-center max-md:hidden">Actions</span></div>diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.tsindex 3721ceca10..2083436649 100644--- a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts+++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts@@ -1,10 +1,5 @@import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants';-import {- ModelCapability,- ModelGroupKind,- ModelsTableGroupKind,- ServerModelStatus-} from '$lib/enums';+import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';import { HuggingFaceService, ModelsService } from '$lib/services';import { modelsStore } from '$lib/stores';import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models';@@ -27,20 +22,6 @@ export function hasActiveFilters(return contextLimit > 0 || modalities.length > 0 || capabilities.length > 0;}-/**- * Order a status sorts behind: loaded first, then a model being worked on- * (loading, sleeping), then the rest.- */-export function statusRank(option: ModelOption): number {- const status = modelsStore.getModelStatus(option.model);-- if (modelsStore.isModelRunning(option.model)) return 2;-- if (status === ServerModelStatus.LOADING || status === ServerModelStatus.SLEEPING) return 1;-- return 0;-}-/** Byte counts of a tracked download: live while it runs, frozen while paused. */export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress | null {return (diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelteindex a7a54e8602..750e5f6a77 100644--- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte@@ -177,7 +177,7 @@<DropdownMenu.Contentalign="end"- class="w-full md:min-w-80 md:w-112 max-w-[calc(100vw-2rem)] p-0! max-h-[min(40rem,calc(var(--bits-dropdown-menu-content-available-height)-1rem))]"+ class="w-full md:min-w-80 md:w-md max-w-[calc(100vw-2rem)] p-0! max-h-[min(40rem,calc(var(--bits-dropdown-menu-content-available-height)-1rem))]"onOpenAutoFocus={(event) => event.preventDefault()}><DropdownMenuSearchablediff --git a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelteindex 83f54958f6..44855b962a 100644--- a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte+++ b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte@@ -17,7 +17,7 @@<DropdownMenuPrimitive.Contentbind:refclass={cn(- 'z-50 max-h-[calc(var(--bits-dropdown-menu-content-available-height)-1rem)] min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border border-border bg-popover p-1.5 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:fill-mode-forwards data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',+ 'z-50 max-h-[calc(var(--bits-dropdown-menu-content-available-height)-1rem)] min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-xl border border-border bg-popover p-1.5 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:fill-mode-forwards data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',className)}data-slot="dropdown-menu-content"diff --git a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelteindex b43a431f34..574089c283 100644--- a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte+++ b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte@@ -17,7 +17,7 @@<DropdownMenuPrimitive.Itembind:refclass={cn(- "relative flex cursor-pointer items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 data-[variant=destructive]:text-destructive data-[variant=destructive]:data-highlighted:bg-destructive/10 data-[variant=destructive]:data-highlighted:text-destructive dark:data-[variant=destructive]:data-highlighted:bg-destructive/20 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 [&_svg:not([class*='text-'])]:text-muted-foreground data-[variant=destructive]:*:[svg]:!text-destructive",+ "relative flex cursor-pointer items-center gap-2 rounded-md px-2 py-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 data-[variant=destructive]:text-destructive data-[variant=destructive]:data-highlighted:bg-destructive/10 data-[variant=destructive]:data-highlighted:text-destructive dark:data-[variant=destructive]:data-highlighted:bg-destructive/20 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 [&_svg:not([class*='text-'])]:text-muted-foreground data-[variant=destructive]:*:[svg]:!text-destructive",className)}data-inset={inset}diff --git a/tools/ui/src/lib/constants/settings.constants.ts b/tools/ui/src/lib/constants/settings.constants.tsindex 6642a4958b..f6596bfb62 100644--- a/tools/ui/src/lib/constants/settings.constants.ts+++ b/tools/ui/src/lib/constants/settings.constants.ts@@ -129,7 +129,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [// Deliberately off for now: the natural place to turn it on is the first// run experience, once onboarding exists to ask the user about it.defaultValue: false,- help: 'Fetch model metadata (avatars, context length, chat template, file sizes) from the Hugging Face Hub. When off, the UI only shows what the server reports for /v1/models and hides the org avatars.',+ help: 'Fetch model metadata (avatars, context length, chat template) from the Hugging Face Hub. When off, the UI only shows what the server reports for /v1/models and hides the org avatars.',key: SETTINGS_KEYS.USE_HUGGING_FACE_HUB,label: 'Use Hugging Face Hub API for models metadata',type: SettingsFieldType.CHECKBOXdiff --git a/tools/ui/src/lib/enums/model.enums.ts b/tools/ui/src/lib/enums/model.enums.tsindex 8e912129f2..e707fe169e 100644--- a/tools/ui/src/lib/enums/model.enums.ts+++ b/tools/ui/src/lib/enums/model.enums.ts@@ -80,6 +80,5 @@ export enum ModelRowDownloadState {/** Column the models manager table can be ordered by. */export enum ModelsTableSortKey {CONTEXT = 'context',- NAME = 'name',- STATUS = 'status'+ NAME = 'name'}diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.tsindex 69f010b70e..a6b081b9cf 100644--- a/tools/ui/src/lib/stores/models/index.svelte.ts+++ b/tools/ui/src/lib/stores/models/index.svelte.ts@@ -622,6 +622,13 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {capabilities: rawCapabilities.filter((value: unknown): value is string =>Boolean(value)),+ // 0 is the server's way of leaving the trained context unknown; a+ // model-mode listing reports it as meta.n_ctx_train instead+ contextLength:+ item.context_length ||+ (typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0+ ? item.meta.n_ctx_train+ : undefined),description: details?.description,details: details?.details,draftSidecars: mergedDraftSidecars(diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.tsindex b52485ade8..70b99eaa0d 100644--- a/tools/ui/src/lib/types/api.d.ts+++ b/tools/ui/src/lib/types/api.d.ts@@ -100,6 +100,8 @@ export interface ApiModelDataEntry {tags?: string[];/** Modality capabilities, reported by the router for every model regardless of load state */architecture?: ApiModelArchitecture;+ /** Trained context of the model, read from its GGUF metadata at registration */+ context_length?: number;/** Legacy meta field (may be present in older responses) */meta?: Record<string, unknown> | null;}diff --git a/tools/ui/tests/client/models-manager-sort-filter.svelte.test.ts b/tools/ui/tests/client/models-manager-sort-filter.svelte.test.tsnew file mode 100644index 0000000000..9e9e0c2468--- /dev/null+++ b/tools/ui/tests/client/models-manager-sort-filter.svelte.test.ts@@ -0,0 +1,243 @@+// Guards the manager table ordering and filtering: the context column sorts and+// filters once a model's context is known, from the option or the Hub record.++import ModelsManagerWrapper from './components/ModelsManagerWrapper.svelte';+import { SETTINGS_KEYS } from '$lib/constants';+import { ServerModelStatus } from '$lib/enums';+import { HuggingFaceService } from '$lib/services';+import { modelsStore, settingsStore } from '$lib/stores';+import type { ApiModelDataEntry } from '$lib/types';+import type { ModelOption } from '$lib/types/models';+import { SvelteMap } from 'svelte/reactivity';+import { beforeEach, expect, it, vi } from 'vitest';+import { render } from 'vitest-browser-svelte';++function option(model: string, contextLength?: number): ModelOption {+ return {+ capabilities: [],+ contextLength,+ id: model,+ model,+ name: model+ };+}++const models = [+ option('org/alpha-8b:Q4_K_M', 8192),+ option('org/beta-8b:Q4_K_M', 131072),+ option('org/gamma-8b:Q4_K_M', 32768)+];+// the router listing carries no context, so a row starts without one+const modelsWithoutContext = models.map((model) => ({ ...model, contextLength: undefined }));++beforeEach(() => {+ modelsStore.routerModels = [];+ modelsStore.models = modelsWithoutContext;+ modelsStore.favoriteModelIds = new Set();+ settingsStore.config[SETTINGS_KEYS.GROUP_MODELS_BY_FAMILY] = false;+});++/** Renders the manager and waits for the test models to show. */+async function renderWithModels(rows: ModelOption[] = modelsWithoutContext) {+ const screen = render(ModelsManagerWrapper);++ modelsStore.models = rows;++ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();++ return screen;+}++function rowNames(container: HTMLElement): string[] {+ return [...container.querySelectorAll('button[aria-pressed]')].map(+ (row) => row.textContent ?? ''+ );+}++/** The Hub details cache the rows read their context from. */+function detailsCache() {+ return (+ HuggingFaceService as unknown as {+ detailsCache: SvelteMap<string, { gguf?: { context_length?: number } } | null>;+ }+ ).detailsCache;+}++function warmCache() {+ const cache = detailsCache();++ cache.set('org/alpha-8b', { gguf: { context_length: 8192 } });+ cache.set('org/beta-8b', { gguf: { context_length: 131072 } });+ cache.set('org/gamma-8b', { gguf: { context_length: 32768 } });+}++it('sorts by context', async () => {+ const screen = await renderWithModels(models);++ // lowest first+ await screen.getByTitle('Sort by context, lowest first').click();++ const ascending = rowNames(screen.container);++ await screen.getByTitle('Sort by context, highest first').click();++ const descending = rowNames(screen.container);++ expect(ascending).not.toEqual(descending);+});++it('sorts by context with family grouping on', async () => {+ settingsStore.config[SETTINGS_KEYS.GROUP_MODELS_BY_FAMILY] = true;++ const screen = await renderWithModels(models);++ // lowest first+ await screen.getByTitle('Sort by context, lowest first').click();++ const ascending = rowNames(screen.container);++ await screen.getByTitle('Sort by context, highest first').click();++ const descending = rowNames(screen.container);++ expect(ascending).not.toEqual(descending);+});++it('lists the favorited quants of a repo as flat rows', async () => {+ // one quant of the beta repo is a favorite, the other stays with the repo+ modelsStore.favoriteModelIds = new Set(['org/alpha-8b:Q4_K_M', 'org/beta-8b:Q4_K_M']);++ const screen = await renderWithModels([+ modelsWithoutContext[0],+ modelsWithoutContext[1],+ option('org/beta-8b:Q8_0'),+ modelsWithoutContext[2]+ ]);++ // the favorite quant is a model row of its own, not a repo with subitems+ expect(screen.container.textContent).not.toContain('2 quants available');++ // the repo appears once in favorites for its favorited quant and once in the+ // local block for the quant left behind+ expect(screen.getByText(/beta\s+8B/).elements().length).toBe(2);+});++/** A router listing entry carrying only the meta context. */+function entry(model: string, nCtxTrain: number): ApiModelDataEntry {+ return {+ created: 0,+ id: model,+ in_cache: false,+ meta: { n_ctx_train: nCtxTrain },+ object: 'model',+ owned_by: 'llamacpp',+ path: `/models/${model}`,+ status: { value: ServerModelStatus.UNLOADED }+ };+}++it('sorts by the meta context of a listing without the router field', async () => {+ // a listing that skips the router's GGUF read reports the trained context+ // only as meta.n_ctx_train, so the option mapping falls back to it+ vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: RequestInfo | URL) => {+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;++ if (url.includes('/props')) {+ return new Response(+ JSON.stringify({+ default_generation_settings: { n_ctx: 0, params: {} },+ model_alias: 'llama-server',+ model_path: 'none',+ role: 'router'+ }),+ { headers: { 'Content-Type': 'application/json' }, status: 200 }+ );+ }++ if (url.includes('/server')) {+ return new Response(+ JSON.stringify({ git_branch: 'test', git_commit: 'test', mode: 'router', version: 'test' }),+ { headers: { 'Content-Type': 'application/json' }, status: 200 }+ );+ }++ if (/\/v1\/models|\/models\b/.test(url)) {+ return new Response(+ JSON.stringify({+ data: [+ entry('org/alpha-8b:Q4_K_M', 8192),+ entry('org/beta-8b:Q4_K_M', 131072),+ entry('org/gamma-8b:Q4_K_M', 32768)+ ],+ object: 'list'+ }),+ { headers: { 'Content-Type': 'application/json' }, status: 200 }+ );+ }++ throw new Error(`unexpected fetch in the test: ${url}`);+ });++ const screen = render(ModelsManagerWrapper);++ // the fetch maps the listing into options, the meta context fills in+ await modelsStore.fetch(true);++ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();++ await screen.getByTitle('Sort by context, lowest first').click();++ const names = rowNames(screen.container).join(' | ');++ expect(names.indexOf('alpha')).toBeLessThan(names.indexOf('gamma'));+ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));+});++it('re-sorts when the Hub details arrive after the sort was clicked', async () => {+ const screen = await renderWithModels();++ // the user sorts while the contexts are still unknown+ await screen.getByTitle('Sort by context, lowest first').click();++ // then the rows fetch their Hub records+ warmCache();++ // the table re-sorts once the cache answers+ await vi.waitFor(() => {+ const names = rowNames(screen.container).join(' | ');++ expect(names.indexOf('alpha')).toBeLessThan(names.indexOf('gamma'));+ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));++ return names;+ });+});++it('filters by search', async () => {+ const screen = await renderWithModels();++ await screen.getByPlaceholder('Search your models').fill('beta');++ await expect.element(screen.getByText(/alpha\s+8B/)).not.toBeVisible();+ await expect.element(screen.getByText(/beta\s+8B/)).toBeVisible();+});++it('filters by context and sorts from the Hub details cache', async () => {+ warmCache();++ const screen = await renderWithModels();++ // open the context filter and ask for 32K or more+ await screen.getByText('Context:').click();+ await screen.getByText('32K or more').click();++ await expect.element(screen.getByText(/alpha\s+8B/)).not.toBeVisible();+ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();++ // sorting re-runs once the cache answers+ await screen.getByTitle('Sort by context, lowest first').click();++ const names = rowNames(screen.container).join(' | ');++ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));+});