Skip to content

LLM inference in C/C++

C++ ggml

ui: Models Manager Follow-up Improvements (#30228)

Aleksander Grygierbaef3ed9a1e8
diff --git a/common/common.cpp b/common/common.cpp
index 48caa80c19..28ab9ac6c8 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1195,7 +1195,9 @@ common_decision_type common_get_decision_type(const struct llama_model * model)
return common_decision_type_from_string(buf);
}
-common_decision_type common_get_decision_type(const std::string & fname) {
+common_gguf_info common_get_gguf_info(const std::string & fname) {
+ common_gguf_info info;
+
struct gguf_init_params gguf_params = {
/* .no_alloc = */ true,
/* .ctx = */ nullptr,
@@ -1203,31 +1205,32 @@ common_decision_type common_get_decision_type(const std::string & fname) {
gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
if (!gguf_ctx) {
- return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file
+ return info; // missing or unreadable file
}
- std::string arch;
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
- if (arch_id < 0) {
- return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata
- }
- if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
- return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
+ if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
+ return info; // no architecture in the metadata
}
- arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
+ const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch.empty()) {
- return COMMON_DECISION_TYPE_UNKNOWN;
+ return info;
}
- const std::string key = arch + ".decision.type";
- const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str());
+ const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str());
if (type_id < 0) {
- return COMMON_DECISION_TYPE_NONE;
+ info.decision_type = COMMON_DECISION_TYPE_NONE;
+ } else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) {
+ info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
}
- if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) {
- return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
+
+ // same key and type as the model loader
+ const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str());
+ if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) {
+ info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
}
- return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
+
+ return info;
}
common_init_result::common_init_result(common_params & params, bool model_only) :
diff --git a/common/common.h b/common/common.h
index 557518e365..d2fe10fd79 100644
--- a/common/common.h
+++ b/common/common.h
@@ -970,9 +970,13 @@ enum common_decision_type {
common_decision_type common_get_decision_type(const struct llama_model * model);
-// same as above, but reads a GGUF file; it does not load the model
-// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid
-common_decision_type common_get_decision_type(const std::string & fname);
+// metadata of a GGUF file, read without loading the model
+struct common_gguf_info {
+ common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid
+ uint32_t n_ctx_train = 0; // 0 if unknown
+};
+
+common_gguf_info common_get_gguf_info(const std::string & fname);
// note: defines the model, context, samplers, ets. lifetimes
struct common_init_result {
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index 35d21c8767..60353240ac 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -540,6 +540,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
void server_model_meta::update_caps(const common_params & base) {
// reset to the default so a failed refresh cannot keep old values
architecture = server_model_architecture_json(false, false, false, {"text"});
+ n_ctx_train = 0;
// resolve the model file offline; do not download
common_params params;
@@ -564,10 +565,12 @@ void server_model_meta::update_caps(const common_params & base) {
return;
}
- // read the output modalities from the GGUF metadata
+ // read the output modalities and the trained context from the GGUF metadata
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
- output_modalities = server_model_output_modalities(common_get_decision_type(params.model.path));
+ const common_gguf_info info = common_get_gguf_info(params.model.path);
+ output_modalities = server_model_output_modalities(info.decision_type);
+ n_ctx_train = info.n_ctx_train;
}
bool inp_image = false;
@@ -2122,9 +2125,13 @@ void server_models_routes::init_routes() {
{"source", server_model_source_to_string(meta.source)},
{"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE},
// {"need_download", meta.need_download},
- // TODO: add other fields, may require reading GGUF metadata
+ // TODO: add other fields from the GGUF metadata
};
+ if (meta.n_ctx_train > 0) {
+ model_info["context_length"] = meta.n_ctx_train;
+ }
+
// merge with loaded_info from the child process if available
if (meta.is_running()) {
for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) {
diff --git a/tools/server/server-models.h b/tools/server/server-models.h
index 238955e7c4..f415b7c5fd 100644
--- a/tools/server/server-models.h
+++ b/tools/server/server-models.h
@@ -86,6 +86,7 @@ struct server_model_meta {
int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
bool hidden = false; // hidden from GET /models, but still accept if requested
json architecture = server_model_architecture_json(false, false, false, {"text"});
+ uint32_t n_ctx_train = 0; // trained context, read from the GGUF metadata; 0 when unknown
bool is_ready() const {
return status == SERVER_MODEL_STATUS_LOADED;
diff --git a/tools/ui/src/lib/components/app/models/ModelRowActions.svelte b/tools/ui/src/lib/components/app/models/ModelRowActions.svelte
index 8f173174aa..b40f88d836 100644
--- a/tools/ui/src/lib/components/app/models/ModelRowActions.svelte
+++ b/tools/ui/src/lib/components/app/models/ModelRowActions.svelte
@@ -12,37 +12,38 @@
}
let { isFav, option, revealOnHover = true }: Props = $props();
+
+ // the favorite heart stays visible, so the reveal applies to the other icons
+ const revealClass =
+ 'pointer-events-none opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100';
</script>
<div
- class={[
- 'flex items-center justify-center gap-1 max-md:gap-2.5',
- revealOnHover
- ? 'pointer-events-none opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100'
- : ''
- ]}
+ class="flex items-center justify-center gap-1 max-md:gap-2.5"
onclick={(event) => event.stopPropagation()}
onkeydown={(event) => event.stopPropagation()}
role="presentation"
>
- <ActionIcon
- class="h-5 w-5 hover:text-foreground"
- icon={Info}
- iconSize="h-4 w-4"
- onclick={() =>
- // a phone has no manager: its information dialog takes the icon
- deviceStore.isMobile
- ? uiStore.openModelInformation(option)
- : uiStore.openModelsManager(option.id)}
- tooltip="Manage model"
- tooltipAsTitle
- />
+ <span class={revealOnHover ? revealClass : ''}>
+ <ActionIcon
+ class="h-5 w-5 hover:text-foreground"
+ icon={Info}
+ iconSize="h-4 w-4"
+ onclick={() =>
+ // a phone has no manager: its information dialog takes the icon
+ deviceStore.isMobile
+ ? uiStore.openModelInformation(option)
+ : uiStore.openModelsManager(option.id)}
+ tooltip="Manage model"
+ tooltipAsTitle
+ />
+ </span>
{#if isFav}
<span class="flex h-5 w-5 items-center justify-center">
<span class="flex group-hover:hidden [@media(pointer:coarse)]:hidden">
<ActionIcon
- class="h-5 w-5 text-rose-500 hover:text-foreground"
+ class="h-5 w-5 text-rose-500"
icon={Heart}
iconSize="h-4 w-4"
onclick={() => modelsStore.toggleFavorite(option.model)}
@@ -63,13 +64,15 @@
</span>
</span>
{:else}
- <ActionIcon
- class="h-5 w-5 hover:text-foreground"
- icon={Heart}
- iconSize="h-4 w-4"
- onclick={() => modelsStore.toggleFavorite(option.model)}
- tooltip="Add to favorites"
- tooltipAsTitle
- />
+ <span class={revealOnHover ? revealClass : ''}>
+ <ActionIcon
+ class="h-5 w-5 hover:text-foreground"
+ icon={Heart}
+ iconSize="h-4 w-4"
+ onclick={() => modelsStore.toggleFavorite(option.model)}
+ tooltip="Add to favorites"
+ tooltipAsTitle
+ />
+ </span>
{/if}
</div>
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte
index e6020e9172..36fd3e22ae 100644
--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte
+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte
@@ -16,7 +16,7 @@
import type { ModelOption } from '$lib/types/models';
import { filterModelOptions } from '$lib/utils';
import { type Snippet, untrack } from 'svelte';
- import { SvelteMap, SvelteSet } from 'svelte/reactivity';
+ import { SvelteMap } from 'svelte/reactivity';
interface Props {
class?: string;
@@ -170,16 +170,29 @@
if (remaining.length > 0) rest.push({ ...entry, base: remaining[0], quants: remaining });
}
- const claimed = new SvelteSet<string>();
- const favorites = rest.filter((entry) =>
- entry.quants.some((q) => modelsStore.favoriteModelIds.has(q.model))
- );
+ // a favorite quant stands on its own, listed flat like a loaded one, and the
+ // quants left behind stay with their repo in the local block
+ const favorites: ModelQuantGroup[] = [];
+ const localRest: ModelQuantGroup[] = [];
+
+ for (const entry of rest) {
+ for (const quant of entry.quants) {
+ if (modelsStore.favoriteModelIds.has(quant.model)) {
+ favorites.push({ ...entry, base: quant, key: quant.id, quants: [quant] });
+ }
+ }
- for (const entry of favorites) claimed.add(entry.key);
+ const remaining = entry.quants.filter(
+ (quant) => !modelsStore.favoriteModelIds.has(quant.model)
+ );
+
+ if (remaining.length > 0) {
+ localRest.push({ ...entry, base: remaining[0], quants: remaining });
+ }
+ }
- const { hidden, local } = splitHiddenQuants(
- rest.filter((entry) => !claimed.has(entry.key)),
- (option) => modelsStore.isHidden(option.id)
+ const { hidden, local } = splitHiddenQuants(localRest, (option) =>
+ modelsStore.isHidden(option.id)
);
const ordered: ModelsTableGroup[] = [];
// loaded models lead the table, then favorites, then the local block
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
index f4d86b7293..2ded1926c7 100644
--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
@@ -7,7 +7,7 @@
import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte';
import { modelRowActions } from './row-actions';
import { configuredContext, downloadProgressFor } from './utils';
- import { MoreHorizontal } from '@lucide/svelte';
+ import { Heart, MoreHorizontal } from '@lucide/svelte';
import { DropdownMenuActions } from '$lib/components/app';
import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '$lib/constants';
import { ModelRowDownloadState } from '$lib/enums';
@@ -74,6 +74,13 @@
<!-- a phone has no width for the modality icons, the id needs it more -->
<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
+
+ {#if favorite}
+ <!-- the heart is decorative, the button's text carries the state -->
+ <Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />
+
+ <span class="sr-only">favorited</span>
+ {/if}
</span>
</button>
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
index 2a3c2206b2..ae4d7f1841 100644
--- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
+++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
@@ -7,8 +7,7 @@
hasActiveFilters,
modelContextLength,
type ModelQuantGroup,
- type ModelsTableGroup,
- statusRank
+ type ModelsTableGroup
} from './utils';
import {
ArrowDown,
@@ -155,10 +154,6 @@
return (modelContextLength(a) ?? 0) - (modelContextLength(b) ?? 0);
case ModelsTableSortKey.NAME:
return a.model.localeCompare(b.model);
- case ModelsTableSortKey.STATUS:
- // a running model leads, then one that is being worked on (loading,
- // sleeping), then the rest; the reported status sorts the row's own cell
- return statusRank(b) - statusRank(a);
default:
return 0;
}
@@ -357,9 +352,7 @@
{@render sortHeader(ModelsTableSortKey.CONTEXT, 'Context')}
</span>
- <span class="justify-self-center max-md:hidden">
- {@render sortHeader(ModelsTableSortKey.STATUS, 'Status')}
- </span>
+ <span class="justify-self-center max-md:hidden">Status</span>
<span class="text-center max-md:hidden">Actions</span>
</div>
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts
index 3721ceca10..2083436649 100644
--- a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts
+++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts
@@ -1,10 +1,5 @@
import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants';
-import {
- ModelCapability,
- ModelGroupKind,
- ModelsTableGroupKind,
- ServerModelStatus
-} from '$lib/enums';
+import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';
import { HuggingFaceService, ModelsService } from '$lib/services';
import { modelsStore } from '$lib/stores';
import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models';
@@ -27,20 +22,6 @@ export function hasActiveFilters(
return contextLimit > 0 || modalities.length > 0 || capabilities.length > 0;
}
-/**
- * Order a status sorts behind: loaded first, then a model being worked on
- * (loading, sleeping), then the rest.
- */
-export function statusRank(option: ModelOption): number {
- const status = modelsStore.getModelStatus(option.model);
-
- if (modelsStore.isModelRunning(option.model)) return 2;
-
- if (status === ServerModelStatus.LOADING || status === ServerModelStatus.SLEEPING) return 1;
-
- return 0;
-}
-
/** Byte counts of a tracked download: live while it runs, frozen while paused. */
export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress | null {
return (
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte
index a7a54e8602..750e5f6a77 100644
--- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte
+++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte
@@ -177,7 +177,7 @@
<DropdownMenu.Content
align="end"
- class="w-full md:min-w-80 md:w-112 max-w-[calc(100vw-2rem)] p-0! max-h-[min(40rem,calc(var(--bits-dropdown-menu-content-available-height)-1rem))]"
+ class="w-full md:min-w-80 md:w-md max-w-[calc(100vw-2rem)] p-0! max-h-[min(40rem,calc(var(--bits-dropdown-menu-content-available-height)-1rem))]"
onOpenAutoFocus={(event) => event.preventDefault()}
>
<DropdownMenuSearchable
diff --git a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte
index 83f54958f6..44855b962a 100644
--- a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte
+++ b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte
@@ -17,7 +17,7 @@
<DropdownMenuPrimitive.Content
bind:ref
class={cn(
- 'z-50 max-h-[calc(var(--bits-dropdown-menu-content-available-height)-1rem)] min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border border-border bg-popover p-1.5 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:fill-mode-forwards data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',
+ 'z-50 max-h-[calc(var(--bits-dropdown-menu-content-available-height)-1rem)] min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-xl border border-border bg-popover p-1.5 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:fill-mode-forwards data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',
className
)}
data-slot="dropdown-menu-content"
diff --git a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte
index b43a431f34..574089c283 100644
--- a/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte
+++ b/tools/ui/src/lib/components/ui/dropdown-menu/dropdown-menu-item.svelte
@@ -17,7 +17,7 @@
<DropdownMenuPrimitive.Item
bind:ref
class={cn(
- "relative flex cursor-pointer items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 data-[variant=destructive]:text-destructive data-[variant=destructive]:data-highlighted:bg-destructive/10 data-[variant=destructive]:data-highlighted:text-destructive dark:data-[variant=destructive]:data-highlighted:bg-destructive/20 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 [&_svg:not([class*='text-'])]:text-muted-foreground data-[variant=destructive]:*:[svg]:!text-destructive",
+ "relative flex cursor-pointer items-center gap-2 rounded-md px-2 py-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 data-[variant=destructive]:text-destructive data-[variant=destructive]:data-highlighted:bg-destructive/10 data-[variant=destructive]:data-highlighted:text-destructive dark:data-[variant=destructive]:data-highlighted:bg-destructive/20 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 [&_svg:not([class*='text-'])]:text-muted-foreground data-[variant=destructive]:*:[svg]:!text-destructive",
className
)}
data-inset={inset}
diff --git a/tools/ui/src/lib/constants/settings.constants.ts b/tools/ui/src/lib/constants/settings.constants.ts
index 6642a4958b..f6596bfb62 100644
--- a/tools/ui/src/lib/constants/settings.constants.ts
+++ b/tools/ui/src/lib/constants/settings.constants.ts
@@ -129,7 +129,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
// Deliberately off for now: the natural place to turn it on is the first
// run experience, once onboarding exists to ask the user about it.
defaultValue: false,
- help: 'Fetch model metadata (avatars, context length, chat template, file sizes) from the Hugging Face Hub. When off, the UI only shows what the server reports for /v1/models and hides the org avatars.',
+ help: 'Fetch model metadata (avatars, context length, chat template) from the Hugging Face Hub. When off, the UI only shows what the server reports for /v1/models and hides the org avatars.',
key: SETTINGS_KEYS.USE_HUGGING_FACE_HUB,
label: 'Use Hugging Face Hub API for models metadata',
type: SettingsFieldType.CHECKBOX
diff --git a/tools/ui/src/lib/enums/model.enums.ts b/tools/ui/src/lib/enums/model.enums.ts
index 8e912129f2..e707fe169e 100644
--- a/tools/ui/src/lib/enums/model.enums.ts
+++ b/tools/ui/src/lib/enums/model.enums.ts
@@ -80,6 +80,5 @@ export enum ModelRowDownloadState {
/** Column the models manager table can be ordered by. */
export enum ModelsTableSortKey {
CONTEXT = 'context',
- NAME = 'name',
- STATUS = 'status'
+ NAME = 'name'
}
diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts
index 69f010b70e..a6b081b9cf 100644
--- a/tools/ui/src/lib/stores/models/index.svelte.ts
+++ b/tools/ui/src/lib/stores/models/index.svelte.ts
@@ -622,6 +622,13 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
capabilities: rawCapabilities.filter((value: unknown): value is string =>
Boolean(value)
),
+ // 0 is the server's way of leaving the trained context unknown; a
+ // model-mode listing reports it as meta.n_ctx_train instead
+ contextLength:
+ item.context_length ||
+ (typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0
+ ? item.meta.n_ctx_train
+ : undefined),
description: details?.description,
details: details?.details,
draftSidecars: mergedDraftSidecars(
diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts
index b52485ade8..70b99eaa0d 100644
--- a/tools/ui/src/lib/types/api.d.ts
+++ b/tools/ui/src/lib/types/api.d.ts
@@ -100,6 +100,8 @@ export interface ApiModelDataEntry {
tags?: string[];
/** Modality capabilities, reported by the router for every model regardless of load state */
architecture?: ApiModelArchitecture;
+ /** Trained context of the model, read from its GGUF metadata at registration */
+ context_length?: number;
/** Legacy meta field (may be present in older responses) */
meta?: Record<string, unknown> | null;
}
diff --git a/tools/ui/tests/client/models-manager-sort-filter.svelte.test.ts b/tools/ui/tests/client/models-manager-sort-filter.svelte.test.ts
new file mode 100644
index 0000000000..9e9e0c2468
--- /dev/null
+++ b/tools/ui/tests/client/models-manager-sort-filter.svelte.test.ts
@@ -0,0 +1,243 @@
+// Guards the manager table ordering and filtering: the context column sorts and
+// filters once a model's context is known, from the option or the Hub record.
+
+import ModelsManagerWrapper from './components/ModelsManagerWrapper.svelte';
+import { SETTINGS_KEYS } from '$lib/constants';
+import { ServerModelStatus } from '$lib/enums';
+import { HuggingFaceService } from '$lib/services';
+import { modelsStore, settingsStore } from '$lib/stores';
+import type { ApiModelDataEntry } from '$lib/types';
+import type { ModelOption } from '$lib/types/models';
+import { SvelteMap } from 'svelte/reactivity';
+import { beforeEach, expect, it, vi } from 'vitest';
+import { render } from 'vitest-browser-svelte';
+
+function option(model: string, contextLength?: number): ModelOption {
+ return {
+ capabilities: [],
+ contextLength,
+ id: model,
+ model,
+ name: model
+ };
+}
+
+const models = [
+ option('org/alpha-8b:Q4_K_M', 8192),
+ option('org/beta-8b:Q4_K_M', 131072),
+ option('org/gamma-8b:Q4_K_M', 32768)
+];
+// the router listing carries no context, so a row starts without one
+const modelsWithoutContext = models.map((model) => ({ ...model, contextLength: undefined }));
+
+beforeEach(() => {
+ modelsStore.routerModels = [];
+ modelsStore.models = modelsWithoutContext;
+ modelsStore.favoriteModelIds = new Set();
+ settingsStore.config[SETTINGS_KEYS.GROUP_MODELS_BY_FAMILY] = false;
+});
+
+/** Renders the manager and waits for the test models to show. */
+async function renderWithModels(rows: ModelOption[] = modelsWithoutContext) {
+ const screen = render(ModelsManagerWrapper);
+
+ modelsStore.models = rows;
+
+ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();
+
+ return screen;
+}
+
+function rowNames(container: HTMLElement): string[] {
+ return [...container.querySelectorAll('button[aria-pressed]')].map(
+ (row) => row.textContent ?? ''
+ );
+}
+
+/** The Hub details cache the rows read their context from. */
+function detailsCache() {
+ return (
+ HuggingFaceService as unknown as {
+ detailsCache: SvelteMap<string, { gguf?: { context_length?: number } } | null>;
+ }
+ ).detailsCache;
+}
+
+function warmCache() {
+ const cache = detailsCache();
+
+ cache.set('org/alpha-8b', { gguf: { context_length: 8192 } });
+ cache.set('org/beta-8b', { gguf: { context_length: 131072 } });
+ cache.set('org/gamma-8b', { gguf: { context_length: 32768 } });
+}
+
+it('sorts by context', async () => {
+ const screen = await renderWithModels(models);
+
+ // lowest first
+ await screen.getByTitle('Sort by context, lowest first').click();
+
+ const ascending = rowNames(screen.container);
+
+ await screen.getByTitle('Sort by context, highest first').click();
+
+ const descending = rowNames(screen.container);
+
+ expect(ascending).not.toEqual(descending);
+});
+
+it('sorts by context with family grouping on', async () => {
+ settingsStore.config[SETTINGS_KEYS.GROUP_MODELS_BY_FAMILY] = true;
+
+ const screen = await renderWithModels(models);
+
+ // lowest first
+ await screen.getByTitle('Sort by context, lowest first').click();
+
+ const ascending = rowNames(screen.container);
+
+ await screen.getByTitle('Sort by context, highest first').click();
+
+ const descending = rowNames(screen.container);
+
+ expect(ascending).not.toEqual(descending);
+});
+
+it('lists the favorited quants of a repo as flat rows', async () => {
+ // one quant of the beta repo is a favorite, the other stays with the repo
+ modelsStore.favoriteModelIds = new Set(['org/alpha-8b:Q4_K_M', 'org/beta-8b:Q4_K_M']);
+
+ const screen = await renderWithModels([
+ modelsWithoutContext[0],
+ modelsWithoutContext[1],
+ option('org/beta-8b:Q8_0'),
+ modelsWithoutContext[2]
+ ]);
+
+ // the favorite quant is a model row of its own, not a repo with subitems
+ expect(screen.container.textContent).not.toContain('2 quants available');
+
+ // the repo appears once in favorites for its favorited quant and once in the
+ // local block for the quant left behind
+ expect(screen.getByText(/beta\s+8B/).elements().length).toBe(2);
+});
+
+/** A router listing entry carrying only the meta context. */
+function entry(model: string, nCtxTrain: number): ApiModelDataEntry {
+ return {
+ created: 0,
+ id: model,
+ in_cache: false,
+ meta: { n_ctx_train: nCtxTrain },
+ object: 'model',
+ owned_by: 'llamacpp',
+ path: `/models/${model}`,
+ status: { value: ServerModelStatus.UNLOADED }
+ };
+}
+
+it('sorts by the meta context of a listing without the router field', async () => {
+ // a listing that skips the router's GGUF read reports the trained context
+ // only as meta.n_ctx_train, so the option mapping falls back to it
+ vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: RequestInfo | URL) => {
+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+
+ if (url.includes('/props')) {
+ return new Response(
+ JSON.stringify({
+ default_generation_settings: { n_ctx: 0, params: {} },
+ model_alias: 'llama-server',
+ model_path: 'none',
+ role: 'router'
+ }),
+ { headers: { 'Content-Type': 'application/json' }, status: 200 }
+ );
+ }
+
+ if (url.includes('/server')) {
+ return new Response(
+ JSON.stringify({ git_branch: 'test', git_commit: 'test', mode: 'router', version: 'test' }),
+ { headers: { 'Content-Type': 'application/json' }, status: 200 }
+ );
+ }
+
+ if (/\/v1\/models|\/models\b/.test(url)) {
+ return new Response(
+ JSON.stringify({
+ data: [
+ entry('org/alpha-8b:Q4_K_M', 8192),
+ entry('org/beta-8b:Q4_K_M', 131072),
+ entry('org/gamma-8b:Q4_K_M', 32768)
+ ],
+ object: 'list'
+ }),
+ { headers: { 'Content-Type': 'application/json' }, status: 200 }
+ );
+ }
+
+ throw new Error(`unexpected fetch in the test: ${url}`);
+ });
+
+ const screen = render(ModelsManagerWrapper);
+
+ // the fetch maps the listing into options, the meta context fills in
+ await modelsStore.fetch(true);
+
+ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();
+
+ await screen.getByTitle('Sort by context, lowest first').click();
+
+ const names = rowNames(screen.container).join(' | ');
+
+ expect(names.indexOf('alpha')).toBeLessThan(names.indexOf('gamma'));
+ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));
+});
+
+it('re-sorts when the Hub details arrive after the sort was clicked', async () => {
+ const screen = await renderWithModels();
+
+ // the user sorts while the contexts are still unknown
+ await screen.getByTitle('Sort by context, lowest first').click();
+
+ // then the rows fetch their Hub records
+ warmCache();
+
+ // the table re-sorts once the cache answers
+ await vi.waitFor(() => {
+ const names = rowNames(screen.container).join(' | ');
+
+ expect(names.indexOf('alpha')).toBeLessThan(names.indexOf('gamma'));
+ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));
+
+ return names;
+ });
+});
+
+it('filters by search', async () => {
+ const screen = await renderWithModels();
+
+ await screen.getByPlaceholder('Search your models').fill('beta');
+
+ await expect.element(screen.getByText(/alpha\s+8B/)).not.toBeVisible();
+ await expect.element(screen.getByText(/beta\s+8B/)).toBeVisible();
+});
+
+it('filters by context and sorts from the Hub details cache', async () => {
+ warmCache();
+
+ const screen = await renderWithModels();
+
+ // open the context filter and ask for 32K or more
+ await screen.getByText('Context:').click();
+ await screen.getByText('32K or more').click();
+
+ await expect.element(screen.getByText(/alpha\s+8B/)).not.toBeVisible();
+ await expect.element(screen.getByText(/gamma\s+8B/)).toBeVisible();
+
+ // sorting re-runs once the cache answers
+ await screen.getByTitle('Sort by context, lowest first').click();
+
+ const names = rowNames(screen.container).join(' | ');
+
+ expect(names.indexOf('gamma')).toBeLessThan(names.indexOf('beta'));
+});