LLM inference in C/C++
C++ ggml
ui: loading bar below the model picker (#24931)
Pascal00139b660b3a
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelteindex a1d02d6ece..14f6c5ca02 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte@@ -180,6 +180,9 @@let displayedModel = $derived(message.model ?? null);+ // model being switched to while it loads, so the selector bar tracks it+ let pendingModel = $state<string | null>(null);+let isCurrentlyLoading = $derived(isLoading());let isStreaming = $derived(isChatStreaming());let hasNoContent = $derived(!message?.content?.trim());@@ -318,13 +321,19 @@>{#if isRouter}<ModelsSelectorDropdown- currentModel={displayedModel}+ currentModel={pendingModel ?? displayedModel}disabled={isLoading()}onModelChange={async (modelId: string, modelName: string) => {const status = modelsStore.getModelStatus(modelId);if (status !== ServerModelStatus.LOADED) {- await modelsStore.loadModel(modelId);+ pendingModel = modelId;++ try {+ await modelsStore.loadModel(modelId);+ } finally {+ pendingModel = null;+ }}onRegenerate(modelName);diff --git a/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte b/tools/ui/src/lib/components/app/models/ModelLoadHighlight.sveltenew file mode 100644index 0000000000..fa9a02108a--- /dev/null+++ b/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte@@ -0,0 +1,11 @@+<script lang="ts">+ let { percent }: { percent: number } = $props();+</script>++<!-- thin determinate load bar pinned to the bottom edge, pulsing while it fills -->+<div class="pointer-events-none absolute inset-x-0 bottom-0 h-0.5 overflow-hidden rounded-b-sm">+ <div+ class="h-full animate-pulse bg-primary transition-[width] duration-200 ease-out"+ style="width: {percent}%"+ ></div>+</div>diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelteindex 40006a4c93..720963a8db 100644--- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte@@ -2,8 +2,10 @@import { ChevronDown, Loader2, Package } from '@lucide/svelte';import * as DropdownMenu from '$lib/components/ui/dropdown-menu';import * as Tooltip from '$lib/components/ui/tooltip';- import { KeyboardKey } from '$lib/enums';+ import { KeyboardKey, ServerModelStatus } from '$lib/enums';import { useModelsSelector } from '$lib/hooks/use-models-selector.svelte';+ import { modelsStore, routerModels } from '$lib/stores/models.svelte';+ import { modelLoadFraction } from '$lib/utils';import {DialogModelInformation,DropdownMenuSearchable,@@ -11,6 +13,7 @@ModelsSelectorList,ModelsSelectorOption} from '$lib/components/app';+ import ModelLoadHighlight from './ModelLoadHighlight.svelte';import type { ModelItem } from './utils';interface Props {@@ -113,6 +116,17 @@{/if}{:else}{@const selectedOption = ms.getDisplayOption()}+ {@const triggerModel = selectedOption?.model}+ {@const triggerStatus = triggerModel+ ? routerModels().find((m) => m.id === triggerModel)?.status?.value+ : undefined}+ {@const triggerLoading =+ !!triggerModel &&+ (triggerStatus === ServerModelStatus.LOADING ||+ modelsStore.isModelOperationInProgress(triggerModel))}+ {@const triggerLoadPercent = triggerLoading+ ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100)+ : 0}{#if ms.isRouter}<DropdownMenu.Root bind:open={isOpen} onOpenChange={ms.handleOpenChange}>@@ -123,7 +137,7 @@<DropdownMenu.Trigger{...props}class={[- `inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,+ `relative inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,!ms.isCurrentModelInCache? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400': forceForegroundText@@ -154,6 +168,10 @@{:else}<ChevronDown class="h-3 w-3.5 shrink-0" />{/if}++ {#if triggerLoading}+ <ModelLoadHighlight percent={triggerLoadPercent} />+ {/if}</DropdownMenu.Trigger>{/snippet}</Tooltip.Trigger>diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelteindex f2a024d31d..981111e201 100644--- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte@@ -10,6 +10,7 @@RotateCw} from '@lucide/svelte';import { ActionIcon, ModelId } from '$lib/components/app';+ import ModelLoadHighlight from './ModelLoadHighlight.svelte';import type { ModelOption } from '$lib/types/models';import { ServerModelStatus } from '$lib/enums';import { modelsStore, routerModels } from '$lib/stores/models.svelte';@@ -119,11 +120,11 @@</div>{#if isLoading}- <div class="flex w-4 [@media(pointer:coarse)]:w-5 items-center justify-center">+ <div class="flex w-4 items-center justify-center [@media(pointer:coarse)]:w-5"><Loader2 class="h-4 w-4 animate-spin text-muted-foreground" /></div>{:else if isFailed}- <div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">+ <div class="flex w-4 items-center justify-center [@media(pointer:coarse)]:w-auto"><CircleAlertclass="h-3.5 w-3.5 text-red-500 group-hover:hidden [@media(pointer:coarse)]:hidden"/>@@ -140,7 +141,7 @@</div></div>{:else if isSleeping}- <div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">+ <div class="flex w-4 items-center justify-center [@media(pointer:coarse)]:w-auto"><spanclass="h-2 w-2 rounded-full bg-orange-400 group-hover:hidden [@media(pointer:coarse)]:hidden"></span>@@ -159,7 +160,7 @@</div></div>{:else if isLoaded}- <div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">+ <div class="flex w-4 items-center justify-center [@media(pointer:coarse)]:w-auto"><spanclass="h-2 w-2 rounded-full bg-green-500 group-hover:hidden [@media(pointer:coarse)]:hidden"></span>@@ -176,7 +177,7 @@</div></div>{:else}- <div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">+ <div class="flex w-4 items-center justify-center [@media(pointer:coarse)]:w-auto"><spanclass="h-2 w-2 rounded-full bg-muted-foreground/50 group-hover:hidden [@media(pointer:coarse)]:hidden"></span>@@ -196,13 +197,6 @@</div>{#if isLoading}- <div- class="pointer-events-none absolute inset-x-0 bottom-0 h-0.5 overflow-hidden rounded-b-sm bg-muted"- >- <div- class="h-full bg-primary transition-[width] duration-200 ease-out"- style="width: {loadPercent}%"- ></div>- </div>+ <ModelLoadHighlight percent={loadPercent} />{/if}</div>diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelteindex e5920a2a0f..a9e9ea1c8d 100644--- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte+++ b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte@@ -8,6 +8,10 @@ModelsSelectorList,SearchInput} from '$lib/components/app';+ import ModelLoadHighlight from './ModelLoadHighlight.svelte';+ import { ServerModelStatus } from '$lib/enums';+ import { modelsStore, routerModels } from '$lib/stores/models.svelte';+ import { modelLoadFraction } from '$lib/utils';interface Props {class?: string;@@ -61,12 +65,23 @@<p class="text-xs text-muted-foreground">No models available.</p>{:else}{@const selectedOption = ms.getDisplayOption()}+ {@const triggerModel = selectedOption?.model}+ {@const triggerStatus = triggerModel+ ? routerModels().find((m) => m.id === triggerModel)?.status?.value+ : undefined}+ {@const triggerLoading =+ !!triggerModel &&+ (triggerStatus === ServerModelStatus.LOADING ||+ modelsStore.isModelOperationInProgress(triggerModel))}+ {@const triggerLoadPercent = triggerLoading+ ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100)+ : 0}{#if ms.isRouter}<buttontype="button"class={[- `inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 max-sm:px-3 max-sm:py-2 text-xs max-sm:text-sm shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,+ `relative inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 max-sm:px-3 max-sm:py-2 max-sm:text-sm dark:bg-muted-foreground/15 dark:text-secondary-foreground`,!ms.isCurrentModelInCache? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400': forceForegroundText@@ -99,6 +114,10 @@{:else}<ChevronDown class="h-3 w-3.5 shrink-0" />{/if}++ {#if triggerLoading}+ <ModelLoadHighlight percent={triggerLoadPercent} />+ {/if}</button><Sheet.Root bind:open={sheetOpen} onOpenChange={handleSheetOpenChange}>