LLM inference in C/C++
C++ ggml
ui : key draft sidecar badges by model instead of repo (#30919)
Aleksander Grygiera6b83d426f90
diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.tsindex 38d734f4ee..3008eb5214 100644--- a/tools/ui/src/lib/stores/models/index.svelte.ts+++ b/tools/ui/src/lib/stores/models/index.svelte.ts@@ -578,7 +578,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {const displayNameSource =details?.name && details.name.trim().length > 0 ? details.name : item.id;const modelId = details?.model || item.id;- const repo = repoOf(modelId);return {aliases: item.aliases ?? [],@@ -594,7 +593,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {: undefined),description: details?.description,details: details?.details,- draftSidecars: this._status.getDraftSidecars(repo),+ draftSidecars: this._status.getDraftSidecars(modelId),id: item.id,meta: item.meta ?? null,modalities: this.props.buildArchitectureModalities(item.architecture),diff --git a/tools/ui/src/lib/stores/models/status.svelte.ts b/tools/ui/src/lib/stores/models/status.svelte.tsindex 4c00adbd78..a2e3b27522 100644--- a/tools/ui/src/lib/stores/models/status.svelte.ts+++ b/tools/ui/src/lib/stores/models/status.svelte.ts@@ -97,10 +97,11 @@ export class ModelStatusManager {/** Sidecar kinds each in-flight or paused download pulls with its model. */private downloadSidecars = new SvelteMap<string, ModelSidecar[]>();/**- * Draft sidecars the registered models pull with them, keyed by the repo of the model- * that pulls them: the router records them in the model's `--model-draft` args.+ * Draft sidecars the registered models pull with them, keyed by the model that pulls+ * them: the router records them in the model's `--model-draft` args, and each quant+ * of a repo can pull a different draft.*/- private draftSidecarsByRepo = $derived.by(() => {+ private draftSidecarsByModel = $derived.by(() => {const result = new SvelteMap<string, ModelSidecarBadge[]>();for (const m of this.host.routerModels) {@@ -108,8 +109,6 @@ export class ModelStatusManager {if (!args) continue;- const baseRepo = repoOf(m.id);-for (let i = 0; i < args.length - 1; i++) {if (args[i] !== CLI_FLAGS.MODEL_DRAFT && args[i] !== CLI_FLAGS.MODEL_DRAFT_SHORT) {continue;@@ -128,7 +127,7 @@ export class ModelStatusManager {quant: meta.quant,repo: parsed.repo};- const kinds = result.get(baseRepo);+ const kinds = result.get(m.id);if (kinds) {if (@@ -139,7 +138,7 @@ export class ModelStatusManager {kinds.push(badge);}} else {- result.set(baseRepo, [badge]);+ result.set(m.id, [badge]);}}}@@ -404,17 +403,19 @@ export class ModelStatusManager {}/**- * Draft sidecars a repo pulls with its model, e.g. `mtp` at `Q8_0`: the- * router records loaded models' sidecars in their `--model-draft` args, and- * lists the sidecars each cached model has on disk.+ * Draft sidecars a model pulls with it, e.g. `mtp` at `Q8_0`: the router+ * records each model's sidecars in its `--model-draft` args, and lists the+ * sidecars each cached repo has on disk.*/- getDraftSidecars(repoId: string): ModelSidecarBadge[] {- // one badge per kind: the args know the quant, the listing carries kinds only,- // and a draft still coming down is pending rather than on disk- const merged = [...(this.draftSidecarsByRepo.get(repoId) ?? [])];+ getDraftSidecars(modelId: string): ModelSidecarBadge[] {+ const repo = repoOf(modelId);+ // one badge per kind: the args know the quant of this model's draft, the+ // listing carries repo-wide kinds only, and a draft still coming down is+ // pending rather than on disk+ const merged = [...(this.draftSidecarsByModel.get(modelId) ?? [])];for (const source of [this.listedSidecarsByRepo, this.pendingSidecarsByRepo]) {- for (const badge of source.get(repoId) ?? []) {+ for (const badge of source.get(repo) ?? []) {if (!merged.some((b) => b.kind === badge.kind && b.repo === badge.repo)) {merged.push(badge);}diff --git a/tools/ui/tests/client/models-sidecar-badges.svelte.test.ts b/tools/ui/tests/client/models-sidecar-badges.svelte.test.tsindex a5754abc1f..03a402b74d 100644--- a/tools/ui/tests/client/models-sidecar-badges.svelte.test.ts+++ b/tools/ui/tests/client/models-sidecar-badges.svelte.test.ts@@ -1,6 +1,7 @@-// Guards the draft badges a repo row shows: a loaded model records its draft in-// the `--model-draft` args, which know the file's quant, and the listing reports-// the kinds of every cached model, which do not. The row shows each kind once.+// Guards the draft badges a model row shows: a registered model records its draft+// in its `--model-draft` args, which know the file's quant, and the listing reports+// the kinds every cached model of the repo has on disk, which do not. The row shows+// each kind once, and two quants of one repo keep their own drafts apart.import { ModelAuxSidecar, ModelDraftSidecar, ServerModelStatus } from '$lib/enums';import { modelsStore } from '$lib/stores';@@ -9,6 +10,11 @@ import { beforeEach, describe, expect, it } from 'vitest';const REPO = 'ggml-org/Qwen3.8-Flash-Next-GGUF';+/** Draft file of a cached model, as a preset records it in `--model-draft`. */+function draftPath(quant: string): string {+ return `/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-${quant}.gguf`;+}+/** A router entry; `args` stand in for the args of a loaded instance. */function entry(id: string,@@ -36,18 +42,28 @@ describe('draft sidecar badges', () => {it('shows one badge for a draft the args and the listing both report', () => {modelsStore.routerModels = [entry(- `${REPO}:IQ4_NL`,- [],- [- '--model-draft',- `/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-Q8_0.gguf`- ]- ),- entry(`${REPO}:Q8_0`, [ModelAuxSidecar.MMPROJ, ModelDraftSidecar.MTP])+ `${REPO}:Q8_0`,+ [ModelAuxSidecar.MMPROJ, ModelDraftSidecar.MTP],+ ['--model-draft', draftPath('Q8_0')]+ )];// the args badge wins: it knows the quant the listing does not- expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([+ expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([+ { kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }+ ]);+ });++ it('gives each quant of a repo the draft it pulls itself', () => {+ modelsStore.routerModels = [+ entry(`${REPO}:IQ4_NL`, [], ['--model-draft', draftPath('Q4_0')]),+ entry(`${REPO}:Q8_0`, [], ['--model-draft', draftPath('Q8_0')])+ ];++ expect(modelsStore.status.getDraftSidecars(`${REPO}:IQ4_NL`)).toEqual([+ { kind: ModelDraftSidecar.MTP, quant: 'Q4_0', repo: REPO }+ ]);+ expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }]);});@@ -58,7 +74,7 @@ describe('draft sidecar badges', () => {entry(`${REPO}:Q8_0`, [ModelDraftSidecar.MTP])];- expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([+ expect(modelsStore.status.getDraftSidecars(`${REPO}:IQ4_NL`)).toEqual([{ kind: ModelDraftSidecar.MTP, quant: null, repo: REPO }]);});@@ -74,7 +90,7 @@ describe('draft sidecar badges', () => {];// the kinds are not on disk yet, so the row reads them as in flight- expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([+ expect(modelsStore.status.getDraftSidecars(`${REPO}:Q4_0`)).toEqual([{ kind: ModelDraftSidecar.MTP, pending: true, quant: null, repo: REPO }]);});@@ -82,17 +98,10 @@ describe('draft sidecar badges', () => {it('keeps the on-disk badge of a kind a download also pulls', () => {modelsStore.routerModels = [entry(`${REPO}:IQ4_NL`, [ModelDraftSidecar.MTP], [], ServerModelStatus.INCOMPLETE),- entry(- `${REPO}:Q8_0`,- [],- [- '--model-draft',- `/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-Q8_0.gguf`- ]- )+ entry(`${REPO}:Q8_0`, [], ['--model-draft', draftPath('Q8_0')])];- expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([+ expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }]);});