Personal AI, On Personal Devices
Python
feat: model catalogue with download/delete and Qwen3.5 auto-pull (#54)
Jon Saad-Falcon5fdd7d0e7596
diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lockindex ba5f0965..55112ab4 100644--- a/desktop/src-tauri/Cargo.lock+++ b/desktop/src-tauri/Cargo.lock@@ -2573,7 +2573,7 @@ dependencies = [[[package]]name = "openjarvis-desktop"-version = "1.0.0"+version = "0.1.0"dependencies = ["reqwest 0.12.28","serde",diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rsindex d1f26918..43bdddb2 100644--- a/desktop/src-tauri/src/lib.rs+++ b/desktop/src-tauri/src/lib.rs@@ -8,7 +8,74 @@ use tokio::sync::Mutex;const OLLAMA_PORT: u16 = 11434;const JARVIS_PORT: u16 = 8222;-const DEFAULT_MODEL: &str = "qwen3:0.6b";++/// Tiny fallback model — always available even on low-RAM machines.+const FALLBACK_MODEL: &str = "qwen3:0.6b";++/// Qwen3.5 model variants, ordered smallest to largest.+/// Each entry is (ollama_tag, approximate_download_size_gb, min_ram_gb).+const QWEN35_MODELS: &[(&str, f64, f64)] = &[+ ("qwen3.5:0.8b", 1.0, 4.0),+ ("qwen3.5:2b", 2.7, 6.0),+ ("qwen3.5:4b", 3.4, 8.0),+ ("qwen3.5:9b", 6.6, 12.0),+ ("qwen3.5:27b", 17.0, 24.0),+ ("qwen3.5:35b", 24.0, 32.0),+ ("qwen3.5:122b", 81.0, 96.0),+];++/// Get total system RAM in GB.+fn total_ram_gb() -> f64 {+ #[cfg(target_os = "macos")]+ {+ use std::process::Command;+ if let Ok(output) = Command::new("sysctl").args(["-n", "hw.memsize"]).output() {+ if let Ok(s) = String::from_utf8(output.stdout) {+ if let Ok(bytes) = s.trim().parse::<u64>() {+ return bytes as f64 / (1024.0 * 1024.0 * 1024.0);+ }+ }+ }+ }+ #[cfg(target_os = "linux")]+ {+ if let Ok(contents) = std::fs::read_to_string("/proc/meminfo") {+ for line in contents.lines() {+ if line.starts_with("MemTotal:") {+ if let Some(kb_str) = line.split_whitespace().nth(1) {+ if let Ok(kb) = kb_str.parse::<u64>() {+ return kb as f64 / (1024.0 * 1024.0);+ }+ }+ }+ }+ }+ }+ 8.0+}++/// Return the list of Qwen3.5 models that fit on this machine, smallest first.+fn models_that_fit() -> Vec<&'static str> {+ let ram = total_ram_gb();+ QWEN35_MODELS+ .iter()+ .filter(|(_, _, min_ram)| ram >= *min_ram)+ .map(|(tag, _, _)| *tag)+ .collect()+}++/// Pick the second-largest Qwen3.5 model that fits on this machine.+/// This leaves headroom for the OS / other apps while still providing+/// a capable model. Falls back to the largest if only one fits, or+/// to FALLBACK_MODEL if none fit.+fn preferred_model() -> &'static str {+ let fitting = models_that_fit();+ match fitting.len() {+ 0 => FALLBACK_MODEL,+ 1 => fitting[0],+ n => fitting[n - 2], // second-largest+ }+}/// Resolve full path to a binary by checking common locations./// macOS .app bundles don't inherit the shell PATH, so we probe manually.@@ -212,7 +279,11 @@ async fn ollama_has_model(model: &str) -> bool {return models.iter().any(|m| {m.get("name").and_then(|n| n.as_str())- .map(|n| n.starts_with(model.split(':').next().unwrap_or(model)))+ .map(|n| {+ n == model+ || n.strip_suffix(":latest") == Some(model)+ || model.strip_suffix(":latest") == Some(n)+ }).unwrap_or(false)});}@@ -285,19 +356,22 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {s.detail = "Inference engine ready.".into();}- // Phase 2: Ensure a default model exists+ // Phase 2: Ensure at least one model exists.+ // Pull the small fallback first so the server can start immediately,+ // then pull the preferred model and remaining Qwen3.5 variants in+ // the background (Phase 4).{let mut s = status.lock().await;s.phase = "model".into();- s.detail = format!("Checking for model {}...", DEFAULT_MODEL);+ s.detail = format!("Checking for model {}...", FALLBACK_MODEL);}- if !ollama_has_model(DEFAULT_MODEL).await {+ if !ollama_has_model(FALLBACK_MODEL).await {{let mut s = status.lock().await;- s.detail = format!("Downloading {}... (this may take a minute)", DEFAULT_MODEL);+ s.detail = format!("Downloading {}... (this may take a minute)", FALLBACK_MODEL);}- if let Err(e) = pull_model(DEFAULT_MODEL).await {+ if let Err(e) = pull_model(FALLBACK_MODEL).await {let mut s = status.lock().await;s.error = Some(format!("Failed to download model: {}", e));return;@@ -331,8 +405,21 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {return;}+ // Use the preferred model if it's already pulled, otherwise fallback.+ let pref = preferred_model();+ let startup_model = if ollama_has_model(pref).await {+ pref+ } else {+ FALLBACK_MODEL+ };+let mut cmd = tokio::process::Command::new(&uv_bin);- cmd.args(["run", "jarvis", "serve", "--port", &JARVIS_PORT.to_string(), "--agent", "simple"])+ cmd.args([+ "run", "jarvis", "serve",+ "--port", &JARVIS_PORT.to_string(),+ "--model", startup_model,+ "--agent", "simple",+ ]).stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).current_dir(project_root.as_ref().unwrap());@@ -368,6 +455,27 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {s.phase = "ready".into();s.detail = "All systems ready.".into();}++ // Phase 4: Pull preferred model + remaining Qwen3.5 variants in the+ // background. The server is already running with the fallback, so+ // the user can chat immediately. As each model finishes pulling it+ // appears in the /v1/models list (Ollama serves it automatically).+ let fitting = models_that_fit();+ let pref_bg = preferred_model().to_string();+ tokio::spawn(async move {+ // Pull the preferred model first so it becomes available quickly.+ if !ollama_has_model(&pref_bg).await {+ let _ = pull_model(&pref_bg).await;+ }+ // Then pull remaining models that fit.+ for model in fitting {+ if model != pref_bg && model != FALLBACK_MODEL {+ if !ollama_has_model(model).await {+ let _ = pull_model(model).await;+ }+ }+ }+ });}// ---------------------------------------------------------------------------diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsxindex 8acc118a..aaa244d5 100644--- a/frontend/src/components/Chat/InputArea.tsx+++ b/frontend/src/components/Chat/InputArea.tsx@@ -26,6 +26,22 @@ export function InputArea() {const { state: speechState, available: speechAvailable, startRecording, stopRecording } = useSpeech();+ // Abort in-flight stream when the user switches models mid-generation.+ // This prevents errors from trying to continue a stream with a stale model.+ const prevModelRef = useRef(selectedModel);+ useEffect(() => {+ if (prevModelRef.current !== selectedModel && streamState.isStreaming) {+ abortRef.current?.abort();+ if (timerRef.current) {+ clearInterval(timerRef.current);+ timerRef.current = null;+ }+ resetStream();+ abortRef.current = null;+ }+ prevModelRef.current = selectedModel;+ }, [selectedModel, streamState.isStreaming, resetStream]);+const micDisabled = !speechEnabled || !speechAvailable || streamState.isStreaming;const micReason: 'not-enabled' | 'no-backend' | 'streaming' | undefined =!speechEnabled ? 'not-enabled'@@ -189,9 +205,13 @@ export function InputArea() {}}} catch (err: any) {- if (err.name !== 'AbortError') {+ if (err.name === 'AbortError') {+ // User cancelled or model switch — keep whatever was accumulated+ if (!accumulatedContent) accumulatedContent = '(Generation stopped)';+ } else {+ const errMsg = err?.message || String(err);accumulatedContent =- accumulatedContent || 'Error: Failed to get response.';+ accumulatedContent || `Error: ${errMsg}`;}} finally {if (!accumulatedContent) {diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsxindex f58d170a..4bb79d6f 100644--- a/frontend/src/components/CommandPalette.tsx+++ b/frontend/src/components/CommandPalette.tsx@@ -1,22 +1,53 @@import { useState, useRef, useEffect } from 'react';-import { Search, Cpu, X } from 'lucide-react';+import { Search, Cpu, X, Download, Loader2, Trash2, Check } from 'lucide-react';import { useAppStore } from '../lib/store';+import { pullModel, deleteModel, fetchModels } from '../lib/api';++/** Popular models that users can download from the catalogue. */+const CATALOGUE_MODELS = [+ { id: 'qwen3.5:0.8b', size: '~1 GB', desc: 'Qwen 3.5 0.8B — fast, lightweight' },+ { id: 'qwen3.5:2b', size: '~2.7 GB', desc: 'Qwen 3.5 2B' },+ { id: 'qwen3.5:4b', size: '~3.4 GB', desc: 'Qwen 3.5 4B — recommended default' },+ { id: 'qwen3.5:9b', size: '~6.6 GB', desc: 'Qwen 3.5 9B' },+ { id: 'qwen3.5:27b', size: '~17 GB', desc: 'Qwen 3.5 27B' },+ { id: 'qwen3.5:35b', size: '~24 GB', desc: 'Qwen 3.5 35B' },+ { id: 'qwen3.5:122b', size: '~81 GB', desc: 'Qwen 3.5 122B — largest' },+ { id: 'llama3.3:latest', size: '~4.9 GB', desc: 'Llama 3.3 8B' },+ { id: 'mistral:latest', size: '~4.1 GB', desc: 'Mistral 7B' },+ { id: 'gemma3:latest', size: '~3.3 GB', desc: 'Gemma 3 4B' },+ { id: 'deepseek-r1:7b', size: '~4.7 GB', desc: 'DeepSeek R1 7B' },+ { id: 'phi4:latest', size: '~9.1 GB', desc: 'Phi-4 14B' },+];++type Tab = 'installed' | 'catalogue';export function CommandPalette() {const [query, setQuery] = useState('');const [selectedIdx, setSelectedIdx] = useState(0);+ const [tab, setTab] = useState<Tab>('installed');+ const [pulling, setPulling] = useState<string | null>(null);+ const [pullError, setPullError] = useState<string | null>(null);+ const [pullSuccess, setPullSuccess] = useState<string | null>(null);+ const [deleting, setDeleting] = useState<string | null>(null);+ const [customModel, setCustomModel] = useState('');const inputRef = useRef<HTMLInputElement>(null);const models = useAppStore((s) => s.models);const selectedModel = useAppStore((s) => s.selectedModel);const setSelectedModel = useAppStore((s) => s.setSelectedModel);+ const setModels = useAppStore((s) => s.setModels);const setCommandPaletteOpen = useAppStore((s) => s.setCommandPaletteOpen);- const filtered = query- ? models.filter((m) =>- m.id.toLowerCase().includes(query.toLowerCase()),- )- : models;+ const installedIds = new Set(models.map((m) => m.id));++ const filtered = tab === 'installed'+ ? (query+ ? models.filter((m) => m.id.toLowerCase().includes(query.toLowerCase()))+ : models)+ : CATALOGUE_MODELS.filter((m) =>+ !installedIds.has(m.id) &&+ (!query || m.id.toLowerCase().includes(query.toLowerCase()) || m.desc.toLowerCase().includes(query.toLowerCase()))+ );useEffect(() => {inputRef.current?.focus();@@ -24,13 +55,66 @@ export function CommandPalette() {useEffect(() => {setSelectedIdx(0);- }, [query]);+ }, [query, tab]);++ // Clear success message after a delay+ useEffect(() => {+ if (pullSuccess) {+ const t = setTimeout(() => setPullSuccess(null), 3000);+ return () => clearTimeout(t);+ }+ }, [pullSuccess]);const handleSelect = (modelId: string) => {setSelectedModel(modelId);setCommandPaletteOpen(false);};+ const refreshModels = async () => {+ try {+ const m = await fetchModels();+ setModels(m);+ } catch {}+ };++ const handlePull = async (modelId: string) => {+ setPulling(modelId);+ setPullError(null);+ try {+ await pullModel(modelId);+ setPullSuccess(modelId);+ await refreshModels();+ // Auto-select the newly pulled model+ setSelectedModel(modelId);+ } catch (e: any) {+ setPullError(e.message || 'Download failed');+ } finally {+ setPulling(null);+ }+ };++ const handleDelete = async (modelId: string) => {+ if (!confirm(`Delete model ${modelId}? You can re-download it later.`)) return;+ setDeleting(modelId);+ try {+ await deleteModel(modelId);+ await refreshModels();+ if (selectedModel === modelId) {+ const remaining = models.filter((m) => m.id !== modelId);+ if (remaining.length > 0) setSelectedModel(remaining[0].id);+ }+ } catch {} finally {+ setDeleting(null);+ }+ };++ const handleCustomPull = async () => {+ const name = customModel.trim();+ if (!name) return;+ await handlePull(name);+ setCustomModel('');+ };+const handleKeyDown = (e: React.KeyboardEvent) => {if (e.key === 'Escape') {setCommandPaletteOpen(false);@@ -40,9 +124,9 @@ export function CommandPalette() {} else if (e.key === 'ArrowUp') {e.preventDefault();setSelectedIdx((i) => Math.max(i - 1, 0));- } else if (e.key === 'Enter' && filtered.length > 0) {+ } else if (e.key === 'Enter' && tab === 'installed' && filtered.length > 0) {e.preventDefault();- handleSelect(filtered[selectedIdx].id);+ handleSelect((filtered[selectedIdx] as any).id);}};@@ -64,6 +148,27 @@ export function CommandPalette() {}}onClick={(e) => e.stopPropagation()}>+ {/* Tabs */}+ <div+ className="flex"+ style={{ borderBottom: '1px solid var(--color-border)' }}+ >+ {(['installed', 'catalogue'] as Tab[]).map((t) => (+ <button+ key={t}+ onClick={() => setTab(t)}+ className="flex-1 px-4 py-2.5 text-xs font-medium transition-colors cursor-pointer"+ style={{+ color: tab === t ? 'var(--color-accent)' : 'var(--color-text-tertiary)',+ borderBottom: tab === t ? '2px solid var(--color-accent)' : '2px solid transparent',+ background: 'transparent',+ }}+ >+ {t === 'installed' ? `Installed (${models.length})` : 'Download Models'}+ </button>+ ))}+ </div>+{/* Search input */}<divclassName="flex items-center gap-3 px-4 py-3"@@ -76,7 +181,7 @@ export function CommandPalette() {value={query}onChange={(e) => setQuery(e.target.value)}onKeyDown={handleKeyDown}- placeholder="Search models..."+ placeholder={tab === 'installed' ? 'Search installed models...' : 'Search models to download...'}className="flex-1 bg-transparent outline-none text-sm"style={{ color: 'var(--color-text)' }}/>@@ -89,49 +194,163 @@ export function CommandPalette() {</button></div>+ {/* Status messages */}+ {pullError && (+ <div className="px-4 py-2 text-xs" style={{ color: 'var(--color-error)', background: 'rgba(220,38,38,0.05)' }}>+ {pullError}+ </div>+ )}+ {pullSuccess && (+ <div className="px-4 py-2 text-xs flex items-center gap-1.5" style={{ color: 'var(--color-success)', background: 'rgba(34,197,94,0.05)' }}>+ <Check size={12} /> Downloaded {pullSuccess} successfully+ </div>+ )}+{/* Results */}<div className="max-h-[300px] overflow-y-auto py-2">- {filtered.length === 0 ? (- <div className="px-4 py-6 text-center text-sm" style={{ color: 'var(--color-text-tertiary)' }}>- {models.length === 0 ? 'No models available — is the server running?' : 'No matching models'}- </div>+ {tab === 'installed' ? (+ /* ── Installed models tab ── */+ filtered.length === 0 ? (+ <div className="px-4 py-6 text-center text-sm" style={{ color: 'var(--color-text-tertiary)' }}>+ {models.length === 0+ ? 'No models available — switch to "Download Models" to get started'+ : 'No matching models'}+ </div>+ ) : (+ (filtered as typeof models).map((model, idx) => {+ const isActive = model.id === selectedModel;+ const isSelected = idx === selectedIdx;+ const isDeleting = deleting === model.id;+ return (+ <div+ key={model.id}+ className="flex items-center gap-3 w-full px-4 py-2.5 transition-colors"+ style={{+ background: isSelected ? 'var(--color-bg-secondary)' : 'transparent',+ }}+ onMouseEnter={() => setSelectedIdx(idx)}+ >+ <button+ onClick={() => handleSelect(model.id)}+ className="flex items-center gap-3 flex-1 min-w-0 text-left cursor-pointer"+ style={{ background: 'none', border: 'none', padding: 0 }}+ >+ <Cpu size={16} style={{ color: isActive ? 'var(--color-accent)' : 'var(--color-text-tertiary)' }} />+ <div className="flex-1 min-w-0">+ <div+ className="text-sm truncate"+ style={{+ color: isActive ? 'var(--color-accent)' : 'var(--color-text)',+ fontWeight: isActive ? 500 : 400,+ }}+ >+ {model.id}+ </div>+ </div>+ {isActive && (+ <span className="text-[10px] px-2 py-0.5 rounded-full" style={{+ background: 'var(--color-accent-subtle)',+ color: 'var(--color-accent)',+ }}>+ Active+ </span>+ )}+ </button>+ <button+ onClick={() => handleDelete(model.id)}+ disabled={isDeleting}+ className="p-1 rounded transition-colors cursor-pointer opacity-0 group-hover:opacity-100"+ style={{ color: 'var(--color-text-tertiary)' }}+ title="Delete model"+ onMouseEnter={(e) => (e.currentTarget.style.opacity = '1', e.currentTarget.style.color = 'var(--color-error)')}+ onMouseLeave={(e) => (e.currentTarget.style.opacity = '0', e.currentTarget.style.color = 'var(--color-text-tertiary)')}+ >+ {isDeleting ? <Loader2 size={14} className="animate-spin" /> : <Trash2 size={14} />}+ </button>+ </div>+ );+ })+ )) : (- filtered.map((model, idx) => {- const isActive = model.id === selectedModel;- const isSelected = idx === selectedIdx;- return (- <button- key={model.id}- onClick={() => handleSelect(model.id)}- className="flex items-center gap-3 w-full px-4 py-2.5 text-left transition-colors cursor-pointer"- style={{- background: isSelected ? 'var(--color-bg-secondary)' : 'transparent',- }}- onMouseEnter={() => setSelectedIdx(idx)}- >- <Cpu size={16} style={{ color: isActive ? 'var(--color-accent)' : 'var(--color-text-tertiary)' }} />- <div className="flex-1 min-w-0">- <div- className="text-sm truncate"+ /* ── Catalogue tab ── */+ <>+ {(filtered as typeof CATALOGUE_MODELS).map((model) => {+ const isPulling = pulling === model.id;+ const justInstalled = pullSuccess === model.id;+ return (+ <div+ key={model.id}+ className="flex items-center gap-3 w-full px-4 py-2.5 transition-colors"+ style={{ background: 'transparent' }}+ >+ <Download size={16} style={{ color: 'var(--color-text-tertiary)' }} />+ <div className="flex-1 min-w-0">+ <div className="text-sm truncate" style={{ color: 'var(--color-text)' }}>+ {model.id}+ </div>+ <div className="text-[11px]" style={{ color: 'var(--color-text-tertiary)' }}>+ {model.desc} · {model.size}+ </div>+ </div>+ <button+ onClick={() => handlePull(model.id)}+ disabled={isPulling || !!pulling}+ className="flex items-center gap-1.5 px-3 py-1 rounded-lg text-xs font-medium transition-colors cursor-pointer"style={{- color: isActive ? 'var(--color-accent)' : 'var(--color-text)',- fontWeight: isActive ? 500 : 400,+ background: justInstalled ? 'var(--color-accent-subtle)' : 'var(--color-accent)',+ color: justInstalled ? 'var(--color-accent)' : '#fff',+ opacity: (isPulling || (pulling && !isPulling)) ? 0.5 : 1,}}>- {model.id}- </div>+ {isPulling ? (+ <><Loader2 size={12} className="animate-spin" /> Downloading...</>+ ) : justInstalled ? (+ <><Check size={12} /> Installed</>+ ) : (+ <><Download size={12} /> Download</>+ )}+ </button></div>- {isActive && (- <span className="text-[10px] px-2 py-0.5 rounded-full" style={{- background: 'var(--color-accent-subtle)',- color: 'var(--color-accent)',- }}>- Active- </span>- )}- </button>- );- })+ );+ })}++ {/* Custom model input */}+ <div+ className="px-4 py-3 mt-1"+ style={{ borderTop: '1px solid var(--color-border)' }}+ >+ <div className="text-[11px] mb-2" style={{ color: 'var(--color-text-tertiary)' }}>+ Or enter any Ollama model name:+ </div>+ <div className="flex gap-2">+ <input+ type="text"+ value={customModel}+ onChange={(e) => setCustomModel(e.target.value)}+ onKeyDown={(e) => { if (e.key === 'Enter') { e.preventDefault(); handleCustomPull(); } }}+ placeholder="e.g. codellama:7b"+ className="flex-1 text-sm px-3 py-1.5 rounded-lg outline-none"+ style={{+ background: 'var(--color-bg-secondary)',+ color: 'var(--color-text)',+ border: '1px solid var(--color-border)',+ }}+ />+ <button+ onClick={handleCustomPull}+ disabled={!customModel.trim() || !!pulling}+ className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-xs font-medium transition-colors cursor-pointer"+ style={{+ background: 'var(--color-accent)',+ color: '#fff',+ opacity: (!customModel.trim() || pulling) ? 0.5 : 1,+ }}+ >+ <Download size={12} /> Pull+ </button>+ </div>+ </div>+ </>)}</div>@@ -140,9 +359,15 @@ export function CommandPalette() {className="flex items-center gap-4 px-4 py-2 text-[11px]"style={{ borderTop: '1px solid var(--color-border)', color: 'var(--color-text-tertiary)' }}>- <span><kbd className="font-mono">↑↓</kbd> Navigate</span>- <span><kbd className="font-mono">Enter</kbd> Select</span>- <span><kbd className="font-mono">Esc</kbd> Close</span>+ {tab === 'installed' ? (+ <>+ <span><kbd className="font-mono">↑↓</kbd> Navigate</span>+ <span><kbd className="font-mono">Enter</kbd> Select</span>+ <span><kbd className="font-mono">Esc</kbd> Close</span>+ </>+ ) : (+ <span>Models are downloaded from the Ollama registry</span>+ )}</div></div></div>diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex c3f3dc3d..44ccda90 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -100,6 +100,28 @@ export async function fetchModels(): Promise<ModelInfo[]> {return data.data || [];}+export async function pullModel(modelName: string): Promise<void> {+ const res = await fetch(`${getBase()}/v1/models/pull`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ model: modelName }),+ });+ if (!res.ok) {+ const detail = await res.text().catch(() => res.statusText);+ throw new Error(`Failed to pull model: ${detail}`);+ }+}++export async function deleteModel(modelName: string): Promise<void> {+ const res = await fetch(`${getBase()}/v1/models/${encodeURIComponent(modelName)}`, {+ method: 'DELETE',+ });+ if (!res.ok) {+ const detail = await res.text().catch(() => res.statusText);+ throw new Error(`Failed to delete model: ${detail}`);+ }+}+export async function fetchSavings(): Promise<SavingsData> {const res = await fetch(`${getBase()}/v1/savings`);if (!res.ok) throw new Error(`Failed to fetch savings: ${res.status}`);diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex 3a3fe8a9..5fc4e62c 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -48,7 +48,11 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Requestif request_body.stream:bus = getattr(request.app.state, "bus", None)- if agent is not None and bus is not None:+ # Use the agent stream bridge only when tools are present (the+ # bridge runs agent.run() synchronously and word-splits the result,+ # so it can't stream tokens in real-time). For plain chat, stream+ # directly from the engine for true token-by-token output.+ if agent is not None and bus is not None and request_body.tools:return await _handle_agent_stream(agent, bus, model, request_body)return await _handle_stream(engine, model, request_body)@@ -181,21 +185,42 @@ async def _handle_stream(engine, model: str, req: ChatCompletionRequest):)yield f"data: {first_chunk.model_dump_json()}\n\n"- # Stream content- async for token in engine.stream(- messages,- model=model,- temperature=req.temperature,- max_tokens=req.max_tokens,- ):- chunk = ChatCompletionChunk(+ try:+ # Stream content+ async for token in engine.stream(+ messages,+ model=model,+ temperature=req.temperature,+ max_tokens=req.max_tokens,+ ):+ chunk = ChatCompletionChunk(+ id=chunk_id,+ model=model,+ choices=[StreamChoice(+ delta=DeltaMessage(content=token),+ )],+ )+ yield f"data: {chunk.model_dump_json()}\n\n"+ except Exception as exc:+ # Surface errors as a content chunk so the frontend can+ # display them instead of silently failing.+ import logging+ logging.getLogger("openjarvis.server").error(+ "Stream error: %s", exc, exc_info=True,+ )+ error_chunk = ChatCompletionChunk(id=chunk_id,model=model,choices=[StreamChoice(- delta=DeltaMessage(content=token),+ delta=DeltaMessage(+ content=f"\n\nError during generation: {exc}",+ ),+ finish_reason="stop",)],)- yield f"data: {chunk.model_dump_json()}\n\n"+ yield f"data: {error_chunk.model_dump_json()}\n\n"+ yield "data: [DONE]\n\n"+ return# Send finish chunkfinish_chunk = ChatCompletionChunk(@@ -226,6 +251,78 @@ async def list_models(request: Request) -> ModelListResponse:)[email protected]("/v1/models/pull")+async def pull_model(request: Request):+ """Pull / download a model from the Ollama registry."""+ body = await request.json()+ model_name = body.get("model", "").strip()+ if not model_name:+ raise HTTPException(status_code=400, detail="'model' field is required")++ engine = request.app.state.engine+ engine_name = getattr(request.app.state, "engine_name", "")+ # Only Ollama supports pulling+ if engine_name != "ollama" and getattr(engine, "engine_id", "") != "ollama":+ raise HTTPException(+ status_code=501,+ detail="Model pulling is only supported with the Ollama engine",+ )++ import httpx as _httpx++ host = getattr(engine, "_host", "http://localhost:11434")+ client = _httpx.Client(base_url=host, timeout=600.0)+ try:+ resp = client.post(+ "/api/pull",+ json={"name": model_name, "stream": False},+ )+ resp.raise_for_status()+ except (_httpx.ConnectError, _httpx.TimeoutException) as exc:+ raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}")+ except _httpx.HTTPStatusError as exc:+ raise HTTPException(+ status_code=exc.response.status_code,+ detail=f"Ollama error: {exc.response.text[:300]}",+ )+ finally:+ client.close()++ return {"status": "ok", "model": model_name}++[email protected]("/v1/models/{model_name:path}")+async def delete_model(model_name: str, request: Request):+ """Delete a model from Ollama."""+ engine = request.app.state.engine+ engine_name = getattr(request.app.state, "engine_name", "")+ if engine_name != "ollama" and getattr(engine, "engine_id", "") != "ollama":+ raise HTTPException(status_code=501, detail="Only supported with Ollama engine")++ import httpx as _httpx++ host = getattr(engine, "_host", "http://localhost:11434")+ client = _httpx.Client(base_url=host, timeout=30.0)+ try:+ resp = client.request(+ "DELETE",+ "/api/delete",+ json={"name": model_name},+ )+ resp.raise_for_status()+ except (_httpx.ConnectError, _httpx.TimeoutException) as exc:+ raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}")+ except _httpx.HTTPStatusError as exc:+ raise HTTPException(+ status_code=exc.response.status_code,+ detail=f"Ollama error: {exc.response.text[:300]}",+ )+ finally:+ client.close()++ return {"status": "deleted", "model": model_name}++@router.get("/v1/savings")async def savings(request: Request):"""Return savings summary compared to cloud providers.diff --git a/tests/server/test_model_management.py b/tests/server/test_model_management.pynew file mode 100644index 00000000..7c5f99ef--- /dev/null+++ b/tests/server/test_model_management.py@@ -0,0 +1,261 @@+"""Tests for model pull / delete API endpoints and streaming resilience."""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++import pytest++fastapi = pytest.importorskip("fastapi")+from fastapi.testclient import TestClient # noqa: E402++from openjarvis.server.app import create_app # noqa: E402++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_engine(engine_id="mock", models=None):+ engine = MagicMock()+ engine.engine_id = engine_id+ engine.health.return_value = True+ engine.list_models.return_value = models or ["test-model"]+ engine.generate.return_value = {+ "content": "Hello",+ "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},+ "model": "test-model",+ "finish_reason": "stop",+ }++ async def mock_stream(messages, *, model, temperature=0.7, max_tokens=1024, **kw):+ for token in ["Hello", " ", "world"]:+ yield token++ engine.stream = mock_stream+ return engine+++def _make_ollama_engine(models=None):+ """Create a mock engine that looks like OllamaEngine."""+ engine = _make_engine(engine_id="ollama", models=models)+ engine._host = "http://localhost:11434"+ return engine+++def _app(engine, engine_name="mock"):+ return create_app(engine, "test-model", engine_name=engine_name)+++# ---------------------------------------------------------------------------+# Model pull endpoint+# ---------------------------------------------------------------------------+++class TestModelPull:+ def test_pull_requires_model_field(self):+ engine = _make_ollama_engine()+ client = TestClient(_app(engine, engine_name="ollama"))+ resp = client.post("/v1/models/pull", json={})+ assert resp.status_code == 400+ assert "model" in resp.json()["detail"].lower()++ def test_pull_rejects_non_ollama_engine(self):+ engine = _make_engine(engine_id="vllm")+ client = TestClient(_app(engine, engine_name="vllm"))+ resp = client.post("/v1/models/pull", json={"model": "foo"})+ assert resp.status_code == 501++ def test_pull_success(self):+ engine = _make_ollama_engine()+ client = TestClient(_app(engine, engine_name="ollama"))+++ mock_resp = MagicMock()+ mock_resp.status_code = 200+ mock_resp.raise_for_status = MagicMock()++ with patch("httpx.Client") as MockClient:+ instance = MockClient.return_value+ instance.post.return_value = mock_resp+ instance.close = MagicMock()++ resp = client.post("/v1/models/pull", json={"model": "qwen3.5:4b"})++ assert resp.status_code == 200+ data = resp.json()+ assert data["status"] == "ok"+ assert data["model"] == "qwen3.5:4b"++ def test_pull_ollama_unreachable(self):+ engine = _make_ollama_engine()+ client = TestClient(_app(engine, engine_name="ollama"))++ import httpx++ with patch("httpx.Client") as MockClient:+ instance = MockClient.return_value+ instance.post.side_effect = httpx.ConnectError("refused")+ instance.close = MagicMock()++ resp = client.post("/v1/models/pull", json={"model": "foo"})++ assert resp.status_code == 502+++# ---------------------------------------------------------------------------+# Model delete endpoint+# ---------------------------------------------------------------------------+++class TestModelDelete:+ def test_delete_rejects_non_ollama(self):+ engine = _make_engine(engine_id="vllm")+ client = TestClient(_app(engine, engine_name="vllm"))+ resp = client.delete("/v1/models/test-model")+ assert resp.status_code == 501++ def test_delete_success(self):+ engine = _make_ollama_engine()+ client = TestClient(_app(engine, engine_name="ollama"))++ mock_resp = MagicMock()+ mock_resp.status_code = 200+ mock_resp.raise_for_status = MagicMock()++ with patch("httpx.Client") as MockClient:+ instance = MockClient.return_value+ instance.request.return_value = mock_resp+ instance.close = MagicMock()++ resp = client.delete("/v1/models/qwen3:0.6b")++ assert resp.status_code == 200+ data = resp.json()+ assert data["status"] == "deleted"+ assert data["model"] == "qwen3:0.6b"+++# ---------------------------------------------------------------------------+# Streaming resilience+# ---------------------------------------------------------------------------+++class TestStreamingResilience:+ """Verify streaming handles errors gracefully."""++ def test_stream_error_returns_error_chunk(self):+ """When the engine raises during streaming, error is sent as content."""+ engine = _make_engine()++ async def failing_stream(messages, *, model, **kw):+ yield "partial"+ raise RuntimeError("model not found")++ engine.stream = failing_stream+ app = create_app(engine, "test-model")+ client = TestClient(app)++ resp = client.post("/v1/chat/completions", json={+ "model": "bad-model",+ "messages": [{"role": "user", "content": "Hello"}],+ "stream": True,+ })+ assert resp.status_code == 200++ # Should contain partial content + error message + [DONE]+ text = resp.text+ assert "partial" in text+ assert "model not found" in text+ assert "[DONE]" in text++ def test_stream_tokens_arrive(self):+ """Verify tokens stream through correctly (not batched)."""+ engine = _make_engine()+ app = create_app(engine, "test-model")+ client = TestClient(app)++ resp = client.post("/v1/chat/completions", json={+ "model": "test-model",+ "messages": [{"role": "user", "content": "Hello"}],+ "stream": True,+ })+ assert resp.status_code == 200++ # Collect tokens+ tokens = []+ for line in resp.text.strip().split("\n"):+ if line.startswith("data:") and "[DONE]" not in line:+ data = json.loads(line[5:].strip())+ content = data.get("choices", [{}])[0].get("delta", {}).get("content")+ if content:+ tokens.append(content)++ assert tokens == ["Hello", " ", "world"]++ def test_stream_without_agent_uses_direct_engine(self):+ """When no tools in request, streaming should use engine.stream directly+ even if an agent is configured (for real token-by-token output)."""+ from openjarvis.agents._stubs import AgentResult++ engine = _make_engine()+ agent = MagicMock()+ agent.agent_id = "simple"+ agent.run.return_value = AgentResult(+ content="agent response", turns=1,+ )++ app = create_app(engine, "test-model", agent=agent)+ client = TestClient(app)++ resp = client.post("/v1/chat/completions", json={+ "model": "test-model",+ "messages": [{"role": "user", "content": "Hello"}],+ "stream": True,+ # No tools — should use direct engine stream+ })+ assert resp.status_code == 200++ # Should get engine tokens, not agent response+ tokens = []+ for line in resp.text.strip().split("\n"):+ if line.startswith("data:") and "[DONE]" not in line:+ data = json.loads(line[5:].strip())+ content = data.get("choices", [{}])[0].get("delta", {}).get("content")+ if content:+ tokens.append(content)++ assert "".join(tokens) == "Hello world"+ # Agent.run should NOT have been called+ agent.run.assert_not_called()+++# ---------------------------------------------------------------------------+# Models endpoint+# ---------------------------------------------------------------------------+++class TestModelsEndpointExtended:+ def test_models_list_multiple(self):+ engine = _make_engine(+ models=["qwen3.5:4b", "qwen3.5:9b", "qwen3:0.6b"],+ )+ app = create_app(engine, "qwen3.5:4b")+ client = TestClient(app)+ resp = client.get("/v1/models")+ assert resp.status_code == 200+ ids = [m["id"] for m in resp.json()["data"]]+ assert "qwen3.5:4b" in ids+ assert "qwen3.5:9b" in ids+ assert "qwen3:0.6b" in ids++ def test_models_empty_engine(self):+ """When engine.list_models() returns empty, endpoint still succeeds."""+ engine = _make_engine(models=[])+ app = create_app(engine, "test-model")+ client = TestClient(app)+ resp = client.get("/v1/models")+ assert resp.status_code == 200+ # The endpoint returns whatever list_models() gives+ assert resp.json()["object"] == "list"