Personal AI, On Personal Devices
Python
feat(desktop): single default model + custom OpenAI-compatible endpoints (#453)
Jon Saad-Falcon21f1becb4d15
diff --git a/frontend/src-tauri/Cargo.lock b/frontend/src-tauri/Cargo.lockindex a33263ca..f36386ac 100644--- a/frontend/src-tauri/Cargo.lock+++ b/frontend/src-tauri/Cargo.lock@@ -1384,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0bb0228f477c0900c880fd78c8759b95c7636dbd7842707f49e132378aa2acdc"dependencies = ["heck 0.4.1",- "proc-macro-crate 2.0.2",+ "proc-macro-crate 2.0.0","proc-macro-error","proc-macro2","quote",@@ -2593,7 +2593,7 @@ dependencies = [[[package]]name = "openjarvis-desktop"-version = "0.1.0"+version = "1.0.1"dependencies = ["dispatch","objc",@@ -2611,6 +2611,7 @@ dependencies = ["tauri-plugin-single-instance","tauri-plugin-updater","tokio",+ "toml_edit 0.22.27",][[package]]@@ -3018,11 +3019,10 @@ dependencies = [[[package]]name = "proc-macro-crate"-version = "2.0.2"+version = "2.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24"+checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8"dependencies = [- "toml_datetime 0.6.3","toml_edit 0.20.2",]@@ -4766,7 +4766,7 @@ checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"dependencies = ["serde","serde_spanned 0.6.9",- "toml_datetime 0.6.3",+ "toml_datetime 0.6.11","toml_edit 0.20.2",]@@ -4802,9 +4802,9 @@ dependencies = [[[package]]name = "toml_datetime"-version = "0.6.3"+version = "0.6.11"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"dependencies = ["serde",]@@ -4834,7 +4834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"dependencies = ["indexmap 2.13.0",- "toml_datetime 0.6.3",+ "toml_datetime 0.6.11","winnow 0.5.40",]@@ -4847,10 +4847,22 @@ dependencies = ["indexmap 2.13.0","serde","serde_spanned 0.6.9",- "toml_datetime 0.6.3",+ "toml_datetime 0.6.11","winnow 0.5.40",]+[[package]]+name = "toml_edit"+version = "0.22.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"+dependencies = [+ "indexmap 2.13.0",+ "toml_datetime 0.6.11",+ "toml_write",+ "winnow 0.7.14",+]+[[package]]name = "toml_edit"version = "0.23.10+spec-1.0.0"@@ -4872,6 +4884,12 @@ dependencies = ["winnow 0.7.14",]+[[package]]+name = "toml_write"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"+[[package]]name = "toml_writer"version = "1.0.6+spec-1.1.0"diff --git a/frontend/src-tauri/Cargo.toml b/frontend/src-tauri/Cargo.tomlindex bf90a7b3..97a33e48 100644--- a/frontend/src-tauri/Cargo.toml+++ b/frontend/src-tauri/Cargo.toml@@ -9,6 +9,7 @@ license = "MIT"tauri-build = { version = "2", features = [] }[dependencies]+toml_edit = "0.22"tauri = { version = "2", features = ["tray-icon"] }tauri-plugin-notification = "2"tauri-plugin-shell = "2"diff --git a/frontend/src-tauri/src/lib.rs b/frontend/src-tauri/src/lib.rsindex b9297442..8c6d57c8 100644--- a/frontend/src-tauri/src/lib.rs+++ b/frontend/src-tauri/src/lib.rs@@ -76,29 +76,102 @@ fn total_ram_gb() -> f64 {8.0}-/// Return the list of Qwen3.5 models that fit on this machine, smallest first.-fn models_that_fit() -> Vec<&'static str> {- let ram = total_ram_gb();+/// Return the Qwen3.5 models that fit in `ram_gb`, smallest first.+fn models_that_fit_in(ram_gb: f64) -> Vec<&'static str> {QWEN35_MODELS.iter()- .filter(|(_, _, min_ram)| ram >= *min_ram)+ .filter(|(_, _, min_ram)| ram_gb >= *min_ram).map(|(tag, _, _)| *tag).collect()}-/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise-/// falls back to the third-largest model that fits on this machine.-fn preferred_model() -> &'static str {- let fitting = models_that_fit();- // Prefer STARTUP_MODEL when it fits (fast, good quality)- if fitting.contains(&STARTUP_MODEL) {- return STARTUP_MODEL;- }+/// The default local model: the second-largest Qwen3.5 model that fits in+/// `ram_gb`. Falls back to the only fitting model, or FALLBACK_MODEL if none+/// fit. Deliberately NOT the largest — leaves RAM headroom for the OS/app.+fn default_local_model(ram_gb: f64) -> &'static str {+ let fitting = models_that_fit_in(ram_gb);match fitting.len() {0 => FALLBACK_MODEL,1 => fitting[0],- 2 => fitting[0],- n => fitting[n - 3], // third-largest+ n => fitting[n - 2],+ }+}++/// A resolved boot plan derived purely from the inference config + RAM.+/// Pure and side-effect-free so it can be unit-tested without spawning+/// processes or touching the network.+#[derive(Debug, Clone, PartialEq, Eq)]+struct BootPlan {+ /// Whether to start and wait for the bundled Ollama.+ launch_ollama: bool,+ /// The single Ollama model to pull (None for custom endpoints).+ model_to_pull: Option<String>,+ /// Optional `(engine_key, bare_host)` override for a custom endpoint,+ /// e.g. `("lmstudio", "http://localhost:1234")`. Written into+ /// ~/.openjarvis/config.toml so `jarvis serve` picks it up.+ engine_host: Option<(String, String)>,+ /// Args appended after `uv run jarvis serve --port <port>`.+ serve_args: Vec<String>,+}++/// Default OpenAI-compatible engine key used when a custom endpoint config+/// omits one (LM Studio is the canonical local server).+const CUSTOM_FALLBACK_ENGINE: &str = "lmstudio";++/// Decide what to launch/pull/serve from the inference config + system RAM.+/// Pure: no I/O, no spawning.+fn boot_plan(cfg: &InferenceConfig, ram_gb: f64) -> BootPlan {+ match cfg.kind {+ SourceKind::Ollama => {+ let model = cfg+ .model+ .clone()+ .unwrap_or_else(|| default_local_model(ram_gb).to_string());+ BootPlan {+ launch_ollama: true,+ model_to_pull: Some(model.clone()),+ engine_host: None,+ serve_args: vec![+ "--engine".into(),+ "ollama".into(),+ "--model".into(),+ model,+ "--agent".into(),+ "simple".into(),+ ],+ }+ }+ SourceKind::Custom => {+ let engine = cfg+ .engine+ .clone()+ .unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());+ // Record (engine_key, bare_host) only when a host is configured, so+ // boot can write `[engine.<key>] host = ...` into config.toml. An+ // empty host is dropped (no override).+ let engine_host = cfg+ .host+ .clone()+ .filter(|h| !h.is_empty())+ .map(|h| (engine.clone(), h));+ // `model` may be empty if the config is malformed; `jarvis serve`+ // surfaces a clear error then (there is no universal default model+ // for an arbitrary endpoint).+ let model = cfg.model.clone().unwrap_or_default();+ BootPlan {+ launch_ollama: false,+ model_to_pull: None,+ engine_host,+ serve_args: vec![+ "--engine".into(),+ engine,+ "--model".into(),+ model,+ "--agent".into(),+ "simple".into(),+ ],+ }+ }}}@@ -343,6 +416,8 @@ struct SetupStatus {server_ready: bool,model_ready: bool,error: Option<String>,+ /// "ollama" | "custom" — lets the setup UI relabel the progress steps.+ source: String,}impl Default for SetupStatus {@@ -354,6 +429,7 @@ impl Default for SetupStatus {server_ready: false,model_ready: false,error: None,+ source: "ollama".into(),}}}@@ -381,6 +457,28 @@ async fn wait_for_url(url: &str, timeout: Duration) -> bool {false}+/// True if a custom OpenAI-compatible endpoint answers at all (any HTTP+/// status counts — even a 404 proves the server is up). `host` is the bare+/// base URL; we probe `<host>/v1/models`.+async fn endpoint_reachable(host: &str, timeout: Duration) -> bool {+ let client = match reqwest::Client::builder()+ .timeout(Duration::from_secs(3))+ .build()+ {+ Ok(c) => c,+ Err(_) => return false,+ };+ let url = format!("{}/v1/models", host.trim_end_matches('/'));+ let deadline = tokio::time::Instant::now() + timeout;+ while tokio::time::Instant::now() < deadline {+ if client.get(&url).send().await.is_ok() {+ return true;+ }+ tokio::time::sleep(Duration::from_millis(500)).await;+ }+ false+}+/// Outcome of waiting for `jarvis serve` to become healthy.////// Unlike [`wait_for_url`] this differentiates "server is up but degraded"@@ -608,80 +706,153 @@ fn format_uv_sync_spawn_error(root: &std::path::Path, uv_bin: &str, err: &str) -// ---------------------------------------------------------------------------async fn boot_backend(backend: SharedBackend, status: SharedStatus) {- // Phase 1: Start Ollama+ // Decide the inference source (default Ollama) before launching anything.+ let cfg = read_inference_config();+ let plan = boot_plan(&cfg, total_ram_gb());{let mut s = status.lock().await;- s.phase = "ollama".into();- s.detail = "Starting inference engine...".into();+ s.source = match cfg.kind {+ SourceKind::Ollama => "ollama",+ SourceKind::Custom => "custom",+ }+ .into();}- // Try the bundled sidecar first, fall back to system ollama- let ollama_child = {- let ollama_bin = resolve_bin("ollama");- let sidecar = tokio::process::Command::new(&ollama_bin)- .arg("serve")- .env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))- .stdout(std::process::Stdio::null())- .stderr(std::process::Stdio::null())- .spawn();- match sidecar {- Ok(child) => Some(child),- Err(_) => None,- }- };+ // For the Ollama path, the model pull may fall back to FALLBACK_MODEL; we+ // record what is actually available here so the serve command below uses+ // it instead of the originally-planned tag. None on the custom path.+ let mut serve_model_override: Option<String> = None;- if let Some(child) = ollama_child {- backend.lock().await.ollama = Some(ChildHandle { child });- }+ if plan.launch_ollama {+ // Phase 1: Start Ollama+ {+ let mut s = status.lock().await;+ s.phase = "ollama".into();+ s.detail = "Starting inference engine...".into();+ }- let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);- let ollama_ok = wait_for_url(&ollama_url, Duration::from_secs(30)).await;+ // Try the bundled sidecar first, fall back to system ollama+ let ollama_child = {+ let ollama_bin = resolve_bin("ollama");+ let sidecar = tokio::process::Command::new(&ollama_bin)+ .arg("serve")+ .env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null())+ .spawn();+ match sidecar {+ Ok(child) => Some(child),+ Err(_) => None,+ }+ };- if !ollama_ok {- let mut s = status.lock().await;- s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());- return;- }+ if let Some(child) = ollama_child {+ backend.lock().await.ollama = Some(ChildHandle { child });+ }- {- let mut s = status.lock().await;- s.ollama_ready = true;- s.detail = "Inference engine ready.".into();- }+ let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);+ if !wait_for_url(&ollama_url, Duration::from_secs(30)).await {+ let mut s = status.lock().await;+ s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());+ return;+ }- // Phase 2: Pull one small model (qwen3.5:2b) so the app can open fast.- // Remaining models are pulled in the background after the server starts.- {- let mut s = status.lock().await;- s.phase = "model".into();- s.detail = format!("Checking for {}...", STARTUP_MODEL);- }+ {+ let mut s = status.lock().await;+ s.ollama_ready = true;+ s.detail = "Inference engine ready.".into();+ }- if !ollama_has_model(STARTUP_MODEL).await {+ // Phase 2: Pull the single default model (see default_local_model /+ // boot_plan). We deliberately do NOT pull any others.+ let model = plan+ .model_to_pull+ .clone()+ .unwrap_or_else(|| STARTUP_MODEL.to_string());{let mut s = status.lock().await;- s.detail = format!("Downloading {}... (this may take a minute)", STARTUP_MODEL);+ s.phase = "model".into();+ s.detail = format!("Checking for {}...", model);}- if let Err(e) = pull_model(STARTUP_MODEL).await {- // If the startup model fails, try the tiny fallback- eprintln!("Warning: failed to pull {}: {}", STARTUP_MODEL, e);- if !ollama_has_model(FALLBACK_MODEL).await {++ if !ollama_has_model(&model).await {+ {let mut s = status.lock().await;- s.detail = format!("Downloading {}...", FALLBACK_MODEL);- drop(s);- if let Err(e2) = pull_model(FALLBACK_MODEL).await {- let mut s = status.lock().await;- s.error = Some(format!("Failed to download model: {}", e2));- return;+ s.detail = format!("Downloading {}... (this may take a minute)", model);+ }+ if let Err(e) = pull_model(&model).await {+ // If the chosen model fails, try the tiny fallback+ eprintln!("Warning: failed to pull {}: {}", model, e);+ if !ollama_has_model(FALLBACK_MODEL).await {+ {+ let mut s = status.lock().await;+ s.detail = format!("Downloading {}...", FALLBACK_MODEL);+ }+ if let Err(e2) = pull_model(FALLBACK_MODEL).await {+ let mut s = status.lock().await;+ s.error = Some(format!("Failed to download model: {}", e2));+ return;+ }}}}- }- {- let mut s = status.lock().await;- s.model_ready = true;- s.detail = "Model ready.".into();+ // The pull may have fallen back to FALLBACK_MODEL; serve and persist+ // whatever is actually available now, not the originally-planned tag.+ let resolved_model = if ollama_has_model(&model).await {+ model+ } else {+ FALLBACK_MODEL.to_string()+ };+ serve_model_override = Some(resolved_model.clone());++ // Persist the resolved model so Settings shows it and future boots reuse it.+ let mut persisted = cfg.clone();+ persisted.model = Some(resolved_model);+ let _ = write_inference_config(&persisted);++ {+ let mut s = status.lock().await;+ s.model_ready = true;+ s.detail = "Model ready.".into();+ }+ } else {+ // Custom OpenAI-compatible endpoint: never start Ollama, never download.+ let host = plan+ .engine_host+ .as_ref()+ .map(|(_, v)| v.clone())+ .unwrap_or_default();+ {+ let mut s = status.lock().await;+ s.phase = "model".into();+ s.detail = format!("Connecting to {}...", host);+ }+ if host.is_empty() || !endpoint_reachable(&host, Duration::from_secs(15)).await {+ let mut s = status.lock().await;+ s.error = Some(format!(+ "Could not reach your custom inference server at {}. \+ Start the server (e.g. LM Studio) and check the URL in Settings, then relaunch.",+ if host.is_empty() { "(no URL set)" } else { host.as_str() }+ ));+ return;+ }+ // Point `jarvis serve` at the user's endpoint by writing the engine+ // host into ~/.openjarvis/config.toml (the env var alone is shadowed by+ // the engine's non-empty default host in the Python layer).+ if let Some((engine, host)) = &plan.engine_host {+ if let Err(e) = set_engine_host_in_config(engine, host) {+ let mut s = status.lock().await;+ s.error = Some(format!("Could not write engine config: {}", e));+ return;+ }+ }+ {+ let mut s = status.lock().await;+ s.ollama_ready = true;+ s.model_ready = true;+ s.detail = "Connected to custom endpoint.".into();+ }}// Phase 3: Start jarvis serve@@ -848,16 +1019,6 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {}}- // Start with STARTUP_MODEL (just pulled) or preferred if already available.- let pref = preferred_model();- let startup_model = if ollama_has_model(pref).await {- pref- } else if ollama_has_model(STARTUP_MODEL).await {- STARTUP_MODEL- } else {- FALLBACK_MODEL- };-let root = project_root.as_ref().unwrap();// Install dependencies automatically (handles fresh clones).@@ -907,28 +1068,35 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {{let mut s = status.lock().await;- s.detail = format!(- "Starting server with {} from {}...",- startup_model,- root.display(),- );+ s.detail = format!("Starting API server from {}...", root.display());}let mut cmd = tokio::process::Command::new(&uv_bin);- cmd.args([- "run",- "jarvis",- "serve",- "--port",- &JARVIS_PORT.to_string(),- "--model",- startup_model,- "--agent",- "simple",- ])- .stdout(std::process::Stdio::null())- .stderr(std::process::Stdio::piped())- .current_dir(root);+ let mut serve_argv: Vec<String> = vec![+ "run".into(),+ "jarvis".into(),+ "serve".into(),+ "--port".into(),+ JARVIS_PORT.to_string(),+ ];+ serve_argv.extend(plan.serve_args.iter().cloned());+ // If the Ollama pull fell back to a different tag than planned, serve the+ // tag that is actually present. boot_plan always emits `--model` followed+ // immediately by its value, so `i + 1` is in bounds.+ if let Some(m) = &serve_model_override {+ match serve_argv.iter().position(|a| a == "--model") {+ Some(i) if i + 1 < serve_argv.len() => serve_argv[i + 1] = m.clone(),+ _ => eprintln!(+ "Warning: resolved model {:?} could not be applied; \+ '--model <value>' not found in serve args {:?}",+ m, serve_argv+ ),+ }+ }+ cmd.args(&serve_argv)+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::piped())+ .current_dir(root);// Inject cloud API keys from ~/.openjarvis/cloud-keys.envfor (key, value) in read_cloud_keys() {@@ -971,11 +1139,16 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {s.error = Some(format!("Jarvis server is running but the inference engine is not available \(HTTP 503). This usually means the configured model couldn't be loaded.\n\n\- Check the server logs, or run 'uv run jarvis serve --port {} --model {}' \+ Check the server logs, or run 'uv run jarvis serve --port {}{}' \from {} to see the engine error.\n\n\Server response:\n{}",JARVIS_PORT,- startup_model,+ // Show the args actually passed (after `serve --port <port>`),+ // including any post-fallback `--model` override.+ match serve_argv.get(5..) {+ Some(rest) if !rest.is_empty() => format!(" {}", rest.join(" ")),+ _ => String::new(),+ },root.display(),body.trim(),));@@ -1435,6 +1608,58 @@ async fn get_cloud_key_status() -> Result<serde_json::Value, String> {Ok(serde_json::json!(status))}+/// Return the current inference-source config for the Settings UI.+#[tauri::command]+async fn get_inference_source() -> Result<InferenceConfig, String> {+ Ok(read_inference_config())+}++/// Persist the chosen inference source. `host` is normalized to a bare base+/// URL. For custom endpoints, an optional API key is stored in cloud-keys.env+/// under `<ENGINE>_API_KEY`. Applies on next app launch.+#[tauri::command]+async fn set_inference_source(+ kind: String,+ model: Option<String>,+ host: Option<String>,+ engine: Option<String>,+ api_key: Option<String>,+) -> Result<(), String> {+ let kind = match kind.as_str() {+ "ollama" => SourceKind::Ollama,+ "custom" => SourceKind::Custom,+ other => return Err(format!("Unknown inference source kind: {:?}", other)),+ };+ let cfg = InferenceConfig {+ kind,+ model: model.filter(|m| !m.is_empty()),+ host: host.map(|h| normalize_host(&h)).filter(|h| !h.is_empty()),+ engine: engine.filter(|e| !e.is_empty()),+ };+ if let SourceKind::Custom = cfg.kind {+ if cfg.host.is_none() {+ return Err("A server URL is required for a custom endpoint.".into());+ }+ if cfg.model.as_deref().unwrap_or("").is_empty() {+ return Err("A model name is required for a custom endpoint.".into());+ }+ if let Some(key) = api_key.filter(|k| !k.is_empty()) {+ let engine = cfg+ .engine+ .clone()+ .unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());+ let key_name = format!("{}_API_KEY", engine.to_ascii_uppercase());+ // Save the key before persisting the config: if the key can't be+ // written, surface it and DON'T record a custom source whose+ // credential is missing (which would fail confusingly at runtime).+ save_cloud_key(key_name, key)+ .await+ .map_err(|e| format!("Could not store the API key: {}", e))?;+ }+ }+ write_inference_config(&cfg)+}+/// Pull a model via Ollama (called from frontend download button).#[tauri::command]async fn pull_ollama_model(model_name: String) -> Result<serde_json::Value, String> {@@ -1464,6 +1689,103 @@ async fn delete_ollama_model(model_name: String) -> Result<serde_json::Value, StOk(serde_json::json!({"status": "deleted", "model": model_name}))}+// ---------------------------------------------------------------------------+// Inference-source selection (~/.openjarvis/inference.json)+// ---------------------------------------------------------------------------++#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+enum SourceKind {+ Ollama,+ Custom,+}++impl Default for SourceKind {+ fn default() -> Self {+ SourceKind::Ollama+ }+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]+struct InferenceConfig {+ #[serde(default)]+ kind: SourceKind,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ model: Option<String>,+ /// Bare base URL (no trailing `/v1`), custom only.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ host: Option<String>,+ /// OpenAI-compatible engine key (e.g. "lmstudio"), custom only.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ engine: Option<String>,+}++/// Path to the inference-source config (~/.openjarvis/inference.json).+fn inference_config_path() -> std::path::PathBuf {+ std::path::PathBuf::from(home_dir())+ .join(".openjarvis")+ .join("inference.json")+}++/// Parse config text. Any error (missing/garbage) yields the Ollama default —+/// a broken file must never strand the user with no working inference source.+fn parse_inference_config(text: &str) -> InferenceConfig {+ serde_json::from_str::<InferenceConfig>(text).unwrap_or_default()+}++/// Read the on-disk inference config, or the Ollama default if absent.+fn read_inference_config() -> InferenceConfig {+ match std::fs::read_to_string(inference_config_path()) {+ Ok(text) => parse_inference_config(&text),+ Err(_) => InferenceConfig::default(),+ }+}++/// Write the inference config to disk (pretty JSON).+fn write_inference_config(cfg: &InferenceConfig) -> Result<(), String> {+ let path = inference_config_path();+ if let Some(parent) = path.parent() {+ let _ = std::fs::create_dir_all(parent);+ }+ let json = serde_json::to_string_pretty(cfg).map_err(|e| e.to_string())?;+ std::fs::write(&path, json + "\n").map_err(|e| format!("Failed to save inference config: {}", e))+}++/// Upsert `[engine.<engine>] host = "<host>"` into an existing config.toml+/// string, preserving all other content/formatting. Pure: string in, string out.+fn upsert_engine_host(existing: &str, engine: &str, host: &str) -> Result<String, String> {+ let mut doc = existing+ .parse::<toml_edit::DocumentMut>()+ .map_err(|e| format!("Invalid config.toml: {}", e))?;+ doc["engine"][engine]["host"] = toml_edit::value(host);+ Ok(doc.to_string())+}++/// Write the custom-endpoint host into ~/.openjarvis/config.toml so+/// `jarvis serve` (which reads that file via load_config) points at it.+/// The `<ENGINE>_HOST` env var is unreliable — it is shadowed by the engine's+/// non-empty default host in the Python layer — so config.toml is the override.+fn set_engine_host_in_config(engine: &str, host: &str) -> Result<(), String> {+ let path = std::path::PathBuf::from(home_dir())+ .join(".openjarvis")+ .join("config.toml");+ if let Some(parent) = path.parent() {+ let _ = std::fs::create_dir_all(parent);+ }+ let existing = std::fs::read_to_string(&path).unwrap_or_default();+ let updated = upsert_engine_host(&existing, engine, host)?;+ std::fs::write(&path, updated).map_err(|e| format!("Failed to write config.toml: {}", e))+}++/// Normalize a user-entered server URL to a bare base host: trim whitespace,+/// drop a trailing `/v1` segment (the engine re-appends its own api prefix),+/// then drop any trailing slash.+fn normalize_host(raw: &str) -> String {+ let s = raw.trim().trim_end_matches('/');+ let s = s.strip_suffix("/v1").unwrap_or(s);+ s.trim_end_matches('/').to_string()+}+/// Check speech backend health.#[tauri::command]async fn speech_health(api_url: String) -> Result<serde_json::Value, String> {@@ -1987,6 +2309,8 @@ pub fn run() {delete_ollama_model,save_cloud_key,get_cloud_key_status,+ get_inference_source,+ set_inference_source,toggle_overlay,hide_overlay,get_overlay_conversation,@@ -2009,7 +2333,11 @@ pub fn run() {#[cfg(test)]mod tests {- use super::{format_uv_sync_failure, format_uv_sync_spawn_error, uv_sync_stderr_tail};+ use super::{+ boot_plan, default_local_model, format_uv_sync_failure, format_uv_sync_spawn_error,+ normalize_host, parse_inference_config, upsert_engine_host, uv_sync_stderr_tail,+ InferenceConfig, SourceKind,+ };use std::path::Path;#[test]@@ -2074,4 +2402,146 @@ mod tests {assert!(msg.contains("/repo"));assert!(msg.contains("No such file or directory"));}++ #[test]+ fn default_local_model_picks_second_largest_that_fits() {+ // QWEN35_MODELS min_ram ladder: 4,6,8,12,24,32,96 GB+ assert_eq!(default_local_model(4.0), "qwen3.5:0.8b"); // only one fits+ assert_eq!(default_local_model(8.0), "qwen3.5:2b"); // fits 0.8/2/4 → 2nd-largest+ assert_eq!(default_local_model(16.0), "qwen3.5:4b"); // fits ..9b → 2nd-largest+ assert_eq!(default_local_model(32.0), "qwen3.5:27b"); // fits 0.8/2/4/9/27/35b → 2nd-largest is 27b+ assert_eq!(default_local_model(128.0), "qwen3.5:35b"); // fits all → 2nd-largest+ }++ #[test]+ fn default_local_model_falls_back_when_nothing_fits() {+ assert_eq!(default_local_model(1.0), super::FALLBACK_MODEL);+ }++ #[test]+ fn parse_defaults_to_ollama_when_file_missing_or_garbage() {+ assert!(matches!(parse_inference_config("").kind, SourceKind::Ollama));+ assert!(matches!(parse_inference_config("not json").kind, SourceKind::Ollama));+ }++ #[test]+ fn parse_reads_custom_endpoint() {+ let cfg = parse_inference_config(+ r#"{"kind":"custom","model":"qwen2.5-7b","host":"http://localhost:1234","engine":"lmstudio"}"#,+ );+ assert!(matches!(cfg.kind, SourceKind::Custom));+ assert_eq!(cfg.model.as_deref(), Some("qwen2.5-7b"));+ assert_eq!(cfg.host.as_deref(), Some("http://localhost:1234"));+ assert_eq!(cfg.engine.as_deref(), Some("lmstudio"));+ }++ #[test]+ fn normalize_host_strips_trailing_slash_and_v1() {+ assert_eq!(normalize_host("http://localhost:1234/v1"), "http://localhost:1234");+ assert_eq!(normalize_host("http://localhost:1234/v1/"), "http://localhost:1234");+ assert_eq!(normalize_host("http://localhost:1234/"), "http://localhost:1234");+ assert_eq!(normalize_host("http://host:8000"), "http://host:8000");+ }++ #[test]+ fn boot_plan_ollama_launches_and_pulls_one_model() {+ let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };+ let plan = boot_plan(&cfg, 16.0);+ assert!(plan.launch_ollama);+ assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:4b"));+ assert!(plan.engine_host.is_none());+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "ollama"]));+ assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen3.5:4b"]));+ }++ #[test]+ fn boot_plan_ollama_respects_pinned_model() {+ let cfg = InferenceConfig {+ kind: SourceKind::Ollama,+ model: Some("qwen3.5:9b".into()),+ ..Default::default()+ };+ let plan = boot_plan(&cfg, 16.0);+ assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:9b"));+ }++ #[test]+ fn boot_plan_custom_skips_ollama_and_sets_engine_host() {+ let cfg = InferenceConfig {+ kind: SourceKind::Custom,+ model: Some("qwen2.5-7b".into()),+ host: Some("http://localhost:1234".into()),+ engine: Some("lmstudio".into()),+ };+ let plan = boot_plan(&cfg, 16.0);+ assert!(!plan.launch_ollama);+ assert!(plan.model_to_pull.is_none());+ assert_eq!(+ plan.engine_host,+ Some(("lmstudio".to_string(), "http://localhost:1234".to_string()))+ );+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));+ assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen2.5-7b"]));+ }++ #[test]+ fn boot_plan_custom_defaults_engine_to_lmstudio() {+ let cfg = InferenceConfig {+ kind: SourceKind::Custom,+ model: Some("m".into()),+ host: Some("http://h:1".into()),+ engine: None,+ };+ let plan = boot_plan(&cfg, 16.0);+ assert_eq!(plan.engine_host.as_ref().unwrap().0, "lmstudio");+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));+ }++ #[test]+ fn boot_plan_custom_omits_engine_host_when_no_host() {+ // No configured host → don't set engine_host (no override to write).+ let cfg = InferenceConfig {+ kind: SourceKind::Custom,+ model: Some("m".into()),+ host: None,+ engine: Some("lmstudio".into()),+ };+ let plan = boot_plan(&cfg, 16.0);+ assert!(plan.engine_host.is_none());+ }++ #[test]+ fn boot_plan_ollama_uses_fallback_model_on_low_ram() {+ // Below the smallest model's min_ram → default_local_model → FALLBACK_MODEL.+ let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };+ let plan = boot_plan(&cfg, 1.0);+ assert_eq!(plan.model_to_pull.as_deref(), Some(super::FALLBACK_MODEL));+ }++ #[test]+ fn upsert_engine_host_writes_into_empty_config() {+ let out = upsert_engine_host("", "lmstudio", "http://localhost:1234").unwrap();+ let doc: toml_edit::DocumentMut = out.parse().unwrap();+ assert_eq!(+ doc["engine"]["lmstudio"]["host"].as_str(),+ Some("http://localhost:1234")+ );+ }++ #[test]+ fn upsert_engine_host_preserves_existing_content() {+ let existing = "[intelligence]\ndefault_model = \"keep-me\"\n";+ let out = upsert_engine_host(existing, "vllm", "http://host:8000").unwrap();+ let doc: toml_edit::DocumentMut = out.parse().unwrap();+ assert_eq!(doc["intelligence"]["default_model"].as_str(), Some("keep-me"));+ assert_eq!(doc["engine"]["vllm"]["host"].as_str(), Some("http://host:8000"));+ }++ #[test]+ fn upsert_engine_host_updates_existing_host() {+ let existing = "[engine.lmstudio]\nhost = \"http://old:1\"\n";+ let out = upsert_engine_host(existing, "lmstudio", "http://new:2").unwrap();+ let doc: toml_edit::DocumentMut = out.parse().unwrap();+ assert_eq!(doc["engine"]["lmstudio"]["host"].as_str(), Some("http://new:2"));+ }}diff --git a/frontend/src/components/SetupScreen.tsx b/frontend/src/components/SetupScreen.tsxindex 15312c6f..1d477e9c 100644--- a/frontend/src/components/SetupScreen.tsx+++ b/frontend/src/components/SetupScreen.tsx@@ -146,7 +146,14 @@ export function SetupScreen({ onReady }: { onReady: () => void }) {{/* Steps */}<div className="flex flex-col gap-2 mb-8">- {STEPS.map((step) => (+ {(status?.source === 'custom'+ ? [+ { key: 'ollama_ready' as const, label: 'Inference Engine', icon: Cpu, detail: 'Connecting to your server...' },+ { key: 'model_ready' as const, label: 'Endpoint', icon: Database, detail: 'Checking endpoint...' },+ { key: 'server_ready' as const, label: 'API Server', icon: Server, detail: 'Starting server...' },+ ]+ : STEPS+ ).map((step) => (<StepRowkey={step.key}icon={step.icon}diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex 38aba42b..ff0b3d24 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -69,6 +69,7 @@ export interface SetupStatus {server_ready: boolean;model_ready: boolean;error: string | null;+ source?: 'ollama' | 'custom'; // drives source-aware setup labels}export async function getSetupStatus(): Promise<SetupStatus | null> {@@ -983,3 +984,46 @@ export async function denyAction(actionId: string): Promise<void> {const res = await fetch(`${getBase()}/v1/approvals/${actionId}/deny`, { method: 'POST' });if (!res.ok) throw new Error(`Failed: ${res.status}`);}++// ---------------------------------------------------------------------------+// Inference source (desktop only)+// ---------------------------------------------------------------------------++export type InferenceSource = {+ kind: 'ollama' | 'custom';+ model?: string;+ host?: string;+ engine?: string;+};++export async function getInferenceSource(): Promise<InferenceSource> {+ if (isTauri()) {+ try {+ const { invoke } = await import('@tauri-apps/api/core');+ return await invoke<InferenceSource>('get_inference_source');+ } catch (e: any) {+ throw new Error(e?.message ?? e ?? 'Failed to read inference source');+ }+ }+ return { kind: 'ollama' };+}++export async function setInferenceSource(+ src: InferenceSource & { apiKey?: string },+): Promise<void> {+ if (!isTauri()) throw new Error('Inference source is configurable in the desktop app only.');+ try {+ const { invoke } = await import('@tauri-apps/api/core');+ await invoke<void>('set_inference_source', {+ kind: src.kind,+ model: src.model ?? null,+ host: src.host ?? null,+ engine: src.engine ?? null,+ apiKey: src.apiKey ?? null,+ });+ } catch (e: any) {+ // Surface the backend's actionable error strings (e.g. "A server URL is+ // required…", "Could not store the API key…") as proper Error instances.+ throw new Error(e?.message ?? e ?? 'Failed to save inference source');+ }+}diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsxindex 8317861b..37788d97 100644--- a/frontend/src/pages/SettingsPage.tsx+++ b/frontend/src/pages/SettingsPage.tsx@@ -19,7 +19,7 @@ import {RefreshCw,} from 'lucide-react';import { useAppStore, type ThemeMode } from '../lib/store';-import { checkHealth, fetchSpeechHealth, getMemoryStats } from '../lib/api';+import { checkHealth, fetchSpeechHealth, getMemoryStats, getInferenceSource, setInferenceSource, type InferenceSource } from '../lib/api';import { isAutoUpdateDisabled, setAutoUpdateDisabled } from '../components/Desktop/UpdateChecker';function OllamaModelList() {@@ -162,6 +162,35 @@ export function SettingsPage() {try { return parseInt(localStorage.getItem('openjarvis-memory-max-tokens') || '2048'); } catch { return 2048; }});+ const [srcKind, setSrcKind] = useState<InferenceSource['kind']>('ollama');+ const [customHost, setCustomHost] = useState('http://localhost:1234/v1');+ const [customModel, setCustomModel] = useState('');+ const [customEngine, setCustomEngine] = useState('lmstudio');+ const [customKey, setCustomKey] = useState('');+ const [srcMsg, setSrcMsg] = useState('');++ useEffect(() => {+ getInferenceSource().then((s) => {+ setSrcKind(s.kind);+ if (s.host) setCustomHost(s.host);+ if (s.model) setCustomModel(s.model);+ if (s.engine) setCustomEngine(s.engine);+ }).catch(() => {});+ }, []);++ const saveSource = useCallback(async () => {+ try {+ if (srcKind === 'custom') {+ await setInferenceSource({ kind: 'custom', host: customHost, model: customModel, engine: customEngine, apiKey: customKey || undefined });+ } else {+ await setInferenceSource({ kind: 'ollama' });+ }+ setSrcMsg('Saved — restart the app to apply.');+ } catch (e: any) {+ setSrcMsg(e?.message ?? 'Failed to save.');+ }+ }, [srcKind, customHost, customModel, customEngine, customKey]);+useEffect(() => {checkHealth().then(setHealthy);fetchSpeechHealth()@@ -318,6 +347,58 @@ export function SettingsPage() {</SettingRow></Section>+ {/* Inference source */}+ <Section title="Inference source">+ <SettingRow label="Source" description="Where the app runs models. Applies after restart.">+ <select+ value={srcKind}+ onChange={(e) => { setSrcKind(e.target.value as InferenceSource['kind']); setSrcMsg(''); }}+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}+ >+ <option value="ollama">Bundled Ollama (default)</option>+ <option value="custom">Custom OpenAI-compatible server</option>+ </select>+ </SettingRow>+ {srcKind === 'custom' && (+ <>+ <SettingRow label="Server URL" description="e.g. LM Studio: http://localhost:1234/v1">+ <input type="text" value={customHost} onChange={(e) => { setCustomHost(e.target.value); setSrcMsg(''); }} placeholder="http://localhost:1234/v1"+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />+ </SettingRow>+ <SettingRow label="Model" description="Model id served by your endpoint">+ <input type="text" value={customModel} onChange={(e) => { setCustomModel(e.target.value); setSrcMsg(''); }} placeholder="qwen2.5-7b-instruct"+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />+ </SettingRow>+ <SettingRow label="Server type" description="OpenAI-compatible engine">+ <select value={customEngine} onChange={(e) => { setCustomEngine(e.target.value); setSrcMsg(''); }}+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>+ <option value="lmstudio">LM Studio</option>+ <option value="vllm">vLLM</option>+ <option value="sglang">SGLang</option>+ <option value="llamacpp">llama.cpp</option>+ <option value="mlx">MLX</option>+ </select>+ </SettingRow>+ <SettingRow label="API key (optional)" description="Only if your server requires one">+ <input type="password" value={customKey} onChange={(e) => { setCustomKey(e.target.value); setSrcMsg(''); }} placeholder="leave blank if none"+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />+ </SettingRow>+ </>+ )}+ <SettingRow label="" description={srcMsg}>+ <button onClick={saveSource}+ className="text-sm px-3 py-1.5 rounded-lg outline-none cursor-pointer"+ style={{ background: 'var(--color-accent, var(--color-bg-tertiary))', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>+ Save inference source+ </button>+ </SettingRow>+ </Section>+{/* Models */}<Section title="Models"><SettingRow label="Local models (Ollama)" description="Models available for local inference">