Skip to content

Personal AI, On Personal Devices

Python

feat(desktop): single default model + custom OpenAI-compatible endpoints (#453)

Jon Saad-Falcon21f1becb4d15
diff --git a/frontend/src-tauri/Cargo.lock b/frontend/src-tauri/Cargo.lock
index a33263ca..f36386ac 100644
--- a/frontend/src-tauri/Cargo.lock
+++ b/frontend/src-tauri/Cargo.lock
@@ -1384,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb0228f477c0900c880fd78c8759b95c7636dbd7842707f49e132378aa2acdc"
dependencies = [
"heck 0.4.1",
- "proc-macro-crate 2.0.2",
+ "proc-macro-crate 2.0.0",
"proc-macro-error",
"proc-macro2",
"quote",
@@ -2593,7 +2593,7 @@ dependencies = [
[[package]]
name = "openjarvis-desktop"
-version = "0.1.0"
+version = "1.0.1"
dependencies = [
"dispatch",
"objc",
@@ -2611,6 +2611,7 @@ dependencies = [
"tauri-plugin-single-instance",
"tauri-plugin-updater",
"tokio",
+ "toml_edit 0.22.27",
]
[[package]]
@@ -3018,11 +3019,10 @@ dependencies = [
[[package]]
name = "proc-macro-crate"
-version = "2.0.2"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24"
+checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8"
dependencies = [
- "toml_datetime 0.6.3",
"toml_edit 0.20.2",
]
@@ -4766,7 +4766,7 @@ checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"
dependencies = [
"serde",
"serde_spanned 0.6.9",
- "toml_datetime 0.6.3",
+ "toml_datetime 0.6.11",
"toml_edit 0.20.2",
]
@@ -4802,9 +4802,9 @@ dependencies = [
[[package]]
name = "toml_datetime"
-version = "0.6.3"
+version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
@@ -4834,7 +4834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
"indexmap 2.13.0",
- "toml_datetime 0.6.3",
+ "toml_datetime 0.6.11",
"winnow 0.5.40",
]
@@ -4847,10 +4847,22 @@ dependencies = [
"indexmap 2.13.0",
"serde",
"serde_spanned 0.6.9",
- "toml_datetime 0.6.3",
+ "toml_datetime 0.6.11",
"winnow 0.5.40",
]
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap 2.13.0",
+ "toml_datetime 0.6.11",
+ "toml_write",
+ "winnow 0.7.14",
+]
+
[[package]]
name = "toml_edit"
version = "0.23.10+spec-1.0.0"
@@ -4872,6 +4884,12 @@ dependencies = [
"winnow 0.7.14",
]
+[[package]]
+name = "toml_write"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+
[[package]]
name = "toml_writer"
version = "1.0.6+spec-1.1.0"
diff --git a/frontend/src-tauri/Cargo.toml b/frontend/src-tauri/Cargo.toml
index bf90a7b3..97a33e48 100644
--- a/frontend/src-tauri/Cargo.toml
+++ b/frontend/src-tauri/Cargo.toml
@@ -9,6 +9,7 @@ license = "MIT"
tauri-build = { version = "2", features = [] }
[dependencies]
+toml_edit = "0.22"
tauri = { version = "2", features = ["tray-icon"] }
tauri-plugin-notification = "2"
tauri-plugin-shell = "2"
diff --git a/frontend/src-tauri/src/lib.rs b/frontend/src-tauri/src/lib.rs
index b9297442..8c6d57c8 100644
--- a/frontend/src-tauri/src/lib.rs
+++ b/frontend/src-tauri/src/lib.rs
@@ -76,29 +76,102 @@ fn total_ram_gb() -> f64 {
8.0
}
-/// Return the list of Qwen3.5 models that fit on this machine, smallest first.
-fn models_that_fit() -> Vec<&'static str> {
- let ram = total_ram_gb();
+/// Return the Qwen3.5 models that fit in `ram_gb`, smallest first.
+fn models_that_fit_in(ram_gb: f64) -> Vec<&'static str> {
QWEN35_MODELS
.iter()
- .filter(|(_, _, min_ram)| ram >= *min_ram)
+ .filter(|(_, _, min_ram)| ram_gb >= *min_ram)
.map(|(tag, _, _)| *tag)
.collect()
}
-/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise
-/// falls back to the third-largest model that fits on this machine.
-fn preferred_model() -> &'static str {
- let fitting = models_that_fit();
- // Prefer STARTUP_MODEL when it fits (fast, good quality)
- if fitting.contains(&STARTUP_MODEL) {
- return STARTUP_MODEL;
- }
+/// The default local model: the second-largest Qwen3.5 model that fits in
+/// `ram_gb`. Falls back to the only fitting model, or FALLBACK_MODEL if none
+/// fit. Deliberately NOT the largest — leaves RAM headroom for the OS/app.
+fn default_local_model(ram_gb: f64) -> &'static str {
+ let fitting = models_that_fit_in(ram_gb);
match fitting.len() {
0 => FALLBACK_MODEL,
1 => fitting[0],
- 2 => fitting[0],
- n => fitting[n - 3], // third-largest
+ n => fitting[n - 2],
+ }
+}
+
+/// A resolved boot plan derived purely from the inference config + RAM.
+/// Pure and side-effect-free so it can be unit-tested without spawning
+/// processes or touching the network.
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct BootPlan {
+ /// Whether to start and wait for the bundled Ollama.
+ launch_ollama: bool,
+ /// The single Ollama model to pull (None for custom endpoints).
+ model_to_pull: Option<String>,
+ /// Optional `(engine_key, bare_host)` override for a custom endpoint,
+ /// e.g. `("lmstudio", "http://localhost:1234")`. Written into
+ /// ~/.openjarvis/config.toml so `jarvis serve` picks it up.
+ engine_host: Option<(String, String)>,
+ /// Args appended after `uv run jarvis serve --port <port>`.
+ serve_args: Vec<String>,
+}
+
+/// Default OpenAI-compatible engine key used when a custom endpoint config
+/// omits one (LM Studio is the canonical local server).
+const CUSTOM_FALLBACK_ENGINE: &str = "lmstudio";
+
+/// Decide what to launch/pull/serve from the inference config + system RAM.
+/// Pure: no I/O, no spawning.
+fn boot_plan(cfg: &InferenceConfig, ram_gb: f64) -> BootPlan {
+ match cfg.kind {
+ SourceKind::Ollama => {
+ let model = cfg
+ .model
+ .clone()
+ .unwrap_or_else(|| default_local_model(ram_gb).to_string());
+ BootPlan {
+ launch_ollama: true,
+ model_to_pull: Some(model.clone()),
+ engine_host: None,
+ serve_args: vec![
+ "--engine".into(),
+ "ollama".into(),
+ "--model".into(),
+ model,
+ "--agent".into(),
+ "simple".into(),
+ ],
+ }
+ }
+ SourceKind::Custom => {
+ let engine = cfg
+ .engine
+ .clone()
+ .unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());
+ // Record (engine_key, bare_host) only when a host is configured, so
+ // boot can write `[engine.<key>] host = ...` into config.toml. An
+ // empty host is dropped (no override).
+ let engine_host = cfg
+ .host
+ .clone()
+ .filter(|h| !h.is_empty())
+ .map(|h| (engine.clone(), h));
+ // `model` may be empty if the config is malformed; `jarvis serve`
+ // surfaces a clear error then (there is no universal default model
+ // for an arbitrary endpoint).
+ let model = cfg.model.clone().unwrap_or_default();
+ BootPlan {
+ launch_ollama: false,
+ model_to_pull: None,
+ engine_host,
+ serve_args: vec![
+ "--engine".into(),
+ engine,
+ "--model".into(),
+ model,
+ "--agent".into(),
+ "simple".into(),
+ ],
+ }
+ }
}
}
@@ -343,6 +416,8 @@ struct SetupStatus {
server_ready: bool,
model_ready: bool,
error: Option<String>,
+ /// "ollama" | "custom" — lets the setup UI relabel the progress steps.
+ source: String,
}
impl Default for SetupStatus {
@@ -354,6 +429,7 @@ impl Default for SetupStatus {
server_ready: false,
model_ready: false,
error: None,
+ source: "ollama".into(),
}
}
}
@@ -381,6 +457,28 @@ async fn wait_for_url(url: &str, timeout: Duration) -> bool {
false
}
+/// True if a custom OpenAI-compatible endpoint answers at all (any HTTP
+/// status counts — even a 404 proves the server is up). `host` is the bare
+/// base URL; we probe `<host>/v1/models`.
+async fn endpoint_reachable(host: &str, timeout: Duration) -> bool {
+ let client = match reqwest::Client::builder()
+ .timeout(Duration::from_secs(3))
+ .build()
+ {
+ Ok(c) => c,
+ Err(_) => return false,
+ };
+ let url = format!("{}/v1/models", host.trim_end_matches('/'));
+ let deadline = tokio::time::Instant::now() + timeout;
+ while tokio::time::Instant::now() < deadline {
+ if client.get(&url).send().await.is_ok() {
+ return true;
+ }
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ }
+ false
+}
+
/// Outcome of waiting for `jarvis serve` to become healthy.
///
/// Unlike [`wait_for_url`] this differentiates "server is up but degraded"
@@ -608,80 +706,153 @@ fn format_uv_sync_spawn_error(root: &std::path::Path, uv_bin: &str, err: &str) -
// ---------------------------------------------------------------------------
async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
- // Phase 1: Start Ollama
+ // Decide the inference source (default Ollama) before launching anything.
+ let cfg = read_inference_config();
+ let plan = boot_plan(&cfg, total_ram_gb());
{
let mut s = status.lock().await;
- s.phase = "ollama".into();
- s.detail = "Starting inference engine...".into();
+ s.source = match cfg.kind {
+ SourceKind::Ollama => "ollama",
+ SourceKind::Custom => "custom",
+ }
+ .into();
}
- // Try the bundled sidecar first, fall back to system ollama
- let ollama_child = {
- let ollama_bin = resolve_bin("ollama");
- let sidecar = tokio::process::Command::new(&ollama_bin)
- .arg("serve")
- .env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))
- .stdout(std::process::Stdio::null())
- .stderr(std::process::Stdio::null())
- .spawn();
- match sidecar {
- Ok(child) => Some(child),
- Err(_) => None,
- }
- };
+ // For the Ollama path, the model pull may fall back to FALLBACK_MODEL; we
+ // record what is actually available here so the serve command below uses
+ // it instead of the originally-planned tag. None on the custom path.
+ let mut serve_model_override: Option<String> = None;
- if let Some(child) = ollama_child {
- backend.lock().await.ollama = Some(ChildHandle { child });
- }
+ if plan.launch_ollama {
+ // Phase 1: Start Ollama
+ {
+ let mut s = status.lock().await;
+ s.phase = "ollama".into();
+ s.detail = "Starting inference engine...".into();
+ }
- let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);
- let ollama_ok = wait_for_url(&ollama_url, Duration::from_secs(30)).await;
+ // Try the bundled sidecar first, fall back to system ollama
+ let ollama_child = {
+ let ollama_bin = resolve_bin("ollama");
+ let sidecar = tokio::process::Command::new(&ollama_bin)
+ .arg("serve")
+ .env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null())
+ .spawn();
+ match sidecar {
+ Ok(child) => Some(child),
+ Err(_) => None,
+ }
+ };
- if !ollama_ok {
- let mut s = status.lock().await;
- s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());
- return;
- }
+ if let Some(child) = ollama_child {
+ backend.lock().await.ollama = Some(ChildHandle { child });
+ }
- {
- let mut s = status.lock().await;
- s.ollama_ready = true;
- s.detail = "Inference engine ready.".into();
- }
+ let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);
+ if !wait_for_url(&ollama_url, Duration::from_secs(30)).await {
+ let mut s = status.lock().await;
+ s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());
+ return;
+ }
- // Phase 2: Pull one small model (qwen3.5:2b) so the app can open fast.
- // Remaining models are pulled in the background after the server starts.
- {
- let mut s = status.lock().await;
- s.phase = "model".into();
- s.detail = format!("Checking for {}...", STARTUP_MODEL);
- }
+ {
+ let mut s = status.lock().await;
+ s.ollama_ready = true;
+ s.detail = "Inference engine ready.".into();
+ }
- if !ollama_has_model(STARTUP_MODEL).await {
+ // Phase 2: Pull the single default model (see default_local_model /
+ // boot_plan). We deliberately do NOT pull any others.
+ let model = plan
+ .model_to_pull
+ .clone()
+ .unwrap_or_else(|| STARTUP_MODEL.to_string());
{
let mut s = status.lock().await;
- s.detail = format!("Downloading {}... (this may take a minute)", STARTUP_MODEL);
+ s.phase = "model".into();
+ s.detail = format!("Checking for {}...", model);
}
- if let Err(e) = pull_model(STARTUP_MODEL).await {
- // If the startup model fails, try the tiny fallback
- eprintln!("Warning: failed to pull {}: {}", STARTUP_MODEL, e);
- if !ollama_has_model(FALLBACK_MODEL).await {
+
+ if !ollama_has_model(&model).await {
+ {
let mut s = status.lock().await;
- s.detail = format!("Downloading {}...", FALLBACK_MODEL);
- drop(s);
- if let Err(e2) = pull_model(FALLBACK_MODEL).await {
- let mut s = status.lock().await;
- s.error = Some(format!("Failed to download model: {}", e2));
- return;
+ s.detail = format!("Downloading {}... (this may take a minute)", model);
+ }
+ if let Err(e) = pull_model(&model).await {
+ // If the chosen model fails, try the tiny fallback
+ eprintln!("Warning: failed to pull {}: {}", model, e);
+ if !ollama_has_model(FALLBACK_MODEL).await {
+ {
+ let mut s = status.lock().await;
+ s.detail = format!("Downloading {}...", FALLBACK_MODEL);
+ }
+ if let Err(e2) = pull_model(FALLBACK_MODEL).await {
+ let mut s = status.lock().await;
+ s.error = Some(format!("Failed to download model: {}", e2));
+ return;
+ }
}
}
}
- }
- {
- let mut s = status.lock().await;
- s.model_ready = true;
- s.detail = "Model ready.".into();
+ // The pull may have fallen back to FALLBACK_MODEL; serve and persist
+ // whatever is actually available now, not the originally-planned tag.
+ let resolved_model = if ollama_has_model(&model).await {
+ model
+ } else {
+ FALLBACK_MODEL.to_string()
+ };
+ serve_model_override = Some(resolved_model.clone());
+
+ // Persist the resolved model so Settings shows it and future boots reuse it.
+ let mut persisted = cfg.clone();
+ persisted.model = Some(resolved_model);
+ let _ = write_inference_config(&persisted);
+
+ {
+ let mut s = status.lock().await;
+ s.model_ready = true;
+ s.detail = "Model ready.".into();
+ }
+ } else {
+ // Custom OpenAI-compatible endpoint: never start Ollama, never download.
+ let host = plan
+ .engine_host
+ .as_ref()
+ .map(|(_, v)| v.clone())
+ .unwrap_or_default();
+ {
+ let mut s = status.lock().await;
+ s.phase = "model".into();
+ s.detail = format!("Connecting to {}...", host);
+ }
+ if host.is_empty() || !endpoint_reachable(&host, Duration::from_secs(15)).await {
+ let mut s = status.lock().await;
+ s.error = Some(format!(
+ "Could not reach your custom inference server at {}. \
+ Start the server (e.g. LM Studio) and check the URL in Settings, then relaunch.",
+ if host.is_empty() { "(no URL set)" } else { host.as_str() }
+ ));
+ return;
+ }
+ // Point `jarvis serve` at the user's endpoint by writing the engine
+ // host into ~/.openjarvis/config.toml (the env var alone is shadowed by
+ // the engine's non-empty default host in the Python layer).
+ if let Some((engine, host)) = &plan.engine_host {
+ if let Err(e) = set_engine_host_in_config(engine, host) {
+ let mut s = status.lock().await;
+ s.error = Some(format!("Could not write engine config: {}", e));
+ return;
+ }
+ }
+ {
+ let mut s = status.lock().await;
+ s.ollama_ready = true;
+ s.model_ready = true;
+ s.detail = "Connected to custom endpoint.".into();
+ }
}
// Phase 3: Start jarvis serve
@@ -848,16 +1019,6 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
}
}
- // Start with STARTUP_MODEL (just pulled) or preferred if already available.
- let pref = preferred_model();
- let startup_model = if ollama_has_model(pref).await {
- pref
- } else if ollama_has_model(STARTUP_MODEL).await {
- STARTUP_MODEL
- } else {
- FALLBACK_MODEL
- };
-
let root = project_root.as_ref().unwrap();
// Install dependencies automatically (handles fresh clones).
@@ -907,28 +1068,35 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
{
let mut s = status.lock().await;
- s.detail = format!(
- "Starting server with {} from {}...",
- startup_model,
- root.display(),
- );
+ s.detail = format!("Starting API server from {}...", root.display());
}
let mut cmd = tokio::process::Command::new(&uv_bin);
- cmd.args([
- "run",
- "jarvis",
- "serve",
- "--port",
- &JARVIS_PORT.to_string(),
- "--model",
- startup_model,
- "--agent",
- "simple",
- ])
- .stdout(std::process::Stdio::null())
- .stderr(std::process::Stdio::piped())
- .current_dir(root);
+ let mut serve_argv: Vec<String> = vec![
+ "run".into(),
+ "jarvis".into(),
+ "serve".into(),
+ "--port".into(),
+ JARVIS_PORT.to_string(),
+ ];
+ serve_argv.extend(plan.serve_args.iter().cloned());
+ // If the Ollama pull fell back to a different tag than planned, serve the
+ // tag that is actually present. boot_plan always emits `--model` followed
+ // immediately by its value, so `i + 1` is in bounds.
+ if let Some(m) = &serve_model_override {
+ match serve_argv.iter().position(|a| a == "--model") {
+ Some(i) if i + 1 < serve_argv.len() => serve_argv[i + 1] = m.clone(),
+ _ => eprintln!(
+ "Warning: resolved model {:?} could not be applied; \
+ '--model <value>' not found in serve args {:?}",
+ m, serve_argv
+ ),
+ }
+ }
+ cmd.args(&serve_argv)
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::piped())
+ .current_dir(root);
// Inject cloud API keys from ~/.openjarvis/cloud-keys.env
for (key, value) in read_cloud_keys() {
@@ -971,11 +1139,16 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
s.error = Some(format!(
"Jarvis server is running but the inference engine is not available \
(HTTP 503). This usually means the configured model couldn't be loaded.\n\n\
- Check the server logs, or run 'uv run jarvis serve --port {} --model {}' \
+ Check the server logs, or run 'uv run jarvis serve --port {}{}' \
from {} to see the engine error.\n\n\
Server response:\n{}",
JARVIS_PORT,
- startup_model,
+ // Show the args actually passed (after `serve --port <port>`),
+ // including any post-fallback `--model` override.
+ match serve_argv.get(5..) {
+ Some(rest) if !rest.is_empty() => format!(" {}", rest.join(" ")),
+ _ => String::new(),
+ },
root.display(),
body.trim(),
));
@@ -1435,6 +1608,58 @@ async fn get_cloud_key_status() -> Result<serde_json::Value, String> {
Ok(serde_json::json!(status))
}
+/// Return the current inference-source config for the Settings UI.
+#[tauri::command]
+async fn get_inference_source() -> Result<InferenceConfig, String> {
+ Ok(read_inference_config())
+}
+
+/// Persist the chosen inference source. `host` is normalized to a bare base
+/// URL. For custom endpoints, an optional API key is stored in cloud-keys.env
+/// under `<ENGINE>_API_KEY`. Applies on next app launch.
+#[tauri::command]
+async fn set_inference_source(
+ kind: String,
+ model: Option<String>,
+ host: Option<String>,
+ engine: Option<String>,
+ api_key: Option<String>,
+) -> Result<(), String> {
+ let kind = match kind.as_str() {
+ "ollama" => SourceKind::Ollama,
+ "custom" => SourceKind::Custom,
+ other => return Err(format!("Unknown inference source kind: {:?}", other)),
+ };
+ let cfg = InferenceConfig {
+ kind,
+ model: model.filter(|m| !m.is_empty()),
+ host: host.map(|h| normalize_host(&h)).filter(|h| !h.is_empty()),
+ engine: engine.filter(|e| !e.is_empty()),
+ };
+ if let SourceKind::Custom = cfg.kind {
+ if cfg.host.is_none() {
+ return Err("A server URL is required for a custom endpoint.".into());
+ }
+ if cfg.model.as_deref().unwrap_or("").is_empty() {
+ return Err("A model name is required for a custom endpoint.".into());
+ }
+ if let Some(key) = api_key.filter(|k| !k.is_empty()) {
+ let engine = cfg
+ .engine
+ .clone()
+ .unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());
+ let key_name = format!("{}_API_KEY", engine.to_ascii_uppercase());
+ // Save the key before persisting the config: if the key can't be
+ // written, surface it and DON'T record a custom source whose
+ // credential is missing (which would fail confusingly at runtime).
+ save_cloud_key(key_name, key)
+ .await
+ .map_err(|e| format!("Could not store the API key: {}", e))?;
+ }
+ }
+ write_inference_config(&cfg)
+}
+
/// Pull a model via Ollama (called from frontend download button).
#[tauri::command]
async fn pull_ollama_model(model_name: String) -> Result<serde_json::Value, String> {
@@ -1464,6 +1689,103 @@ async fn delete_ollama_model(model_name: String) -> Result<serde_json::Value, St
Ok(serde_json::json!({"status": "deleted", "model": model_name}))
}
+// ---------------------------------------------------------------------------
+// Inference-source selection (~/.openjarvis/inference.json)
+// ---------------------------------------------------------------------------
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+enum SourceKind {
+ Ollama,
+ Custom,
+}
+
+impl Default for SourceKind {
+ fn default() -> Self {
+ SourceKind::Ollama
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]
+struct InferenceConfig {
+ #[serde(default)]
+ kind: SourceKind,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ model: Option<String>,
+ /// Bare base URL (no trailing `/v1`), custom only.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ host: Option<String>,
+ /// OpenAI-compatible engine key (e.g. "lmstudio"), custom only.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ engine: Option<String>,
+}
+
+/// Path to the inference-source config (~/.openjarvis/inference.json).
+fn inference_config_path() -> std::path::PathBuf {
+ std::path::PathBuf::from(home_dir())
+ .join(".openjarvis")
+ .join("inference.json")
+}
+
+/// Parse config text. Any error (missing/garbage) yields the Ollama default —
+/// a broken file must never strand the user with no working inference source.
+fn parse_inference_config(text: &str) -> InferenceConfig {
+ serde_json::from_str::<InferenceConfig>(text).unwrap_or_default()
+}
+
+/// Read the on-disk inference config, or the Ollama default if absent.
+fn read_inference_config() -> InferenceConfig {
+ match std::fs::read_to_string(inference_config_path()) {
+ Ok(text) => parse_inference_config(&text),
+ Err(_) => InferenceConfig::default(),
+ }
+}
+
+/// Write the inference config to disk (pretty JSON).
+fn write_inference_config(cfg: &InferenceConfig) -> Result<(), String> {
+ let path = inference_config_path();
+ if let Some(parent) = path.parent() {
+ let _ = std::fs::create_dir_all(parent);
+ }
+ let json = serde_json::to_string_pretty(cfg).map_err(|e| e.to_string())?;
+ std::fs::write(&path, json + "\n").map_err(|e| format!("Failed to save inference config: {}", e))
+}
+
+/// Upsert `[engine.<engine>] host = "<host>"` into an existing config.toml
+/// string, preserving all other content/formatting. Pure: string in, string out.
+fn upsert_engine_host(existing: &str, engine: &str, host: &str) -> Result<String, String> {
+ let mut doc = existing
+ .parse::<toml_edit::DocumentMut>()
+ .map_err(|e| format!("Invalid config.toml: {}", e))?;
+ doc["engine"][engine]["host"] = toml_edit::value(host);
+ Ok(doc.to_string())
+}
+
+/// Write the custom-endpoint host into ~/.openjarvis/config.toml so
+/// `jarvis serve` (which reads that file via load_config) points at it.
+/// The `<ENGINE>_HOST` env var is unreliable — it is shadowed by the engine's
+/// non-empty default host in the Python layer — so config.toml is the override.
+fn set_engine_host_in_config(engine: &str, host: &str) -> Result<(), String> {
+ let path = std::path::PathBuf::from(home_dir())
+ .join(".openjarvis")
+ .join("config.toml");
+ if let Some(parent) = path.parent() {
+ let _ = std::fs::create_dir_all(parent);
+ }
+ let existing = std::fs::read_to_string(&path).unwrap_or_default();
+ let updated = upsert_engine_host(&existing, engine, host)?;
+ std::fs::write(&path, updated).map_err(|e| format!("Failed to write config.toml: {}", e))
+}
+
+/// Normalize a user-entered server URL to a bare base host: trim whitespace,
+/// drop a trailing `/v1` segment (the engine re-appends its own api prefix),
+/// then drop any trailing slash.
+fn normalize_host(raw: &str) -> String {
+ let s = raw.trim().trim_end_matches('/');
+ let s = s.strip_suffix("/v1").unwrap_or(s);
+ s.trim_end_matches('/').to_string()
+}
+
/// Check speech backend health.
#[tauri::command]
async fn speech_health(api_url: String) -> Result<serde_json::Value, String> {
@@ -1987,6 +2309,8 @@ pub fn run() {
delete_ollama_model,
save_cloud_key,
get_cloud_key_status,
+ get_inference_source,
+ set_inference_source,
toggle_overlay,
hide_overlay,
get_overlay_conversation,
@@ -2009,7 +2333,11 @@ pub fn run() {
#[cfg(test)]
mod tests {
- use super::{format_uv_sync_failure, format_uv_sync_spawn_error, uv_sync_stderr_tail};
+ use super::{
+ boot_plan, default_local_model, format_uv_sync_failure, format_uv_sync_spawn_error,
+ normalize_host, parse_inference_config, upsert_engine_host, uv_sync_stderr_tail,
+ InferenceConfig, SourceKind,
+ };
use std::path::Path;
#[test]
@@ -2074,4 +2402,146 @@ mod tests {
assert!(msg.contains("/repo"));
assert!(msg.contains("No such file or directory"));
}
+
+ #[test]
+ fn default_local_model_picks_second_largest_that_fits() {
+ // QWEN35_MODELS min_ram ladder: 4,6,8,12,24,32,96 GB
+ assert_eq!(default_local_model(4.0), "qwen3.5:0.8b"); // only one fits
+ assert_eq!(default_local_model(8.0), "qwen3.5:2b"); // fits 0.8/2/4 → 2nd-largest
+ assert_eq!(default_local_model(16.0), "qwen3.5:4b"); // fits ..9b → 2nd-largest
+ assert_eq!(default_local_model(32.0), "qwen3.5:27b"); // fits 0.8/2/4/9/27/35b → 2nd-largest is 27b
+ assert_eq!(default_local_model(128.0), "qwen3.5:35b"); // fits all → 2nd-largest
+ }
+
+ #[test]
+ fn default_local_model_falls_back_when_nothing_fits() {
+ assert_eq!(default_local_model(1.0), super::FALLBACK_MODEL);
+ }
+
+ #[test]
+ fn parse_defaults_to_ollama_when_file_missing_or_garbage() {
+ assert!(matches!(parse_inference_config("").kind, SourceKind::Ollama));
+ assert!(matches!(parse_inference_config("not json").kind, SourceKind::Ollama));
+ }
+
+ #[test]
+ fn parse_reads_custom_endpoint() {
+ let cfg = parse_inference_config(
+ r#"{"kind":"custom","model":"qwen2.5-7b","host":"http://localhost:1234","engine":"lmstudio"}"#,
+ );
+ assert!(matches!(cfg.kind, SourceKind::Custom));
+ assert_eq!(cfg.model.as_deref(), Some("qwen2.5-7b"));
+ assert_eq!(cfg.host.as_deref(), Some("http://localhost:1234"));
+ assert_eq!(cfg.engine.as_deref(), Some("lmstudio"));
+ }
+
+ #[test]
+ fn normalize_host_strips_trailing_slash_and_v1() {
+ assert_eq!(normalize_host("http://localhost:1234/v1"), "http://localhost:1234");
+ assert_eq!(normalize_host("http://localhost:1234/v1/"), "http://localhost:1234");
+ assert_eq!(normalize_host("http://localhost:1234/"), "http://localhost:1234");
+ assert_eq!(normalize_host("http://host:8000"), "http://host:8000");
+ }
+
+ #[test]
+ fn boot_plan_ollama_launches_and_pulls_one_model() {
+ let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };
+ let plan = boot_plan(&cfg, 16.0);
+ assert!(plan.launch_ollama);
+ assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:4b"));
+ assert!(plan.engine_host.is_none());
+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "ollama"]));
+ assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen3.5:4b"]));
+ }
+
+ #[test]
+ fn boot_plan_ollama_respects_pinned_model() {
+ let cfg = InferenceConfig {
+ kind: SourceKind::Ollama,
+ model: Some("qwen3.5:9b".into()),
+ ..Default::default()
+ };
+ let plan = boot_plan(&cfg, 16.0);
+ assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:9b"));
+ }
+
+ #[test]
+ fn boot_plan_custom_skips_ollama_and_sets_engine_host() {
+ let cfg = InferenceConfig {
+ kind: SourceKind::Custom,
+ model: Some("qwen2.5-7b".into()),
+ host: Some("http://localhost:1234".into()),
+ engine: Some("lmstudio".into()),
+ };
+ let plan = boot_plan(&cfg, 16.0);
+ assert!(!plan.launch_ollama);
+ assert!(plan.model_to_pull.is_none());
+ assert_eq!(
+ plan.engine_host,
+ Some(("lmstudio".to_string(), "http://localhost:1234".to_string()))
+ );
+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));
+ assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen2.5-7b"]));
+ }
+
+ #[test]
+ fn boot_plan_custom_defaults_engine_to_lmstudio() {
+ let cfg = InferenceConfig {
+ kind: SourceKind::Custom,
+ model: Some("m".into()),
+ host: Some("http://h:1".into()),
+ engine: None,
+ };
+ let plan = boot_plan(&cfg, 16.0);
+ assert_eq!(plan.engine_host.as_ref().unwrap().0, "lmstudio");
+ assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));
+ }
+
+ #[test]
+ fn boot_plan_custom_omits_engine_host_when_no_host() {
+ // No configured host → don't set engine_host (no override to write).
+ let cfg = InferenceConfig {
+ kind: SourceKind::Custom,
+ model: Some("m".into()),
+ host: None,
+ engine: Some("lmstudio".into()),
+ };
+ let plan = boot_plan(&cfg, 16.0);
+ assert!(plan.engine_host.is_none());
+ }
+
+ #[test]
+ fn boot_plan_ollama_uses_fallback_model_on_low_ram() {
+ // Below the smallest model's min_ram → default_local_model → FALLBACK_MODEL.
+ let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };
+ let plan = boot_plan(&cfg, 1.0);
+ assert_eq!(plan.model_to_pull.as_deref(), Some(super::FALLBACK_MODEL));
+ }
+
+ #[test]
+ fn upsert_engine_host_writes_into_empty_config() {
+ let out = upsert_engine_host("", "lmstudio", "http://localhost:1234").unwrap();
+ let doc: toml_edit::DocumentMut = out.parse().unwrap();
+ assert_eq!(
+ doc["engine"]["lmstudio"]["host"].as_str(),
+ Some("http://localhost:1234")
+ );
+ }
+
+ #[test]
+ fn upsert_engine_host_preserves_existing_content() {
+ let existing = "[intelligence]\ndefault_model = \"keep-me\"\n";
+ let out = upsert_engine_host(existing, "vllm", "http://host:8000").unwrap();
+ let doc: toml_edit::DocumentMut = out.parse().unwrap();
+ assert_eq!(doc["intelligence"]["default_model"].as_str(), Some("keep-me"));
+ assert_eq!(doc["engine"]["vllm"]["host"].as_str(), Some("http://host:8000"));
+ }
+
+ #[test]
+ fn upsert_engine_host_updates_existing_host() {
+ let existing = "[engine.lmstudio]\nhost = \"http://old:1\"\n";
+ let out = upsert_engine_host(existing, "lmstudio", "http://new:2").unwrap();
+ let doc: toml_edit::DocumentMut = out.parse().unwrap();
+ assert_eq!(doc["engine"]["lmstudio"]["host"].as_str(), Some("http://new:2"));
+ }
}
diff --git a/frontend/src/components/SetupScreen.tsx b/frontend/src/components/SetupScreen.tsx
index 15312c6f..1d477e9c 100644
--- a/frontend/src/components/SetupScreen.tsx
+++ b/frontend/src/components/SetupScreen.tsx
@@ -146,7 +146,14 @@ export function SetupScreen({ onReady }: { onReady: () => void }) {
{/* Steps */}
<div className="flex flex-col gap-2 mb-8">
- {STEPS.map((step) => (
+ {(status?.source === 'custom'
+ ? [
+ { key: 'ollama_ready' as const, label: 'Inference Engine', icon: Cpu, detail: 'Connecting to your server...' },
+ { key: 'model_ready' as const, label: 'Endpoint', icon: Database, detail: 'Checking endpoint...' },
+ { key: 'server_ready' as const, label: 'API Server', icon: Server, detail: 'Starting server...' },
+ ]
+ : STEPS
+ ).map((step) => (
<StepRow
key={step.key}
icon={step.icon}
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index 38aba42b..ff0b3d24 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -69,6 +69,7 @@ export interface SetupStatus {
server_ready: boolean;
model_ready: boolean;
error: string | null;
+ source?: 'ollama' | 'custom'; // drives source-aware setup labels
}
export async function getSetupStatus(): Promise<SetupStatus | null> {
@@ -983,3 +984,46 @@ export async function denyAction(actionId: string): Promise<void> {
const res = await fetch(`${getBase()}/v1/approvals/${actionId}/deny`, { method: 'POST' });
if (!res.ok) throw new Error(`Failed: ${res.status}`);
}
+
+// ---------------------------------------------------------------------------
+// Inference source (desktop only)
+// ---------------------------------------------------------------------------
+
+export type InferenceSource = {
+ kind: 'ollama' | 'custom';
+ model?: string;
+ host?: string;
+ engine?: string;
+};
+
+export async function getInferenceSource(): Promise<InferenceSource> {
+ if (isTauri()) {
+ try {
+ const { invoke } = await import('@tauri-apps/api/core');
+ return await invoke<InferenceSource>('get_inference_source');
+ } catch (e: any) {
+ throw new Error(e?.message ?? e ?? 'Failed to read inference source');
+ }
+ }
+ return { kind: 'ollama' };
+}
+
+export async function setInferenceSource(
+ src: InferenceSource & { apiKey?: string },
+): Promise<void> {
+ if (!isTauri()) throw new Error('Inference source is configurable in the desktop app only.');
+ try {
+ const { invoke } = await import('@tauri-apps/api/core');
+ await invoke<void>('set_inference_source', {
+ kind: src.kind,
+ model: src.model ?? null,
+ host: src.host ?? null,
+ engine: src.engine ?? null,
+ apiKey: src.apiKey ?? null,
+ });
+ } catch (e: any) {
+ // Surface the backend's actionable error strings (e.g. "A server URL is
+ // required…", "Could not store the API key…") as proper Error instances.
+ throw new Error(e?.message ?? e ?? 'Failed to save inference source');
+ }
+}
diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx
index 8317861b..37788d97 100644
--- a/frontend/src/pages/SettingsPage.tsx
+++ b/frontend/src/pages/SettingsPage.tsx
@@ -19,7 +19,7 @@ import {
RefreshCw,
} from 'lucide-react';
import { useAppStore, type ThemeMode } from '../lib/store';
-import { checkHealth, fetchSpeechHealth, getMemoryStats } from '../lib/api';
+import { checkHealth, fetchSpeechHealth, getMemoryStats, getInferenceSource, setInferenceSource, type InferenceSource } from '../lib/api';
import { isAutoUpdateDisabled, setAutoUpdateDisabled } from '../components/Desktop/UpdateChecker';
function OllamaModelList() {
@@ -162,6 +162,35 @@ export function SettingsPage() {
try { return parseInt(localStorage.getItem('openjarvis-memory-max-tokens') || '2048'); } catch { return 2048; }
});
+ const [srcKind, setSrcKind] = useState<InferenceSource['kind']>('ollama');
+ const [customHost, setCustomHost] = useState('http://localhost:1234/v1');
+ const [customModel, setCustomModel] = useState('');
+ const [customEngine, setCustomEngine] = useState('lmstudio');
+ const [customKey, setCustomKey] = useState('');
+ const [srcMsg, setSrcMsg] = useState('');
+
+ useEffect(() => {
+ getInferenceSource().then((s) => {
+ setSrcKind(s.kind);
+ if (s.host) setCustomHost(s.host);
+ if (s.model) setCustomModel(s.model);
+ if (s.engine) setCustomEngine(s.engine);
+ }).catch(() => {});
+ }, []);
+
+ const saveSource = useCallback(async () => {
+ try {
+ if (srcKind === 'custom') {
+ await setInferenceSource({ kind: 'custom', host: customHost, model: customModel, engine: customEngine, apiKey: customKey || undefined });
+ } else {
+ await setInferenceSource({ kind: 'ollama' });
+ }
+ setSrcMsg('Saved — restart the app to apply.');
+ } catch (e: any) {
+ setSrcMsg(e?.message ?? 'Failed to save.');
+ }
+ }, [srcKind, customHost, customModel, customEngine, customKey]);
+
useEffect(() => {
checkHealth().then(setHealthy);
fetchSpeechHealth()
@@ -318,6 +347,58 @@ export function SettingsPage() {
</SettingRow>
</Section>
+ {/* Inference source */}
+ <Section title="Inference source">
+ <SettingRow label="Source" description="Where the app runs models. Applies after restart.">
+ <select
+ value={srcKind}
+ onChange={(e) => { setSrcKind(e.target.value as InferenceSource['kind']); setSrcMsg(''); }}
+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}
+ >
+ <option value="ollama">Bundled Ollama (default)</option>
+ <option value="custom">Custom OpenAI-compatible server</option>
+ </select>
+ </SettingRow>
+ {srcKind === 'custom' && (
+ <>
+ <SettingRow label="Server URL" description="e.g. LM Studio: http://localhost:1234/v1">
+ <input type="text" value={customHost} onChange={(e) => { setCustomHost(e.target.value); setSrcMsg(''); }} placeholder="http://localhost:1234/v1"
+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
+ </SettingRow>
+ <SettingRow label="Model" description="Model id served by your endpoint">
+ <input type="text" value={customModel} onChange={(e) => { setCustomModel(e.target.value); setSrcMsg(''); }} placeholder="qwen2.5-7b-instruct"
+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
+ </SettingRow>
+ <SettingRow label="Server type" description="OpenAI-compatible engine">
+ <select value={customEngine} onChange={(e) => { setCustomEngine(e.target.value); setSrcMsg(''); }}
+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>
+ <option value="lmstudio">LM Studio</option>
+ <option value="vllm">vLLM</option>
+ <option value="sglang">SGLang</option>
+ <option value="llamacpp">llama.cpp</option>
+ <option value="mlx">MLX</option>
+ </select>
+ </SettingRow>
+ <SettingRow label="API key (optional)" description="Only if your server requires one">
+ <input type="password" value={customKey} onChange={(e) => { setCustomKey(e.target.value); setSrcMsg(''); }} placeholder="leave blank if none"
+ className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
+ style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
+ </SettingRow>
+ </>
+ )}
+ <SettingRow label="" description={srcMsg}>
+ <button onClick={saveSource}
+ className="text-sm px-3 py-1.5 rounded-lg outline-none cursor-pointer"
+ style={{ background: 'var(--color-accent, var(--color-bg-tertiary))', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>
+ Save inference source
+ </button>
+ </SettingRow>
+ </Section>
+
{/* Models */}
<Section title="Models">
<SettingRow label="Local models (Ollama)" description="Models available for local inference">