Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Add llama.cpp as a language model provider (#59964)
célina4b119fc54758
diff --git a/Cargo.lock b/Cargo.lockindex f43363e23c2..f5803b5baa6 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -9823,6 +9823,7 @@ dependencies = ["language","language_model","language_models_cloud",+ "llama_cpp","lmstudio","log","menu",@@ -10354,6 +10355,19 @@ dependencies = ["zed-scap",]+[[package]]+name = "llama_cpp"+version = "0.1.0"+dependencies = [+ "anyhow",+ "futures 0.3.32",+ "http_client",+ "schemars 1.0.4",+ "serde",+ "serde_json",+ "url",+]+[[package]]name = "lmdb-master-sys"version = "0.2.5"diff --git a/Cargo.toml b/Cargo.tomlindex 4f36916d2dd..4d36e276a57 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -126,6 +126,7 @@ members = ["crates/line_ending_selector","crates/livekit_api","crates/livekit_client",+ "crates/llama_cpp","crates/lmstudio","crates/lsp","crates/markdown",@@ -384,6 +385,7 @@ languages = { path = "crates/languages" }line_ending_selector = { path = "crates/line_ending_selector" }livekit_api = { path = "crates/livekit_api" }livekit_client = { path = "crates/livekit_client" }+llama_cpp = { path = "crates/llama_cpp" }lmstudio = { path = "crates/lmstudio" }lsp = { path = "crates/lsp" }markdown = { path = "crates/markdown" }@@ -977,6 +979,7 @@ edit_prediction_ui = { codegen-units = 1 }install_cli = { codegen-units = 1 }journal = { codegen-units = 1 }json_schema_store = { codegen-units = 1 }+llama_cpp = { codegen-units = 1 }lmstudio = { codegen-units = 1 }menu = { codegen-units = 1 }notifications = { codegen-units = 1 }diff --git a/assets/icons/ai_llama_cpp.svg b/assets/icons/ai_llama_cpp.svgnew file mode 100644index 00000000000..310ec3793a5--- /dev/null+++ b/assets/icons/ai_llama_cpp.svg@@ -0,0 +1,4 @@+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">+<path d="M14 9.84L12.085 13.16L12.0827 13.1586C11.745 13.6814 11.1639 14 10.5373 14H4.4L6.8 9.84H14Z" fill="currentColor"/>+<path d="M6.8 9.84H2L5.992 2.9205C6.3206 2.3509 6.9282 2 7.5858 2H11.3231L6.8 9.84Z" fill="currentColor"/>+</svg>diff --git a/assets/settings/default.json b/assets/settings/default.jsonindex e4051ee8f02..059608ae046 100644--- a/assets/settings/default.json+++ b/assets/settings/default.json@@ -2427,6 +2427,9 @@"ollama": {"api_url": "http://localhost:11434",},+ "llama_cpp": {+ "api_url": "http://localhost:8080",+ },"openai": {"api_url": "https://api.openai.com/v1",},diff --git a/crates/icons/src/icons.rs b/crates/icons/src/icons.rsindex a94b43e8017..56182da2133 100644--- a/crates/icons/src/icons.rs+++ b/crates/icons/src/icons.rs@@ -17,6 +17,7 @@ pub enum IconName {AiEdit,AiGemini,AiGoogle,+ AiLlamaCpp,AiLmStudio,AiMistral,AiOllama,diff --git a/crates/language_models/Cargo.toml b/crates/language_models/Cargo.tomlindex b96a377f02c..57a0d530e45 100644--- a/crates/language_models/Cargo.toml+++ b/crates/language_models/Cargo.toml@@ -43,6 +43,7 @@ http_client.workspace = truelanguage.workspace = truelanguage_model.workspace = truelanguage_models_cloud.workspace = true+llama_cpp = { workspace = true, features = ["schemars"] }lmstudio = { workspace = true, features = ["schemars"] }log.workspace = truemenu.workspace = truediff --git a/crates/language_models/src/language_models.rs b/crates/language_models/src/language_models.rsindex dc13cde1378..1133ee2d477 100644--- a/crates/language_models/src/language_models.rs+++ b/crates/language_models/src/language_models.rs@@ -24,6 +24,7 @@ use crate::provider::bedrock::BedrockLanguageModelProvider;use crate::provider::cloud::CloudLanguageModelProvider;use crate::provider::copilot_chat::CopilotChatLanguageModelProvider;use crate::provider::google::GoogleLanguageModelProvider;+use crate::provider::llama_cpp::LlamaCppLanguageModelProvider;use crate::provider::lmstudio::LmStudioLanguageModelProvider;pub use crate::provider::mistral::MistralLanguageModelProvider;use crate::provider::ollama::OllamaLanguageModelProvider;@@ -307,6 +308,14 @@ fn register_language_model_providers()),cx,);+ registry.register_provider(+ Arc::new(LlamaCppLanguageModelProvider::new(+ client.http_client(),+ credentials_provider.clone(),+ cx,+ )),+ cx,+ );registry.register_provider(Arc::new(DeepSeekLanguageModelProvider::new(client.http_client(),diff --git a/crates/language_models/src/provider.rs b/crates/language_models/src/provider.rsindex 3a1369a2d1c..65fc039b074 100644--- a/crates/language_models/src/provider.rs+++ b/crates/language_models/src/provider.rs@@ -10,6 +10,7 @@ pub mod cloud;pub mod copilot_chat;pub mod deepseek;pub mod google;+pub mod llama_cpp;pub mod lmstudio;pub mod mistral;pub mod ollama;diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rsnew file mode 100644index 00000000000..935ec74f9a4--- /dev/null+++ b/crates/language_models/src/provider/llama_cpp.rs@@ -0,0 +1,2073 @@+use anyhow::Result;+use collections::{HashMap, HashSet};+use credentials_provider::CredentialsProvider;+use fs::Fs;+use futures::Stream;+use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};+use gpui::{AnyView, App, AsyncApp, Context, CursorStyle, Entity, Task, TaskExt};+use http_client::{CustomHeaders, HttpClient};+use language_model::util::parse_tool_arguments;+use language_model::{+ ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel, LanguageModelCompletionError,+ LanguageModelCompletionEvent, LanguageModelId, LanguageModelName, LanguageModelProvider,+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,+ LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolResultContent,+ LanguageModelToolUse, MessageContent, RateLimiter, Role, StopReason, TokenUsage, env_var,+};+use llama_cpp::{+ LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,+ stream_model_events,+};+pub use settings::LlamaCppAvailableModel as AvailableModel;+use settings::{Settings, SettingsStore, update_settings_file};+use std::pin::Pin;+use std::sync::LazyLock;+use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};+use std::time::Duration;+use ui::{+ ButtonLike, ButtonLink, ConfiguredApiCard, ElevationIndex, List, ListBulletItem, Tooltip,+ prelude::*,+};+use ui_input::InputField;+use util::ResultExt;++use crate::AllLanguageModelSettings;++const LLAMA_CPP_DOWNLOAD_URL: &str = "https://llama.app";+const LLAMA_CPP_MODELS_URL: &str = "https://huggingface.co/models?library=gguf&sort=trending";++const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("llama_cpp");+const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new("llama.cpp");++const API_KEY_ENV_VAR_NAME: &str = "LLAMACPP_API_KEY";+static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME);++/// How long to wait before reconnecting to `/models/sse` after the stream ends.+const MODEL_EVENT_RECONNECT_INTERVAL: Duration = Duration::from_secs(5);++/// Context length assumed for an unloaded router model (it can't be probed+/// without loading it). Generous so early messages work; re-discovery refines+/// it once the model loads.+const ASSUMED_UNLOADED_CONTEXT: u64 = 131_072;++#[derive(Default, Debug, Clone, PartialEq)]+pub struct LlamaCppSettings {+ pub api_url: String,+ pub auto_discover: bool,+ pub available_models: Vec<AvailableModel>,+ pub context_window: Option<u64>,+ pub custom_headers: CustomHeaders,+}++pub struct LlamaCppLanguageModelProvider {+ http_client: Arc<dyn HttpClient>,+ state: Entity<State>,+ /// Live capabilities shared with the agent's models (see [`LiveCapabilities`]).+ capability_cells: CapabilityCells,+ /// Live model-load progress shared with the models (see [`LoadingProgress`]).+ loading_progress: LoadingProgress,+}++pub struct State {+ api_key_state: ApiKeyState,+ credentials_provider: Arc<dyn CredentialsProvider>,+ http_client: Arc<dyn HttpClient>,+ fetched_models: Vec<llama_cpp::Model>,+ fetch_model_task: Option<Task<Result<()>>>,+ /// Router-mode task on `/models/sse`; re-runs discovery as models load/unload.+ model_event_task: Option<Task<()>>,+ /// Same `Arc` as the provider's; re-discovery keeps these cells in sync.+ capability_cells: CapabilityCells,+ /// Same `Arc` as the provider's; the event stream updates it as a model loads.+ loading_progress: LoadingProgress,+}++impl State {+ fn is_authenticated(&self) -> bool {+ !self.fetched_models.is_empty()+ }++ fn set_api_key(&mut self, api_key: Option<String>, cx: &mut Context<Self>) -> Task<Result<()>> {+ let credentials_provider = self.credentials_provider.clone();+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let task = self.api_key_state.store(+ api_url,+ api_key,+ |this| &mut this.api_key_state,+ credentials_provider,+ cx,+ );++ self.fetched_models.clear();+ // Drop the event stream so it reconnects with the new key (re-fetch+ // below restarts it).+ self.model_event_task = None;+ write_recover(&self.loading_progress).clear();+ cx.spawn(async move |this, cx| {+ let result = task.await;+ this.update(cx, |this, cx| this.restart_fetch_models_task(cx))+ .ok();+ result+ })+ }++ fn authenticate(&mut self, cx: &mut Context<Self>) -> Task<Result<(), AuthenticateError>> {+ let credentials_provider = self.credentials_provider.clone();+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let load_key_task = self.api_key_state.load_if_needed(+ api_url,+ |this| &mut this.api_key_state,+ credentials_provider,+ cx,+ );++ if self.is_authenticated() {+ return Task::ready(Ok(()));+ }++ cx.spawn(async move |this, cx| {+ match load_key_task.await {+ Ok(()) | Err(AuthenticateError::CredentialsNotFound) => {}+ Err(error) => {+ log::warn!("failed to load llama.cpp API key: {error}");+ }+ }+ let fetch_models_task = this.update(cx, |this, cx| this.fetch_models(cx))?;+ match fetch_models_task.await {+ Ok(()) => Ok(()),+ Err(err) => {+ // A refused connection means the server isn't running yet, not an error.+ let connection_refused = err.chain().any(|cause| {+ cause+ .downcast_ref::<std::io::Error>()+ .is_some_and(|io_err| {+ io_err.kind() == std::io::ErrorKind::ConnectionRefused+ })+ });+ if connection_refused {+ Err(AuthenticateError::ConnectionRefused)+ } else {+ Err(AuthenticateError::Other(err))+ }+ }+ }+ })+ }++ fn fetch_models(&mut self, cx: &mut Context<Self>) -> Task<Result<()>> {+ let http_client = Arc::clone(&self.http_client);+ let settings = LlamaCppLanguageModelProvider::settings(cx);+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let api_key = self.api_key_state.key(&api_url);+ let extra_headers = settings.custom_headers.clone();++ cx.spawn(async move |this, cx| {+ let entries = get_models(+ http_client.as_ref(),+ &api_url,+ api_key.as_deref(),+ &extra_headers,+ )+ .await?;++ let is_router = entries.iter().any(ModelEntry::is_router_entry);++ // Models the server reports as loading, used below to prune stale+ // progress labels by reconciling against the live listing (a preempted+ // load or missed SSE event can skip the terminal event).+ let loading_ids: HashSet<String> = entries+ .iter()+ .filter(|entry| entry.is_loading())+ .map(|entry| entry.id.clone())+ .collect();++ let models: Vec<llama_cpp::Model> = if is_router {+ // Router mode: metadata comes from `/v1/models`. We probe+ // `/props` only for loaded models so listing never triggers a+ // load; unloaded models use the listing's hints and overrides.+ let tasks = entries.into_iter().map(|entry| {+ let http_client = Arc::clone(&http_client);+ let api_url = api_url.clone();+ let api_key = api_key.clone();+ let extra_headers = extra_headers.clone();+ async move {+ let props = if entry.is_loaded() {+ get_props(+ http_client.as_ref(),+ &api_url,+ api_key.as_deref(),+ Some(&entry.id),+ &extra_headers,+ )+ .await+ .log_err()+ } else {+ None+ };+ model_from_entry(&entry, props.as_ref())+ }+ });+ futures::stream::iter(tasks)+ .buffer_unordered(5)+ .collect()+ .await+ } else {+ // Single-model mode: one `/props` call describes the loaded model.+ let props = get_props(+ http_client.as_ref(),+ &api_url,+ api_key.as_deref(),+ None,+ &extra_headers,+ )+ .await+ .log_err();+ entries+ .iter()+ .map(|entry| model_from_entry(entry, props.as_ref()))+ .collect()+ };++ this.update(cx, |this, cx| {+ this.fetched_models = models;+ let effective = compute_effective_models(+ &this.fetched_models,+ LlamaCppLanguageModelProvider::settings(cx),+ );+ sync_capability_cells(&this.capability_cells, &effective);+ // Drop progress labels for models no longer loading, so a stale+ // "Loading …" can't stick after a preempted load or missed event.+ write_recover(&this.loading_progress).retain(|id, _| loading_ids.contains(id));+ // Router mode loads models on demand: subscribe so capabilities+ // self-correct as they load/unload. Start it once (events trigger+ // re-discovery, not a re-spawn); single-model mode needs no stream.+ if is_router {+ if this.model_event_task.is_none() {+ this.start_model_event_stream(cx);+ }+ } else {+ this.model_event_task = None;+ }+ cx.notify();+ })+ })+ }++ /// Subscribes to `/models/sse` and re-runs discovery as models load, unload,+ /// or the list changes, so capabilities stay current. Reconnects if the stream+ /// drops; on builds without `/models/sse` the refresh is simply skipped.+ fn start_model_event_stream(&mut self, cx: &mut Context<Self>) {+ let http_client = Arc::clone(&self.http_client);+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let api_key = self.api_key_state.key(&api_url);+ let extra_headers = LlamaCppLanguageModelProvider::settings(cx)+ .custom_headers+ .clone();++ self.model_event_task = Some(cx.spawn(async move |this, cx| {+ loop {+ match stream_model_events(+ http_client.as_ref(),+ &api_url,+ api_key.as_deref(),+ &extra_headers,+ )+ .await+ {+ Ok(mut events) => {+ while let Some(event) = events.next().await {+ let Some(event) = event.log_err() else {+ continue;+ };+ if let Some(exit_code) = event.load_failure() {+ log::error!(+ "llama.cpp model {} failed to load (exit code {exit_code})",+ event.model+ );+ }+ // Loading-progress tick: record it for the selector (no+ // re-discovery). `cx.notify()` drives `ProviderStateChanged`.+ if let Some(progress) = event.load_progress() {+ let label = SharedString::from(progress.progress_label());+ if this+ .update(cx, |this, cx| {+ write_recover(&this.loading_progress)+ .insert(event.model.clone(), label);+ cx.notify();+ })+ .is_err()+ {+ return;+ }+ continue;+ }+ if !event.changes_model_state() {+ continue;+ }+ // Terminal load/unload (or list change): drop the+ // progress label and re-discover to refresh capabilities.+ if this+ .update(cx, |this, cx| {+ write_recover(&this.loading_progress).remove(&event.model);+ this.restart_fetch_models_task(cx);+ })+ .is_err()+ {+ return;+ }+ }+ }+ // Endpoint missing (older build) or connection failed; retry after a backoff.+ Err(error) => {+ log::warn!("llama.cpp model event stream unavailable: {error:#}");+ }+ }++ cx.background_executor()+ .timer(MODEL_EVENT_RECONNECT_INTERVAL)+ .await;+ if this.update(cx, |_, _| ()).is_err() {+ return;+ }+ }+ }));+ }++ fn restart_fetch_models_task(&mut self, cx: &mut Context<Self>) {+ let task = self.fetch_models(cx);+ self.fetch_model_task.replace(task);+ }+}++/// Capabilities that only become known once a router model loads. `LanguageModel`'s+/// methods take no `cx`, yet the agent reads them live each turn, so we share+/// them through a map that re-discovery updates — an already-selected model+/// picks up real values without re-selection. Image support is available from+/// `/v1/models` hints, so it stays a field.+#[derive(Clone, Copy, Debug, PartialEq)]+struct LiveCapabilities {+ max_tokens: u64,+ supports_tools: bool,+ supports_thinking: bool,+}++impl LiveCapabilities {+ fn of(model: &llama_cpp::Model) -> Self {+ Self {+ max_tokens: model.max_tokens,+ supports_tools: model.supports_tools,+ supports_thinking: model.supports_thinking,+ }+ }+}++/// Live capabilities keyed by model name, shared by the provider and its models.+type CapabilityCells = Arc<RwLock<HashMap<String, LiveCapabilities>>>;++/// Model name → load-status label (e.g. `"Loading weights 42%"`) while a router+/// model loads, shared so the model selector can show progress. Absent once loaded.+type LoadingProgress = Arc<RwLock<HashMap<String, SharedString>>>;++/// Locks for reading, recovering instead of panicking on a poisoned lock. The+/// critical sections are infallible map ops, so poisoning is unreachable anyway.+fn read_recover<T>(lock: &RwLock<T>) -> RwLockReadGuard<'_, T> {+ lock.read().unwrap_or_else(|poisoned| poisoned.into_inner())+}++/// Locks for writing; see [`read_recover`].+fn write_recover<T>(lock: &RwLock<T>) -> RwLockWriteGuard<'_, T> {+ lock.write()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+}++/// The exact set of models `provided_models` exposes: discovery merged with the+/// `available_models` and `context_window` overrides. Shared with re-discovery.+fn compute_effective_models(+ fetched_models: &[llama_cpp::Model],+ settings: &LlamaCppSettings,+) -> HashMap<String, llama_cpp::Model> {+ let mut models: HashMap<String, llama_cpp::Model> = HashMap::default();+ if settings.auto_discover {+ for model in fetched_models {+ let mut model = model.clone();+ if let Some(context_window) = settings.context_window {+ model.max_tokens = context_window;+ }+ models.insert(model.name.clone(), model);+ }+ }+ merge_settings_into_models(+ &mut models,+ &settings.available_models,+ settings.context_window,+ );+ models+}++/// Updates the shared capability map from the effective models, so a model held+/// by an open conversation observes the new values (it reads the map by name).+fn sync_capability_cells(cells: &CapabilityCells, effective: &HashMap<String, llama_cpp::Model>) {+ let mut cells = write_recover(cells);+ for model in effective.values() {+ cells.insert(model.name.clone(), LiveCapabilities::of(model));+ }+}++/// Builds a model from a `/v1/models` entry, refined by `/props` when the model+/// is loaded. An unloaded router model can't be probed, so we assume optimistic+/// capabilities and let re-discovery refine them on load.+fn model_from_entry(entry: &ModelEntry, props: Option<&Props>) -> llama_cpp::Model {+ let max_tokens = props+ .and_then(Props::context_length)+ .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx))+ .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx_train))+ .unwrap_or(ASSUMED_UNLOADED_CONTEXT);+ // Trust `/props` when present. Without it, assume tools for an unloaded model+ // (re-discovery corrects on load) but not for a loaded model whose probe failed.+ let supports_tools = match props {+ Some(props) => props.supports_tools(),+ None => !entry.is_loaded(),+ };+ let supports_images = props.is_some_and(Props::supports_images) || entry.supports_images_hint();+ let supports_thinking = props.is_some_and(Props::supports_thinking);++ llama_cpp::Model::new(+ &entry.id,+ Some(&display_name_for(&entry.id)),+ Some(max_tokens),+ supports_tools,+ supports_images,+ supports_thinking,+ )+}++/// Friendly display name from a model id, which is often a `.gguf` file path.+fn display_name_for(id: &str) -> String {+ let base = id.rsplit(['/', '\\']).next().unwrap_or(id);+ base.strip_suffix(".gguf").unwrap_or(base).to_string()+}++fn telemetry_id_for(id: &str) -> String {+ format!("llama_cpp/{}", display_name_for(id))+}++impl LlamaCppLanguageModelProvider {+ pub fn new(+ http_client: Arc<dyn HttpClient>,+ credentials_provider: Arc<dyn CredentialsProvider>,+ cx: &mut App,+ ) -> Self {+ let capability_cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default()));+ let loading_progress: LoadingProgress = Arc::new(RwLock::new(HashMap::default()));+ let this = Self {+ http_client: http_client.clone(),+ capability_cells: capability_cells.clone(),+ loading_progress: loading_progress.clone(),+ state: cx.new(|cx| {+ cx.observe_global::<SettingsStore>({+ let mut last_settings = LlamaCppLanguageModelProvider::settings(cx).clone();+ move |this: &mut State, cx| {+ let current_settings = LlamaCppLanguageModelProvider::settings(cx);+ let settings_changed = current_settings != &last_settings;+ if settings_changed {+ let url_changed = last_settings.api_url != current_settings.api_url;+ last_settings = current_settings.clone();+ if url_changed {+ let credentials_provider = this.credentials_provider.clone();+ let api_url = Self::api_url(cx);+ this.api_key_state.handle_url_change(+ api_url,+ |this| &mut this.api_key_state,+ credentials_provider,+ cx,+ );+ this.fetched_models.clear();+ // Drop the event stream so it reconnects against+ // the new URL (re-auth below restarts it).+ this.model_event_task = None;+ write_recover(&this.loading_progress).clear();+ this.authenticate(cx).detach();+ }+ cx.notify();+ }+ }+ })+ .detach();++ State {+ http_client,+ fetched_models: Default::default(),+ fetch_model_task: None,+ model_event_task: None,+ capability_cells,+ loading_progress,+ api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()),+ credentials_provider,+ }+ }),+ };+ // Discover eagerly so a running server is picked up without opening settings.+ this.state+ .update(cx, |state, cx| state.restart_fetch_models_task(cx));+ this+ }++ fn settings(cx: &App) -> &LlamaCppSettings {+ &AllLanguageModelSettings::get_global(cx).llama_cpp+ }++ fn api_url(cx: &App) -> SharedString {+ let api_url = &Self::settings(cx).api_url;+ if api_url.is_empty() {+ LLAMA_CPP_API_URL.into()+ } else {+ SharedString::new(api_url.as_str())+ }+ }++ fn has_custom_url(cx: &App) -> bool {+ Self::settings(cx).api_url != LLAMA_CPP_API_URL+ }+}++impl LanguageModelProviderState for LlamaCppLanguageModelProvider {+ type ObservableEntity = State;++ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {+ Some(self.state.clone())+ }+}++impl LanguageModelProvider for LlamaCppLanguageModelProvider {+ fn id(&self) -> LanguageModelProviderId {+ PROVIDER_ID+ }++ fn name(&self) -> LanguageModelProviderName {+ PROVIDER_NAME+ }++ fn icon(&self) -> IconOrSvg {+ IconOrSvg::Icon(IconName::AiLlamaCpp)+ }++ fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {+ // No default model: in router mode it could trigger an expensive load of+ // an unloaded model on a constrained machine.+ None+ }++ fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {+ // See explanation for default_model.+ None+ }++ fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {+ let settings = LlamaCppLanguageModelProvider::settings(cx);+ let effective = compute_effective_models(&self.state.read(cx).fetched_models, settings);++ // Refresh the shared capability map so open conversations pick up settings changes.+ sync_capability_cells(&self.capability_cells, &effective);+ let mut models = effective+ .into_values()+ .map(|model| {+ Arc::new(LlamaCppLanguageModel {+ id: LanguageModelId::from(model.name.clone()),+ name: model.name.clone(),+ display_name: model.display_name().to_string(),+ fallback_capabilities: LiveCapabilities::of(&model),+ supports_images: model.supports_images,+ capability_cells: self.capability_cells.clone(),+ loading_progress: self.loading_progress.clone(),+ http_client: self.http_client.clone(),+ request_limiter: RateLimiter::new(4),+ state: self.state.clone(),+ }) as Arc<dyn LanguageModel>+ })+ .collect::<Vec<_>>();+ models.sort_by_key(|model| model.name());+ models+ }++ fn is_authenticated(&self, cx: &App) -> bool {+ self.state.read(cx).is_authenticated()+ }++ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {+ self.state.update(cx, |state, cx| state.authenticate(cx))+ }++ fn configuration_view(+ &self,+ _target_agent: language_model::ConfigurationViewTargetAgent,+ window: &mut Window,+ cx: &mut App,+ ) -> AnyView {+ let state = self.state.clone();+ cx.new(|cx| ConfigurationView::new(state, window, cx))+ .into()+ }++ fn reset_credentials(&self, cx: &mut App) -> Task<Result<()>> {+ self.state+ .update(cx, |state, cx| state.set_api_key(None, cx))+ }+}++pub struct LlamaCppLanguageModel {+ id: LanguageModelId,+ /// The model id sent to the server.+ name: String,+ display_name: String,+ /// Live capabilities shared with the provider, read fresh on each access so an+ /// open conversation reflects the model's real values once it has loaded.+ capability_cells: CapabilityCells,+ /// Used when `capability_cells` has no entry (e.g. model removed mid-conversation).+ fallback_capabilities: LiveCapabilities,+ /// Available from `/v1/models` hints, so captured at build time.+ supports_images: bool,+ /// Shared with the provider; this model's load progress, read by `name` so the+ /// selector can show a loading indicator.+ loading_progress: LoadingProgress,+ http_client: Arc<dyn HttpClient>,+ request_limiter: RateLimiter,+ state: Entity<State>,+}++impl LlamaCppLanguageModel {+ /// The model's live capabilities, or the build-time fallback if the map lacks it.+ fn capabilities(&self) -> LiveCapabilities {+ read_recover(&self.capability_cells)+ .get(&self.name)+ .copied()+ .unwrap_or(self.fallback_capabilities)+ }++ /// This model's load-status label while loading, read live from the shared map.+ fn loading_label(&self) -> Option<SharedString> {+ read_recover(&self.loading_progress)+ .get(&self.name)+ .cloned()+ }++ fn to_llama_cpp_request(+ &self,+ request: LanguageModelRequest,+ ) -> llama_cpp::ChatCompletionRequest {+ build_llama_cpp_request(+ &self.name,+ self.supports_images,+ self.capabilities(),+ request,+ )+ }++ fn stream_completion(+ &self,+ request: llama_cpp::ChatCompletionRequest,+ cx: &AsyncApp,+ ) -> BoxFuture<+ 'static,+ Result<futures::stream::BoxStream<'static, Result<llama_cpp::ResponseStreamEvent>>>,+ > {+ let http_client = self.http_client.clone();+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let extra_headers = LlamaCppLanguageModelProvider::settings(cx)+ .custom_headers+ .clone();+ (state.api_key_state.key(&api_url), api_url, extra_headers)+ });++ let future = self.request_limiter.stream(async move {+ let stream = stream_chat_completion(+ http_client.as_ref(),+ &api_url,+ api_key.as_deref(),+ request,+ &extra_headers,+ )+ .await?;+ Ok(stream)+ });++ async move { Ok(future.await?.boxed()) }.boxed()+ }+}++fn build_llama_cpp_request(+ model_name: &str,+ supports_images: bool,+ capabilities: LiveCapabilities,+ request: LanguageModelRequest,+) -> llama_cpp::ChatCompletionRequest {+ let supports_tools = capabilities.supports_tools;+ let supports_thinking = capabilities.supports_thinking;+ let mut messages = Vec::new();++ for message in request.messages {+ let mut reasoning_content: Option<String> = None;+ for content in message.content {+ match content {+ MessageContent::Text(text) => add_message_content_part(+ llama_cpp::MessagePart::Text { text },+ message.role,+ &mut messages,+ if supports_thinking && message.role == Role::Assistant {+ reasoning_content.take()+ } else {+ None+ },+ ),+ MessageContent::Thinking { text, .. } => {+ if supports_thinking && message.role == Role::Assistant && !text.is_empty() {+ reasoning_content.get_or_insert_default().push_str(&text);+ }+ }+ MessageContent::RedactedThinking(_) => {}+ MessageContent::Compaction(_) => {}+ MessageContent::Image(image) => {+ if supports_images {+ add_message_content_part(+ llama_cpp::MessagePart::Image {+ image_url: llama_cpp::ImageUrl {+ url: image.to_base64_url(),+ detail: None,+ },+ },+ message.role,+ &mut messages,+ if supports_thinking && message.role == Role::Assistant {+ reasoning_content.take()+ } else {+ None+ },+ );+ }+ }+ MessageContent::ToolUse(tool_use) => {+ let tool_call = llama_cpp::ToolCall {+ id: tool_use.id.to_string(),+ content: llama_cpp::ToolCallContent::Function {+ function: llama_cpp::FunctionContent {+ name: tool_use.name.to_string(),+ arguments: serde_json::to_string(&tool_use.input)+ .unwrap_or_default(),+ },+ },+ };++ if let Some(llama_cpp::ChatMessage::Assistant {+ tool_calls,+ reasoning_content: message_reasoning_content,+ ..+ }) = messages.last_mut()+ {+ append_reasoning_content(+ message_reasoning_content,+ reasoning_content.take(),+ );+ tool_calls.push(tool_call);+ } else {+ messages.push(llama_cpp::ChatMessage::Assistant {+ content: None,+ reasoning_content: reasoning_content.take(),+ tool_calls: vec![tool_call],+ });+ }+ }+ MessageContent::ToolResult(tool_result) => {+ let content: Vec<llama_cpp::MessagePart> = tool_result+ .content+ .iter()+ .filter_map(|part| match part {+ LanguageModelToolResultContent::Text(text) => {+ Some(llama_cpp::MessagePart::Text {+ text: text.to_string(),+ })+ }+ LanguageModelToolResultContent::Image(image) => {+ if supports_images {+ Some(llama_cpp::MessagePart::Image {+ image_url: llama_cpp::ImageUrl {+ url: image.to_base64_url(),+ detail: None,+ },+ })+ } else {+ None+ }+ }+ })+ .collect();++ messages.push(llama_cpp::ChatMessage::Tool {+ content: content.into(),+ tool_call_id: tool_result.tool_use_id.to_string(),+ });+ }+ }+ }+ }++ let tools: Vec<llama_cpp::ToolDefinition> = if supports_tools {+ request+ .tools+ .into_iter()+ .map(|tool| llama_cpp::ToolDefinition::Function {+ function: llama_cpp::FunctionDefinition {+ name: tool.name,+ description: Some(tool.description),+ parameters: Some(tool.input_schema),+ },+ })+ .collect()+ } else {+ Vec::new()+ };+ // Only send `tool_choice` with actual tools; some OpenAI-compatible servers+ // reject it otherwise.+ let tool_choice = if tools.is_empty() {+ None+ } else {+ request.tool_choice.map(|choice| match choice {+ LanguageModelToolChoice::Auto => llama_cpp::ToolChoice::Auto,+ LanguageModelToolChoice::Any => llama_cpp::ToolChoice::Required,+ LanguageModelToolChoice::None => llama_cpp::ToolChoice::None,+ })+ };++ llama_cpp::ChatCompletionRequest {+ model: model_name.to_string(),+ messages,+ stream: true,+ // Let the server decide the output length (its `n_predict` default).+ max_tokens: None,+ stop: if request.stop.is_empty() {+ None+ } else {+ Some(request.stop)+ },+ // llama.cpp models often ship recommended sampler settings, so override+ // temperature only when the request sets one.+ temperature: request.temperature,+ tools,+ tool_choice,+ stream_options: Some(llama_cpp::StreamOptions {+ include_usage: true,+ }),+ }+}++impl LanguageModel for LlamaCppLanguageModel {+ fn id(&self) -> LanguageModelId {+ self.id.clone()+ }++ fn name(&self) -> LanguageModelName {+ match self.loading_label() {+ // Surface load progress in the display name so it shows wherever the+ // model is named, without provider-agnostic UI changes. The agent+ // rebuilds the name on `ProviderStateChanged`, which our ticks emit.+ Some(label) => LanguageModelName::from(format!("{} · {}", self.display_name, label)),+ None => LanguageModelName::from(self.display_name.clone()),+ }+ }++ fn provider_id(&self) -> LanguageModelProviderId {+ PROVIDER_ID+ }++ fn provider_name(&self) -> LanguageModelProviderName {+ PROVIDER_NAME+ }++ fn supports_tools(&self) -> bool {+ self.capabilities().supports_tools+ }++ fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {+ self.supports_tools()+ && match choice {+ LanguageModelToolChoice::Auto => true,+ LanguageModelToolChoice::Any => true,+ LanguageModelToolChoice::None => true,+ }+ }++ fn supports_images(&self) -> bool {+ self.supports_images+ }++ fn supports_thinking(&self) -> bool {+ self.capabilities().supports_thinking+ }++ fn telemetry_id(&self) -> String {+ telemetry_id_for(&self.name)+ }++ fn max_token_count(&self) -> u64 {+ self.capabilities().max_tokens+ }++ fn stream_completion(+ &self,+ request: LanguageModelRequest,+ cx: &AsyncApp,+ ) -> BoxFuture<+ 'static,+ Result<+ BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,+ LanguageModelCompletionError,+ >,+ > {+ let request = self.to_llama_cpp_request(request);+ let completions = self.stream_completion(request, cx);+ async move {+ let mapper = LlamaCppEventMapper::new();+ Ok(mapper.map_stream(completions.await?).boxed())+ }+ .boxed()+ }+}++struct LlamaCppEventMapper {+ tool_calls_by_index: HashMap<usize, RawToolCall>,+}++impl LlamaCppEventMapper {+ fn new() -> Self {+ Self {+ tool_calls_by_index: HashMap::default(),+ }+ }++ pub fn map_stream(+ mut self,+ events: Pin<Box<dyn Send + Stream<Item = Result<llama_cpp::ResponseStreamEvent>>>>,+ ) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>+ {+ events.flat_map(move |event| {+ futures::stream::iter(match event {+ Ok(event) => self.map_event(event),+ Err(error) => vec![Err(LanguageModelCompletionError::from(error))],+ })+ })+ }++ pub fn map_event(+ &mut self,+ event: llama_cpp::ResponseStreamEvent,+ ) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>> {+ let mut events = Vec::new();++ if let Some(usage) = event.usage {+ events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: usage.prompt_tokens,+ output_tokens: usage.completion_tokens,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })));+ }++ if let Some(choice) = event.choices.into_iter().next() {+ if let Some(reasoning_content) = choice.delta.reasoning_content {+ events.push(Ok(LanguageModelCompletionEvent::Thinking {+ text: reasoning_content,+ signature: None,+ }));+ }++ if let Some(content) = choice.delta.content {+ if !content.is_empty() {+ events.push(Ok(LanguageModelCompletionEvent::Text(content)));+ }+ }++ if let Some(tool_calls) = choice.delta.tool_calls {+ for tool_call in tool_calls {+ let entry = self.tool_calls_by_index.entry(tool_call.index).or_default();++ if let Some(tool_id) = tool_call.id {+ entry.id = tool_id;+ }++ if let Some(function) = tool_call.function {+ if let Some(name) = function.name {+ // Only the first chunk carries the function name;+ // later chunks send an empty name with arguments.+ if !name.is_empty() {+ entry.name = name;+ }+ }++ if let Some(arguments) = function.arguments {+ entry.arguments.push_str(&arguments);+ }+ }+ }+ }++ if let Some(finish_reason) = choice.finish_reason.as_deref() {+ match finish_reason {+ "stop" => {+ events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));+ }+ "tool_calls" => {+ events.extend(self.tool_calls_by_index.drain().map(|(_, tool_call)| {+ match parse_tool_arguments(&tool_call.arguments) {+ Ok(input) => Ok(LanguageModelCompletionEvent::ToolUse(+ LanguageModelToolUse {+ id: tool_call.id.into(),+ name: tool_call.name.into(),+ is_input_complete: true,+ input,+ raw_input: tool_call.arguments,+ thought_signature: None,+ },+ )),+ Err(error) => {+ Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {+ id: tool_call.id.into(),+ tool_name: tool_call.name.into(),+ raw_input: tool_call.arguments.into(),+ json_parse_error: error.to_string(),+ })+ }+ }+ }));++ events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)));+ }+ "length" => {+ events.push(Ok(LanguageModelCompletionEvent::Stop(+ StopReason::MaxTokens,+ )));+ }+ unexpected => {+ log::warn!("Unexpected llama.cpp finish_reason: {unexpected:?}");+ events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));+ }+ }+ }+ }++ events+ }+}++#[derive(Default)]+struct RawToolCall {+ id: String,+ name: String,+ arguments: String,+}++fn add_message_content_part(+ new_part: llama_cpp::MessagePart,+ role: Role,+ messages: &mut Vec<llama_cpp::ChatMessage>,+ reasoning_content: Option<String>,+) {+ match (role, messages.last_mut()) {+ (Role::User, Some(llama_cpp::ChatMessage::User { content }))+ | (Role::System, Some(llama_cpp::ChatMessage::System { content })) => {+ content.push_part(new_part);+ }+ (+ Role::Assistant,+ Some(llama_cpp::ChatMessage::Assistant {+ content: Some(content),+ reasoning_content: message_reasoning_content,+ ..+ }),+ ) => {+ append_reasoning_content(message_reasoning_content, reasoning_content);+ content.push_part(new_part);+ }+ _ => {+ messages.push(match role {+ Role::User => llama_cpp::ChatMessage::User {+ content: llama_cpp::MessageContent::from(vec![new_part]),+ },+ Role::Assistant => llama_cpp::ChatMessage::Assistant {+ content: Some(llama_cpp::MessageContent::from(vec![new_part])),+ reasoning_content,+ tool_calls: Vec::new(),+ },+ Role::System => llama_cpp::ChatMessage::System {+ content: llama_cpp::MessageContent::from(vec![new_part]),+ },+ });+ }+ }+}++fn append_reasoning_content(target: &mut Option<String>, content: Option<String>) {+ let Some(content) = content else {+ return;+ };+ if content.is_empty() {+ return;+ }+ target.get_or_insert_default().push_str(&content);+}++fn merge_settings_into_models(+ models: &mut HashMap<String, llama_cpp::Model>,+ available_models: &[AvailableModel],+ context_window: Option<u64>,+) {+ for setting_model in available_models {+ if let Some(model) = models.get_mut(&setting_model.name) {+ if context_window.is_none() {+ model.max_tokens = setting_model.max_tokens;+ }+ if setting_model.display_name.is_some() {+ model.display_name = setting_model.display_name.clone();+ }+ if let Some(supports_tools) = setting_model.supports_tools {+ model.supports_tools = supports_tools;+ }+ if let Some(supports_images) = setting_model.supports_images {+ model.supports_images = supports_images;+ }+ if let Some(supports_thinking) = setting_model.supports_thinking {+ model.supports_thinking = supports_thinking;+ }+ } else {+ models.insert(+ setting_model.name.clone(),+ llama_cpp::Model {+ name: setting_model.name.clone(),+ display_name: setting_model.display_name.clone(),+ max_tokens: context_window.unwrap_or(setting_model.max_tokens),+ supports_tools: setting_model.supports_tools.unwrap_or(false),+ supports_images: setting_model.supports_images.unwrap_or(false),+ supports_thinking: setting_model.supports_thinking.unwrap_or(false),+ },+ );+ }+ }+}++struct ConfigurationView {+ api_key_editor: Entity<InputField>,+ api_url_editor: Entity<InputField>,+ context_window_editor: Entity<InputField>,+ state: Entity<State>,+}++impl ConfigurationView {+ pub fn new(state: Entity<State>, window: &mut Window, cx: &mut Context<Self>) -> Self {+ let api_key_editor = cx.new(|cx| InputField::new(window, cx, "sk-...").label("API key"));++ let api_url_editor = cx.new(|cx| {+ let input = InputField::new(window, cx, LLAMA_CPP_API_URL).label("API URL");+ input.set_text(&LlamaCppLanguageModelProvider::api_url(cx), window, cx);+ input+ });++ let context_window_editor = cx.new(|cx| {+ let input = InputField::new(window, cx, "8192").label("Context Window");+ if let Some(context_window) = LlamaCppLanguageModelProvider::settings(cx).context_window+ {+ input.set_text(&context_window.to_string(), window, cx);+ }+ input+ });++ cx.observe(&state, |_, _, cx| {+ cx.notify();+ })+ .detach();++ Self {+ api_key_editor,+ api_url_editor,+ context_window_editor,+ state,+ }+ }++ fn retry_connection(&mut self, window: &mut Window, cx: &mut Context<Self>) {+ let has_api_url = LlamaCppLanguageModelProvider::has_custom_url(cx);+ let has_api_key = self+ .state+ .read_with(cx, |state, _| state.api_key_state.has_key());+ if !has_api_url {+ self.save_api_url(cx);+ }+ if !has_api_key {+ self.save_api_key(&Default::default(), window, cx);+ }++ self.state.update(cx, |state, cx| {+ state.restart_fetch_models_task(cx);+ });+ }++ fn save_api_key(&mut self, _: &menu::Confirm, window: &mut Window, cx: &mut Context<Self>) {+ let api_key = self.api_key_editor.read(cx).text(cx).trim().to_string();+ if api_key.is_empty() {+ return;+ }++ // A URL change can cause the editor to be shown again.+ self.api_key_editor+ .update(cx, |input, cx| input.set_text("", window, cx));++ let state = self.state.clone();+ cx.spawn_in(window, async move |_, cx| {+ state+ .update(cx, |state, cx| state.set_api_key(Some(api_key), cx))+ .await+ })+ .detach_and_log_err(cx);+ }++ fn reset_api_key(&mut self, window: &mut Window, cx: &mut Context<Self>) {+ self.api_key_editor+ .update(cx, |input, cx| input.set_text("", window, cx));++ let state = self.state.clone();+ cx.spawn_in(window, async move |_, cx| {+ state+ .update(cx, |state, cx| state.set_api_key(None, cx))+ .await+ })+ .detach_and_log_err(cx);++ cx.notify();+ }++ fn save_api_url(&self, cx: &mut Context<Self>) {+ let api_url = self.api_url_editor.read(cx).text(cx).trim().to_string();+ let current_url = LlamaCppLanguageModelProvider::api_url(cx);+ if !api_url.is_empty() && &api_url != ¤t_url {+ let fs = <dyn Fs>::global(cx);+ update_settings_file(fs, cx, move |settings, _| {+ settings+ .language_models+ .get_or_insert_default()+ .llama_cpp+ .get_or_insert_default()+ .api_url = Some(api_url);+ });+ }+ }++ fn reset_api_url(&mut self, window: &mut Window, cx: &mut Context<Self>) {+ self.api_url_editor+ .update(cx, |input, cx| input.set_text("", window, cx));+ let fs = <dyn Fs>::global(cx);+ update_settings_file(fs, cx, |settings, _cx| {+ if let Some(settings) = settings+ .language_models+ .as_mut()+ .and_then(|models| models.llama_cpp.as_mut())+ {+ settings.api_url = Some(LLAMA_CPP_API_URL.into());+ }+ });+ cx.notify();+ }++ fn save_context_window(&mut self, cx: &mut Context<Self>) {+ let context_window_str = self+ .context_window_editor+ .read(cx)+ .text(cx)+ .trim()+ .to_string();+ let current_context_window = LlamaCppLanguageModelProvider::settings(cx).context_window;++ if let Ok(context_window) = context_window_str.parse::<u64>() {+ if Some(context_window) != current_context_window {+ let fs = <dyn Fs>::global(cx);+ update_settings_file(fs, cx, move |settings, _| {+ settings+ .language_models+ .get_or_insert_default()+ .llama_cpp+ .get_or_insert_default()+ .context_window = Some(context_window);+ });+ }+ } else if context_window_str.is_empty() && current_context_window.is_some() {+ let fs = <dyn Fs>::global(cx);+ update_settings_file(fs, cx, move |settings, _| {+ settings+ .language_models+ .get_or_insert_default()+ .llama_cpp+ .get_or_insert_default()+ .context_window = None;+ });+ }+ }++ fn reset_context_window(&mut self, window: &mut Window, cx: &mut Context<Self>) {+ self.context_window_editor+ .update(cx, |input, cx| input.set_text("", window, cx));+ let fs = <dyn Fs>::global(cx);+ update_settings_file(fs, cx, |settings, _cx| {+ if let Some(settings) = settings+ .language_models+ .as_mut()+ .and_then(|models| models.llama_cpp.as_mut())+ {+ settings.context_window = None;+ }+ });+ cx.notify();+ }++ fn render_instructions(cx: &App) -> Div {+ v_flex()+ .gap_2()+ .child(Label::new(+ "Run open models locally with llama.cpp's built-in server, or connect to a \+ remote llama.cpp server.",+ ))+ .child(Label::new("To use a local llama.cpp server:"))+ .child(+ List::new()+ .child(+ ListBulletItem::new("")+ .child(Label::new("Install llama.cpp from"))+ .child(ButtonLink::new("llama.app", LLAMA_CPP_DOWNLOAD_URL)),+ )+ .child(+ ListBulletItem::new("")+ .child(Label::new("Start the server in router mode:"))+ .child(Label::new("llama serve").inline_code(cx)),+ )+ .child(ListBulletItem::new(+ "Click 'Connect' below to start using llama.cpp in Zed",+ )),+ )+ .child(Label::new(+ "Alternatively, you can connect to a remote llama.cpp server by specifying its \+ URL and API key (set with --api-key, may not be required):",+ ))+ }++ fn render_api_key_editor(&self, cx: &Context<Self>) -> impl IntoElement {+ let state = self.state.read(cx);+ let env_var_set = state.api_key_state.is_from_env_var();+ let configured_card_label = if env_var_set {+ format!("API key set in {API_KEY_ENV_VAR_NAME} environment variable.")+ } else {+ "API key configured".to_string()+ };++ if !state.api_key_state.has_key() {+ v_flex()+ .on_action(cx.listener(Self::save_api_key))+ .child(self.api_key_editor.clone())+ .child(+ Label::new(format!(+ "You can also set the {API_KEY_ENV_VAR_NAME} environment variable and restart Zed."+ ))+ .size(LabelSize::Small)+ .color(Color::Muted),+ )+ .into_any_element()+ } else {+ ConfiguredApiCard::new(configured_card_label)+ .disabled(env_var_set)+ .on_click(cx.listener(|this, _, window, cx| this.reset_api_key(window, cx)))+ .when(env_var_set, |this| {+ this.tooltip_label(format!(+ "To reset your API key, unset the {API_KEY_ENV_VAR_NAME} environment variable."+ ))+ })+ .into_any_element()+ }+ }++ fn render_context_window_editor(&self, cx: &Context<Self>) -> Div {+ let settings = LlamaCppLanguageModelProvider::settings(cx);+ let custom_context_window_set = settings.context_window.is_some();++ if custom_context_window_set {+ h_flex()+ .p_3()+ .justify_between()+ .rounded_md()+ .border_1()+ .border_color(cx.theme().colors().border)+ .bg(cx.theme().colors().elevated_surface_background)+ .child(+ h_flex()+ .gap_2()+ .child(Icon::new(IconName::Check).color(Color::Success))+ .child(v_flex().gap_1().child(Label::new(format!(+ "Context Window: {}",+ settings.context_window.unwrap_or_default()+ )))),+ )+ .child(+ Button::new("reset-context-window", "Reset")+ .label_size(LabelSize::Small)+ .start_icon(Icon::new(IconName::Undo).size(IconSize::Small))+ .layer(ElevationIndex::ModalSurface)+ .on_click(+ cx.listener(|this, _, window, cx| {+ this.reset_context_window(window, cx)+ }),+ ),+ )+ } else {+ v_flex()+ .on_action(+ cx.listener(|this, _: &menu::Confirm, _window, cx| {+ this.save_context_window(cx)+ }),+ )+ .child(self.context_window_editor.clone())+ .child(+ Label::new("Default: discovered from the server")+ .size(LabelSize::Small)+ .color(Color::Muted),+ )+ }+ }++ fn render_api_url_editor(&self, cx: &Context<Self>) -> Div {+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);+ let custom_api_url_set = api_url != LLAMA_CPP_API_URL;++ if custom_api_url_set {+ h_flex()+ .p_3()+ .justify_between()+ .rounded_md()+ .border_1()+ .border_color(cx.theme().colors().border)+ .bg(cx.theme().colors().elevated_surface_background)+ .child(+ h_flex()+ .gap_2()+ .child(Icon::new(IconName::Check).color(Color::Success))+ .child(v_flex().gap_1().child(Label::new(api_url))),+ )+ .child(+ Button::new("reset-api-url", "Reset API URL")+ .label_size(LabelSize::Small)+ .start_icon(Icon::new(IconName::Undo).size(IconSize::Small))+ .layer(ElevationIndex::ModalSurface)+ .on_click(+ cx.listener(|this, _, window, cx| this.reset_api_url(window, cx)),+ ),+ )+ } else {+ v_flex()+ .on_action(cx.listener(|this, _: &menu::Confirm, _window, cx| {+ this.save_api_url(cx);+ cx.notify();+ }))+ .gap_2()+ .child(self.api_url_editor.clone())+ }+ }+}++impl Render for ConfigurationView {+ fn render(&mut self, _: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {+ let is_authenticated = self.state.read(cx).is_authenticated();++ v_flex()+ .gap_2()+ .child(Self::render_instructions(cx))+ .child(self.render_api_url_editor(cx))+ .child(self.render_context_window_editor(cx))+ .child(self.render_api_key_editor(cx))+ .child(+ h_flex()+ .w_full()+ .justify_between()+ .gap_2()+ .child(+ h_flex()+ .w_full()+ .gap_2()+ .map(|this| {+ if is_authenticated {+ this.child(+ Button::new("llama-cpp-webui", "Open WebUI")+ .style(ButtonStyle::Subtle)+ .end_icon(+ Icon::new(IconName::ArrowUpRight)+ .size(IconSize::XSmall)+ .color(Color::Muted),+ )+ .on_click(move |_, _, cx| {+ let url =+ LlamaCppLanguageModelProvider::api_url(cx);+ cx.open_url(&url);+ })+ .into_any_element(),+ )+ .child(+ Button::new("llama-cpp-site", "llama.cpp")+ .style(ButtonStyle::Subtle)+ .end_icon(+ Icon::new(IconName::ArrowUpRight)+ .size(IconSize::XSmall)+ .color(Color::Muted),+ )+ .on_click(move |_, _, cx| {+ cx.open_url(LLAMA_CPP_DOWNLOAD_URL)+ })+ .into_any_element(),+ )+ } else {+ this.child(+ Button::new("download_llama_cpp_button", "Get llama.cpp")+ .style(ButtonStyle::Subtle)+ .end_icon(+ Icon::new(IconName::ArrowUpRight)+ .size(IconSize::XSmall)+ .color(Color::Muted),+ )+ .on_click(move |_, _, cx| {+ cx.open_url(LLAMA_CPP_DOWNLOAD_URL)+ })+ .into_any_element(),+ )+ }+ })+ .child(+ Button::new("view-models", "Browse GGUF Models")+ .style(ButtonStyle::Subtle)+ .end_icon(+ Icon::new(IconName::ArrowUpRight)+ .size(IconSize::XSmall)+ .color(Color::Muted),+ )+ .on_click(move |_, _, cx| cx.open_url(LLAMA_CPP_MODELS_URL)),+ ),+ )+ .map(|this| {+ if is_authenticated {+ this.child(+ ButtonLike::new("connected")+ .disabled(true)+ .cursor_style(CursorStyle::Arrow)+ .child(+ h_flex()+ .gap_2()+ .child(Icon::new(IconName::Check).color(Color::Success))+ .child(Label::new("Connected"))+ .into_any_element(),+ )+ .child(+ IconButton::new("refresh-models", IconName::RotateCcw)+ .tooltip(Tooltip::text("Refresh Models"))+ .on_click(cx.listener(|this, _, window, cx| {+ this.state.update(cx, |state, _| {+ state.fetched_models.clear();+ });+ this.retry_connection(window, cx);+ })),+ ),+ )+ } else {+ this.child(+ Button::new("retry_llama_cpp_models", "Connect")+ .start_icon(+ Icon::new(IconName::PlayOutlined).size(IconSize::XSmall),+ )+ .on_click(cx.listener(move |this, _, window, cx| {+ this.retry_connection(window, cx)+ })),+ )+ }+ }),+ )+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use gpui::TestAppContext;+ use http_client::FakeHttpClient;+ use parking_lot::Mutex;+ use std::sync::atomic::{AtomicUsize, Ordering};++ struct FakeCredentialsProvider {+ api_key: Vec<u8>,+ }++ impl CredentialsProvider for FakeCredentialsProvider {+ fn read_credentials<'a>(+ &'a self,+ _url: &'a str,+ _cx: &'a AsyncApp,+ ) -> std::pin::Pin<+ Box<dyn std::future::Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>,+ > {+ let api_key = self.api_key.clone();+ Box::pin(async move { Ok(Some(("Bearer".to_string(), api_key))) })+ }++ fn write_credentials<'a>(+ &'a self,+ _url: &'a str,+ _username: &'a str,+ _password: &'a [u8],+ _cx: &'a AsyncApp,+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {+ Box::pin(async { Ok(()) })+ }++ fn delete_credentials<'a>(+ &'a self,+ _url: &'a str,+ _cx: &'a AsyncApp,+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {+ Box::pin(async { Ok(()) })+ }+ }++ fn entry(id: &str, n_ctx: Option<u64>, n_ctx_train: Option<u64>) -> ModelEntry {+ ModelEntry {+ id: id.to_string(),+ meta: Some(llama_cpp::ModelMeta { n_ctx, n_ctx_train }),+ architecture: None,+ status: None,+ }+ }++ #[test]+ fn display_name_strips_path_and_extension() {+ assert_eq!(+ display_name_for("../models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),+ "Qwen2.5-Coder-7B-Instruct-Q4_K_M"+ );+ assert_eq!(display_name_for("my-alias"), "my-alias");+ }++ #[test]+ fn telemetry_id_strips_local_model_paths() {+ assert_eq!(+ telemetry_id_for("/Users/alice/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),+ "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M"+ );+ assert_eq!(+ telemetry_id_for(r"C:\Users\alice\models\Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),+ "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M"+ );+ assert_eq!(telemetry_id_for("my-alias"), "llama_cpp/my-alias");+ }++ #[test]+ fn model_uses_props_then_meta_for_context() {+ let props = Props {+ default_generation_settings: Some(llama_cpp::GenerationSettings { n_ctx: Some(8192) }),+ modalities: Some(llama_cpp::Modalities { vision: true }),+ chat_template_caps: Some(llama_cpp::ChatTemplateCaps {+ supports_tool_calls: true,+ supports_preserve_reasoning: true,+ ..Default::default()+ }),+ };+ // /props wins when present.+ let model = model_from_entry(&entry("m", Some(4096), Some(131072)), Some(&props));+ assert_eq!(model.max_tokens, 8192);+ assert!(model.supports_tools);+ assert!(model.supports_images);+ assert!(model.supports_thinking);++ // Unprobed: falls back to the listing's runtime context, then trained+ // context. Tools are assumed supported until the model loads.+ let model = model_from_entry(&entry("m", Some(4096), Some(131072)), None);+ assert_eq!(model.max_tokens, 4096);+ assert!(model.supports_tools);+ assert!(!model.supports_thinking);++ let model = model_from_entry(&entry("m", None, Some(131072)), None);+ assert_eq!(model.max_tokens, 131072);++ // Nothing reported -> the optimistic unloaded-context assumption.+ let model = model_from_entry(&entry("m", None, None), None);+ assert_eq!(model.max_tokens, ASSUMED_UNLOADED_CONTEXT);+ assert!(model.supports_tools);+ }++ #[test]+ fn router_entry_detects_vision_from_modalities() {+ let router_entry = ModelEntry {+ id: "vlm".to_string(),+ meta: None,+ architecture: Some(llama_cpp::Architecture {+ input_modalities: vec!["text".to_string(), "image".to_string()],+ }),+ status: Some(llama_cpp::ModelStatus {+ value: "unloaded".to_string(),+ }),+ };+ let model = model_from_entry(&router_entry, None);+ assert!(model.supports_images);+ // Unprobed router models optimistically advertise tools until loaded.+ assert!(model.supports_tools);+ }++ #[test]+ fn settings_override_capabilities_and_context() {+ let mut models: HashMap<String, llama_cpp::Model> = HashMap::default();+ models.insert(+ "qwen".to_string(),+ llama_cpp::Model::new("qwen", Some("qwen"), Some(8192), false, false, false),+ );++ let available = vec![AvailableModel {+ name: "qwen".to_string(),+ display_name: Some("Qwen Coder".to_string()),+ max_tokens: 16384,+ supports_tools: Some(true),+ supports_images: None,+ supports_thinking: Some(true),+ }];++ merge_settings_into_models(&mut models, &available, None);++ let model = models.get("qwen").unwrap();+ assert_eq!(model.display_name.as_deref(), Some("Qwen Coder"));+ assert_eq!(model.max_tokens, 16384);+ assert!(model.supports_tools);+ assert!(model.supports_thinking);+ // Unspecified capability keeps the discovered value.+ assert!(!model.supports_images);+ }++ #[test]+ fn capability_cells_update_when_a_model_loads() {+ let cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default()));+ let settings = LlamaCppSettings {+ auto_discover: true,+ ..Default::default()+ };++ // Cold: the optimistic unloaded-context assumption.+ let cold = vec![llama_cpp::Model::new(+ "m",+ Some("m"),+ Some(ASSUMED_UNLOADED_CONTEXT),+ true,+ false,+ false,+ )];+ sync_capability_cells(&cells, &compute_effective_models(&cold, &settings));+ assert_eq!(+ cells.read().unwrap().get("m").unwrap().max_tokens,+ ASSUMED_UNLOADED_CONTEXT+ );++ // The model loads and reports its real context. The shared map must+ // reflect the new value so a model reading it by name (an open+ // conversation) is no longer stuck on the cold-start assumption.+ let loaded = vec![llama_cpp::Model::new(+ "m",+ Some("m"),+ Some(262_144),+ true,+ false,+ true,+ )];+ sync_capability_cells(&cells, &compute_effective_models(&loaded, &settings));+ assert_eq!(cells.read().unwrap().get("m").unwrap().max_tokens, 262_144);+ assert!(cells.read().unwrap().get("m").unwrap().supports_thinking);+ }++ #[test]+ fn request_preserves_assistant_thinking_when_supported() {+ let request = build_llama_cpp_request(+ "test-model",+ false,+ LiveCapabilities {+ max_tokens: 8192,+ supports_tools: false,+ supports_thinking: true,+ },+ LanguageModelRequest {+ messages: vec![language_model::LanguageModelRequestMessage {+ role: Role::Assistant,+ content: vec![+ MessageContent::Thinking {+ text: "reasoning".to_string(),+ signature: None,+ },+ MessageContent::Text("answer".to_string()),+ ],+ cache: false,+ reasoning_details: None,+ }],+ ..Default::default()+ },+ );++ assert_eq!(request.messages.len(), 1);+ match &request.messages[0] {+ llama_cpp::ChatMessage::Assistant {+ content: Some(llama_cpp::MessageContent::Plain(content)),+ reasoning_content: Some(reasoning_content),+ tool_calls,+ } => {+ assert_eq!(content, "answer");+ assert_eq!(reasoning_content, "reasoning");+ assert!(tool_calls.is_empty());+ }+ message => panic!("unexpected message: {message:?}"),+ }+ }++ #[test]+ fn request_skips_assistant_thinking_when_unsupported() {+ let request = build_llama_cpp_request(+ "test-model",+ false,+ LiveCapabilities {+ max_tokens: 8192,+ supports_tools: false,+ supports_thinking: false,+ },+ LanguageModelRequest {+ messages: vec![language_model::LanguageModelRequestMessage {+ role: Role::Assistant,+ content: vec![+ MessageContent::Thinking {+ text: "reasoning".to_string(),+ signature: None,+ },+ MessageContent::RedactedThinking("encrypted".to_string()),+ MessageContent::Text("answer".to_string()),+ ],+ cache: false,+ reasoning_details: None,+ }],+ ..Default::default()+ },+ );++ assert_eq!(request.messages.len(), 1);+ match &request.messages[0] {+ llama_cpp::ChatMessage::Assistant {+ content: Some(llama_cpp::MessageContent::Plain(content)),+ reasoning_content,+ tool_calls,+ } => {+ assert_eq!(content, "answer");+ assert_eq!(reasoning_content, &None);+ assert!(tool_calls.is_empty());+ }+ message => panic!("unexpected message: {message:?}"),+ }+ }++ #[test]+ fn request_preserves_thinking_for_assistant_tool_calls_when_supported() {+ let request = build_llama_cpp_request(+ "test-model",+ false,+ LiveCapabilities {+ max_tokens: 8192,+ supports_tools: true,+ supports_thinking: true,+ },+ LanguageModelRequest {+ messages: vec![language_model::LanguageModelRequestMessage {+ role: Role::Assistant,+ content: vec![+ MessageContent::Thinking {+ text: "reasoning".to_string(),+ signature: None,+ },+ MessageContent::ToolUse(LanguageModelToolUse {+ id: "call_1".into(),+ name: "weather".into(),+ raw_input: r#"{"city":"Oslo"}"#.to_string(),+ input: serde_json::json!({ "city": "Oslo" }),+ is_input_complete: true,+ thought_signature: None,+ }),+ ],+ cache: false,+ reasoning_details: None,+ }],+ ..Default::default()+ },+ );++ assert_eq!(request.messages.len(), 1);+ match &request.messages[0] {+ llama_cpp::ChatMessage::Assistant {+ content: None,+ reasoning_content: Some(reasoning_content),+ tool_calls,+ } => {+ assert_eq!(reasoning_content, "reasoning");+ assert_eq!(tool_calls.len(), 1);+ }+ message => panic!("unexpected message: {message:?}"),+ }+ }++ #[test]+ fn usage_event_precedes_stop_event() {+ let mut mapper = LlamaCppEventMapper::new();+ let events = mapper.map_event(llama_cpp::ResponseStreamEvent {+ model: "test-model".to_string(),+ object: "chat.completion.chunk".to_string(),+ choices: vec![llama_cpp::ChoiceDelta {+ index: 0,+ delta: llama_cpp::ResponseMessageDelta {+ content: None,+ reasoning_content: None,+ tool_calls: None,+ },+ finish_reason: Some("stop".to_string()),+ }],+ usage: Some(llama_cpp::Usage {+ prompt_tokens: 11,+ completion_tokens: 7,+ total_tokens: 18,+ }),+ });++ assert!(matches!(+ events.as_slice(),+ [+ Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: 11,+ output_tokens: 7,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })),+ Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)),+ ]+ ));+ }++ #[test]+ fn usage_event_precedes_tool_use_stop_event() {+ let mut mapper = LlamaCppEventMapper::new();+ let events = mapper.map_event(llama_cpp::ResponseStreamEvent {+ model: "test-model".to_string(),+ object: "chat.completion.chunk".to_string(),+ choices: vec![llama_cpp::ChoiceDelta {+ index: 0,+ delta: llama_cpp::ResponseMessageDelta {+ content: None,+ reasoning_content: None,+ tool_calls: Some(vec![llama_cpp::ToolCallChunk {+ index: 0,+ id: Some("tool-call-id".to_string()),+ function: Some(llama_cpp::FunctionChunk {+ name: Some("test_tool".to_string()),+ arguments: Some(r#"{"value":1}"#.to_string()),+ }),+ }]),+ },+ finish_reason: Some("tool_calls".to_string()),+ }],+ usage: Some(llama_cpp::Usage {+ prompt_tokens: 13,+ completion_tokens: 5,+ total_tokens: 18,+ }),+ });++ assert!(matches!(+ events.as_slice(),+ [+ Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: 13,+ output_tokens: 5,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })),+ Ok(LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {+ id,+ name,+ ..+ })),+ Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)),+ ] if id.to_string() == "tool-call-id" && name.as_ref() == "test_tool"+ ));+ }++ #[gpui::test]+ async fn authenticate_fetches_models_after_loading_api_key(cx: &mut TestAppContext) {+ cx.update(|cx| {+ let settings_store = SettingsStore::test(cx);+ cx.set_global(settings_store);+ });++ let model_request_authorizations = Arc::new(Mutex::new(Vec::new()));+ let model_request_count = Arc::new(AtomicUsize::new(0));+ let http_client = FakeHttpClient::create({+ let model_request_authorizations = model_request_authorizations.clone();+ let model_request_count = model_request_count.clone();+ move |request| {+ let model_request_authorizations = model_request_authorizations.clone();+ let model_request_count = model_request_count.clone();+ async move {+ let path = request.uri().path();+ let authorization = request+ .headers()+ .get("Authorization")+ .and_then(|value| value.to_str().ok())+ .map(str::to_string);++ if path == "/v1/models" {+ model_request_authorizations.lock().push(authorization);+ let request_index = model_request_count.fetch_add(1, Ordering::SeqCst);+ if request_index == 0 {+ return Ok(http_client::Response::builder()+ .status(503)+ .body(http_client::AsyncBody::from("not ready"))?);+ }++ return Ok(http_client::Response::builder().status(200).body(+ http_client::AsyncBody::from(+ r#"{"data":[{"id":"test-model","meta":{"n_ctx":4096}}]}"#,+ ),+ )?);+ }++ if path == "/props" {+ return Ok(http_client::Response::builder()+ .status(200)+ .body(http_client::AsyncBody::from("{}"))?);+ }++ Ok(http_client::Response::builder()+ .status(404)+ .body(http_client::AsyncBody::default())?)+ }+ }+ });+ let credentials_provider = Arc::new(FakeCredentialsProvider {+ api_key: b"loaded-key".to_vec(),+ });+ let provider = cx+ .update(|cx| LlamaCppLanguageModelProvider::new(http_client, credentials_provider, cx));++ cx.run_until_parked();++ let result = cx.update(|cx| provider.authenticate(cx)).await;+ assert!(+ result.is_ok(),+ "authenticate should discover models after loading credentials"+ );+ assert_eq!(+ &*model_request_authorizations.lock(),+ &[None, Some("Bearer loaded-key".to_string())]+ );+ }+}diff --git a/crates/language_models/src/settings.rs b/crates/language_models/src/settings.rsindex d2fc9f59a26..1ce7caa3a67 100644--- a/crates/language_models/src/settings.rs+++ b/crates/language_models/src/settings.rs@@ -6,10 +6,11 @@ use settings::RegisterSetting;use crate::provider::{anthropic, anthropic::AnthropicSettings, anthropic_compatible::AnthropicCompatibleSettings,bedrock, bedrock::AmazonBedrockSettings, cloud::ZedDotDevSettings, deepseek::DeepSeekSettings,- google::GoogleSettings, lmstudio::LmStudioSettings, mistral, mistral::MistralSettings,- ollama::OllamaSettings, open_ai::OpenAiSettings, open_ai_compatible::OpenAiCompatibleSettings,- open_router, open_router::OpenRouterSettings, opencode, opencode::OpenCodeSettings,- resolve_custom_headers, vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings,+ google::GoogleSettings, llama_cpp::LlamaCppSettings, lmstudio::LmStudioSettings, mistral,+ mistral::MistralSettings, ollama::OllamaSettings, open_ai::OpenAiSettings,+ open_ai_compatible::OpenAiCompatibleSettings, open_router, open_router::OpenRouterSettings,+ opencode, opencode::OpenCodeSettings, resolve_custom_headers,+ vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings,};#[derive(Debug, RegisterSetting)]@@ -19,6 +20,7 @@ pub struct AllLanguageModelSettings {pub bedrock: AmazonBedrockSettings,pub deepseek: DeepSeekSettings,pub google: GoogleSettings,+ pub llama_cpp: LlamaCppSettings,pub lmstudio: LmStudioSettings,pub mistral: MistralSettings,pub ollama: OllamaSettings,@@ -52,6 +54,7 @@ impl settings::Settings for AllLanguageModelSettings {let bedrock = language_models.bedrock.unwrap();let deepseek = language_models.deepseek.unwrap();let google = language_models.google.unwrap();+ let llama_cpp = language_models.llama_cpp.unwrap();let lmstudio = language_models.lmstudio.unwrap();let mistral = language_models.mistral.unwrap();let ollama = language_models.ollama.unwrap();@@ -116,6 +119,13 @@ impl settings::Settings for AllLanguageModelSettings {available_models: google.available_models.unwrap_or_default(),custom_headers: custom_headers_from("Google AI", google.custom_headers, &[]),},+ llama_cpp: LlamaCppSettings {+ api_url: llama_cpp.api_url.unwrap(),+ auto_discover: llama_cpp.auto_discover.unwrap_or(true),+ available_models: llama_cpp.available_models.unwrap_or_default(),+ context_window: llama_cpp.context_window,+ custom_headers: custom_headers_from("llama.cpp", llama_cpp.custom_headers, &[]),+ },lmstudio: LmStudioSettings {api_url: lmstudio.api_url.unwrap(),available_models: lmstudio.available_models.unwrap_or_default(),diff --git a/crates/llama_cpp/Cargo.toml b/crates/llama_cpp/Cargo.tomlnew file mode 100644index 00000000000..c77c2dd8171--- /dev/null+++ b/crates/llama_cpp/Cargo.toml@@ -0,0 +1,25 @@+[package]+name = "llama_cpp"+version = "0.1.0"+edition.workspace = true+publish.workspace = true+license = "GPL-3.0-or-later"++[lints]+workspace = true++[lib]+path = "src/llama_cpp.rs"++[features]+default = []+schemars = ["dep:schemars"]++[dependencies]+anyhow.workspace = true+futures.workspace = true+http_client.workspace = true+schemars = { workspace = true, optional = true }+serde.workspace = true+serde_json.workspace = true+url.workspace = truediff --git a/crates/llama_cpp/LICENSE-GPL b/crates/llama_cpp/LICENSE-GPLnew file mode 120000index 00000000000..89e542f750c--- /dev/null+++ b/crates/llama_cpp/LICENSE-GPL@@ -0,0 +1 @@+../../LICENSE-GPL\ No newline at end of filediff --git a/crates/llama_cpp/src/llama_cpp.rs b/crates/llama_cpp/src/llama_cpp.rsnew file mode 100644index 00000000000..5d5238c4b1b--- /dev/null+++ b/crates/llama_cpp/src/llama_cpp.rs@@ -0,0 +1,911 @@+use anyhow::{Context as _, Result, anyhow};+use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};+use http_client::{+ AsyncBody, CustomHeaders, HttpClient, HttpRequestExt, Method, Request as HttpRequest,+ RequestBuilderExt, http,+};+use serde::{Deserialize, Serialize};+use serde_json::Value;+use url::Url;++pub const LLAMA_CPP_API_URL: &str = "http://localhost:8080";++const DEFAULT_CONTEXT_LENGTH: u64 = 4096;++/// A model exposed to the rest of Zed, after merging API discovery with+/// user-configured overrides.+#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]+pub struct Model {+ pub name: String,+ pub display_name: Option<String>,+ pub max_tokens: u64,+ pub supports_tools: bool,+ pub supports_images: bool,+ pub supports_thinking: bool,+}++impl Model {+ pub fn new(+ name: &str,+ display_name: Option<&str>,+ max_tokens: Option<u64>,+ supports_tools: bool,+ supports_images: bool,+ supports_thinking: bool,+ ) -> Self {+ Self {+ name: name.to_owned(),+ display_name: display_name.map(ToString::to_string),+ max_tokens: max_tokens.unwrap_or(DEFAULT_CONTEXT_LENGTH),+ supports_tools,+ supports_images,+ supports_thinking,+ }+ }++ pub fn display_name(&self) -> &str {+ self.display_name.as_deref().unwrap_or(&self.name)+ }+}++#[derive(Debug, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum ToolChoice {+ Auto,+ Required,+ None,+}++#[derive(Clone, Deserialize, Serialize, Debug)]+#[serde(tag = "type", rename_all = "snake_case")]+pub enum ToolDefinition {+ Function { function: FunctionDefinition },+}++#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct FunctionDefinition {+ pub name: String,+ pub description: Option<String>,+ pub parameters: Option<Value>,+}++#[derive(Serialize, Deserialize, Debug)]+#[serde(tag = "role", rename_all = "lowercase")]+pub enum ChatMessage {+ Assistant {+ #[serde(default)]+ content: Option<MessageContent>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ reasoning_content: Option<String>,+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ tool_calls: Vec<ToolCall>,+ },+ User {+ content: MessageContent,+ },+ System {+ content: MessageContent,+ },+ Tool {+ content: MessageContent,+ tool_call_id: String,+ },+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+#[serde(untagged)]+pub enum MessageContent {+ Plain(String),+ Multipart(Vec<MessagePart>),+}++impl MessageContent {+ pub fn push_part(&mut self, part: MessagePart) {+ match self {+ MessageContent::Plain(text) => {+ *self =+ MessageContent::Multipart(vec![MessagePart::Text { text: text.clone() }, part]);+ }+ MessageContent::Multipart(parts) if parts.is_empty() => match part {+ MessagePart::Text { text } => *self = MessageContent::Plain(text),+ MessagePart::Image { .. } => *self = MessageContent::Multipart(vec![part]),+ },+ MessageContent::Multipart(parts) => parts.push(part),+ }+ }+}++impl From<Vec<MessagePart>> for MessageContent {+ fn from(mut parts: Vec<MessagePart>) -> Self {+ if let [MessagePart::Text { text }] = parts.as_mut_slice() {+ MessageContent::Plain(std::mem::take(text))+ } else {+ MessageContent::Multipart(parts)+ }+ }+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+#[serde(tag = "type", rename_all = "snake_case")]+pub enum MessagePart {+ Text {+ text: String,+ },+ #[serde(rename = "image_url")]+ Image {+ image_url: ImageUrl,+ },+}++#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]+pub struct ImageUrl {+ pub url: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub detail: Option<String>,+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+pub struct ToolCall {+ pub id: String,+ #[serde(flatten)]+ pub content: ToolCallContent,+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+#[serde(tag = "type", rename_all = "lowercase")]+pub enum ToolCallContent {+ Function { function: FunctionContent },+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+pub struct FunctionContent {+ pub name: String,+ pub arguments: String,+}++#[derive(Serialize, Debug)]+pub struct ChatCompletionRequest {+ pub model: String,+ pub messages: Vec<ChatMessage>,+ pub stream: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub max_tokens: Option<i32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub stop: Option<Vec<String>>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub temperature: Option<f32>,+ #[serde(skip_serializing_if = "Vec::is_empty")]+ pub tools: Vec<ToolDefinition>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub tool_choice: Option<ToolChoice>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub stream_options: Option<StreamOptions>,+}++/// Asks the server to include a final `usage` chunk in the stream.+#[derive(Serialize, Debug)]+pub struct StreamOptions {+ pub include_usage: bool,+}++#[derive(Serialize, Deserialize, Debug)]+pub struct Usage {+ pub prompt_tokens: u64,+ pub completion_tokens: u64,+ pub total_tokens: u64,+}++#[derive(Serialize, Deserialize, Debug)]+pub struct LlamaCppError {+ pub message: String,+}++#[derive(Serialize, Deserialize, Debug)]+#[serde(untagged)]+pub enum ResponseStreamResult {+ Ok(ResponseStreamEvent),+ Err { error: LlamaCppError },+}++#[derive(Serialize, Deserialize, Debug)]+pub struct ResponseStreamEvent {+ pub model: String,+ pub object: String,+ pub choices: Vec<ChoiceDelta>,+ pub usage: Option<Usage>,+}++#[derive(Serialize, Deserialize, Debug)]+pub struct ChoiceDelta {+ pub index: u32,+ pub delta: ResponseMessageDelta,+ pub finish_reason: Option<String>,+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+pub struct ResponseMessageDelta {+ pub content: Option<String>,+ /// `llama-server` emits reasoning as a dedicated `reasoning_content` field+ /// when started with a reasoning format (e.g. `--reasoning-format deepseek`).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub reasoning_content: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub tool_calls: Option<Vec<ToolCallChunk>>,+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+pub struct ToolCallChunk {+ pub index: usize,+ pub id: Option<String>,+ pub function: Option<FunctionChunk>,+}++#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]+pub struct FunctionChunk {+ pub name: Option<String>,+ pub arguments: Option<String>,+}++/// Response of `GET /v1/models`.+///+/// In single-model mode `data` has exactly one entry describing the loaded+/// model; in router mode it lists every model the server knows about.+#[derive(Deserialize, Debug)]+pub struct ListModelsResponse {+ #[serde(default)]+ pub data: Vec<ModelEntry>,+}++#[derive(Clone, Debug, Deserialize, PartialEq)]+pub struct ModelEntry {+ pub id: String,+ /// Present in single-model mode; carries context information.+ #[serde(default)]+ pub meta: Option<ModelMeta>,+ /// Present in router mode; reports the modalities the model accepts.+ #[serde(default)]+ pub architecture: Option<Architecture>,+ /// Present in router mode; reports whether the model is currently loaded.+ #[serde(default)]+ pub status: Option<ModelStatus>,+}++impl ModelEntry {+ /// Whether this entry came from a server running in router mode.+ pub fn is_router_entry(&self) -> bool {+ self.status.is_some()+ }++ /// Whether the model is loaded and can be probed for capabilities without+ /// triggering a (potentially expensive) load.+ pub fn is_loaded(&self) -> bool {+ self.status+ .as_ref()+ .is_some_and(|status| status.value == "loaded")+ }++ /// Whether the model is currently loading, so a progress label applies.+ pub fn is_loading(&self) -> bool {+ self.status+ .as_ref()+ .is_some_and(|status| status.value == "loading")+ }++ pub fn supports_images_hint(&self) -> bool {+ self.architecture+ .as_ref()+ .is_some_and(|architecture| architecture.input_modalities.iter().any(|m| m == "image"))+ }+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct ModelMeta {+ /// The runtime per-slot context size.+ #[serde(default)]+ pub n_ctx: Option<u64>,+ /// The context size the model was trained with.+ #[serde(default)]+ pub n_ctx_train: Option<u64>,+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct Architecture {+ #[serde(default)]+ pub input_modalities: Vec<String>,+}++#[derive(Clone, Debug, Deserialize, PartialEq)]+pub struct ModelStatus {+ /// One of `loaded`, `loading`, `unloaded`, `downloading`, `downloaded`, `sleeping`.+ pub value: String,+}++/// Response of `GET /props`, describing the loaded model.+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct Props {+ #[serde(default)]+ pub default_generation_settings: Option<GenerationSettings>,+ #[serde(default)]+ pub modalities: Option<Modalities>,+ #[serde(default)]+ pub chat_template_caps: Option<ChatTemplateCaps>,+}++impl Props {+ /// The runtime context length the loaded model was configured with.+ pub fn context_length(&self) -> Option<u64> {+ self.default_generation_settings+ .as_ref()+ .and_then(|settings| settings.n_ctx)+ .filter(|n_ctx| *n_ctx > 0)+ }++ pub fn supports_images(&self) -> bool {+ self.modalities+ .as_ref()+ .is_some_and(|modalities| modalities.vision)+ }++ pub fn supports_tools(&self) -> bool {+ self.chat_template_caps+ .as_ref()+ .is_some_and(|caps| caps.supports_tool_calls || caps.supports_tools)+ }++ pub fn supports_thinking(&self) -> bool {+ self.chat_template_caps+ .as_ref()+ .is_some_and(|caps| caps.supports_preserve_reasoning)+ }+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct GenerationSettings {+ #[serde(default)]+ pub n_ctx: Option<u64>,+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct Modalities {+ #[serde(default)]+ pub vision: bool,+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct ChatTemplateCaps {+ /// llama.cpp's `/props` reports both of these keys, so we read them into+ /// separate fields (aliasing one to the other would be a serde duplicate-field+ /// error) and treat either being true as tool support. `supports_tools` is the+ /// older name; some builds report only one of the two.+ #[serde(default)]+ pub supports_tool_calls: bool,+ #[serde(default)]+ pub supports_tools: bool,+ #[serde(default)]+ pub supports_preserve_reasoning: bool,+}++/// An event from the router's `/models/sse` feed, which the provider subscribes+/// to so model capabilities stay current as models load and unload. `model` is+/// `*` for events that aren't about a single model (e.g. the list reloading).+#[derive(Clone, Debug, Deserialize, PartialEq)]+pub struct ModelEvent {+ #[serde(default)]+ pub model: String,+ pub event: String,+ #[serde(default)]+ pub data: Option<ModelEventData>,+}++#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct ModelEventData {+ #[serde(default)]+ pub status: Option<String>,+ /// Present on an `unloaded` status; non-zero means the model failed to load.+ #[serde(default)]+ pub exit_code: Option<i32>,+ /// Present on a `loading` status, reporting per-stage load progress.+ #[serde(default)]+ pub progress: Option<LoadProgress>,+}++/// Per-stage load progress carried by a `loading` event. A model loads its+/// stages in order (the text model, plus an optional draft and/or multimodal+/// projector), each reporting a `0.0..=1.0` fraction.+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+pub struct LoadProgress {+ #[serde(default)]+ pub stages: Vec<String>,+ #[serde(default)]+ pub current: String,+ #[serde(default)]+ pub value: f32,+}++impl LoadProgress {+ /// A human label for the stage currently loading, matching the llama.cpp+ /// WebUI's labels. We report the current stage's own `value` rather than a+ /// blended overall percentage (as the WebUI does), so each stage runs+ /// `0→100%` and the label says which stage it is.+ pub fn stage_label(&self) -> &'static str {+ match self.current.as_str() {+ "text_model" => "Loading weights",+ "spec_model" => "Loading draft",+ "mmproj_model" => "Loading projector",+ _ => "Loading",+ }+ }++ /// The full load-status label shown in the model selector, e.g.+ /// `"Loading weights 42%"`: the current stage plus its own progress rounded+ /// to a percentage.+ pub fn progress_label(&self) -> String {+ format!(+ "{} {}%",+ self.stage_label(),+ (self.value * 100.0).round() as u32+ )+ }+}++impl ModelEvent {+ /// Whether this event means the set of models or their loaded state changed,+ /// so the provider should re-run discovery. Intermediate `loading` progress+ /// ticks return `false` — only terminal load/unload and list changes matter.+ pub fn changes_model_state(&self) -> bool {+ match self.event.as_str() {+ "models_reload" | "model_remove" => true,+ _ => matches!(+ self.data.as_ref().and_then(|data| data.status.as_deref()),+ Some("loaded" | "unloaded")+ ),+ }+ }++ /// The non-zero exit code of a failed load, if this event reports one.+ pub fn load_failure(&self) -> Option<i32> {+ let data = self.data.as_ref()?;+ if data.status.as_deref() == Some("unloaded") {+ data.exit_code.filter(|code| *code != 0)+ } else {+ None+ }+ }++ /// This event's load progress if it is a loading event carrying usable stage+ /// data, else `None`.+ pub fn load_progress(&self) -> Option<&LoadProgress> {+ let data = self.data.as_ref()?;+ if data.status.as_deref() != Some("loading") {+ return None;+ }+ let progress = data.progress.as_ref()?;+ // The server also emits bare stage-transition markers (e.g.+ // `{"stage": "mmproj_model"}`) with no `stages`/`current`/`value`. Skip+ // them so the indicator holds its last value rather than dropping to 0%.+ if progress.stages.is_empty() || progress.current.is_empty() {+ return None;+ }+ Some(progress)+ }+}++pub async fn stream_chat_completion(+ client: &dyn HttpClient,+ api_url: &str,+ api_key: Option<&str>,+ request: ChatCompletionRequest,+ extra_headers: &CustomHeaders,+) -> Result<BoxStream<'static, Result<ResponseStreamEvent>>> {+ let uri = format!("{api_url}/v1/chat/completions");+ let request_builder = http::Request::builder()+ .method(Method::POST)+ .uri(uri)+ .header("Content-Type", "application/json")+ .when_some(api_key, |builder, api_key| {+ builder.header("Authorization", format!("Bearer {api_key}"))+ });++ let request = request_builder+ .extra_headers(extra_headers)+ .body(AsyncBody::from(serde_json::to_string(&request)?))?;+ let mut response = client.send(request).await?;+ if response.status().is_success() {+ let reader = BufReader::new(response.into_body());+ Ok(reader+ .lines()+ .filter_map(|line| async move {+ match line {+ Ok(line) => {+ let line = line.strip_prefix("data: ")?;+ if line == "[DONE]" {+ None+ } else {+ match serde_json::from_str(line) {+ Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)),+ Ok(ResponseStreamResult::Err { error }) => {+ Some(Err(anyhow!(error.message)))+ }+ Err(error) => Some(Err(anyhow!(error))),+ }+ }+ }+ Err(error) => Some(Err(anyhow!(error))),+ }+ })+ .boxed())+ } else {+ let mut body = String::new();+ response.body_mut().read_to_string(&mut body).await?;+ anyhow::bail!(+ "Failed to connect to llama.cpp API: {} {}",+ response.status(),+ body,+ );+ }+}++/// Lists the models the server is serving via `GET /v1/models`.+pub async fn get_models(+ client: &dyn HttpClient,+ api_url: &str,+ api_key: Option<&str>,+ extra_headers: &CustomHeaders,+) -> Result<Vec<ModelEntry>> {+ let uri = format!("{api_url}/v1/models");+ let request = HttpRequest::builder()+ .method(Method::GET)+ .uri(uri)+ .header("Accept", "application/json")+ .when_some(api_key, |builder, api_key| {+ builder.header("Authorization", format!("Bearer {api_key}"))+ })+ .extra_headers(extra_headers)+ .body(AsyncBody::default())?;++ let mut response = client.send(request).await?;++ let mut body = String::new();+ response.body_mut().read_to_string(&mut body).await?;++ anyhow::ensure!(+ response.status().is_success(),+ "Failed to connect to llama.cpp API: {} {}",+ response.status(),+ body,+ );+ let response: ListModelsResponse =+ serde_json::from_str(&body).context("Unable to parse llama.cpp models response")?;+ Ok(response.data)+}++/// Fetches `GET /props` to discover the loaded model's capabilities.+///+/// In router mode `model` selects which model instance to query; passing+/// `None` queries the single loaded model in single-model mode.+pub async fn get_props(+ client: &dyn HttpClient,+ api_url: &str,+ api_key: Option<&str>,+ model: Option<&str>,+ extra_headers: &CustomHeaders,+) -> Result<Props> {+ // Router-mode `/props` takes a `model` query parameter selecting which+ // instance to describe. Model ids contain `/` and `:` (e.g.+ // `unsloth/Qwen3.5-2B-GGUF:Q4_1`), so the value is URL-encoded.+ let uri = match model {+ Some(model) => Url::parse_with_params(&format!("{api_url}/props"), [("model", model)])+ .context("invalid llama.cpp API URL")?+ .to_string(),+ None => format!("{api_url}/props"),+ };+ let request = HttpRequest::builder()+ .method(Method::GET)+ .uri(uri)+ .header("Accept", "application/json")+ .when_some(api_key, |builder, api_key| {+ builder.header("Authorization", format!("Bearer {api_key}"))+ })+ .extra_headers(extra_headers)+ .body(AsyncBody::default())?;++ let mut response = client.send(request).await?;++ let mut body = String::new();+ response.body_mut().read_to_string(&mut body).await?;++ anyhow::ensure!(+ response.status().is_success(),+ "Failed to connect to llama.cpp API: {} {}",+ response.status(),+ body,+ );+ let props: Props =+ serde_json::from_str(&body).context("Unable to parse llama.cpp props response")?;+ Ok(props)+}++/// Opens the router's `GET /models/sse` event stream. Each item is one parsed+/// event; the stream ends when the connection closes. Only available on builds+/// that expose `/models/sse` (router mode).+pub async fn stream_model_events(+ client: &dyn HttpClient,+ api_url: &str,+ api_key: Option<&str>,+ extra_headers: &CustomHeaders,+) -> Result<BoxStream<'static, Result<ModelEvent>>> {+ let uri = format!("{api_url}/models/sse");+ let request = HttpRequest::builder()+ .method(Method::GET)+ .uri(uri)+ .header("Accept", "text/event-stream")+ .when_some(api_key, |builder, api_key| {+ builder.header("Authorization", format!("Bearer {api_key}"))+ })+ .extra_headers(extra_headers)+ .body(AsyncBody::default())?;++ let mut response = client.send(request).await?;+ if !response.status().is_success() {+ let mut body = String::new();+ response.body_mut().read_to_string(&mut body).await?;+ anyhow::bail!(+ "Failed to open llama.cpp model event stream: {} {}",+ response.status(),+ body,+ );+ }++ let reader = BufReader::new(response.into_body());+ Ok(reader+ .lines()+ .filter_map(|line| async move {+ // Each event is a single `data:` line carrying the JSON envelope;+ // other SSE lines (comments, blank separators) are ignored.+ let line = line.ok()?;+ let payload = line.strip_prefix("data:")?.trim_start();+ Some(serde_json::from_str::<ModelEvent>(payload).map_err(|error| anyhow!(error)))+ })+ .boxed())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parse_single_model_listing() {+ let response = serde_json::json!({+ "object": "list",+ "data": [+ {+ "id": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",+ "object": "model",+ "created": 1735142223u64,+ "owned_by": "llamacpp",+ "meta": {+ "n_ctx": 8192,+ "n_ctx_train": 131072,+ "n_embd": 4096,+ }+ }+ ]+ });+ let response: ListModelsResponse = serde_json::from_value(response).unwrap();+ assert_eq!(response.data.len(), 1);+ let entry = &response.data[0];+ assert!(!entry.is_router_entry());+ assert_eq!(entry.meta.as_ref().unwrap().n_ctx, Some(8192));+ assert_eq!(entry.meta.as_ref().unwrap().n_ctx_train, Some(131072));+ }++ #[test]+ fn parse_router_model_listing() {+ let response = serde_json::json!({+ "object": "list",+ "data": [+ {+ "id": "qwen2.5-coder",+ "object": "model",+ "owned_by": "llamacpp",+ "status": { "value": "loaded", "args": [] },+ "architecture": {+ "input_modalities": ["text", "image"],+ "output_modalities": ["text"]+ }+ },+ {+ "id": "gemma-3",+ "object": "model",+ "owned_by": "llamacpp",+ "status": { "value": "unloaded" },+ "architecture": { "input_modalities": ["text"] }+ }+ ]+ });+ let response: ListModelsResponse = serde_json::from_value(response).unwrap();+ assert_eq!(response.data.len(), 2);++ let loaded = &response.data[0];+ assert!(loaded.is_router_entry());+ assert!(loaded.is_loaded());+ assert!(loaded.supports_images_hint());++ let unloaded = &response.data[1];+ assert!(unloaded.is_router_entry());+ assert!(!unloaded.is_loaded());+ assert!(!unloaded.supports_images_hint());+ }++ #[test]+ fn parse_props() {+ let response = serde_json::json!({+ "default_generation_settings": {+ "id": 0,+ "n_ctx": 8192,+ "params": {}+ },+ "total_slots": 1,+ "model_path": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",+ "modalities": { "vision": true, "audio": false },+ // The real `/props` reports both keys; parsing must not treat that as+ // a duplicate field.+ "chat_template_caps": {+ "supports_tools": true,+ "supports_tool_calls": true,+ "supports_preserve_reasoning": true,+ "supports_system_role": true+ }+ });+ let props: Props = serde_json::from_value(response).unwrap();+ assert_eq!(props.context_length(), Some(8192));+ assert!(props.supports_images());+ assert!(props.supports_tools());+ assert!(props.supports_thinking());+ }++ fn model_event(value: serde_json::Value) -> ModelEvent {+ serde_json::from_value(value).unwrap()+ }++ #[test]+ fn model_event_state_changes() {+ // Terminal load/unload and list changes warrant re-discovery.+ assert!(+ model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "loaded" }+ }))+ .changes_model_state()+ );+ assert!(+ model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "unloaded" }+ }))+ .changes_model_state()+ );+ assert!(+ model_event(serde_json::json!({ "model": "*", "event": "models_reload" }))+ .changes_model_state()+ );+ assert!(+ model_event(serde_json::json!({ "model": "m", "event": "model_remove" }))+ .changes_model_state()+ );++ // Intermediate loading-progress ticks do not.+ assert!(+ !model_event(serde_json::json!({+ "model": "m", "event": "status_change",+ "data": { "status": "loading", "progress": { "value": 0.4 } }+ }))+ .changes_model_state()+ );+ assert!(+ !model_event(serde_json::json!({ "model": "m", "event": "download_progress" }))+ .changes_model_state()+ );+ }++ #[test]+ fn model_event_load_failure() {+ let failed = model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 1 }+ }));+ assert_eq!(failed.load_failure(), Some(1));++ // A clean unload (exit code 0 or absent) is not a failure.+ let clean = model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 0 }+ }));+ assert_eq!(clean.load_failure(), None);+ let loaded = model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "loaded" }+ }));+ assert_eq!(loaded.load_failure(), None);+ }++ #[test]+ fn model_event_load_progress() {+ // We report the current stage's own value (0→1), not a blended overall.+ let weights = model_event(serde_json::json!({+ "model": "m", "event": "status_change",+ "data": { "status": "loading",+ "progress": { "stages": ["text_model"], "current": "text_model", "value": 0.4 } }+ }));+ let progress = weights.load_progress().unwrap();+ assert!((progress.value - 0.4).abs() < 1e-4);+ assert_eq!(progress.stage_label(), "Loading weights");+ assert_eq!(progress.progress_label(), "Loading weights 40%");++ // The projector stage runs 0→1 on its own (not 90→100%).+ let projector = model_event(serde_json::json!({+ "model": "m", "event": "status_change",+ "data": { "status": "loading",+ "progress": { "stages": ["text_model", "mmproj_model"],+ "current": "mmproj_model", "value": 0.5 } }+ }));+ let progress = projector.load_progress().unwrap();+ assert!((progress.value - 0.5).abs() < 1e-4);+ assert_eq!(progress.stage_label(), "Loading projector");++ // Non-loading events carry no progress.+ let loaded = model_event(serde_json::json!({+ "model": "m", "event": "status_change", "data": { "status": "loaded" }+ }));+ assert!(loaded.load_progress().is_none());++ // Bare stage-transition markers (no stages/current/value) are skipped so+ // the indicator doesn't flicker to 0% between stages.+ let transition = model_event(serde_json::json!({+ "model": "m", "event": "status_change",+ "data": { "status": "loading", "progress": { "stage": "mmproj_model" } }+ }));+ assert!(transition.load_progress().is_none());+ }++ #[test]+ fn parse_streaming_reasoning_and_tool_calls() {+ let event = serde_json::json!({+ "model": "llama",+ "object": "chat.completion.chunk",+ "choices": [+ {+ "index": 0,+ "delta": {+ "role": "assistant",+ "content": null,+ "reasoning_content": "thinking...",+ "tool_calls": [+ {+ "index": 0,+ "id": "call_1",+ "function": { "name": "weather", "arguments": "{\"city\":" }+ }+ ]+ },+ "finish_reason": null+ }+ ]+ });+ let event: ResponseStreamEvent = serde_json::from_value(event).unwrap();+ let delta = &event.choices[0].delta;+ assert_eq!(delta.reasoning_content.as_deref(), Some("thinking..."));+ assert_eq!(delta.tool_calls.as_ref().unwrap().len(), 1);+ }++ #[test]+ fn encodes_router_model_id_for_props_query() {+ // Router ids contain `/` and `:`, which must be URL-encoded in the+ // `?model=` query, while unreserved characters (`.`, `-`) stay literal.+ let url = Url::parse_with_params(+ "http://localhost:8080/props",+ [("model", "unsloth/Qwen3.5-2B-GGUF:Q4_1")],+ )+ .unwrap();+ assert_eq!(+ url.as_str(),+ "http://localhost:8080/props?model=unsloth%2FQwen3.5-2B-GGUF%3AQ4_1"+ );+ }+}diff --git a/crates/settings_content/src/language_model.rs b/crates/settings_content/src/language_model.rsindex a673ecc7a88..ab11097728c 100644--- a/crates/settings_content/src/language_model.rs+++ b/crates/settings_content/src/language_model.rs@@ -15,6 +15,7 @@ pub struct AllLanguageModelSettingsContent {pub bedrock: Option<AmazonBedrockSettingsContent>,pub deepseek: Option<DeepseekSettingsContent>,pub google: Option<GoogleSettingsContent>,+ pub llama_cpp: Option<LlamaCppSettingsContent>,pub lmstudio: Option<LmStudioSettingsContent>,pub mistral: Option<MistralSettingsContent>,pub ollama: Option<OllamaSettingsContent>,@@ -262,6 +263,36 @@ pub struct LmStudioAvailableModel {pub supports_images: bool,}+#[with_fallible_options]+#[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)]+pub struct LlamaCppSettingsContent {+ pub api_url: Option<String>,+ /// Whether to automatically discover models served by the llama.cpp server.+ /// Defaults to true.+ pub auto_discover: Option<bool>,+ pub available_models: Option<Vec<LlamaCppAvailableModel>>,+ /// Overrides the context length reported for every llama.cpp model.+ pub context_window: Option<u64>,+ pub custom_headers: Option<HashMap<String, String>>,+}++#[with_fallible_options]+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema, MergeFrom)]+pub struct LlamaCppAvailableModel {+ /// The model id reported by the llama.cpp server (its `--alias` or the model file path).+ pub name: String,+ /// The model's name in Zed's UI, such as in the model selector dropdown menu in the agent panel.+ pub display_name: Option<String>,+ /// The Context Length parameter to the model (aka n_ctx).+ pub max_tokens: u64,+ /// Whether the model supports tools.+ pub supports_tools: Option<bool>,+ /// Whether the model supports vision.+ pub supports_images: Option<bool>,+ /// Whether the model emits reasoning/thinking content.+ pub supports_thinking: Option<bool>,+}+#[with_fallible_options]#[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)]pub struct DeepseekSettingsContent {diff --git a/docs/src/ai/by-company.md b/docs/src/ai/by-company.mdindex 5eb3d27a7c3..e477bdf6b2f 100644--- a/docs/src/ai/by-company.md+++ b/docs/src/ai/by-company.md@@ -97,8 +97,9 @@ Paid DeepSeek usage is API access in Zed, not subscription sign-in.| Tool | Support level | What you get | Account / billing | Setup || --------------------------------- | ----------------- | -------------------------------------- | ----------------- | ------------------------------------------------------------- |-| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) |+| llama.cpp | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#llama-cpp) || LM Studio | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#lm-studio) |+| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) || Local OpenAI-compatible server | Configured in Zed | Local or self-hosted model endpoint | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#openai-compatible) || Local/self-hosted edit prediction | Configured in Zed | Edit predictions from a local provider | Local/self-hosted | [Edit Prediction](./edit-prediction.md) |diff --git a/docs/src/ai/use-a-local-model.md b/docs/src/ai/use-a-local-model.mdindex 7cd7836b698..89984d10f34 100644--- a/docs/src/ai/use-a-local-model.md+++ b/docs/src/ai/use-a-local-model.md@@ -1,6 +1,6 @@---title: Use a Local Model - Zed-description: Configure Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed.+description: Configure llama.cpp, Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed.---# Use a Local Model@@ -9,11 +9,69 @@ Use local models when you run the model on your machine or on infrastructure you| Local path | Zed AI features | External Agents | Terminal Threads | Notes || --------------------------------- | -------------------- | --------------- | ---------------- | -------------------------------------------------- |-| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features |+| llama.cpp | Yes | Separate config | Separate config | Configure a llama.cpp server for Zed AI features || LM Studio | Yes | Separate config | Separate config | Configure LM Studio for Zed AI features |+| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features || Local OpenAI-compatible server | Yes | Separate config | Separate config | Configure base URL, model, and key if needed || Local/self-hosted edit prediction | Edit Prediction only | No | No | Uses [Edit Prediction](./edit-prediction.md) setup |+## llama.cpp {#llama-cpp}++Use [llama.cpp](https://llama.app) and its built-in server for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features.++1. Install llama.cpp from [llama.app](https://llama.app).+2. Start the server in [router mode](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md):++ ```sh+ llama serve+ ```++ It loads models from the llama.cpp cache on demand. To download and run a specific model in one step, pass `-hf`:++ ```sh+ llama serve -hf unsloth/gemma-4-26B-A4B-it-GGUF:BF16+ ```++3. In Zed, select a llama.cpp model from the model dropdown.++Zed automatically discovers the served models with their context length and tool/vision capabilities. In router mode these are refined once a model loads, via the server's `/models/sse` stream (which requires a recent llama.cpp build). To list models yourself instead, set `auto_discover` to `false`:++```json [settings]+{+ "language_models": {+ "llama_cpp": {+ "api_url": "http://localhost:8080",+ "auto_discover": false,+ "available_models": [+ {+ "name": "gemma-4-12b-it-GGUF:BF16",+ "display_name": "gemma-4-12b-it-GGUF:BF16",+ "max_tokens": 32768,+ "supports_tools": true,+ "supports_images": false+ }+ ]+ }+ }+}+```++### llama.cpp Context Length {#llama-cpp-context}++Zed uses the context length the server reports (`/props`). Override it for all models with `context_window`, or per model with `max_tokens` in `available_models`:++```json [settings]+{+ "language_models": {+ "llama_cpp": {+ "context_window": 8192+ }+ }+}+```++If your llama.cpp server requires a key, enter it in the provider UI or set `LLAMACPP_API_KEY`. For a remote server, set the API URL to its endpoint and provide the key (set on the server with `--api-key`).+## Ollama {#ollama}Use Ollama for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features.