Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

language_model: Make LanguageModel plain data served by its provider (#64750)

Mikayla Maki975845875bb2
diff --git a/Cargo.lock b/Cargo.lock
index f2f375f5ceb..edff55c3997 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9729,7 +9729,6 @@ dependencies = [
"ai_onboarding",
"anthropic",
"anyhow",
- "async-lock",
"aws-config",
"aws-credential-types",
"aws-sigv4",
@@ -9781,7 +9780,6 @@ dependencies = [
"settings",
"sha2",
"strum 0.28.0",
- "tokio",
"ui",
"ui_input",
"util",
diff --git a/crates/agent/src/agent.rs b/crates/agent/src/agent.rs
index 89dc13d6bea..ebede3a87d9 100644
--- a/crates/agent/src/agent.rs
+++ b/crates/agent/src/agent.rs
@@ -254,7 +254,7 @@ struct PendingSession {
pub struct LanguageModels {
/// Access language model by ID
- models: HashMap<AgentModelId, Arc<dyn LanguageModel>>,
+ models: HashMap<AgentModelId, LanguageModel>,
/// Cached list for returning language model information
model_list: acp_thread::AgentModelList,
refresh_models_rx: watch::Receiver<()>,
@@ -332,12 +332,12 @@ impl LanguageModels {
self.refresh_models_tx.send(()).ok();
}
- pub fn model_from_id(&self, model_id: &AgentModelId) -> Option<Arc<dyn LanguageModel>> {
+ pub fn model_from_id(&self, model_id: &AgentModelId) -> Option<LanguageModel> {
self.models.get(model_id).cloned()
}
fn map_language_model_to_info(
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
provider: &Arc<dyn LanguageModelProvider>,
) -> acp_thread::AgentModelInfo {
acp_thread::AgentModelInfo {
@@ -354,7 +354,7 @@ impl LanguageModels {
}
}
- fn model_id(model: &Arc<dyn LanguageModel>) -> AgentModelId {
+ fn model_id(model: &LanguageModel) -> AgentModelId {
AgentModelId::new(format!("{}/{}", model.provider_id().0, model.id().0))
}
@@ -771,7 +771,7 @@ impl NativeAgent {
let default_model = registry.default_model().and_then(|default_model| {
self.models
- .model_from_id(&LanguageModels::model_id(&default_model.model))
+ .model_from_id(&LanguageModels::model_id(&default_model))
});
let thread = cx.new(|cx| {
Thread::new(
@@ -824,7 +824,7 @@ impl NativeAgent {
});
let registry = LanguageModelRegistry::read_global(cx);
- let summarization_model = registry.thread_summary_model(cx).map(|c| c.model);
+ let summarization_model = registry.thread_summary_model(cx);
let weak = cx.weak_entity();
let weak_thread = thread_handle.downgrade();
@@ -1444,12 +1444,15 @@ impl NativeAgent {
self.models.refresh_list(cx);
let registry = LanguageModelRegistry::read_global(cx);
- let default_model = registry.default_model().map(|m| m.model);
- let summarization_model = registry.thread_summary_model(cx).map(|m| m.model);
+ let default_model = registry.default_model();
+ let summarization_model = registry.thread_summary_model(cx);
for session in self.sessions.values_mut() {
session.thread.update(cx, |thread, cx| {
thread.ensure_model(default_model.as_ref(), cx);
+ if let language_model::Event::ProviderStateChanged(provider_id) = event {
+ thread.refresh_model(provider_id, cx);
+ }
if let Some(model) = summarization_model.clone() {
if thread.summarization_model().is_none()
@@ -1655,9 +1658,8 @@ impl NativeAgent {
.projects
.get(&project_id)
.context("project state not found")?;
- let summarization_model = LanguageModelRegistry::read_global(cx)
- .thread_summary_model(cx)
- .map(|c| c.model);
+ let summarization_model =
+ LanguageModelRegistry::read_global(cx).thread_summary_model(cx);
Ok(cx.new(|cx| {
let mut thread = Thread::from_db(
@@ -2746,7 +2748,7 @@ pub fn available_native_agent(cx: &App) -> AvailableAgent {
for model in provider.provided_models(cx) {
let id = format!("{}/{}", provider_id.0, model.id().0);
let is_default = default.as_ref().is_some_and(|default| {
- default.provider.id() == provider_id && default.model.id() == model.id()
+ default.provider_id == provider_id && default.id == model.id
});
models.push(AvailableModel {
id,
@@ -3922,10 +3924,10 @@ mod internal_tests {
use fs::FakeFs;
use gpui::TestAppContext;
use indoc::formatdoc;
- use language_model::fake_provider::{FakeLanguageModel, FakeLanguageModelProvider};
+ use language_model::fake_provider::FakeLanguageModelProvider;
use language_model::{
CompletionIntent, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelProviderId, LanguageModelProviderName, Speed,
+ LanguageModelName, LanguageModelProviderId, LanguageModelProviderName, Speed,
};
use serde_json::json;
use settings::{LanguageModelProviderSetting, SettingsStore};
@@ -3957,19 +3959,14 @@ mod internal_tests {
let thread_store = cx.new(|cx| ThreadStore::new(cx));
let agent = cx.update(|cx| NativeAgent::new(thread_store, Templates::new(), fs, cx));
let connection = Rc::new(NativeAgentConnection(agent.clone()));
- let model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "subagent-model",
- "Subagent Model",
- true,
+ let provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![model.clone()]),
- );
+ let model = provider.update_model("subagent-model", |model| {
+ model.name = LanguageModelName::from("Subagent Model".to_string());
+ model.supports_thinking = true;
+ });
cx.update(|cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(provider, cx);
@@ -4230,11 +4227,11 @@ mod internal_tests {
AcpBetaFeatureFlag, FeatureFlag as _, FeatureFlagAppExt as _, FeatureFlagsSettings,
};
- init_test(cx);
+ let fake = init_test(cx);
let (connection, agent, project, acp_thread) = setup_native_agent_session(cx).await;
let session_id = cx.update(|cx| acp_thread.read(cx).session_id().clone());
let thread = cx.update(|cx| native_thread_for_session(&agent, &session_id, cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let old_message_id = ClientUserMessageId::new();
cx.update(|cx| {
@@ -4273,7 +4270,7 @@ mod internal_tests {
});
cx.run_until_parked();
- let request = model.pending_completions().pop().unwrap();
+ let request = fake.pending_completions().pop().unwrap();
assert_eq!(
request.intent,
Some(CompletionIntent::ThreadContextSummarization)
@@ -4299,7 +4296,7 @@ mod internal_tests {
compaction.id.clone()
});
- model.send_completion_stream_text_chunk(&request, "retained ");
+ fake.send_text(&model, &request, "retained ");
cx.run_until_parked();
let summary = acp_thread.read_with(cx, |thread, cx| {
let Some(acp_thread::AgentThreadEntry::ContextCompaction(compaction)) =
@@ -4318,8 +4315,8 @@ mod internal_tests {
assert_eq!(markdown.read(cx).source().as_ref(), "retained ");
markdown.clone()
});
- model.send_completion_stream_text_chunk(&request, "context");
- model.end_completion_stream(&request);
+ fake.send_text(&model, &request, "context");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
prompt_task
.await
@@ -4454,12 +4451,11 @@ mod internal_tests {
#[gpui::test]
async fn test_retried_auto_compaction_terminalizes_each_attempt(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let (connection, agent, _project, acp_thread) = setup_native_agent_session(cx).await;
let session_id = cx.update(|cx| acp_thread.read(cx).session_id().clone());
let thread = cx.update(|cx| native_thread_for_session(&agent, &session_id, cx));
- let model = Arc::new(FakeLanguageModel::default());
- model.set_max_token_count(1_000_000);
+ let model = fake.model("fake");
cx.update(|cx| {
let mut settings = agent_settings::AgentSettings::get_global(cx).clone();
@@ -4481,19 +4477,20 @@ mod internal_tests {
});
cx.run_until_parked();
- let first_request = model
+ let first_request = fake
.pending_completions()
.pop()
.expect("first user prompt should start a model request");
- model.send_completion_stream_event(
+ fake.send_event(
+ &model,
&first_request,
LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 750_000,
..Default::default()
}),
);
- model.send_completion_stream_text_chunk(&first_request, "old assistant");
- model.end_completion_stream(&first_request);
+ fake.send_text(&model, &first_request, "old assistant");
+ fake.end_stream(&model, &first_request);
cx.run_until_parked();
first_prompt
.await
@@ -4509,7 +4506,7 @@ mod internal_tests {
});
cx.run_until_parked();
- let first_compaction_request = model
+ let first_compaction_request = fake
.pending_completions()
.pop()
.expect("token threshold should start automatic compaction");
@@ -4517,7 +4514,8 @@ mod internal_tests {
first_compaction_request.intent,
Some(CompletionIntent::ThreadContextSummarization)
);
- model.send_completion_stream_error(
+ fake.send_error(
+ &model,
&first_compaction_request,
LanguageModelCompletionError::from_provider_response(
language_model::ANTHROPIC_PROVIDER_NAME,
@@ -4528,10 +4526,10 @@ mod internal_tests {
language_model::ProviderErrorCategory::RateLimit,
),
);
- model.end_completion_stream(&first_compaction_request);
+ fake.end_stream(&model, &first_compaction_request);
cx.run_until_parked();
- let second_compaction_request = model
+ let second_compaction_request = fake
.pending_completions()
.pop()
.expect("retryable error should start another compaction attempt");
@@ -4560,8 +4558,8 @@ mod internal_tests {
assert!(thread.is_compacting());
});
- model.send_completion_stream_text_chunk(&second_compaction_request, "summary");
- model.end_completion_stream(&second_compaction_request);
+ fake.send_text(&model, &second_compaction_request, "summary");
+ fake.end_stream(&model, &second_compaction_request);
cx.run_until_parked();
acp_thread.read_with(cx, |thread, _cx| {
@@ -4588,13 +4586,13 @@ mod internal_tests {
);
});
- let final_request = model
+ let final_request = fake
.pending_completions()
.pop()
.expect("successful compaction should continue the user turn");
assert_eq!(final_request.intent, Some(CompletionIntent::UserPrompt));
- model.send_completion_stream_text_chunk(&final_request, "new assistant");
- model.end_completion_stream(&final_request);
+ fake.send_text(&model, &final_request, "new assistant");
+ fake.end_stream(&model, &final_request);
cx.run_until_parked();
second_prompt
.await
@@ -4603,7 +4601,7 @@ mod internal_tests {
#[gpui::test]
async fn test_native_compaction_cancellation_is_bridged_before_stop(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
for (scenario, cancel_before_first_poll, partial_summary) in [
("before first poll", true, None),
@@ -4617,7 +4615,7 @@ mod internal_tests {
let (connection, agent, project, acp_thread) = setup_native_agent_session(cx).await;
let session_id = acp_thread.read_with(cx, |thread, _| thread.session_id().clone());
let thread = cx.update(|cx| native_thread_for_session(&agent, &session_id, cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
cx.update(|cx| {
let path_style = project.read(cx).path_style(cx);
@@ -4649,12 +4647,12 @@ mod internal_tests {
.expect("manual compaction should start");
cx.run_until_parked();
- let request = model
+ let request = fake
.pending_completions()
.pop()
.expect("manual compaction should reach the model");
if let Some(partial_summary) = partial_summary {
- model.send_completion_stream_text_chunk(&request, partial_summary);
+ fake.send_text(&model, &request, partial_summary);
cx.run_until_parked();
}
@@ -4723,11 +4721,11 @@ mod internal_tests {
async fn test_native_tool_names_survive_reused_ids_and_reload(cx: &mut TestAppContext) {
use language_model::{LanguageModelToolUse, LanguageModelToolUseInput};
- init_test(cx);
+ let fake = init_test(cx);
let (connection, agent, project, acp_thread) = setup_native_agent_session(cx).await;
let session_id = acp_thread.read_with(cx, |thread, _| thread.session_id().clone());
let thread = cx.update(|cx| native_thread_for_session(&agent, &session_id, cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("thread");
let fs = project.read_with(cx, |project, _| project.fs().clone());
fs.write(Path::new("/a/file.txt"), b"retained file contents")
.await
@@ -4750,12 +4748,13 @@ mod internal_tests {
)
});
cx.run_until_parked();
- let request = model
- .pending_completions()
+ let request = fake
+ .pending_completions_for(&model)
.pop()
.expect("user prompt should reach the model");
let input = json!({"path": "a/file.txt"});
- model.send_completion_stream_event(
+ fake.send_event(
+ &model,
&request,
LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "reused_id".into(),
@@ -4766,15 +4765,15 @@ mod internal_tests {
thought_signature: None,
}),
);
- model.end_completion_stream(&request);
+ fake.end_stream(&model, &request);
cx.run_until_parked();
- let request = model
- .pending_completions()
+ let request = fake
+ .pending_completions_for(&model)
.pop()
.expect("tool result should reach the model");
assert_eq!(request.intent, Some(CompletionIntent::ToolResults));
- model.send_completion_stream_text_chunk(&request, "done");
- model.end_completion_stream(&request);
+ fake.send_text(&model, &request, "done");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
prompt_task.await.expect("native tool turn should complete");
}
@@ -6653,19 +6652,14 @@ mod internal_tests {
// Register a thinking model and select it.
cx.update(|cx| {
- let thinking_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "fake-thinking",
- "Fake Thinking",
- true,
+ let thinking_provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let thinking_provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![thinking_model]),
- );
+ thinking_provider.update_model("fake-thinking", |model| {
+ model.name = LanguageModelName::from("Fake Thinking".to_string());
+ model.supports_thinking = true;
+ });
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(thinking_provider, cx);
});
@@ -6717,19 +6711,14 @@ mod internal_tests {
// Register a second provider with a thinking model.
cx.update(|cx| {
- let thinking_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "fake-thinking",
- "Fake Thinking",
- true,
+ let thinking_provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let thinking_provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![thinking_model]),
- );
+ thinking_provider.update_model("fake-thinking", |model| {
+ model.name = LanguageModelName::from("Fake Thinking".to_string());
+ model.supports_thinking = true;
+ });
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(thinking_provider, cx);
});
@@ -6847,22 +6836,17 @@ mod internal_tests {
let connection = Rc::new(NativeAgentConnection(agent.clone()));
// Register a thinking model.
- let thinking_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "fake-thinking",
- "Fake Thinking",
- true,
+ let thinking_provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let thinking_provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![thinking_model.clone()]),
- );
+ let thinking_model = thinking_provider.update_model("fake-thinking", |model| {
+ model.name = LanguageModelName::from("Fake Thinking".to_string());
+ model.supports_thinking = true;
+ });
cx.update(|cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
- registry.register_provider(thinking_provider, cx);
+ registry.register_provider(thinking_provider.clone(), cx);
});
});
agent.update(cx, |agent, cx| agent.models.refresh_list(cx));
@@ -6901,8 +6885,8 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- thinking_model.send_last_completion_stream_text_chunk("Response.");
- thinking_model.end_last_completion_stream();
+ thinking_provider.send_last_text(&thinking_model, "Response.");
+ thinking_provider.end_last(&thinking_model);
send.await.unwrap();
cx.run_until_parked();
@@ -6948,22 +6932,16 @@ mod internal_tests {
// Register a model where id() != name(), like real Anthropic models
// (e.g. id="claude-sonnet-4-5-thinking-latest", name="Claude Sonnet 4.5 Thinking").
- let model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "custom-model-id",
- "Custom Model Display Name",
- false,
+ let provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![model.clone()]),
- );
+ let model = provider.update_model("custom-model-id", |model| {
+ model.name = LanguageModelName::from("Custom Model Display Name".to_string());
+ });
cx.update(|cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
- registry.register_provider(provider, cx);
+ registry.register_provider(provider.clone(), cx);
});
});
agent.update(cx, |agent, cx| agent.models.refresh_list(cx));
@@ -7002,8 +6980,8 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("Response.");
- model.end_last_completion_stream();
+ provider.send_last_text(&model, "Response.");
+ provider.end_last(&model);
send.await.unwrap();
cx.run_until_parked();
@@ -7057,19 +7035,13 @@ mod internal_tests {
.update(|cx| NativeAgent::new(thread_store.clone(), Templates::new(), fs.clone(), cx));
let connection = Rc::new(NativeAgentConnection(agent.clone()));
- let model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "custom-model-id",
- "Custom Model Display Name",
- false,
+ let provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
- let provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![model.clone()]),
- );
+ let model = provider.update_model("custom-model-id", |model| {
+ model.name = LanguageModelName::from("Custom Model Display Name".to_string());
+ });
cx.update(|cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(provider.clone(), cx);
@@ -7097,8 +7069,8 @@ mod internal_tests {
let send = acp_thread.update(cx, |thread, cx| thread.send(vec!["Hello".into()], cx));
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("Response.");
- model.end_last_completion_stream();
+ provider.send_last_text(&model, "Response.");
+ provider.end_last(&model);
send.await.unwrap();
cx.run_until_parked();
@@ -7196,19 +7168,13 @@ mod internal_tests {
});
// The user explicitly picks a different, available model.
- let other_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "other-corp",
- "other-model-id",
- "Other Model",
- false,
+ let other_provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("other-corp".to_string()),
+ LanguageModelProviderName::from("Other Corp".to_string()),
));
- let other_provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("other-corp".to_string()),
- LanguageModelProviderName::from("Other Corp".to_string()),
- )
- .with_models(vec![other_model.clone()]),
- );
+ other_provider.update_model("other-model-id", |model| {
+ model.name = LanguageModelName::from("Other Model".to_string());
+ });
cx.update(|cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(other_provider, cx);
@@ -7246,7 +7212,7 @@ mod internal_tests {
#[gpui::test]
async fn test_save_load_thread(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(
"/",
@@ -7277,8 +7243,8 @@ mod internal_tests {
});
// Ensure empty threads are not saved, even if they get mutated.
- let model = Arc::new(FakeLanguageModel::default());
- let summary_model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
+ let summary_model = fake.model("summary");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
thread.set_summarization_model(Some(summary_model.clone()), cx);
@@ -7306,19 +7272,26 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("Lorem.");
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "Lorem.");
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 150,
output_tokens: 75,
..Default::default()
- },
- ));
- model.end_last_completion_stream();
+ }),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
- summary_model
- .send_last_completion_stream_text_chunk(&format!("Explaining {}", path!("/a/b.md")));
- summary_model.end_last_completion_stream();
+ let summary_request = fake.pending_completions_for(&summary_model).pop().unwrap();
+ fake.send_text(
+ &summary_model,
+ &summary_request,
+ format!("Explaining {}", path!("/a/b.md")),
+ );
+ fake.end_stream(&summary_model, &summary_request);
send.await.unwrap();
let uri = MentionUri::File {
@@ -7425,7 +7398,7 @@ mod internal_tests {
#[gpui::test]
async fn test_releasing_session_saves_thread(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(
"/",
@@ -7455,7 +7428,7 @@ mod internal_tests {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
});
@@ -7465,8 +7438,8 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("world");
- model.end_last_completion_stream();
+ fake.send_last_text(&model, "world");
+ fake.end_last(&model);
send.await.unwrap();
cx.run_until_parked();
@@ -7589,7 +7562,7 @@ mod internal_tests {
#[gpui::test]
async fn test_save_burst_preserves_in_flight_write(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(
"/",
@@ -7619,7 +7592,7 @@ mod internal_tests {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
});
@@ -7628,8 +7601,8 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("world");
- model.end_last_completion_stream();
+ fake.send_last_text(&model, "world");
+ fake.end_last(&model);
send.await.unwrap();
cx.run_until_parked();
@@ -7680,7 +7653,7 @@ mod internal_tests {
#[gpui::test]
async fn test_thread_summary_releases_loaded_session(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(
"/",
@@ -7710,8 +7683,8 @@ mod internal_tests {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
- let summary_model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
+ let summary_model = fake.model("summary");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
thread.set_summarization_model(Some(summary_model.clone()), cx);
@@ -7721,8 +7694,9 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("world");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "world");
+ fake.end_stream(&model, &request);
send.await.unwrap();
cx.run_until_parked();
@@ -7731,8 +7705,9 @@ mod internal_tests {
});
cx.run_until_parked();
- summary_model.send_last_completion_stream_text_chunk("summary");
- summary_model.end_last_completion_stream();
+ let summary_request = fake.pending_completions_for(&summary_model).pop().unwrap();
+ fake.send_text(&summary_model, &summary_request, "summary");
+ fake.end_stream(&summary_model, &summary_request);
assert_eq!(summary.await.unwrap(), "summary");
cx.run_until_parked();
@@ -7758,7 +7733,7 @@ mod internal_tests {
#[gpui::test]
async fn test_loaded_sessions_keep_state_until_last_close(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(
"/",
@@ -7788,13 +7763,7 @@ mod internal_tests {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = cx.update(|cx| {
- LanguageModelRegistry::read_global(cx)
- .default_model()
- .map(|default_model| default_model.model)
- .expect("default test model should be available")
- });
- let fake_model = model.as_fake();
+ let model = fake.model("fake");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
});
@@ -7803,8 +7772,8 @@ mod internal_tests {
let send = cx.foreground_executor().spawn(send);
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("world");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "world");
+ fake.end_last(&model);
send.await.unwrap();
cx.run_until_parked();
@@ -7861,8 +7830,8 @@ mod internal_tests {
let follow_up = cx.foreground_executor().spawn(follow_up);
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("yes");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "yes");
+ fake.end_last(&model);
follow_up.await.unwrap();
cx.run_until_parked();
@@ -7973,14 +7942,15 @@ mod internal_tests {
})
}
- fn init_test(cx: &mut TestAppContext) {
+ /// Returns the fake provider that serves the registry's default model.
+ fn init_test(cx: &mut TestAppContext) -> Arc<FakeLanguageModelProvider> {
env_logger::try_init().ok();
cx.update(|cx| {
let settings_store = SettingsStore::test(cx);
cx.set_global(settings_store);
- LanguageModelRegistry::test(cx);
- });
+ LanguageModelRegistry::test(cx)
+ })
}
#[test]
diff --git a/crates/agent/src/tests/mod.rs b/crates/agent/src/tests/mod.rs
index c93e11d031f..4cf9dc4e2f1 100644
--- a/crates/agent/src/tests/mod.rs
+++ b/crates/agent/src/tests/mod.rs
@@ -25,12 +25,11 @@ use gpui::{
};
use indoc::indoc;
use language_model::{
- CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelId, LanguageModelImageExt, LanguageModelProviderId, LanguageModelProviderName,
+ CompletionIntent, LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId,
+ LanguageModelImageExt, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage,
LanguageModelToolResult, LanguageModelToolUse, MessageContent, ProviderErrorCategory, Role,
- StopReason, TokenUsage,
- fake_provider::{FakeLanguageModel, FakeLanguageModelProvider},
+ StopReason, TokenUsage, fake_provider::FakeLanguageModelProvider,
};
use pretty_assertions::assert_eq;
use project::{
@@ -57,11 +56,13 @@ use util::path;
mod test_tools;
use test_tools::*;
-pub(crate) fn init_test(cx: &mut TestAppContext) {
+/// Returns the fake provider that serves the registry's default model.
+pub(crate) fn init_test(cx: &mut TestAppContext) -> Arc<FakeLanguageModelProvider> {
cx.update(|cx| {
let settings_store = SettingsStore::test(cx);
cx.set_global(settings_store);
- });
+ LanguageModelRegistry::test(cx)
+ })
}
pub(crate) fn release_dropped_entities(cx: &mut TestAppContext) {
@@ -324,8 +325,12 @@ fn disable_sandboxing(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_echo(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events = thread
.update(cx, |thread, cx| {
@@ -337,10 +342,12 @@ async fn test_echo(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hello");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hello");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events = events.collect().await;
thread.update(cx, |thread, _cx| {
@@ -479,8 +486,12 @@ async fn test_terminal_tool_without_timeout_does_not_kill_handle(cx: &mut TestAp
#[gpui::test]
async fn test_thinking(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events = thread
.update(cx, |thread, cx| {
@@ -497,14 +508,19 @@ async fn test_thinking(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Thinking {
- text: "Think".to_string(),
- signature: None,
- });
- fake_model.send_last_completion_stream_text_chunk("Hello");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Thinking {
+ text: "Think".to_string(),
+ signature: None,
+ },
+ );
+ fake.send_last_text(&model, "Hello");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events = events.collect().await;
thread.update(cx, |thread, _cx| {
@@ -528,13 +544,18 @@ async fn test_thinking(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_thinking_allowed_when_model_cannot_disable_thinking(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
- fake_model.set_supports_thinking(true);
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
+ fake.update_model("fake", |model| model.supports_thinking = true);
// With thinking toggled off, a model that can disable thinking honors
// the toggle...
thread.update(cx, |thread, cx| {
+ thread.refresh_model(&model.provider_id, cx);
thread.set_thinking_enabled(false, cx);
let request = thread
.build_completion_request(CompletionIntent::UserPrompt, cx)
@@ -543,8 +564,9 @@ async fn test_thinking_allowed_when_model_cannot_disable_thinking(cx: &mut TestA
});
// ...but a model that always thinks ignores the stale toggle state.
- fake_model.set_supports_disabling_thinking(false);
+ fake.update_model("fake", |model| model.supports_disabling_thinking = false);
thread.update(cx, |thread, cx| {
+ thread.refresh_model(&model.provider_id, cx);
let request = thread
.build_completion_request(CompletionIntent::UserPrompt, cx)
.unwrap();
@@ -555,12 +577,11 @@ async fn test_thinking_allowed_when_model_cannot_disable_thinking(cx: &mut TestA
#[gpui::test]
async fn test_system_prompt(cx: &mut TestAppContext) {
let ThreadTest {
- model,
+ fake,
thread,
project_context,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
project_context.update(cx, |project_context, _cx| {
project_context.shell = "test-shell".into()
@@ -572,7 +593,7 @@ async fn test_system_prompt(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- let mut pending_completions = fake_model.pending_completions();
+ let mut pending_completions = fake.pending_completions();
assert_eq!(
pending_completions.len(),
1,
@@ -601,8 +622,7 @@ async fn test_system_prompt(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_system_prompt_without_tools(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest { fake, thread, .. } = setup(cx, TestModel::Fake).await;
thread
.update(cx, |thread, cx| {
@@ -610,7 +630,7 @@ async fn test_system_prompt_without_tools(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- let mut pending_completions = fake_model.pending_completions();
+ let mut pending_completions = fake.pending_completions();
assert_eq!(
pending_completions.len(),
1,
@@ -639,8 +659,12 @@ async fn test_system_prompt_without_tools(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_prompt_caching(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
// Send initial user message and verify it's cached
thread
@@ -650,7 +674,7 @@ async fn test_prompt_caching(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
completion.messages[1..],
vec![LanguageModelRequestMessage {
@@ -660,10 +684,11 @@ async fn test_prompt_caching(cx: &mut TestAppContext) {
reasoning_details: None,
}]
);
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Text(
- "Response to Message 1".into(),
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Text("Response to Message 1".into()),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// Send another user message and verify only the latest is cached
@@ -674,7 +699,7 @@ async fn test_prompt_caching(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
completion.messages[1..],
vec![
@@ -698,10 +723,11 @@ async fn test_prompt_caching(cx: &mut TestAppContext) {
}
]
);
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Text(
- "Response to Message 2".into(),
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Text("Response to Message 2".into()),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// Simulate a tool call and verify that the latest tool result is cached
@@ -721,12 +747,14 @@ async fn test_prompt_caching(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(tool_use.clone()));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(tool_use.clone()),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_result = LanguageModelToolResult {
tool_use_id: "tool_1".into(),
tool_name: EchoTool::NAME.into(),
@@ -905,8 +933,12 @@ async fn test_streaming_tool_calls(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_tool_authorization(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -915,37 +947,40 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_1".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_2".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_interrupted".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let tool_call_auth_1 = next_tool_call_authorization(&mut events).await;
let tool_call_auth_2 = next_tool_call_authorization(&mut events).await;
let interrupted_tool_call_auth = next_tool_call_authorization(&mut events).await;
@@ -977,7 +1012,7 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
assert_eq!(
message.content,
@@ -1007,17 +1042,18 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
);
// Simulate yet another tool call.
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_3".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Respond by always allowing tools - send transformed option_id
// (UI transforms "always:tool_requiring_permission" to "always_allow:tool_requiring_permission")
@@ -1030,7 +1066,7 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
))
.unwrap();
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
assert_eq!(
message.content,
@@ -1046,19 +1082,20 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
);
// Simulate a final tool call, ensuring we don't trigger authorization.
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_4".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
assert_eq!(
message.content,
@@ -1076,8 +1113,12 @@ async fn test_tool_authorization(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_tool_hallucination(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -1085,17 +1126,18 @@ async fn test_tool_hallucination(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_1".into(),
name: "nonexistent_tool".into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let tool_call = expect_tool_call(&mut events).await;
assert_eq!(tool_call.title, "nonexistent_tool");
@@ -1111,8 +1153,12 @@ async fn test_tool_hallucination(cx: &mut TestAppContext) {
/// one in `AcpThread::upsert_tool_call`.
#[gpui::test]
async fn test_tool_call_id_scoped_per_completion_request(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| {
thread.add_tool(EchoTool);
@@ -1125,39 +1171,43 @@ async fn test_tool_call_id_scoped_per_completion_request(cx: &mut TestAppContext
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "call_1".into(),
name: EchoTool::NAME.into(),
raw_input: json!({"text": "first"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "first"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let first_tool_call = next_tool_call(&mut events).await;
cx.run_until_parked();
// Same turn, second cycle: the id counter has reset, so "call_1" recurs
// for a different tool call.
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "call_1".into(),
name: EchoTool::NAME.into(),
raw_input: json!({"text": "second"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "second"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let second_tool_call = next_tool_call(&mut events).await;
cx.run_until_parked();
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
assert_ne!(
@@ -1175,11 +1225,11 @@ async fn test_tool_call_id_scoped_per_completion_request(cx: &mut TestAppContext
async fn test_replayed_tool_call_ids_scoped_across_messages(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
project_context,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
thread.update(cx, |thread, _cx| {
thread.add_tool(EchoTool);
@@ -1191,20 +1241,21 @@ async fn test_replayed_tool_call_ids_scoped_across_messages(cx: &mut TestAppCont
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "call_1".into(),
name: EchoTool::NAME.into(),
raw_input: json!({"text": "first"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "first"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Done with first");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done with first");
+ fake.end_last(&model);
cx.run_until_parked();
// Different turn: the id counter has reset, so "call_1" recurs in a
@@ -1215,20 +1266,21 @@ async fn test_replayed_tool_call_ids_scoped_across_messages(cx: &mut TestAppCont
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "call_1".into(),
name: EchoTool::NAME.into(),
raw_input: json!({"text": "second"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "second"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Done with second");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done with second");
+ fake.end_last(&model);
cx.run_until_parked();
let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx)).await;
@@ -1637,9 +1689,12 @@ async fn test_concurrent_tool_calls(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_profiles(cx: &mut TestAppContext) {
let ThreadTest {
- model, thread, fs, ..
+ model,
+ fake,
+ thread,
+ fs,
+ ..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
thread.update(cx, |thread, _cx| {
thread.add_tool(DelayTool);
@@ -1684,7 +1739,7 @@ async fn test_profiles(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let mut pending_completions = fake_model.pending_completions();
+ let mut pending_completions = fake.pending_completions();
assert_eq!(pending_completions.len(), 1);
let completion = pending_completions.pop().unwrap();
let tool_names: Vec<String> = completion
@@ -1693,7 +1748,7 @@ async fn test_profiles(cx: &mut TestAppContext) {
.map(|tool| tool.name.clone())
.collect();
assert_eq!(tool_names, vec![DelayTool::NAME, EchoTool::NAME]);
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
// Switch to test-2 profile, and verify that it has only the infinite tool.
thread
@@ -1703,7 +1758,7 @@ async fn test_profiles(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- let mut pending_completions = fake_model.pending_completions();
+ let mut pending_completions = fake.pending_completions();
assert_eq!(pending_completions.len(), 1);
let completion = pending_completions.pop().unwrap();
let tool_names: Vec<String> = completion
@@ -1718,12 +1773,12 @@ async fn test_profiles(cx: &mut TestAppContext) {
async fn test_mcp_tools(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
context_server_store,
fs,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
// Override profiles and wait for settings to be loaded.
fs.insert_file(
@@ -1773,19 +1828,20 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
cx.run_until_parked();
// Simulate the model calling the MCP tool.
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(tool_names_for_completion(&completion), vec!["echo"]);
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "echo".into(),
raw_input: json!({"text": "test"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "test"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let (tool_call_params, tool_call_response) = mcp_tool_calls.next().await.unwrap();
@@ -1804,8 +1860,8 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
cx.run_until_parked();
assert_eq!(tool_names_for_completion(&completion), vec!["echo"]);
- fake_model.send_last_completion_stream_text_chunk("Done!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done!");
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
// Send again after adding the echo tool, ensuring the name collision is resolved.
@@ -1814,32 +1870,34 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
thread.send(ClientUserMessageId::new(), ["Go"], cx).unwrap()
});
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
tool_names_for_completion(&completion),
vec!["echo", "test_server_echo"]
);
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_2".into(),
name: "test_server_echo".into(),
raw_input: json!({"text": "mcp"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "mcp"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_3".into(),
name: "echo".into(),
raw_input: json!({"text": "native"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "native"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let (tool_call_params, tool_call_response) = mcp_tool_calls.next().await.unwrap();
@@ -1856,7 +1914,7 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
cx.run_until_parked();
// Ensure the tool results were inserted with the correct names.
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
completion.messages.last().unwrap().content,
vec![
@@ -1876,7 +1934,7 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
},),
]
);
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
}
@@ -1884,12 +1942,12 @@ async fn test_mcp_tools(cx: &mut TestAppContext) {
async fn test_mcp_tool_names_are_sanitized_for_providers(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
context_server_store,
fs,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
fs.insert_file(
paths::settings_file(),
@@ -1934,22 +1992,23 @@ async fn test_mcp_tool_names_are_sanitized_for_providers(cx: &mut TestAppContext
});
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
tool_names_for_completion(&completion),
vec!["snake_case_PascalCase"]
);
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "snake_case_PascalCase".into(),
raw_input: json!({}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let (tool_call_params, tool_call_response) = mcp_tool_calls.next().await.unwrap();
@@ -1966,8 +2025,8 @@ async fn test_mcp_tool_names_are_sanitized_for_providers(cx: &mut TestAppContext
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Done!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done!");
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
}
@@ -1975,13 +2034,16 @@ async fn test_mcp_tool_names_are_sanitized_for_providers(cx: &mut TestAppContext
async fn test_mcp_tool_multi_content_response(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
context_server_store,
fs,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
- fake_model.set_supports_images(true);
+ fake.update_model("fake", |model| model.supports_images = true);
+ thread.update(cx, |thread, cx| {
+ thread.refresh_model(&model.provider_id, cx)
+ });
fs.insert_file(
paths::settings_file(),
@@ -2027,18 +2089,19 @@ async fn test_mcp_tool_multi_content_response(cx: &mut TestAppContext) {
});
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ let completion = fake.pending_completions().pop().unwrap();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "screenshot".into(),
raw_input: json!({}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let _ = completion;
@@ -2067,7 +2130,7 @@ async fn test_mcp_tool_multi_content_response(cx: &mut TestAppContext) {
cx.run_until_parked();
// Verify the tool result round-trips back to the model as a multi-part Vec.
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_result = completion
.messages
.last()
@@ -2101,7 +2164,7 @@ async fn test_mcp_tool_multi_content_response(cx: &mut TestAppContext) {
tool_result.content[2],
language_model::LanguageModelToolResultContent::Text(Arc::from("Some more text"))
);
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
}
@@ -2109,12 +2172,12 @@ async fn test_mcp_tool_multi_content_response(cx: &mut TestAppContext) {
async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
context_server_store,
fs,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
// Setup settings to allow MCP tools
fs.insert_file(
@@ -2169,7 +2232,7 @@ async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAp
cx.run_until_parked();
// Verify the MCP tool is available to the model
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
tool_names_for_completion(&completion),
vec!["issue_read"],
@@ -2177,8 +2240,9 @@ async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAp
);
// Simulate the model calling the MCP tool
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "issue_read".into(),
raw_input: json!({"issue_url": "https://github.com/zed-industries/zed/issues/47404"})
@@ -2188,9 +2252,9 @@ async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAp
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// The MCP server receives the tool call and responds with content
@@ -2211,11 +2275,13 @@ async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAp
// After tool completes, the model continues with a new completion request
// that includes the tool results. We need to respond to this.
- let _completion = fake_model.pending_completions().pop().unwrap();
- fake_model.send_last_completion_stream_text_chunk("I found the issue!");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ let _completion = fake.pending_completions().pop().unwrap();
+ fake.send_last_text(&model, "I found the issue!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
events.collect::<Vec<_>>().await;
// Verify the tool result is stored in the thread by checking the markdown output.
@@ -2297,13 +2363,12 @@ async fn test_mcp_tool_result_displayed_when_server_disconnected(cx: &mut TestAp
#[gpui::test]
async fn test_mcp_tool_truncation(cx: &mut TestAppContext) {
let ThreadTest {
- model,
+ fake,
thread,
context_server_store,
fs,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
// Set up a profile with all tools enabled
fs.insert_file(
@@ -2457,7 +2522,7 @@ async fn test_mcp_tool_truncation(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
tool_names_for_completion(&completion),
vec![
@@ -2567,10 +2632,14 @@ async fn test_cancellation(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_terminal_tool_cancellation_captures_output(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
disable_sandboxing(cx);
- let fake_model = model.as_fake();
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
@@ -2590,8 +2659,9 @@ async fn test_terminal_tool_cancellation_captures_output(cx: &mut TestAppContext
cx.run_until_parked();
// Simulate the model calling the terminal tool
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_1".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 1000", "cd": "."}"#.into(),
@@ -2600,9 +2670,9 @@ async fn test_terminal_tool_cancellation_captures_output(cx: &mut TestAppContext
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Wait for the terminal tool to start running
wait_for_terminal_tool_started(&mut events, cx).await;
@@ -2659,7 +2729,7 @@ async fn test_terminal_tool_cancellation_captures_output(cx: &mut TestAppContext
});
// Verify we can send a new message after cancellation
- verify_thread_recovery(&thread, &fake_model, cx).await;
+ verify_thread_recovery(&thread, &fake, cx).await;
}
#[gpui::test]
@@ -2667,9 +2737,13 @@ async fn test_cancellation_aware_tool_responds_to_cancellation(cx: &mut TestAppC
// This test verifies that tools which properly handle cancellation via
// `event_stream.cancelled_by_user()` (like edit_file_tool) respond promptly
// to cancellation and report that they were cancelled.
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
- let fake_model = model.as_fake();
let (tool, was_cancelled) = CancellationAwareTool::new();
@@ -2687,17 +2761,18 @@ async fn test_cancellation_aware_tool_responds_to_cancellation(cx: &mut TestAppC
cx.run_until_parked();
// Simulate the model calling the cancellation-aware tool
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "cancellation_aware_1".into(),
name: "cancellation_aware".into(),
raw_input: r#"{}"#.into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
@@ -2754,13 +2829,13 @@ async fn test_cancellation_aware_tool_responds_to_cancellation(cx: &mut TestAppC
);
// Verify we can send a new message after cancellation
- verify_thread_recovery(&thread, &fake_model, cx).await;
+ verify_thread_recovery(&thread, &fake, cx).await;
}
/// Helper to verify thread can recover after cancellation by sending a simple message.
async fn verify_thread_recovery(
thread: &Entity<Thread>,
- fake_model: &FakeLanguageModel,
+ fake: &FakeLanguageModelProvider,
cx: &mut TestAppContext,
) {
let events = thread
@@ -2773,10 +2848,13 @@ async fn verify_thread_recovery(
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hello");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ let model = thread.read_with(cx, |thread, _| thread.model().cloned().unwrap());
+ fake.send_last_text(&model, "Hello");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events = events.collect::<Vec<_>>().await;
thread.update(cx, |thread, _cx| {
@@ -2849,10 +2927,14 @@ async fn collect_events_until_stop(
#[gpui::test]
async fn test_truncate_while_terminal_tool_running(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
disable_sandboxing(cx);
- let fake_model = model.as_fake();
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
@@ -2873,8 +2955,9 @@ async fn test_truncate_while_terminal_tool_running(cx: &mut TestAppContext) {
cx.run_until_parked();
// Simulate the model calling the terminal tool
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_1".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 1000", "cd": "."}"#.into(),
@@ -2883,9 +2966,9 @@ async fn test_truncate_while_terminal_tool_running(cx: &mut TestAppContext) {
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Wait for the terminal tool to start running
wait_for_terminal_tool_started(&mut events, cx).await;
@@ -2914,16 +2997,20 @@ async fn test_truncate_while_terminal_tool_running(cx: &mut TestAppContext) {
});
// Verify we can send a new message after truncation
- verify_thread_recovery(&thread, &fake_model, cx).await;
+ verify_thread_recovery(&thread, &fake, cx).await;
}
#[gpui::test]
async fn test_cancel_multiple_concurrent_terminal_tools(cx: &mut TestAppContext) {
// Tests that cancellation properly kills all running terminal tools when multiple are active.
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
disable_sandboxing(cx);
- let fake_model = model.as_fake();
let environment = Rc::new(MultiTerminalEnvironment::new());
@@ -2940,8 +3027,9 @@ async fn test_cancel_multiple_concurrent_terminal_tools(cx: &mut TestAppContext)
cx.run_until_parked();
// Simulate the model calling two terminal tools
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_1".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 1000", "cd": "."}"#.into(),
@@ -2950,10 +3038,11 @@ async fn test_cancel_multiple_concurrent_terminal_tools(cx: &mut TestAppContext)
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_2".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 2000", "cd": "."}"#.into(),
@@ -2962,9 +3051,9 @@ async fn test_cancel_multiple_concurrent_terminal_tools(cx: &mut TestAppContext)
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Wait for both terminal tools to start by counting terminal content updates
let mut terminals_started = 0;
@@ -3035,10 +3124,14 @@ async fn test_cancel_multiple_concurrent_terminal_tools(cx: &mut TestAppContext)
async fn test_terminal_tool_stopped_via_terminal_card_button(cx: &mut TestAppContext) {
// Tests that clicking the stop button on the terminal card (as opposed to the main
// cancel button) properly reports user stopped via the was_stopped_by_user path.
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
disable_sandboxing(cx);
- let fake_model = model.as_fake();
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
@@ -3058,8 +3151,9 @@ async fn test_terminal_tool_stopped_via_terminal_card_button(cx: &mut TestAppCon
cx.run_until_parked();
// Simulate the model calling the terminal tool
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_1".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 1000", "cd": "."}"#.into(),
@@ -3068,9 +3162,9 @@ async fn test_terminal_tool_stopped_via_terminal_card_button(cx: &mut TestAppCon
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Wait for the terminal tool to start running
wait_for_terminal_tool_started(&mut events, cx).await;
@@ -3085,9 +3179,11 @@ async fn test_terminal_tool_stopped_via_terminal_card_button(cx: &mut TestAppCon
cx.run_until_parked();
// The thread continues after tool completion - simulate the model ending its turn
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
// Collect remaining events
let remaining_events = collect_events_until_stop(&mut events, cx).await;
@@ -3129,10 +3225,14 @@ async fn test_terminal_tool_stopped_via_terminal_card_button(cx: &mut TestAppCon
#[gpui::test]
async fn test_terminal_tool_timeout_expires(cx: &mut TestAppContext) {
// Tests that when a timeout is configured and expires, the tool result indicates timeout.
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
always_allow_tools(cx);
disable_sandboxing(cx);
- let fake_model = model.as_fake();
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
@@ -3156,8 +3256,9 @@ async fn test_terminal_tool_timeout_expires(cx: &mut TestAppContext) {
cx.run_until_parked();
// Simulate the model calling the terminal tool with a short timeout
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "terminal_tool_1".into(),
name: TerminalTool::NAME.into(),
raw_input: r#"{"command": "sleep 1000", "cd": ".", "timeout_ms": 100}"#.into(),
@@ -3166,9 +3267,9 @@ async fn test_terminal_tool_timeout_expires(cx: &mut TestAppContext) {
),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
// Wait for the terminal tool to start running
wait_for_terminal_tool_started(&mut events, cx).await;
@@ -3178,9 +3279,11 @@ async fn test_terminal_tool_timeout_expires(cx: &mut TestAppContext) {
cx.run_until_parked();
// The thread continues after tool completion - simulate the model ending its turn
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
// Collect remaining events
let remaining_events = collect_events_until_stop(&mut events, cx).await;
@@ -3231,8 +3334,12 @@ async fn test_terminal_tool_timeout_expires(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_in_progress_send_canceled_by_next_send(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events_1 = thread
.update(cx, |thread, cx| {
@@ -3240,7 +3347,7 @@ async fn test_in_progress_send_canceled_by_next_send(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey 1!");
+ fake.send_last_text(&model, "Hey 1!");
cx.run_until_parked();
let events_2 = thread
@@ -3249,10 +3356,12 @@ async fn test_in_progress_send_canceled_by_next_send(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey 2!");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey 2!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events_1 = events_1.collect::<Vec<_>>().await;
assert_eq!(stop_events(events_1), vec![acp::StopReason::Cancelled]);
@@ -3266,8 +3375,12 @@ async fn test_retry_cancelled_promptly_on_new_send(cx: &mut TestAppContext) {
// the retry loop waits on a timer. If the user switches models and sends a new message
// during that delay, the old turn should exit immediately instead of retrying with the
// stale model.
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let model_a = model.as_fake();
+ let ThreadTest {
+ model: model_a,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
// Start a turn with model_a.
let events_1 = thread
@@ -3276,26 +3389,29 @@ async fn test_retry_cancelled_promptly_on_new_send(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- assert_eq!(model_a.completion_count(), 1);
+ assert_eq!(fake.pending_completions_for(&model_a).len(), 1);
// Model returns a retryable upstream 500. The turn enters the retry delay.
- model_a.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- language_model::LanguageModelProviderName::new("test"),
- http_client::StatusCode::INTERNAL_SERVER_ERROR,
- "Internal server error".to_string(),
- None,
- ));
- model_a.end_last_completion_stream();
+ fake.send_last_error(
+ &model_a,
+ LanguageModelCompletionError::from_http_status(
+ language_model::LanguageModelProviderName::new("test"),
+ http_client::StatusCode::INTERNAL_SERVER_ERROR,
+ "Internal server error".to_string(),
+ None,
+ ),
+ );
+ fake.end_last(&model_a);
cx.run_until_parked();
// The old completion was consumed; model_a has no pending requests yet because the
// retry timer hasn't fired.
- assert_eq!(model_a.completion_count(), 0);
+ assert_eq!(fake.pending_completions_for(&model_a).len(), 0);
// Switch to model_b and send a new message. This cancels the old turn.
- let model_b = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake", "model-b", "Model B", false,
- ));
+ let model_b = fake.update_model("model-b", |model| {
+ model.name = LanguageModelName::from("Model B".to_string())
+ });
thread.update(cx, |thread, cx| {
thread.set_model(model_b.clone(), cx);
});
@@ -3307,7 +3423,7 @@ async fn test_retry_cancelled_promptly_on_new_send(cx: &mut TestAppContext) {
cx.run_until_parked();
// model_b should have received its completion request.
- assert_eq!(model_b.as_fake().completion_count(), 1);
+ assert_eq!(fake.pending_completions_for(&model_b).len(), 1);
// Advance the clock well past the retry delay (BASE_RETRY_DELAY = 5s).
cx.executor().advance_clock(Duration::from_secs(10));
@@ -3316,19 +3432,20 @@ async fn test_retry_cancelled_promptly_on_new_send(cx: &mut TestAppContext) {
// model_a must NOT have received another completion request — the cancelled turn
// should have exited during the retry delay rather than retrying with the old model.
assert_eq!(
- model_a.completion_count(),
+ fake.pending_completions_for(&model_a).len(),
0,
"old model should not receive a retry request after cancellation"
);
// Complete model_b's turn.
- model_b
- .as_fake()
- .send_last_completion_stream_text_chunk("Done!");
- model_b
- .as_fake()
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- model_b.as_fake().end_last_completion_stream();
+ let model_b_request = fake.pending_completions_for(&model_b).pop().unwrap();
+ fake.send_text(&model_b, &model_b_request, "Done!");
+ fake.send_event(
+ &model_b,
+ &model_b_request,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_stream(&model_b, &model_b_request);
let events_1 = events_1.collect::<Vec<_>>().await;
assert_eq!(stop_events(events_1), vec![acp::StopReason::Cancelled]);
@@ -3339,8 +3456,12 @@ async fn test_retry_cancelled_promptly_on_new_send(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_subsequent_successful_sends_dont_cancel(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events_1 = thread
.update(cx, |thread, cx| {
@@ -3348,10 +3469,12 @@ async fn test_subsequent_successful_sends_dont_cancel(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey 1!");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey 1!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events_1 = events_1.collect::<Vec<_>>().await;
let events_2 = thread
@@ -3360,10 +3483,12 @@ async fn test_subsequent_successful_sends_dont_cancel(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey 2!");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey 2!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let events_2 = events_2.collect::<Vec<_>>().await;
assert_eq!(stop_events(events_1), vec![acp::StopReason::EndTurn]);
@@ -3372,8 +3497,12 @@ async fn test_subsequent_successful_sends_dont_cancel(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_refusal(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events = thread
.update(cx, |thread, cx| {
@@ -3392,7 +3521,7 @@ async fn test_refusal(cx: &mut TestAppContext) {
);
});
- fake_model.send_last_completion_stream_text_chunk("Hey!");
+ fake.send_last_text(&model, "Hey!");
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
assert_eq!(
@@ -3410,8 +3539,10 @@ async fn test_refusal(cx: &mut TestAppContext) {
});
// If the model refuses to continue, the thread should remove all the messages after the last user message.
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::Refusal));
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::Refusal),
+ );
let events = events.collect::<Vec<_>>().await;
assert_eq!(stop_events(events), vec![acp::StopReason::Refusal]);
thread.read_with(cx, |thread, _| {
@@ -3421,8 +3552,12 @@ async fn test_refusal(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_truncate_first_message(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let message_id = ClientUserMessageId::new();
thread
@@ -3443,15 +3578,16 @@ async fn test_truncate_first_message(cx: &mut TestAppContext) {
assert_eq!(thread.latest_token_usage(), None);
});
- fake_model.send_last_completion_stream_text_chunk("Hey!");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Hey!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 32_000,
output_tokens: 16_000,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
+ }),
+ );
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
assert_eq!(
@@ -3504,15 +3640,16 @@ async fn test_truncate_first_message(cx: &mut TestAppContext) {
);
});
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Ahoy!");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Ahoy!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 40_000,
output_tokens: 20_000,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
+ }),
+ );
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
assert_eq!(
@@ -3543,8 +3680,12 @@ async fn test_truncate_first_message(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let message_1_id = ClientUserMessageId::new();
thread
@@ -3554,16 +3695,17 @@ async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppCont
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Response 1");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 1");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 100,
output_tokens: 50,
cache_creation_input_tokens: 25,
cache_read_input_tokens: 75,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3594,8 +3736,12 @@ async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppCont
#[gpui::test]
async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread
.update(cx, |thread, cx| {
@@ -3604,15 +3750,16 @@ async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContex
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Response 1");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 1");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 100,
output_tokens: 50,
..Default::default()
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3629,13 +3776,16 @@ async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContex
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- LanguageModelProviderName::new("test"),
- http_client::StatusCode::PAYLOAD_TOO_LARGE,
- "prompt too large".to_string(),
- None,
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_error(
+ &model,
+ LanguageModelCompletionError::from_http_status(
+ LanguageModelProviderName::new("test"),
+ http_client::StatusCode::PAYLOAD_TOO_LARGE,
+ "prompt too large".to_string(),
+ None,
+ ),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3648,8 +3798,12 @@ async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContex
#[gpui::test]
async fn test_prompt_too_large_uses_reported_token_count(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread
.update(cx, |thread, cx| {
@@ -3658,13 +3812,16 @@ async fn test_prompt_too_large_uses_reported_token_count(cx: &mut TestAppContext
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- LanguageModelProviderName::new("test"),
- http_client::StatusCode::PAYLOAD_TOO_LARGE,
- "prompt is too long: 1500000 tokens".to_string(),
- None,
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_error(
+ &model,
+ LanguageModelCompletionError::from_http_status(
+ LanguageModelProviderName::new("test"),
+ http_client::StatusCode::PAYLOAD_TOO_LARGE,
+ "prompt is too long: 1500000 tokens".to_string(),
+ None,
+ ),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3678,11 +3835,11 @@ async fn test_prompt_too_large_uses_reported_token_count(cx: &mut TestAppContext
async fn test_cumulative_token_usage(cx: &mut TestAppContext) {
let ThreadTest {
model,
+ fake,
thread,
project_context,
..
} = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
thread
.update(cx, |thread, cx| {
@@ -3694,43 +3851,47 @@ async fn test_cumulative_token_usage(cx: &mut TestAppContext) {
// The first request emits two cumulative snapshots; only the final values
// must be counted, exactly once.
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 100,
output_tokens: 10,
..Default::default()
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 100,
output_tokens: 50,
..Default::default()
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: EchoTool::NAME.into(),
raw_input: json!({"text": "hello"}).to_string(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "hello"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// The second request (after the tool call) is counted in addition to the first.
- fake_model.send_last_completion_stream_text_chunk("Done");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
+ fake.send_last_text(&model, "Done");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 200,
output_tokens: 30,
..Default::default()
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let expected = TokenUsage {
@@ -3772,8 +3933,12 @@ async fn test_cumulative_token_usage(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_cumulative_token_usage_keeps_accounted_usage_monotonic(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread
.update(cx, |thread, cx| {
@@ -3782,24 +3947,27 @@ async fn test_cumulative_token_usage_keeps_accounted_usage_monotonic(cx: &mut Te
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 100,
output_tokens: 10,
..Default::default()
- },
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage::default(),
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage::default()),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 100,
output_tokens: 50,
..Default::default()
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3816,8 +3984,12 @@ async fn test_cumulative_token_usage_keeps_accounted_usage_monotonic(cx: &mut Te
#[gpui::test]
async fn test_truncate_second_message(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread
.update(cx, |thread, cx| {
@@ -3825,16 +3997,17 @@ async fn test_truncate_second_message(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Message 1 response");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Message 1 response");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 32_000,
output_tokens: 16_000,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let assert_first_message_state = |cx: &mut TestAppContext| {
@@ -3875,16 +4048,17 @@ async fn test_truncate_second_message(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Message 2 response");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Message 2 response");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 40_000,
output_tokens: 20_000,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3931,10 +4105,14 @@ async fn test_truncate_second_message(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_title_generation(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
- let summary_model = Arc::new(FakeLanguageModel::default());
+ let summary_model = fake.model("summary");
thread.update(cx, |thread, cx| {
thread.set_summarization_model(Some(summary_model.clone()), cx)
});
@@ -3946,16 +4124,17 @@ async fn test_title_generation(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey!");
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _| assert_eq!(thread.title(), None));
// Ensure the summary model has been invoked to generate a title.
- summary_model.send_last_completion_stream_text_chunk("Hello ");
- summary_model.send_last_completion_stream_text_chunk("world\nG");
- summary_model.send_last_completion_stream_text_chunk("oodnight Moon");
- summary_model.end_last_completion_stream();
+ let summary_request = fake.pending_completions_for(&summary_model).pop().unwrap();
+ fake.send_text(&summary_model, &summary_request, "Hello ");
+ fake.send_text(&summary_model, &summary_request, "world\nG");
+ fake.send_text(&summary_model, &summary_request, "oodnight Moon");
+ fake.end_stream(&summary_model, &summary_request);
send.collect::<Vec<_>>().await;
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -3969,10 +4148,10 @@ async fn test_title_generation(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey again!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey again!");
+ fake.end_last(&model);
cx.run_until_parked();
- assert_eq!(summary_model.pending_completions(), Vec::new());
+ assert_eq!(fake.pending_completions_for(&summary_model), Vec::new());
send.collect::<Vec<_>>().await;
thread.read_with(cx, |thread, _| {
assert_eq!(thread.title(), Some("Hello world".into()))
@@ -3981,18 +4160,19 @@ async fn test_title_generation(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_stream_thread_title_keeps_only_first_line(cx: &mut TestAppContext) {
- let model = Arc::new(FakeLanguageModel::default());
let request = LanguageModelRequest::default();
+ let fake = cx.update(LanguageModelRegistry::test);
+ let registered_model = fake.model("fake");
let title_task = cx.spawn({
- let model = model.clone();
- async move |cx| crate::stream_thread_title(model, request, &cx).await
+ let registered_model = registered_model.clone();
+ async move |cx| crate::stream_thread_title(registered_model, request, &cx).await
});
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("Hello world\nGoodnight Moon");
- model.end_last_completion_stream();
+ fake.send_last_text(&registered_model, "Hello world\nGoodnight Moon");
+ fake.end_last(&registered_model);
let title = title_task.await.unwrap();
assert_eq!(title, "Hello world");
@@ -4000,19 +4180,20 @@ async fn test_stream_thread_title_keeps_only_first_line(cx: &mut TestAppContext)
#[gpui::test]
async fn test_stream_thread_title_stops_when_newline_ends_chunk(cx: &mut TestAppContext) {
- let model = Arc::new(FakeLanguageModel::default());
let request = LanguageModelRequest::default();
+ let fake = cx.update(LanguageModelRegistry::test);
+ let registered_model = fake.model("fake");
let title_task = cx.spawn({
- let model = model.clone();
- async move |cx| crate::stream_thread_title(model, request, &cx).await
+ let registered_model = registered_model.clone();
+ async move |cx| crate::stream_thread_title(registered_model, request, &cx).await
});
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("Hello world\n");
- model.send_last_completion_stream_text_chunk("Goodnight Moon");
- model.end_last_completion_stream();
+ fake.send_last_text(&registered_model, "Hello world\n");
+ fake.send_last_text(&registered_model, "Goodnight Moon");
+ fake.end_last(&registered_model);
let title = title_task.await.unwrap();
assert_eq!(title, "Hello world");
@@ -4024,8 +4205,12 @@ async fn test_stream_thread_title_stops_when_newline_ends_chunk(cx: &mut TestApp
// they share a single rendering path.
#[gpui::test]
async fn test_db_thread_markdown_matches_live_thread(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let send = thread
.update(cx, |thread, cx| {
@@ -4033,8 +4218,8 @@ async fn test_db_thread_markdown_matches_live_thread(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey there!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey there!");
+ fake.end_last(&model);
send.collect::<Vec<_>>().await;
cx.run_until_parked();
@@ -4047,11 +4232,14 @@ async fn test_db_thread_markdown_matches_live_thread(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_title_generation_failure_allows_retry(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
- let summary_model = Arc::new(FakeLanguageModel::default());
- let fake_summary_model = summary_model.as_fake();
+ let summary_model = fake.model("summary");
thread.update(cx, |thread, cx| {
thread.set_summarization_model(Some(summary_model.clone()), cx)
});
@@ -4063,11 +4251,14 @@ async fn test_title_generation_failure_allows_retry(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey!");
+ fake.end_last(&model);
cx.run_until_parked();
- fake_summary_model.send_last_completion_stream_error(
+ let summary_request = fake.pending_completions_for(&summary_model).pop().unwrap();
+ fake.send_error(
+ &summary_model,
+ &summary_request,
LanguageModelCompletionError::from_http_status(
language_model::LanguageModelProviderName::new("test"),
gpui::http_client::StatusCode::INTERNAL_SERVER_ERROR,
@@ -4075,7 +4266,7 @@ async fn test_title_generation_failure_allows_retry(cx: &mut TestAppContext) {
None,
),
);
- fake_summary_model.end_last_completion_stream();
+ fake.end_stream(&summary_model, &summary_request);
send.collect::<Vec<_>>().await;
cx.run_until_parked();
@@ -4101,8 +4292,9 @@ async fn test_title_generation_failure_allows_retry(cx: &mut TestAppContext) {
assert!(thread.is_generating_title());
});
- fake_summary_model.send_last_completion_stream_text_chunk("Retried title");
- fake_summary_model.end_last_completion_stream();
+ let summary_request = fake.pending_completions_for(&summary_model).pop().unwrap();
+ fake.send_text(&summary_model, &summary_request, "Retried title");
+ fake.end_stream(&summary_model, &summary_request);
cx.run_until_parked();
thread.read_with(cx, |thread, _| {
@@ -4115,8 +4307,12 @@ async fn test_title_generation_failure_allows_retry(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_building_request_with_pending_tools(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let _events = thread
.update(cx, |thread, cx| {
@@ -4143,14 +4339,16 @@ async fn test_building_request_with_pending_tools(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- fake_model.send_last_completion_stream_text_chunk("Hi!");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- permission_tool_use,
- ));
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- echo_tool_use.clone(),
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hi!");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(permission_tool_use),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(echo_tool_use.clone()),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// Ensure pending tools are skipped when building a request.
@@ -4211,8 +4409,8 @@ async fn test_agent_connection(cx: &mut TestAppContext) {
language_model::init(cx);
RefreshLlmTokenListener::register(client.clone(), user_store.clone(), cx);
language_models::init(user_store, client.clone(), cx);
- LanguageModelRegistry::test(cx);
});
+ let fake = cx.update(LanguageModelRegistry::test);
cx.executor().forbid_parking();
// Create a project for new_thread
@@ -4268,12 +4466,11 @@ async fn test_agent_connection(cx: &mut TestAppContext) {
let model = cx
.update(|cx| agent.read(cx).models().model_from_id(&model.id))
.unwrap();
- let model = model.as_fake();
assert_eq!(model.id().0, "fake", "should return default model");
let request = acp_thread.update(cx, |thread, cx| thread.send(vec!["abc".into()], cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("def");
+ fake.send_last_text(&model, "def");
cx.run_until_parked();
acp_thread.read_with(cx, |thread, cx| {
assert_eq!(
@@ -4318,9 +4515,13 @@ async fn test_agent_connection(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_tool_updates_to_completion(cx: &mut TestAppContext) {
- let ThreadTest { thread, model, .. } = setup(cx, TestModel::Fake).await;
+ let ThreadTest {
+ model,
+ thread,
+ fake,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| thread.add_tool(EchoTool));
- let fake_model = model.as_fake();
let mut events = thread
.update(cx, |thread, cx| {
@@ -4331,30 +4532,32 @@ async fn test_tool_updates_to_completion(cx: &mut TestAppContext) {
// Simulate streaming partial input.
let input = json!({});
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "1".into(),
name: EchoTool::NAME.into(),
raw_input: input.to_string(),
input: language_model::LanguageModelToolUseInput::Json(input),
is_input_complete: false,
thought_signature: None,
- },
- ));
+ }),
+ );
// Input streaming completed
let input = json!({ "text": "Hello!" });
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "1".into(),
name: "echo".into(),
raw_input: input.to_string(),
input: language_model::LanguageModelToolUseInput::Json(input),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// User message is index 0, so the tool call is scoped to index 1 (see
@@ -4401,8 +4604,12 @@ async fn test_tool_updates_to_completion(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_send_no_retry_on_success(cx: &mut TestAppContext) {
- let ThreadTest { thread, model, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ thread,
+ fake,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -4411,8 +4618,8 @@ async fn test_send_no_retry_on_success(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey!");
+ fake.end_last(&model);
let mut retry_events = Vec::new();
while let Some(Ok(event)) = events.next().await {
@@ -4444,8 +4651,12 @@ async fn test_send_no_retry_on_success(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_send_retry_on_error(cx: &mut TestAppContext) {
- let ThreadTest { thread, model, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ thread,
+ fake,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -4454,14 +4665,17 @@ async fn test_send_retry_on_error(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Hey,");
- fake_model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- LanguageModelProviderName::new("Anthropic"),
- http_client::StatusCode::SERVICE_UNAVAILABLE,
- "Anthropic's API servers are overloaded right now".to_string(),
- Some(Duration::from_secs(3)),
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Hey,");
+ fake.send_last_error(
+ &model,
+ LanguageModelCompletionError::from_http_status(
+ LanguageModelProviderName::new("Anthropic"),
+ http_client::StatusCode::SERVICE_UNAVAILABLE,
+ "Anthropic's API servers are overloaded right now".to_string(),
+ Some(Duration::from_secs(3)),
+ ),
+ );
+ fake.end_last(&model);
cx.executor()
.advance_clock(crate::maximum_retry_delay_with_jitter(Duration::from_secs(
@@ -4469,8 +4683,8 @@ async fn test_send_retry_on_error(cx: &mut TestAppContext) {
)));
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("there!");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "there!");
+ fake.end_last(&model);
cx.run_until_parked();
let mut retry_events = Vec::new();
@@ -4513,8 +4727,12 @@ async fn test_send_retry_on_error(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_send_retry_finishes_tool_calls_on_error(cx: &mut TestAppContext) {
- let ThreadTest { thread, model, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ thread,
+ fake,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let events = thread
.update(cx, |thread, cx| {
@@ -4532,22 +4750,26 @@ async fn test_send_retry_finishes_tool_calls_on_error(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- tool_use_1.clone(),
- ));
- fake_model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- LanguageModelProviderName::new("Anthropic"),
- http_client::StatusCode::SERVICE_UNAVAILABLE,
- "Anthropic's API servers are overloaded right now".to_string(),
- Some(Duration::from_secs(3)),
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(tool_use_1.clone()),
+ );
+ fake.send_last_error(
+ &model,
+ LanguageModelCompletionError::from_http_status(
+ LanguageModelProviderName::new("Anthropic"),
+ http_client::StatusCode::SERVICE_UNAVAILABLE,
+ "Anthropic's API servers are overloaded right now".to_string(),
+ Some(Duration::from_secs(3)),
+ ),
+ );
+ fake.end_last(&model);
cx.executor()
.advance_clock(crate::maximum_retry_delay_with_jitter(Duration::from_secs(
3,
)));
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
assert_eq!(
completion.messages[1..],
vec![
@@ -4580,8 +4802,8 @@ async fn test_send_retry_finishes_tool_calls_on_error(cx: &mut TestAppContext) {
]
);
- fake_model.send_last_completion_stream_text_chunk("Done");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done");
+ fake.end_last(&model);
cx.run_until_parked();
events.collect::<Vec<_>>().await;
thread.read_with(cx, |thread, _cx| {
@@ -4598,8 +4820,12 @@ async fn test_send_retry_finishes_tool_calls_on_error(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_send_max_retries_exceeded(cx: &mut TestAppContext) {
- let ThreadTest { thread, model, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ thread,
+ fake,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -4609,7 +4835,8 @@ async fn test_send_max_retries_exceeded(cx: &mut TestAppContext) {
cx.run_until_parked();
for _ in 0..crate::thread::MAX_RETRY_ATTEMPTS + 1 {
- fake_model.send_last_completion_stream_error(
+ fake.send_last_error(
+ &model,
LanguageModelCompletionError::from_http_status(
LanguageModelProviderName::new("Anthropic"),
http_client::StatusCode::SERVICE_UNAVAILABLE,
@@ -4617,7 +4844,7 @@ async fn test_send_max_retries_exceeded(cx: &mut TestAppContext) {
Some(Duration::from_secs(3)),
),
);
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
cx.executor()
.advance_clock(crate::maximum_retry_delay_with_jitter(Duration::from_secs(
3,
@@ -4665,8 +4892,12 @@ async fn test_streaming_tool_completes_when_llm_stream_ends_without_final_input(
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| {
thread.add_tool(StreamingEchoTool::new());
@@ -4693,21 +4924,26 @@ async fn test_streaming_tool_completes_when_llm_stream_ends_without_final_input(
is_input_complete: false,
thought_signature: None,
};
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(tool_use.clone()));
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(tool_use.clone()),
+ );
cx.run_until_parked();
// Send a stream error WITHOUT ever sending is_input_complete = true.
// Before the fix, this would deadlock: the tool waits for more partials
// (or cancellation), run_turn_internal waits for the tool, and the sender
// keeping the channel open lives inside RunningTurn.
- fake_model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- language_model::LanguageModelProviderName::new("test"),
- http_client::StatusCode::INTERNAL_SERVER_ERROR,
- "Internal server error".to_string(),
- None,
- ));
- fake_model.end_last_completion_stream();
+ fake.send_last_error(
+ &model,
+ LanguageModelCompletionError::from_http_status(
+ language_model::LanguageModelProviderName::new("test"),
+ http_client::StatusCode::INTERNAL_SERVER_ERROR,
+ "Internal server error".to_string(),
+ None,
+ ),
+ );
+ fake.end_last(&model);
// Advance past the retry delay so run_turn_internal retries.
cx.executor()
@@ -4718,10 +4954,7 @@ async fn test_streaming_tool_completes_when_llm_stream_ends_without_final_input(
// The retry request should contain the streaming tool's error result,
// proving the tool terminated and its result was forwarded.
- let completion = fake_model
- .pending_completions()
- .pop()
- .expect("No running turn");
+ let completion = fake.pending_completions().pop().expect("No running turn");
assert_eq!(
completion.messages[1..],
vec![
@@ -4755,8 +4988,8 @@ async fn test_streaming_tool_completes_when_llm_stream_ends_without_final_input(
);
// Finish the retry round so the turn completes cleanly.
- fake_model.send_last_completion_stream_text_chunk("Done");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done");
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _cx| {
@@ -4774,8 +5007,12 @@ async fn test_streaming_tool_json_parse_error_is_forwarded_to_running_tool(
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| {
thread.add_tool(StreamingJsonErrorContextTool);
@@ -4800,10 +5037,11 @@ async fn test_streaming_tool_json_parse_error_is_forwarded_to_running_tool(
is_input_complete: false,
thought_signature: None,
};
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(tool_use));
+ fake.send_last_event(&model, LanguageModelCompletionEvent::ToolUse(tool_use));
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(
+ fake.send_last_event(
+ &model,
LanguageModelCompletionEvent::ToolUseJsonParseError {
id: "tool_1".into(),
tool_name: StreamingJsonErrorContextTool::NAME.into(),
@@ -4811,18 +5049,17 @@ async fn test_streaming_tool_json_parse_error_is_forwarded_to_running_tool(
json_parse_error: "EOF while parsing a string at line 1 column 17".into(),
},
);
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::ToolUse));
- fake_model.end_last_completion_stream();
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::ToolUse),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
cx.executor().advance_clock(Duration::from_secs(5));
cx.run_until_parked();
- let completion = fake_model
- .pending_completions()
- .pop()
- .expect("No running turn");
+ let completion = fake.pending_completions().pop().expect("No running turn");
let tool_results: Vec<_> = completion
.messages
@@ -4863,8 +5100,8 @@ async fn test_streaming_tool_json_parse_error_is_forwarded_to_running_tool(
"Should not contain orphaned sender error, got: {content_text}"
);
- fake_model.send_last_completion_stream_text_chunk("Done");
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "Done");
+ fake.end_last(&model);
cx.run_until_parked();
thread.read_with(cx, |thread, _cx| {
@@ -4887,7 +5124,9 @@ fn stop_events(result_events: Vec<Result<ThreadEvent>>) -> Vec<acp::StopReason>
}
struct ThreadTest {
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
+ /// Serves `model`; unused with `TestModel::Sonnet4`.
+ fake: Arc<FakeLanguageModelProvider>,
thread: Entity<Thread>,
project_context: Entity<ProjectContext>,
context_server_store: Entity<ContextServerStore>,
@@ -4945,11 +5184,11 @@ async fn setup(cx: &mut TestAppContext, model: TestModel) -> ThreadTest {
)
.await;
- cx.update(|cx| {
+ let fake = cx.update(|cx| {
settings::init(cx);
- match model {
- TestModel::Fake => {}
+ let fake = match model {
+ TestModel::Fake => LanguageModelRegistry::test(cx),
TestModel::Sonnet4 => {
<dyn fs::Fs>::set_global(fs.clone(), cx);
gpui_tokio::init(cx);
@@ -4960,10 +5199,12 @@ async fn setup(cx: &mut TestAppContext, model: TestModel) -> ThreadTest {
language_model::init(cx);
RefreshLlmTokenListener::register(client.clone(), user_store.clone(), cx);
language_models::init(user_store, client.clone(), cx);
+ Arc::new(FakeLanguageModelProvider::default())
}
};
watch_settings(fs.clone(), cx);
+ fake
});
let templates = Templates::new();
@@ -4974,7 +5215,7 @@ async fn setup(cx: &mut TestAppContext, model: TestModel) -> ThreadTest {
let model = cx
.update(|cx| {
if let TestModel::Fake = model {
- Task::ready(Arc::new(FakeLanguageModel::default()) as Arc<_>)
+ Task::ready(fake.model("fake"))
} else {
let model_id = model.id();
let models = LanguageModelRegistry::read_global(cx);
@@ -5010,6 +5251,7 @@ async fn setup(cx: &mut TestAppContext, model: TestModel) -> ThreadTest {
});
ThreadTest {
model,
+ fake,
thread,
project_context,
context_server_store,
@@ -5141,8 +5383,12 @@ fn setup_context_server(
#[gpui::test]
async fn test_tokens_before_message(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
// First message
let message_1_id = ClientUserMessageId::new();
@@ -5163,16 +5409,17 @@ async fn test_tokens_before_message(cx: &mut TestAppContext) {
});
// Complete first message with usage
- fake_model.send_last_completion_stream_text_chunk("Response 1");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 1");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 100,
output_tokens: 50,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// First message still has no tokens before it
@@ -5203,16 +5450,17 @@ async fn test_tokens_before_message(cx: &mut TestAppContext) {
});
// Complete second message
- fake_model.send_last_completion_stream_text_chunk("Response 2");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 2");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 250, // Total for this request (includes previous context)
output_tokens: 75,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// Third message
@@ -5248,8 +5496,12 @@ async fn test_tokens_before_message(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_tokens_before_message_after_truncate(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
// Set up three messages with responses
let message_1_id = ClientUserMessageId::new();
@@ -5259,16 +5511,17 @@ async fn test_tokens_before_message_after_truncate(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Response 1");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 1");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 100,
output_tokens: 50,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
let message_2_id = ClientUserMessageId::new();
@@ -5278,16 +5531,17 @@ async fn test_tokens_before_message_after_truncate(cx: &mut TestAppContext) {
})
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_text_chunk("Response 2");
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- language_model::TokenUsage {
+ fake.send_last_text(&model, "Response 2");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::UsageUpdate(language_model::TokenUsage {
input_tokens: 250,
output_tokens: 75,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
cx.run_until_parked();
// Verify initial state
@@ -5612,10 +5866,7 @@ async fn test_spawn_agent_tool_rejects_model_when_resuming(cx: &mut TestAppConte
#[gpui::test]
async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
- init_test(cx);
- cx.update(|cx| {
- LanguageModelRegistry::test(cx);
- });
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
});
@@ -5648,7 +5899,7 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
let thread = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("thread");
// Ensure empty threads are not saved, even if they get mutated.
thread.update(cx, |thread, cx| {
@@ -5658,7 +5909,8 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
let send = acp_thread.update(cx, |thread, cx| thread.send_raw("Prompt", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("spawning subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "spawning subagent");
let subagent_tool_input = SpawnAgentToolInput {
label: "label".to_string(),
message: "subagent task prompt".to_string(),
@@ -5675,10 +5927,12 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- subagent_tool_use,
- ));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(subagent_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -5700,8 +5954,9 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
.expect("subagent thread should be alive")
});
- model.send_last_completion_stream_text_chunk("subagent task response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "subagent task response");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -5719,8 +5974,9 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
"}
);
- model.send_last_completion_stream_text_chunk("Response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "Response");
+ fake.end_stream(&model, &request);
send.await.unwrap();
@@ -5750,10 +6006,7 @@ async fn test_subagent_tool_call_end_to_end(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppContext) {
- init_test(cx);
- cx.update(|cx| {
- LanguageModelRegistry::test(cx);
- });
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
});
@@ -5786,7 +6039,7 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
let thread = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("thread");
// Ensure empty threads are not saved, even if they get mutated.
thread.update(cx, |thread, cx| {
@@ -5796,7 +6049,8 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
let send = acp_thread.update(cx, |thread, cx| thread.send_raw("Prompt", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("spawning subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "spawning subagent");
let subagent_tool_input = SpawnAgentToolInput {
label: "label".to_string(),
message: "subagent task prompt".to_string(),
@@ -5813,10 +6067,12 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- subagent_tool_use,
- ));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(subagent_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -5838,13 +6094,18 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
.expect("subagent thread should be alive")
});
- model.send_last_completion_stream_text_chunk("subagent task response 1");
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::Thinking {
- text: "thinking more about the subagent task".into(),
- signature: None,
- });
- model.send_last_completion_stream_text_chunk("subagent task response 2");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "subagent task response 1");
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::Thinking {
+ text: "thinking more about the subagent task".into(),
+ signature: None,
+ },
+ );
+ fake.send_text(&model, &request, "subagent task response 2");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -5868,8 +6129,9 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
"}
);
- model.send_last_completion_stream_text_chunk("Response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "Response");
+ fake.end_stream(&model, &request);
send.await.unwrap();
@@ -5901,10 +6163,7 @@ async fn test_subagent_tool_output_does_not_include_thinking(cx: &mut TestAppCon
#[gpui::test]
async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAppContext) {
- init_test(cx);
- cx.update(|cx| {
- LanguageModelRegistry::test(cx);
- });
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
});
@@ -5937,7 +6196,7 @@ async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAp
let thread = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
// Ensure empty threads are not saved, even if they get mutated.
thread.update(cx, |thread, cx| {
@@ -5947,7 +6206,8 @@ async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAp
let send = acp_thread.update(cx, |thread, cx| thread.send_raw("Prompt", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("spawning subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "spawning subagent");
let subagent_tool_input = SpawnAgentToolInput {
label: "label".to_string(),
message: "subagent task prompt".to_string(),
@@ -5964,10 +6224,12 @@ async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAp
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- subagent_tool_use,
- ));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(subagent_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -5988,8 +6250,8 @@ async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAp
.expect("subagent thread should be alive")
});
- // model.send_last_completion_stream_text_chunk("subagent task response");
- // model.end_last_completion_stream();
+ // fake.send_last_text(&model, "subagent task response");
+ // fake.end_last(&model);
// cx.run_until_parked();
@@ -6034,10 +6296,7 @@ async fn test_subagent_tool_call_cancellation_during_task_prompt(cx: &mut TestAp
#[gpui::test]
async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
- init_test(cx);
- cx.update(|cx| {
- LanguageModelRegistry::test(cx);
- });
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
});
@@ -6070,7 +6329,7 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
let thread = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("thread");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
@@ -6080,7 +6339,8 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
// === First turn: create subagent ===
let send = acp_thread.update(cx, |thread, cx| thread.send_raw("First prompt", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("spawning subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "spawning subagent");
let subagent_tool_input = SpawnAgentToolInput {
label: "initial task".to_string(),
message: "do the first task".to_string(),
@@ -6097,10 +6357,12 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- subagent_tool_use,
- ));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(subagent_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -6123,14 +6385,16 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
});
// Subagent responds
- model.send_last_completion_stream_text_chunk("first task response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "first task response");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
// Parent model responds to complete first turn
- model.send_last_completion_stream_text_chunk("First response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "First response");
+ fake.end_stream(&model, &request);
send.await.unwrap();
@@ -6145,7 +6409,8 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
// === Second turn: resume subagent with session_id ===
let send2 = acp_thread.update(cx, |thread, cx| thread.send_raw("Follow up", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("resuming subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "resuming subagent");
let resume_tool_input = SpawnAgentToolInput {
label: "follow-up task".to_string(),
message: "do the follow-up task".to_string(),
@@ -6162,8 +6427,12 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(resume_tool_use));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(resume_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -6175,14 +6444,16 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
});
// Subagent responds to follow-up
- model.send_last_completion_stream_text_chunk("follow-up task response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "follow-up task response");
+ fake.end_stream(&model, &request);
cx.run_until_parked();
// Parent model responds to complete second turn
- model.send_last_completion_stream_text_chunk("Second response");
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "Second response");
+ fake.end_stream(&model, &request);
send2.await.unwrap();
@@ -6220,7 +6491,7 @@ async fn test_subagent_tool_resume_session(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_subagent_thread_inherits_parent_thread_properties(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
@@ -6233,7 +6504,7 @@ async fn test_subagent_thread_inherits_parent_thread_properties(cx: &mut TestApp
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let parent_thread = cx.new(|cx| {
Thread::new(
@@ -6268,7 +6539,7 @@ async fn test_subagent_thread_inherits_parent_thread_properties(cx: &mut TestApp
#[gpui::test]
async fn test_subagent_thread_model_selection(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(path!("/test"), json!({})).await;
@@ -6277,30 +6548,20 @@ async fn test_subagent_thread_model_selection(cx: &mut TestAppContext) {
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let parent_model = Arc::new(FakeLanguageModel::default());
- let subagent_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "subagent-model",
- "Subagent Model",
- true,
- ));
- let explicit_model = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake-corp",
- "explicit-model",
- "Explicit Model",
- false,
+ let parent_model = fake.model("fake");
+ let provider = Arc::new(FakeLanguageModelProvider::new(
+ LanguageModelProviderId::from("fake-corp".to_string()),
+ LanguageModelProviderName::from("Fake Corp".to_string()),
));
+ let subagent_model = provider.update_model("subagent-model", |model| {
+ model.name = LanguageModelName::from("Subagent Model".to_string());
+ model.supports_thinking = true;
+ });
+ let explicit_model = provider.update_model("explicit-model", |model| {
+ model.name = LanguageModelName::from("Explicit Model".to_string());
+ });
cx.update(|cx| {
- LanguageModelRegistry::test(cx);
-
- let provider = Arc::new(
- FakeLanguageModelProvider::new(
- LanguageModelProviderId::from("fake-corp".to_string()),
- LanguageModelProviderName::from("Fake Corp".to_string()),
- )
- .with_models(vec![subagent_model.clone(), explicit_model.clone()]),
- );
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.register_provider(provider, cx);
});
@@ -6386,7 +6647,7 @@ async fn test_subagent_thread_model_selection(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_max_subagent_depth_prevents_tool_registration(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
@@ -6399,7 +6660,7 @@ async fn test_max_subagent_depth_prevents_tool_registration(cx: &mut TestAppCont
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
}));
@@ -6436,7 +6697,7 @@ async fn test_max_subagent_depth_prevents_tool_registration(cx: &mut TestAppCont
#[gpui::test]
async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
let fs = FakeFs::new(cx.executor());
fs.insert_tree(path!("/test"), json!({})).await;
@@ -6445,7 +6706,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
}));
@@ -6456,7 +6717,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
project_context,
context_server_registry,
Templates::new(),
- Some(model.clone() as Arc<dyn LanguageModel>),
+ Some(model.clone()),
cx,
);
thread.add_default_tools(environment, cx);
@@ -6497,7 +6758,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_names = tool_names_for_completion(&completion);
for name in &lsp_tool_names {
assert!(
@@ -6516,7 +6777,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
tool_names.iter().any(|t| t == ReadFileTool::NAME),
"expected non-LSP tools to still be exposed, got: {tool_names:?}"
);
- model.end_last_completion_stream();
+ fake.end_last(&model);
cx.run_until_parked();
// Enable the `lsp-tool` flag and send another message; the LSP tools
@@ -6532,7 +6793,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_names = tool_names_for_completion(&completion);
for name in &lsp_tool_names {
assert!(
@@ -6550,7 +6811,7 @@ async fn test_lsp_tools_gated_by_feature_flag(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
// `CreateThreadToolFeatureFlag::enabled_for_staff()` returns true, which
// means tests in debug builds resolve it to ON unless we explicitly
@@ -6585,7 +6846,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let environment = Rc::new(cx.update(|cx| {
FakeThreadEnvironment::default().with_terminal(FakeTerminalHandle::new_never_exits(cx))
}));
@@ -6596,7 +6857,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
project_context,
context_server_registry,
Templates::new(),
- Some(model.clone() as Arc<dyn LanguageModel>),
+ Some(model.clone()),
cx,
);
thread.add_default_tools(environment, cx);
@@ -6619,7 +6880,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_names = tool_names_for_completion(&completion);
assert!(!tool_names.iter().any(|tool| tool == CreateThreadTool::NAME));
assert!(
@@ -6632,7 +6893,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
tool_names.iter().any(|t| t == ReadFileTool::NAME),
"expected non-sibling-thread tools to still be exposed, got: {tool_names:?}"
);
- model.end_last_completion_stream();
+ fake.end_last(&model);
cx.run_until_parked();
// Flag explicitly on: the next completion request includes both tools.
@@ -6644,7 +6905,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
.unwrap();
cx.run_until_parked();
- let completion = model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let tool_names = tool_names_for_completion(&completion);
assert!(tool_names.iter().any(|tool| tool == CreateThreadTool::NAME));
assert!(
@@ -6656,7 +6917,7 @@ async fn test_thread_tools_feature_gating(cx: &mut TestAppContext) {
#[gpui::test]
async fn test_parent_cancel_stops_subagent(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
cx.update(|cx| {
cx.update_flags(true, vec!["subagents".to_string()]);
@@ -6669,7 +6930,7 @@ async fn test_parent_cancel_stops_subagent(cx: &mut TestAppContext) {
let context_server_store = project.read_with(cx, |project, _| project.context_server_store());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(context_server_store.clone(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
let parent = cx.new(|cx| {
Thread::new(
@@ -6741,7 +7002,7 @@ async fn test_subagent_auto_compaction(cx: &mut TestAppContext) {
cx,
);
test.assert_running(cx);
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
cx.executor()
.advance_clock(crate::maximum_retry_delay_with_jitter(Duration::from_secs(
3,
@@ -6751,12 +7012,12 @@ async fn test_subagent_auto_compaction(cx: &mut TestAppContext) {
assert_eq!(retry_request.messages, request.messages);
let request = retry_request;
- test.model
- .send_completion_stream_text_chunk(&request, "subagent summary");
- test.model.end_completion_stream(&request);
+ test.fake
+ .send_text(&test.model, &request, "subagent summary");
+ test.fake.end_stream(&test.model, &request);
cx.run_until_parked();
- let request = test.model.pending_completions().pop().unwrap();
+ let request = test.fake.pending_completions().pop().unwrap();
assert_eq!(request.intent, Some(CompletionIntent::ToolResults));
assert_eq!(request.thread_id, Some(test.handle.id().to_string()));
assert_eq!(
@@ -6771,9 +7032,9 @@ async fn test_subagent_auto_compaction(cx: &mut TestAppContext) {
"The previous conversation was compacted. Use this summary as context:\n\nsubagent summary",
],
);
- test.model
- .send_completion_stream_text_chunk(&request, "subagent answer");
- test.model.end_completion_stream(&request);
+ test.fake
+ .send_text(&test.model, &request, "subagent answer");
+ test.fake.end_stream(&test.model, &request);
assert_eq!(send.await.unwrap(), "subagent answer");
test.assert_stopped(cx);
@@ -6786,15 +7047,15 @@ async fn test_subagent_auto_compaction(cx: &mut TestAppContext) {
assert_eq!(handle.id(), test.handle.id());
let send = cx.update(|cx| handle.send("follow-up task".to_string(), &cx.to_async()));
cx.run_until_parked();
- let request = test.model.pending_completions().pop().unwrap();
+ let request = test.fake.pending_completions().pop().unwrap();
assert_eq!(request.intent, Some(CompletionIntent::Subagent));
assert_eq!(
request.messages.last().unwrap().string_contents(),
"follow-up task"
);
- test.model
- .send_completion_stream_text_chunk(&request, "follow-up answer");
- test.model.end_completion_stream(&request);
+ test.fake
+ .send_text(&test.model, &request, "follow-up answer");
+ test.fake.end_stream(&test.model, &request);
assert_eq!(send.await.unwrap(), "follow-up answer");
test.assert_stopped(cx);
@@ -6803,11 +7064,11 @@ async fn test_subagent_auto_compaction(cx: &mut TestAppContext) {
if use_tool {
test.tool_round(0, cx);
}
- test.model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(
- StopReason::Refusal,
- ));
- test.model.end_last_completion_stream();
+ test.fake.send_last_event(
+ &test.model,
+ LanguageModelCompletionEvent::Stop(StopReason::Refusal),
+ );
+ test.fake.end_last(&test.model);
assert_eq!(
send.await.unwrap_err().to_string(),
"The agent refused to process that prompt. Try again."
@@ -6828,12 +7089,12 @@ async fn test_subagent_compaction_respects_settings_and_context_window(cx: &mut
test.configure_compaction(enabled, max_tokens, max_output_tokens, cx);
let send = test.send("subagent task prompt", cx);
test.tool_round(input_tokens, cx);
- let request = test.model.pending_completions().pop().unwrap();
+ let request = test.fake.pending_completions().pop().unwrap();
assert_eq!(request.intent, Some(CompletionIntent::ToolResults));
test.assert_running(cx);
- test.model
- .send_completion_stream_text_chunk(&request, "subagent answer");
- test.model.end_completion_stream(&request);
+ test.fake
+ .send_text(&test.model, &request, "subagent answer");
+ test.fake.end_stream(&test.model, &request);
assert_eq!(send.await.unwrap(), "subagent answer");
test.assert_stopped(cx);
}
@@ -6849,12 +7110,11 @@ async fn test_subagent_context_limit_warning_when_compaction_unavailable(cx: &mu
let test = SubagentCompactionTest::new(cx).await;
test.configure_compaction(enabled, max_tokens, max_output_tokens, cx);
let send = test.send("subagent task prompt", cx);
- let request = test.model.pending_completions().pop().unwrap();
- test.model
- .send_completion_stream_text_chunk(&request, "partial work");
+ let request = test.fake.pending_completions().pop().unwrap();
+ test.fake.send_text(&test.model, &request, "partial work");
test.update_usage(warning_tokens - 1, cx);
test.assert_running(cx);
- assert!(!test.model.is_completion_stream_closed(&request));
+ assert!(!test.fake.is_stream_closed(&test.model, &request));
test.update_usage(warning_tokens, cx);
assert_eq!(
send.await.unwrap_err().to_string(),
@@ -6863,18 +7123,17 @@ async fn test_subagent_context_limit_warning_when_compaction_unavailable(cx: &mu
),
);
test.assert_stopped(cx);
- assert!(test.model.is_completion_stream_closed(&request));
- test.model.end_completion_stream(&request);
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert!(test.fake.is_stream_closed(&test.model, &request));
+ test.fake.end_stream(&test.model, &request);
+ assert_eq!(test.fake.pending_completions(), Vec::new());
test.assert_no_compaction(cx).await;
let send = test.send("wrap up", cx);
- let request = test.model.pending_completions().pop().unwrap();
- test.model
- .send_completion_stream_text_chunk(&request, "wrapped up");
+ let request = test.fake.pending_completions().pop().unwrap();
+ test.fake.send_text(&test.model, &request, "wrapped up");
test.update_usage(warning_tokens + 1, cx);
test.assert_running(cx);
- test.model.end_completion_stream(&request);
+ test.fake.end_stream(&test.model, &request);
assert_eq!(send.await.unwrap(), "wrapped up");
test.assert_stopped(cx);
}
@@ -6885,16 +7144,16 @@ async fn test_subagent_context_limit_exceeded_without_partial_output(cx: &mut Te
let test = SubagentCompactionTest::new(cx).await;
test.configure_compaction(false, 1_000_000, None, cx);
let send = test.send("subagent task prompt", cx);
- let request = test.model.pending_completions().pop().unwrap();
+ let request = test.fake.pending_completions().pop().unwrap();
test.update_usage(1_000_000, cx);
assert_eq!(
send.await.unwrap_err().to_string(),
SUBAGENT_CONTEXT_LIMIT_WARNING,
);
test.assert_stopped(cx);
- assert!(test.model.is_completion_stream_closed(&request));
- test.model.end_completion_stream(&request);
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert!(test.fake.is_stream_closed(&test.model, &request));
+ test.fake.end_stream(&test.model, &request);
+ assert_eq!(test.fake.pending_completions(), Vec::new());
}
#[gpui::test]
@@ -6918,10 +7177,10 @@ async fn test_subagent_context_limit_preserves_terminal_result_during_checkpoint
let checkpoint_job = repository.update(cx, |repository, _| {
repository.send_job("hold checkpoint", None, move |_, _| checkpoint_gate)
});
- test.model
- .send_last_completion_stream_text_chunk("partial work");
+ test.fake.send_last_text(&test.model, "partial work");
if provider_error {
- test.model.send_last_completion_stream_error(
+ test.fake.send_last_error(
+ &test.model,
LanguageModelCompletionError::from_http_status(
LanguageModelProviderName::new("test"),
http_client::StatusCode::BAD_REQUEST,
@@ -6930,7 +7189,7 @@ async fn test_subagent_context_limit_preserves_terminal_result_during_checkpoint
),
);
}
- test.model.end_last_completion_stream();
+ test.fake.end_last(&test.model);
cx.run_until_parked();
test.thread
@@ -6978,7 +7237,7 @@ async fn test_subagent_context_limit_preserves_terminal_result_during_checkpoint
assert_eq!(send.await.unwrap(), "partial work");
}
test.assert_stopped(cx);
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
}
}
@@ -6993,7 +7252,7 @@ async fn test_subagent_compaction_exhausts_transient_retries(cx: &mut TestAppCon
Some(Duration::from_secs(3)),
cx,
);
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
if request_index < 4 {
test.assert_running(cx);
cx.executor()
@@ -7018,7 +7277,7 @@ async fn test_subagent_compaction_exhausts_transient_retries(cx: &mut TestAppCon
3,
)));
cx.run_until_parked();
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
}
#[gpui::test]
@@ -7027,19 +7286,19 @@ async fn test_subagent_compaction_parent_cancellation(cx: &mut TestAppContext) {
let send = test.send("subagent task prompt", cx);
test.tool_round(950_000, cx);
let request = test.compaction_request();
- test.model
- .send_completion_stream_text_chunk(&request, "partial summary");
+ test.fake
+ .send_text(&test.model, &request, "partial summary");
cx.run_until_parked();
- assert!(!test.model.is_completion_stream_closed(&request));
+ assert!(!test.fake.is_stream_closed(&test.model, &request));
test.parent.update(cx, |thread, cx| thread.cancel(cx)).await;
assert_eq!(send.await.unwrap_err().to_string(), "User canceled");
cx.run_until_parked();
- assert!(test.model.is_completion_stream_closed(&request));
+ assert!(test.fake.is_stream_closed(&test.model, &request));
test.assert_stopped(cx);
test.assert_no_compaction(cx).await;
- test.model.end_completion_stream(&request);
+ test.fake.end_stream(&test.model, &request);
cx.run_until_parked();
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
}
#[gpui::test]
@@ -7052,7 +7311,8 @@ async fn test_subagent_compaction_error_propagation(cx: &mut TestAppContext) {
test.fail_compaction(http_client::StatusCode::BAD_REQUEST, None, cx);
"compaction provider error"
} else {
- test.model.end_completion_stream(&test.compaction_request());
+ test.fake
+ .end_stream(&test.model, &test.compaction_request());
"Compaction produced an empty summary"
};
assert_eq!(
@@ -7065,15 +7325,14 @@ async fn test_subagent_compaction_error_propagation(cx: &mut TestAppContext) {
);
test.assert_stopped(cx);
test.assert_no_compaction(cx).await;
- assert_eq!(test.model.pending_completions(), Vec::new());
+ assert_eq!(test.fake.pending_completions(), Vec::new());
}
}
#[gpui::test]
async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
- init_test(cx);
+ let fake = init_test(cx);
cx.update(|cx| {
- LanguageModelRegistry::test(cx);
let mut settings = AgentSettings::get_global(cx).clone();
settings.auto_compact.enabled = false;
AgentSettings::override_global(settings, cx);
@@ -7110,7 +7369,7 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
let thread = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("thread");
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
@@ -7120,7 +7379,8 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
// Start the parent turn
let send = acp_thread.update(cx, |thread, cx| thread.send_raw("Prompt", cx));
cx.run_until_parked();
- model.send_last_completion_stream_text_chunk("spawning subagent");
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "spawning subagent");
let subagent_tool_input = SpawnAgentToolInput {
label: "label".to_string(),
message: "subagent task prompt".to_string(),
@@ -7137,10 +7397,12 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
is_input_complete: true,
thought_signature: None,
};
- model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- subagent_tool_use,
- ));
- model.end_last_completion_stream();
+ fake.send_event(
+ &model,
+ &request,
+ LanguageModelCompletionEvent::ToolUse(subagent_tool_use),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -7150,14 +7412,19 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
subagent_ids.into_iter().next().unwrap()
});
- model.send_last_completion_stream_text_chunk("partial work");
- model.send_last_completion_stream_error(LanguageModelCompletionError::from_http_status(
- LanguageModelProviderName::new("test"),
- http_client::StatusCode::PAYLOAD_TOO_LARGE,
- "prompt too large".to_string(),
- None,
- ));
- model.end_last_completion_stream();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
+ fake.send_text(&model, &request, "partial work");
+ fake.send_error(
+ &model,
+ &request,
+ LanguageModelCompletionError::from_http_status(
+ LanguageModelProviderName::new("test"),
+ http_client::StatusCode::PAYLOAD_TOO_LARGE,
+ "prompt too large".to_string(),
+ None,
+ ),
+ );
+ fake.end_stream(&model, &request);
cx.run_until_parked();
@@ -7169,7 +7436,7 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
);
});
- let request = model.pending_completions().pop().unwrap();
+ let request = fake.pending_completions_for(&model).pop().unwrap();
assert_eq!(request.intent, Some(CompletionIntent::ToolResults));
let tool_results = request
.messages
@@ -7197,8 +7464,8 @@ async fn test_subagent_error_propagation(cx: &mut TestAppContext) {
})
.to_string(),
);
- model.send_completion_stream_text_chunk(&request, "Response after error");
- model.end_completion_stream(&request);
+ fake.send_text(&model, &request, "Response after error");
+ fake.end_stream(&model, &request);
send.await.unwrap();
}
@@ -8114,8 +8381,12 @@ async fn test_fetch_tool_follows_same_host_redirect(cx: &mut TestAppContext) {
/// sibling pending authorizations for the same tool in the same turn.
#[gpui::test]
async fn test_always_allow_resolves_pending_authorizations(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -8127,18 +8398,19 @@ async fn test_always_allow_resolves_pending_authorizations(cx: &mut TestAppConte
// Two parallel tool calls, both require permission.
for id in ["tool_id_1", "tool_id_2"] {
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: id.into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
+ }),
+ );
}
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
let tool_call_auth_1 = next_tool_call_authorization(&mut events).await;
let tool_call_auth_2 = next_tool_call_authorization(&mut events).await;
@@ -8168,7 +8440,7 @@ async fn test_always_allow_resolves_pending_authorizations(cx: &mut TestAppConte
"expected tool 2's response receiver to be dropped after auto-resolve"
);
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
let results: Vec<_> = message
.content
@@ -8195,8 +8467,12 @@ async fn test_always_allow_resolves_pending_authorizations(cx: &mut TestAppConte
/// for tool calls that match the new rule.
#[gpui::test]
async fn test_external_settings_edit_resolves_pending_authorization(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -8206,17 +8482,18 @@ async fn test_external_settings_edit_resolves_pending_authorization(cx: &mut Tes
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_1".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let tool_call_auth = next_tool_call_authorization(&mut events).await;
@@ -8249,7 +8526,7 @@ async fn test_external_settings_edit_resolves_pending_authorization(cx: &mut Tes
"response receiver should have been dropped after settings-driven auto-resolve"
);
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
let result = message
.content
@@ -8266,8 +8543,12 @@ async fn test_external_settings_edit_resolves_pending_authorization(cx: &mut Tes
/// authorization prompt and returns the tool call as denied.
#[gpui::test]
async fn test_external_deny_rule_resolves_pending_authorization(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -8277,17 +8558,18 @@ async fn test_external_deny_rule_resolves_pending_authorization(cx: &mut TestApp
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_1".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let tool_call_auth = next_tool_call_authorization(&mut events).await;
@@ -8319,7 +8601,7 @@ async fn test_external_deny_rule_resolves_pending_authorization(cx: &mut TestApp
"response receiver should have been dropped after deny auto-resolve"
);
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
let result = message
.content
@@ -8342,8 +8624,12 @@ async fn test_external_deny_rule_resolves_pending_authorization(cx: &mut TestApp
async fn test_unrelated_settings_change_does_not_resolve_pending_authorization(
cx: &mut TestAppContext,
) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -8353,17 +8639,18 @@ async fn test_unrelated_settings_change_does_not_resolve_pending_authorization(
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_id_1".into(),
name: ToolRequiringPermission::NAME.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model);
let tool_call_auth = next_tool_call_authorization(&mut events).await;
@@ -8386,7 +8673,7 @@ async fn test_unrelated_settings_change_does_not_resolve_pending_authorization(
.expect("response receiver should still be alive");
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
let result = message
.content
@@ -8405,8 +8692,12 @@ async fn test_unrelated_settings_change_does_not_resolve_pending_authorization(
/// and waits for the user.
#[gpui::test]
async fn test_always_allow_does_not_resolve_unrelated_tool_authorization(cx: &mut TestAppContext) {
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let mut events = thread
.update(cx, |thread, cx| {
@@ -8423,18 +8714,19 @@ async fn test_always_allow_does_not_resolve_unrelated_tool_authorization(cx: &mu
("tool_id_1", ToolRequiringPermission::NAME),
("tool_id_2", ToolRequiringPermission2::NAME),
] {
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: id.into(),
name: name.into(),
raw_input: "{}".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({})),
is_input_complete: true,
thought_signature: None,
- },
- ));
+ }),
+ );
}
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
let auth_a = next_tool_call_authorization(&mut events).await;
let auth_b = next_tool_call_authorization(&mut events).await;
@@ -8476,7 +8768,7 @@ async fn test_always_allow_does_not_resolve_unrelated_tool_authorization(cx: &mu
.expect("tool 2's response receiver should still be alive");
cx.run_until_parked();
- let completion = fake_model.pending_completions().pop().unwrap();
+ let completion = fake.pending_completions().pop().unwrap();
let message = completion.messages.last().unwrap();
let results: Vec<_> = message
.content
@@ -8503,8 +8795,12 @@ async fn test_queued_message_ends_turn_at_boundary(cx: &mut TestAppContext) {
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
// Add a tool so we can simulate tool calls
thread.update(cx, |thread, _cx| {
@@ -8520,18 +8816,21 @@ async fn test_queued_message_ends_turn_at_boundary(cx: &mut TestAppContext) {
cx.run_until_parked();
// Simulate the model making a tool call
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "echo".into(),
raw_input: r#"{"text": "hello"}"#.into(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "hello"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::ToolUse));
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::ToolUse),
+ );
// Request that the turn end at the next boundary (a "steering" queued message)
thread.update(cx, |thread, _cx| {
@@ -8539,7 +8838,7 @@ async fn test_queued_message_ends_turn_at_boundary(cx: &mut TestAppContext) {
});
// Now end the stream - tool will run, and the boundary check should see the queue
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
// Collect all events until the turn stops
let all_events = collect_events_until_stop(&mut events, cx).await;
@@ -8590,8 +8889,12 @@ async fn test_queued_message_does_not_end_turn_without_boundary_flag(cx: &mut Te
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| {
thread.add_tool(EchoTool);
@@ -8604,28 +8907,31 @@ async fn test_queued_message_does_not_end_turn_without_boundary_flag(cx: &mut Te
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "echo".into(),
raw_input: r#"{"text": "hello"}"#.into(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "hello"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::ToolUse));
+ }),
+ );
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::ToolUse),
+ );
// Default behavior: even though a message is conceptually queued, we do NOT
// set the boundary flag, so the agent must keep going past the tool boundary
// (running to completion) rather than ending the turn early.
- fake_model.end_last_completion_stream();
+ fake.end_last(&model);
cx.run_until_parked();
// The agent should have issued a fresh completion request with the tool
// results instead of stopping — proof it continued past the boundary.
- let continuation = fake_model.pending_completions();
+ let continuation = fake.pending_completions();
assert_eq!(
continuation.len(),
1,
@@ -8633,10 +8939,12 @@ async fn test_queued_message_does_not_end_turn_without_boundary_flag(cx: &mut Te
);
// Let the continuation finish the turn naturally.
- fake_model.send_last_completion_stream_text_chunk("All done");
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));
- fake_model.end_last_completion_stream();
+ fake.send_last_text(&model, "All done");
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::Stop(StopReason::EndTurn),
+ );
+ fake.end_last(&model);
let all_events = collect_events_until_stop(&mut events, cx).await;
let stop_reasons = stop_events(all_events);
@@ -8652,8 +8960,12 @@ async fn test_streaming_tool_error_breaks_stream_loop_immediately(cx: &mut TestA
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
thread.update(cx, |thread, _cx| {
thread.add_tool(StreamingFailingEchoTool {
@@ -8681,12 +8993,14 @@ async fn test_streaming_tool_error_breaks_stream_loop_immediately(cx: &mut TestA
thought_signature: None,
};
- fake_model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(tool_use.clone()));
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(tool_use.clone()),
+ );
cx.run_until_parked();
- let completions = fake_model.pending_completions();
+ let completions = fake.pending_completions();
let last_completion = completions.last().unwrap();
assert_eq!(
@@ -8727,8 +9041,12 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
init_test(cx);
always_allow_tools(cx);
- let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
- let fake_model = model.as_fake();
+ let ThreadTest {
+ model,
+ fake,
+ thread,
+ ..
+ } = setup(cx, TestModel::Fake).await;
let (complete_streaming_echo_tool_call_tx, complete_streaming_echo_tool_call_rx) =
oneshot::channel();
@@ -8753,16 +9071,17 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
.unwrap();
cx.run_until_parked();
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "call_1".into(),
name: StreamingEchoTool::NAME.into(),
raw_input: "hello".into(),
input: language_model::LanguageModelToolUseInput::Json(json!({ "text": "hello" })),
is_input_complete: false,
thought_signature: None,
- },
- ));
+ }),
+ );
let first_tool_use = LanguageModelToolUse {
id: "call_1".into(),
name: StreamingEchoTool::NAME.into(),
@@ -8771,9 +9090,10 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
is_input_complete: true,
thought_signature: None,
};
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- first_tool_use.clone(),
- ));
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(first_tool_use.clone()),
+ );
let second_tool_use = LanguageModelToolUse {
name: StreamingFailingEchoTool::NAME.into(),
raw_input: "hello".into(),
@@ -8782,9 +9102,10 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
thought_signature: None,
id: "call_2".into(),
};
- fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- second_tool_use.clone(),
- ));
+ fake.send_last_event(
+ &model,
+ LanguageModelCompletionEvent::ToolUse(second_tool_use.clone()),
+ );
cx.run_until_parked();
@@ -8792,7 +9113,7 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
cx.run_until_parked();
- let completions = fake_model.pending_completions();
+ let completions = fake.pending_completions();
let last_completion = completions.last().unwrap();
assert_eq!(
@@ -8843,9 +9164,12 @@ async fn test_streaming_tool_error_waits_for_prior_tools_to_complete(cx: &mut Te
#[gpui::test]
async fn test_mid_turn_model_and_settings_refresh(cx: &mut TestAppContext) {
let ThreadTest {
- model, thread, fs, ..
+ model: model_a,
+ fake,
+ thread,
+ fs,
+ ..
} = setup(cx, TestModel::Fake).await;
- let fake_model_a = model.as_fake();
thread.update(cx, |thread, _cx| {
thread.add_tool(EchoTool);
@@ -8894,36 +9218,35 @@ async fn test_mid_turn_model_and_settings_refresh(cx: &mut TestAppContext) {
cx.run_until_parked();
// Verify first request has both tools and thinking disabled.
- let completions = fake_model_a.pending_completions();
+ let completions = fake.pending_completions_for(&model_a);
assert_eq!(completions.len(), 1);
let first_tools = tool_names_for_completion(&completions[0]);
assert_eq!(first_tools, vec![DelayTool::NAME, EchoTool::NAME]);
assert!(!completions[0].thinking_allowed);
// Model A responds with an echo tool call.
- fake_model_a.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
+ fake.send_last_event(
+ &model_a,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
id: "tool_1".into(),
name: "echo".into(),
raw_input: r#"{"text":"hello"}"#.into(),
input: language_model::LanguageModelToolUseInput::Json(json!({"text": "hello"})),
is_input_complete: true,
thought_signature: None,
- },
- ));
- fake_model_a.end_last_completion_stream();
+ }),
+ );
+ fake.end_last(&model_a);
// Before the next iteration runs, switch to profile-b (only DelayTool),
// swap in a new model, and enable thinking.
- let fake_model_b = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "test-provider",
- "model-b",
- "Model B",
- true,
- ));
+ let model_b = fake.update_model("model-b", |model| {
+ model.name = LanguageModelName::from("Model B".to_string());
+ model.supports_thinking = true;
+ });
thread.update(cx, |thread, cx| {
thread.set_profile(AgentProfileId("profile-b".into()), cx);
- thread.set_model(fake_model_b.clone() as Arc<dyn LanguageModel>, cx);
+ thread.set_model(model_b.clone(), cx);
thread.set_thinking_enabled(true, cx);
});
@@ -8932,7 +9255,7 @@ async fn test_mid_turn_model_and_settings_refresh(cx: &mut TestAppContext) {
cx.run_until_parked();
// The second request should have gone to model B.
- let model_b_completions = fake_model_b.pending_completions();
+ let model_b_completions = fake.pending_completions_for(&model_b);
assert_eq!(
model_b_completions.len(),
1,
@@ -8956,7 +9279,9 @@ struct SubagentCompactionTest {
thread: Entity<Thread>,
environment: NativeThreadEnvironment,
handle: Rc<dyn SubagentHandle>,
- model: Arc<FakeLanguageModel>,
+ fake: Arc<FakeLanguageModelProvider>,
+ /// The model the subagent thread sends to.
+ model: LanguageModel,
}
impl SubagentCompactionTest {
@@ -8965,9 +9290,8 @@ impl SubagentCompactionTest {
}
async fn new_with_files(files: serde_json::Value, cx: &mut TestAppContext) -> Self {
- init_test(cx);
+ let fake = init_test(cx);
cx.update(|cx| {
- LanguageModelRegistry::test(cx);
let mut settings = AgentSettings::get_global(cx).clone();
settings.auto_compact.enabled = true;
settings.auto_compact.threshold = AutoCompactThreshold::Percentage(0.9);
@@ -8992,7 +9316,7 @@ impl SubagentCompactionTest {
let parent = agent.read_with(cx, |agent, _| {
agent.sessions.get(&session_id).unwrap().thread.clone()
});
- let model = Arc::new(FakeLanguageModel::default());
+ let model = fake.model("fake");
parent.update(cx, |thread, cx| thread.set_model(model.clone(), cx));
let environment = NativeThreadEnvironment {
agent: agent.downgrade(),
@@ -9014,6 +9338,7 @@ impl SubagentCompactionTest {
thread,
environment,
handle,
+ fake,
model,
}
}
@@ -9030,8 +9355,13 @@ impl SubagentCompactionTest {
settings.auto_compact.enabled = enabled;
AgentSettings::override_global(settings, cx);
});
- self.model.set_max_token_count(max_tokens);
- self.model.set_max_output_tokens(max_output_tokens);
+ let model = self.fake.update_model("fake", |model| {
+ model.max_token_count = max_tokens;
+ model.max_input_tokens = max_tokens;
+ model.max_output_tokens = max_output_tokens;
+ });
+ self.thread
+ .update(cx, |thread, cx| thread.set_model(model, cx));
}
fn send(&self, message: &str, cx: &mut TestAppContext) -> Task<Result<String>> {
@@ -9041,40 +9371,39 @@ impl SubagentCompactionTest {
}
fn update_usage(&self, input_tokens: u64, cx: &mut TestAppContext) {
- self.model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
- TokenUsage {
- input_tokens,
- ..TokenUsage::default()
- },
- ));
+ self.fake.send_last_event(
+ &self.model,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens,
+ ..TokenUsage::default()
+ }),
+ );
cx.run_until_parked();
}
fn tool_round(&self, input_tokens: u64, cx: &mut TestAppContext) {
- self.model
- .send_last_completion_stream_text_chunk("partial work");
+ self.fake.send_last_text(&self.model, "partial work");
self.update_usage(input_tokens, cx);
self.assert_running(cx);
- self.model
- .send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
- LanguageModelToolUse {
- id: "echo_1".into(),
- name: EchoTool::NAME.into(),
- raw_input: r#"{"text":"tool output"}"#.to_string(),
- input: language_model::LanguageModelToolUseInput::Json(
- json!({"text": "tool output"}),
- ),
- is_input_complete: true,
- thought_signature: None,
- },
- ));
- self.model.end_last_completion_stream();
+ self.fake.send_last_event(
+ &self.model,
+ LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
+ id: "echo_1".into(),
+ name: EchoTool::NAME.into(),
+ raw_input: r#"{"text":"tool output"}"#.to_string(),
+ input: language_model::LanguageModelToolUseInput::Json(
+ json!({"text": "tool output"}),
+ ),
+ is_input_complete: true,
+ thought_signature: None,
+ }),
+ );
+ self.fake.end_last(&self.model);
cx.run_until_parked();
}
fn compaction_request(&self) -> LanguageModelRequest {
- let mut requests = self.model.pending_completions();
+ let mut requests = self.fake.pending_completions();
assert_eq!(requests.len(), 1);
let request = requests.pop().unwrap();
assert_eq!(
@@ -9091,9 +9420,9 @@ impl SubagentCompactionTest {
cx: &mut TestAppContext,
) {
let request = self.compaction_request();
- self.model
- .send_completion_stream_text_chunk(&request, "failed summary");
- self.model.send_completion_stream_error(
+ self.fake.send_text(&self.model, &request, "failed summary");
+ self.fake.send_error(
+ &self.model,
&request,
LanguageModelCompletionError::from_http_status(
LanguageModelProviderName::new("test"),
@@ -9102,7 +9431,7 @@ impl SubagentCompactionTest {
retry_after,
),
);
- self.model.end_completion_stream(&request);
+ self.fake.end_stream(&self.model, &request);
cx.run_until_parked();
}
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index f8dbde992b2..2969b1beb1a 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -1239,13 +1239,13 @@ enum CompletionError {
}
pub enum ThreadModel {
- Ready(Arc<dyn LanguageModel>),
+ Ready(LanguageModel),
Unresolved(SelectedModel),
Unset,
}
impl ThreadModel {
- fn as_model(&self) -> Option<&Arc<dyn LanguageModel>> {
+ fn as_model(&self) -> Option<&LanguageModel> {
match self {
Self::Ready(model) => Some(model),
Self::Unresolved(_) | Self::Unset => None,
@@ -1307,7 +1307,7 @@ pub struct Thread {
project_context: Entity<ProjectContext>,
pub(crate) templates: Arc<Templates>,
model: ThreadModel,
- summarization_model: Option<Arc<dyn LanguageModel>>,
+ summarization_model: Option<LanguageModel>,
thinking_enabled: bool,
thinking_effort: Option<String>,
speed: Option<Speed>,
@@ -1332,7 +1332,7 @@ pub struct Thread {
}
impl Thread {
- fn prompt_capabilities(model: Option<&dyn LanguageModel>) -> acp::PromptCapabilities {
+ fn prompt_capabilities(model: Option<&LanguageModel>) -> acp::PromptCapabilities {
let image = model.map_or(true, |model| model.supports_images());
acp::PromptCapabilities::new()
.image(image)
@@ -1381,7 +1381,7 @@ impl Thread {
project_context: Entity<ProjectContext>,
context_server_registry: Entity<ContextServerRegistry>,
templates: Arc<Templates>,
- model: Option<Arc<dyn LanguageModel>>,
+ model: Option<LanguageModel>,
cx: &mut Context<Self>,
) -> Self {
Self::new_internal(
@@ -1400,7 +1400,7 @@ impl Thread {
project_context: Entity<ProjectContext>,
context_server_registry: Entity<ContextServerRegistry>,
templates: Arc<Templates>,
- model: Option<Arc<dyn LanguageModel>>,
+ model: Option<LanguageModel>,
action_log: Entity<ActionLog>,
cx: &mut Context<Self>,
) -> Self {
@@ -1420,7 +1420,7 @@ impl Thread {
.as_ref()
.and_then(|model| model.speed);
let (prompt_capabilities_tx, prompt_capabilities_rx) =
- watch::channel(Self::prompt_capabilities(model.as_deref()));
+ watch::channel(Self::prompt_capabilities(model.as_ref()));
let model = match model {
Some(model) => ThreadModel::Ready(model),
None => Self::user_configured_model_selection(cx)
@@ -1508,7 +1508,7 @@ impl Thread {
self.thinking_effort = selection.effort.clone();
self.speed = selection.speed.filter(|_| model.supports_fast_mode());
self.prompt_capabilities_tx
- .send(Self::prompt_capabilities(Some(model.as_ref())))
+ .send(Self::prompt_capabilities(Some(&model)))
.log_err();
self.model = ThreadModel::Ready(model);
}
@@ -1784,7 +1784,6 @@ impl Thread {
saved_selection
.as_ref()
.and_then(|selection| registry.select_model(selection, cx))
- .map(|configured| configured.model)
});
let model = match (resolved_saved_model, saved_selection) {
@@ -1792,16 +1791,14 @@ impl Thread {
(None, Some(selection)) => ThreadModel::Unresolved(selection),
(None, None) => Self::resolve_profile_model(&profile_id, cx)
.or_else(|| {
- LanguageModelRegistry::global(cx).update(cx, |registry, _cx| {
- registry.default_model().map(|model| model.model)
- })
+ LanguageModelRegistry::global(cx)
+ .update(cx, |registry, _cx| registry.default_model())
})
.map_or(ThreadModel::Unset, ThreadModel::Ready),
};
- let (prompt_capabilities_tx, prompt_capabilities_rx) = watch::channel(
- Self::prompt_capabilities(model.as_model().map(|model| model.as_ref())),
- );
+ let (prompt_capabilities_tx, prompt_capabilities_rx) =
+ watch::channel(Self::prompt_capabilities(model.as_model()));
let action_log = cx.new(|_| ActionLog::new(project.clone()));
@@ -2009,7 +2006,7 @@ impl Thread {
self.ui_scroll_position = position;
}
- pub fn model(&self) -> Option<&Arc<dyn LanguageModel>> {
+ pub fn model(&self) -> Option<&LanguageModel> {
self.model.as_model()
}
@@ -2019,18 +2016,13 @@ impl Thread {
pub(crate) fn ensure_model(
&mut self,
- default_model: Option<&Arc<dyn LanguageModel>>,
+ default_model: Option<&LanguageModel>,
cx: &mut Context<Self>,
) {
let resolved = match &self.model {
ThreadModel::Ready(_) => return,
- ThreadModel::Unresolved(selection) => {
- LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
- registry
- .select_model(selection, cx)
- .map(|configured| configured.model)
- })
- }
+ ThreadModel::Unresolved(selection) => LanguageModelRegistry::global(cx)
+ .update(cx, |registry, cx| registry.select_model(selection, cx)),
ThreadModel::Unset => default_model.cloned(),
};
@@ -2039,10 +2031,10 @@ impl Thread {
}
}
- pub fn set_model(&mut self, model: Arc<dyn LanguageModel>, cx: &mut Context<Self>) {
+ pub fn set_model(&mut self, model: LanguageModel, cx: &mut Context<Self>) {
let old_usage = self.latest_token_usage();
self.model = ThreadModel::Ready(model.clone());
- let new_caps = Self::prompt_capabilities(self.model.as_model().map(|model| model.as_ref()));
+ let new_caps = Self::prompt_capabilities(self.model.as_model());
let new_usage = self.latest_token_usage();
if old_usage != new_usage {
cx.emit(TokenUsageUpdated(new_usage));
@@ -2063,13 +2055,37 @@ impl Thread {
cx.notify()
}
- pub fn summarization_model(&self) -> Option<&Arc<dyn LanguageModel>> {
+ /// Re-selects this thread's model from its provider when that provider's
+ /// state changes, so names and capabilities that update after a model is
+ /// created (e.g. a local model finishing loading) reach the thread.
+ pub fn refresh_model(&mut self, provider_id: &LanguageModelProviderId, cx: &mut Context<Self>) {
+ let Some(current) = self.model() else {
+ return;
+ };
+ if &current.provider_id != provider_id {
+ return;
+ }
+ let Some(model) = LanguageModelRegistry::read_global(cx)
+ .provider(provider_id)
+ .into_iter()
+ .flat_map(|provider| provider.provided_models(cx))
+ .find(|model| model.id == current.id)
+ else {
+ return;
+ };
+ if model == *current {
+ return;
+ }
+ self.set_model(model, cx);
+ }
+
+ pub fn summarization_model(&self) -> Option<&LanguageModel> {
self.summarization_model.as_ref()
}
pub fn set_summarization_model(
&mut self,
- model: Option<Arc<dyn LanguageModel>>,
+ model: Option<LanguageModel>,
cx: &mut Context<Self>,
) {
self.summarization_model = model.clone();
@@ -2495,7 +2511,7 @@ impl Thread {
fn resolve_profile_model(
profile_id: &AgentProfileId,
cx: &mut Context<Self>,
- ) -> Option<Arc<dyn LanguageModel>> {
+ ) -> Option<LanguageModel> {
let selection = AgentSettings::get_global(cx)
.profiles
.get(profile_id)?
@@ -2522,16 +2538,13 @@ impl Thread {
fn resolve_model_from_selection(
selection: &LanguageModelSelection,
cx: &mut Context<Self>,
- ) -> Option<Arc<dyn LanguageModel>> {
+ ) -> Option<LanguageModel> {
let selected = SelectedModel {
provider: LanguageModelProviderId::from(selection.provider.0.clone()),
model: LanguageModelId::from(selection.model.clone()),
};
- LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
- registry
- .select_model(&selected, cx)
- .map(|configured| configured.model)
- })
+ LanguageModelRegistry::global(cx)
+ .update(cx, |registry, cx| registry.select_model(&selected, cx))
}
pub fn resume(
@@ -2786,7 +2799,7 @@ impl Thread {
let mut attempt = 0;
let mut intent = CompletionIntent::UserPrompt;
// Set when a refusal fallback occurs so subsequent iterations use the fallback model.
- let mut refusal_fallback_model: Option<Arc<dyn LanguageModel>> = None;
+ let mut refusal_fallback_model: Option<LanguageModel> = None;
loop {
match Self::perform_compaction_if_needed(
this,
@@ -2869,15 +2882,16 @@ impl Thread {
// mid-turn changes (e.g. the user switches model, toggles tools,
// or changes profile) take effect between tool-call rounds.
// If a refusal fallback is active, use that model instead.
- let (model, request) = this.update(cx, |this, cx| {
+ let (model, provider, request) = this.update(cx, |this, cx| {
let model = refusal_fallback_model
.clone()
.or_else(|| this.model().cloned())
.ok_or_else(|| anyhow!(NoModelConfiguredError))?;
+ let provider = LanguageModelRegistry::read_global(cx).provider_for_model(&model);
this.refresh_turn_tools(cx);
let request = this.build_completion_request(intent, cx)?;
this.current_request_token_usage = TokenUsage::default();
- anyhow::Ok((model, request))
+ anyhow::Ok((model, provider, request))
})??;
telemetry::event!(
@@ -2894,7 +2908,11 @@ impl Thread {
log::debug!("Calling model.stream_completion, attempt {}", attempt);
- let (mut events, mut error) = match model.stream_completion(request, cx).await {
+ let events = match provider {
+ Ok(provider) => provider.stream_completion(&model, request, cx).await,
+ Err(error) => Err(error),
+ };
+ let (mut events, mut error) = match events {
Ok(events) => (events.fuse(), None),
Err(err) => (stream::empty().boxed().fuse(), Some(err)),
};
@@ -3019,8 +3037,10 @@ impl Thread {
})?;
if had_refusal {
- let maybe_fallback = this.update(cx, |this, cx| -> Option<Arc<dyn LanguageModel>> {
- let current_model = refusal_fallback_model.as_ref().or(this.model())?;
+ let maybe_fallback = this.update(cx, |this, cx| -> Option<LanguageModel> {
+ let current_model = refusal_fallback_model
+ .as_ref()
+ .or(this.model())?;
let fallback_id = match current_model.refusal_fallback_model_id() {
Some(id) => id,
None => {
@@ -3199,7 +3219,7 @@ impl Thread {
this: &WeakEntity<Self>,
event_stream: &ThreadEventStream,
mut cancellation_rx: watch::Receiver<bool>,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
request: LanguageModelRequest,
insertion: CompactionInsertion,
cx: &mut AsyncApp,
@@ -3215,8 +3235,10 @@ impl Thread {
acp_thread::ContextCompactionStatus::InProgress,
);
let result: Result<ControlFlow<()>> = async {
+ let provider =
+ cx.update(|cx| LanguageModelRegistry::read_global(cx).provider_for_model(&model))?;
let stream = futures::select! {
- result = model.stream_completion(request, cx).fuse() => result,
+ result = provider.stream_completion(&model, request, cx).fuse() => result,
_ = cancellation_rx.changed().fuse() => {
if *cancellation_rx.borrow() {
log::debug!("Compaction cancelled before request started");
@@ -3911,7 +3933,13 @@ impl Thread {
let task = cx
.spawn(async move |this, cx| {
let mut summary = String::new();
- let mut messages = model.stream_completion(request, cx).await.log_err()?;
+ let provider = cx
+ .update(|cx| LanguageModelRegistry::read_global(cx).provider_for_model(&model))
+ .log_err()?;
+ let mut messages = provider
+ .stream_completion(&model, request, cx)
+ .await
+ .log_err()?;
while let Some(event) = messages.next().await {
let event = event.log_err()?;
let text = match event {
@@ -3974,7 +4002,7 @@ impl Thread {
fn spawn_title_generation(
&mut self,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
on_generated_title: Option<Box<dyn FnOnce(SharedString, &mut Context<Self>)>>,
cx: &mut Context<Self>,
) {
@@ -4423,7 +4451,7 @@ impl Thread {
fn build_compaction_telemetry(
&self,
trigger: &'static str,
- compaction_model: &Arc<dyn LanguageModel>,
+ compaction_model: &LanguageModel,
cx: &App,
) -> Option<CompactionTelemetry> {
let model = self.model()?;
@@ -4438,8 +4466,8 @@ impl Thread {
thread_id: self.id.to_string(),
parent_thread_id: self.parent_thread_id().map(|id| id.to_string()),
prompt_id: self.prompt_id.to_string(),
- model: model.telemetry_id(),
- compaction_model: compaction_model.telemetry_id(),
+ model: model.telemetry_id.to_string(),
+ compaction_model: compaction_model.telemetry_id.to_string(),
thinking_effort: self.thinking_effort.clone(),
max_tokens,
tokens_before,
@@ -4524,24 +4552,23 @@ impl Thread {
Some(self.messages.len())
}
- fn compaction_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn compaction_model(&self, cx: &App) -> Option<LanguageModel> {
LanguageModelRegistry::read_global(cx)
.compaction_model()
- .map(|m| m.model)
.or_else(|| self.model().cloned())
}
fn build_compaction_request(
&self,
insertion_ix: usize,
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
cx: &App,
) -> LanguageModelRequest {
let mut request = LanguageModelRequest {
thread_id: Some(self.id.to_string()),
prompt_id: Some(self.prompt_id.to_string()),
intent: Some(CompletionIntent::ThreadContextSummarization),
- temperature: AgentSettings::temperature_for_model(model, cx),
+ temperature: AgentSettings::temperature_for_model(&model, cx),
messages: self.build_request_messages_until(Vec::new(), insertion_ix, cx),
..Default::default()
};
@@ -4612,6 +4639,8 @@ impl Thread {
// Retrying won't help until the user consents to data retention
// or switches models.
DataRetentionConsentRequired { .. } => None,
+ // Retrying won't help until the thread picks another model.
+ ModelUnavailable { .. } => None,
// `Other` includes mid-stream mapping failures that can be caused by
// a transient malformed or interrupted provider event.
Other(..) => Some(RetryStrategy::FixedDelay {
@@ -4984,12 +5013,14 @@ pub fn build_thread_title_request(
}
pub async fn stream_thread_title(
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
) -> Result<String> {
let mut title = String::new();
- let mut events = model.stream_completion(request, cx).await?;
+ let provider =
+ cx.update(|cx| LanguageModelRegistry::read_global(cx).provider_for_model(&model))?;
+ let mut events = provider.stream_completion(&model, request, cx).await?;
while let Some(event) = events.next().await {
let LanguageModelCompletionEvent::Text(text) = event? else {
continue;
@@ -6962,7 +6993,7 @@ mod tests {
use super::*;
use gpui::TestAppContext;
use language_model::LanguageModelToolUseId;
- use language_model::fake_provider::FakeLanguageModel;
+ use language_model::fake_provider::FakeLanguageModelProvider;
use serde_json::json;
use settings::LanguageModelProviderSetting;
use std::sync::Arc;
@@ -6991,12 +7022,18 @@ mod tests {
);
}
- async fn setup_thread_for_test(cx: &mut TestAppContext) -> (Entity<Thread>, ThreadEventStream) {
- cx.update(|cx| {
+ async fn setup_thread_for_test(
+ cx: &mut TestAppContext,
+ ) -> (
+ Entity<Thread>,
+ ThreadEventStream,
+ Arc<FakeLanguageModelProvider>,
+ ) {
+ let fake = cx.update(|cx| {
let settings_store = settings::SettingsStore::test(cx);
cx.set_global(settings_store);
- LanguageModelRegistry::test(cx);
+ LanguageModelRegistry::test(cx)
});
let fs = fs::FakeFs::new(cx.background_executor.clone());
@@ -7023,7 +7060,7 @@ mod tests {
let (event_tx, _event_rx) = mpsc::unbounded();
let event_stream = ThreadEventStream::new(event_tx);
- (thread, event_stream)
+ (thread, event_stream, fake)
})
}
@@ -7033,16 +7070,9 @@ mod tests {
AgentSettings::override_global(settings, cx);
}
- fn set_registry_compaction_model(cx: &mut App, model: Option<Arc<dyn LanguageModel>>) {
- use language_model::fake_provider::FakeLanguageModelProvider;
- use language_model::{ConfiguredModel, LanguageModelProvider};
+ fn set_registry_compaction_model(cx: &mut App, model: Option<LanguageModel>) {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
- let configured = model.map(|m| ConfiguredModel {
- provider: Arc::new(FakeLanguageModelProvider::default())
- as Arc<dyn LanguageModelProvider>,
- model: m,
- });
- registry.set_compaction_model(configured, cx);
+ registry.set_compaction_model(model, cx);
});
}
@@ -7233,9 +7263,9 @@ mod tests {
#[gpui::test]
async fn test_thread_summary_request_uses_compacted_history(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let thread_id = thread.read_with(cx, |thread, _| thread.id().to_string());
- let summary_model = Arc::new(FakeLanguageModel::default());
+ let summary_model = fake.model("summary");
let summary_task = cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -7263,7 +7293,7 @@ mod tests {
});
cx.run_until_parked();
- let summary_request = summary_model.pending_completions().pop().unwrap();
+ let summary_request = fake.pending_completions().pop().unwrap();
assert_eq!(
summary_request.thread_id.as_deref(),
Some(thread_id.as_str())
@@ -7284,8 +7314,8 @@ mod tests {
]
);
- summary_model.send_completion_stream_text_chunk(&summary_request, "thread summary");
- summary_model.end_completion_stream(&summary_request);
+ fake.send_text(&summary_model, &summary_request, "thread summary");
+ fake.end_stream(&summary_model, &summary_request);
assert_eq!(summary_task.await.as_deref(), Some("thread summary"));
}
@@ -7323,13 +7353,13 @@ mod tests {
#[gpui::test]
async fn test_compaction_threshold_uses_percentage_setting(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread
.messages
.push(user_text_message(user_message_id.clone(), "below limit"));
@@ -7358,14 +7388,13 @@ mod tests {
#[gpui::test]
async fn test_compaction_threshold_accounts_for_max_output_tokens(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
- model.set_max_output_tokens(Some(32_000));
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| model.max_output_tokens = Some(32_000));
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread.messages.push(user_text_message(
user_message_id.clone(),
"near input limit",
@@ -7422,12 +7451,12 @@ mod tests {
#[gpui::test]
async fn test_compaction_threshold_respects_independent_input_limit(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let mut model = FakeLanguageModel::default();
- model.set_max_token_count(200_000);
- model.set_max_input_tokens(90_000);
- model.set_max_output_tokens(Some(16_384));
- let model = Arc::new(model);
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| {
+ model.max_token_count = 200_000;
+ model.max_input_tokens = 90_000;
+ model.max_output_tokens = Some(16_384);
+ });
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
@@ -7439,7 +7468,7 @@ mod tests {
},
);
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread.messages.push(user_text_message(
user_message_id.clone(),
"near the independent input limit",
@@ -7463,8 +7492,8 @@ mod tests {
#[gpui::test]
async fn test_compaction_threshold_respects_enabled_setting(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
@@ -7476,7 +7505,7 @@ mod tests {
},
);
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread
.messages
.push(user_text_message(user_message_id.clone(), "near limit"));
@@ -7495,8 +7524,8 @@ mod tests {
#[gpui::test]
async fn test_compaction_threshold_respects_token_settings(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
@@ -7508,7 +7537,7 @@ mod tests {
},
);
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread.messages.push(user_text_message(
user_message_id.clone(),
"fixed token limit",
@@ -7565,15 +7594,17 @@ mod tests {
#[gpui::test]
async fn test_compaction_unavailable_for_small_context_window(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
// A context window below the minimum disables auto-compaction.
- model.set_max_token_count(MIN_COMPACTION_CONTEXT_WINDOW - 1);
+ let model = fake.update_model("fake", |model| {
+ model.max_token_count = MIN_COMPACTION_CONTEXT_WINDOW - 1;
+ model.max_input_tokens = MIN_COMPACTION_CONTEXT_WINDOW - 1;
+ });
let user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
thread.update(cx, |thread, cx| {
- thread.set_model(model, cx);
+ thread.set_model(model.clone(), cx);
thread
.messages
.push(user_text_message(user_message_id.clone(), "near limit"));
@@ -7594,8 +7625,8 @@ mod tests {
async fn test_compaction_inserts_before_new_user_and_requests_compacted_window(
cx: &mut TestAppContext,
) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
let old_user_message_id = ClientUserMessageId::new();
let new_user_message_id = ClientUserMessageId::new();
@@ -7625,7 +7656,7 @@ mod tests {
.unwrap();
cx.run_until_parked();
- let compaction_request = model.pending_completions().pop().unwrap();
+ let compaction_request = fake.pending_completions().pop().unwrap();
assert_eq!(
compaction_request.intent,
Some(CompletionIntent::ThreadContextSummarization)
@@ -7636,11 +7667,11 @@ mod tests {
assert_eq!(compaction_texts[1], "old assistant");
assert_eq!(compaction_texts[2], COMPACTION_PROMPT);
- model.send_completion_stream_text_chunk(&compaction_request, "compacted old context");
- model.end_completion_stream(&compaction_request);
+ fake.send_text(&model, &compaction_request, "compacted old context");
+ fake.end_stream(&model, &compaction_request);
cx.run_until_parked();
- let final_request = model.pending_completions().pop().unwrap();
+ let final_request = fake.pending_completions().pop().unwrap();
assert_eq!(final_request.intent, Some(CompletionIntent::UserPrompt));
assert_eq!(
request_texts_after_system(&final_request.messages),
@@ -7651,8 +7682,8 @@ mod tests {
]
);
- model.send_completion_stream_text_chunk(&final_request, "answer");
- model.end_completion_stream(&final_request);
+ fake.send_text(&model, &final_request, "answer");
+ fake.end_stream(&model, &final_request);
cx.run_until_parked();
cx.update(|cx| {
@@ -7670,11 +7701,13 @@ mod tests {
#[gpui::test]
async fn test_manual_compact_forces_summary(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
// A context window below the minimum and no recorded token usage would
// both disable *automatic* compaction. Manual compaction forces it anyway.
- model.set_max_token_count(MIN_COMPACTION_CONTEXT_WINDOW - 1);
+ let model = fake.update_model("fake", |model| {
+ model.max_token_count = MIN_COMPACTION_CONTEXT_WINDOW - 1;
+ model.max_input_tokens = MIN_COMPACTION_CONTEXT_WINDOW - 1;
+ });
let user_message_id = ClientUserMessageId::new();
let compact_message_id = ClientUserMessageId::new();
@@ -7699,7 +7732,7 @@ mod tests {
.unwrap();
cx.run_until_parked();
- let compaction_request = model.pending_completions().pop().unwrap();
+ let compaction_request = fake.pending_completions().pop().unwrap();
assert_eq!(
compaction_request.intent,
Some(CompletionIntent::ThreadContextSummarization)
@@ -7710,14 +7743,14 @@ mod tests {
assert_eq!(compaction_texts[1], "old assistant");
assert_eq!(compaction_texts[2], COMPACTION_PROMPT);
- model.send_completion_stream_text_chunk(&compaction_request, "summary of old context");
- model.end_completion_stream(&compaction_request);
+ fake.send_text(&model, &compaction_request, "summary of old context");
+ fake.end_stream(&model, &compaction_request);
cx.run_until_parked();
// The compaction summary is appended after a zero-content user message
// marker, and no follow-up model turn is requested — `/compact` only
// compacts.
- assert!(model.pending_completions().is_empty());
+ assert!(fake.pending_completions().is_empty());
cx.update(|cx| {
thread.read_with(cx, |thread, _cx| {
assert!(matches!(&*thread.messages[0], Message::User(_)));
@@ -7751,8 +7784,8 @@ mod tests {
/// rewind marker (or a partial summary) dangling at the end of the thread.
#[gpui::test]
async fn test_manual_compact_cancelled_leaves_no_marker(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -7773,7 +7806,7 @@ mod tests {
.unwrap();
cx.run_until_parked();
// The compaction request is in flight but hasn't streamed a summary.
- assert_eq!(model.pending_completions().len(), 1);
+ assert_eq!(fake.pending_completions().len(), 1);
cx.update(|cx| thread.update(cx, |thread, cx| thread.cancel(cx)))
.await;
@@ -7790,8 +7823,8 @@ mod tests {
/// the thread untouched — no marker, no compaction.
#[gpui::test]
async fn test_manual_compact_empty_summary_leaves_no_marker(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -7812,9 +7845,9 @@ mod tests {
.unwrap();
cx.run_until_parked();
- let request = model.pending_completions().pop().unwrap();
+ let request = fake.pending_completions().pop().unwrap();
// End the stream without emitting any summary text.
- model.end_completion_stream(&request);
+ fake.end_stream(&model, &request);
cx.run_until_parked();
// An error is surfaced, and the thread is left exactly as it was. The
@@ -7838,8 +7871,8 @@ mod tests {
/// issues no model request and adds no marker.
#[gpui::test]
async fn test_manual_compact_noop_on_empty_thread(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
cx.update(|cx| thread.update(cx, |thread, cx| thread.set_model(model.clone(), cx)));
let _events = cx
@@ -7851,7 +7884,7 @@ mod tests {
.unwrap();
cx.run_until_parked();
- assert!(model.pending_completions().is_empty());
+ assert!(fake.pending_completions().is_empty());
thread.read_with(cx, |thread, _cx| {
assert!(thread.messages.is_empty());
});
@@ -7862,7 +7895,7 @@ mod tests {
/// a compacted thread doesn't surface an empty `/compact` bubble.
#[gpui::test]
async fn test_manual_compact_marker_replays_as_empty_user_message(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, _) = setup_thread_for_test(cx).await;
let marker_id = ClientUserMessageId::new();
let mut replay_events = cx.update(|cx| {
@@ -7907,9 +7940,9 @@ mod tests {
/// to the configured model rather than the thread's primary model.
#[gpui::test]
async fn test_compaction_uses_configured_compaction_model(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let thread_model = Arc::new(FakeLanguageModel::default());
- let compaction_model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let thread_model = fake.model("fake");
+ let compaction_model = fake.model("compaction");
cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -7919,7 +7952,8 @@ mod tests {
.push(user_text_message(ClientUserMessageId::new(), "old user"));
thread.messages.push(agent_text_message("old assistant"));
});
- set_registry_compaction_model(cx, Some(compaction_model.clone()));
+ let compaction_model = compaction_model.clone();
+ set_registry_compaction_model(cx, Some(compaction_model));
});
let _events = cx
@@ -7932,12 +7966,12 @@ mod tests {
cx.run_until_parked();
assert_eq!(
- thread_model.pending_completions().len(),
+ fake.pending_completions_for(&thread_model).len(),
0,
"thread's primary model should not have been used for compaction"
);
- let request = compaction_model
- .pending_completions()
+ let request = fake
+ .pending_completions_for(&compaction_model)
.pop()
.expect("compaction model should have received the request");
assert_eq!(
@@ -7953,8 +7987,8 @@ mod tests {
assert_eq!(telemetry.model, compaction_model.telemetry_id());
});
- compaction_model.send_completion_stream_text_chunk(&request, "summary");
- compaction_model.end_completion_stream(&request);
+ fake.send_text(&compaction_model, &request, "summary");
+ fake.end_stream(&compaction_model, &request);
cx.run_until_parked();
}
@@ -7964,8 +7998,8 @@ mod tests {
/// model — not the one the user tried to configure.
#[gpui::test]
async fn test_compaction_falls_back_when_compaction_model_unavailable(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let thread_model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let thread_model = fake.model("fake");
cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -7997,8 +8031,8 @@ mod tests {
.unwrap();
cx.run_until_parked();
- let request = thread_model
- .pending_completions()
+ let request = fake
+ .pending_completions_for(&thread_model)
.pop()
.expect("thread model should have received the fallback request");
assert_eq!(
@@ -8014,8 +8048,8 @@ mod tests {
assert_eq!(telemetry.model, thread_model.telemetry_id());
});
- thread_model.send_completion_stream_text_chunk(&request, "summary");
- thread_model.end_completion_stream(&request);
+ fake.send_text(&thread_model, &request, "summary");
+ fake.end_stream(&thread_model, &request);
cx.run_until_parked();
}
@@ -8023,9 +8057,9 @@ mod tests {
/// `agent.compaction_model`.
#[gpui::test]
async fn test_auto_compaction_uses_compaction_model(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let thread_model = Arc::new(FakeLanguageModel::default());
- let compaction_model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let thread_model = fake.model("fake");
+ let compaction_model = fake.model("compaction");
let old_user_message_id = ClientUserMessageId::new();
cx.update(|cx| {
@@ -8050,7 +8084,8 @@ mod tests {
threshold: agent_settings::AutoCompactThreshold::Percentage(0.5),
},
);
- set_registry_compaction_model(cx, Some(compaction_model.clone()));
+ let compaction_model = compaction_model.clone();
+ set_registry_compaction_model(cx, Some(compaction_model));
});
// The auto-compact gate fires inside `run_turn` when we kick off a
@@ -8065,9 +8100,9 @@ mod tests {
.unwrap();
cx.run_until_parked();
- assert_eq!(thread_model.pending_completions().len(), 0);
- let request = compaction_model
- .pending_completions()
+ assert_eq!(fake.pending_completions_for(&thread_model).len(), 0);
+ let request = fake
+ .pending_completions_for(&compaction_model)
.pop()
.expect("compaction model should have received the auto-compaction request");
assert_eq!(
@@ -8075,15 +8110,15 @@ mod tests {
Some(CompletionIntent::ThreadContextSummarization)
);
- compaction_model.send_completion_stream_text_chunk(&request, "summary");
- compaction_model.end_completion_stream(&request);
+ fake.send_text(&compaction_model, &request, "summary");
+ fake.end_stream(&compaction_model, &request);
cx.run_until_parked();
}
#[gpui::test]
async fn test_compaction_usage_counts_toward_cumulative_usage(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
- let model = Arc::new(FakeLanguageModel::default());
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
let old_user_message_id = ClientUserMessageId::new();
let new_user_message_id = ClientUserMessageId::new();
let prior_usage = TokenUsage {
@@ -8127,13 +8162,14 @@ mod tests {
.unwrap();
cx.run_until_parked();
- let compaction_request = model.pending_completions().pop().unwrap();
+ let compaction_request = fake.pending_completions().pop().unwrap();
assert_eq!(
compaction_request.intent,
Some(CompletionIntent::ThreadContextSummarization)
);
- model.send_completion_stream_event(
+ fake.send_event(
+ &model,
&compaction_request,
LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: 40,
@@ -8141,12 +8177,13 @@ mod tests {
..Default::default()
}),
);
- model.send_completion_stream_event(
+ fake.send_event(
+ &model,
&compaction_request,
LanguageModelCompletionEvent::UsageUpdate(compaction_usage),
);
- model.send_completion_stream_text_chunk(&compaction_request, "compacted old context");
- model.end_completion_stream(&compaction_request);
+ fake.send_text(&model, &compaction_request, "compacted old context");
+ fake.end_stream(&model, &compaction_request);
cx.run_until_parked();
let expected_after_compaction = prior_usage + compaction_usage;
@@ -8159,14 +8196,15 @@ mod tests {
);
});
- let final_request = model.pending_completions().pop().unwrap();
+ let final_request = fake.pending_completions().pop().unwrap();
assert_eq!(final_request.intent, Some(CompletionIntent::UserPrompt));
- model.send_completion_stream_event(
+ fake.send_event(
+ &model,
&final_request,
LanguageModelCompletionEvent::UsageUpdate(final_usage),
);
- model.end_completion_stream(&final_request);
+ fake.end_stream(&model, &final_request);
cx.run_until_parked();
thread.read_with(cx, |thread, _cx| {
@@ -8183,7 +8221,7 @@ mod tests {
#[gpui::test]
async fn test_replay_emits_context_compaction(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, _) = setup_thread_for_test(cx).await;
let user_message_id = ClientUserMessageId::new();
let mut replay_events = cx.update(|cx| {
@@ -8232,7 +8270,7 @@ mod tests {
#[gpui::test]
async fn test_native_compaction_boundary(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, _) = setup_thread_for_test(cx).await;
let request_messages = cx.update(|cx| {
thread.update(cx, |thread, cx| {
@@ -8263,7 +8301,7 @@ mod tests {
#[gpui::test]
async fn test_retained_users_truncate_oldest(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, _) = setup_thread_for_test(cx).await;
let mut long_text = "START".to_string();
long_text.push_str(&"x".repeat(COMPACTION_RETAINED_USER_MESSAGES_BYTE_BUDGET));
long_text.push_str("END");
@@ -8860,7 +8898,7 @@ mod tests {
#[gpui::test]
async fn test_replay_tool_call_replays_image_content(cx: &mut TestAppContext) {
- let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let (thread, _event_stream, _) = setup_thread_for_test(cx).await;
let registered_tool_use_id = LanguageModelToolUseId::from("registered_tool_id");
let missing_tool_use_id = LanguageModelToolUseId::from("missing_tool_id");
@@ -8976,21 +9014,44 @@ mod tests {
);
}
+ #[gpui::test]
+ async fn test_refresh_model_picks_up_provider_capability_changes(cx: &mut TestAppContext) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.model("fake");
+ let provider_id = model.provider_id.clone();
+ let model_changes = Rc::new(std::cell::Cell::new(0));
+ let _subscription = cx.update(|cx| {
+ thread.update(cx, |thread, cx| thread.set_model(model, cx));
+ let model_changes = model_changes.clone();
+ cx.subscribe(&thread, move |_, _: &ModelChanged, _| {
+ model_changes.set(model_changes.get() + 1)
+ })
+ });
+
+ thread.update(cx, |thread, cx| thread.refresh_model(&provider_id, cx));
+ assert_eq!(model_changes.get(), 0, "an unchanged model is not re-set");
+
+ fake.update_model("fake", |model| model.supports_images = true);
+ thread.update(cx, |thread, cx| {
+ thread.refresh_model(&LanguageModelProviderId::from("other".to_string()), cx);
+ assert!(!thread.model().unwrap().supports_images());
+
+ thread.refresh_model(&provider_id, cx);
+ assert!(thread.model().unwrap().supports_images());
+ });
+ assert_eq!(model_changes.get(), 1);
+ }
+
#[gpui::test]
async fn test_set_model_propagates_to_subagents(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, fake) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 2);
- let new_model: Arc<dyn LanguageModel> = Arc::new(FakeLanguageModel::with_id_and_thinking(
- "test-provider",
- "new-model",
- "New Model",
- false,
- ));
+ let new_model = fake.model("new-model");
cx.update(|cx| {
parent.update(cx, |thread, cx| {
- thread.set_model(new_model, cx);
+ thread.set_model(new_model.clone(), cx);
});
for subagent in &subagents {
@@ -9006,20 +9067,14 @@ mod tests {
#[gpui::test]
async fn test_set_summarization_model_propagates_to_subagents(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, fake) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 2);
- let summary_model: Arc<dyn LanguageModel> =
- Arc::new(FakeLanguageModel::with_id_and_thinking(
- "test-provider",
- "summary-model",
- "Summary Model",
- false,
- ));
+ let summary_model = fake.model("summary-model");
cx.update(|cx| {
parent.update(cx, |thread, cx| {
- thread.set_summarization_model(Some(summary_model), cx);
+ thread.set_summarization_model(Some(summary_model.clone()), cx);
});
for subagent in &subagents {
@@ -9035,7 +9090,7 @@ mod tests {
#[gpui::test]
async fn test_set_thinking_enabled_propagates_to_subagents(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, _) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 2);
cx.update(|cx| {
@@ -9065,7 +9120,7 @@ mod tests {
#[gpui::test]
async fn test_set_thinking_effort_propagates_to_subagents(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, _) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 2);
cx.update(|cx| {
@@ -9097,7 +9152,7 @@ mod tests {
#[gpui::test]
async fn test_subagent_inherits_settings_at_creation(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, _) = setup_thread_for_test(cx).await;
cx.update(|cx| {
parent.update(cx, |thread, cx| {
@@ -9121,7 +9176,7 @@ mod tests {
#[gpui::test]
async fn test_set_speed_propagates_to_subagents(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, _) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 2);
cx.update(|cx| {
@@ -9141,7 +9196,7 @@ mod tests {
#[gpui::test]
async fn test_dropped_subagent_does_not_panic(cx: &mut TestAppContext) {
- let (parent, _event_stream) = setup_thread_for_test(cx).await;
+ let (parent, _event_stream, _) = setup_thread_for_test(cx).await;
let subagents = setup_parent_with_subagents(cx, &parent, 1);
// Drop the subagent so the WeakEntity can no longer be upgraded
@@ -9161,7 +9216,7 @@ mod tests {
async fn test_handle_tool_use_json_parse_error_adds_tool_use_to_content(
cx: &mut TestAppContext,
) {
- let (thread, event_stream) = setup_thread_for_test(cx).await;
+ let (thread, event_stream, _) = setup_thread_for_test(cx).await;
let tool_use_id = LanguageModelToolUseId::from("test_tool_id");
let tool_name: Arc<str> = Arc::from("test_tool");
diff --git a/crates/agent/src/tools/edit_file_tool.rs b/crates/agent/src/tools/edit_file_tool.rs
index 64f6af0a49c..e645273d799 100644
--- a/crates/agent/src/tools/edit_file_tool.rs
+++ b/crates/agent/src/tools/edit_file_tool.rs
@@ -289,7 +289,7 @@ mod tests {
use crate::{ContextServerRegistry, Templates, ToolInputSender};
use fs::Fs as _;
use gpui::{AppContext as _, TestAppContext, UpdateGlobal};
- use language_model::fake_provider::FakeLanguageModel;
+ use language_model::LanguageModelRegistry;
use project::ProjectPath;
use prompt_store::ProjectContext;
use serde_json::json;
@@ -3089,7 +3089,7 @@ mod tests {
let language_registry = project.read_with(cx, |project, _cx| project.languages().clone());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(project.read(cx).context_server_store(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = cx.update(|cx| LanguageModelRegistry::test(cx).model("fake"));
let thread = cx.new(|cx| {
crate::Thread::new(
project.clone(),
diff --git a/crates/agent/src/tools/evals/edit_file.rs b/crates/agent/src/tools/evals/edit_file.rs
index 124664fd39d..9ec852b3f9f 100644
--- a/crates/agent/src/tools/evals/edit_file.rs
+++ b/crates/agent/src/tools/evals/edit_file.rs
@@ -75,7 +75,7 @@ trait AssertionFn: 'static + Send + Sync {
fn assert<'a>(
&'a self,
sample: &'a EvalSample,
- judge_model: Arc<dyn LanguageModel>,
+ judge_model: LanguageModel,
cx: &'a mut TestAppContext,
) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>>;
}
@@ -85,16 +85,12 @@ where
F: 'static
+ Send
+ Sync
- + AsyncFn(
- &EvalSample,
- Arc<dyn LanguageModel>,
- &mut TestAppContext,
- ) -> Result<EvalAssertionOutcome>,
+ + AsyncFn(&EvalSample, LanguageModel, &mut TestAppContext) -> Result<EvalAssertionOutcome>,
{
fn assert<'a>(
&'a self,
sample: &'a EvalSample,
- judge_model: Arc<dyn LanguageModel>,
+ judge_model: LanguageModel,
cx: &'a mut TestAppContext,
) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>> {
(self)(sample, judge_model, cx).boxed_local()
@@ -110,11 +106,7 @@ impl EvalAssertion {
F: 'static
+ Send
+ Sync
- + AsyncFn(
- &EvalSample,
- Arc<dyn LanguageModel>,
- &mut TestAppContext,
- ) -> Result<EvalAssertionOutcome>,
+ + AsyncFn(&EvalSample, LanguageModel, &mut TestAppContext) -> Result<EvalAssertionOutcome>,
{
EvalAssertion(Arc::new(f))
}
@@ -159,9 +151,11 @@ impl EvalAssertion {
.map(|effort_level| effort_level.value.to_string()),
..Default::default()
};
+ let provider =
+ cx.update(|cx| LanguageModelRegistry::read_global(cx).provider_for_model(&judge))?;
let mut response = retry_on_rate_limit(async || {
- Ok(judge
- .stream_completion_text(request.clone(), &cx.to_async())
+ Ok(provider
+ .stream_completion_text(&judge, request.clone(), &cx.to_async())
.await?)
})
.await?;
@@ -190,7 +184,7 @@ impl EvalAssertion {
async fn run(
&self,
input: &EvalSample,
- judge_model: Arc<dyn LanguageModel>,
+ judge_model: LanguageModel,
cx: &mut TestAppContext,
) -> Result<EvalAssertionOutcome> {
self.0.assert(input, judge_model, cx).await
@@ -224,8 +218,8 @@ struct EvalAssertionOutcome {
struct EditToolTest {
fs: Arc<FakeFs>,
project: Entity<Project>,
- model: Arc<dyn LanguageModel>,
- judge_model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
+ judge_model: LanguageModel,
model_thinking_effort: Option<String>,
}
@@ -309,7 +303,7 @@ impl EditToolTest {
async fn load_model(
selected_model: &SelectedModel,
cx: &mut AsyncApp,
- ) -> Result<Arc<dyn LanguageModel>> {
+ ) -> Result<LanguageModel> {
cx.update(|cx| {
let registry = LanguageModelRegistry::read_global(cx);
let provider = registry
@@ -319,13 +313,14 @@ impl EditToolTest {
})
.await?;
Ok(cx.update(|cx| {
- let models = LanguageModelRegistry::read_global(cx);
- models
- .available_models(cx)
- .find(|model| {
- model.provider_id() == selected_model.provider
- && model.id() == selected_model.model
- })
+ let registry = LanguageModelRegistry::read_global(cx);
+ let provider = registry
+ .provider(&selected_model.provider)
+ .expect("Provider not found");
+ provider
+ .provided_models(cx)
+ .into_iter()
+ .find(|model| model.id() == selected_model.model)
.unwrap_or_else(|| panic!("Model {} not found", selected_model.model.0))
}))
}
@@ -482,9 +477,13 @@ impl EditToolTest {
let model = self.model.clone();
let events = cx
.update(|cx| {
+ let provider = LanguageModelRegistry::read_global(cx).provider_for_model(&model);
let async_cx = cx.to_async();
- cx.foreground_executor()
- .spawn(async move { model.stream_completion(request, &async_cx).await })
+ cx.foreground_executor().spawn(async move {
+ provider?
+ .stream_completion(&model, request, &async_cx)
+ .await
+ })
})
.await
.map_err(|err| anyhow::anyhow!("completion error: {}", err))?;
diff --git a/crates/agent/src/tools/evals/terminal_tool.rs b/crates/agent/src/tools/evals/terminal_tool.rs
index 03b6c3a07c2..8e5bcbc49a3 100644
--- a/crates/agent/src/tools/evals/terminal_tool.rs
+++ b/crates/agent/src/tools/evals/terminal_tool.rs
@@ -145,7 +145,7 @@ impl Display for EvalOutput {
}
struct TerminalToolTest {
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
model_thinking_effort: Option<String>,
}
@@ -274,10 +274,7 @@ impl TerminalToolTest {
}
}
-async fn load_model(
- selected_model: &SelectedModel,
- cx: &mut AsyncApp,
-) -> Result<Arc<dyn LanguageModel>> {
+async fn load_model(selected_model: &SelectedModel, cx: &mut AsyncApp) -> Result<LanguageModel> {
cx.update(|cx| {
let registry = LanguageModelRegistry::read_global(cx);
let provider = registry
@@ -287,12 +284,14 @@ async fn load_model(
})
.await?;
Ok(cx.update(|cx| {
- let models = LanguageModelRegistry::read_global(cx);
- models
- .available_models(cx)
- .find(|model| {
- model.provider_id() == selected_model.provider && model.id() == selected_model.model
- })
+ let registry = LanguageModelRegistry::read_global(cx);
+ let provider = registry
+ .provider(&selected_model.provider)
+ .expect("Provider not found");
+ provider
+ .provided_models(cx)
+ .into_iter()
+ .find(|model| model.id() == selected_model.model)
.unwrap_or_else(|| panic!("Model {} not found", selected_model.model.0))
}))
}
@@ -300,16 +299,20 @@ async fn load_model(
/// Stream the model completion and extract the first complete tool use whose
/// name matches `TerminalTool::NAME`, parsed as `TerminalToolInput`.
async fn extract_tool_use(
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &mut TestAppContext,
) -> Result<TerminalToolInput> {
let model = model.clone();
let events = cx
.update(|cx| {
+ let provider = LanguageModelRegistry::read_global(cx).provider_for_model(&model);
let async_cx = cx.to_async();
- cx.foreground_executor()
- .spawn(async move { model.stream_completion(request, &async_cx).await })
+ cx.foreground_executor().spawn(async move {
+ provider?
+ .stream_completion(&model, request, &async_cx)
+ .await
+ })
})
.await
.map_err(|err| anyhow::anyhow!("completion error: {}", err))?;
diff --git a/crates/agent/src/tools/evals/write_file.rs b/crates/agent/src/tools/evals/write_file.rs
index 4c471ab61af..6e93c4a8879 100644
--- a/crates/agent/src/tools/evals/write_file.rs
+++ b/crates/agent/src/tools/evals/write_file.rs
@@ -67,7 +67,7 @@ impl Display for WriteEvalOutput {
struct WriteToolTest {
fs: Arc<FakeFs>,
project: Entity<Project>,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
model_thinking_effort: Option<String>,
}
@@ -143,7 +143,7 @@ impl WriteToolTest {
async fn load_model(
selected_model: &SelectedModel,
cx: &mut AsyncApp,
- ) -> Result<Arc<dyn LanguageModel>> {
+ ) -> Result<LanguageModel> {
cx.update(|cx| {
let registry = LanguageModelRegistry::read_global(cx);
let provider = registry
@@ -153,13 +153,14 @@ impl WriteToolTest {
})
.await?;
Ok(cx.update(|cx| {
- let models = LanguageModelRegistry::read_global(cx);
- models
- .available_models(cx)
- .find(|model| {
- model.provider_id() == selected_model.provider
- && model.id() == selected_model.model
- })
+ let registry = LanguageModelRegistry::read_global(cx);
+ let provider = registry
+ .provider(&selected_model.provider)
+ .expect("Provider not found");
+ provider
+ .provided_models(cx)
+ .into_iter()
+ .find(|model| model.id() == selected_model.model)
.unwrap_or_else(|| panic!("Model {} not found", selected_model.model.0))
}))
}
@@ -302,9 +303,13 @@ impl WriteToolTest {
let model = self.model.clone();
let events = cx
.update(|cx| {
+ let provider = LanguageModelRegistry::read_global(cx).provider_for_model(&model);
let async_cx = cx.to_async();
- cx.foreground_executor()
- .spawn(async move { model.stream_completion(request, &async_cx).await })
+ cx.foreground_executor().spawn(async move {
+ provider?
+ .stream_completion(&model, request, &async_cx)
+ .await
+ })
})
.await
.map_err(|err| anyhow::anyhow!("completion error: {}", err))?;
diff --git a/crates/agent/src/tools/write_file_tool.rs b/crates/agent/src/tools/write_file_tool.rs
index 59561556947..451a0416aa7 100644
--- a/crates/agent/src/tools/write_file_tool.rs
+++ b/crates/agent/src/tools/write_file_tool.rs
@@ -274,7 +274,7 @@ mod tests {
use futures::StreamExt as _;
use gpui::{AppContext as _, Entity, TestAppContext, UpdateGlobal};
use language::language_settings::FormatOnSave;
- use language_model::fake_provider::FakeLanguageModel;
+ use language_model::LanguageModelRegistry;
use project::{Project, ProjectPath};
use prompt_store::ProjectContext;
use serde_json::json;
@@ -1382,7 +1382,7 @@ mod tests {
let language_registry = project.read_with(cx, |project, _cx| project.languages().clone());
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(project.read(cx).context_server_store(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = cx.update(|cx| LanguageModelRegistry::test(cx).model("fake"));
let thread = cx.new(|cx| {
crate::Thread::new(
project.clone(),
diff --git a/crates/agent_settings/src/agent_settings.rs b/crates/agent_settings/src/agent_settings.rs
index 193cc5a8c0f..56f97f643eb 100644
--- a/crates/agent_settings/src/agent_settings.rs
+++ b/crates/agent_settings/src/agent_settings.rs
@@ -263,7 +263,7 @@ impl AgentSettings {
self.enabled && !DisableAiSettings::get_global(cx).disable_ai
}
- pub fn temperature_for_model(model: &Arc<dyn LanguageModel>, cx: &App) -> Option<f32> {
+ pub fn temperature_for_model(model: &LanguageModel, cx: &App) -> Option<f32> {
let settings = Self::get_global(cx);
for setting in settings.model_parameters.iter().rev() {
if let Some(provider) = &setting.provider
@@ -301,7 +301,7 @@ impl AgentSettings {
}
pub fn language_model_to_selection(
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
override_selection: Option<&LanguageModelSelection>,
) -> LanguageModelSelection {
let provider = model.provider_id().0.to_string().into();
diff --git a/crates/agent_ui/src/agent_configuration/manage_profiles_modal.rs b/crates/agent_ui/src/agent_configuration/manage_profiles_modal.rs
index d38db4a0855..314eb7afae2 100644
--- a/crates/agent_ui/src/agent_configuration/manage_profiles_modal.rs
+++ b/crates/agent_ui/src/agent_configuration/manage_profiles_modal.rs
@@ -108,7 +108,7 @@ pub struct NewProfileMode {
pub struct ManageProfilesModal {
fs: Arc<dyn Fs>,
context_server_registry: Entity<ContextServerRegistry>,
- active_model: Option<Arc<dyn LanguageModel>>,
+ active_model: Option<LanguageModel>,
focus_handle: FocusHandle,
mode: Mode,
_settings_subscription: Subscription,
@@ -144,7 +144,7 @@ impl ManageProfilesModal {
pub fn new(
fs: Arc<dyn Fs>,
- active_model: Option<Arc<dyn LanguageModel>>,
+ active_model: Option<LanguageModel>,
context_server_registry: Entity<ContextServerRegistry>,
window: &mut Window,
cx: &mut Context<Self>,
@@ -244,13 +244,11 @@ impl ManageProfilesModal {
let provider_id = language_model::LanguageModelProviderId(
gpui::SharedString::from(selection.provider.0.clone()),
);
- let provider = registry.provider(&provider_id)?;
- let model = provider
+ registry
+ .provider(&provider_id)?
.provided_models(cx)
- .iter()
- .find(|m| m.id().0 == selection.model.as_str())?
- .clone();
- Some(language_model::ConfiguredModel { provider, model })
+ .into_iter()
+ .find(|m| m.id().0 == selection.model.as_str())
})
}
},
diff --git a/crates/agent_ui/src/agent_model_selector.rs b/crates/agent_ui/src/agent_model_selector.rs
index cbffced96df..89eeccc1e3c 100644
--- a/crates/agent_ui/src/agent_model_selector.rs
+++ b/crates/agent_ui/src/agent_model_selector.rs
@@ -5,7 +5,7 @@ use crate::{
};
use fs::Fs;
use gpui::{Entity, FocusHandle, SharedString};
-use language_model::IconOrSvg;
+use language_model::{IconOrSvg, LanguageModelRegistry};
use picker::popover_menu::PickerPopoverMenu;
use settings::update_settings_file;
use std::sync::Arc;
@@ -30,7 +30,7 @@ impl AgentModelSelector {
language_model_selector(
{
let model_context = model_usage_context.clone();
- move |cx| model_context.configured_model(cx)
+ move |cx| model_context.model(cx)
},
{
let fs = fs.clone();
@@ -74,7 +74,7 @@ impl AgentModelSelector {
self.menu_handle.toggle(window, cx);
}
- pub fn active_model(&self, cx: &App) -> Option<language_model::ConfiguredModel> {
+ pub fn active_model(&self, cx: &App) -> Option<language_model::LanguageModel> {
self.selector.read(cx).delegate.active_model(cx)
}
@@ -90,10 +90,14 @@ impl Render for AgentModelSelector {
let model = self.selector.read(cx).delegate.active_model(cx);
let model_name = model
.as_ref()
- .map(|model| model.model.name().0)
+ .map(|model| model.name().0)
.unwrap_or_else(|| SharedString::from("Select a Model"));
- let provider_icon = model.as_ref().map(|model| model.provider.icon());
+ let provider_icon = model.as_ref().and_then(|model| {
+ LanguageModelRegistry::read_global(cx)
+ .provider(&model.provider_id)
+ .map(|provider| provider.icon())
+ });
let color = if self.menu_handle.is_deployed() {
Color::Accent
} else {
diff --git a/crates/agent_ui/src/agent_panel.rs b/crates/agent_ui/src/agent_panel.rs
index 8a146247b87..4942a140a7d 100644
--- a/crates/agent_ui/src/agent_panel.rs
+++ b/crates/agent_ui/src/agent_panel.rs
@@ -4710,16 +4710,16 @@ pub(crate) fn apply_native_model_override(
);
return;
};
- let configured = LanguageModelRegistry::global(cx)
+ let model = LanguageModelRegistry::global(cx)
.update(cx, |registry, cx| registry.select_model(&selected, cx));
- let Some(configured) = configured else {
+ let Some(model) = model else {
log::warn!(
"create_thread: no model registered for {model_id:?}; using thread's default model"
);
return;
};
thread.update(cx, |thread, cx| {
- thread.set_model(configured.model, cx);
+ thread.set_model(model, cx);
});
}
@@ -6178,9 +6178,7 @@ impl AgentPanel {
if LanguageModelRegistry::global(cx)
.read(cx)
.default_model()
- .is_some_and(|model| {
- model.provider.id() != language_model::ZED_CLOUD_PROVIDER_ID
- })
+ .is_some_and(|model| model.provider_id != language_model::ZED_CLOUD_PROVIDER_ID)
{
return false;
}
diff --git a/crates/agent_ui/src/agent_ui.rs b/crates/agent_ui/src/agent_ui.rs
index c9e823bd26e..b3a4d9fd7d3 100644
--- a/crates/agent_ui/src/agent_ui.rs
+++ b/crates/agent_ui/src/agent_ui.rs
@@ -56,7 +56,7 @@ use language::{
language_settings::{AllLanguageSettings, EditPredictionProvider},
};
use language_model::{
- ConfiguredModel, LanguageModelId, LanguageModelProviderId, LanguageModelRegistry,
+ LanguageModel, LanguageModelId, LanguageModelProviderId, LanguageModelRegistry,
};
use project::{AgentId, DisableAiSettings};
use prompt_store::{self, PromptBuilder, rules_to_skills_migration};
@@ -538,7 +538,7 @@ pub(crate) enum ModelUsageContext {
}
impl ModelUsageContext {
- pub fn configured_model(&self, cx: &App) -> Option<ConfiguredModel> {
+ pub fn model(&self, cx: &App) -> Option<LanguageModel> {
match self {
Self::InlineAssistant => {
LanguageModelRegistry::read_global(cx).inline_assistant_model()
diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rs
index 38798cadd99..636758c7abb 100644
--- a/crates/agent_ui/src/buffer_codegen.rs
+++ b/crates/agent_ui/src/buffer_codegen.rs
@@ -9,7 +9,6 @@ use futures::{
channel::mpsc,
future::{LocalBoxFuture, Shared},
join,
- stream::BoxStream,
};
use gpui::{App, AppContext as _, AsyncApp, Context, Entity, EventEmitter, Subscription, Task};
use language::{
@@ -17,9 +16,10 @@ use language::{
};
use language_model::{
CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage,
- LanguageModelRequestTool, LanguageModelTextStream, LanguageModelToolChoice,
- LanguageModelToolUse, LanguageModelToolUseId, Role, StopReason, TokenUsage,
+ LanguageModelCompletionStream, LanguageModelRegistry, LanguageModelRequest,
+ LanguageModelRequestMessage, LanguageModelRequestTool, LanguageModelTextStream,
+ LanguageModelToolChoice, LanguageModelToolUse, LanguageModelToolUseId, Role, StopReason,
+ TokenUsage,
};
use language_models::provider::anthropic::telemetry::{
AnthropicCompletionType, AnthropicEventData, AnthropicEventReporter, AnthropicEventType,
@@ -174,7 +174,7 @@ impl BufferCodegen {
pub fn start(
&mut self,
- primary_model: Arc<dyn LanguageModel>,
+ primary_model: LanguageModel,
user_prompt: String,
context_task: Shared<Task<Option<LoadedContext>>>,
cx: &mut Context<Self>,
@@ -401,7 +401,7 @@ impl CodegenAlternative {
&self.last_equal_ranges
}
- pub fn use_streaming_tools(model: &dyn LanguageModel, cx: &App) -> bool {
+ pub fn use_streaming_tools(model: &LanguageModel, cx: &App) -> bool {
model.supports_streaming_tools()
&& AgentSettings::get_global(cx).inline_assistant_use_streaming_tools
}
@@ -410,7 +410,7 @@ impl CodegenAlternative {
&mut self,
user_prompt: String,
context_task: Shared<Task<Option<LoadedContext>>>,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
cx: &mut Context<Self>,
) -> Result<()> {
// Clear the model explanation since the user has started a new generation.
@@ -425,11 +425,12 @@ impl CodegenAlternative {
self.edit_position = Some(self.range.start.bias_right(&self.snapshot));
- if Self::use_streaming_tools(model.as_ref(), cx) {
+ let provider = LanguageModelRegistry::read_global(cx).provider_for_model(&model)?;
+ if Self::use_streaming_tools(&model, cx) {
let request = self.build_request(&model, user_prompt, context_task, cx)?;
let completion_events = cx.spawn({
let model = model.clone();
- async move |_, cx| model.stream_completion(request.await, cx).await
+ async move |_, cx| provider.stream_completion(&model, request.await, cx).await
});
self.generation = self.handle_completion(model, completion_events, cx);
} else {
@@ -441,7 +442,9 @@ impl CodegenAlternative {
cx.spawn({
let model = model.clone();
async move |_, cx| {
- Ok(model.stream_completion_text(request.await, cx).await?)
+ Ok(provider
+ .stream_completion_text(&model, request.await, cx)
+ .await?)
}
})
.boxed_local()
@@ -455,7 +458,7 @@ impl CodegenAlternative {
fn build_request_tools(
&self,
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
user_prompt: String,
context_task: Shared<Task<Option<LoadedContext>>>,
cx: &mut App,
@@ -500,7 +503,8 @@ impl CodegenAlternative {
let session_id = self.session_id.to_string();
let tool_choice = model
- .supports_tool_choice(LanguageModelToolChoice::Any)
+ .tool_choice_support
+ .supports(LanguageModelToolChoice::Any)
.then_some(LanguageModelToolChoice::Any);
Ok(cx.spawn(async move |_cx| {
@@ -565,12 +569,12 @@ impl CodegenAlternative {
fn build_request(
&self,
- model: &Arc<dyn LanguageModel>,
+ model: &LanguageModel,
user_prompt: String,
context_task: Shared<Task<Option<LoadedContext>>>,
cx: &mut App,
) -> Result<Task<LanguageModelRequest>> {
- if Self::use_streaming_tools(model.as_ref(), cx) {
+ if Self::use_streaming_tools(model, cx) {
return self.build_request_tools(model, user_prompt, context_task, cx);
}
@@ -649,7 +653,7 @@ impl CodegenAlternative {
pub fn handle_stream(
&mut self,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
strip_invalid_spans: bool,
stream: impl 'static + Future<Output = Result<LanguageModelTextStream>>,
cx: &mut Context<Self>,
@@ -1135,15 +1139,9 @@ impl CodegenAlternative {
fn handle_completion(
&mut self,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
completion_stream: Task<
- Result<
- BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
+ Result<LanguageModelCompletionStream, LanguageModelCompletionError>,
>,
cx: &mut Context<Self>,
) -> Task<()> {
@@ -1646,7 +1644,7 @@ mod tests {
use gpui::TestAppContext;
use indoc::indoc;
use language::{Buffer, Point};
- use language_model::fake_provider::FakeLanguageModel;
+ use language_model::fake_provider::FakeLanguageModelProvider;
use language_model::{
LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelRegistry,
LanguageModelToolUse, StopReason, TokenUsage,
@@ -2507,7 +2505,9 @@ mod tests {
.start(
"Rewrite the selection".into(),
Task::ready(None).shared(),
- Arc::new(FakeLanguageModel::default()),
+ LanguageModelRegistry::read_global(cx)
+ .default_model()
+ .unwrap(),
cx,
)
.unwrap();
@@ -2593,7 +2593,7 @@ mod tests {
cx: &mut TestAppContext,
) -> mpsc::UnboundedSender<String> {
let (chunks_tx, chunks_rx) = mpsc::unbounded();
- let model = Arc::new(FakeLanguageModel::default());
+ let model = FakeLanguageModelProvider::default().model("fake");
codegen.update(cx, |codegen, cx| {
codegen.generation = codegen.handle_stream(
model,
@@ -2615,7 +2615,7 @@ mod tests {
) -> mpsc::UnboundedSender<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
{
let (events_tx, events_rx) = mpsc::unbounded();
- let model = Arc::new(FakeLanguageModel::default());
+ let model = FakeLanguageModelProvider::default().model("fake");
codegen.update(cx, |codegen, cx| {
codegen.generation =
codegen.handle_completion(model, Task::ready(Ok(events_rx.boxed())), cx);
@@ -2628,13 +2628,11 @@ mod tests {
cx: &mut TestAppContext,
) -> mpsc::UnboundedSender<LanguageModelCompletionEvent> {
let (events_tx, events_rx) = mpsc::unbounded();
- let model = Arc::new(FakeLanguageModel::default());
+ let model = FakeLanguageModelProvider::default().model("fake");
codegen.update(cx, |codegen, cx| {
- let completion_stream = Task::ready(Ok(events_rx.map(Ok).boxed()
- as BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >));
+ let completion_stream = Task::ready(Ok(
+ events_rx.map(Ok).boxed() as LanguageModelCompletionStream
+ ));
codegen.generation = codegen.handle_completion(model, completion_stream, cx);
});
events_tx
diff --git a/crates/agent_ui/src/conversation_view/thread_view.rs b/crates/agent_ui/src/conversation_view/thread_view.rs
index f31a1227ae7..97fc7ea325a 100644
--- a/crates/agent_ui/src/conversation_view/thread_view.rs
+++ b/crates/agent_ui/src/conversation_view/thread_view.rs
@@ -5144,9 +5144,9 @@ impl ThreadView {
let enable_thinking = !thread.thinking_enabled();
thread.set_thinking_enabled(enable_thinking, cx);
- let favorite_key = thread.model().map(|model| {
- (model.provider_id().0.to_string(), model.id().0.to_string())
- });
+ let favorite_key = thread
+ .model()
+ .map(|model| (model.provider_id.0.to_string(), model.id.0.to_string()));
let fs = thread.project().read(cx).fs().clone();
update_settings_file(fs, cx, move |settings, _| {
if let Some(agent) = settings.agent.as_mut() {
@@ -11960,7 +11960,7 @@ impl ThreadView {
/// Returns the model to offer as a downgrade target when the current model
/// requires data retention consent (e.g. Opus 4.8 for Fable).
- fn data_retention_fallback_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn data_retention_fallback_model(&self, cx: &App) -> Option<LanguageModel> {
let thread = self.as_native_thread(cx)?;
let model = thread.read(cx).model()?.clone();
let fallback_id = model.refusal_fallback_model_id()?;
diff --git a/crates/agent_ui/src/favorite_models.rs b/crates/agent_ui/src/favorite_models.rs
index c655f9b6a55..563da9dd098 100644
--- a/crates/agent_ui/src/favorite_models.rs
+++ b/crates/agent_ui/src/favorite_models.rs
@@ -7,7 +7,7 @@ use settings::{Settings as _, update_settings_file};
use ui::App;
pub fn toggle_in_settings(
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
should_be_favorite: bool,
fs: Arc<dyn Fs>,
cx: &mut App,
diff --git a/crates/agent_ui/src/inline_assistant.rs b/crates/agent_ui/src/inline_assistant.rs
index 8f96989ac89..e65a1f2acd7 100644
--- a/crates/agent_ui/src/inline_assistant.rs
+++ b/crates/agent_ui/src/inline_assistant.rs
@@ -39,7 +39,7 @@ use gpui::{
UpdateGlobal, WeakEntity, Window, point,
};
use language::{Buffer, Point, Selection, TransactionId};
-use language_model::{ConfigurationError, ConfiguredModel, LanguageModelRegistry};
+use language_model::{ConfigurationError, LanguageModelRegistry};
use multi_buffer::MultiBufferRow;
use parking_lot::Mutex;
use project::{DisableAiSettings, Project};
@@ -407,13 +407,13 @@ impl InlineAssistant {
"Assistant Invoked",
kind = "inline",
phase = "invoked",
- model = model.model.telemetry_id(),
- model_provider = model.provider.id().to_string(),
+ model = model.telemetry_id(),
+ model_provider = model.provider_id().to_string(),
language_name = buffer.language().map(|language| language.name().to_proto())
);
report_anthropic_event(
- &model.model,
+ &model,
AnthropicEventData {
completion_type: AnthropicCompletionType::Editor,
event: AnthropicEventType::Invoked,
@@ -990,8 +990,8 @@ impl InlineAssistant {
let codegen = assist.codegen.read(cx);
let session_id = codegen.session_id();
let message_id = active_alternative.read(cx).message_id.clone();
- let model_telemetry_id = model.model.telemetry_id();
- let model_provider_id = model.model.provider_id().to_string();
+ let model_telemetry_id = model.telemetry_id();
+ let model_provider_id = model.provider_id().to_string();
let (phase, event_type, anthropic_event_type) = if undo {
(
@@ -1019,7 +1019,7 @@ impl InlineAssistant {
);
report_anthropic_event(
- &model.model,
+ &model,
AnthropicEventData {
completion_type: AnthropicCompletionType::Editor,
event: anthropic_event_type,
@@ -1245,9 +1245,7 @@ impl InlineAssistant {
self.prompt_history.pop_front();
}
- let Some(ConfiguredModel { model, .. }) =
- LanguageModelRegistry::read_global(cx).inline_assistant_model()
- else {
+ let Some(model) = LanguageModelRegistry::read_global(cx).inline_assistant_model() else {
return;
};
diff --git a/crates/agent_ui/src/inline_prompt_editor.rs b/crates/agent_ui/src/inline_prompt_editor.rs
index 9d506aa5e4d..3d5baa652fd 100644
--- a/crates/agent_ui/src/inline_prompt_editor.rs
+++ b/crates/agent_ui/src/inline_prompt_editor.rs
@@ -568,8 +568,8 @@ impl<T: 'static> PromptEditor<T> {
return;
};
- let model_telemetry_id = model.model.telemetry_id();
- let model_provider_id = model.provider.id().to_string();
+ let model_telemetry_id = model.telemetry_id();
+ let model_provider_id = model.provider_id().to_string();
let (kind, language_name) = match &self.mode {
PromptEditorMode::Buffer { codegen, .. } => {
@@ -610,16 +610,13 @@ impl<T: 'static> PromptEditor<T> {
CompletionState::Generated { completion_text } => {
let model_info = self.model_selector.read(cx).active_model(cx);
let (model_id, use_streaming_tools) = {
- let Some(configured_model) = model_info else {
+ let Some(model) = model_info else {
self.toast("No configured model", None, cx);
return;
};
(
- configured_model.model.telemetry_id(),
- CodegenAlternative::use_streaming_tools(
- configured_model.model.as_ref(),
- cx,
- ),
+ model.telemetry_id(),
+ CodegenAlternative::use_streaming_tools(&model, cx),
)
};
@@ -673,16 +670,13 @@ impl<T: 'static> PromptEditor<T> {
CompletionState::Generated { completion_text } => {
let model_info = self.model_selector.read(cx).active_model(cx);
let (model_telemetry_id, use_streaming_tools) = {
- let Some(configured_model) = model_info else {
+ let Some(model) = model_info else {
self.toast("No configured model", None, cx);
return;
};
(
- configured_model.model.telemetry_id(),
- CodegenAlternative::use_streaming_tools(
- configured_model.model.as_ref(),
- cx,
- ),
+ model.telemetry_id(),
+ CodegenAlternative::use_streaming_tools(&model, cx),
)
};
@@ -1022,11 +1016,11 @@ impl<T: 'static> PromptEditor<T> {
let disabled = matches!(codegen.status(cx), CodegenStatus::Idle);
let model_registry = LanguageModelRegistry::read_global(cx);
- let default_model = model_registry.default_model().map(|default| default.model);
+ let default_model = model_registry.default_model();
let alternative_models = model_registry.inline_alternative_models();
let get_model_name = |index: usize| -> String {
- let name = |model: &Arc<dyn LanguageModel>| model.name().0.to_string();
+ let name = |model: &LanguageModel| model.name.0.to_string();
match index {
0 => default_model.as_ref().map_or_else(String::new, name),
@@ -1201,7 +1195,7 @@ struct PromptEditorCompletionProviderDelegate;
fn inline_assistant_model_supports_images(cx: &App) -> bool {
LanguageModelRegistry::read_global(cx)
.inline_assistant_model()
- .map_or(false, |m| m.model.supports_images())
+ .map_or(false, |m| m.supports_images())
}
impl PromptCompletionProviderDelegate for PromptEditorCompletionProviderDelegate {
diff --git a/crates/agent_ui/src/language_model_selector.rs b/crates/agent_ui/src/language_model_selector.rs
index 2c19e111571..ba60f9c2db0 100644
--- a/crates/agent_ui/src/language_model_selector.rs
+++ b/crates/agent_ui/src/language_model_selector.rs
@@ -8,8 +8,8 @@ use gpui::{
Subscription, Task,
};
use language_model::{
- ConfiguredModel, IconOrSvg, LanguageModel, LanguageModelId, LanguageModelProvider,
- LanguageModelProviderId, LanguageModelRegistry,
+ IconOrSvg, LanguageModel, LanguageModelId, LanguageModelProvider, LanguageModelProviderId,
+ LanguageModelRegistry,
};
use ordered_float::OrderedFloat;
use picker::{Picker, PickerDelegate};
@@ -19,16 +19,16 @@ use zed_actions::agent::OpenSettings;
use crate::ui::{ModelSelectorFooter, ModelSelectorHeader, ModelSelectorListItem};
-type OnModelChanged = Arc<dyn Fn(Arc<dyn LanguageModel>, &mut App) + 'static>;
-type GetActiveModel = Arc<dyn Fn(&App) -> Option<ConfiguredModel> + 'static>;
-type OnToggleFavorite = Arc<dyn Fn(Arc<dyn LanguageModel>, bool, &mut App) + 'static>;
+type OnModelChanged = Arc<dyn Fn(LanguageModel, &mut App) + 'static>;
+type GetActiveModel = Arc<dyn Fn(&App) -> Option<LanguageModel> + 'static>;
+type OnToggleFavorite = Arc<dyn Fn(LanguageModel, bool, &mut App) + 'static>;
pub type LanguageModelSelector = Picker<LanguageModelPickerDelegate>;
pub fn language_model_selector(
- get_active_model: impl Fn(&App) -> Option<ConfiguredModel> + 'static,
- on_model_changed: impl Fn(Arc<dyn LanguageModel>, &mut App) + 'static,
- on_toggle_favorite: impl Fn(Arc<dyn LanguageModel>, bool, &mut App) + 'static,
+ get_active_model: impl Fn(&App) -> Option<LanguageModel> + 'static,
+ on_model_changed: impl Fn(LanguageModel, &mut App) + 'static,
+ on_toggle_favorite: impl Fn(LanguageModel, bool, &mut App) + 'static,
popover_styles: bool,
focus_handle: FocusHandle,
window: &mut Window,
@@ -96,7 +96,7 @@ type FavoritesIndex = HashMap<LanguageModelProviderId, HashSet<LanguageModelId>>
#[derive(Clone)]
struct ModelInfo {
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
icon: IconOrSvg,
is_favorite: bool,
}
@@ -104,7 +104,7 @@ struct ModelInfo {
impl ModelInfo {
fn new(
provider: &dyn LanguageModelProvider,
- model: Arc<dyn LanguageModel>,
+ model: LanguageModel,
favorites_index: &FavoritesIndex,
) -> Self {
let is_favorite = favorites_index
@@ -133,9 +133,9 @@ pub struct LanguageModelPickerDelegate {
impl LanguageModelPickerDelegate {
fn new(
- get_active_model: impl Fn(&App) -> Option<ConfiguredModel> + 'static,
- on_model_changed: impl Fn(Arc<dyn LanguageModel>, &mut App) + 'static,
- on_toggle_favorite: impl Fn(Arc<dyn LanguageModel>, bool, &mut App) + 'static,
+ get_active_model: impl Fn(&App) -> Option<LanguageModel> + 'static,
+ on_model_changed: impl Fn(LanguageModel, &mut App) + 'static,
+ on_toggle_favorite: impl Fn(LanguageModel, bool, &mut App) + 'static,
popover_styles: bool,
focus_handle: FocusHandle,
window: &mut Window,
@@ -177,7 +177,7 @@ impl LanguageModelPickerDelegate {
fn get_active_model_index(
entries: &[LanguageModelPickerEntry],
- active_model: Option<ConfiguredModel>,
+ active_model: Option<LanguageModel>,
) -> usize {
entries
.iter()
@@ -185,10 +185,7 @@ impl LanguageModelPickerDelegate {
if let LanguageModelPickerEntry::Model(model) = entry {
active_model
.as_ref()
- .map(|active_model| {
- active_model.model.id() == model.model.id()
- && active_model.provider.id() == model.model.provider_id()
- })
+ .map(|active_model| active_model.is_same_as(&model.model))
.unwrap_or_default()
} else {
false
@@ -197,7 +194,7 @@ impl LanguageModelPickerDelegate {
.unwrap_or(0)
}
- pub fn active_model(&self, cx: &App) -> Option<ConfiguredModel> {
+ pub fn active_model(&self, cx: &App) -> Option<LanguageModel> {
(self.get_active_model)(cx)
}
@@ -211,8 +208,8 @@ impl LanguageModelPickerDelegate {
}
let active_model = (self.get_active_model)(cx);
- let active_provider_id = active_model.as_ref().map(|m| m.provider.id());
- let active_model_id = active_model.as_ref().map(|m| m.model.id());
+ let active_provider_id = active_model.as_ref().map(|m| m.provider_id());
+ let active_model_id = active_model.as_ref().map(|m| m.id());
let current_index = self
.all_models
@@ -484,7 +481,7 @@ impl PickerDelegate for LanguageModelPickerDelegate {
self.filtered_entries.get(self.selected_index)
{
let model = model_info.model.clone();
- (self.on_model_changed)(model.clone(), cx);
+ (self.on_model_changed)(model, cx);
let current_index = self.selected_index;
self.set_selected_index(current_index, window, cx);
@@ -510,15 +507,16 @@ impl PickerDelegate for LanguageModelPickerDelegate {
}
LanguageModelPickerEntry::Model(model_info) => {
let active_model = (self.get_active_model)(cx);
- let active_provider_id = active_model.as_ref().map(|m| m.provider.id());
- let active_model_id = active_model.map(|m| m.model.id());
+ let active_provider_id = active_model.as_ref().map(|m| m.provider_id());
+ let active_model_id = active_model.map(|m| m.id());
let is_selected = Some(model_info.model.provider_id()) == active_provider_id
&& Some(model_info.model.id()) == active_model_id;
let model_cost = model_info
.model
- .model_cost_info()
+ .cost_info
+ .as_ref()
.map(|cost| cost.to_shared_string());
let is_favorite = model_info.is_favorite;
@@ -567,87 +565,21 @@ impl PickerDelegate for LanguageModelPickerDelegate {
#[cfg(test)]
mod tests {
use super::*;
- use futures::{future::BoxFuture, stream::BoxStream};
- use gpui::{AsyncApp, TestAppContext};
+ use gpui::TestAppContext;
use language_model::{
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId,
- LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelRequest, LanguageModelToolChoice,
+ LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
};
use ui::IconName;
- #[derive(Clone)]
- struct TestLanguageModel {
- name: LanguageModelName,
- id: LanguageModelId,
- provider_id: LanguageModelProviderId,
- provider_name: LanguageModelProviderName,
- }
-
- impl TestLanguageModel {
- fn new(name: &str, provider: &str) -> Self {
- Self {
- name: LanguageModelName::from(name.to_string()),
- id: LanguageModelId::from(name.to_string()),
- provider_id: LanguageModelProviderId::from(provider.to_string()),
- provider_name: LanguageModelProviderName::from(provider.to_string()),
- }
- }
- }
-
- impl LanguageModel for TestLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- self.name.clone()
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- self.provider_id.clone()
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- self.provider_name.clone()
- }
-
- fn supports_tools(&self) -> bool {
- false
- }
-
- fn supports_tool_choice(&self, _choice: LanguageModelToolChoice) -> bool {
- false
- }
-
- fn supports_images(&self) -> bool {
- false
- }
-
- fn telemetry_id(&self) -> String {
- format!("{}/{}", self.provider_id.0, self.name.0)
- }
-
- fn max_token_count(&self) -> u64 {
- 1000
- }
-
- fn stream_completion(
- &self,
- _: LanguageModelRequest,
- _: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- unimplemented!()
- }
+ fn test_language_model(name: &str, provider: &str) -> LanguageModel {
+ LanguageModel::new(
+ LanguageModelId::from(name.to_string()),
+ LanguageModelName::from(name.to_string()),
+ LanguageModelProviderId::from(provider.to_string()),
+ LanguageModelProviderName::from(provider.to_string()),
+ format!("{provider}/{name}"),
+ 1000,
+ )
}
fn create_models(model_specs: Vec<(&str, &str)>) -> Vec<ModelInfo> {
@@ -665,7 +597,7 @@ mod tests {
.iter()
.any(|(fav_provider, fav_name)| *fav_provider == provider && *fav_name == name);
ModelInfo {
- model: Arc::new(TestLanguageModel::new(name, provider)),
+ model: test_language_model(name, provider),
icon: IconOrSvg::Icon(IconName::ZedAgent),
is_favorite,
}
diff --git a/crates/agent_ui/src/terminal_codegen.rs b/crates/agent_ui/src/terminal_codegen.rs
index 19adf2d8806..ec83b7710f6 100644
--- a/crates/agent_ui/src/terminal_codegen.rs
+++ b/crates/agent_ui/src/terminal_codegen.rs
@@ -1,7 +1,7 @@
use crate::inline_prompt_editor::CodegenStatus;
use futures::{SinkExt, StreamExt, channel::mpsc};
use gpui::{App, AppContext as _, Context, Entity, EventEmitter, Task};
-use language_model::{ConfiguredModel, LanguageModelRegistry, LanguageModelRequest};
+use language_model::{LanguageModelRegistry, LanguageModelRequest};
use language_models::provider::anthropic::telemetry::{
AnthropicCompletionType, AnthropicEventData, AnthropicEventReporter, AnthropicEventType,
};
@@ -37,11 +37,11 @@ impl TerminalCodegen {
}
pub fn start(&mut self, prompt_task: Task<LanguageModelRequest>, cx: &mut Context<Self>) {
- let Some(ConfiguredModel { model, .. }) =
- LanguageModelRegistry::read_global(cx).inline_assistant_model()
- else {
+ let registry = LanguageModelRegistry::read_global(cx);
+ let Some(model) = registry.inline_assistant_model() else {
return;
};
+ let provider = registry.provider_for_model(&model);
let anthropic_reporter = AnthropicEventReporter::new(&model, cx);
let session_id = self.session_id;
@@ -52,7 +52,10 @@ impl TerminalCodegen {
self.transaction = Some(TerminalTransaction::start(self.terminal.clone()));
self.generation = cx.spawn(async move |this, cx| {
let prompt = prompt_task.await;
- let response = model.stream_completion_text(prompt, cx).await;
+ let response = match provider {
+ Ok(provider) => provider.stream_completion_text(&model, prompt, cx).await,
+ Err(error) => Err(error),
+ };
let generate = async {
let message_id = response
.as_ref()
diff --git a/crates/agent_ui/src/terminal_inline_assistant.rs b/crates/agent_ui/src/terminal_inline_assistant.rs
index c7c9e7fb8e2..a3eca1d845b 100644
--- a/crates/agent_ui/src/terminal_inline_assistant.rs
+++ b/crates/agent_ui/src/terminal_inline_assistant.rs
@@ -15,8 +15,8 @@ use fs::Fs;
use gpui::{App, Entity, Focusable, Global, Subscription, Task, UpdateGlobal, WeakEntity};
use language::Buffer;
use language_model::{
- CompletionIntent, ConfiguredModel, LanguageModelRegistry, LanguageModelRequest,
- LanguageModelRequestMessage, Role,
+ CompletionIntent, LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage,
+ Role,
};
use language_models::provider::anthropic::telemetry::{
AnthropicCompletionType, AnthropicEventData, AnthropicEventType, report_anthropic_event,
@@ -214,7 +214,7 @@ impl TerminalInlineAssistant {
assist_id: TerminalInlineAssistId,
cx: &mut App,
) -> Result<Task<LanguageModelRequest>> {
- let ConfiguredModel { model, .. } = LanguageModelRegistry::read_global(cx)
+ let model = LanguageModelRegistry::read_global(cx)
.inline_assistant_model()
.context("No inline assistant model")?;
@@ -301,9 +301,7 @@ impl TerminalInlineAssistant {
})
.log_err();
- if let Some(ConfiguredModel { model, .. }) =
- LanguageModelRegistry::read_global(cx).inline_assistant_model()
- {
+ if let Some(model) = LanguageModelRegistry::read_global(cx).inline_assistant_model() {
let codegen = assist.codegen.read(cx);
let session_id = codegen.session_id();
let message_id = codegen.message_id.clone();
diff --git a/crates/benchmarks/benches/edit_file_tool.rs b/crates/benchmarks/benches/edit_file_tool.rs
index 2adf7cd8ad6..204628cfe80 100644
--- a/crates/benchmarks/benches/edit_file_tool.rs
+++ b/crates/benchmarks/benches/edit_file_tool.rs
@@ -26,7 +26,7 @@ use gpui::{
UpdateGlobal as _,
};
use language::{FakeLspAdapter, rust_lang};
-use language_model::fake_provider::FakeLanguageModel;
+use language_model::LanguageModelRegistry;
use project::{FakeFs, Project};
use prompt_store::ProjectContext;
use rand::{Rng as _, SeedableRng as _, rngs::StdRng};
@@ -206,7 +206,7 @@ async fn setup_editor_and_tool(cx: &mut TestAppContext, file_text: String) -> Ha
let context_server_registry =
cx.new(|cx| ContextServerRegistry::new(project.read(cx).context_server_store(), cx));
- let model = Arc::new(FakeLanguageModel::default());
+ let model = cx.update(|cx| LanguageModelRegistry::test(cx).model("fake"));
let thread = cx.new(|cx| {
Thread::new(
project.clone(),
diff --git a/crates/copilot_chat/src/copilot_chat.rs b/crates/copilot_chat/src/copilot_chat.rs
index 9b9b208a1b7..c56bee14dbe 100644
--- a/crates/copilot_chat/src/copilot_chat.rs
+++ b/crates/copilot_chat/src/copilot_chat.rs
@@ -2,7 +2,7 @@ pub mod copilot_oauth;
mod model;
pub mod responses;
-pub use model::{PROVIDER_ID, PROVIDER_NAME, create_language_model};
+pub use model::{PROVIDER_ID, PROVIDER_NAME, language_model, stream_completion};
use std::sync::Arc;
@@ -768,7 +768,7 @@ impl CopilotChat {
Self::get_auth_details(&copilot_chat, &mut cx).await?;
let api_url = configuration.chat_completions_url(&api_endpoint);
- stream_completion(
+ stream_chat_completion(
client.clone(),
oauth_token,
api_url.into(),
@@ -1083,7 +1083,7 @@ async fn request_models(
Ok(models)
}
-async fn stream_completion(
+async fn stream_chat_completion(
client: Arc<dyn HttpClient>,
oauth_token: String,
completion_url: Arc<str>,
diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs
index e778a182583..ee7e2ddd29b 100644
--- a/crates/copilot_chat/src/model.rs
+++ b/crates/copilot_chat/src/model.rs
@@ -1,4 +1,4 @@
-use std::{pin::Pin, str::FromStr as _, sync::Arc};
+use std::{pin::Pin, str::FromStr as _};
use crate::responses as copilot_responses;
use crate::{
@@ -13,17 +13,16 @@ use anthropic::{
use anyhow::{Result, anyhow};
use collections::HashMap;
use futures::future::BoxFuture;
-use futures::stream::BoxStream;
use futures::{FutureExt, Stream, StreamExt};
use gpui::{AsyncApp, Entity};
use http_client::StatusCode;
use language_model::{
CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelCostInfo, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
- LanguageModelProviderId, LanguageModelProviderName, LanguageModelRequest,
- LanguageModelRequestMessage, LanguageModelToolChoice, LanguageModelToolResultContent,
- LanguageModelToolUse, MessageContent, ProviderErrorCategory, RateLimiter, Role, StopReason,
- TokenUsage,
+ LanguageModelCompletionStream, LanguageModelCostInfo, LanguageModelEffortLevel,
+ LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
+ LanguageModelRequest, LanguageModelRequestMessage, LanguageModelToolChoice,
+ LanguageModelToolChoiceSupport, LanguageModelToolResultContent, LanguageModelToolUse,
+ MessageContent, ProviderErrorCategory, RateLimiter, Role, StopReason, TokenUsage,
};
use util::debug_panic;
@@ -33,61 +32,13 @@ pub const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("c
pub const PROVIDER_NAME: LanguageModelProviderName =
LanguageModelProviderName::new("GitHub Copilot Chat");
-pub fn create_language_model(
- model: CopilotChatModel,
- copilot_chat: Entity<CopilotChat>,
-) -> Arc<dyn LanguageModel> {
- Arc::new(CopilotChatLanguageModel {
- model,
- copilot_chat,
- request_limiter: RateLimiter::new(4),
- })
-}
-
-struct CopilotChatLanguageModel {
- model: CopilotChatModel,
- copilot_chat: Entity<CopilotChat>,
- request_limiter: RateLimiter,
-}
-
-impl LanguageModel for CopilotChatLanguageModel {
- fn id(&self) -> LanguageModelId {
- LanguageModelId::from(self.model.id().to_string())
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_vision()
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.can_think()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- let levels = self.model.reasoning_effort_levels();
- if levels.is_empty() {
- return vec![];
- }
+/// Describes a Copilot Chat model as a [`LanguageModel`].
+pub fn language_model(model: &CopilotChatModel) -> LanguageModel {
+ let supports_tools = model.supports_tools();
+ let levels = model.reasoning_effort_levels();
+ let supported_effort_levels = if levels.is_empty() {
+ vec![]
+ } else {
levels
.iter()
.map(|level| {
@@ -105,228 +56,216 @@ impl LanguageModel for CopilotChatLanguageModel {
}
})
.collect()
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto
- | LanguageModelToolChoice::Any
- | LanguageModelToolChoice::None => self.supports_tools(),
- }
- }
-
- fn model_cost_info(&self) -> Option<LanguageModelCostInfo> {
- LanguageModelCostInfo::RequestCost {
- cost_per_request: self.model.multiplier(),
+ };
+ LanguageModel {
+ supports_tools,
+ supports_streaming_tools: true,
+ supports_images: model.supports_vision(),
+ supports_thinking: model.can_think(),
+ supported_effort_levels: supported_effort_levels.into(),
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: supports_tools,
+ any: supports_tools,
+ none: supports_tools,
+ },
+ cost_info: LanguageModelCostInfo::RequestCost {
+ cost_per_request: model.multiplier(),
}
- .into()
- }
-
- fn telemetry_id(&self) -> String {
- format!("copilot_chat/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_input_tokens(&self) -> u64 {
- self.model
- .max_prompt_tokens()
- .unwrap_or_else(|| self.model.max_token_count())
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
+ .into(),
+ max_input_tokens: model.max_prompt_tokens().unwrap_or(model.max_token_count()),
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("copilot_chat/{}", model.id()),
+ model.max_token_count(),
+ )
}
+}
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let is_user_initiated = request.intent.is_none_or(|intent| match intent {
- CompletionIntent::UserPrompt
- | CompletionIntent::ThreadContextSummarization
- | CompletionIntent::InlineAssist
- | CompletionIntent::TerminalInlineAssist
- | CompletionIntent::GenerateGitCommitMessage => true,
-
- CompletionIntent::Subagent
- | CompletionIntent::ToolResults
- | CompletionIntent::ThreadSummarization
- | CompletionIntent::CreateFile
- | CompletionIntent::EditFile => false,
- });
+/// Streams a completion of `request` from the Copilot Chat model `model`.
+pub fn stream_completion(
+ model: &CopilotChatModel,
+ copilot_chat: &Entity<CopilotChat>,
+ request_limiter: &RateLimiter,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>> {
+ let is_user_initiated = request.intent.is_none_or(|intent| match intent {
+ CompletionIntent::UserPrompt
+ | CompletionIntent::ThreadContextSummarization
+ | CompletionIntent::InlineAssist
+ | CompletionIntent::TerminalInlineAssist
+ | CompletionIntent::GenerateGitCommitMessage => true,
+
+ CompletionIntent::Subagent
+ | CompletionIntent::ToolResults
+ | CompletionIntent::ThreadSummarization
+ | CompletionIntent::CreateFile
+ | CompletionIntent::EditFile => false,
+ });
- if self.model.supports_messages() {
- let location = intent_to_chat_location(request.intent);
- let model = self.model.clone();
- let copilot_chat = self.copilot_chat.clone();
- let request_limiter = self.request_limiter.clone();
- let future = cx.spawn(async move |cx| {
- let max_output_tokens = model
- .max_output_tokens()
- .or(request.max_output_tokens)
- .ok_or_else(|| {
- anyhow!("Copilot did not provide an output limit for this model")
- })?;
- let effort = request
- .thinking_effort
- .as_ref()
- .and_then(|e| anthropic::Effort::from_str(e).ok());
-
- let mut anthropic_request = into_anthropic(
- request,
- model.id().to_string(),
- 0.0,
- max_output_tokens,
- if model.supports_adaptive_thinking() {
- AnthropicModelMode::Thinking {
- budget_tokens: None,
- }
- } else if model.supports_thinking() {
- AnthropicModelMode::Thinking {
- budget_tokens: compute_thinking_budget(
- model.min_thinking_budget(),
- model.max_thinking_budget(),
- max_output_tokens.min(u32::MAX as u64) as u32,
- ),
- }
- } else {
- AnthropicModelMode::Default
- },
- AnthropicPromptCacheMode::Legacy,
- &PROVIDER_ID,
- )?;
+ if model.supports_messages() {
+ let location = intent_to_chat_location(request.intent);
+ let model = model.clone();
+ let copilot_chat = copilot_chat.clone();
+ let request_limiter = request_limiter.clone();
+ let future = cx.spawn(async move |cx| {
+ let max_output_tokens = model
+ .max_output_tokens()
+ .or(request.max_output_tokens)
+ .ok_or_else(|| anyhow!("Copilot did not provide an output limit for this model"))?;
+ let effort = request
+ .thinking_effort
+ .as_ref()
+ .and_then(|e| anthropic::Effort::from_str(e).ok());
+
+ let mut anthropic_request = into_anthropic(
+ request,
+ model.id().to_string(),
+ 0.0,
+ max_output_tokens,
+ if model.supports_adaptive_thinking() {
+ AnthropicModelMode::Thinking {
+ budget_tokens: None,
+ }
+ } else if model.supports_thinking() {
+ AnthropicModelMode::Thinking {
+ budget_tokens: compute_thinking_budget(
+ model.min_thinking_budget(),
+ model.max_thinking_budget(),
+ max_output_tokens.min(u32::MAX as u64) as u32,
+ ),
+ }
+ } else {
+ AnthropicModelMode::Default
+ },
+ AnthropicPromptCacheMode::Legacy,
+ &PROVIDER_ID,
+ )?;
- anthropic_request.temperature = None;
+ anthropic_request.temperature = None;
- // The Copilot proxy doesn't support eager_input_streaming on tools.
- for tool in &mut anthropic_request.tools {
- tool.eager_input_streaming = false;
- }
+ // The Copilot proxy doesn't support eager_input_streaming on tools.
+ for tool in &mut anthropic_request.tools {
+ tool.eager_input_streaming = false;
+ }
- if model.supports_adaptive_thinking() {
- if anthropic_request.thinking.is_some() {
- anthropic_request.thinking = Some(anthropic::Thinking::Adaptive {
- display: Some(anthropic::AdaptiveThinkingDisplay::Summarized),
- // Thinking block binding needs a beta header on
- // the upstream Anthropic request, which the
- // Copilot proxy controls, so opting in belongs
- // server-side.
- block_binding: None,
+ if model.supports_adaptive_thinking() {
+ if anthropic_request.thinking.is_some() {
+ anthropic_request.thinking = Some(anthropic::Thinking::Adaptive {
+ display: Some(anthropic::AdaptiveThinkingDisplay::Summarized),
+ // Thinking block binding needs a beta header on
+ // the upstream Anthropic request, which the
+ // Copilot proxy controls, so opting in belongs
+ // server-side.
+ block_binding: None,
+ });
+ anthropic_request.output_config =
+ effort.map(|effort| anthropic::OutputConfig {
+ effort: Some(effort),
});
- anthropic_request.output_config =
- effort.map(|effort| anthropic::OutputConfig {
- effort: Some(effort),
- });
- }
}
+ }
- let anthropic_beta =
- if !model.supports_adaptive_thinking() && model.supports_thinking() {
- Some("interleaved-thinking-2025-05-14".to_string())
- } else {
- None
- };
+ let anthropic_beta = if !model.supports_adaptive_thinking() && model.supports_thinking()
+ {
+ Some("interleaved-thinking-2025-05-14".to_string())
+ } else {
+ None
+ };
- let body = serde_json::to_string(&anthropic::StreamingRequest {
- base: anthropic_request,
- stream: true,
- })
- .map_err(|e| anyhow::anyhow!(e))?;
-
- let stream = CopilotChat::stream_messages(
- copilot_chat,
- body,
- location,
- is_user_initiated,
- anthropic_beta,
- cx.clone(),
- );
- let executor = cx.background_executor().clone();
-
- request_limiter
- .stream(async move {
- let events = stream.await?;
- let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);
- Ok(language_model::stream_in_background(
- mapper.map_stream(events).boxed(),
- executor,
- ))
- })
- .await
- });
- return async move { Ok(future.await?.boxed()) }.boxed();
- }
+ let body = serde_json::to_string(&anthropic::StreamingRequest {
+ base: anthropic_request,
+ stream: true,
+ })
+ .map_err(|e| anyhow::anyhow!(e))?;
- if self.model.supports_response() {
- let location = intent_to_chat_location(request.intent);
- let responses_request = match into_copilot_responses(&self.model, request) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let copilot_chat = self.copilot_chat.clone();
- let request_limiter = self.request_limiter.clone();
- let future = cx.spawn(async move |cx| {
- let request = CopilotChat::stream_response(
- copilot_chat,
- responses_request,
- location,
- is_user_initiated,
- cx.clone(),
- );
- let executor = cx.background_executor().clone();
- request_limiter
- .stream(async move {
- let stream = request.await?;
- let mapper = CopilotResponsesEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(stream).boxed(),
- executor,
- ))
- })
- .await
- });
- return async move { Ok(future.await?.boxed()) }.boxed();
- }
+ let stream = CopilotChat::stream_messages(
+ copilot_chat,
+ body,
+ location,
+ is_user_initiated,
+ anthropic_beta,
+ cx.clone(),
+ );
+ let executor = cx.background_executor().clone();
+
+ request_limiter
+ .stream(async move {
+ let events = stream.await?;
+ let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(events).boxed(),
+ executor,
+ ))
+ })
+ .await
+ });
+ return async move { Ok(future.await?.boxed()) }.boxed();
+ }
+ if model.supports_response() {
let location = intent_to_chat_location(request.intent);
- let copilot_request = match into_copilot_chat(&self.model, request) {
+ let responses_request = match into_copilot_responses(model, request) {
Ok(request) => request,
- Err(err) => return futures::future::ready(Err(err.into())).boxed(),
+ Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let is_streaming = copilot_request.stream;
-
- let copilot_chat = self.copilot_chat.clone();
- let request_limiter = self.request_limiter.clone();
+ let copilot_chat = copilot_chat.clone();
+ let request_limiter = request_limiter.clone();
let future = cx.spawn(async move |cx| {
- let request = CopilotChat::stream_completion(
+ let request = CopilotChat::stream_response(
copilot_chat,
- copilot_request,
+ responses_request,
location,
is_user_initiated,
cx.clone(),
);
+ let executor = cx.background_executor().clone();
request_limiter
.stream(async move {
- let response = request.await?;
- Ok(map_to_language_model_completion_events(
- response,
- is_streaming,
+ let stream = request.await?;
+ let mapper = CopilotResponsesEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream).boxed(),
+ executor,
))
})
.await
});
- async move { Ok(future.await?.boxed()) }.boxed()
+ return async move { Ok(future.await?.boxed()) }.boxed();
}
+
+ let location = intent_to_chat_location(request.intent);
+ let copilot_request = match into_copilot_chat(model, request) {
+ Ok(request) => request,
+ Err(err) => return futures::future::ready(Err(err.into())).boxed(),
+ };
+ let is_streaming = copilot_request.stream;
+
+ let copilot_chat = copilot_chat.clone();
+ let request_limiter = request_limiter.clone();
+ let future = cx.spawn(async move |cx| {
+ let request = CopilotChat::stream_completion(
+ copilot_chat,
+ copilot_request,
+ location,
+ is_user_initiated,
+ cx.clone(),
+ );
+ request_limiter
+ .stream(async move {
+ let response = request.await?;
+ Ok(map_to_language_model_completion_events(
+ response,
+ is_streaming,
+ ))
+ })
+ .await
+ });
+ async move { Ok(future.await?.boxed()) }.boxed()
}
pub fn map_to_language_model_completion_events(
@@ -1311,19 +1250,10 @@ mod tests {
use futures::StreamExt;
use language_model::ProviderErrorCategory;
use serde_json::json;
+ use std::sync::Arc;
- #[gpui::test]
- fn language_model_exposes_token_limits(cx: &mut gpui::TestAppContext) {
- use gpui::AppContext as _;
-
- let copilot_chat = cx.new(|cx| {
- CopilotChat::new(
- Arc::new(http_client::BlockedHttpClient::new()),
- Arc::new(EmptyCredentialsProvider),
- crate::CopilotChatConfiguration::default(),
- cx,
- )
- });
+ #[test]
+ fn language_model_exposes_token_limits() {
for (limits, input, context, output) in [
(
json!({"max_context_window_tokens": 200_000, "max_prompt_tokens": 90_000, "max_output_tokens": 16_384}),
@@ -1353,41 +1283,10 @@ mod tests {
let mut value = serde_json::to_value(test_responses_model()).unwrap();
value["capabilities"]["limits"] = limits;
let descriptor = serde_json::from_value(value).unwrap();
- let model = create_language_model(descriptor, copilot_chat.clone());
- assert_eq!(model.max_token_count(), context);
- assert_eq!(model.max_input_tokens(), input);
- assert_eq!(model.max_total_tokens(), Some(context));
- assert_eq!(model.max_output_tokens(), output);
- }
- }
-
- struct EmptyCredentialsProvider;
-
- impl credentials_provider::CredentialsProvider for EmptyCredentialsProvider {
- fn read_credentials<'a>(
- &'a self,
- _url: &'a str,
- _cx: &'a AsyncApp,
- ) -> Pin<Box<dyn Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>> {
- Box::pin(async { Ok(None) })
- }
-
- fn write_credentials<'a>(
- &'a self,
- _url: &'a str,
- _username: &'a str,
- _password: &'a [u8],
- _cx: &'a AsyncApp,
- ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
- Box::pin(async { Ok(()) })
- }
-
- fn delete_credentials<'a>(
- &'a self,
- _url: &'a str,
- _cx: &'a AsyncApp,
- ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
- Box::pin(async { Ok(()) })
+ let model = language_model(&descriptor);
+ assert_eq!(model.max_token_count, context);
+ assert_eq!(model.max_input_tokens, input);
+ assert_eq!(model.max_output_tokens, output);
}
}
diff --git a/crates/eval_cli/src/main.rs b/crates/eval_cli/src/main.rs
index e25730dc5e0..b223fb2f2ed 100644
--- a/crates/eval_cli/src/main.rs
+++ b/crates/eval_cli/src/main.rs
@@ -463,10 +463,7 @@ fn ensure_provider_authenticated(selected: &SelectedModel, cx: &gpui::App) -> Re
Ok(())
}
-fn find_available_model(
- selected: &SelectedModel,
- cx: &gpui::App,
-) -> Option<Arc<dyn LanguageModel>> {
+fn find_available_model(selected: &SelectedModel, cx: &gpui::App) -> Option<LanguageModel> {
let registry = LanguageModelRegistry::global(cx);
let models = registry.read(cx).available_models(cx).collect::<Vec<_>>();
@@ -619,19 +616,11 @@ async fn run_agent(
let registry = LanguageModelRegistry::global(cx);
let model = find_available_model(&selected, cx)
.ok_or_else(|| model_not_found_error(model_name, cx))?;
- let provider = registry
- .read(cx)
- .provider(&model.provider_id())
- .context("Provider not found")?;
-
let supports_thinking = model.supports_thinking();
let model_id = model.id().0.to_string();
registry.update(cx, |registry, cx| {
- registry.set_default_model(
- Some(language_model::ConfiguredModel { provider, model }),
- cx,
- );
+ registry.set_default_model(Some(model), cx);
});
let enable_thinking = thinking_override.unwrap_or(supports_thinking);
diff --git a/crates/git_ui/src/git_panel.rs b/crates/git_ui/src/git_panel.rs
index 31c7e55cb4e..1e3e49da653 100644
--- a/crates/git_ui/src/git_panel.rs
+++ b/crates/git_ui/src/git_panel.rs
@@ -57,8 +57,8 @@ use gpui::{
use itertools::Itertools;
use language::{Buffer, BufferEvent, File};
use language_model::{
- CompletionIntent, ConfiguredModel, Event as LanguageModelEvent, LanguageModelRegistry,
- LanguageModelRequest, LanguageModelRequestMessage, Role,
+ CompletionIntent, Event as LanguageModelEvent, LanguageModelRegistry, LanguageModelRequest,
+ LanguageModelRequestMessage, Role,
};
use menu;
use multi_buffer::ExcerptBoundaryInfo;
@@ -4117,11 +4117,22 @@ impl GitPanel {
return;
}
- let Some(ConfiguredModel { provider, model }) =
- LanguageModelRegistry::read_global(cx).commit_message_model(cx)
- else {
+ let registry = LanguageModelRegistry::read_global(cx);
+ let Some(model) = registry.commit_message_model(cx) else {
return;
};
+ let provider = match registry.provider_for_model(&model) {
+ Ok(provider) => provider,
+ Err(error) => {
+ if let Some(workspace) = self.workspace.upgrade() {
+ workspace.update(cx, |workspace, cx| {
+ workspace
+ .show_error(format!("Failed to generate commit message: {error}"), cx);
+ });
+ }
+ return;
+ }
+ };
let Some(repo) = self.active_repository.as_ref() else {
return;
@@ -4155,11 +4166,7 @@ impl GitPanel {
});
if let Some(task) = cx.update(|cx| {
- if !provider.is_authenticated(cx) {
- Some(provider.authenticate(cx))
- } else {
- None
- }
+ (!provider.is_authenticated(cx)).then(|| provider.authenticate(cx))
}) {
task.await.log_err();
}
@@ -4240,7 +4247,7 @@ impl GitPanel {
max_output_tokens: None,
};
- let stream = model.stream_completion_text(request, cx);
+ let stream = provider.stream_completion_text(&model, request, cx);
match stream.await {
Ok(mut messages) => {
if !text_empty {
diff --git a/crates/language_model/src/fake_provider.rs b/crates/language_model/src/fake_provider.rs
index 4aa9d34cfeb..d8dee6dcba0 100644
--- a/crates/language_model/src/fake_provider.rs
+++ b/crates/language_model/src/fake_provider.rs
@@ -1,34 +1,48 @@
use crate::{
AuthenticateError, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice,
+ LanguageModelCompletionStream, LanguageModelId, LanguageModelName, LanguageModelProvider,
+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
+ LanguageModelRequest,
};
use anyhow::anyhow;
-use futures::{FutureExt, channel::mpsc, future::BoxFuture, stream::BoxStream, stream::StreamExt};
+use futures::{FutureExt, channel::mpsc, future::BoxFuture, stream::StreamExt};
use gpui::{App, AsyncApp, Entity, Task};
use http_client::Result;
use parking_lot::Mutex;
use std::collections::VecDeque;
-use std::sync::{
- Arc,
- atomic::{AtomicBool, AtomicU64, Ordering::SeqCst},
-};
-#[derive(Clone)]
+/// A provider for tests that serves any model id and records every request.
+///
+/// Tests get models with [`Self::model`] and drive responses to the requests
+/// the provider has received, like `send_last_text`.
pub struct FakeLanguageModelProvider {
id: LanguageModelProviderId,
name: LanguageModelProviderName,
- models: Vec<Arc<dyn LanguageModel>>,
+ state: Mutex<FakeProviderState>,
+}
+
+#[derive(Default)]
+struct FakeProviderState {
+ models: Vec<LanguageModel>,
+ pending_completions: Vec<PendingCompletion>,
+ forbid_requests: bool,
+ input_token_counts: VecDeque<u64>,
+ input_token_count_requests: Vec<LanguageModelRequest>,
+}
+
+struct PendingCompletion {
+ model_id: LanguageModelId,
+ request: LanguageModelRequest,
+ events:
+ mpsc::UnboundedSender<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
}
impl Default for FakeLanguageModelProvider {
fn default() -> Self {
- Self {
- id: LanguageModelProviderId::from("fake".to_string()),
- name: LanguageModelProviderName::from("Fake".to_string()),
- models: vec![Arc::new(FakeLanguageModel::default())],
- }
+ Self::new(
+ LanguageModelProviderId::from("fake".to_string()),
+ LanguageModelProviderName::from("Fake".to_string()),
+ )
}
}
@@ -49,16 +63,16 @@ impl LanguageModelProvider for FakeLanguageModelProvider {
self.name.clone()
}
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- self.models.first().cloned()
+ fn default_model(&self, _cx: &App) -> Option<LanguageModel> {
+ self.state.lock().models.first().cloned()
}
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- self.models.first().cloned()
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ self.state.lock().models.first().cloned()
}
- fn provided_models(&self, _: &App) -> Vec<Arc<dyn LanguageModel>> {
- self.models.clone()
+ fn provided_models(&self, _: &App) -> Vec<LanguageModel> {
+ self.state.lock().models.clone()
}
fn is_authenticated(&self, _: &App) -> bool {
@@ -72,348 +86,256 @@ impl LanguageModelProvider for FakeLanguageModelProvider {
fn settings_view(&self, _: &mut App) -> Option<crate::ProviderSettingsView> {
None
}
-}
-impl FakeLanguageModelProvider {
- pub fn new(id: LanguageModelProviderId, name: LanguageModelProviderName) -> Self {
- Self {
- id,
- name,
- models: vec![Arc::new(FakeLanguageModel::default())],
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ _: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let mut state = self.state.lock();
+ if state.forbid_requests {
+ return async { Err(anyhow!("requests are forbidden").into()) }.boxed();
}
+ let (events, rx) = mpsc::unbounded();
+ state.pending_completions.push(PendingCompletion {
+ model_id: model.id.clone(),
+ request,
+ events,
+ });
+ async move { Ok(rx.boxed()) }.boxed()
}
- pub fn with_models(mut self, models: Vec<Arc<dyn LanguageModel>>) -> Self {
- self.models = models;
- self
- }
-
- pub fn test_model(&self) -> FakeLanguageModel {
- FakeLanguageModel::default()
- }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ToolUseRequest {
- pub request: LanguageModelRequest,
- pub name: String,
- pub description: String,
- pub schema: serde_json::Value,
-}
-
-pub struct FakeLanguageModel {
- id: LanguageModelId,
- name: LanguageModelName,
- provider_id: LanguageModelProviderId,
- provider_name: LanguageModelProviderName,
- current_completion_txs: Mutex<
- Vec<(
- LanguageModelRequest,
- mpsc::UnboundedSender<
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- )>,
- >,
- forbid_requests: AtomicBool,
- supports_thinking: AtomicBool,
- supports_disabling_thinking: AtomicBool,
- supports_streaming_tools: AtomicBool,
- supports_images: AtomicBool,
- supports_server_side_compaction: AtomicBool,
- max_token_count: AtomicU64,
- max_input_tokens: Option<u64>,
- max_output_tokens: AtomicU64,
- input_token_counts: Mutex<VecDeque<u64>>,
- input_token_count_requests: Mutex<Vec<LanguageModelRequest>>,
-}
-
-impl Default for FakeLanguageModel {
- fn default() -> Self {
- Self {
- id: LanguageModelId::from("fake".to_string()),
- name: LanguageModelName::from("Fake".to_string()),
- provider_id: LanguageModelProviderId::from("fake".to_string()),
- provider_name: LanguageModelProviderName::from("Fake".to_string()),
- current_completion_txs: Mutex::new(Vec::new()),
- forbid_requests: AtomicBool::new(false),
- supports_thinking: AtomicBool::new(false),
- supports_disabling_thinking: AtomicBool::new(true),
- supports_streaming_tools: AtomicBool::new(false),
- supports_images: AtomicBool::new(false),
- supports_server_side_compaction: AtomicBool::new(false),
- max_token_count: AtomicU64::new(1_000_000),
- max_input_tokens: None,
- max_output_tokens: AtomicU64::new(0),
- input_token_counts: Mutex::new(VecDeque::new()),
- input_token_count_requests: Mutex::new(Vec::new()),
+ fn count_input_tokens(
+ &self,
+ _model: &LanguageModel,
+ request: LanguageModelRequest,
+ _: &AsyncApp,
+ ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
+ let mut state = self.state.lock();
+ if state.forbid_requests {
+ return async { Err(anyhow!("requests are forbidden").into()) }.boxed();
}
+ state.input_token_count_requests.push(request);
+ let count = state.input_token_counts.pop_front();
+ async move { Ok(count) }.boxed()
}
}
-impl FakeLanguageModel {
- pub fn with_id_and_thinking(
- provider_id: &str,
- id: &str,
- name: &str,
- supports_thinking: bool,
- ) -> Self {
- Self {
- id: LanguageModelId::from(id.to_string()),
- name: LanguageModelName::from(name.to_string()),
- provider_id: LanguageModelProviderId::from(provider_id.to_string()),
- supports_thinking: AtomicBool::new(supports_thinking),
- ..Default::default()
- }
+impl FakeLanguageModelProvider {
+ /// A provider offering one model, with id `fake`.
+ pub fn new(id: LanguageModelProviderId, name: LanguageModelProviderName) -> Self {
+ let provider = Self {
+ id,
+ name,
+ state: Mutex::default(),
+ };
+ provider.update_model("fake", |model| {
+ model.name = LanguageModelName::from("Fake".to_string())
+ });
+ provider
+ }
+
+ /// The model this provider offers as `id`, adding it with default
+ /// capabilities if it isn't offered yet.
+ pub fn model(&self, id: &str) -> LanguageModel {
+ self.update_model(id, |_| {})
+ }
+
+ /// Changes the capabilities of the model offered as `id`, adding it first
+ /// if needed, and returns the updated model.
+ pub fn update_model(&self, id: &str, update: impl FnOnce(&mut LanguageModel)) -> LanguageModel {
+ let mut state = self.state.lock();
+ let index = match state
+ .models
+ .iter()
+ .position(|model| model.id.0.as_ref() == id)
+ {
+ Some(index) => index,
+ None => {
+ state.models.push(LanguageModel {
+ supports_disabling_thinking: true,
+ ..LanguageModel::new(
+ LanguageModelId::from(id.to_string()),
+ LanguageModelName::from(id.to_string()),
+ self.id.clone(),
+ self.name.clone(),
+ "fake",
+ 1_000_000,
+ )
+ });
+ state.models.len() - 1
+ }
+ };
+ update(&mut state.models[index]);
+ state.models[index].clone()
}
pub fn allow_requests(&self) {
- self.forbid_requests.store(false, SeqCst);
+ self.state.lock().forbid_requests = false;
}
pub fn forbid_requests(&self) {
- self.forbid_requests.store(true, SeqCst);
- }
-
- pub fn set_supports_thinking(&self, supports: bool) {
- self.supports_thinking.store(supports, SeqCst);
- }
-
- pub fn set_supports_disabling_thinking(&self, supports: bool) {
- self.supports_disabling_thinking.store(supports, SeqCst);
- }
-
- pub fn set_supports_streaming_tools(&self, supports: bool) {
- self.supports_streaming_tools.store(supports, SeqCst);
- }
-
- pub fn set_supports_images(&self, supports: bool) {
- self.supports_images.store(supports, SeqCst);
- }
-
- pub fn set_supports_server_side_compaction(&self, supports: bool) {
- self.supports_server_side_compaction.store(supports, SeqCst);
- }
-
- pub fn set_max_token_count(&self, count: u64) {
- self.max_token_count.store(count, SeqCst);
- }
-
- pub fn set_max_input_tokens(&mut self, count: u64) {
- self.max_input_tokens = Some(count);
- }
-
- pub fn set_max_output_tokens(&self, count: Option<u64>) {
- self.max_output_tokens
- .store(count.unwrap_or_default(), SeqCst);
+ self.state.lock().forbid_requests = true;
}
pub fn queue_input_token_count(&self, count: u64) {
- self.input_token_counts.lock().push_back(count);
+ self.state.lock().input_token_counts.push_back(count);
}
pub fn input_token_count_requests(&self) -> Vec<LanguageModelRequest> {
- self.input_token_count_requests.lock().clone()
+ self.state.lock().input_token_count_requests.clone()
}
+ /// Requests still streaming, to any model, oldest first.
pub fn pending_completions(&self) -> Vec<LanguageModelRequest> {
- self.current_completion_txs
+ self.state
.lock()
+ .pending_completions
.iter()
- .map(|(request, _)| request.clone())
+ .map(|pending| pending.request.clone())
+ .collect()
+ }
+
+ /// Requests still streaming to `model`, oldest first.
+ pub fn pending_completions_for(&self, model: &LanguageModel) -> Vec<LanguageModelRequest> {
+ self.state
+ .lock()
+ .pending_completions
+ .iter()
+ .filter(|pending| pending.model_id == model.id)
+ .map(|pending| pending.request.clone())
.collect()
}
pub fn completion_count(&self) -> usize {
- self.current_completion_txs.lock().len()
+ self.state.lock().pending_completions.len()
}
- pub fn send_completion_stream_text_chunk(
+ /// Streams `chunk` as text to the oldest pending `request` to `model`.
+ pub fn send_text(
&self,
+ model: &LanguageModel,
request: &LanguageModelRequest,
chunk: impl Into<String>,
) {
- self.send_completion_stream_event(
+ self.send_event(
+ model,
request,
LanguageModelCompletionEvent::Text(chunk.into()),
);
}
- pub fn send_completion_stream_event(
+ /// Streams `event` to the oldest pending `request` to `model`.
+ pub fn send_event(
&self,
+ model: &LanguageModel,
request: &LanguageModelRequest,
event: impl Into<LanguageModelCompletionEvent>,
) {
- let current_completion_txs = self.current_completion_txs.lock();
- let tx = current_completion_txs
- .iter()
- .find(|(req, _)| req == request)
- .map(|(_, tx)| tx)
- .unwrap();
- tx.unbounded_send(Ok(event.into())).unwrap();
+ let state = self.state.lock();
+ state.send(state.pending_index(model, request), Ok(event.into()));
}
- pub fn send_completion_stream_error(
+ /// Streams `error` to the oldest pending `request` to `model`.
+ pub fn send_error(
&self,
+ model: &LanguageModel,
request: &LanguageModelRequest,
error: impl Into<LanguageModelCompletionError>,
) {
- let current_completion_txs = self.current_completion_txs.lock();
- let tx = current_completion_txs
- .iter()
- .find(|(req, _)| req == request)
- .map(|(_, tx)| tx)
- .unwrap();
- tx.unbounded_send(Err(error.into())).unwrap();
+ let state = self.state.lock();
+ state.send(state.pending_index(model, request), Err(error.into()));
}
- pub fn end_completion_stream(&self, request: &LanguageModelRequest) {
- self.current_completion_txs
- .lock()
- .retain(|(req, _)| req != request);
+ /// Ends the stream of the oldest pending `request` to `model`.
+ pub fn end_stream(&self, model: &LanguageModel, request: &LanguageModelRequest) {
+ let mut state = self.state.lock();
+ let index = state.pending_index(model, request);
+ state.pending_completions.remove(index);
}
- pub fn is_completion_stream_closed(&self, request: &LanguageModelRequest) -> bool {
- self.current_completion_txs
+ /// Whether the oldest pending `request` to `model` has ended or had its
+ /// receiver dropped.
+ pub fn is_stream_closed(&self, model: &LanguageModel, request: &LanguageModelRequest) -> bool {
+ self.state
.lock()
+ .pending_completions
.iter()
- .find(|(pending_request, _)| pending_request == request)
- .is_none_or(|(_, sender)| sender.is_closed())
+ .find(|pending| pending.model_id == model.id && &pending.request == request)
+ .is_none_or(|pending| pending.events.is_closed())
}
- pub fn send_last_completion_stream_text_chunk(&self, chunk: impl Into<String>) {
- self.send_completion_stream_text_chunk(self.pending_completions().last().unwrap(), chunk);
+ /// Streams `chunk` as text to the most recent pending request to `model`.
+ pub fn send_last_text(&self, model: &LanguageModel, chunk: impl Into<String>) {
+ self.send_last_event(model, LanguageModelCompletionEvent::Text(chunk.into()));
}
- pub fn send_last_completion_stream_event(
+ /// Streams `event` to the most recent pending request to `model`.
+ pub fn send_last_event(
&self,
+ model: &LanguageModel,
event: impl Into<LanguageModelCompletionEvent>,
) {
- self.send_completion_stream_event(self.pending_completions().last().unwrap(), event);
+ let state = self.state.lock();
+ state.send(state.last_pending_index(model), Ok(event.into()));
}
- pub fn send_last_completion_stream_error(
+ /// Streams `error` to the most recent pending request to `model`.
+ pub fn send_last_error(
&self,
+ model: &LanguageModel,
error: impl Into<LanguageModelCompletionError>,
) {
- self.send_completion_stream_error(self.pending_completions().last().unwrap(), error);
+ let state = self.state.lock();
+ state.send(state.last_pending_index(model), Err(error.into()));
}
- pub fn end_last_completion_stream(&self) {
- self.end_completion_stream(self.pending_completions().last().unwrap());
+ /// Ends the stream of the most recent pending request to `model`.
+ pub fn end_last(&self, model: &LanguageModel) {
+ let mut state = self.state.lock();
+ let index = state.last_pending_index(model);
+ state.pending_completions.remove(index);
}
}
-impl LanguageModel for FakeLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- self.name.clone()
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- self.provider_id.clone()
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- self.provider_name.clone()
- }
-
- fn supports_tools(&self) -> bool {
- false
- }
-
- fn supports_tool_choice(&self, _choice: LanguageModelToolChoice) -> bool {
- false
- }
-
- fn supports_images(&self) -> bool {
- self.supports_images.load(SeqCst)
- }
-
- fn supports_server_side_compaction(&self) -> bool {
- self.supports_server_side_compaction.load(SeqCst)
- }
-
- fn supports_thinking(&self) -> bool {
- self.supports_thinking.load(SeqCst)
- }
-
- fn supports_disabling_thinking(&self) -> bool {
- self.supports_disabling_thinking.load(SeqCst)
- }
-
- fn supports_streaming_tools(&self) -> bool {
- self.supports_streaming_tools.load(SeqCst)
- }
-
- fn telemetry_id(&self) -> String {
- "fake".to_string()
- }
-
- fn max_token_count(&self) -> u64 {
- self.max_token_count.load(SeqCst)
- }
-
- fn max_input_tokens(&self) -> u64 {
- self.max_input_tokens
- .unwrap_or_else(|| self.max_token_count())
+impl FakeProviderState {
+ fn pending_index(&self, model: &LanguageModel, request: &LanguageModelRequest) -> usize {
+ self.pending_completions
+ .iter()
+ .position(|pending| pending.model_id == model.id && &pending.request == request)
+ .unwrap_or_else(|| {
+ panic!(
+ "no pending completion to model `{}` matches the request",
+ model.id.0
+ )
+ })
}
- fn max_output_tokens(&self) -> Option<u64> {
- let max_output_tokens = self.max_output_tokens.load(SeqCst);
- if max_output_tokens == 0 {
- None
- } else {
- Some(max_output_tokens)
- }
+ fn last_pending_index(&self, model: &LanguageModel) -> usize {
+ self.pending_completions
+ .iter()
+ .rposition(|pending| pending.model_id == model.id)
+ .unwrap_or_else(|| panic!("no pending completion to model `{}`", model.id.0))
}
- fn count_input_tokens(
+ fn send(
&self,
- request: LanguageModelRequest,
- _: &AsyncApp,
- ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
- if self.forbid_requests.load(SeqCst) {
- return async {
- Err(LanguageModelCompletionError::Other(anyhow!(
- "requests are forbidden"
- )))
- }
- .boxed();
- }
- self.input_token_count_requests.lock().push(request);
- let count = self.input_token_counts.lock().pop_front();
- async move { Ok(count) }.boxed()
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- _: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- if self.forbid_requests.load(SeqCst) {
- async move {
- Err(LanguageModelCompletionError::Other(anyhow!(
- "requests are forbidden"
- )))
- }
- .boxed()
- } else {
- let (tx, rx) = mpsc::unbounded();
- self.current_completion_txs.lock().push((request, tx));
- async move { Ok(rx.boxed()) }.boxed()
- }
+ index: usize,
+ event: Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
+ ) {
+ self.pending_completions[index]
+ .events
+ .unbounded_send(event)
+ .unwrap();
}
+}
- fn as_fake(&self) -> &Self {
- self
- }
+#[derive(Debug, PartialEq)]
+pub struct ToolUseRequest {
+ pub request: LanguageModelRequest,
+ pub name: String,
+ pub description: String,
+ pub schema: serde_json::Value,
}
diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs
index dc12a4559bf..5cd16b10063 100644
--- a/crates/language_model/src/language_model.rs
+++ b/crates/language_model/src/language_model.rs
@@ -88,187 +88,324 @@ impl Default for LanguageModelTextStream {
}
}
-pub trait LanguageModel: Send + Sync {
- fn id(&self) -> LanguageModelId;
- fn name(&self) -> LanguageModelName;
- fn provider_id(&self) -> LanguageModelProviderId;
- fn provider_name(&self) -> LanguageModelProviderName;
- fn upstream_provider_id(&self) -> LanguageModelProviderId {
- self.provider_id()
+/// A language model offered by a [`LanguageModelProvider`].
+///
+/// This is plain data: identity, capabilities, and limits. Everything that
+/// talks to a backend (streaming completions, counting tokens, compaction)
+/// lives on the provider, which is handed the model it should serve and
+/// resolves its own configuration for it by `id`. Find that provider with
+/// [`LanguageModelRegistry::provider_for_model`].
+#[derive(Clone, Debug, PartialEq)]
+pub struct LanguageModel {
+ /// Identifies the model within its provider. Stable across provider
+ /// refreshes; the name and capabilities may change.
+ pub id: LanguageModelId,
+ pub name: LanguageModelName,
+ pub provider_id: LanguageModelProviderId,
+ pub provider_name: LanguageModelProviderName,
+ /// The provider that ultimately serves requests, when it differs from
+ /// `provider_id` (for example, a model offered through a gateway).
+ pub upstream_provider_id: Option<LanguageModelProviderId>,
+ pub upstream_provider_name: Option<LanguageModelProviderName>,
+ pub telemetry_id: SharedString,
+ /// Whether this model is the "latest", so we can highlight it in the UI.
+ pub is_latest: bool,
+ /// Why the model is currently disabled, if it is.
+ pub disabled_reason: Option<DisabledReason>,
+ /// Whether requests to this model require the user to consent to the
+ /// upstream provider retaining inference logs (i.e. the model cannot be
+ /// offered with Zero Data Retention).
+ pub requires_data_retention: bool,
+ /// When this model refuses a request, the model ID to fall back to (same provider).
+ pub refusal_fallback_model_id: Option<&'static str>,
+ /// Information about the cost of using this model, if available.
+ pub cost_info: Option<LanguageModelCostInfo>,
+ pub supports_thinking: bool,
+ /// Whether thinking can be turned off entirely for this model. Some
+ /// models (e.g. Claude Fable 5) always think and cannot honor an "off"
+ /// request. Only meaningful when `supports_thinking` is `true`.
+ pub supports_disabling_thinking: bool,
+ pub supports_fast_mode: bool,
+ /// The effort levels that can be used when thinking.
+ pub supported_effort_levels: Arc<[LanguageModelEffortLevel]>,
+ /// Whether this model supports provider-side automatic context
+ /// compaction (requested via `LanguageModelRequest::compact_at_tokens`).
+ pub supports_server_side_compaction: bool,
+ pub supports_explicit_compaction: bool,
+ /// Whether native compaction honors `LanguageModelRequest::max_output_tokens`.
+ pub supports_explicit_compaction_output_limit: bool,
+ /// The provider-enforced input size required for explicit compaction.
+ pub minimum_explicit_compaction_input_tokens: Option<u64>,
+ pub supports_images: bool,
+ pub supports_tools: bool,
+ pub tool_choice_support: LanguageModelToolChoiceSupport,
+ /// Whether this model or provider supports streaming tool calls.
+ pub supports_streaming_tools: bool,
+ /// Whether this model/provider reports accurate split input/output token
+ /// counts. When true, the UI may show separate input/output token indicators.
+ pub supports_split_token_display: bool,
+ /// The model's context-window capacity.
+ pub max_token_count: u64,
+ /// The input ceiling before reserving output from any shared window.
+ /// Equals `max_token_count` unless the model has a separate prompt limit.
+ pub max_input_tokens: u64,
+ pub max_output_tokens: Option<u64>,
+}
+
+/// Which [`LanguageModelToolChoice`] values a model accepts.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct LanguageModelToolChoiceSupport {
+ pub auto: bool,
+ pub any: bool,
+ pub none: bool,
+}
+
+impl LanguageModelToolChoiceSupport {
+ pub const ALL: Self = Self {
+ auto: true,
+ any: true,
+ none: true,
+ };
+
+ pub fn supports(&self, choice: LanguageModelToolChoice) -> bool {
+ match choice {
+ LanguageModelToolChoice::Auto => self.auto,
+ LanguageModelToolChoice::Any => self.any,
+ LanguageModelToolChoice::None => self.none,
+ }
}
- fn upstream_provider_name(&self) -> LanguageModelProviderName {
- self.provider_name()
+}
+
+impl LanguageModel {
+ /// Creates a model with the given identity and conservative defaults for
+ /// every capability. Providers override the capabilities they support
+ /// with struct update syntax.
+ pub fn new(
+ id: LanguageModelId,
+ name: LanguageModelName,
+ provider_id: LanguageModelProviderId,
+ provider_name: LanguageModelProviderName,
+ telemetry_id: impl Into<SharedString>,
+ max_token_count: u64,
+ ) -> Self {
+ Self {
+ id,
+ name,
+ provider_id,
+ provider_name,
+ upstream_provider_id: None,
+ upstream_provider_name: None,
+ telemetry_id: telemetry_id.into(),
+ is_latest: false,
+ disabled_reason: None,
+ requires_data_retention: false,
+ refusal_fallback_model_id: None,
+ cost_info: None,
+ supports_thinking: false,
+ supports_disabling_thinking: true,
+ supports_fast_mode: false,
+ supported_effort_levels: Arc::default(),
+ supports_server_side_compaction: false,
+ supports_explicit_compaction: false,
+ supports_explicit_compaction_output_limit: false,
+ minimum_explicit_compaction_input_tokens: None,
+ supports_images: false,
+ supports_tools: false,
+ tool_choice_support: LanguageModelToolChoiceSupport::default(),
+ supports_streaming_tools: false,
+ supports_split_token_display: false,
+ max_token_count,
+ max_input_tokens: max_token_count,
+ max_output_tokens: None,
+ }
}
- /// Returns whether this model is the "latest", so we can highlight it in the UI.
- fn is_latest(&self) -> bool {
- false
+ pub fn id(&self) -> LanguageModelId {
+ self.id.clone()
}
- /// Whether the model is currently disabled and, if so, why this is the case.
- fn is_disabled(&self) -> Option<DisabledReason> {
- None
+ pub fn name(&self) -> LanguageModelName {
+ self.name.clone()
}
- /// Whether requests to this model require the user to consent to the
- /// upstream provider retaining inference logs (i.e. the model cannot be
- /// offered with Zero Data Retention).
- fn requires_data_retention(&self) -> bool {
- false
+ pub fn provider_id(&self) -> LanguageModelProviderId {
+ self.provider_id.clone()
}
- /// When this model refuses a request, the model ID to fall back to (same provider).
- fn refusal_fallback_model_id(&self) -> Option<&'static str> {
- None
+ pub fn provider_name(&self) -> LanguageModelProviderName {
+ self.provider_name.clone()
}
- fn telemetry_id(&self) -> String;
+ pub fn upstream_provider_id(&self) -> LanguageModelProviderId {
+ self.upstream_provider_id
+ .clone()
+ .unwrap_or_else(|| self.provider_id.clone())
+ }
- fn api_key(&self, _cx: &App) -> Option<String> {
- None
+ pub fn upstream_provider_name(&self) -> LanguageModelProviderName {
+ self.upstream_provider_name
+ .clone()
+ .unwrap_or_else(|| self.provider_name.clone())
}
- /// Information about the cost of using this model, if available.
- fn model_cost_info(&self) -> Option<LanguageModelCostInfo> {
- None
+ pub fn is_latest(&self) -> bool {
+ self.is_latest
}
- /// Whether this model supports thinking.
- fn supports_thinking(&self) -> bool {
- false
+ pub fn is_disabled(&self) -> Option<DisabledReason> {
+ self.disabled_reason.clone()
}
- /// Whether thinking can be turned off entirely for this model. Some
- /// models (e.g. Claude Fable 5) always think and cannot honor an "off"
- /// request. Only meaningful when `supports_thinking` returns `true`.
- fn supports_disabling_thinking(&self) -> bool {
- true
+ pub fn requires_data_retention(&self) -> bool {
+ self.requires_data_retention
}
- fn supports_fast_mode(&self) -> bool {
- false
+ pub fn refusal_fallback_model_id(&self) -> Option<&'static str> {
+ self.refusal_fallback_model_id
}
- /// Returns the list of supported effort levels that can be used when thinking.
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- Vec::new()
+ pub fn telemetry_id(&self) -> String {
+ self.telemetry_id.to_string()
+ }
+
+ pub fn model_cost_info(&self) -> Option<LanguageModelCostInfo> {
+ self.cost_info.clone()
+ }
+
+ pub fn supports_thinking(&self) -> bool {
+ self.supports_thinking
+ }
+
+ pub fn supports_disabling_thinking(&self) -> bool {
+ self.supports_disabling_thinking
+ }
+
+ pub fn supports_fast_mode(&self) -> bool {
+ self.supports_fast_mode
+ }
+
+ pub fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
+ self.supported_effort_levels.to_vec()
}
/// Returns the default effort level to use when thinking.
- fn default_effort_level(&self) -> Option<LanguageModelEffortLevel> {
- self.supported_effort_levels()
- .into_iter()
+ pub fn default_effort_level(&self) -> Option<LanguageModelEffortLevel> {
+ self.supported_effort_levels
+ .iter()
.find(|effort_level| effort_level.is_default)
+ .cloned()
}
- /// Whether this model supports provider-side automatic context
- /// compaction (requested via `LanguageModelRequest::compact_at_tokens`).
- fn supports_server_side_compaction(&self) -> bool {
- false
+ pub fn supports_server_side_compaction(&self) -> bool {
+ self.supports_server_side_compaction
}
- fn supports_explicit_compaction(&self) -> bool {
- false
+ pub fn supports_explicit_compaction(&self) -> bool {
+ self.supports_explicit_compaction
}
- /// Whether native compaction honors `LanguageModelRequest::max_output_tokens`.
- fn supports_explicit_compaction_output_limit(&self) -> bool {
- false
+ pub fn supports_explicit_compaction_output_limit(&self) -> bool {
+ self.supports_explicit_compaction_output_limit
}
- /// The provider-enforced input size required for explicit compaction.
- fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
- None
+ pub fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
+ self.minimum_explicit_compaction_input_tokens
}
- fn compact(
- &self,
- _request: LanguageModelRequest,
- _cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- let provider = self.provider_name();
- async move {
- Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
- "{provider} does not support explicit compaction"
- )))
- }
- .boxed()
+ pub fn supports_images(&self) -> bool {
+ self.supports_images
}
- /// Whether this model supports images
- fn supports_images(&self) -> bool;
-
- /// Whether this model supports tools.
- fn supports_tools(&self) -> bool;
+ pub fn supports_tools(&self) -> bool {
+ self.supports_tools
+ }
- /// Whether this model supports choosing which tool to use.
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool;
+ pub fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
+ self.tool_choice_support.supports(choice)
+ }
- /// Returns whether this model or provider supports streaming tool calls;
- fn supports_streaming_tools(&self) -> bool {
- false
+ pub fn supports_streaming_tools(&self) -> bool {
+ self.supports_streaming_tools
}
- /// Returns whether this model/provider reports accurate split input/output token counts.
- /// When true, the UI may show separate input/output token indicators.
- fn supports_split_token_display(&self) -> bool {
- false
+ pub fn supports_split_token_display(&self) -> bool {
+ self.supports_split_token_display
}
- /// Returns the model's context-window capacity.
- fn max_token_count(&self) -> u64;
+ pub fn max_token_count(&self) -> u64 {
+ self.max_token_count
+ }
/// Returns the input ceiling before reserving output from any shared window.
- ///
- /// Models with a separate prompt limit override the context-window default.
- fn max_input_tokens(&self) -> u64 {
- self.max_token_count()
+ pub fn max_input_tokens(&self) -> u64 {
+ self.max_input_tokens
}
- /// Counts request input without generating output, when supported by the provider.
+ /// Returns the combined input and output ceiling, if one applies.
///
- /// Counts may be estimates and differ from subsequent measured usage. Callers
- /// choose the content to count; this does not infer which input is already
- /// covered by a previous usage report. Unsupported providers return `None`.
- fn count_input_tokens(
- &self,
- _request: LanguageModelRequest,
- _cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
- async { Ok(None) }.boxed()
+ /// This shares the context window with output.
+ pub fn max_total_tokens(&self) -> Option<u64> {
+ Some(self.max_token_count)
}
- /// Returns the combined input and output ceiling, if one applies.
- ///
- /// The conservative default shares the context window with output. `None`
- /// means generation does not consume that window, not merely that the API
- /// validates input separately or stops generation at the window boundary.
- fn max_total_tokens(&self) -> Option<u64> {
- Some(self.max_token_count())
+ pub fn max_output_tokens(&self) -> Option<u64> {
+ self.max_output_tokens
}
+}
- fn max_output_tokens(&self) -> Option<u64> {
- None
+/// The error for a request to `model` when its provider doesn't offer it.
+pub fn unavailable_error(model: &LanguageModel) -> LanguageModelCompletionError {
+ LanguageModelCompletionError::ModelUnavailable {
+ provider: model.provider_name.clone(),
+ model: model.id.clone(),
+ }
+}
+
+/// Either a built-in icon name or a path to an external SVG.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum IconOrSvg {
+ /// A built-in icon from Zed's icon set.
+ Icon(IconName),
+ /// Path to a custom SVG icon file.
+ Svg(SharedString),
+}
+
+impl Default for IconOrSvg {
+ fn default() -> Self {
+ Self::Icon(IconName::ZedAssistant)
}
+}
+pub trait LanguageModelProvider: 'static {
+ fn id(&self) -> LanguageModelProviderId;
+ fn name(&self) -> LanguageModelProviderName;
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::default()
+ }
+ fn default_model(&self, cx: &App) -> Option<LanguageModel>;
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel>;
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel>;
+ fn recommended_models(&self, _cx: &App) -> Vec<LanguageModel> {
+ Vec::new()
+ }
+
+ /// Streams a completion of `request` from `model`, which must be one of
+ /// this provider's models.
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- >;
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>;
+ /// Streams the text of a completion of `request` from `model`.
fn stream_completion_text(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<LanguageModelTextStream, LanguageModelCompletionError>> {
- let future = self.stream_completion(request, cx);
+ let future = self.stream_completion(model, request, cx);
async move {
let events = future.await?;
@@ -329,18 +466,19 @@ pub trait LanguageModel: Send + Sync {
.boxed()
}
+ /// Completes `request` from `model`, resolving to its first complete tool use.
fn stream_completion_tool(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<LanguageModelToolUse, LanguageModelCompletionError>> {
- let future = self.stream_completion(request, cx);
+ let future = self.stream_completion(model, request, cx);
async move {
let events = future.await?;
let mut events = events.fuse();
- // Iterate through events until we find a complete ToolUse
while let Some(event) = events.next().await {
match event {
Ok(LanguageModelCompletionEvent::ToolUse(tool_use))
@@ -355,7 +493,6 @@ pub trait LanguageModel: Send + Sync {
}
}
- // Stream ended without a complete tool use
Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
"Stream ended without receiving a complete tool use"
)))
@@ -363,54 +500,42 @@ pub trait LanguageModel: Send + Sync {
.boxed()
}
- #[cfg(any(test, feature = "test-support"))]
- fn as_fake(&self) -> &fake_provider::FakeLanguageModel {
- unimplemented!()
+ /// Counts request input without generating output, when supported by the provider.
+ ///
+ /// Counts may be estimates and differ from subsequent measured usage. Callers
+ /// choose the content to count; this does not infer which input is already
+ /// covered by a previous usage report. Unsupported providers return `None`.
+ fn count_input_tokens(
+ &self,
+ _model: &LanguageModel,
+ _request: LanguageModelRequest,
+ _cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
+ async { Ok(None) }.boxed()
}
-}
-impl std::fmt::Debug for dyn LanguageModel {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("<dyn LanguageModel>")
- .field("id", &self.id())
- .field("name", &self.name())
- .field("provider_id", &self.provider_id())
- .field("provider_name", &self.provider_name())
- .field("upstream_provider_name", &self.upstream_provider_name())
- .field("upstream_provider_id", &self.upstream_provider_id())
- .field("upstream_provider_id", &self.upstream_provider_id())
- .field("supports_streaming_tools", &self.supports_streaming_tools())
- .finish()
+ /// Compacts `request` into replacement context, for models that report
+ /// [`LanguageModel::supports_explicit_compaction`].
+ fn compact(
+ &self,
+ model: &LanguageModel,
+ _request: LanguageModelRequest,
+ _cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let provider = model.provider_name.clone();
+ async move {
+ Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
+ "{provider} does not support explicit compaction"
+ )))
+ }
+ .boxed()
}
-}
-
-/// Either a built-in icon name or a path to an external SVG.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum IconOrSvg {
- /// A built-in icon from Zed's icon set.
- Icon(IconName),
- /// Path to a custom SVG icon file.
- Svg(SharedString),
-}
-impl Default for IconOrSvg {
- fn default() -> Self {
- Self::Icon(IconName::ZedAssistant)
+ /// The API key used to serve `model`, if this provider uses one.
+ fn api_key(&self, _model: &LanguageModel, _cx: &App) -> Option<String> {
+ None
}
-}
-pub trait LanguageModelProvider: 'static {
- fn id(&self) -> LanguageModelProviderId;
- fn name(&self) -> LanguageModelProviderName;
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::default()
- }
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>>;
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>>;
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>>;
- fn recommended_models(&self, _cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- Vec::new()
- }
fn is_authenticated(&self, cx: &App) -> bool;
fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>>;
fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView>;
diff --git a/crates/language_model/src/registry.rs b/crates/language_model/src/registry.rs
index 43e243c7100..011a1d4efb3 100644
--- a/crates/language_model/src/registry.rs
+++ b/crates/language_model/src/registry.rs
@@ -1,6 +1,6 @@
use crate::{
- LanguageModel, LanguageModelId, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderState, ZED_CLOUD_PROVIDER_ID,
+ LanguageModel, LanguageModelCompletionError, LanguageModelId, LanguageModelProvider,
+ LanguageModelProviderId, LanguageModelProviderState, ZED_CLOUD_PROVIDER_ID, unavailable_error,
};
use collections::{BTreeMap, HashSet};
use gpui::{App, Context, Entity, EventEmitter, Global, prelude::*};
@@ -46,16 +46,16 @@ impl std::fmt::Debug for ConfigurationError {
pub struct LanguageModelRegistry {
/// True if the user has *NO* default model configured in settings
should_use_fallback: bool,
- default_model: Option<ConfiguredModel>,
+ default_model: Option<LanguageModel>,
/// This model is automatically configured by a user's environment after
/// authenticating all providers. It's only used when `default_model` is not set.
- available_fallback_model: Option<ConfiguredModel>,
- inline_assistant_model: Option<ConfiguredModel>,
- commit_message_model: Option<ConfiguredModel>,
- thread_summary_model: Option<ConfiguredModel>,
- compaction_model: Option<ConfiguredModel>,
+ available_fallback_model: Option<LanguageModel>,
+ inline_assistant_model: Option<LanguageModel>,
+ commit_message_model: Option<LanguageModel>,
+ thread_summary_model: Option<LanguageModel>,
+ compaction_model: Option<LanguageModel>,
providers: BTreeMap<LanguageModelProviderId, Arc<dyn LanguageModelProvider>>,
- inline_alternatives: Vec<Arc<dyn LanguageModel>>,
+ inline_alternatives: Vec<LanguageModel>,
/// Set of installed extension IDs that provide language models.
/// Used to determine which built-in providers should be hidden.
installed_llm_extension_ids: HashSet<Arc<str>>,
@@ -92,19 +92,15 @@ impl FromStr for SelectedModel {
}
}
-#[derive(Clone)]
-pub struct ConfiguredModel {
- pub provider: Arc<dyn LanguageModelProvider>,
- pub model: Arc<dyn LanguageModel>,
-}
-
-impl ConfiguredModel {
- pub fn is_same_as(&self, other: &ConfiguredModel) -> bool {
- self.model.id() == other.model.id() && self.provider.id() == other.provider.id()
+impl LanguageModel {
+ /// Whether `other` is the same model from the same provider, regardless
+ /// of changes to its capabilities.
+ pub fn is_same_as(&self, other: &LanguageModel) -> bool {
+ self.id == other.id && self.provider_id == other.provider_id
}
pub fn is_provided_by_zed(&self) -> bool {
- self.provider.id() == ZED_CLOUD_PROVIDER_ID
+ self.provider_id == ZED_CLOUD_PROVIDER_ID
}
}
@@ -133,28 +129,20 @@ impl LanguageModelRegistry {
}
#[cfg(any(test, feature = "test-support"))]
+ /// Installs a registry whose only provider is a fake, with its model as
+ /// the default, and returns the fake.
pub fn test(cx: &mut App) -> Arc<crate::fake_provider::FakeLanguageModelProvider> {
let fake_provider = Arc::new(crate::fake_provider::FakeLanguageModelProvider::default());
let registry = cx.new(|cx| {
let mut registry = Self::default();
registry.register_provider(fake_provider.clone(), cx);
- let model = fake_provider.provided_models(cx)[0].clone();
- let configured_model = ConfiguredModel {
- provider: fake_provider.clone(),
- model,
- };
- registry.set_default_model(Some(configured_model), cx);
+ registry.set_default_model(Some(fake_provider.model("fake")), cx);
registry
});
cx.set_global(GlobalLanguageModelRegistry(registry));
fake_provider
}
- #[cfg(any(test, feature = "test-support"))]
- pub fn fake_model(&self) -> Arc<dyn LanguageModel> {
- self.default_model.as_ref().unwrap().model.clone()
- }
-
pub fn set_should_use_fallback(&mut self, value: bool) {
self.should_use_fallback = value;
}
@@ -258,7 +246,7 @@ impl LanguageModelRegistry {
pub fn configuration_error(
&self,
- model: Option<ConfiguredModel>,
+ model: Option<LanguageModel>,
cx: &App,
) -> Option<ConfigurationError> {
let Some(model) = model else {
@@ -268,8 +256,11 @@ impl LanguageModelRegistry {
return Some(ConfigurationError::ModelNotFound);
};
- if !model.provider.is_authenticated(cx) {
- return Some(ConfigurationError::ProviderNotAuthenticated(model.provider));
+ let Some(provider) = self.provider(&model.provider_id) else {
+ return Some(ConfigurationError::ModelNotFound);
+ };
+ if !provider.is_authenticated(cx) {
+ return Some(ConfigurationError::ProviderNotAuthenticated(provider));
}
None
@@ -280,10 +271,7 @@ impl LanguageModelRegistry {
self.providers.values().any(|p| p.is_authenticated(cx))
}
- pub fn available_models<'a>(
- &'a self,
- cx: &'a App,
- ) -> impl Iterator<Item = Arc<dyn LanguageModel>> + 'a {
+ pub fn available_models<'a>(&'a self, cx: &'a App) -> impl Iterator<Item = LanguageModel> + 'a {
self.providers
.values()
.filter(|provider| provider.is_authenticated(cx))
@@ -294,6 +282,20 @@ impl LanguageModelRegistry {
self.providers.get(id).cloned()
}
+ /// The registered provider that serves `model`.
+ ///
+ /// # Errors
+ ///
+ /// Returns [`LanguageModelCompletionError::ModelUnavailable`] when no
+ /// provider is registered under the model's provider id.
+ pub fn provider_for_model(
+ &self,
+ model: &LanguageModel,
+ ) -> Result<Arc<dyn LanguageModelProvider>, LanguageModelCompletionError> {
+ self.provider(&model.provider_id)
+ .ok_or_else(|| unavailable_error(model))
+ }
+
pub fn select_default_model(&mut self, model: Option<&SelectedModel>, cx: &mut Context<Self>) {
let configured_model = model.and_then(|model| self.select_model(model, cx));
self.set_default_model(configured_model, cx);
@@ -344,10 +346,7 @@ impl LanguageModelRegistry {
) {
self.inline_alternatives = alternatives
.into_iter()
- .flat_map(|alternative| {
- self.select_model(&alternative, cx)
- .map(|configured_model| configured_model.model)
- })
+ .flat_map(|alternative| self.select_model(&alternative, cx))
.collect::<Vec<_>>();
}
@@ -355,17 +354,14 @@ impl LanguageModelRegistry {
&mut self,
selected_model: &SelectedModel,
cx: &mut Context<Self>,
- ) -> Option<ConfiguredModel> {
- let provider = self.provider(&selected_model.provider)?;
- let model = provider
+ ) -> Option<LanguageModel> {
+ self.provider(&selected_model.provider)?
.provided_models(cx)
- .iter()
- .find(|model| model.id() == selected_model.model)?
- .clone();
- Some(ConfiguredModel { provider, model })
+ .into_iter()
+ .find(|model| model.id == selected_model.model)
}
- pub fn set_default_model(&mut self, model: Option<ConfiguredModel>, cx: &mut Context<Self>) {
+ pub fn set_default_model(&mut self, model: Option<LanguageModel>, cx: &mut Context<Self>) {
match (self.default_model(), model.as_ref()) {
(Some(old), Some(new)) if old.is_same_as(new) => {}
(None, None) => {}
@@ -385,19 +381,15 @@ impl LanguageModelRegistry {
.iter()
.filter(|provider| provider.is_authenticated(cx))
.find_map(|provider| {
- let model = provider
+ provider
.default_model(cx)
- .or_else(|| provider.recommended_models(cx).first().cloned())?;
- Some(ConfiguredModel {
- provider: provider.clone(),
- model,
- })
+ .or_else(|| provider.recommended_models(cx).first().cloned())
});
self.set_fallback_model(fallback_model, cx);
}
- fn set_fallback_model(&mut self, model: Option<ConfiguredModel>, cx: &mut Context<Self>) {
+ fn set_fallback_model(&mut self, model: Option<LanguageModel>, cx: &mut Context<Self>) {
if self.default_model.is_none() {
match (self.available_fallback_model.as_ref(), model.as_ref()) {
(Some(old), Some(new)) if old.is_same_as(new) => {}
@@ -410,7 +402,7 @@ impl LanguageModelRegistry {
pub fn set_inline_assistant_model(
&mut self,
- model: Option<ConfiguredModel>,
+ model: Option<LanguageModel>,
cx: &mut Context<Self>,
) {
match (self.inline_assistant_model.as_ref(), model.as_ref()) {
@@ -423,7 +415,7 @@ impl LanguageModelRegistry {
pub fn set_commit_message_model(
&mut self,
- model: Option<ConfiguredModel>,
+ model: Option<LanguageModel>,
cx: &mut Context<Self>,
) {
match (self.commit_message_model.as_ref(), model.as_ref()) {
@@ -436,7 +428,7 @@ impl LanguageModelRegistry {
pub fn set_thread_summary_model(
&mut self,
- model: Option<ConfiguredModel>,
+ model: Option<LanguageModel>,
cx: &mut Context<Self>,
) {
match (self.thread_summary_model.as_ref(), model.as_ref()) {
@@ -447,7 +439,7 @@ impl LanguageModelRegistry {
self.thread_summary_model = model;
}
- pub fn set_compaction_model(&mut self, model: Option<ConfiguredModel>, cx: &mut Context<Self>) {
+ pub fn set_compaction_model(&mut self, model: Option<LanguageModel>, cx: &mut Context<Self>) {
match (self.compaction_model.as_ref(), model.as_ref()) {
(Some(old), Some(new)) if old.is_same_as(new) => {}
(None, None) => {}
@@ -456,7 +448,7 @@ impl LanguageModelRegistry {
self.compaction_model = model;
}
- pub fn default_model(&self) -> Option<ConfiguredModel> {
+ pub fn default_model(&self) -> Option<LanguageModel> {
#[cfg(debug_assertions)]
if std::env::var("ZED_SIMULATE_NO_LLM_PROVIDER").is_ok() {
return None;
@@ -471,16 +463,13 @@ impl LanguageModelRegistry {
})
}
- pub fn default_fast_model(&self, cx: &App) -> Option<ConfiguredModel> {
- let configured = self.default_model()?;
- let fast_model = configured.provider.default_fast_model(cx)?;
- Some(ConfiguredModel {
- provider: configured.provider,
- model: fast_model,
- })
+ pub fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ let default_model = self.default_model()?;
+ self.provider(&default_model.provider_id)?
+ .default_fast_model(cx)
}
- pub fn inline_assistant_model(&self) -> Option<ConfiguredModel> {
+ pub fn inline_assistant_model(&self) -> Option<LanguageModel> {
#[cfg(debug_assertions)]
if std::env::var("ZED_SIMULATE_NO_LLM_PROVIDER").is_ok() {
return None;
@@ -491,7 +480,7 @@ impl LanguageModelRegistry {
.or_else(|| self.default_model())
}
- pub fn commit_message_model(&self, cx: &App) -> Option<ConfiguredModel> {
+ pub fn commit_message_model(&self, cx: &App) -> Option<LanguageModel> {
#[cfg(debug_assertions)]
if std::env::var("ZED_SIMULATE_NO_LLM_PROVIDER").is_ok() {
return None;
@@ -503,7 +492,7 @@ impl LanguageModelRegistry {
.or_else(|| self.default_model())
}
- pub fn thread_summary_model(&self, cx: &App) -> Option<ConfiguredModel> {
+ pub fn thread_summary_model(&self, cx: &App) -> Option<LanguageModel> {
#[cfg(debug_assertions)]
if std::env::var("ZED_SIMULATE_NO_LLM_PROVIDER").is_ok() {
return None;
@@ -518,7 +507,7 @@ impl LanguageModelRegistry {
/// Returns the configured compaction model without falling back through
/// `default_fast_model`/`default_model`. Callers that want a fallback to
/// the thread's primary model should handle `None` themselves.
- pub fn compaction_model(&self) -> Option<ConfiguredModel> {
+ pub fn compaction_model(&self) -> Option<LanguageModel> {
#[cfg(debug_assertions)]
if std::env::var("ZED_SIMULATE_NO_LLM_PROVIDER").is_ok() {
return None;
@@ -530,7 +519,7 @@ impl LanguageModelRegistry {
/// The models to use for inline assists. Returns the union of the active
/// model and all inline alternatives. When there are multiple models, the
/// user will be able to cycle through results.
- pub fn inline_alternative_models(&self) -> &[Arc<dyn LanguageModel>] {
+ pub fn inline_alternative_models(&self) -> &[LanguageModel] {
&self.inline_alternatives
}
}
@@ -538,7 +527,7 @@ impl LanguageModelRegistry {
#[cfg(test)]
mod tests {
use super::*;
- use crate::fake_provider::{FakeLanguageModel, FakeLanguageModelProvider};
+ use crate::fake_provider::FakeLanguageModelProvider;
#[test]
fn selected_model_allows_slashes_in_model_id() {
@@ -692,13 +681,7 @@ mod tests {
let provider = registry.provider(&provider.id()).unwrap();
let model = provider.default_model(cx).unwrap();
- registry.set_fallback_model(
- Some(ConfiguredModel {
- provider: provider.clone(),
- model: model.clone(),
- }),
- cx,
- );
+ registry.set_fallback_model(Some(model.clone()), cx);
assert!(registry.default_model().is_none());
assert!(registry.inline_assistant_model().is_none());
@@ -706,8 +689,8 @@ mod tests {
registry.set_should_use_fallback(true);
let default_model = registry.default_model().unwrap();
- assert_eq!(default_model.model.id(), model.id());
- assert_eq!(default_model.provider.id(), provider.id());
+ assert_eq!(default_model.id, model.id);
+ assert_eq!(default_model.provider_id, provider.id());
assert!(
registry
.inline_assistant_model()
@@ -719,14 +702,9 @@ mod tests {
#[gpui::test]
fn test_inline_assistant_model_precedence(cx: &mut App) {
let registry = cx.new(|_| LanguageModelRegistry::default());
- let provider = Arc::new(FakeLanguageModelProvider::default());
+ let provider = FakeLanguageModelProvider::default();
let [inline_model, default_model, fallback_model] =
- ["inline", "default", "fallback"].map(|model_id| ConfiguredModel {
- provider: provider.clone(),
- model: Arc::new(FakeLanguageModel::with_id_and_thinking(
- "fake", model_id, model_id, false,
- )),
- });
+ ["inline", "default", "fallback"].map(|model_id| provider.model(model_id));
registry.update(cx, |registry, cx| {
registry.set_should_use_fallback(true);
diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rs
index a91ba901219..25f911d4ced 100644
--- a/crates/language_model_core/src/language_model_core.rs
+++ b/crates/language_model_core/src/language_model_core.rs
@@ -31,6 +31,12 @@ pub use crate::util::{
};
pub use gpui_shared_string::SharedString;
+/// The events of a streaming completion, as produced by a provider.
+pub type LanguageModelCompletionStream = futures::stream::BoxStream<
+ 'static,
+ Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
+>;
+
/// A completion event from a language model.
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
pub enum LanguageModelCompletionEvent {
@@ -243,6 +249,13 @@ pub enum LanguageModelCompletionError {
},
#[error("stream from {provider} ended unexpectedly")]
StreamEndedUnexpectedly { provider: LanguageModelProviderName },
+ /// The provider no longer offers a model with this id, for example after
+ /// its settings or fetched model list changed.
+ #[error("{provider} no longer offers the model {}", model.0)]
+ ModelUnavailable {
+ provider: LanguageModelProviderName,
+ model: LanguageModelId,
+ },
#[error(transparent)]
Other(#[from] anyhow::Error),
}
@@ -397,6 +410,7 @@ impl LanguageModelCompletionError {
| Self::BuildRequestBody { .. }
| Self::DeserializeResponse { .. }
| Self::StreamEndedUnexpectedly { .. }
+ | Self::ModelUnavailable { .. }
| Self::Other(_) => false,
}
}
@@ -426,6 +440,7 @@ impl LanguageModelCompletionError {
| Self::BuildRequestBody { .. }
| Self::DeserializeResponse { .. }
| Self::StreamEndedUnexpectedly { .. }
+ | Self::ModelUnavailable { .. }
| Self::Other(_) => None,
}
}
@@ -674,7 +689,7 @@ impl LanguageModelToolUseInput {
}
}
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LanguageModelEffortLevel {
pub name: SharedString,
pub value: SharedString,
diff --git a/crates/language_model_core/src/rate_limiter.rs b/crates/language_model_core/src/rate_limiter.rs
index 28e92056f9d..a2f6a7df9f8 100644
--- a/crates/language_model_core/src/rate_limiter.rs
+++ b/crates/language_model_core/src/rate_limiter.rs
@@ -7,7 +7,53 @@ use std::{
task::{Context, Poll},
};
-use crate::LanguageModelCompletionError;
+use crate::{LanguageModelCompletionError, LanguageModelId};
+use collections::HashMap;
+use std::sync::Mutex;
+
+/// Default number of concurrent requests a provider allows per model.
+///
+/// One limit is shared by every request to a model across the app: threads,
+/// subagents, summaries, and inline assists. It is set well above normal
+/// parallel use so it only stops runaway fan-out.
+pub const DEFAULT_MODEL_CONCURRENCY: usize = 16;
+
+/// A provider's local concurrency caps, one [`RateLimiter`] per model id.
+///
+/// Upstream rate limits are per credential and model family, and exceeding
+/// them surfaces as retryable errors. This only bounds how many requests Zed
+/// has in flight to one model at once.
+pub struct ModelRateLimiters {
+ limit: usize,
+ limiters: Mutex<HashMap<LanguageModelId, RateLimiter>>,
+}
+
+impl ModelRateLimiters {
+ pub fn new(limit: usize) -> Self {
+ Self {
+ limit,
+ limiters: Mutex::default(),
+ }
+ }
+
+ /// Returns the limiter shared by every request to `model_id`.
+ pub fn for_model(&self, model_id: &LanguageModelId) -> RateLimiter {
+ let mut limiters = self
+ .limiters
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ limiters
+ .entry(model_id.clone())
+ .or_insert_with(|| RateLimiter::new(self.limit))
+ .clone()
+ }
+}
+
+impl Default for ModelRateLimiters {
+ fn default() -> Self {
+ Self::new(DEFAULT_MODEL_CONCURRENCY)
+ }
+}
#[derive(Clone)]
pub struct RateLimiter {
diff --git a/crates/language_models/Cargo.toml b/crates/language_models/Cargo.toml
index 552e9e296fc..b4ccec6d853 100644
--- a/crates/language_models/Cargo.toml
+++ b/crates/language_models/Cargo.toml
@@ -17,7 +17,6 @@ test-support = []
[dependencies]
ai_onboarding.workspace = true
-async-lock.workspace = true
anthropic = { workspace = true, features = ["schemars"] }
anyhow.workspace = true
aws-config = { workspace = true, features = ["behavior-version-latest"] }
@@ -66,7 +65,6 @@ serde_json.workspace = true
settings.workspace = true
sha2.workspace = true
strum.workspace = true
-tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
ui.workspace = true
ui_input.workspace = true
util.workspace = true
diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs
index e8d380b00fc..91c039c41ca 100644
--- a/crates/language_models/src/provider/anthropic.rs
+++ b/crates/language_models/src/provider/anthropic.rs
@@ -10,10 +10,10 @@ use http_client::{CustomHeaders, HttpClient};
use language_model::{
ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, ApiKeyConfiguration, ApiKeyState,
AuthenticateError, CompactionResult, EnvVar, FastModeConfirmation, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
- LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
- ProviderSettingsView, RateLimiter, env_var,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelId,
+ LanguageModelName, LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
+ LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
+ ModelRateLimiters, ProviderSettingsView, env_var, unavailable_error,
};
use settings::{Settings, SettingsStore};
use std::sync::{Arc, LazyLock};
@@ -38,6 +38,7 @@ pub struct AnthropicSettings {
pub struct AnthropicLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
const API_KEY_ENV_VAR_NAME: &str = "ANTHROPIC_API_KEY";
@@ -173,17 +174,93 @@ impl AnthropicLanguageModelProvider {
}
});
- Self { http_client, state }
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
- fn create_language_model(&self, model: anthropic::Model) -> Arc<dyn LanguageModel> {
- Arc::new(AnthropicModel {
- id: LanguageModelId::from(model.id.to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
+ /// Every model this provider offers, keyed by id: the fetched list, with
+ /// settings entries added or overriding fetched ones.
+ fn anthropic_models(&self, cx: &App) -> BTreeMap<String, anthropic::Model> {
+ let mut models: BTreeMap<String, anthropic::Model> = BTreeMap::default();
+
+ // Models reported by Anthropic's `/v1/models` endpoint are the
+ // primary source. The list will be empty until authentication has
+ // succeeded and the first fetch completes.
+ for model in &self.state.read(cx).fetched_models {
+ models.insert(model.id.to_string(), model.clone());
+ }
+
+ // User-defined `available_models` from settings can either add
+ // entirely new entries or override fields on a fetched model with
+ // the same id (e.g. enable Fast mode or set a tool override).
+ for available in &AnthropicLanguageModelProvider::settings(cx).available_models {
+ let model = available_model_to_anthropic_model(available);
+ models.insert(model.id.to_string(), model);
+ }
+
+ models
+ }
+
+ /// Describes the offered model with `id`.
+ fn offered_model(&self, id: &str, cx: &App) -> Option<LanguageModel> {
+ self.anthropic_models(cx).get(id).map(language_model)
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<anthropic::Model, LanguageModelCompletionError> {
+ self.anthropic_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
+ }
+
+ fn stream_anthropic_request(
+ &self,
+ config: &anthropic::Model,
+ request: anthropic::Request,
+ cx: &AsyncApp,
+ ) -> BoxFuture<
+ 'static,
+ Result<
+ BoxStream<'static, Result<anthropic::Event, AnthropicError>>,
+ LanguageModelCompletionError,
+ >,
+ > {
+ let http_client = self.http_client.clone();
+
+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
+ let api_url = AnthropicLanguageModelProvider::api_url(cx);
+ let extra_headers = AnthropicLanguageModelProvider::settings(cx)
+ .custom_headers
+ .clone();
+ (state.api_key_state.key(&api_url), api_url, extra_headers)
+ });
+
+ let beta_headers = config.beta_headers();
+
+ async move {
+ let Some(api_key) = api_key else {
+ return Err(LanguageModelCompletionError::NoApiKey {
+ provider: PROVIDER_NAME,
+ });
+ };
+ let request = anthropic::stream_completion(
+ http_client.as_ref(),
+ &api_url,
+ &api_key,
+ request,
+ beta_headers,
+ &extra_headers,
+ );
+ request.await.map_err(Into::into)
+ }
+ .boxed()
}
fn settings(cx: &App) -> &AnthropicSettings {
@@ -221,49 +298,34 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider {
IconOrSvg::Icon(IconName::AiAnthropic)
}
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
let fetched = self.state.read(cx).fetched_models.clone();
// Pick the highest-version Sonnet we know about; otherwise the first
// Claude model returned. Returning `None` until the fetch completes
// matches the Ollama provider's behavior.
- pick_preferred_model(&fetched, &["claude-sonnet-", "claude-opus-", "claude-"])
- .map(|model| self.create_language_model(model))
+ let preferred =
+ pick_preferred_model(&fetched, &["claude-sonnet-", "claude-opus-", "claude-"])?;
+ self.offered_model(&preferred.id, cx)
}
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
let fetched = self.state.read(cx).fetched_models.clone();
- pick_preferred_model(&fetched, &["claude-haiku-", "claude-"])
- .map(|model| self.create_language_model(model))
+ let preferred = pick_preferred_model(&fetched, &["claude-haiku-", "claude-"])?;
+ self.offered_model(&preferred.id, cx)
}
- fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn recommended_models(&self, cx: &App) -> Vec<LanguageModel> {
let fetched = self.state.read(cx).fetched_models.clone();
pick_preferred_model(&fetched, &["claude-sonnet-"])
- .map(|model| vec![self.create_language_model(model)])
- .unwrap_or_default()
+ .and_then(|preferred| self.offered_model(&preferred.id, cx))
+ .into_iter()
+ .collect()
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let mut models: BTreeMap<String, anthropic::Model> = BTreeMap::default();
-
- // Models reported by Anthropic's `/v1/models` endpoint are the
- // primary source. The list will be empty until authentication has
- // succeeded and the first fetch completes.
- for model in &self.state.read(cx).fetched_models {
- models.insert(model.id.to_string(), model.clone());
- }
-
- // User-defined `available_models` from settings can either add
- // entirely new entries or override fields on a fetched model with
- // the same id (e.g. enable Fast mode or set a tool override).
- for available in &AnthropicLanguageModelProvider::settings(cx).available_models {
- let model = available_model_to_anthropic_model(available);
- models.insert(model.id.to_string(), model);
- }
-
- models
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.anthropic_models(cx)
.into_values()
- .map(|model| self.create_language_model(model))
+ .map(|model| language_model(&model))
.collect()
}
@@ -290,6 +352,153 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider {
.update(cx, |state, cx| state.set_api_key(api_key, cx))
}
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let has_tools = !request.tools.is_empty();
+ let request_id = config.request_id(has_tools).to_string();
+ let mut request = match into_anthropic(
+ request,
+ request_id,
+ config.default_temperature,
+ config.max_output_tokens,
+ config.mode.clone(),
+ AnthropicPromptCacheMode::Automatic,
+ &PROVIDER_ID,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ if !config.supports_speed {
+ request.speed = None;
+ }
+ let request = self.stream_anthropic_request(&config, request, cx);
+ let executor = cx.background_executor().clone();
+ let future = request_limiter.stream(async move {
+ let response = request.await?;
+ let events = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
+ ))
+ });
+ async move { Ok(future.await?.boxed()) }.boxed()
+ }
+
+ fn count_input_tokens(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request_id = config.request_id(!request.tools.is_empty()).to_string();
+ let request = into_anthropic(
+ request,
+ request_id,
+ config.default_temperature,
+ config.max_output_tokens,
+ config.mode.clone(),
+ AnthropicPromptCacheMode::Automatic,
+ &PROVIDER_ID,
+ );
+ let http_client = self.http_client.clone();
+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
+ let api_url = AnthropicLanguageModelProvider::api_url(cx);
+ let extra_headers = AnthropicLanguageModelProvider::settings(cx)
+ .custom_headers
+ .clone();
+ (state.api_key_state.key(&api_url), api_url, extra_headers)
+ });
+ let beta_headers = config.beta_headers();
+ request_limiter
+ .run(async move {
+ let request = request?.into_count_tokens_request();
+ let api_key = api_key.ok_or(LanguageModelCompletionError::NoApiKey {
+ provider: PROVIDER_NAME,
+ })?;
+ anthropic::count_input_tokens(
+ http_client.as_ref(),
+ &api_url,
+ &api_key,
+ request,
+ beta_headers,
+ &extra_headers,
+ )
+ .await
+ .map(Some)
+ .map_err(Into::into)
+ })
+ .boxed()
+ }
+
+ fn compact(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if !config.supports_compaction {
+ return async {
+ Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
+ "this Anthropic model does not support explicit compaction"
+ )))
+ }
+ .boxed();
+ }
+
+ let mut request = match into_anthropic(
+ request,
+ config.request_id(false).to_string(),
+ config.default_temperature,
+ config.max_output_tokens,
+ config.mode.clone(),
+ AnthropicPromptCacheMode::Automatic,
+ &PROVIDER_ID,
+ ) {
+ Ok(request) => request.into_compact_request(),
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ if !config.supports_speed {
+ request.speed = None;
+ }
+ let request = self.stream_anthropic_request(&config, request, cx);
+ let executor = cx.background_executor().clone();
+ let future = request_limiter.run(async move {
+ let response = request.await?;
+ let stream = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
+ let stream = language_model::stream_in_background(stream.boxed(), executor);
+ let (context, usage) = collect_compaction_result(stream.boxed(), PROVIDER_NAME).await?;
+ Ok(CompactionResult { context, usage })
+ });
+ future.boxed()
+ }
+
+ fn api_key(&self, _model: &LanguageModel, cx: &App) -> Option<String> {
+ self.state.read_with(cx, |state, cx| {
+ let api_url = AnthropicLanguageModelProvider::api_url(cx);
+ state.api_key_state.key(&api_url).map(|key| key.to_string())
+ })
+ }
+
fn fast_mode_confirmation(&self, _cx: &App) -> Option<FastModeConfirmation> {
Some(FastModeConfirmation {
title: "Enable Fast Mode for Anthropic?".into(),
@@ -464,10 +673,9 @@ mod tests {
cx: &mut gpui::TestAppContext,
) {
let provider = direct_anthropic_test_provider(FakeHttpClient::with_404_response(), cx);
- let model = direct_anthropic_test_model(&provider);
+ let model = direct_anthropic_test_model(&provider, cx);
- assert!(model.supports_explicit_compaction());
- assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));
+ assert!(model.supports_explicit_compaction);
assert_eq!(
model.minimum_explicit_compaction_input_tokens(),
Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
@@ -569,7 +777,7 @@ mod tests {
let provider = direct_anthropic_test_provider(http_client, cx);
let store_key = cx.update(|cx| provider.set_api_key(Some("test-key".to_string()), cx));
store_key.await.unwrap();
- let model = direct_anthropic_test_model(&provider);
+ let model = direct_anthropic_test_model(&provider, cx);
let request = LanguageModelRequest {
max_output_tokens: Some(8192),
messages: vec![LanguageModelRequestMessage {
@@ -581,7 +789,10 @@ mod tests {
..Default::default()
};
- let result = model.compact(request, &cx.to_async()).await.unwrap();
+ let result = provider
+ .compact(&model, request, &cx.to_async())
+ .await
+ .unwrap();
assert_eq!(
result.usage,
@@ -644,16 +855,23 @@ mod tests {
})
}
+ /// Adds a listed model to the provider's fetched list, as a successful
+ /// `/v1/models` fetch would, and returns it.
fn direct_anthropic_test_model(
provider: &AnthropicLanguageModelProvider,
- ) -> Arc<dyn LanguageModel> {
- provider.create_language_model(anthropic::Model::from_listed(anthropic::ListModelEntry {
+ cx: &mut gpui::TestAppContext,
+ ) -> LanguageModel {
+ let model = anthropic::Model::from_listed(anthropic::ListModelEntry {
id: "claude-opus-4-6".to_string(),
display_name: "Claude Opus 4.6".to_string(),
max_input_tokens: 1_000_000,
max_tokens: 128_000,
capabilities: None,
- }))
+ });
+ provider
+ .state
+ .update(cx, |state, _| state.fetched_models.push(model.clone()));
+ language_model(&model)
}
struct TestCredentialsProvider;
@@ -689,170 +907,29 @@ mod tests {
}
}
-pub struct AnthropicModel {
- id: LanguageModelId,
- model: anthropic::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-impl AnthropicModel {
- fn stream_completion(
- &self,
- request: anthropic::Request,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<anthropic::Event, AnthropicError>>,
- LanguageModelCompletionError,
- >,
- > {
- let http_client = self.http_client.clone();
-
- let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
- let api_url = AnthropicLanguageModelProvider::api_url(cx);
- let extra_headers = AnthropicLanguageModelProvider::settings(cx)
- .custom_headers
- .clone();
- (state.api_key_state.key(&api_url), api_url, extra_headers)
- });
-
- let beta_headers = self.model.beta_headers();
-
- async move {
- let Some(api_key) = api_key else {
- return Err(LanguageModelCompletionError::NoApiKey {
- provider: PROVIDER_NAME,
- });
- };
- let request = anthropic::stream_completion(
- http_client.as_ref(),
- &api_url,
- &api_key,
- request,
- beta_headers,
- &extra_headers,
- );
- request.await.map_err(Into::into)
- }
- .boxed()
- }
-}
-
-impl LanguageModel for AnthropicModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name.clone())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- true
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::None => true,
- LanguageModelToolChoice::Any => anthropic::supports_forced_tool_use(&self.model.id),
- }
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking
- }
-
- fn supports_fast_mode(&self) -> bool {
- self.model.supports_speed
- }
-
- fn refusal_fallback_model_id(&self) -> Option<&'static str> {
- if self.model.id.starts_with(anthropic::FABLE_MODEL_ID_PREFIX) {
- Some(anthropic::FABLE_FALLBACK_MODEL_ID)
- } else {
- None
- }
- }
-
- fn supports_server_side_compaction(&self) -> bool {
- self.model.supports_compaction
- }
-
- fn supports_explicit_compaction(&self) -> bool {
- self.model.supports_compaction
- }
-
- fn supports_explicit_compaction_output_limit(&self) -> bool {
- self.supports_explicit_compaction()
- }
-
- fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
- self.supports_explicit_compaction()
- .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
- }
-
- fn compact(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- if !self.supports_explicit_compaction() {
- return async {
- Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
- "this Anthropic model does not support explicit compaction"
- )))
- }
- .boxed();
- }
-
- let mut request = match into_anthropic(
- request,
- self.model.request_id(false).to_string(),
- self.model.default_temperature,
- self.model.max_output_tokens,
- self.model.mode.clone(),
- AnthropicPromptCacheMode::Automatic,
- &PROVIDER_ID,
- ) {
- Ok(request) => request.into_compact_request(),
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- if !self.model.supports_speed {
- request.speed = None;
- }
- let request = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
- let future = self.request_limiter.run(async move {
- let response = request.await?;
- let stream = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
- let stream = language_model::stream_in_background(stream.boxed(), executor);
- let (context, usage) = collect_compaction_result(stream.boxed(), PROVIDER_NAME).await?;
- Ok(CompactionResult { context, usage })
- });
- future.boxed()
- }
-
- fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
- self.model
+fn language_model(model: &anthropic::Model) -> LanguageModel {
+ let supports_explicit_compaction = model.supports_compaction;
+ LanguageModel {
+ supports_tools: true,
+ supports_images: model.supports_images,
+ supports_streaming_tools: true,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: true,
+ none: true,
+ any: anthropic::supports_forced_tool_use(&model.id),
+ },
+ supports_thinking: model.supports_thinking,
+ supports_fast_mode: model.supports_speed,
+ refusal_fallback_model_id: model
+ .id
+ .starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
+ .then_some(anthropic::FABLE_FALLBACK_MODEL_ID),
+ supports_server_side_compaction: model.supports_compaction,
+ supports_explicit_compaction,
+ supports_explicit_compaction_output_limit: supports_explicit_compaction,
+ minimum_explicit_compaction_input_tokens: supports_explicit_compaction
+ .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS),
+ supported_effort_levels: model
.supported_effort_levels
.iter()
.map(|e| {
@@ -870,111 +947,15 @@ impl LanguageModel for AnthropicModel {
is_default,
}
})
- .collect::<Vec<_>>()
- }
-
- fn telemetry_id(&self) -> String {
- format!("anthropic/{}", self.model.id)
- }
-
- fn api_key(&self, cx: &App) -> Option<String> {
- self.state.read_with(cx, |state, cx| {
- let api_url = AnthropicLanguageModelProvider::api_url(cx);
- state.api_key_state.key(&api_url).map(|key| key.to_string())
- })
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_input_tokens
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- Some(self.model.max_output_tokens)
- }
-
- fn count_input_tokens(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
- let request_id = self.model.request_id(!request.tools.is_empty()).to_string();
- let request = into_anthropic(
- request,
- request_id,
- self.model.default_temperature,
- self.model.max_output_tokens,
- self.model.mode.clone(),
- AnthropicPromptCacheMode::Automatic,
- &PROVIDER_ID,
- );
- let http_client = self.http_client.clone();
- let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
- let api_url = AnthropicLanguageModelProvider::api_url(cx);
- let extra_headers = AnthropicLanguageModelProvider::settings(cx)
- .custom_headers
- .clone();
- (state.api_key_state.key(&api_url), api_url, extra_headers)
- });
- let beta_headers = self.model.beta_headers();
- self.request_limiter
- .run(async move {
- let request = request?.into_count_tokens_request();
- let api_key = api_key.ok_or(LanguageModelCompletionError::NoApiKey {
- provider: PROVIDER_NAME,
- })?;
- anthropic::count_input_tokens(
- http_client.as_ref(),
- &api_url,
- &api_key,
- request,
- beta_headers,
- &extra_headers,
- )
- .await
- .map(Some)
- .map_err(Into::into)
- })
- .boxed()
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let has_tools = !request.tools.is_empty();
- let request_id = self.model.request_id(has_tools).to_string();
- let mut request = match into_anthropic(
- request,
- request_id,
- self.model.default_temperature,
- self.model.max_output_tokens,
- self.model.mode.clone(),
- AnthropicPromptCacheMode::Automatic,
- &PROVIDER_ID,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- if !self.model.supports_speed {
- request.speed = None;
- }
- let request = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
- let response = request.await?;
- let events = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
- Ok(language_model::stream_in_background(
- events.boxed(),
- executor,
- ))
- });
- async move { Ok(future.await?.boxed()) }.boxed()
+ .collect(),
+ max_output_tokens: Some(model.max_output_tokens),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id.to_string()),
+ LanguageModelName::from(model.display_name.clone()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("anthropic/{}", model.id),
+ model.max_input_tokens,
+ )
}
}
diff --git a/crates/language_models/src/provider/anthropic/telemetry.rs b/crates/language_models/src/provider/anthropic/telemetry.rs
index 75fb11a81b4..ffb9223377b 100644
--- a/crates/language_models/src/provider/anthropic/telemetry.rs
+++ b/crates/language_models/src/provider/anthropic/telemetry.rs
@@ -2,7 +2,7 @@ use anthropic::ANTHROPIC_API_URL;
use anyhow::{Context as _, anyhow};
use gpui::BackgroundExecutor;
use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
-use language_model::{ANTHROPIC_PROVIDER_ID, LanguageModel};
+use language_model::{ANTHROPIC_PROVIDER_ID, LanguageModel, LanguageModelRegistry};
use std::env;
use std::sync::Arc;
use util::ResultExt;
@@ -51,11 +51,7 @@ impl AnthropicEventType {
}
}
-pub fn report_anthropic_event(
- model: &Arc<dyn LanguageModel>,
- event: AnthropicEventData,
- cx: &gpui::App,
-) {
+pub fn report_anthropic_event(model: &LanguageModel, event: AnthropicEventData, cx: &gpui::App) {
let reporter = AnthropicEventReporter::new(model, cx);
reporter.report(event);
}
@@ -69,12 +65,15 @@ pub struct AnthropicEventReporter {
}
impl AnthropicEventReporter {
- pub fn new(model: &Arc<dyn LanguageModel>, cx: &gpui::App) -> Self {
+ pub fn new(model: &LanguageModel, cx: &gpui::App) -> Self {
Self {
http_client: cx.http_client(),
executor: cx.background_executor().clone(),
- api_key: model.api_key(cx),
- is_anthropic: model.provider_id() == ANTHROPIC_PROVIDER_ID,
+ api_key: LanguageModelRegistry::read_global(cx)
+ .provider_for_model(model)
+ .ok()
+ .and_then(|provider| provider.api_key(model, cx)),
+ is_anthropic: model.provider_id == ANTHROPIC_PROVIDER_ID,
}
}
diff --git a/crates/language_models/src/provider/anthropic_compatible.rs b/crates/language_models/src/provider/anthropic_compatible.rs
index 0ad754a913b..8542c08fdcf 100644
--- a/crates/language_models/src/provider/anthropic_compatible.rs
+++ b/crates/language_models/src/provider/anthropic_compatible.rs
@@ -7,10 +7,10 @@ use gpui::{App, AppContext, AsyncApp, Entity, Task};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
AuthenticateError, IconOrSvg, LanguageModel, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelId, LanguageModelName, LanguageModelProvider,
+ LanguageModelCompletionStream, LanguageModelId, LanguageModelName, LanguageModelProvider,
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
- LanguageModelRequest, LanguageModelToolChoice, ProviderSettingsView, RateLimiter,
- SubPageProviderSettings,
+ LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView,
+ SubPageProviderSettings, unavailable_error,
};
use settings::Settings;
use std::sync::Arc;
@@ -38,6 +38,7 @@ pub struct AnthropicCompatibleLanguageModelProvider {
name: LanguageModelProviderName,
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
impl ApiCompatibleProviderSettings for AnthropicCompatibleSettings {
@@ -185,126 +186,85 @@ impl AnthropicCompatibleLanguageModelProvider {
name: id.into(),
http_client,
state,
+ request_limiters: ModelRateLimiters::default(),
}
}
- fn create_language_model(&self, model: AvailableModel) -> Arc<dyn LanguageModel> {
- let capabilities = model.capabilities.clone();
+ /// Every model this provider offers: the settings entries, in order.
+ fn available_models<'a>(&self, cx: &'a App) -> &'a [AvailableModel] {
+ &self.state.read(cx).settings.available_models
+ }
+
+ fn create_language_model(&self, available: &AvailableModel) -> LanguageModel {
+ let capabilities = &available.capabilities;
+ let model = available_model_to_anthropic_model(available);
+ LanguageModel {
+ supports_tools: capabilities.tools,
+ supports_images: capabilities.images,
+ supports_streaming_tools: capabilities.tools,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: capabilities.tools,
+ any: capabilities.tools,
+ none: true,
+ },
+ supports_thinking: model.supports_thinking,
+ supported_effort_levels: model
+ .supported_effort_levels
+ .iter()
+ .map(|effort| {
+ let is_default = matches!(effort, anthropic::Effort::High);
+ let (name, value) = match effort {
+ anthropic::Effort::Low => ("Low".into(), "low".into()),
+ anthropic::Effort::Medium => ("Medium".into(), "medium".into()),
+ anthropic::Effort::High => ("High".into(), "high".into()),
+ anthropic::Effort::XHigh => ("XHigh".into(), "xhigh".into()),
+ anthropic::Effort::Max => ("Max".into(), "max".into()),
+ };
+ language_model::LanguageModelEffortLevel {
+ name,
+ value,
+ is_default,
+ }
+ })
+ .collect(),
+ max_output_tokens: Some(model.max_output_tokens),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id.clone()),
+ LanguageModelName::from(model.display_name.clone()),
+ self.id.clone(),
+ self.name.clone(),
+ format!("anthropic/{}", model.id),
+ model.max_input_tokens,
+ )
+ }
+ }
+
+ /// The current configuration of `model` and the prompt cache mode to
+ /// request it with, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<(anthropic::Model, AnthropicPromptCacheMode), LanguageModelCompletionError> {
+ let available = self
+ .available_models(cx)
+ .iter()
+ .find(|available| available.name == model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))?;
// Compatible providers may not support Anthropic's automatic prompt
// caching; only request explicit (legacy) cache breakpoints when the
// user has opted in via the `prompt_caching` capability.
- let cache_mode = if capabilities.prompt_caching {
+ let cache_mode = if available.capabilities.prompt_caching {
AnthropicPromptCacheMode::Legacy
} else {
AnthropicPromptCacheMode::Disabled
};
- let model = available_model_to_anthropic_model(&model);
-
- Arc::new(AnthropicCompatibleLanguageModel {
- id: LanguageModelId::from(model.id.clone()),
- provider_id: self.id.clone(),
- provider_name: self.name.clone(),
- model,
- capabilities,
- cache_mode,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-}
-
-impl LanguageModelProviderState for AnthropicCompatibleLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for AnthropicCompatibleLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelProviderName {
- self.name.clone()
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiAnthropicCompat)
- }
-
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
- self.state
- .read(cx)
- .settings
- .available_models
- .first()
- .map(|model| self.create_language_model(model.clone()))
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- None
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- self.state
- .read(cx)
- .settings
- .available_models
- .iter()
- .map(|model| self.create_language_model(model.clone()))
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.clone();
- Some(ProviderSettingsView::SubPage(SubPageProviderSettings::new(
- move |window, cx| {
- cx.new(|cx| {
- ApiCompatibleProviderConfigurationView::new(
- state.clone(),
- "Anthropic",
- API_KEY_PLACEHOLDER,
- window,
- cx,
- )
- })
- .into()
- },
- )))
+ Ok((available_model_to_anthropic_model(available), cache_mode))
}
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
- }
-}
-
-pub struct AnthropicCompatibleLanguageModel {
- id: LanguageModelId,
- provider_id: LanguageModelProviderId,
- provider_name: LanguageModelProviderName,
- model: anthropic::Model,
- capabilities: ModelCapabilities,
- cache_mode: AnthropicPromptCacheMode,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-impl AnthropicCompatibleLanguageModel {
- fn stream_completion(
+ fn stream_anthropic_request(
&self,
+ config: &anthropic::Model,
request: anthropic::Request,
cx: &AsyncApp,
) -> BoxFuture<
@@ -315,7 +275,7 @@ impl AnthropicCompatibleLanguageModel {
>,
> {
let http_client = self.http_client.clone();
- let provider_name = self.provider_name.clone();
+ let provider_name = self.name.clone();
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, _cx| {
let api_url = state.settings.api_url.clone();
@@ -326,7 +286,7 @@ impl AnthropicCompatibleLanguageModel {
)
});
- let beta_headers = self.model.beta_headers();
+ let beta_headers = config.beta_headers();
async move {
let Some(api_key) = api_key else {
@@ -352,113 +312,109 @@ impl AnthropicCompatibleLanguageModel {
}
}
-impl LanguageModel for AnthropicCompatibleLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name.clone())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- self.provider_id.clone()
- }
+impl LanguageModelProviderState for AnthropicCompatibleLanguageModelProvider {
+ type ObservableEntity = State;
- fn provider_name(&self) -> LanguageModelProviderName {
- self.provider_name.clone()
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn supports_tools(&self) -> bool {
- self.capabilities.tools
+impl LanguageModelProvider for AnthropicCompatibleLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
+ self.id.clone()
}
- fn supports_images(&self) -> bool {
- self.capabilities.images
+ fn name(&self) -> LanguageModelProviderName {
+ self.name.clone()
}
- fn supports_streaming_tools(&self) -> bool {
- self.capabilities.tools
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiAnthropicCompat)
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::Any => self.capabilities.tools,
- LanguageModelToolChoice::None => true,
- }
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.available_models(cx)
+ .first()
+ .map(|model| self.create_language_model(model))
}
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ None
}
- fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
- self.model
- .supported_effort_levels
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.available_models(cx)
.iter()
- .map(|effort| {
- let is_default = matches!(effort, anthropic::Effort::High);
- let (name, value) = match effort {
- anthropic::Effort::Low => ("Low".into(), "low".into()),
- anthropic::Effort::Medium => ("Medium".into(), "medium".into()),
- anthropic::Effort::High => ("High".into(), "high".into()),
- anthropic::Effort::XHigh => ("XHigh".into(), "xhigh".into()),
- anthropic::Effort::Max => ("Max".into(), "max".into()),
- };
- language_model::LanguageModelEffortLevel {
- name,
- value,
- is_default,
- }
- })
+ .map(|model| self.create_language_model(model))
.collect()
}
- fn telemetry_id(&self) -> String {
- format!("anthropic/{}", self.model.id)
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn max_token_count(&self) -> u64 {
- self.model.max_input_tokens
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
+ }
+
+ fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.clone();
+ Some(ProviderSettingsView::SubPage(SubPageProviderSettings::new(
+ move |window, cx| {
+ cx.new(|cx| {
+ ApiCompatibleProviderConfigurationView::new(
+ state.clone(),
+ "Anthropic",
+ API_KEY_PLACEHOLDER,
+ window,
+ cx,
+ )
+ })
+ .into()
+ },
+ )))
}
- fn max_output_tokens(&self) -> Option<u64> {
- Some(self.model.max_output_tokens)
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let (config, cache_mode) = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
let has_tools = !request.tools.is_empty();
- let request_id = self.model.request_id(has_tools).to_string();
+ let request_id = config.request_id(has_tools).to_string();
let mut request = match into_anthropic(
request,
request_id,
- self.model.default_temperature,
- self.model.max_output_tokens,
- self.model.mode.clone(),
- self.cache_mode,
- &self.provider_id,
+ config.default_temperature,
+ config.max_output_tokens,
+ config.mode.clone(),
+ cache_mode,
+ &self.id,
) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- if !self.model.supports_speed {
+ if !config.supports_speed {
request.speed = None;
}
- let completion_request = self.stream_completion(request, cx);
- let provider_name = self.provider_name.clone();
- let provider_id = self.provider_id.clone();
+ let completion_request = self.stream_anthropic_request(&config, request, cx);
+ let provider_name = self.name.clone();
+ let provider_id = self.id.clone();
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let response = completion_request.await?;
let events = AnthropicEventMapper::new(provider_name, provider_id).map_stream(response);
Ok(language_model::stream_in_background(
diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs
index 092dcd1ae10..239d7fd0911 100644
--- a/crates/language_models/src/provider/bedrock.rs
+++ b/crates/language_models/src/provider/bedrock.rs
@@ -2,10 +2,9 @@ use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Context as _, Result, anyhow};
-use async_lock::OnceCell;
use aws_config::stalled_stream_protection::StalledStreamProtectionConfig;
use aws_config::{BehaviorVersion, Region};
-use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider};
+use aws_credential_types::provider::ProvideCredentials;
use aws_credential_types::{Credentials, Token};
use aws_http_client::AwsHttpClient;
use aws_sigv4::http_request::{SignableBody, SignableRequest, SigningSettings, sign};
@@ -25,7 +24,7 @@ use bedrock::{
BedrockToolResultContentBlock, BedrockToolResultStatus, BedrockToolSpec, BedrockToolUseBlock,
ConverseModel, MantleModel, MantleProtocol, value_to_aws_document,
};
-use collections::{BTreeMap, HashMap};
+use collections::{BTreeMap, HashMap, IndexMap};
use credentials_provider::CredentialsProvider;
use futures::{
AsyncBufReadExt, AsyncReadExt, FutureExt, Stream, StreamExt, future::BoxFuture, io::BufReader,
@@ -41,12 +40,13 @@ use http_client::{
};
use language_model::{
AuthenticateError, EnvVar, IconOrSvg, InlineDescription, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelEffortLevel,
- LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse, MessageContent,
+ LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelCompletionStream,
+ LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProvider,
+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
+ LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
+ LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ModelRateLimiters,
ProviderErrorCategory, ProviderSettingsView, RateLimiter, Role, SubPageProviderSettings,
- TokenUsage, env_var,
+ TokenUsage, env_var, unavailable_error,
};
use open_ai::responses::Request as OpenAiResponseRequest;
use open_ai::responses::{ResponseOutputItem, StreamEvent as OpenAiResponseStreamEvent};
@@ -58,7 +58,7 @@ use settings::{
Settings, SettingsStore,
};
use std::sync::LazyLock;
-use std::time::SystemTime;
+use std::time::{Duration, SystemTime};
use strum::{EnumIter, IntoEnumIterator, IntoStaticStr};
use ui::{ButtonLink, ConfiguredApiCard, Divider, List, ListBulletItem, prelude::*};
use ui_input::InputField;
@@ -254,20 +254,22 @@ fn mantle_endpoint_url(region: &str) -> String {
format!("https://bedrock-mantle.{region}.api.aws/openai/v1")
}
-enum MantleAuth {
+/// Auth resolved for one request, shared by the Converse and Mantle APIs.
+#[derive(Clone)]
+enum BedrockRequestAuth {
ApiKey { api_key: String },
SigV4 { credentials: Credentials },
}
-impl MantleAuth {
+impl BedrockRequestAuth {
fn apply(&self, request: &mut HttpRequest<AsyncBody>, body: &[u8], region: &str) -> Result<()> {
match self {
- MantleAuth::ApiKey { api_key } => {
+ BedrockRequestAuth::ApiKey { api_key } => {
let value = HeaderValue::from_str(&format!("Bearer {}", api_key.trim()))
.context("building Mantle bearer token authorization header")?;
request.headers_mut().insert(AUTHORIZATION, value);
}
- MantleAuth::SigV4 { credentials } => {
+ BedrockRequestAuth::SigV4 { credentials } => {
sign_mantle_request_sigv4(request, body, credentials, region)?;
}
}
@@ -348,10 +350,54 @@ pub struct State {
/// Whether credentials came from environment variables (only relevant for static credentials)
credentials_from_env: bool,
credentials_provider: Arc<dyn CredentialsProvider>,
+ /// AWS credentials that expire, such as those from SSO, STS or a credential
+ /// process, reused until shortly before they expire. Credentials without an
+ /// expiry are resolved for every request instead, because the AWS SDK never
+ /// rereads credential files and external tools rewrite them.
+ expiring_credentials: Option<ExpiringCredentials>,
+ /// Incremented on every auth change, so credentials resolved for earlier
+ /// auth are not cached once they arrive.
+ auth_generation: u64,
_subscription: Subscription,
}
+struct ExpiringCredentials {
+ auth: Option<BedrockAuth>,
+ region: String,
+ credentials: Credentials,
+}
+
+/// How long before expiry cached AWS credentials are replaced, so a request
+/// that streams for a while does not outlive them.
+const CREDENTIALS_EXPIRY_MARGIN: Duration = Duration::from_secs(5 * 60);
+
+/// How long resolving AWS credentials may take before the request fails,
+/// matching the AWS SDK's default identity cache load timeout. Without it a
+/// hung credential process or metadata endpoint would stall the request.
+const CREDENTIALS_LOAD_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// The Bedrock bearer token the AWS SDK would find in the environment, looked
+/// up with `environment_variable`.
+///
+/// Unless a token is configured explicitly, the SDK prefers this token over any
+/// AWS credentials when it builds a client, so it is honored before resolving
+/// credentials. Like the SDK, this checks the Bedrock-specific variable, then
+/// the generic one, and uses a set value even when it is empty.
+fn environment_bearer_token(
+ environment_variable: impl Fn(&str) -> Option<String>,
+) -> Option<String> {
+ environment_variable("AWS_BEARER_TOKEN_BEDROCK")
+ .or_else(|| environment_variable("AWS_BEARER_TOKEN"))
+}
+
impl State {
+ /// Sets the auth method and drops credentials resolved for the old one.
+ fn set_auth(&mut self, auth: Option<BedrockAuth>) {
+ self.auth = auth;
+ self.expiring_credentials = None;
+ self.auth_generation += 1;
+ }
+
fn reset_auth(&self, cx: &mut Context<Self>) -> Task<Result<()>> {
let credentials_provider = self.credentials_provider.clone();
cx.spawn(async move |this, cx| {
@@ -360,7 +406,7 @@ impl State {
.await
.log_err();
this.update(cx, |this, cx| {
- this.auth = None;
+ this.set_auth(None);
this.credentials_from_env = false;
cx.notify();
})
@@ -384,7 +430,7 @@ impl State {
)
.await?;
this.update(cx, |this, cx| {
- this.auth = auth;
+ this.set_auth(auth);
this.credentials_from_env = false;
cx.notify();
})
@@ -422,7 +468,7 @@ impl State {
return cx.spawn(async move |this, cx| {
this.update(cx, |this, cx| {
- this.auth = Some(auth);
+ this.set_auth(Some(auth));
this.credentials_from_env = false;
cx.notify();
})?;
@@ -483,7 +529,7 @@ impl State {
// If we got auth from env vars, use it
if let Some(auth) = auth {
this.update(cx, |this, cx| {
- this.auth = Some(auth);
+ this.set_auth(Some(auth));
this.credentials_from_env = from_env;
cx.notify();
})?;
@@ -507,7 +553,7 @@ impl State {
.ok_or(AuthenticateError::CredentialsNotFound)?;
this.update(cx, |this, cx| {
- this.auth = Some(auth);
+ this.set_auth(Some(auth));
this.credentials_from_env = false;
cx.notify();
})?;
@@ -548,8 +594,14 @@ impl State {
pub struct BedrockLanguageModelProvider {
http_client: AwsHttpClient,
plain_http_client: Arc<dyn HttpClient>,
- handle: tokio::runtime::Handle,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
+}
+
+/// A model this provider offers, served by either the Converse API or Mantle.
+enum BedrockModelConfig {
+ Converse(ConverseModel),
+ Mantle(MantleModel),
}
impl BedrockLanguageModelProvider {
@@ -563,6 +615,8 @@ impl BedrockLanguageModelProvider {
settings: Some(AllLanguageModelSettings::get_global(cx).bedrock.clone()),
credentials_from_env: false,
credentials_provider,
+ expiring_credentials: None,
+ auth_generation: 0,
_subscription: cx.observe_global::<SettingsStore>(|_, cx| {
cx.notify();
}),
@@ -571,58 +625,16 @@ impl BedrockLanguageModelProvider {
Self {
http_client: AwsHttpClient::new(http_client.clone()),
plain_http_client: http_client,
- handle: Tokio::handle(cx),
state,
+ request_limiters: ModelRateLimiters::default(),
}
}
- fn create_language_model(&self, model: bedrock::ConverseModel) -> Arc<dyn LanguageModel> {
- Arc::new(BedrockModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- http_client: self.http_client.clone(),
- handle: self.handle.clone(),
- state: self.state.clone(),
- client: OnceCell::new(),
- request_limiter: RateLimiter::new(4),
- })
- }
-
- fn create_mantle_language_model(&self, model: bedrock::MantleModel) -> Arc<dyn LanguageModel> {
- Arc::new(BedrockMantleModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- http_client: self.plain_http_client.clone(),
- state: self.state.clone(),
- credentials_provider: Arc::new(OnceCell::new()),
- request_limiter: RateLimiter::new(4),
- })
- }
-}
-
-impl LanguageModelProvider for BedrockLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiBedrock)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(bedrock::ConverseModel::default()))
- }
-
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
- let region = self.state.read(cx).get_region();
- Some(self.create_language_model(bedrock::ConverseModel::default_fast(region.as_str())))
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: the built-in Converse
+ /// models with settings entries added or overriding them, then the same
+ /// for Mantle. A Mantle model replaces a Converse model with the same id,
+ /// so each id is offered and served exactly once.
+ fn bedrock_models(&self, cx: &App) -> IndexMap<String, BedrockModelConfig> {
let bedrock_settings = &AllLanguageModelSettings::get_global(cx).bedrock;
let mut models = BTreeMap::default();
@@ -632,7 +644,6 @@ impl LanguageModelProvider for BedrockLanguageModelProvider {
}
}
- // Override with available models from settings
for model in bedrock_settings.available_models.iter() {
models.insert(
model.name.clone(),
@@ -652,11 +663,6 @@ impl LanguageModelProvider for BedrockLanguageModelProvider {
);
}
- let mut models: Vec<Arc<dyn LanguageModel>> = models
- .into_values()
- .map(|model| self.create_language_model(model))
- .collect();
-
let mut mantle_models = BTreeMap::default();
for model in bedrock::MantleModel::iter() {
@@ -665,7 +671,6 @@ impl LanguageModelProvider for BedrockLanguageModelProvider {
}
}
- // Override with available Mantle models from settings
for model in bedrock_settings.mantle_available_models.iter() {
mantle_models.insert(
model.name.clone(),
@@ -682,13 +687,92 @@ impl LanguageModelProvider for BedrockLanguageModelProvider {
);
}
- models.extend(
- mantle_models
- .into_values()
- .map(|model| self.create_mantle_language_model(model)),
- );
+ let mut all_models = IndexMap::default();
+ for (id, model) in models {
+ all_models.insert(id, BedrockModelConfig::Converse(model));
+ }
+ for (id, model) in mantle_models {
+ all_models.insert(id, BedrockModelConfig::Mantle(model));
+ }
+ all_models
+ }
+
+ /// Describes the offered model with `id`.
+ fn offered_model(&self, id: &str, cx: &App) -> Option<LanguageModel> {
+ self.bedrock_models(cx).get(id).map(language_model)
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<BedrockModelConfig, LanguageModelCompletionError> {
+ self.bedrock_models(cx)
+ .swap_remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
+ }
+}
+
+fn language_model(config: &BedrockModelConfig) -> LanguageModel {
+ match config {
+ BedrockModelConfig::Converse(model) => converse_language_model(model),
+ BedrockModelConfig::Mantle(model) => mantle_language_model(model),
+ }
+}
+
+impl LanguageModelProvider for BedrockLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
+ PROVIDER_ID
+ }
+
+ fn name(&self) -> LanguageModelProviderName {
+ PROVIDER_NAME
+ }
+
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiBedrock)
+ }
+
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.offered_model(bedrock::ConverseModel::default().id(), cx)
+ }
- models
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ let region = self.state.read(cx).get_region();
+ self.offered_model(
+ bedrock::ConverseModel::default_fast(region.as_str()).id(),
+ cx,
+ )
+ }
+
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.bedrock_models(cx)
+ .values()
+ .map(language_model)
+ .collect()
+ }
+
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ match config {
+ BedrockModelConfig::Converse(config) => {
+ self.stream_converse_completion(&config, &request_limiter, request, cx)
+ }
+ BedrockModelConfig::Mantle(config) => {
+ self.stream_mantle_completion(&config, &request_limiter, request, cx)
+ }
+ }
}
fn is_authenticated(&self, cx: &App) -> bool {
@@ -721,79 +805,85 @@ impl LanguageModelProviderState for BedrockLanguageModelProvider {
}
}
-struct BedrockModel {
- id: LanguageModelId,
- model: ConverseModel,
- http_client: AwsHttpClient,
- handle: tokio::runtime::Handle,
- client: OnceCell<BedrockClient>,
- state: Entity<State>,
- request_limiter: RateLimiter,
-}
-
-impl BedrockModel {
- fn get_or_init_client(&self, cx: &AsyncApp) -> anyhow::Result<&BedrockClient> {
- self.client
- .get_or_try_init_blocking(|| {
- let (auth, endpoint, region) = cx.read_entity(&self.state, |state, _cx| {
- let endpoint = state.settings.as_ref().and_then(|s| s.endpoint.clone());
- let region = state.get_region();
- (state.auth.clone(), endpoint, region)
- });
-
- let mut config_builder = aws_config::defaults(BehaviorVersion::latest())
- .stalled_stream_protection(StalledStreamProtectionConfig::disabled())
- .http_client(self.http_client.clone())
- .region(Region::new(region))
- .timeout_config(TimeoutConfig::disabled());
+impl BedrockLanguageModelProvider {
+ /// Resolves auth for one request, reusing unexpired cached credentials.
+ fn resolve_request_auth(&self, cx: &AsyncApp) -> Task<Result<BedrockRequestAuth>> {
+ let http_client = self.http_client.clone();
+ self.resolve_request_auth_with(
+ cx,
+ |name| std::env::var(name).ok(),
+ move |auth, region| resolve_request_auth(http_client, auth, region),
+ )
+ }
- if let Some(endpoint_url) = endpoint
- && !endpoint_url.is_empty()
- {
- config_builder = config_builder.endpoint_url(endpoint_url);
- }
+ /// [`Self::resolve_request_auth`], reading environment variables with
+ /// `environment_variable` and resolving uncached auth with `resolve`.
+ fn resolve_request_auth_with<ResolveFuture>(
+ &self,
+ cx: &AsyncApp,
+ environment_variable: impl Fn(&str) -> Option<String>,
+ resolve: impl FnOnce(Option<BedrockAuth>, String) -> ResolveFuture,
+ ) -> Task<Result<BedrockRequestAuth>>
+ where
+ ResolveFuture: Future<Output = Result<BedrockRequestAuth>> + Send + 'static,
+ {
+ let (auth, region, auth_generation, cached_credentials) =
+ cx.read_entity(&self.state, |state, _cx| {
+ let (auth, region) = (state.auth.clone(), state.get_region());
+ let fresh_until = SystemTime::now() + CREDENTIALS_EXPIRY_MARGIN;
+ let cached_credentials = state
+ .expiring_credentials
+ .as_ref()
+ .filter(|cached| {
+ cached.auth == auth
+ && cached.region == region
+ && cached
+ .credentials
+ .expiry()
+ .is_some_and(|expiry| expiry > fresh_until)
+ })
+ .map(|cached| cached.credentials.clone());
+ (auth, region, state.auth_generation, cached_credentials)
+ });
+ if !matches!(auth, Some(BedrockAuth::ApiKey { .. }))
+ && let Some(api_key) = environment_bearer_token(environment_variable)
+ {
+ return Task::ready(Ok(BedrockRequestAuth::ApiKey { api_key }));
+ }
+ if let Some(credentials) = cached_credentials {
+ return Task::ready(Ok(BedrockRequestAuth::SigV4 { credentials }));
+ }
- match auth {
- Some(BedrockAuth::Automatic) | None => {
- // Use default AWS credential provider chain
- }
- Some(BedrockAuth::NamedProfile { profile_name })
- | Some(BedrockAuth::SingleSignOn { profile_name }) => {
- if !profile_name.is_empty() {
- config_builder = config_builder.profile_name(profile_name);
- }
- }
- Some(BedrockAuth::IamCredentials {
- access_key_id,
- secret_access_key,
- session_token,
- }) => {
- let aws_creds = Credentials::new(
- access_key_id,
- secret_access_key,
- session_token,
- None,
- "zed-bedrock-provider",
- );
- config_builder = config_builder.credentials_provider(aws_creds);
- }
- Some(BedrockAuth::ApiKey { api_key }) => {
- config_builder = config_builder
- .auth_scheme_preference(["httpBearerAuth".into()]) // https://github.com/smithy-lang/smithy-rs/pull/4241
- .token_provider(Token::new(api_key, None));
+ let resolve_task = Tokio::spawn_result(cx, resolve(auth.clone(), region.clone()));
+ let timeout = cx.background_executor().timer(CREDENTIALS_LOAD_TIMEOUT);
+ let state = self.state.clone();
+ cx.spawn(async move |cx| {
+ // Dropping `resolve_task` on timeout cancels the Tokio task.
+ let request_auth = futures::select_biased! {
+ request_auth = resolve_task.fuse() => request_auth?,
+ _ = timeout.fuse() => anyhow::bail!(
+ "timed out after {CREDENTIALS_LOAD_TIMEOUT:?} resolving AWS credentials"
+ ),
+ };
+ if let BedrockRequestAuth::SigV4 { credentials } = &request_auth
+ && credentials.expiry().is_some()
+ {
+ let credentials = credentials.clone();
+ state.update(cx, |state, _| {
+ if state.auth_generation == auth_generation {
+ state.expiring_credentials = Some(ExpiringCredentials {
+ auth,
+ region,
+ credentials,
+ });
}
- }
-
- let config = self.handle.block_on(config_builder.load());
-
- anyhow::Ok(BedrockClient::new(&config))
- })
- .context("initializing Bedrock client")?;
-
- self.client.get().context("Bedrock client not initialized")
+ });
+ }
+ Ok(request_auth)
+ })
}
- fn stream_completion(
+ fn stream_bedrock_request(
&self,
request: bedrock::Request,
cx: &AsyncApp,
@@ -801,72 +891,78 @@ impl BedrockModel {
'static,
Result<BoxStream<'static, Result<BedrockStreamingResponse, anyhow::Error>>, BedrockError>,
> {
- let Ok(runtime_client) = self
- .get_or_init_client(cx)
- .cloned()
- .context("Bedrock client not initialized")
- else {
- return futures::future::ready(Err(BedrockError::Other(anyhow!("App state dropped"))))
- .boxed();
- };
- let extra_headers = self.state.read_with(cx, |_, cx| {
- AllLanguageModelSettings::get_global(cx)
- .bedrock
- .custom_headers
- .clone()
- });
+ let http_client = self.http_client.clone();
+ let auth_task = self.resolve_request_auth(cx);
+ let (profile_name, endpoint, region, extra_headers) =
+ self.state.read_with(cx, |state, cx| {
+ (
+ aws_profile_name(&state.auth),
+ state.settings.as_ref().and_then(|s| s.endpoint.clone()),
+ state.get_region(),
+ AllLanguageModelSettings::get_global(cx)
+ .bedrock
+ .custom_headers
+ .clone(),
+ )
+ });
- let task = Tokio::spawn(
- cx,
- bedrock::stream_completion(runtime_client, request, extra_headers),
- );
+ let task = Tokio::spawn(cx, async move {
+ let request_auth = auth_task.await.map_err(BedrockError::Other)?;
+ let client =
+ build_converse_client(http_client, request_auth, profile_name, endpoint, region)
+ .await;
+ bedrock::stream_completion(client, request, extra_headers).await
+ });
async move { task.await.map_err(|e| BedrockError::Other(e.into()))? }.boxed()
}
}
-impl LanguageModel for BedrockModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
+/// Builds a Converse client for one request from already-resolved auth. The
+/// profile still applies, since it can configure more than credentials.
+async fn build_converse_client(
+ http_client: AwsHttpClient,
+ request_auth: BedrockRequestAuth,
+ profile_name: Option<String>,
+ endpoint: Option<String>,
+ region: String,
+) -> BedrockClient {
+ let mut config_builder = aws_config::defaults(BehaviorVersion::latest())
+ .stalled_stream_protection(StalledStreamProtectionConfig::disabled())
+ .http_client(http_client)
+ .region(Region::new(region))
+ .timeout_config(TimeoutConfig::disabled());
+
+ if let Some(profile_name) = profile_name {
+ config_builder = config_builder.profile_name(profile_name);
}
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tool_use()
+ if let Some(endpoint_url) = endpoint
+ && !endpoint_url.is_empty()
+ {
+ config_builder = config_builder.endpoint_url(endpoint_url);
}
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
+ config_builder = match request_auth {
+ BedrockRequestAuth::SigV4 { credentials } => {
+ config_builder.credentials_provider(credentials)
+ }
+ BedrockRequestAuth::ApiKey { api_key } => config_builder
+ .auth_scheme_preference(["httpBearerAuth".into()]) // https://github.com/smithy-lang/smithy-rs/pull/4241
+ .token_provider(Token::new(api_key, None)),
+ };
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking()
- }
+ BedrockClient::new(&config_builder.load().await)
+}
- fn refusal_fallback_model_id(&self) -> Option<&'static str> {
- if self
- .model
+fn converse_language_model(model: &ConverseModel) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.supports_tool_use(),
+ supports_images: model.supports_images(),
+ supports_thinking: model.supports_thinking(),
+ refusal_fallback_model_id: model
.id()
.starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
- {
- Some(anthropic::FABLE_FALLBACK_MODEL_ID)
- } else {
- None
- }
- }
-
- fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
- if self.model.supports_adaptive_thinking() {
+ .then_some(anthropic::FABLE_FALLBACK_MODEL_ID),
+ supported_effort_levels: if model.supports_adaptive_thinking() {
vec![
language_model::LanguageModelEffortLevel {
name: "Low".into(),
@@ -896,51 +992,40 @@ impl LanguageModel for BedrockModel {
]
.into_iter()
.filter(|effort_level| {
- effort_level.value != "xhigh" || self.model.supports_xhigh_adaptive_thinking()
+ effort_level.value != "xhigh" || model.supports_xhigh_adaptive_thinking()
})
.collect()
} else {
- Vec::new()
- }
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::Any => {
- self.model.supports_tool_use()
- }
- // Add support for None - we'll filter tool calls at response
- LanguageModelToolChoice::None => self.model.supports_tool_use(),
- }
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn telemetry_id(&self) -> String {
- format!("bedrock/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- Some(self.model.max_output_tokens())
+ Arc::default()
+ },
+ // Add support for None - we'll filter tool calls at response
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: model.supports_tool_use(),
+ any: model.supports_tool_use(),
+ none: model.supports_tool_use(),
+ },
+ supports_streaming_tools: true,
+ max_output_tokens: Some(model.max_output_tokens()),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("bedrock/{}", model.id()),
+ model.max_token_count(),
+ )
}
+}
- fn stream_completion(
+impl BedrockLanguageModelProvider {
+ fn stream_converse_completion(
&self,
+ config: &ConverseModel,
+ request_limiter: &RateLimiter,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
if request.contains_custom_tool_input() {
return async move {
Err(anyhow::anyhow!("Bedrock does not support custom tools").into())
@@ -954,7 +1039,7 @@ impl LanguageModel for BedrockModel {
(state.get_region(), state.get_allow_global(), gid, gv)
});
- let model_id = match self.model.cross_region_inference_id(&region, allow_global) {
+ let model_id = match config.cross_region_inference_id(&region, allow_global) {
Ok(s) => s,
Err(e) => {
return async move { Err(e.into()) }.boxed();
@@ -966,11 +1051,11 @@ impl LanguageModel for BedrockModel {
let request = match into_bedrock(
request,
model_id,
- self.model.default_temperature(),
- self.model.max_output_tokens(),
- self.model.thinking_mode(),
- self.model.supports_caching(),
- self.model.supports_tool_use(),
+ config.default_temperature(),
+ config.max_output_tokens(),
+ config.thinking_mode(),
+ config.supports_caching(),
+ config.supports_tool_use(),
guardrail_identifier,
guardrail_version,
) {
@@ -978,10 +1063,10 @@ impl LanguageModel for BedrockModel {
Err(err) => return futures::future::ready(Err(err.into())).boxed(),
};
- let request = self.stream_completion(request, cx);
- let display_name = self.model.display_name().to_string();
+ let request = self.stream_bedrock_request(request, cx);
+ let display_name = config.display_name().to_string();
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let response = request.await.map_err(|err| match err {
BedrockError::Validation(ref msg) => {
if msg.contains("model identifier is invalid") {
@@ -1135,82 +1220,61 @@ fn map_mantle_error(model: &MantleModel, error: RequestError) -> LanguageModelCo
error.into()
}
-/// Resolves an AWS credentials provider for profile/SSO/automatic auth.
-/// Cached in `cell` since building it may read config files from disk;
-/// credentials themselves are still re-resolved on every call. Async so this
-/// never blocks the foreground thread (unlike `BedrockModel::get_or_init_client`).
-async fn resolve_mantle_credentials_provider(
- cell: &OnceCell<SharedCredentialsProvider>,
- profile_name: Option<String>,
- region: String,
-) -> Result<SharedCredentialsProvider> {
- let provider = cell
- .get_or_try_init(move || async move {
- let mut config_builder =
- aws_config::defaults(BehaviorVersion::latest()).region(Region::new(region));
-
- if let Some(profile_name) = profile_name.filter(|name| !name.is_empty()) {
- config_builder = config_builder.profile_name(profile_name);
- }
-
- let config = config_builder.load().await;
- config
- .credentials_provider()
- .context("no AWS credentials provider is configured")
- })
- .await
- .context("resolving AWS credentials for Bedrock Mantle")?;
- Ok(provider.clone())
+/// The AWS profile named by profile or SSO auth, if any.
+fn aws_profile_name(auth: &Option<BedrockAuth>) -> Option<String> {
+ match auth {
+ Some(BedrockAuth::NamedProfile { profile_name })
+ | Some(BedrockAuth::SingleSignOn { profile_name }) => {
+ Some(profile_name.clone()).filter(|name| !name.is_empty())
+ }
+ _ => None,
+ }
}
-/// Resolves provider settings into concrete Mantle request auth. A configured
-/// Bedrock API key is sent as bearer auth; every AWS-credential-based method
-/// signs the Mantle HTTP request directly with SigV4.
-async fn resolve_mantle_auth(
- credentials_provider: Arc<OnceCell<SharedCredentialsProvider>>,
+/// Resolves provider settings into concrete request auth. A configured Bedrock
+/// API key is sent as bearer auth; every other method resolves AWS
+/// credentials to sign the request with SigV4.
+async fn resolve_request_auth(
+ http_client: AwsHttpClient,
auth: Option<BedrockAuth>,
region: String,
-) -> Result<MantleAuth> {
- match auth {
- Some(BedrockAuth::ApiKey { api_key }) => Ok(MantleAuth::ApiKey { api_key }),
+) -> Result<BedrockRequestAuth> {
+ let profile_name = match auth {
+ Some(BedrockAuth::ApiKey { api_key }) => return Ok(BedrockRequestAuth::ApiKey { api_key }),
Some(BedrockAuth::IamCredentials {
access_key_id,
secret_access_key,
session_token,
- }) => Ok(MantleAuth::SigV4 {
- credentials: Credentials::new(
- access_key_id,
- secret_access_key,
- session_token,
- None,
- "zed-bedrock-provider",
- ),
- }),
- Some(BedrockAuth::NamedProfile { profile_name })
- | Some(BedrockAuth::SingleSignOn { profile_name }) => {
- let provider = resolve_mantle_credentials_provider(
- &credentials_provider,
- Some(profile_name),
- region.clone(),
- )
- .await?;
- let credentials = provider
- .provide_credentials()
- .await
- .context("failed to resolve AWS credentials")?;
- Ok(MantleAuth::SigV4 { credentials })
- }
- Some(BedrockAuth::Automatic) | None => {
- let provider =
- resolve_mantle_credentials_provider(&credentials_provider, None, region.clone())
- .await?;
- let credentials = provider
- .provide_credentials()
- .await
- .context("failed to resolve AWS credentials")?;
- Ok(MantleAuth::SigV4 { credentials })
+ }) => {
+ return Ok(BedrockRequestAuth::SigV4 {
+ credentials: Credentials::new(
+ access_key_id,
+ secret_access_key,
+ session_token,
+ None,
+ "zed-bedrock-provider",
+ ),
+ });
}
+ _ => aws_profile_name(&auth),
+ };
+
+ let mut config_builder = aws_config::defaults(BehaviorVersion::latest())
+ .http_client(http_client)
+ .region(Region::new(region));
+ if let Some(profile_name) = profile_name {
+ config_builder = config_builder.profile_name(profile_name);
}
+ let credentials = config_builder
+ .load()
+ .await
+ .credentials_provider()
+ .context("no AWS credentials provider is configured")
+ .context("resolving AWS credentials for Bedrock")?
+ .provide_credentials()
+ .await
+ .context("failed to resolve AWS credentials")?;
+ Ok(BedrockRequestAuth::SigV4 { credentials })
}
fn parse_mantle_chat_stream_line(line: &str) -> Result<ResponseStreamEvent> {
@@ -1244,7 +1308,7 @@ async fn stream_mantle_sse<Request, Event>(
provider_name: &str,
url: &str,
region: &str,
- auth: &MantleAuth,
+ auth: &BedrockRequestAuth,
request: Request,
extra_headers: &CustomHeaders,
parse_stream_line: fn(&str) -> Result<Event>,
@@ -1363,8 +1427,7 @@ impl MantleResponseEventMapper {
fn map_stream(
mut self,
events: BoxStream<'static, Result<OpenAiResponseStreamEvent>>,
- ) -> BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
- {
+ ) -> LanguageModelCompletionStream {
events
.flat_map(move |event| {
futures::stream::iter(match event {
@@ -1738,18 +1801,11 @@ impl MantleResponseEventMapper {
}
}
-struct BedrockMantleModel {
- id: LanguageModelId,
- model: MantleModel,
- http_client: Arc<dyn HttpClient>,
- state: Entity<State>,
- credentials_provider: Arc<OnceCell<SharedCredentialsProvider>>,
- request_limiter: RateLimiter,
-}
-
-impl BedrockMantleModel {
+impl BedrockLanguageModelProvider {
fn stream_mantle_request<Request, Event>(
&self,
+ config: &MantleModel,
+ request_limiter: &RateLimiter,
request: Request,
cx: &AsyncApp,
endpoint: &'static str,
@@ -1759,12 +1815,9 @@ impl BedrockMantleModel {
Request: Serialize + Send + 'static,
Event: Send + 'static,
{
- let http_client = self.http_client.clone();
- let model = self.model.clone();
- let credentials_provider = self.credentials_provider.clone();
- let (auth, region) = cx.read_entity(&self.state, |state, _cx| {
- (state.auth.clone(), state.get_region())
- });
+ let http_client = self.plain_http_client.clone();
+ let model = config.clone();
+ let region = cx.read_entity(&self.state, |state, _cx| state.get_region());
let url = format!("{}/{}", mantle_endpoint_url(&region), endpoint);
let extra_headers = cx.read_entity(&self.state, |_, cx| {
AllLanguageModelSettings::get_global(cx)
@@ -1773,12 +1826,9 @@ impl BedrockMantleModel {
.clone()
});
let provider_name = PROVIDER_NAME.0.to_string();
- let auth_task = Tokio::spawn_result(
- cx,
- resolve_mantle_auth(credentials_provider, auth, region.clone()),
- );
+ let auth_task = self.resolve_request_auth(cx);
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let auth = auth_task
.await
.map_err(LanguageModelCompletionError::Other)?;
@@ -1799,8 +1849,10 @@ impl BedrockMantleModel {
async move { Ok(future.await?.boxed()) }.boxed()
}
- fn stream_completion(
+ fn stream_chat_completion(
&self,
+ config: &MantleModel,
+ request_limiter: &RateLimiter,
request: open_ai::Request,
cx: &AsyncApp,
) -> BoxFuture<
@@ -1808,6 +1860,8 @@ impl BedrockMantleModel {
Result<BoxStream<'static, Result<ResponseStreamEvent>>, LanguageModelCompletionError>,
> {
self.stream_mantle_request(
+ config,
+ request_limiter,
request,
cx,
"chat/completions",
@@ -1817,6 +1871,8 @@ impl BedrockMantleModel {
fn stream_response(
&self,
+ config: &MantleModel,
+ request_limiter: &RateLimiter,
request: OpenAiResponseRequest,
cx: &AsyncApp,
) -> BoxFuture<
@@ -1828,87 +1884,55 @@ impl BedrockMantleModel {
> {
let mut request = request;
strip_unsupported_mantle_response_fields(&mut request);
- self.stream_mantle_request(request, cx, "responses", parse_mantle_response_stream_line)
+ self.stream_mantle_request(
+ config,
+ request_limiter,
+ request,
+ cx,
+ "responses",
+ parse_mantle_response_stream_line,
+ )
}
}
-impl LanguageModel for BedrockMantleModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::Any => {
- self.model.supports_tools()
- }
- LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- mantle_supported_effort_levels(&self.model)
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn telemetry_id(&self) -> String {
- format!("bedrock-mantle/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- Some(self.model.max_output_tokens())
+fn mantle_language_model(model: &MantleModel) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.supports_tools(),
+ supports_images: model.supports_images(),
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: model.supports_tools(),
+ any: model.supports_tools(),
+ none: true,
+ },
+ supports_streaming_tools: true,
+ supports_thinking: model.supports_thinking(),
+ supported_effort_levels: mantle_supported_effort_levels(model).into(),
+ supports_split_token_display: true,
+ max_output_tokens: Some(model.max_output_tokens()),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("bedrock-mantle/{}", model.id()),
+ model.max_token_count(),
+ )
}
+}
- fn stream_completion(
+impl BedrockLanguageModelProvider {
+ fn stream_mantle_completion(
&self,
+ config: &MantleModel,
+ request_limiter: &RateLimiter,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
let region = cx.read_entity(&self.state, |state, _cx| state.get_region());
if !MANTLE_SUPPORTED_REGIONS.contains(&region.as_str()) {
- let display_name = self.model.display_name().to_string();
+ let display_name = config.display_name().to_string();
let supported = MANTLE_SUPPORTED_REGIONS.join(", ");
return futures::future::ready(Err(LanguageModelCompletionError::Other(anyhow!(
"{display_name} is not available in {region} because Bedrock Mantle isn't offered \
@@ -1917,25 +1941,25 @@ impl LanguageModel for BedrockMantleModel {
.boxed();
}
- let model_id = self.model.request_id().to_string();
- let max_output_tokens = Some(self.model.max_output_tokens());
+ let model_id = config.request_id().to_string();
+ let max_output_tokens = Some(config.max_output_tokens());
- match self.model.protocol() {
+ match config.protocol() {
MantleProtocol::Responses => {
let request = match into_open_ai_response(
request,
&model_id,
- self.model.supports_tools(),
+ config.supports_tools(),
false,
max_output_tokens,
- mantle_default_reasoning_effort(&self.model),
- self.model.supports_thinking(),
+ mantle_default_reasoning_effort(config),
+ config.supports_thinking(),
&PROVIDER_ID,
) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let completions = self.stream_response(request, cx);
+ let completions = self.stream_response(config, request_limiter, request, cx);
let executor = cx.background_executor().clone();
async move {
let mapper = MantleResponseEventMapper::new();
@@ -1947,11 +1971,11 @@ impl LanguageModel for BedrockMantleModel {
.boxed()
}
MantleProtocol::ChatCompletions => {
- let reasoning_effort = mantle_selected_reasoning_effort(&request, &self.model);
+ let reasoning_effort = mantle_selected_reasoning_effort(&request, config);
let request = match into_open_ai(
request,
&model_id,
- self.model.supports_tools(),
+ config.supports_tools(),
false,
max_output_tokens,
ChatCompletionMaxTokensParameter::MaxCompletionTokens,
@@ -1961,7 +1985,7 @@ impl LanguageModel for BedrockMantleModel {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let completions = self.stream_completion(request, cx);
+ let completions = self.stream_chat_completion(config, request_limiter, request, cx);
let executor = cx.background_executor().clone();
async move {
let mapper = ChatCompletionEventMapper::new();
@@ -2946,6 +2970,246 @@ mod tests {
.unwrap()
}
+ #[gpui::test]
+ fn mantle_model_replaces_converse_model_with_the_same_id(cx: &mut gpui::TestAppContext) {
+ cx.update(|cx| {
+ let settings_store = SettingsStore::test(cx);
+ cx.set_global(settings_store);
+ gpui_tokio::init(cx);
+ let content = serde_json::json!({
+ "language_models": {
+ "bedrock": {
+ "available_models": [{
+ "name": "shared-model",
+ "display_name": "Converse Shared",
+ "max_tokens": 1000,
+ }],
+ "mantle_available_models": [{
+ "name": "shared-model",
+ "display_name": "Mantle Shared",
+ "max_tokens": 2000,
+ "protocol": "chat_completions",
+ }],
+ }
+ }
+ })
+ .to_string();
+ cx.update_global::<SettingsStore, _>(|store, cx| {
+ store
+ .set_user_settings(&content, cx)
+ .expect("test settings should parse");
+ });
+ });
+ let provider = cx.update(|cx| {
+ BedrockLanguageModelProvider::new(
+ http_client::FakeHttpClient::with_404_response(),
+ Arc::new(NoCredentialsProvider),
+ cx,
+ )
+ });
+
+ let shared = cx
+ .update(|cx| provider.provided_models(cx))
+ .into_iter()
+ .filter(|model| model.id.0.as_ref() == "shared-model")
+ .collect::<Vec<_>>();
+ assert_eq!(shared.len(), 1, "each id should be offered once");
+ assert_eq!(shared[0].name.0.as_ref(), "Mantle Shared");
+
+ let config = cx
+ .update(|cx| provider.config(&shared[0], cx))
+ .expect("the offered model should resolve");
+ assert!(matches!(
+ config,
+ BedrockModelConfig::Mantle(model) if model.display_name() == "Mantle Shared"
+ ));
+ }
+
+ #[test]
+ fn environment_bearer_token_prefers_the_bedrock_variable_like_the_aws_sdk() {
+ let environment = |variables: &[(&str, &str)]| {
+ let variables: HashMap<String, String> = variables
+ .iter()
+ .map(|(name, value)| (name.to_string(), value.to_string()))
+ .collect();
+ move |name: &str| variables.get(name).cloned()
+ };
+
+ assert_eq!(environment_bearer_token(environment(&[])), None);
+ assert_eq!(
+ environment_bearer_token(environment(&[("AWS_BEARER_TOKEN", "generic")])),
+ Some("generic".to_string())
+ );
+ assert_eq!(
+ environment_bearer_token(environment(&[
+ ("AWS_BEARER_TOKEN", "generic"),
+ ("AWS_BEARER_TOKEN_BEDROCK", "bedrock"),
+ ])),
+ Some("bedrock".to_string())
+ );
+ assert_eq!(
+ environment_bearer_token(environment(&[
+ ("AWS_BEARER_TOKEN", "generic"),
+ ("AWS_BEARER_TOKEN_BEDROCK", ""),
+ ])),
+ Some(String::new()),
+ "a set but empty Bedrock variable still wins, as in the SDK"
+ );
+ }
+
+ #[gpui::test]
+ async fn expiring_credentials_are_cached_per_auth_until_near_expiry(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ let provider = test_provider(cx);
+ let auth = Some(BedrockAuth::NamedProfile {
+ profile_name: "work".into(),
+ });
+ let other_auth = Some(BedrockAuth::NamedProfile {
+ profile_name: "other".into(),
+ });
+ provider
+ .state
+ .update(cx, |state, _| state.set_auth(auth.clone()));
+
+ // Resolved credentials that expire are cached and reused.
+ let resolved = provider
+ .resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| {
+ resolved_credentials("first", Some(Duration::from_secs(60 * 60)))
+ })
+ .await;
+ assert_eq!(access_key_id(resolved), "first");
+ let reused = provider
+ .resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| {
+ resolved_credentials("second", None)
+ })
+ .await;
+ assert_eq!(
+ access_key_id(reused),
+ "first",
+ "unexpired credentials are reused"
+ );
+
+ // Credentials close to expiry, or cached for other auth, are replaced.
+ for (cached_auth, expires_in, case) in [
+ (&auth, Duration::from_secs(60), "near expiry"),
+ (&other_auth, Duration::from_secs(60 * 60), "different auth"),
+ ] {
+ provider.state.update(cx, |state, _| {
+ state.expiring_credentials = Some(ExpiringCredentials {
+ auth: cached_auth.clone(),
+ region: state.get_region(),
+ credentials: Credentials::new(
+ "cached",
+ "secret",
+ None,
+ Some(SystemTime::now() + expires_in),
+ "test",
+ ),
+ });
+ });
+ let resolved = provider
+ .resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| {
+ resolved_credentials("fresh", None)
+ })
+ .await;
+ assert_eq!(access_key_id(resolved), "fresh", "{case}");
+ }
+
+ // Credentials without an expiry are never cached.
+ provider
+ .state
+ .update(cx, |state, _| state.expiring_credentials = None);
+ provider
+ .resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| {
+ resolved_credentials("static", None)
+ })
+ .await
+ .expect("auth should resolve");
+ assert!(
+ provider
+ .state
+ .read_with(cx, |state, _| state.expiring_credentials.is_none()),
+ "credentials without an expiry should not be cached"
+ );
+
+ // A resolution that completes after an auth change, even back to the
+ // same auth, is not cached.
+ let (send_credentials, receive_credentials) = futures::channel::oneshot::channel();
+ let pending =
+ provider.resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| async move {
+ receive_credentials
+ .await
+ .expect("the test sends credentials")
+ });
+ provider
+ .state
+ .update(cx, |state, _| state.set_auth(auth.clone()));
+ assert!(
+ send_credentials
+ .send(resolved_credentials("stale", Some(Duration::from_secs(60 * 60))).await)
+ .is_ok(),
+ "the pending resolution should still be waiting"
+ );
+ pending.await.expect("auth should resolve");
+ assert!(
+ provider
+ .state
+ .read_with(cx, |state, _| state.expiring_credentials.is_none()),
+ "credentials resolved for earlier auth should not be cached"
+ );
+ }
+
+ #[gpui::test]
+ async fn resolving_request_auth_prefers_the_environment_token_and_times_out(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ let provider = test_provider(cx);
+ provider.state.update(cx, |state, _| {
+ state.set_auth(Some(BedrockAuth::Automatic));
+ });
+
+ let request_auth = provider
+ .resolve_request_auth_with(
+ &cx.to_async(),
+ |name| (name == "AWS_BEARER_TOKEN").then(|| "token".to_string()),
+ |_, _| async { panic!("credentials should not be resolved") },
+ )
+ .await
+ .expect("auth should resolve");
+ assert!(matches!(
+ request_auth,
+ BedrockRequestAuth::ApiKey { api_key } if api_key == "token"
+ ));
+
+ // An API key configured in Zed takes precedence over the environment.
+ provider.state.update(cx, |state, _| {
+ state.set_auth(Some(BedrockAuth::ApiKey {
+ api_key: "configured".into(),
+ }));
+ });
+ let http_client = provider.http_client.clone();
+ let request_auth = provider
+ .resolve_request_auth_with(
+ &cx.to_async(),
+ |name| (name == "AWS_BEARER_TOKEN").then(|| "token".to_string()),
+ move |auth, region| resolve_request_auth(http_client, auth, region),
+ )
+ .await
+ .expect("auth should resolve");
+ assert!(matches!(
+ request_auth,
+ BedrockRequestAuth::ApiKey { api_key } if api_key == "configured"
+ ));
+
+ let pending = provider.resolve_request_auth_with(&cx.to_async(), no_environment, |_, _| {
+ futures::future::pending()
+ });
+ cx.executor().advance_clock(CREDENTIALS_LOAD_TIMEOUT);
+ let error = pending.await.err().expect("resolution should time out");
+ assert!(error.to_string().contains("timed out"), "{error}");
+ }
+
#[test]
fn test_thinking_disallowed_sends_explicit_opt_out_only_on_opus_5() {
// Claude Opus 5 runs adaptive thinking by default when the `thinking`
@@ -3822,4 +4086,74 @@ mod tests {
let request = serde_json::to_value(&request).unwrap();
assert!(request.get("context_management").is_none());
}
+
+ struct NoCredentialsProvider;
+
+ impl CredentialsProvider for NoCredentialsProvider {
+ fn read_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _cx: &'a AsyncApp,
+ ) -> Pin<Box<dyn Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>> {
+ Box::pin(async { Ok(None) })
+ }
+
+ fn write_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _username: &'a str,
+ _password: &'a [u8],
+ _cx: &'a AsyncApp,
+ ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
+ Box::pin(async { Ok(()) })
+ }
+
+ fn delete_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _cx: &'a AsyncApp,
+ ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
+ Box::pin(async { Ok(()) })
+ }
+ }
+
+ fn test_provider(cx: &mut gpui::TestAppContext) -> BedrockLanguageModelProvider {
+ cx.executor().allow_parking();
+ cx.update(|cx| {
+ let settings_store = SettingsStore::test(cx);
+ cx.set_global(settings_store);
+ gpui_tokio::init(cx);
+ BedrockLanguageModelProvider::new(
+ http_client::FakeHttpClient::with_404_response(),
+ Arc::new(NoCredentialsProvider),
+ cx,
+ )
+ })
+ }
+
+ fn no_environment(_: &str) -> Option<String> {
+ None
+ }
+
+ async fn resolved_credentials(
+ access_key_id: &str,
+ expires_in: Option<Duration>,
+ ) -> Result<BedrockRequestAuth> {
+ Ok(BedrockRequestAuth::SigV4 {
+ credentials: Credentials::new(
+ access_key_id,
+ "secret",
+ None,
+ expires_in.map(|expires_in| SystemTime::now() + expires_in),
+ "test",
+ ),
+ })
+ }
+
+ fn access_key_id(request_auth: Result<BedrockRequestAuth>) -> String {
+ match request_auth.expect("auth should resolve") {
+ BedrockRequestAuth::SigV4 { credentials } => credentials.access_key_id().to_string(),
+ BedrockRequestAuth::ApiKey { .. } => panic!("expected SigV4 credentials"),
+ }
+ }
}
diff --git a/crates/language_models/src/provider/cloud.rs b/crates/language_models/src/provider/cloud.rs
index a577d8b43a8..f24c6a5ffee 100644
--- a/crates/language_models/src/provider/cloud.rs
+++ b/crates/language_models/src/provider/cloud.rs
@@ -9,14 +9,16 @@ use cloud_api_types::Plan;
use futures::FutureExt;
use futures::StreamExt;
use futures::future::BoxFuture;
-use gpui::{AnyElement, App, AppContext, Context, Entity, Subscription, Task, TaskExt};
+
+use gpui::{AnyElement, App, AppContext, AsyncApp, Context, Entity, Subscription, Task, TaskExt};
use language_model::{
- AuthenticateError, FastModeConfirmation, IconOrSvg, InlineDescription, LanguageModel,
+ AuthenticateError, CompactionResult, FastModeConfirmation, IconOrSvg, InlineDescription,
+ LanguageModel, LanguageModelCompletionError, LanguageModelCompletionStream,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, ProviderSettingsView, ZED_CLOUD_PROVIDER_ID,
+ LanguageModelProviderState, LanguageModelRequest, ProviderSettingsView, ZED_CLOUD_PROVIDER_ID,
ZED_CLOUD_PROVIDER_NAME,
};
-use language_models_cloud::{CloudLlmTokenProvider, CloudModelProvider};
+use language_models_cloud::{CloudLlmTokenProvider, CloudModelProvider, language_model};
use rand::{Rng as _, SeedableRng as _, rngs::StdRng};
use release_channel::AppVersion;
@@ -287,40 +289,84 @@ impl LanguageModelProvider for CloudLanguageModelProvider {
IconOrSvg::Icon(IconName::AiZed)
}
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
let state = self.state.read(cx);
let provider = state.provider.read(cx);
- let model = provider.default_model()?;
- Some(provider.create_model(model))
+ Some(language_model(provider.default_model()?))
}
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
let state = self.state.read(cx);
let provider = state.provider.read(cx);
- let model = provider.default_fast_model()?;
- Some(provider.create_model(model))
+ Some(language_model(provider.default_fast_model()?))
}
- fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn recommended_models(&self, cx: &App) -> Vec<LanguageModel> {
let state = self.state.read(cx);
let provider = state.provider.read(cx);
provider
.recommended_models()
.iter()
- .map(|model| provider.create_model(model))
+ .map(|model| language_model(model))
.collect()
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
let state = self.state.read(cx);
let provider = state.provider.read(cx);
provider
.models()
.iter()
- .map(|model| provider.create_model(model))
+ .map(|model| language_model(model))
.collect()
}
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ cx.update(|cx| {
+ self.state
+ .read(cx)
+ .provider
+ .read(cx)
+ .stream_completion(model, request, cx)
+ })
+ }
+
+ fn count_input_tokens(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
+ cx.update(|cx| {
+ self.state
+ .read(cx)
+ .provider
+ .read(cx)
+ .count_input_tokens(model, request, cx)
+ })
+ }
+
+ fn compact(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ cx.update(|cx| {
+ self.state
+ .read(cx)
+ .provider
+ .read(cx)
+ .compact(model, request, cx)
+ })
+ }
+
fn is_authenticated(&self, cx: &App) -> bool {
let state = self.state.read(cx);
!state.is_signed_out(cx)
diff --git a/crates/language_models/src/provider/copilot_chat.rs b/crates/language_models/src/provider/copilot_chat.rs
index db39f17c68f..218ffddc446 100644
--- a/crates/language_models/src/provider/copilot_chat.rs
+++ b/crates/language_models/src/provider/copilot_chat.rs
@@ -2,20 +2,23 @@ use std::sync::Arc;
use anyhow::Result;
use copilot_chat::{
- CopilotChat, CopilotChatConfiguration, Model as CopilotChatModel, PROVIDER_ID, PROVIDER_NAME,
- create_language_model,
+ CopilotChat, CopilotChatConfiguration, PROVIDER_ID, PROVIDER_NAME, language_model,
};
-use gpui::{App, Entity, Subscription, Task};
+use futures::{FutureExt as _, future::BoxFuture};
+use gpui::{App, AsyncApp, Entity, Subscription, Task};
use language::language_settings::all_language_settings;
use language_model::{
- AuthenticateError, IconOrSvg, LanguageModel, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelProviderState, ProviderSettingsView,
+ AuthenticateError, IconOrSvg, LanguageModel, LanguageModelCompletionError,
+ LanguageModelCompletionStream, LanguageModelProvider, LanguageModelProviderId,
+ LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest, ModelRateLimiters,
+ ProviderSettingsView, unavailable_error,
};
use settings::SettingsStore;
use ui::prelude::*;
pub struct CopilotChatLanguageModelProvider {
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -57,15 +60,32 @@ impl CopilotChatLanguageModelProvider {
}
});
- Self { state }
+ Self {
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
- fn create_language_model(
+ /// The current configuration of `model`, and the Copilot Chat client that
+ /// serves it, if Copilot Chat still offers it.
+ fn config(
&self,
- model: CopilotChatModel,
- copilot_chat: Entity<CopilotChat>,
- ) -> Arc<dyn LanguageModel> {
- create_language_model(model, copilot_chat)
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<(copilot_chat::Model, Entity<CopilotChat>), LanguageModelCompletionError> {
+ let unavailable = || unavailable_error(model);
+ let copilot_chat = CopilotChat::global(cx).ok_or_else(unavailable)?;
+ let config = copilot_chat
+ .read(cx)
+ .models()
+ .and_then(|models| {
+ models
+ .iter()
+ .find(|config| config.id() == model.id.0.as_ref())
+ })
+ .cloned()
+ .ok_or_else(unavailable)?;
+ Ok((config, copilot_chat))
}
}
@@ -90,30 +110,40 @@ impl LanguageModelProvider for CopilotChatLanguageModelProvider {
IconOrSvg::Icon(IconName::Copilot)
}
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
let copilot_chat = CopilotChat::global(cx)?;
- let model = copilot_chat.read(cx).models()?.first()?.clone();
- Some(self.create_language_model(model, copilot_chat))
+ Some(language_model(copilot_chat.read(cx).models()?.first()?))
}
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
// The default model should be Copilot Chat's 'base model', which is likely a relatively fast
// model (e.g. 4o) and a sensible choice when considering premium requests
self.default_model(cx)
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
let Some(copilot_chat) = CopilotChat::global(cx) else {
return Vec::new();
};
let Some(models) = copilot_chat.read(cx).models() else {
return Vec::new();
};
- models
- .iter()
- .cloned()
- .map(|model| self.create_language_model(model, copilot_chat.clone()))
- .collect()
+ models.iter().map(language_model).collect()
+ }
+
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let (config, copilot_chat) = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ copilot_chat::stream_completion(&config, &copilot_chat, &request_limiter, request, cx)
}
fn is_authenticated(&self, cx: &App) -> bool {
diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rs
index 64fe5fbd690..7185f0b8fde 100644
--- a/crates/language_models/src/provider/deepseek.rs
+++ b/crates/language_models/src/provider/deepseek.rs
@@ -9,11 +9,12 @@ use gpui::{App, AppContext, AsyncApp, Context, Entity, SharedString, Task};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelEffortLevel,
- LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse, MessageContent,
- ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var,
+ LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelCompletionStream,
+ LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProvider,
+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
+ LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
+ LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ModelRateLimiters,
+ ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var, unavailable_error,
};
pub use settings::DeepseekAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -46,6 +47,7 @@ pub struct DeepSeekSettings {
pub struct DeepSeekLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -107,17 +109,46 @@ impl DeepSeekLanguageModelProvider {
}
});
- Self { http_client, state }
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
- fn create_language_model(&self, model: deepseek::Model) -> Arc<dyn LanguageModel> {
- Arc::new(DeepSeekLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
+ /// Every model this provider offers, keyed by id: the built-in models,
+ /// with settings entries added or overriding built-in ones.
+ fn deepseek_models(&self, cx: &App) -> IndexMap<String, deepseek::Model> {
+ let mut models = IndexMap::default();
+
+ for model in [deepseek::Model::V4_1Flash, deepseek::Model::V4Pro] {
+ models.insert(model.id().to_string(), model);
+ }
+
+ for available_model in &Self::settings(cx).available_models {
+ models.insert(
+ available_model.name.clone(),
+ deepseek::Model::Custom {
+ name: available_model.name.clone(),
+ display_name: available_model.display_name.clone(),
+ max_tokens: available_model.max_tokens,
+ max_output_tokens: available_model.max_output_tokens,
+ },
+ );
+ }
+
+ models
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<deepseek::Model, LanguageModelCompletionError> {
+ self.deepseek_models(cx)
+ .swap_remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
fn settings(cx: &App) -> &DeepSeekSettings {
@@ -155,35 +186,22 @@ impl LanguageModelProvider for DeepSeekLanguageModelProvider {
IconOrSvg::Icon(IconName::AiDeepSeek)
}
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(deepseek::Model::default()))
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.deepseek_models(cx)
+ .get(deepseek::Model::default().id())
+ .map(language_model)
}
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(deepseek::Model::default_fast()))
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.deepseek_models(cx)
+ .get(deepseek::Model::default_fast().id())
+ .map(language_model)
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let mut models = IndexMap::default();
-
- models.insert("deepseek-flash", deepseek::Model::V4_1Flash);
- models.insert("deepseek-v4-pro", deepseek::Model::V4Pro);
-
- for available_model in &Self::settings(cx).available_models {
- models.insert(
- &available_model.name,
- deepseek::Model::Custom {
- name: available_model.name.clone(),
- display_name: available_model.display_name.clone(),
- max_tokens: available_model.max_tokens,
- max_output_tokens: available_model.max_output_tokens,
- },
- );
- }
-
- models
- .into_values()
- .map(|model| self.create_language_model(model))
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.deepseek_models(cx)
+ .values()
+ .map(language_model)
.collect()
}
@@ -209,20 +227,42 @@ impl LanguageModelProvider for DeepSeekLanguageModelProvider {
self.state
.update(cx, |state, cx| state.set_api_key(api_key, cx))
}
-}
-pub struct DeepSeekLanguageModel {
- id: LanguageModelId,
- model: deepseek::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request = match into_deepseek(request, &config, config.max_output_tokens()) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let stream = self.stream_deepseek_request(request, &request_limiter, cx);
+ let executor = cx.background_executor().clone();
+
+ async move {
+ let mapper = DeepSeekEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
+ }
}
-impl DeepSeekLanguageModel {
- fn stream_completion(
+impl DeepSeekLanguageModelProvider {
+ fn stream_deepseek_request(
&self,
request: deepseek::Request,
+ request_limiter: &RateLimiter,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<BoxStream<'static, Result<deepseek::StreamResponse>>>> {
let http_client = self.http_client.clone();
@@ -235,7 +275,7 @@ impl DeepSeekLanguageModel {
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -256,109 +296,45 @@ impl DeepSeekLanguageModel {
}
}
-impl LanguageModel for DeepSeekLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- true
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_thinking(&self) -> bool {
- matches!(
- self.model,
- deepseek::Model::V4_1Flash | deepseek::Model::V4Pro
+fn language_model(model: &deepseek::Model) -> LanguageModel {
+ let supports_thinking = matches!(model, deepseek::Model::V4_1Flash | deepseek::Model::V4Pro);
+ LanguageModel {
+ supports_tools: true,
+ supports_streaming_tools: true,
+ supports_thinking,
+ supported_effort_levels: if supports_thinking {
+ Arc::new([
+ LanguageModelEffortLevel {
+ name: "Low".into(),
+ value: "low".into(),
+ is_default: false,
+ },
+ LanguageModelEffortLevel {
+ name: "High".into(),
+ value: "high".into(),
+ is_default: true,
+ },
+ LanguageModelEffortLevel {
+ name: "Max".into(),
+ value: "max".into(),
+ is_default: false,
+ },
+ ])
+ } else {
+ Arc::default()
+ },
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_images: model.supports_images(),
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("deepseek/{}", model.id()),
+ model.max_token_count(),
)
}
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- if !self.supports_thinking() {
- return Vec::new();
- }
-
- vec![
- LanguageModelEffortLevel {
- name: "Low".into(),
- value: "low".into(),
- is_default: false,
- },
- LanguageModelEffortLevel {
- name: "High".into(),
- value: "high".into(),
- is_default: true,
- },
- LanguageModelEffortLevel {
- name: "Max".into(),
- value: "max".into(),
- is_default: false,
- },
- ]
- }
-
- fn supports_tool_choice(&self, _choice: LanguageModelToolChoice) -> bool {
- true
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
-
- fn telemetry_id(&self) -> String {
- format!("deepseek/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let request = match into_deepseek(request, &self.model, self.max_output_tokens()) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let stream = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
-
- async move {
- let mapper = DeepSeekEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(stream.await?).boxed(),
- executor,
- ))
- }
- .boxed()
- }
}
pub fn into_deepseek(
diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rs
index 57e00f8d021..e7e69940421 100644
--- a/crates/language_models/src/provider/google.rs
+++ b/crates/language_models/src/provider/google.rs
@@ -8,13 +8,13 @@ use gpui::{App, AppContext, AsyncApp, Context, Entity, SharedString, Task};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyConfiguration, AuthenticateError, EnvVar, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelToolChoice,
+ LanguageModelCompletionStream, LanguageModelToolChoiceSupport, unavailable_error,
};
use language_model::{
GOOGLE_PROVIDER_ID, GOOGLE_PROVIDER_NAME, IconOrSvg, LanguageModel, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- ProviderSettingsView, RateLimiter,
+ LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest, ModelRateLimiters,
+ ProviderSettingsView,
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -50,6 +50,7 @@ pub enum ModelMode {
pub struct GoogleLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -119,73 +120,24 @@ impl GoogleLanguageModelProvider {
}
});
- Self { http_client, state }
- }
-
- fn create_language_model(&self, model: google_ai::Model) -> Arc<dyn LanguageModel> {
- Arc::new(GoogleLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-
- fn settings(cx: &App) -> &GoogleSettings {
- &crate::AllLanguageModelSettings::get_global(cx).google
- }
-
- fn api_url(cx: &App) -> SharedString {
- let api_url = &Self::settings(cx).api_url;
- if api_url.is_empty() {
- google_ai::API_URL.into()
- } else {
- SharedString::new(api_url.as_str())
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
}
}
-}
-
-impl LanguageModelProviderState for GoogleLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for GoogleLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiGoogle)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(google_ai::Model::default()))
- }
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(google_ai::Model::default_fast()))
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: the built-in models,
+ /// with settings entries added or overriding built-in ones.
+ fn google_models(&self, cx: &App) -> BTreeMap<String, google_ai::Model> {
let mut models = BTreeMap::default();
- // Add base models from google_ai::Model::iter()
for model in google_ai::Model::iter() {
if !matches!(model, google_ai::Model::Custom { .. }) {
models.insert(model.id().to_string(), model);
}
}
- // Override with available models from settings
for model in &GoogleLanguageModelProvider::settings(cx).available_models {
models.insert(
model.name.clone(),
@@ -199,53 +151,20 @@ impl LanguageModelProvider for GoogleLanguageModelProvider {
}
models
- .into_values()
- .map(|model| {
- Arc::new(GoogleLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- }) as Arc<dyn LanguageModel>
- })
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.read(cx);
- Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
- state.api_key_state.has_key(),
- state.api_key_state.is_from_env_var(),
- state.api_key_state.env_var_name().clone(),
- "https://aistudio.google.com/app/apikey".into(),
- )))
- }
-
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<google_ai::Model, LanguageModelCompletionError> {
+ self.google_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
-}
-
-pub struct GoogleLanguageModel {
- id: LanguageModelId,
- model: google_ai::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-impl GoogleLanguageModel {
- fn stream_completion(
+ fn stream_google_request(
&self,
request: google_ai::GenerateContentRequest,
cx: &AsyncApp,
@@ -276,102 +195,138 @@ impl GoogleLanguageModel {
}
.boxed()
}
-}
-impl LanguageModel for GoogleLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
+ fn settings(cx: &App) -> &GoogleSettings {
+ &crate::AllLanguageModelSettings::get_global(cx).google
}
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
+ fn api_url(cx: &App) -> SharedString {
+ let api_url = &Self::settings(cx).api_url;
+ if api_url.is_empty() {
+ google_ai::API_URL.into()
+ } else {
+ SharedString::new(api_url.as_str())
+ }
}
+}
- fn provider_id(&self) -> LanguageModelProviderId {
+impl LanguageModelProviderState for GoogleLanguageModelProvider {
+ type ObservableEntity = State;
+
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
+ }
+}
+
+impl LanguageModelProvider for GoogleLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
PROVIDER_ID
}
- fn provider_name(&self) -> LanguageModelProviderName {
+ fn name(&self) -> LanguageModelProviderName {
PROVIDER_NAME
}
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiGoogle)
}
- fn supports_images(&self) -> bool {
- self.model.supports_images()
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.google_models(cx)
+ .get(google_ai::Model::default().id())
+ .map(language_model)
}
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking()
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.google_models(cx)
+ .get(google_ai::Model::default_fast().id())
+ .map(language_model)
}
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- let default_level = self.model.default_thinking_level();
- self.model
- .supported_thinking_levels()
- .iter()
- .map(|level| LanguageModelEffortLevel {
- name: level.name().into(),
- value: level.value().into(),
- is_default: Some(*level) == default_level,
- })
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.google_models(cx)
+ .values()
+ .map(language_model)
.collect()
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto
- | LanguageModelToolChoice::Any
- | LanguageModelToolChoice::None => true,
- }
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn telemetry_id(&self) -> String {
- format!("google/{}", self.model.request_id())
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
+ fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.read(cx);
+ Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
+ state.api_key_state.has_key(),
+ state.api_key_state.is_from_env_var(),
+ state.api_key_state.env_var_name().clone(),
+ "https://aistudio.google.com/app/apikey".into(),
+ )))
}
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
let mut request = request;
if request.max_output_tokens.is_some() {
request.max_output_tokens =
- request.effective_max_output_tokens(self.max_output_tokens());
+ request.effective_max_output_tokens(config.max_output_tokens());
}
- let request = match into_google(
- request,
- self.model.request_id().to_string(),
- self.model.mode(),
- ) {
+ let request = match into_google(request, config.request_id().to_string(), config.mode()) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let request = self.stream_completion(request, cx);
- let future = self.request_limiter.stream(async move {
+ let request = self.stream_google_request(request, cx);
+ let future = request_limiter.stream(async move {
let response = request.await.map_err(LanguageModelCompletionError::from)?;
Ok(GoogleEventMapper::new().map_stream(response))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
}
+
+fn language_model(model: &google_ai::Model) -> LanguageModel {
+ let default_level = model.default_thinking_level();
+ LanguageModel {
+ supports_tools: model.supports_tools(),
+ supports_images: model.supports_images(),
+ supports_thinking: model.supports_thinking(),
+ supported_effort_levels: model
+ .supported_thinking_levels()
+ .iter()
+ .map(|level| LanguageModelEffortLevel {
+ name: level.name().into(),
+ value: level.value().into(),
+ is_default: Some(*level) == default_level,
+ })
+ .collect(),
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("google/{}", model.request_id()),
+ model.max_token_count(),
+ )
+ }
+}
diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rs
index c21494ffc4f..e33fe2d7d5d 100644
--- a/crates/language_models/src/provider/llama_cpp.rs
+++ b/crates/language_models/src/provider/llama_cpp.rs
@@ -2,16 +2,17 @@ use anyhow::Result;
use collections::{HashMap, HashSet};
use credentials_provider::CredentialsProvider;
use fs::Fs;
-use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};
+use futures::{FutureExt, StreamExt, future::BoxFuture};
use gpui::{App, AsyncApp, Context, Entity, Task, TaskExt};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, InlineDescription, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
- LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelId,
+ LanguageModelName, LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
- LanguageModelToolResultContent, MessageContent, ProviderSettingsView, RateLimiter, Role,
- SubPageProviderSettings, env_var,
+ LanguageModelToolChoiceSupport, LanguageModelToolResultContent, MessageContent,
+ ModelRateLimiters, ProviderSettingsView, RateLimiter, Role, SubPageProviderSettings, env_var,
+ unavailable_error,
};
use llama_cpp::{
LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,
@@ -60,10 +61,11 @@ pub struct LlamaCppSettings {
pub struct LlamaCppLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
- /// Live capabilities shared with the agent's models (see [`LiveCapabilities`]).
+ /// Live capabilities shared with [`State`] (see [`LiveCapabilities`]).
capability_cells: CapabilityCells,
- /// Live model-load progress shared with the models (see [`LoadingProgress`]).
+ /// Live model-load progress shared with [`State`] (see [`LoadingProgress`]).
loading_progress: LoadingProgress,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -337,11 +339,10 @@ impl State {
}
}
-/// Capabilities that only become known once a router model loads. `LanguageModel`'s
-/// methods take no `cx`, yet the agent reads them live each turn, so we share
-/// them through a map that re-discovery updates — an already-selected model
-/// picks up real values without re-selection. Image support is available from
-/// `/v1/models` hints, so it stays a field.
+/// Capabilities that only become known once a router model loads. Re-discovery
+/// updates them in a shared map, which each request reads when it is bound, so
+/// a request uses the model's real values as soon as it has loaded. Image
+/// support is available from `/v1/models` hints, so it is not tracked here.
#[derive(Clone, Copy, Debug, PartialEq)]
struct LiveCapabilities {
max_tokens: u64,
@@ -359,7 +360,7 @@ impl LiveCapabilities {
}
}
-/// Live capabilities keyed by model name, shared by the provider and its models.
+/// Live capabilities keyed by model name, shared by the provider and its state.
type CapabilityCells = Arc<RwLock<HashMap<String, LiveCapabilities>>>;
/// Model name → load-status label (e.g. `"Loading weights 42%"`) while a router
@@ -402,8 +403,8 @@ fn compute_effective_models(
models
}
-/// Updates the shared capability map from the effective models, so a model held
-/// by an open conversation observes the new values (it reads the map by name).
+/// Updates the shared capability map from the effective models, so requests
+/// for a model held by an open conversation observe the new values.
fn sync_capability_cells(cells: &CapabilityCells, effective: &HashMap<String, llama_cpp::Model>) {
let mut cells = write_recover(cells);
for model in effective.values() {
@@ -461,6 +462,7 @@ impl LlamaCppLanguageModelProvider {
http_client: http_client.clone(),
capability_cells: capability_cells.clone(),
loading_progress: loading_progress.clone(),
+ request_limiters: ModelRateLimiters::default(),
state: cx.new(|cx| {
cx.observe_global::<SettingsStore>({
let mut last_settings = LlamaCppLanguageModelProvider::settings(cx).clone();
@@ -526,6 +528,103 @@ impl LlamaCppLanguageModelProvider {
fn has_custom_url(cx: &App) -> bool {
Self::settings(cx).api_url != LLAMA_CPP_API_URL
}
+
+ /// Every model this provider offers, keyed by id; see [`compute_effective_models`].
+ fn llama_cpp_models(&self, cx: &App) -> HashMap<String, llama_cpp::Model> {
+ compute_effective_models(
+ &self.state.read(cx).fetched_models,
+ LlamaCppLanguageModelProvider::settings(cx),
+ )
+ }
+
+ /// The model's live capabilities, or `model`'s own if the map lacks it.
+ fn capabilities(&self, model: &llama_cpp::Model) -> LiveCapabilities {
+ read_recover(&self.capability_cells)
+ .get(&model.name)
+ .copied()
+ .unwrap_or_else(|| LiveCapabilities::of(model))
+ }
+
+ fn create_language_model(&self, model: &llama_cpp::Model) -> LanguageModel {
+ let capabilities = self.capabilities(model);
+ let display_name = model.display_name();
+ let loading_label = read_recover(&self.loading_progress)
+ .get(&model.name)
+ .cloned();
+ let name = match loading_label {
+ // Surface load progress in the display name so it shows wherever the
+ // model is named, without provider-agnostic UI changes. The agent
+ // rebuilds the name on `ProviderStateChanged`, which our ticks emit.
+ Some(label) => LanguageModelName::from(format!("{display_name} · {label}")),
+ None => LanguageModelName::from(display_name.to_string()),
+ };
+ LanguageModel {
+ supports_tools: capabilities.supports_tools,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: capabilities.supports_tools,
+ any: capabilities.supports_tools,
+ none: capabilities.supports_tools,
+ },
+ supports_images: model.supports_images,
+ supports_thinking: capabilities.supports_thinking,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.name.clone()),
+ name,
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ telemetry_id_for(&model.name),
+ capabilities.max_tokens,
+ )
+ }
+ }
+
+ /// The current configuration of `model` and its live capabilities, if
+ /// this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<(llama_cpp::Model, LiveCapabilities), LanguageModelCompletionError> {
+ let config = self
+ .llama_cpp_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))?;
+ let capabilities = self.capabilities(&config);
+ Ok((config, capabilities))
+ }
+
+ fn stream_llama_cpp_request(
+ &self,
+ request: llama_cpp::ChatCompletionRequest,
+ request_limiter: &RateLimiter,
+ cx: &AsyncApp,
+ ) -> BoxFuture<
+ 'static,
+ Result<futures::stream::BoxStream<'static, Result<llama_cpp::ResponseStreamEvent>>>,
+ > {
+ let http_client = self.http_client.clone();
+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
+ let api_url = LlamaCppLanguageModelProvider::api_url(cx);
+ let extra_headers = LlamaCppLanguageModelProvider::settings(cx)
+ .custom_headers
+ .clone();
+ (state.api_key_state.key(&api_url), api_url, extra_headers)
+ });
+
+ let future = request_limiter.stream(async move {
+ let stream = stream_chat_completion(
+ http_client.as_ref(),
+ &api_url,
+ api_key.as_deref(),
+ request,
+ &extra_headers,
+ )
+ .await?;
+ Ok(stream)
+ });
+
+ async move { Ok(future.await?.boxed()) }.boxed()
+ }
}
impl LanguageModelProviderState for LlamaCppLanguageModelProvider {
@@ -549,39 +648,25 @@ impl LanguageModelProvider for LlamaCppLanguageModelProvider {
IconOrSvg::Icon(IconName::AiLlamaCpp)
}
- fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, _: &App) -> Option<LanguageModel> {
// No default model: in router mode it could trigger an expensive load of
// an unloaded model on a constrained machine.
None
}
- fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, _: &App) -> Option<LanguageModel> {
// See explanation for default_model.
None
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let settings = LlamaCppLanguageModelProvider::settings(cx);
- let effective = compute_effective_models(&self.state.read(cx).fetched_models, settings);
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ let effective = self.llama_cpp_models(cx);
// Refresh the shared capability map so open conversations pick up settings changes.
sync_capability_cells(&self.capability_cells, &effective);
let mut models = effective
- .into_values()
- .map(|model| {
- Arc::new(LlamaCppLanguageModel {
- id: LanguageModelId::from(model.name.clone()),
- name: model.name.clone(),
- display_name: model.display_name().to_string(),
- fallback_capabilities: LiveCapabilities::of(&model),
- supports_images: model.supports_images,
- capability_cells: self.capability_cells.clone(),
- loading_progress: self.loading_progress.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- state: self.state.clone(),
- }) as Arc<dyn LanguageModel>
- })
+ .values()
+ .map(|model| self.create_language_model(model))
.collect::<Vec<_>>();
models.sort_by_key(|model| model.name());
models
@@ -607,86 +692,34 @@ impl LanguageModelProvider for LlamaCppLanguageModelProvider {
)),
))
}
-}
-
-pub struct LlamaCppLanguageModel {
- id: LanguageModelId,
- /// The model id sent to the server.
- name: String,
- display_name: String,
- /// Live capabilities shared with the provider, read fresh on each access so an
- /// open conversation reflects the model's real values once it has loaded.
- capability_cells: CapabilityCells,
- /// Used when `capability_cells` has no entry (e.g. model removed mid-conversation).
- fallback_capabilities: LiveCapabilities,
- /// Available from `/v1/models` hints, so captured at build time.
- supports_images: bool,
- /// Shared with the provider; this model's load progress, read by `name` so the
- /// selector can show a loading indicator.
- loading_progress: LoadingProgress,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
- state: Entity<State>,
-}
-
-impl LlamaCppLanguageModel {
- /// The model's live capabilities, or the build-time fallback if the map lacks it.
- fn capabilities(&self) -> LiveCapabilities {
- read_recover(&self.capability_cells)
- .get(&self.name)
- .copied()
- .unwrap_or(self.fallback_capabilities)
- }
-
- /// This model's load-status label while loading, read live from the shared map.
- fn loading_label(&self) -> Option<SharedString> {
- read_recover(&self.loading_progress)
- .get(&self.name)
- .cloned()
- }
-
- fn to_llama_cpp_request(
- &self,
- request: LanguageModelRequest,
- ) -> Result<llama_cpp::ChatCompletionRequest> {
- build_llama_cpp_request(
- &self.name,
- self.supports_images,
- self.capabilities(),
- request,
- )
- }
fn stream_completion(
&self,
- request: llama_cpp::ChatCompletionRequest,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<futures::stream::BoxStream<'static, Result<llama_cpp::ResponseStreamEvent>>>,
- > {
- let http_client = self.http_client.clone();
- let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
- let api_url = LlamaCppLanguageModelProvider::api_url(cx);
- let extra_headers = LlamaCppLanguageModelProvider::settings(cx)
- .custom_headers
- .clone();
- (state.api_key_state.key(&api_url), api_url, extra_headers)
- });
-
- let future = self.request_limiter.stream(async move {
- let stream = stream_chat_completion(
- http_client.as_ref(),
- &api_url,
- api_key.as_deref(),
- request,
- &extra_headers,
- )
- .await?;
- Ok(stream)
- });
-
- async move { Ok(future.await?.boxed()) }.boxed()
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let (config, capabilities) = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request = match build_llama_cpp_request(
+ &config.name,
+ config.supports_images,
+ capabilities,
+ request,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_llama_cpp_request(request, &request_limiter, cx);
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(mapper.map_stream(completions.await?).boxed())
+ }
+ .boxed()
}
}
@@ -871,82 +904,6 @@ fn build_llama_cpp_request(
})
}
-impl LanguageModel for LlamaCppLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- match self.loading_label() {
- // Surface load progress in the display name so it shows wherever the
- // model is named, without provider-agnostic UI changes. The agent
- // rebuilds the name on `ProviderStateChanged`, which our ticks emit.
- Some(label) => LanguageModelName::from(format!("{} · {}", self.display_name, label)),
- None => LanguageModelName::from(self.display_name.clone()),
- }
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.capabilities().supports_tools
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- self.supports_tools()
- && match choice {
- LanguageModelToolChoice::Auto => true,
- LanguageModelToolChoice::Any => true,
- LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_images(&self) -> bool {
- self.supports_images
- }
-
- fn supports_thinking(&self) -> bool {
- self.capabilities().supports_thinking
- }
-
- fn telemetry_id(&self) -> String {
- telemetry_id_for(&self.name)
- }
-
- fn max_token_count(&self) -> u64 {
- self.capabilities().max_tokens
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let request = match self.to_llama_cpp_request(request) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_completion(request, cx);
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
- }
- .boxed()
- }
-}
-
fn add_message_content_part(
new_part: llama_cpp::MessagePart,
role: Role,
@@ -1837,6 +1794,80 @@ mod tests {
}
}
+ #[gpui::test]
+ fn config_uses_live_capabilities_for_a_stable_model_id(cx: &mut TestAppContext) {
+ cx.update(|cx| {
+ let settings_store = SettingsStore::test(cx);
+ cx.set_global(settings_store);
+ });
+ let provider = cx.update(|cx| {
+ LlamaCppLanguageModelProvider::new(
+ FakeHttpClient::with_404_response(),
+ Arc::new(FakeCredentialsProvider {
+ api_key: Vec::new(),
+ }),
+ cx,
+ )
+ });
+ cx.run_until_parked();
+
+ let cold = llama_cpp::Model::new(
+ "m",
+ Some("m"),
+ Some(ASSUMED_UNLOADED_CONTEXT),
+ true,
+ false,
+ false,
+ );
+ provider
+ .state
+ .update(cx, |state, _| state.fetched_models = vec![cold]);
+ write_recover(&provider.loading_progress)
+ .insert("m".to_string(), SharedString::from("Loading 42%"));
+ let selected = cx.update(|cx| provider.provided_models(cx)).remove(0);
+ assert_eq!(selected.id.0.as_ref(), "m");
+ assert_eq!(selected.name.0.as_ref(), "m · Loading 42%");
+ assert!(!selected.supports_thinking);
+
+ // The model finishes loading and re-discovery reports its real values.
+ let loaded = llama_cpp::Model::new("m", Some("m"), Some(262_144), false, false, true);
+ provider.state.update(cx, |state, cx| {
+ state.fetched_models = vec![loaded];
+ sync_capability_cells(
+ &state.capability_cells,
+ &compute_effective_models(
+ &state.fetched_models,
+ LlamaCppLanguageModelProvider::settings(cx),
+ ),
+ );
+ });
+ write_recover(&provider.loading_progress).remove("m");
+
+ let (config, capabilities) = cx
+ .update(|cx| provider.config(&selected, cx))
+ .expect("model is still offered");
+ assert_eq!(config.name, "m");
+ assert_eq!(
+ capabilities,
+ LiveCapabilities {
+ max_tokens: 262_144,
+ supports_tools: false,
+ supports_thinking: true,
+ }
+ );
+ let refreshed = cx.update(|cx| provider.provided_models(cx)).remove(0);
+ assert_eq!(refreshed.id, selected.id);
+ assert_eq!(refreshed.name.0.as_ref(), "m");
+
+ provider
+ .state
+ .update(cx, |state, _| state.fetched_models.clear());
+ assert!(matches!(
+ cx.update(|cx| provider.config(&selected, cx)),
+ Err(LanguageModelCompletionError::ModelUnavailable { .. })
+ ));
+ }
+
#[gpui::test]
async fn authenticate_fetches_models_after_loading_api_key(cx: &mut TestAppContext) {
cx.update(|cx| {
diff --git a/crates/language_models/src/provider/lmstudio.rs b/crates/language_models/src/provider/lmstudio.rs
index 310932cfc45..f7ee4ab6ffd 100644
--- a/crates/language_models/src/provider/lmstudio.rs
+++ b/crates/language_models/src/provider/lmstudio.rs
@@ -1,18 +1,19 @@
use anyhow::{Result, anyhow};
use credentials_provider::CredentialsProvider;
use fs::Fs;
-use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};
+use futures::{FutureExt, StreamExt, future::BoxFuture};
use gpui::{App, AsyncApp, Context, Entity, Subscription, Task, TaskExt};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelToolChoice, LanguageModelToolResultContent,
- MessageContent, env_var,
+ LanguageModelCompletionStream, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
+ LanguageModelToolResultContent, MessageContent, env_var, unavailable_error,
};
use language_model::{
InlineDescription, LanguageModelId, LanguageModelName, LanguageModelProvider,
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
- LanguageModelRequest, ProviderSettingsView, RateLimiter, Role, SubPageProviderSettings,
+ LanguageModelRequest, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role,
+ SubPageProviderSettings,
};
use lmstudio::{LMSTUDIO_API_URL, ModelType, get_models};
@@ -46,6 +47,7 @@ pub struct LmStudioSettings {
pub struct LmStudioLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -208,6 +210,7 @@ impl LmStudioLanguageModelProvider {
_subscription: subscription,
}
}),
+ request_limiters: ModelRateLimiters::default(),
};
this.state
.update(cx, |state, cx| state.restart_fetch_models_task(cx));
@@ -224,6 +227,46 @@ impl LmStudioLanguageModelProvider {
fn has_custom_url(cx: &App) -> bool {
Self::api_url(cx) != LMSTUDIO_API_URL
}
+
+ /// Every model this provider offers, keyed by id: the models LM Studio
+ /// reports, with settings entries added or overriding reported ones.
+ fn lmstudio_models(&self, cx: &App) -> BTreeMap<String, lmstudio::Model> {
+ let mut models: BTreeMap<String, lmstudio::Model> = BTreeMap::default();
+
+ for model in self.state.read(cx).available_models.iter() {
+ models.insert(model.name.clone(), model.clone());
+ }
+
+ for model in AllLanguageModelSettings::get_global(cx)
+ .lmstudio
+ .available_models
+ .iter()
+ {
+ models.insert(
+ model.name.clone(),
+ lmstudio::Model {
+ name: model.name.clone(),
+ display_name: model.display_name.clone(),
+ max_tokens: model.max_tokens,
+ supports_tool_calls: model.supports_tool_calls,
+ supports_images: model.supports_images,
+ },
+ );
+ }
+
+ models
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<lmstudio::Model, LanguageModelCompletionError> {
+ self.lmstudio_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
+ }
}
impl LanguageModelProviderState for LmStudioLanguageModelProvider {
@@ -247,55 +290,22 @@ impl LanguageModelProvider for LmStudioLanguageModelProvider {
IconOrSvg::Icon(IconName::AiLmStudio)
}
- fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, _: &App) -> Option<LanguageModel> {
// We shouldn't try to select default model, because it might lead to a load call for an unloaded model.
// In a constrained environment where user might not have enough resources it'll be a bad UX to select something
// to load by default.
None
}
- fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, _: &App) -> Option<LanguageModel> {
// See explanation for default_model.
None
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let mut models: BTreeMap<String, lmstudio::Model> = BTreeMap::default();
-
- // Add models from the LM Studio API
- for model in self.state.read(cx).available_models.iter() {
- models.insert(model.name.clone(), model.clone());
- }
-
- // Override with available models from settings
- for model in AllLanguageModelSettings::get_global(cx)
- .lmstudio
- .available_models
- .iter()
- {
- models.insert(
- model.name.clone(),
- lmstudio::Model {
- name: model.name.clone(),
- display_name: model.display_name.clone(),
- max_tokens: model.max_tokens,
- supports_tool_calls: model.supports_tool_calls,
- supports_images: model.supports_images,
- },
- );
- }
-
- models
- .into_values()
- .map(|model| {
- Arc::new(LmStudioLanguageModel {
- id: LanguageModelId::from(model.name.clone()),
- model,
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- state: self.state.clone(),
- }) as Arc<dyn LanguageModel>
- })
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.lmstudio_models(cx)
+ .values()
+ .map(language_model)
.collect()
}
@@ -319,157 +329,175 @@ impl LanguageModelProvider for LmStudioLanguageModelProvider {
)),
))
}
-}
-
-pub struct LmStudioLanguageModel {
- id: LanguageModelId,
- model: lmstudio::Model,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
- state: Entity<State>,
-}
-impl LmStudioLanguageModel {
- fn to_lmstudio_request(
+ fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
- ) -> Result<lmstudio::ChatCompletionRequest> {
- if request.contains_custom_tool_input() {
- anyhow::bail!("LM Studio does not support custom tools");
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request = match to_lmstudio_request(&config, request) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_lmstudio_request(request, &request_limiter, cx);
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(mapper.map_stream(completions.await?).boxed())
}
+ .boxed()
+ }
+}
- let mut messages = Vec::new();
-
- for message in request.messages {
- for content in message.content {
- match content {
- MessageContent::Text(text) => add_message_content_part(
- lmstudio::MessagePart::Text { text },
+fn to_lmstudio_request(
+ config: &lmstudio::Model,
+ request: LanguageModelRequest,
+) -> Result<lmstudio::ChatCompletionRequest> {
+ if request.contains_custom_tool_input() {
+ anyhow::bail!("LM Studio does not support custom tools");
+ }
+
+ let mut messages = Vec::new();
+
+ for message in request.messages {
+ for content in message.content {
+ match content {
+ MessageContent::Text(text) => add_message_content_part(
+ lmstudio::MessagePart::Text { text },
+ message.role,
+ &mut messages,
+ ),
+ MessageContent::Thinking { .. } => {}
+ MessageContent::RedactedThinking(_) => {}
+ MessageContent::Compaction(_) => {}
+ MessageContent::Image(image) => {
+ add_message_content_part(
+ lmstudio::MessagePart::Image {
+ image_url: lmstudio::ImageUrl {
+ url: image.to_base64_url(),
+ detail: None,
+ },
+ },
message.role,
&mut messages,
- ),
- MessageContent::Thinking { .. } => {}
- MessageContent::RedactedThinking(_) => {}
- MessageContent::Compaction(_) => {}
- MessageContent::Image(image) => {
- add_message_content_part(
- lmstudio::MessagePart::Image {
- image_url: lmstudio::ImageUrl {
- url: image.to_base64_url(),
- detail: None,
- },
- },
- message.role,
- &mut messages,
- );
- }
- MessageContent::ToolUse(tool_use) => {
- let input = tool_use.input.as_json().ok_or_else(|| {
- anyhow!("LM Studio does not support custom tool calls")
- })?;
- let tool_call = lmstudio::ToolCall {
- id: tool_use.id.to_string(),
- content: lmstudio::ToolCallContent::Function {
- function: lmstudio::FunctionContent {
- name: tool_use.name.to_string(),
- arguments: serde_json::to_string(input).unwrap_or_default(),
- },
+ );
+ }
+ MessageContent::ToolUse(tool_use) => {
+ let input = tool_use
+ .input
+ .as_json()
+ .ok_or_else(|| anyhow!("LM Studio does not support custom tool calls"))?;
+ let tool_call = lmstudio::ToolCall {
+ id: tool_use.id.to_string(),
+ content: lmstudio::ToolCallContent::Function {
+ function: lmstudio::FunctionContent {
+ name: tool_use.name.to_string(),
+ arguments: serde_json::to_string(input).unwrap_or_default(),
},
- };
+ },
+ };
- if let Some(lmstudio::ChatMessage::Assistant { tool_calls, .. }) =
- messages.last_mut()
- {
- tool_calls.push(tool_call);
- } else {
- messages.push(lmstudio::ChatMessage::Assistant {
- content: None,
- tool_calls: vec![tool_call],
- });
- }
+ if let Some(lmstudio::ChatMessage::Assistant { tool_calls, .. }) =
+ messages.last_mut()
+ {
+ tool_calls.push(tool_call);
+ } else {
+ messages.push(lmstudio::ChatMessage::Assistant {
+ content: None,
+ tool_calls: vec![tool_call],
+ });
}
- MessageContent::ToolResult(tool_result) => {
- let content: Vec<lmstudio::MessagePart> = tool_result
- .content
- .iter()
- .map(|part| match part {
- LanguageModelToolResultContent::Text(text) => {
- lmstudio::MessagePart::Text {
- text: text.to_string(),
- }
+ }
+ MessageContent::ToolResult(tool_result) => {
+ let content: Vec<lmstudio::MessagePart> = tool_result
+ .content
+ .iter()
+ .map(|part| match part {
+ LanguageModelToolResultContent::Text(text) => {
+ lmstudio::MessagePart::Text {
+ text: text.to_string(),
}
- LanguageModelToolResultContent::Image(image) => {
- lmstudio::MessagePart::Image {
- image_url: lmstudio::ImageUrl {
- url: image.to_base64_url(),
- detail: None,
- },
- }
+ }
+ LanguageModelToolResultContent::Image(image) => {
+ lmstudio::MessagePart::Image {
+ image_url: lmstudio::ImageUrl {
+ url: image.to_base64_url(),
+ detail: None,
+ },
}
- })
- .collect();
+ }
+ })
+ .collect();
- messages.push(lmstudio::ChatMessage::Tool {
- content: content.into(),
- tool_call_id: tool_result.tool_use_id.to_string(),
- });
- }
+ messages.push(lmstudio::ChatMessage::Tool {
+ content: content.into(),
+ tool_call_id: tool_result.tool_use_id.to_string(),
+ });
}
}
}
+ }
- Ok(lmstudio::ChatCompletionRequest {
- model: self.model.name.clone(),
- messages,
- stream: true,
- stream_options: Some(lmstudio::StreamOptions {
- include_usage: true,
- }),
- max_tokens: Some(
- request
- .max_output_tokens
- .map(i32::try_from)
- .transpose()?
- .unwrap_or(-1),
- ),
- stop: Some(request.stop),
- // In LM Studio you can configure specific settings you'd like to use for your model.
- // For example Qwen3 is recommended to be used with 0.7 temperature.
- // It would be a bad UX to silently override these settings from Zed, so we pass no temperature as a default.
- temperature: request.temperature.or(None),
- tools: request
- .tools
- .into_iter()
- .map(|tool| {
- let input_schema = match tool.input {
- language_model::LanguageModelRequestToolInput::Function {
- input_schema,
- ..
- } => input_schema,
- language_model::LanguageModelRequestToolInput::Custom { .. } => {
- return Err(anyhow::anyhow!("LM Studio does not support custom tools"));
- }
- };
- Ok(lmstudio::ToolDefinition::Function {
- function: lmstudio::FunctionDefinition {
- name: tool.name,
- description: Some(tool.description),
- parameters: Some(input_schema),
- },
- })
+ Ok(lmstudio::ChatCompletionRequest {
+ model: config.name.clone(),
+ messages,
+ stream: true,
+ stream_options: Some(lmstudio::StreamOptions {
+ include_usage: true,
+ }),
+ max_tokens: Some(
+ request
+ .max_output_tokens
+ .map(i32::try_from)
+ .transpose()?
+ .unwrap_or(-1),
+ ),
+ stop: Some(request.stop),
+ // In LM Studio you can configure specific settings you'd like to use for your model.
+ // For example Qwen3 is recommended to be used with 0.7 temperature.
+ // It would be a bad UX to silently override these settings from Zed, so we pass no temperature as a default.
+ temperature: request.temperature.or(None),
+ tools: request
+ .tools
+ .into_iter()
+ .map(|tool| {
+ let input_schema = match tool.input {
+ language_model::LanguageModelRequestToolInput::Function {
+ input_schema,
+ ..
+ } => input_schema,
+ language_model::LanguageModelRequestToolInput::Custom { .. } => {
+ return Err(anyhow::anyhow!("LM Studio does not support custom tools"));
+ }
+ };
+ Ok(lmstudio::ToolDefinition::Function {
+ function: lmstudio::FunctionDefinition {
+ name: tool.name,
+ description: Some(tool.description),
+ parameters: Some(input_schema),
+ },
})
- .collect::<Result<_>>()?,
- tool_choice: request.tool_choice.map(|choice| match choice {
- LanguageModelToolChoice::Auto => lmstudio::ToolChoice::Auto,
- LanguageModelToolChoice::Any => lmstudio::ToolChoice::Required,
- LanguageModelToolChoice::None => lmstudio::ToolChoice::None,
- }),
- })
- }
+ })
+ .collect::<Result<_>>()?,
+ tool_choice: request.tool_choice.map(|choice| match choice {
+ LanguageModelToolChoice::Auto => lmstudio::ToolChoice::Auto,
+ LanguageModelToolChoice::Any => lmstudio::ToolChoice::Required,
+ LanguageModelToolChoice::None => lmstudio::ToolChoice::None,
+ }),
+ })
+}
- fn stream_completion(
+impl LmStudioLanguageModelProvider {
+ fn stream_lmstudio_request(
&self,
request: lmstudio::ChatCompletionRequest,
+ request_limiter: &RateLimiter,
cx: &AsyncApp,
) -> BoxFuture<
'static,
@@ -485,7 +513,7 @@ impl LmStudioLanguageModel {
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let stream = lmstudio::stream_chat_completion(
http_client.as_ref(),
&api_url,
@@ -501,69 +529,24 @@ impl LmStudioLanguageModel {
}
}
-impl LanguageModel for LmStudioLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tool_calls()
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- self.supports_tools()
- && match choice {
- LanguageModelToolChoice::Auto => true,
- LanguageModelToolChoice::Any => true,
- LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images
- }
-
- fn telemetry_id(&self) -> String {
- format!("lmstudio/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let request = match self.to_lmstudio_request(request) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_completion(request, cx);
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
- }
- .boxed()
+fn language_model(model: &lmstudio::Model) -> LanguageModel {
+ let supports_tools = model.supports_tool_calls();
+ LanguageModel {
+ supports_tools,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: supports_tools,
+ any: supports_tools,
+ none: supports_tools,
+ },
+ supports_images: model.supports_images,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.name.clone()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("lmstudio/{}", model.id()),
+ model.max_token_count(),
+ )
}
}
diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs
index 3884896cb8b..bf105035b2b 100644
--- a/crates/language_models/src/provider/mistral.rs
+++ b/crates/language_models/src/provider/mistral.rs
@@ -2,16 +2,17 @@ use anyhow::{Result, anyhow};
use collections::{BTreeMap, HashMap};
use credentials_provider::CredentialsProvider;
-use futures::{FutureExt, Stream, StreamExt, future::BoxFuture, stream::BoxStream};
+use futures::{FutureExt, Stream, StreamExt, future::BoxFuture};
use gpui::{App, AppContext, AsyncApp, Context, Entity, Global, SharedString, Task};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
- LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
- LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ProviderSettingsView,
- RateLimiter, Role, StopReason, TokenUsage, env_var,
+ LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelCompletionStream,
+ LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
+ LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
+ LanguageModelToolChoice, LanguageModelToolChoiceSupport, LanguageModelToolResultContent,
+ LanguageModelToolUse, MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter,
+ Role, StopReason, TokenUsage, env_var, unavailable_error,
};
pub use mistral::{MISTRAL_API_URL, StreamResponse};
pub use settings::MistralAvailableModel as AvailableModel;
@@ -40,6 +41,7 @@ pub struct MistralSettings {
pub struct MistralLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
pub state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -113,75 +115,26 @@ impl MistralLanguageModelProvider {
}
});
- let this = Arc::new(Self { http_client, state });
+ let this = Arc::new(Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ });
cx.set_global(GlobalMistralLanguageModelProvider(this));
cx.global::<GlobalMistralLanguageModelProvider>().0.clone()
}
- fn create_language_model(&self, model: mistral::Model) -> Arc<dyn LanguageModel> {
- Arc::new(MistralLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-
- fn settings(cx: &App) -> &MistralSettings {
- &crate::AllLanguageModelSettings::get_global(cx).mistral
- }
-
- pub fn api_url(cx: &App) -> SharedString {
- let api_url = &Self::settings(cx).api_url;
- if api_url.is_empty() {
- mistral::MISTRAL_API_URL.into()
- } else {
- SharedString::new(api_url.as_str())
- }
- }
-}
-
-impl LanguageModelProviderState for MistralLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for MistralLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiMistral)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(mistral::Model::default()))
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(mistral::Model::default_fast()))
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: the built-in models,
+ /// with settings entries added or overriding built-in ones.
+ fn mistral_models(&self, cx: &App) -> BTreeMap<String, mistral::Model> {
let mut models = BTreeMap::default();
- // Add base models from mistral::Model::iter()
for model in mistral::Model::iter() {
if !matches!(model, mistral::Model::Custom { .. }) {
models.insert(model.id().to_string(), model);
}
}
- // Override with available models from settings
for model in &Self::settings(cx).available_models {
models.insert(
model.name.clone(),
@@ -199,54 +152,22 @@ impl LanguageModelProvider for MistralLanguageModelProvider {
}
models
- .into_values()
- .map(|model| {
- Arc::new(MistralLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- }) as Arc<dyn LanguageModel>
- })
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.read(cx);
- Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
- state.api_key_state.has_key(),
- state.api_key_state.is_from_env_var(),
- state.api_key_state.env_var_name().clone(),
- "https://console.mistral.ai/api-keys".into(),
- )))
}
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<mistral::Model, LanguageModelCompletionError> {
+ self.mistral_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
-}
-
-pub struct MistralLanguageModel {
- id: LanguageModelId,
- model: mistral::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-impl MistralLanguageModel {
- fn stream_completion(
+ fn stream_mistral_request(
&self,
+ request_limiter: &RateLimiter,
request: mistral::Request,
affinity: Option<String>,
cx: &AsyncApp,
@@ -264,7 +185,7 @@ impl MistralLanguageModel {
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -284,78 +205,102 @@ impl MistralLanguageModel {
async move { Ok(future.await?.boxed()) }.boxed()
}
-}
-impl LanguageModel for MistralLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
+ fn settings(cx: &App) -> &MistralSettings {
+ &crate::AllLanguageModelSettings::get_global(cx).mistral
}
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
+ pub fn api_url(cx: &App) -> SharedString {
+ let api_url = &Self::settings(cx).api_url;
+ if api_url.is_empty() {
+ mistral::MISTRAL_API_URL.into()
+ } else {
+ SharedString::new(api_url.as_str())
+ }
}
+}
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
+impl LanguageModelProviderState for MistralLanguageModelProvider {
+ type ObservableEntity = State;
+
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
+impl LanguageModelProvider for MistralLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
+ PROVIDER_ID
}
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
+ fn name(&self) -> LanguageModelProviderName {
+ PROVIDER_NAME
}
- fn supports_streaming_tools(&self) -> bool {
- true
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiMistral)
}
- fn supports_tool_choice(&self, _choice: LanguageModelToolChoice) -> bool {
- self.model.supports_tools()
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.mistral_models(cx)
+ .get(mistral::Model::default().id())
+ .map(language_model)
}
- fn supports_images(&self) -> bool {
- self.model.supports_images()
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.mistral_models(cx)
+ .get(mistral::Model::default_fast().id())
+ .map(language_model)
}
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking()
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.mistral_models(cx)
+ .values()
+ .map(language_model)
+ .collect()
}
- fn supports_disabling_thinking(&self) -> bool {
- self.model.supports_disabling_thinking()
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn telemetry_id(&self) -> String {
- format!("mistral/{}", self.model.id())
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
+ fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.read(cx);
+ Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
+ state.api_key_state.has_key(),
+ state.api_key_state.is_from_env_var(),
+ state.api_key_state.env_var_name().clone(),
+ "https://console.mistral.ai/api-keys".into(),
+ )))
}
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
let (request, affinity) =
- match into_mistral(request, self.model.clone(), self.max_output_tokens()) {
+ match into_mistral(request, config.clone(), config.max_output_tokens()) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let stream = self.stream_completion(request, affinity, cx);
+ let stream = self.stream_mistral_request(&request_limiter, request, affinity, cx);
let executor = cx.background_executor().clone();
async move {
@@ -370,6 +315,31 @@ impl LanguageModel for MistralLanguageModel {
}
}
+fn language_model(model: &mistral::Model) -> LanguageModel {
+ let supports_tools = model.supports_tools();
+ LanguageModel {
+ supports_tools,
+ supports_streaming_tools: true,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: supports_tools,
+ any: supports_tools,
+ none: supports_tools,
+ },
+ supports_images: model.supports_images(),
+ supports_thinking: model.supports_thinking(),
+ supports_disabling_thinking: model.supports_disabling_thinking(),
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("mistral/{}", model.id()),
+ model.max_token_count(),
+ )
+ }
+}
+
pub fn into_mistral(
request: LanguageModelRequest,
model: mistral::Model,
diff --git a/crates/language_models/src/provider/ollama.rs b/crates/language_models/src/provider/ollama.rs
index 486111cadca..d17edc278de 100644
--- a/crates/language_models/src/provider/ollama.rs
+++ b/crates/language_models/src/provider/ollama.rs
@@ -2,18 +2,18 @@ use anyhow::{Result, anyhow};
use collections::HashMap;
use credentials_provider::CredentialsProvider;
use fs::Fs;
-use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};
+use futures::{FutureExt, StreamExt, future::BoxFuture};
use futures::{Stream, TryFutureExt, stream};
use gpui::{App, AsyncApp, Context, Entity, Task, TaskExt};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyState, AuthenticateError, DisabledReason, EnvVar, IconOrSvg, InlineDescription,
- LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId,
- LanguageModelName, LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelRequestTool,
- LanguageModelToolChoice, LanguageModelToolUse, LanguageModelToolUseId, MessageContent,
- ProviderSettingsView, RateLimiter, Role, StopReason, SubPageProviderSettings, TokenUsage,
- env_var,
+ LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
+ LanguageModelCompletionStream, LanguageModelId, LanguageModelName, LanguageModelProvider,
+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
+ LanguageModelRequest, LanguageModelRequestTool, LanguageModelToolUse, LanguageModelToolUseId,
+ MessageContent, ModelRateLimiters, ProviderSettingsView, Role, StopReason,
+ SubPageProviderSettings, TokenUsage, env_var, unavailable_error,
};
use menu;
use ollama::{
@@ -54,6 +54,7 @@ pub struct OllamaSettings {
pub struct OllamaLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -234,67 +235,18 @@ impl OllamaLanguageModelProvider {
credentials_provider,
}
}),
+ request_limiters: ModelRateLimiters::default(),
};
this
}
- fn settings(cx: &App) -> &OllamaSettings {
- &AllLanguageModelSettings::get_global(cx).ollama
- }
-
- fn api_url(cx: &App) -> SharedString {
- let api_url = &Self::settings(cx).api_url;
- if api_url.is_empty() {
- OLLAMA_API_URL.into()
- } else {
- SharedString::new(api_url.as_str())
- }
- }
-
- fn has_custom_url(cx: &App) -> bool {
- Self::settings(cx).api_url != OLLAMA_API_URL
- }
-}
-
-impl LanguageModelProviderState for OllamaLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for OllamaLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiOllama)
- }
-
- fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
- // We shouldn't try to select default model, because it might lead to a load call for an unloaded model.
- // In a constrained environment where user might not have enough resources it'll be a bad UX to select something
- // to load by default.
- None
- }
-
- fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
- // See explanation for default_model.
- None
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: the fetched list when
+ /// auto-discovery is on, with settings entries merged over it.
+ fn ollama_models(&self, cx: &App) -> HashMap<String, ollama::Model> {
let mut models: HashMap<String, ollama::Model> = HashMap::default();
let settings = OllamaLanguageModelProvider::settings(cx);
if settings.auto_discover {
- // Add models from the Ollama API
for model in self.state.read(cx).fetched_models.iter() {
let mut model = model.clone();
if let Some(context_window) = settings.context_window {
@@ -304,68 +256,35 @@ impl LanguageModelProvider for OllamaLanguageModelProvider {
}
}
- // Override with available models from settings
merge_settings_into_models(
&mut models,
&settings.available_models,
settings.context_window,
);
- let mut models = models
- .into_values()
- .map(|model| {
- Arc::new(OllamaLanguageModel {
- id: LanguageModelId::from(model.name.clone()),
- disabled: model.disabled.as_ref().map(|d| DisabledReason::new(d)),
- model,
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- state: self.state.clone(),
- }) as Arc<dyn LanguageModel>
- })
- .collect::<Vec<_>>();
- models.sort_by_key(|model| model.name());
models
}
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.clone();
- Some(ProviderSettingsView::SubPage(
- SubPageProviderSettings::new(move |window, cx| {
- cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))
- .into()
- })
- .description(InlineDescription::Text(
- "Run local models on your machine with Ollama.".into(),
- )),
- ))
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<ollama::Model, LanguageModelCompletionError> {
+ self.ollama_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
-}
-
-pub struct OllamaLanguageModel {
- id: LanguageModelId,
- model: ollama::Model,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
- state: Entity<State>,
- disabled: Option<DisabledReason>,
-}
-impl OllamaLanguageModel {
- fn to_ollama_request(&self, request: LanguageModelRequest) -> Result<ChatRequest> {
+ fn to_ollama_request(
+ config: &ollama::Model,
+ request: LanguageModelRequest,
+ ) -> Result<ChatRequest> {
if request.contains_custom_tool_input() {
anyhow::bail!("Ollama does not support custom tools");
}
- let supports_vision = self.model.supports_vision.unwrap_or(false);
+ let supports_vision = config.supports_vision.unwrap_or(false);
let mut messages = Vec::with_capacity(request.messages.len());
@@ -468,12 +387,12 @@ impl OllamaLanguageModel {
}
}
Ok(ChatRequest {
- model: self.model.name.clone(),
+ model: config.name.clone(),
messages,
- keep_alive: self.model.keep_alive.clone().unwrap_or_default(),
+ keep_alive: config.keep_alive.clone().unwrap_or_default(),
stream: true,
options: Some(ChatOptions {
- num_ctx: Some(self.model.max_tokens),
+ num_ctx: Some(config.max_tokens),
num_predict: request.max_output_tokens.map(isize::try_from).transpose()?,
// Only send stop tokens if explicitly provided. When empty/None,
// Ollama will use the model's default stop tokens from its Modelfile.
@@ -486,11 +405,10 @@ impl OllamaLanguageModel {
temperature: request.temperature.or(Some(1.0)),
..Default::default()
}),
- think: self
- .model
+ think: config
.supports_thinking
.map(|supports_thinking| supports_thinking && request.thinking_allowed),
- tools: if self.model.supports_tools.unwrap_or(false) {
+ tools: if config.supports_tools.unwrap_or(false) {
request
.tools
.into_iter()
@@ -501,69 +419,102 @@ impl OllamaLanguageModel {
},
})
}
-}
-impl LanguageModel for OllamaLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
+ fn settings(cx: &App) -> &OllamaSettings {
+ &AllLanguageModelSettings::get_global(cx).ollama
+ }
+
+ fn api_url(cx: &App) -> SharedString {
+ let api_url = &Self::settings(cx).api_url;
+ if api_url.is_empty() {
+ OLLAMA_API_URL.into()
+ } else {
+ SharedString::new(api_url.as_str())
+ }
+ }
+
+ fn has_custom_url(cx: &App) -> bool {
+ Self::settings(cx).api_url != OLLAMA_API_URL
}
+}
+
+impl LanguageModelProviderState for OllamaLanguageModelProvider {
+ type ObservableEntity = State;
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn provider_id(&self) -> LanguageModelProviderId {
+impl LanguageModelProvider for OllamaLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
PROVIDER_ID
}
- fn provider_name(&self) -> LanguageModelProviderName {
+ fn name(&self) -> LanguageModelProviderName {
PROVIDER_NAME
}
- fn supports_tools(&self) -> bool {
- self.model.supports_tools.unwrap_or(false)
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiOllama)
}
- fn supports_images(&self) -> bool {
- self.model.supports_vision.unwrap_or(false)
+ fn default_model(&self, _: &App) -> Option<LanguageModel> {
+ // We shouldn't try to select default model, because it might lead to a load call for an unloaded model.
+ // In a constrained environment where user might not have enough resources it'll be a bad UX to select something
+ // to load by default.
+ None
}
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking.unwrap_or(false)
+ fn default_fast_model(&self, _: &App) -> Option<LanguageModel> {
+ // See explanation for default_model.
+ None
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto => false,
- LanguageModelToolChoice::Any => false,
- LanguageModelToolChoice::None => false,
- }
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ let mut models = self
+ .ollama_models(cx)
+ .values()
+ .map(language_model)
+ .collect::<Vec<_>>();
+ models.sort_by_key(|model| model.name());
+ models
}
- fn telemetry_id(&self) -> String {
- format!("ollama/{}", self.model.id())
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn is_disabled(&self) -> Option<DisabledReason> {
- self.disabled.clone()
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
+ fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.clone();
+ Some(ProviderSettingsView::SubPage(
+ SubPageProviderSettings::new(move |window, cx| {
+ cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))
+ .into()
+ })
+ .description(InlineDescription::Text(
+ "Run local models on your machine with Ollama.".into(),
+ )),
+ ))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- let request = match self.to_ollama_request(request) {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request = match Self::to_ollama_request(&config, request) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
@@ -577,7 +528,7 @@ impl LanguageModel for OllamaLanguageModel {
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let stream = stream_chat_completion(
http_client.as_ref(),
&api_url,
@@ -594,6 +545,23 @@ impl LanguageModel for OllamaLanguageModel {
}
}
+fn language_model(model: &ollama::Model) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.supports_tools.unwrap_or(false),
+ supports_images: model.supports_vision.unwrap_or(false),
+ supports_thinking: model.supports_thinking.unwrap_or(false),
+ disabled_reason: model.disabled.as_ref().map(|d| DisabledReason::new(d)),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.name.clone()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("ollama/{}", model.id()),
+ model.max_token_count(),
+ )
+ }
+}
+
fn map_to_language_model_completion_events(
stream: Pin<Box<dyn Stream<Item = anyhow::Result<ChatResponseDelta>> + Send>>,
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>> {
diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs
index 496c7b867ff..6af07c57edd 100644
--- a/crates/language_models/src/provider/open_ai.rs
+++ b/crates/language_models/src/provider/open_ai.rs
@@ -7,10 +7,11 @@ use http_client::{CustomHeaders, HttpClient};
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, CompactionResult, EnvVar,
FastModeConfirmation, IconOrSvg, LanguageModel, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
+ LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice, OPEN_AI_PROVIDER_ID,
- OPEN_AI_PROVIDER_NAME, ProviderSettingsView, RateLimiter, env_var, stream_in_background,
+ LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
+ ModelRateLimiters, OPEN_AI_PROVIDER_ID, OPEN_AI_PROVIDER_NAME, ProviderSettingsView,
+ RateLimiter, env_var, stream_in_background, unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -48,6 +49,7 @@ pub struct OpenAiSettings {
pub struct OpenAiLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -109,78 +111,25 @@ impl OpenAiLanguageModelProvider {
}
});
- Self { http_client, state }
- }
-
- fn create_language_model(&self, model: open_ai::Model) -> Arc<dyn LanguageModel> {
- Arc::new(OpenAiLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-
- fn settings(cx: &App) -> &OpenAiSettings {
- &crate::AllLanguageModelSettings::get_global(cx).openai
- }
-
- fn api_url(cx: &App) -> SharedString {
- let api_url = &Self::settings(cx).api_url;
- if api_url.is_empty() {
- open_ai::OPEN_AI_API_URL.into()
- } else {
- SharedString::new(api_url.as_str())
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
}
}
-}
-
-impl LanguageModelProviderState for OpenAiLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for OpenAiLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiOpenAi)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(open_ai::Model::default()))
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(open_ai::Model::default_fast()))
- }
-
- fn recommended_models(&self, _cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- vec![self.create_language_model(open_ai::Model::FivePointSixSol)]
- }
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: the built-in models,
+ /// with settings entries added or overriding built-in ones.
+ fn open_ai_models(&self, cx: &App) -> BTreeMap<String, open_ai::Model> {
let mut models = BTreeMap::default();
- // Add base models from open_ai::Model::iter()
for model in open_ai::Model::iter() {
if !matches!(model, open_ai::Model::Custom { .. }) {
models.insert(model.id().to_string(), model);
}
}
- // Override with available models from settings
- for model in &OpenAiLanguageModelProvider::settings(cx).available_models {
+ for model in &Self::settings(cx).available_models {
models.insert(
model.name.clone(),
open_ai::Model::Custom {
@@ -197,174 +146,22 @@ impl LanguageModelProvider for OpenAiLanguageModelProvider {
}
models
- .into_values()
- .map(|model| self.create_language_model(model))
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.read(cx);
- Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
- state.api_key_state.has_key(),
- state.api_key_state.is_from_env_var(),
- state.api_key_state.env_var_name().clone(),
- "https://platform.openai.com/api-keys".into(),
- )))
- }
-
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
- }
-
- fn fast_mode_confirmation(&self, _cx: &App) -> Option<FastModeConfirmation> {
- Some(FastModeConfirmation {
- title: "Enable Fast Mode for OpenAI?".into(),
- message: "Fast mode sends requests using OpenAI's Priority processing tier, which \
- targets significantly lower latency than the standard tier and is billed at a \
- premium per-token rate."
- .into(),
- })
- }
-}
-
-fn default_thinking_reasoning_effort(model: &open_ai::Model) -> Option<open_ai::ReasoningEffort> {
- use open_ai::ReasoningEffort;
-
- model
- .reasoning_effort()
- .filter(|effort| open_ai_reasoning_effort_is_supported(*effort))
- .or_else(|| {
- let supported_efforts = model.supported_reasoning_efforts();
- if supported_efforts.contains(&ReasoningEffort::Medium) {
- Some(ReasoningEffort::Medium)
- } else {
- supported_efforts
- .iter()
- .copied()
- .find(|effort| open_ai_reasoning_effort_is_supported(*effort))
- }
- })
-}
-
-fn open_ai_reasoning_effort_is_supported(effort: open_ai::ReasoningEffort) -> bool {
- effort != open_ai::ReasoningEffort::None
-}
-
-fn normalize_open_ai_response_thinking_effort(
- request: &mut LanguageModelRequest,
- model: &open_ai::Model,
-) {
- let selected_effort_is_supported = request
- .thinking_effort
- .as_deref()
- .and_then(|effort| effort.parse::<open_ai::ReasoningEffort>().ok())
- .is_some_and(|effort| {
- open_ai_reasoning_effort_is_supported(effort)
- && model.supported_reasoning_efforts().contains(&effort)
- });
-
- if !selected_effort_is_supported {
- request.thinking_effort = None;
- }
-}
-
-fn supports_selectable_thinking_effort(model: &open_ai::Model) -> bool {
- model.uses_responses_api()
- && model
- .supported_reasoning_efforts()
- .iter()
- .any(|effort| open_ai_reasoning_effort_is_supported(*effort))
-}
-
-fn supported_thinking_effort_levels(model: &open_ai::Model) -> Vec<LanguageModelEffortLevel> {
- if !supports_selectable_thinking_effort(model) {
- return Vec::new();
- }
-
- let default_effort = default_thinking_reasoning_effort(model);
- model
- .supported_reasoning_efforts()
- .iter()
- .copied()
- .filter_map(|effort| {
- if !open_ai_reasoning_effort_is_supported(effort) {
- return None;
- }
-
- Some(LanguageModelEffortLevel {
- name: effort.label().into(),
- value: effort.value().into(),
- is_default: Some(effort) == default_effort,
- })
- })
- .collect()
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn supported_thinking_effort_levels_hide_none() {
- let effort_levels = supported_thinking_effort_levels(&open_ai::Model::FivePointTwo);
- let values = effort_levels
- .iter()
- .map(|level| level.value.as_ref())
- .collect::<Vec<_>>();
-
- assert_eq!(values, ["low", "medium", "high", "xhigh"]);
- assert_eq!(
- effort_levels
- .iter()
- .find(|level| level.is_default)
- .map(|level| level.value.as_ref()),
- Some("medium")
- );
}
- #[test]
- fn models_supporting_only_none_have_no_selectable_thinking_effort() {
- let model = open_ai::Model::Custom {
- name: "custom-model".to_string(),
- display_name: None,
- max_tokens: 128_000,
- max_output_tokens: None,
- max_completion_tokens: None,
- reasoning_effort: Some(open_ai::ReasoningEffort::None),
- supports_chat_completions: false,
- supports_images: true,
- };
-
- assert!(!supports_selectable_thinking_effort(&model));
- assert!(supported_thinking_effort_levels(&model).is_empty());
- assert!(
- model
- .supported_reasoning_efforts()
- .contains(&open_ai::ReasoningEffort::None)
- );
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<open_ai::Model, LanguageModelCompletionError> {
+ self.open_ai_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
-}
-
-pub struct OpenAiLanguageModel {
- id: LanguageModelId,
- model: open_ai::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-impl OpenAiLanguageModel {
- fn stream_completion(
+ fn stream_chat_completion(
&self,
+ request_limiter: &RateLimiter,
request: open_ai::Request,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>>>
@@ -379,7 +176,7 @@ impl OpenAiLanguageModel {
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let provider = PROVIDER_NAME;
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey { provider });
@@ -401,6 +198,7 @@ impl OpenAiLanguageModel {
fn stream_response(
&self,
+ request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
@@ -416,7 +214,7 @@ impl OpenAiLanguageModel {
});
let provider = PROVIDER_NAME;
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey { provider });
};
@@ -437,6 +235,7 @@ impl OpenAiLanguageModel {
fn compact_response(
&self,
+ request_limiter: &RateLimiter,
request: CompactRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<CompactedResponse, LanguageModelCompletionError>> {
@@ -451,7 +250,7 @@ impl OpenAiLanguageModel {
});
let provider = PROVIDER_NAME;
- let future = self.request_limiter.run(async move {
+ let future = request_limiter.run(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey { provider });
};
@@ -468,167 +267,184 @@ impl OpenAiLanguageModel {
future.boxed()
}
-}
-impl LanguageModel for OpenAiLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
+ fn settings(cx: &App) -> &OpenAiSettings {
+ &crate::AllLanguageModelSettings::get_global(cx).openai
+ }
+
+ fn api_url(cx: &App) -> SharedString {
+ let api_url = &Self::settings(cx).api_url;
+ if api_url.is_empty() {
+ open_ai::OPEN_AI_API_URL.into()
+ } else {
+ SharedString::new(api_url.as_str())
+ }
}
+}
+
+impl LanguageModelProviderState for OpenAiLanguageModelProvider {
+ type ObservableEntity = State;
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn provider_id(&self) -> LanguageModelProviderId {
+impl LanguageModelProvider for OpenAiLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
PROVIDER_ID
}
- fn provider_name(&self) -> LanguageModelProviderName {
+ fn name(&self) -> LanguageModelProviderName {
PROVIDER_NAME
}
- fn supports_tools(&self) -> bool {
- true
- }
-
- fn supports_images(&self) -> bool {
- use open_ai::Model;
- match &self.model {
- Model::FourOmniMini
- | Model::Five
- | Model::FiveMini
- | Model::FiveNano
- | Model::FivePointOne
- | Model::FivePointTwo
- | Model::FivePointThreeCodex
- | Model::FivePointFour
- | Model::FivePointFourMini
- | Model::FivePointFourNano
- | Model::FivePointFourPro
- | Model::FivePointFive
- | Model::FivePointFivePro
- | Model::FivePointSixSol
- | Model::FivePointSixTerra
- | Model::FivePointSixLuna
- | Model::SixAstra
- | Model::SixSol
- | Model::SixLuna
- | Model::O3 => true,
- Model::Four => false,
- Model::Custom {
- supports_images, ..
- } => *supports_images,
- }
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiOpenAi)
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto => true,
- LanguageModelToolChoice::Any => true,
- LanguageModelToolChoice::None => true,
- }
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.open_ai_models(cx)
+ .get(open_ai::Model::default().id())
+ .map(language_model)
}
- fn supports_streaming_tools(&self) -> bool {
- true
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.open_ai_models(cx)
+ .get(open_ai::Model::default_fast().id())
+ .map(language_model)
}
- fn supports_thinking(&self) -> bool {
- supports_selectable_thinking_effort(&self.model)
+ fn recommended_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.open_ai_models(cx)
+ .get(open_ai::Model::FivePointSixSol.id())
+ .map(language_model)
+ .into_iter()
+ .collect()
}
- fn supports_fast_mode(&self) -> bool {
- self.model.supports_priority()
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.open_ai_models(cx)
+ .values()
+ .map(language_model)
+ .collect()
}
- fn supports_server_side_compaction(&self) -> bool {
- self.model.supports_compaction()
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn supports_explicit_compaction(&self) -> bool {
- self.model.supports_compaction()
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn compact(
+ fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.read(cx);
+ Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
+ state.api_key_state.has_key(),
+ state.api_key_state.is_from_env_var(),
+ state.api_key_state.env_var_name().clone(),
+ "https://platform.openai.com/api-keys".into(),
+ )))
+ }
+
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ }
+
+ fn stream_completion(
&self,
+ model: &LanguageModel,
mut request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- if !self.supports_explicit_compaction() {
- return async {
- Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
- "this OpenAI model does not support explicit compaction"
- )))
- }
- .boxed();
- }
-
- normalize_open_ai_response_thinking_effort(&mut request, &self.model);
- let request = match into_open_ai_response(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- self.max_output_tokens(),
- default_thinking_reasoning_effort(&self.model),
- self.model
- .supported_reasoning_efforts()
- .contains(&open_ai::ReasoningEffort::None),
- &OPEN_AI_PROVIDER_ID,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
};
- let request = request.into_compact_request();
- let response = self.compact_response(request, cx);
- async move {
- let response = response.await?;
- let usage = token_usage_from_response_usage(&response.usage);
- let context = response
- .into_compacted_context(OPEN_AI_PROVIDER_ID)
- .map_err(LanguageModelCompletionError::Other)?;
- Ok(CompactionResult { context, usage })
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if !config.supports_priority() {
+ request.speed = None;
+ }
+ if config.uses_responses_api() {
+ normalize_open_ai_response_thinking_effort(&mut request, &config);
+ let request = match into_open_ai_response(
+ request,
+ config.id(),
+ config.supports_parallel_tool_calls(),
+ config.supports_prompt_cache_key(),
+ config.max_output_tokens(),
+ default_thinking_reasoning_effort(&config),
+ config
+ .supported_reasoning_efforts()
+ .contains(&open_ai::ReasoningEffort::None),
+ &OPEN_AI_PROVIDER_ID,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_response(&request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);
+ Ok(stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
+ } else {
+ let request = match into_open_ai(
+ request,
+ config.id(),
+ config.supports_parallel_tool_calls(),
+ config.supports_prompt_cache_key(),
+ config.max_output_tokens(),
+ ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ None,
+ false,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_chat_completion(&request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
}
- .boxed()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- supported_thinking_effort_levels(&self.model)
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn telemetry_id(&self) -> String {
- format!("openai/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
}
fn count_input_tokens(
&self,
+ model: &LanguageModel,
mut request: LanguageModelRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
- if !self.model.uses_responses_api() {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if !config.uses_responses_api() {
return async { Ok(None) }.boxed();
}
- normalize_open_ai_response_thinking_effort(&mut request, &self.model);
+ normalize_open_ai_response_thinking_effort(&mut request, &config);
let request = into_open_ai_response(
request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- self.max_output_tokens(),
- default_thinking_reasoning_effort(&self.model),
- self.model
+ config.id(),
+ config.supports_parallel_tool_calls(),
+ config.supports_prompt_cache_key(),
+ config.max_output_tokens(),
+ default_thinking_reasoning_effort(&config),
+ config
.supported_reasoning_efforts()
.contains(&open_ai::ReasoningEffort::None),
&OPEN_AI_PROVIDER_ID,
@@ -641,7 +457,7 @@ impl LanguageModel for OpenAiLanguageModel {
.clone();
(state.api_key_state.key(&api_url), api_url, extra_headers)
});
- self.request_limiter
+ request_limiter
.run(async move {
let request = request?.into_count_tokens_request();
let api_key = api_key.ok_or(LanguageModelCompletionError::NoApiKey {
@@ -662,74 +478,231 @@ impl LanguageModel for OpenAiLanguageModel {
.boxed()
}
- fn stream_completion(
+ fn compact(
&self,
+ model: &LanguageModel,
mut request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- if !self.model.supports_priority() {
- request.speed = None;
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if !config.supports_compaction() {
+ return async {
+ Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
+ "this OpenAI model does not support explicit compaction"
+ )))
+ }
+ .boxed();
}
- if self.model.uses_responses_api() {
- normalize_open_ai_response_thinking_effort(&mut request, &self.model);
- let request = match into_open_ai_response(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- self.max_output_tokens(),
- default_thinking_reasoning_effort(&self.model),
- self.model
- .supported_reasoning_efforts()
- .contains(&open_ai::ReasoningEffort::None),
- &OPEN_AI_PROVIDER_ID,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_response(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);
- Ok(stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
+
+ normalize_open_ai_response_thinking_effort(&mut request, &config);
+ let request = match into_open_ai_response(
+ request,
+ config.id(),
+ config.supports_parallel_tool_calls(),
+ config.supports_prompt_cache_key(),
+ config.max_output_tokens(),
+ default_thinking_reasoning_effort(&config),
+ config
+ .supported_reasoning_efforts()
+ .contains(&open_ai::ReasoningEffort::None),
+ &OPEN_AI_PROVIDER_ID,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let request = request.into_compact_request();
+ let response = self.compact_response(&request_limiter, request, cx);
+ async move {
+ let response = response.await?;
+ let usage = token_usage_from_response_usage(&response.usage);
+ let context = response
+ .into_compacted_context(OPEN_AI_PROVIDER_ID)
+ .map_err(LanguageModelCompletionError::Other)?;
+ Ok(CompactionResult { context, usage })
+ }
+ .boxed()
+ }
+
+ fn fast_mode_confirmation(&self, _cx: &App) -> Option<FastModeConfirmation> {
+ Some(FastModeConfirmation {
+ title: "Enable Fast Mode for OpenAI?".into(),
+ message: "Fast mode sends requests using OpenAI's Priority processing tier, which \
+ targets significantly lower latency than the standard tier and is billed at a \
+ premium per-token rate."
+ .into(),
+ })
+ }
+}
+
+fn default_thinking_reasoning_effort(model: &open_ai::Model) -> Option<open_ai::ReasoningEffort> {
+ use open_ai::ReasoningEffort;
+
+ model
+ .reasoning_effort()
+ .filter(|effort| open_ai_reasoning_effort_is_supported(*effort))
+ .or_else(|| {
+ let supported_efforts = model.supported_reasoning_efforts();
+ if supported_efforts.contains(&ReasoningEffort::Medium) {
+ Some(ReasoningEffort::Medium)
+ } else {
+ supported_efforts
+ .iter()
+ .copied()
+ .find(|effort| open_ai_reasoning_effort_is_supported(*effort))
}
- .boxed()
- } else {
- let request = match into_open_ai(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- self.max_output_tokens(),
- ChatCompletionMaxTokensParameter::MaxCompletionTokens,
- None,
- false,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
+ })
+}
+
+fn open_ai_reasoning_effort_is_supported(effort: open_ai::ReasoningEffort) -> bool {
+ effort != open_ai::ReasoningEffort::None
+}
+
+fn normalize_open_ai_response_thinking_effort(
+ request: &mut LanguageModelRequest,
+ model: &open_ai::Model,
+) {
+ let selected_effort_is_supported = request
+ .thinking_effort
+ .as_deref()
+ .and_then(|effort| effort.parse::<open_ai::ReasoningEffort>().ok())
+ .is_some_and(|effort| {
+ open_ai_reasoning_effort_is_supported(effort)
+ && model.supported_reasoning_efforts().contains(&effort)
+ });
+
+ if !selected_effort_is_supported {
+ request.thinking_effort = None;
+ }
+}
+
+fn supports_selectable_thinking_effort(model: &open_ai::Model) -> bool {
+ model.uses_responses_api()
+ && model
+ .supported_reasoning_efforts()
+ .iter()
+ .any(|effort| open_ai_reasoning_effort_is_supported(*effort))
+}
+
+fn supported_thinking_effort_levels(model: &open_ai::Model) -> Vec<LanguageModelEffortLevel> {
+ if !supports_selectable_thinking_effort(model) {
+ return Vec::new();
+ }
+
+ let default_effort = default_thinking_reasoning_effort(model);
+ model
+ .supported_reasoning_efforts()
+ .iter()
+ .copied()
+ .filter_map(|effort| {
+ if !open_ai_reasoning_effort_is_supported(effort) {
+ return None;
}
- .boxed()
- }
+
+ Some(LanguageModelEffortLevel {
+ name: effort.label().into(),
+ value: effort.value().into(),
+ is_default: Some(effort) == default_effort,
+ })
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn supported_thinking_effort_levels_hide_none() {
+ let effort_levels = supported_thinking_effort_levels(&open_ai::Model::FivePointTwo);
+ let values = effort_levels
+ .iter()
+ .map(|level| level.value.as_ref())
+ .collect::<Vec<_>>();
+
+ assert_eq!(values, ["low", "medium", "high", "xhigh"]);
+ assert_eq!(
+ effort_levels
+ .iter()
+ .find(|level| level.is_default)
+ .map(|level| level.value.as_ref()),
+ Some("medium")
+ );
+ }
+
+ #[test]
+ fn models_supporting_only_none_have_no_selectable_thinking_effort() {
+ let model = open_ai::Model::Custom {
+ name: "custom-model".to_string(),
+ display_name: None,
+ max_tokens: 128_000,
+ max_output_tokens: None,
+ max_completion_tokens: None,
+ reasoning_effort: Some(open_ai::ReasoningEffort::None),
+ supports_chat_completions: false,
+ supports_images: true,
+ };
+
+ assert!(!supports_selectable_thinking_effort(&model));
+ assert!(supported_thinking_effort_levels(&model).is_empty());
+ assert!(
+ model
+ .supported_reasoning_efforts()
+ .contains(&open_ai::ReasoningEffort::None)
+ );
+ }
+}
+
+fn language_model(model: &open_ai::Model) -> LanguageModel {
+ use open_ai::Model;
+ let supports_images = match model {
+ Model::FourOmniMini
+ | Model::Five
+ | Model::FiveMini
+ | Model::FiveNano
+ | Model::FivePointOne
+ | Model::FivePointTwo
+ | Model::FivePointThreeCodex
+ | Model::FivePointFour
+ | Model::FivePointFourMini
+ | Model::FivePointFourNano
+ | Model::FivePointFourPro
+ | Model::FivePointFive
+ | Model::FivePointFivePro
+ | Model::FivePointSixSol
+ | Model::FivePointSixTerra
+ | Model::FivePointSixLuna
+ | Model::SixAstra
+ | Model::SixSol
+ | Model::SixLuna
+ | Model::O3 => true,
+ Model::Four => false,
+ Model::Custom {
+ supports_images, ..
+ } => *supports_images,
+ };
+ LanguageModel {
+ supports_tools: true,
+ supports_images,
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_streaming_tools: true,
+ supports_thinking: supports_selectable_thinking_effort(model),
+ supports_fast_mode: model.supports_priority(),
+ supports_server_side_compaction: model.supports_compaction(),
+ supports_explicit_compaction: model.supports_compaction(),
+ supported_effort_levels: supported_thinking_effort_levels(model).into(),
+ supports_split_token_display: true,
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("openai/{}", model.id()),
+ model.max_token_count(),
+ )
}
}
diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs
index 090c348f167..e4bcf241162 100644
--- a/crates/language_models/src/provider/open_ai_compatible.rs
+++ b/crates/language_models/src/provider/open_ai_compatible.rs
@@ -6,10 +6,11 @@ use http_client::{CustomHeaders, HttpClient};
use language_model::chat_completion::ChatCompletionEventMapper;
use language_model::{
AuthenticateError, IconOrSvg, LanguageModel, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
+ LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
- ProviderSettingsView, RateLimiter, SubPageProviderSettings,
+ LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
+ ModelRateLimiters, ProviderSettingsView, RateLimiter, SubPageProviderSettings,
+ unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -50,6 +51,7 @@ pub struct OpenAiCompatibleLanguageModelProvider {
name: LanguageModelProviderName,
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
impl OpenAiCompatibleLanguageModelProvider {
@@ -75,111 +77,61 @@ impl OpenAiCompatibleLanguageModelProvider {
name: id.into(),
http_client,
state,
+ request_limiters: ModelRateLimiters::default(),
}
}
- fn create_language_model(&self, model: AvailableModel) -> Arc<dyn LanguageModel> {
- Arc::new(OpenAiCompatibleLanguageModel {
- id: LanguageModelId::from(model.name.clone()),
- provider_id: self.id.clone(),
- provider_name: self.name.clone(),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
+ /// Every model this provider offers, in settings order.
+ fn available_models<'a>(&self, cx: &'a App) -> &'a [AvailableModel] {
+ &self.state.read(cx).settings.available_models
}
-}
-
-impl LanguageModelProviderState for OpenAiCompatibleLanguageModelProvider {
- type ObservableEntity = State;
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for OpenAiCompatibleLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelProviderName {
- self.name.clone()
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiOpenAiCompat)
- }
-
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
- self.state
- .read(cx)
- .settings
- .available_models
- .first()
- .map(|model| self.create_language_model(model.clone()))
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- None
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- self.state
- .read(cx)
- .settings
- .available_models
- .iter()
- .map(|model| self.create_language_model(model.clone()))
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
- }
-
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.clone();
- Some(ProviderSettingsView::SubPage(SubPageProviderSettings::new(
- move |window, cx| {
- cx.new(|cx| {
- ApiCompatibleProviderConfigurationView::new(
- state.clone(),
- "OpenAI",
- API_KEY_PLACEHOLDER,
- window,
- cx,
- )
- })
- .into()
+ fn language_model(&self, model: &AvailableModel) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.capabilities.tools,
+ supports_images: model.capabilities.images,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: model.capabilities.tools,
+ any: model.capabilities.tools,
+ none: true,
},
- )))
+ supports_streaming_tools: true,
+ supports_thinking: default_thinking_reasoning_effort(model).is_some(),
+ supported_effort_levels: supported_thinking_effort_levels(model).into(),
+ supports_split_token_display: true,
+ max_output_tokens: model.max_output_tokens,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.name.clone()),
+ LanguageModelName::from(
+ model
+ .display_name
+ .clone()
+ .unwrap_or_else(|| model.name.clone()),
+ ),
+ self.id.clone(),
+ self.name.clone(),
+ format!("openai/{}", model.name),
+ model.max_tokens,
+ )
+ }
}
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<AvailableModel, LanguageModelCompletionError> {
+ self.available_models(cx)
+ .iter()
+ .find(|available| available.name == model.id.0.as_ref())
+ .cloned()
+ .ok_or_else(|| unavailable_error(model))
}
-}
-
-pub struct OpenAiCompatibleLanguageModel {
- id: LanguageModelId,
- provider_id: LanguageModelProviderId,
- provider_name: LanguageModelProviderName,
- model: AvailableModel,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-impl OpenAiCompatibleLanguageModel {
- fn stream_completion(
+ fn stream_chat_completion(
&self,
+ request_limiter: &RateLimiter,
request: open_ai::Request,
cx: &AsyncApp,
) -> BoxFuture<
@@ -200,8 +152,8 @@ impl OpenAiCompatibleLanguageModel {
)
});
- let provider = self.provider_name.clone();
- let future = self.request_limiter.stream(async move {
+ let provider = self.name.clone();
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey { provider });
};
@@ -222,6 +174,7 @@ impl OpenAiCompatibleLanguageModel {
fn stream_response(
&self,
+ request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
@@ -237,8 +190,8 @@ impl OpenAiCompatibleLanguageModel {
)
});
- let provider = self.provider_name.clone();
- let future = self.request_limiter.stream(async move {
+ let provider = self.name.clone();
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey { provider });
};
@@ -258,6 +211,146 @@ impl OpenAiCompatibleLanguageModel {
}
}
+impl LanguageModelProviderState for OpenAiCompatibleLanguageModelProvider {
+ type ObservableEntity = State;
+
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
+ }
+}
+
+impl LanguageModelProvider for OpenAiCompatibleLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
+ self.id.clone()
+ }
+
+ fn name(&self) -> LanguageModelProviderName {
+ self.name.clone()
+ }
+
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiOpenAiCompat)
+ }
+
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.available_models(cx)
+ .first()
+ .map(|model| self.language_model(model))
+ }
+
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ None
+ }
+
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.available_models(cx)
+ .iter()
+ .map(|model| self.language_model(model))
+ .collect()
+ }
+
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
+ }
+
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
+ }
+
+ fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.clone();
+ Some(ProviderSettingsView::SubPage(SubPageProviderSettings::new(
+ move |window, cx| {
+ cx.new(|cx| {
+ ApiCompatibleProviderConfigurationView::new(
+ state.clone(),
+ "OpenAI",
+ API_KEY_PLACEHOLDER,
+ window,
+ cx,
+ )
+ })
+ .into()
+ },
+ )))
+ }
+
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ }
+
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ mut request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ // `speed` can leak in from a parent thread's model; this provider never
+ // supports fast mode, and arbitrary compatible endpoints reject `service_tier`.
+ request.speed = None;
+
+ if config.capabilities.chat_completions {
+ let reasoning_effort = chat_completion_reasoning_effort(&request, &config);
+ let request = match into_open_ai(
+ request,
+ &config.name,
+ config.capabilities.parallel_tool_calls,
+ config.capabilities.prompt_cache_key,
+ config.max_output_tokens,
+ chat_completion_max_tokens_parameter(&config),
+ reasoning_effort,
+ config.capabilities.interleaved_reasoning,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_chat_completion(&request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
+ } else {
+ disable_response_thinking_for_none_effort(&mut request, &config);
+ let request = match into_open_ai_response(
+ request,
+ &config.name,
+ config.capabilities.parallel_tool_calls,
+ config.capabilities.prompt_cache_key,
+ config.max_output_tokens,
+ default_thinking_reasoning_effort(&config),
+ supports_none_reasoning_effort(&config),
+ &self.id,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_response(&request_limiter, request, cx);
+ let compaction_state_owner = self.id.clone();
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = OpenAiResponseEventMapper::new(compaction_state_owner);
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
+ }
+ }
+}
+
fn default_thinking_reasoning_effort(model: &AvailableModel) -> Option<open_ai::ReasoningEffort> {
model
.reasoning_effort
@@ -331,147 +424,6 @@ fn disable_response_thinking_for_none_effort(
}
}
-impl LanguageModel for OpenAiCompatibleLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(
- self.model
- .display_name
- .clone()
- .unwrap_or_else(|| self.model.name.clone()),
- )
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- self.provider_id.clone()
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- self.provider_name.clone()
- }
-
- fn supports_tools(&self) -> bool {
- self.model.capabilities.tools
- }
-
- fn supports_images(&self) -> bool {
- self.model.capabilities.images
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto => self.model.capabilities.tools,
- LanguageModelToolChoice::Any => self.model.capabilities.tools,
- LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_thinking(&self) -> bool {
- default_thinking_reasoning_effort(&self.model).is_some()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- supported_thinking_effort_levels(&self.model)
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn telemetry_id(&self) -> String {
- format!("openai/{}", self.model.name)
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_tokens
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens
- }
-
- fn stream_completion(
- &self,
- mut request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- // `speed` can leak in from a parent thread's model; this provider never
- // supports fast mode, and arbitrary compatible endpoints reject `service_tier`.
- if !self.supports_fast_mode() {
- request.speed = None;
- }
-
- if self.model.capabilities.chat_completions {
- let reasoning_effort = chat_completion_reasoning_effort(&request, &self.model);
- let request = match into_open_ai(
- request,
- &self.model.name,
- self.model.capabilities.parallel_tool_calls,
- self.model.capabilities.prompt_cache_key,
- self.max_output_tokens(),
- chat_completion_max_tokens_parameter(&self.model),
- reasoning_effort,
- self.model.capabilities.interleaved_reasoning,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
- }
- .boxed()
- } else {
- disable_response_thinking_for_none_effort(&mut request, &self.model);
- let request = match into_open_ai_response(
- request,
- &self.model.name,
- self.model.capabilities.parallel_tool_calls,
- self.model.capabilities.prompt_cache_key,
- self.max_output_tokens(),
- default_thinking_reasoning_effort(&self.model),
- supports_none_reasoning_effort(&self.model),
- &self.provider_id,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_response(request, cx);
- let compaction_state_owner = self.provider_id.clone();
- let executor = cx.background_executor().clone();
- async move {
- let mapper = OpenAiResponseEventMapper::new(compaction_state_owner);
- Ok(language_model::stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
- }
- .boxed()
- }
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 9a11aedce59..c7a23b4e959 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -7,11 +7,11 @@ use http_client::{CustomHeaders, HttpClient};
use language_model::chat_completion::ChatCompletionEventMapper;
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelEffortLevel,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice, LanguageModelToolResultContent, MessageContent, ProviderSettingsView,
- RateLimiter, Role, env_var,
+ LanguageModelToolChoice, LanguageModelToolChoiceSupport, LanguageModelToolResultContent,
+ MessageContent, ModelRateLimiters, ProviderSettingsView, Role, env_var, unavailable_error,
};
use open_router::{
Model, ModelMode as OpenRouterModelMode, OPEN_ROUTER_API_URL, ReasoningEffort,
@@ -39,6 +39,7 @@ pub struct OpenRouterSettings {
pub struct OpenRouterLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -159,7 +160,11 @@ impl OpenRouterLanguageModelProvider {
}
});
- Self { http_client, state }
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
fn settings(cx: &App) -> &OpenRouterSettings {
@@ -175,47 +180,9 @@ impl OpenRouterLanguageModelProvider {
}
}
- fn create_language_model(&self, model: open_router::Model) -> Arc<dyn LanguageModel> {
- Arc::new(OpenRouterLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-}
-
-impl LanguageModelProviderState for OpenRouterLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for OpenRouterLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiOpenRouter)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(open_router::Model::default()))
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- None
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider lists, in order: the fetched list, with
+ /// settings entries replacing fetched ones of the same name or appended.
+ fn open_router_models(&self, cx: &App) -> Vec<open_router::Model> {
let mut models_from_api = self.state.read(cx).available_models.clone();
let mut settings_models = Vec::new();
@@ -257,45 +224,31 @@ impl LanguageModelProvider for OpenRouterLanguageModelProvider {
}
models_from_api
- .into_iter()
- .map(|model| self.create_language_model(model))
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
}
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.read(cx);
- Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
- state.api_key_state.has_key(),
- state.api_key_state.is_from_env_var(),
- state.api_key_state.env_var_name().clone(),
- "https://openrouter.ai/keys".into(),
- )))
+ /// The listed model with `id`, or the built-in default model when `id`
+ /// names it, even if the fetched list doesn't include it.
+ fn open_router_model(&self, id: &str, cx: &App) -> Option<open_router::Model> {
+ self.open_router_models(cx)
+ .into_iter()
+ .find(|listed| listed.id() == id)
+ .or_else(|| {
+ let default = open_router::Model::default();
+ (default.id() == id).then_some(default)
+ })
}
- fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
- self.state
- .update(cx, |state, cx| state.set_api_key(api_key, cx))
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<open_router::Model, LanguageModelCompletionError> {
+ self.open_router_model(model.id.0.as_ref(), cx)
+ .ok_or_else(|| unavailable_error(model))
}
-}
-pub struct OpenRouterLanguageModel {
- id: LanguageModelId,
- model: open_router::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-impl OpenRouterLanguageModel {
- fn stream_completion(
+ fn stream_open_router_request(
&self,
request: open_router::Request,
cx: &AsyncApp,
@@ -337,111 +290,87 @@ impl OpenRouterLanguageModel {
}
}
-impl LanguageModel for OpenRouterLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
+impl LanguageModelProviderState for OpenRouterLanguageModelProvider {
+ type ObservableEntity = State;
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn provider_id(&self) -> LanguageModelProviderId {
+impl LanguageModelProvider for OpenRouterLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
PROVIDER_ID
}
- fn provider_name(&self) -> LanguageModelProviderName {
+ fn name(&self) -> LanguageModelProviderName {
PROVIDER_NAME
}
- fn supports_tools(&self) -> bool {
- self.model.supports_tool_calls()
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiOpenRouter)
}
- fn supports_thinking(&self) -> bool {
- matches!(
- self.model.mode,
- OpenRouterModelMode::Thinking { .. } | OpenRouterModelMode::Adaptive
- )
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ let default = open_router::Model::default();
+ self.open_router_model(default.id(), cx)
+ .map(|model| language_model(&model))
}
- fn supports_disabling_thinking(&self) -> bool {
- !self.model.mandatory_reasoning
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ None
}
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- let efforts: &[ReasoningEffort] = if !self.model.supported_efforts.is_empty() {
- &self.model.supported_efforts
- } else if self.model.supports_max_tokens {
- &ReasoningEffort::OPENAI_COMPATIBLE_SELECTABLE
- } else {
- return Vec::new();
- };
- let default_effort = self.model.default_effort.or_else(|| {
- self.model
- .supports_max_tokens
- .then_some(ReasoningEffort::Medium)
- });
- efforts
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.open_router_models(cx)
.iter()
- .map(|&effort| LanguageModelEffortLevel {
- name: effort.label().into(),
- value: effort.value().into(),
- is_default: Some(effort) == default_effort,
- })
+ .map(language_model)
.collect()
}
- fn telemetry_id(&self) -> String {
- format!("openrouter/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto => true,
- LanguageModelToolChoice::Any => true,
- LanguageModelToolChoice::None => true,
- }
+ fn settings_view(&self, cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.read(cx);
+ Some(ProviderSettingsView::ApiKey(ApiKeyConfiguration::new(
+ state.api_key_state.has_key(),
+ state.api_key_state.is_from_env_var(),
+ state.api_key_state.env_var_name().clone(),
+ "https://openrouter.ai/keys".into(),
+ )))
}
- fn supports_images(&self) -> bool {
- self.model.supports_images.unwrap_or(false)
+ fn set_api_key(&self, api_key: Option<String>, cx: &mut App) -> Task<Result<()>> {
+ self.state
+ .update(cx, |state, cx| state.set_api_key(api_key, cx))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
let openrouter_request =
- match into_open_router(request, &self.model, self.max_output_tokens()) {
+ match into_open_router(request, &config, config.max_output_tokens()) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let request = self.stream_completion(openrouter_request, cx);
+ let request = self.stream_open_router_request(openrouter_request, cx);
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let response = request.await?;
let events = ChatCompletionEventMapper::new().map_stream(response);
Ok(language_model::stream_in_background(
@@ -453,6 +382,48 @@ impl LanguageModel for OpenRouterLanguageModel {
}
}
+fn language_model(model: &open_router::Model) -> LanguageModel {
+ let efforts: &[ReasoningEffort] = if !model.supported_efforts.is_empty() {
+ &model.supported_efforts
+ } else if model.supports_max_tokens {
+ &ReasoningEffort::OPENAI_COMPATIBLE_SELECTABLE
+ } else {
+ &[]
+ };
+ let default_effort = model
+ .default_effort
+ .or_else(|| model.supports_max_tokens.then_some(ReasoningEffort::Medium));
+ let supported_effort_levels = efforts
+ .iter()
+ .map(|&effort| LanguageModelEffortLevel {
+ name: effort.label().into(),
+ value: effort.value().into(),
+ is_default: Some(effort) == default_effort,
+ })
+ .collect();
+ LanguageModel {
+ supports_tools: model.supports_tool_calls(),
+ supports_streaming_tools: true,
+ supports_thinking: matches!(
+ model.mode,
+ OpenRouterModelMode::Thinking { .. } | OpenRouterModelMode::Adaptive
+ ),
+ supports_disabling_thinking: !model.mandatory_reasoning,
+ supported_effort_levels,
+ max_output_tokens: model.max_output_tokens(),
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_images: model.supports_images.unwrap_or(false),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("openrouter/{}", model.id()),
+ model.max_token_count(),
+ )
+ }
+}
+
pub fn into_open_router(
request: LanguageModelRequest,
model: &Model,
diff --git a/crates/language_models/src/provider/openai_subscribed.rs b/crates/language_models/src/provider/openai_subscribed.rs
index bac7831a6d3..3ab8e6bb868 100644
--- a/crates/language_models/src/provider/openai_subscribed.rs
+++ b/crates/language_models/src/provider/openai_subscribed.rs
@@ -1,14 +1,18 @@
use anyhow::{Result, anyhow};
use credentials_provider::CredentialsProvider;
-use futures::future::Shared;
-use gpui::{App, Context, Entity, SharedString, Task, Window};
+use futures::FutureExt as _;
+use futures::future::{BoxFuture, Shared};
+
+use gpui::{App, AsyncApp, Context, Entity, SharedString, Task, Window};
use http_client::HttpClient;
use language_model::{
- AuthenticateError, FastModeConfirmation, IconOrSvg, InlineDescription, LanguageModel,
+ AuthenticateError, CompactionResult, FastModeConfirmation, IconOrSvg, InlineDescription,
+ LanguageModel, LanguageModelCompletionError, LanguageModelCompletionStream,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, ProviderSettingsView,
+ LanguageModelProviderState, LanguageModelRequest, ModelRateLimiters, ProviderSettingsView,
+ unavailable_error,
};
-use openai_subscribed::{PROVIDER_ID, PROVIDER_NAME, State, create_language_model};
+use openai_subscribed::{ChatGptModel, PROVIDER_ID, PROVIDER_NAME, State, language_model};
use std::sync::Arc;
use ui::{ConfiguredApiCard, prelude::*};
@@ -17,6 +21,7 @@ const SUBSCRIPTION_DESCRIPTION: &str =
pub struct OpenAiSubscribedProvider {
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
impl OpenAiSubscribedProvider {
@@ -26,7 +31,25 @@ impl OpenAiSubscribedProvider {
cx: &mut App,
) -> Self {
let state = cx.new(|cx| State::new(http_client, credentials_provider, cx));
- Self { state }
+ Self {
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<ChatGptModel, LanguageModelCompletionError> {
+ self.state
+ .read(cx)
+ .available_models()
+ .iter()
+ .find(|config| config.id() == model.id.0.as_ref())
+ .cloned()
+ .ok_or_else(|| unavailable_error(model))
}
}
@@ -51,31 +74,30 @@ impl LanguageModelProvider for OpenAiSubscribedProvider {
IconOrSvg::Icon(IconName::AiOpenAiGptSub)
}
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
self.state
.read(cx)
.default_model()
- .map(|model| create_language_model(model, &self.state, cx))
+ .map(|model| language_model(&model))
}
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
self.state
.read(cx)
.default_fast_model()
- .map(|model| create_language_model(model, &self.state, cx))
+ .map(|model| language_model(&model))
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
self.state
.read(cx)
.available_models()
.iter()
- .cloned()
- .map(|model| create_language_model(model, &self.state, cx))
+ .map(language_model)
.collect()
}
- fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn recommended_models(&self, cx: &App) -> Vec<LanguageModel> {
self.default_model(cx).into_iter().collect()
}
@@ -151,6 +173,35 @@ impl LanguageModelProvider for OpenAiSubscribedProvider {
.into()
}
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ openai_subscribed::stream_completion(&config, &self.state, &request_limiter, request, cx)
+ }
+
+ fn compact(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ openai_subscribed::compact(&config, &self.state, &request_limiter, request, cx)
+ }
+
fn fast_mode_confirmation(&self, _cx: &App) -> Option<FastModeConfirmation> {
Some(FastModeConfirmation {
title: "Enable Fast Mode for OpenAI?".into(),
diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rs
index 17c490e8242..f446610a98b 100644
--- a/crates/language_models/src/provider/opencode.rs
+++ b/crates/language_models/src/provider/opencode.rs
@@ -9,11 +9,11 @@ use gpui::{
use http_client::{AsyncBody, CustomHeaders, HttpClient, Method, Request, http};
use language_model::{
ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, InlineDescription, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelEffortLevel,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice, ProviderSettingsView, RateLimiter, ReasoningEffort,
- SubPageProviderSettings, env_var,
+ LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView, RateLimiter,
+ ReasoningEffort, SubPageProviderSettings, env_var, unavailable_error,
};
use opencode::{ApiProtocol, OPENCODE_API_URL, OpenCodeSubscription};
use serde::Deserialize;
@@ -96,6 +96,7 @@ pub struct OpenCodeSettings {
pub struct OpenCodeLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -370,81 +371,19 @@ impl OpenCodeLanguageModelProvider {
}
});
- Self { http_client, state }
- }
-
- fn create_language_model_with_capabilities(
- &self,
- model: opencode::Model,
- subscription: OpenCodeSubscription,
- supports_images: bool,
- supports_thinking: bool,
- ) -> Arc<dyn LanguageModel> {
- let id_str = format!("{}/{}", subscription.id_prefix(), model.id());
- Arc::new(OpenCodeLanguageModel {
- id: LanguageModelId::from(id_str),
- model,
- subscription,
- supports_images,
- supports_thinking,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
-
- pub fn settings(cx: &App) -> &OpenCodeSettings {
- &crate::AllLanguageModelSettings::get_global(cx).opencode
- }
-
- fn subscription_enabled(subscription: OpenCodeSubscription, cx: &App) -> bool {
- let settings = Self::settings(cx);
- match subscription {
- OpenCodeSubscription::Zen => settings.show_zen_models,
- OpenCodeSubscription::Go => settings.show_go_models,
- }
- }
-
- fn api_url(cx: &App) -> SharedString {
- let api_url = &Self::settings(cx).api_url;
- if api_url.is_empty() {
- OPENCODE_API_URL.into()
- } else {
- SharedString::new(api_url.as_str())
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
}
}
-}
-
-impl LanguageModelProviderState for OpenCodeLanguageModelProvider {
- type ObservableEntity = State;
-
- fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
- Some(self.state.clone())
- }
-}
-
-impl LanguageModelProvider for OpenCodeLanguageModelProvider {
- fn id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
- fn icon(&self) -> IconOrSvg {
- IconOrSvg::Icon(IconName::AiOpenCode)
- }
-
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- None
- }
-
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- None
- }
-
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ /// Every model this provider offers, keyed by id: discovered models of
+ /// enabled subscriptions, with settings entries added or overriding them.
+ fn opencode_models(
+ &self,
+ cx: &App,
+ ) -> BTreeMap<String, (DiscoveredModel, OpenCodeSubscription)> {
let mut models: BTreeMap<String, (DiscoveredModel, OpenCodeSubscription)> =
BTreeMap::default();
let settings = Self::settings(cx);
@@ -454,7 +393,7 @@ impl LanguageModelProvider for OpenCodeLanguageModelProvider {
if Self::subscription_enabled(subscription, cx) {
if let Some(discovered) = discovered_models.get(&subscription) {
for model in discovered {
- let key = format!("{}/{}", subscription.id_prefix(), model.model.id());
+ let key = model_id(&model.model, subscription);
models.insert(key, (model.clone(), subscription));
}
}
@@ -486,7 +425,7 @@ impl LanguageModelProvider for OpenCodeLanguageModelProvider {
model.custom_model_api_url.clone(),
model.interleaved_reasoning,
);
- let key = format!("{}/{}", subscription.id_prefix(), model.name);
+ let key = model_id(&custom_model, subscription);
models.insert(
key,
(
@@ -501,88 +440,27 @@ impl LanguageModelProvider for OpenCodeLanguageModelProvider {
}
models
- .into_values()
- .map(|(model, subscription)| {
- self.create_language_model_with_capabilities(
- model.model,
- subscription,
- model.supports_images,
- model.supports_thinking,
- )
- })
- .collect()
- }
-
- fn is_authenticated(&self, cx: &App) -> bool {
- self.state.read(cx).is_authenticated()
}
- fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
- self.state.update(cx, |state, cx| state.authenticate(cx))
- }
-
- fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
- let state = self.state.clone();
- Some(ProviderSettingsView::SubPage(
- SubPageProviderSettings::new(move |window, cx| {
- cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))
- .into()
- })
- .description(InlineDescription::Text(
- "To use OpenCode models in Zed, you need an API key.".into(),
- )),
- ))
- }
-}
-
-pub struct OpenCodeLanguageModel {
- id: LanguageModelId,
- model: opencode::Model,
- subscription: OpenCodeSubscription,
- supports_images: bool,
- supports_thinking: bool,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-struct InjectHeaderClient {
- inner: Arc<dyn HttpClient>,
- name: http::HeaderName,
- value: http::HeaderValue,
-}
-
-impl HttpClient for InjectHeaderClient {
- fn user_agent(&self) -> Option<&http::HeaderValue> {
- self.inner.user_agent()
- }
-
- fn proxy(&self) -> Option<&http_client::Url> {
- self.inner.proxy()
- }
-
- fn send(
+ /// The current configuration of `model` and the subscription serving it,
+ /// if this provider still offers it.
+ fn config(
&self,
- mut req: http::Request<AsyncBody>,
- ) -> futures::future::BoxFuture<'static, anyhow::Result<http::Response<AsyncBody>>> {
- req.headers_mut()
- .insert(self.name.clone(), self.value.clone());
- self.inner.send(req)
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<(DiscoveredModel, OpenCodeSubscription), LanguageModelCompletionError> {
+ self.opencode_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
-}
-// Standalone requests do not have a conversation ID, but OpenCode requires a
-// non-empty session header.
-fn opencode_session_header_value(thread_id: Option<&str>) -> http::HeaderValue {
- thread_id
- .filter(|thread_id| !thread_id.is_empty())
- .and_then(|thread_id| http::HeaderValue::from_str(thread_id).ok())
- .unwrap_or_else(|| http::HeaderValue::from(rand::random::<u64>()))
-}
-
-impl OpenCodeLanguageModel {
- fn base_api_url(&self, cx: &AsyncApp) -> SharedString {
- if let Some(url) = self.model.custom_model_api_url() {
+ fn base_api_url(
+ &self,
+ config: &DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ cx: &AsyncApp,
+ ) -> SharedString {
+ if let Some(url) = config.model.custom_model_api_url() {
if !url.is_empty() {
return url.to_string().into();
}
@@ -593,12 +471,12 @@ impl OpenCodeLanguageModel {
.state
.read_with(cx, |_, cx| OpenCodeLanguageModelProvider::api_url(cx));
- let suffix = self.subscription.api_path_suffix();
+ let suffix = subscription.api_path_suffix();
let base_str = base.as_ref().trim_end_matches('/');
format!("{}{}", base_str, suffix).into()
}
- fn api_key(&self, cx: &AsyncApp) -> Option<Arc<str>> {
+ fn current_api_key(&self, cx: &AsyncApp) -> Option<Arc<str>> {
self.state.read_with(cx, |state, cx| {
let api_url = OpenCodeLanguageModelProvider::api_url(cx);
state.api_key_state.key(&api_url)
@@ -615,6 +493,9 @@ impl OpenCodeLanguageModel {
fn stream_anthropic(
&self,
+ config: &DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ request_limiter: &RateLimiter,
request: anthropic::Request,
http_client: Arc<dyn HttpClient>,
extra_headers: CustomHeaders,
@@ -630,10 +511,10 @@ impl OpenCodeLanguageModel {
>,
> {
// Anthropic crate appends /v1/messages to api_url
- let api_url = self.base_api_url(cx);
- let api_key = self.api_key(cx);
+ let api_url = self.base_api_url(config, subscription, cx);
+ let api_key = self.current_api_key(cx);
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -656,6 +537,9 @@ impl OpenCodeLanguageModel {
fn stream_openai_chat(
&self,
+ config: &DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ request_limiter: &RateLimiter,
request: open_ai::Request,
http_client: Arc<dyn HttpClient>,
extra_headers: CustomHeaders,
@@ -663,12 +547,12 @@ impl OpenCodeLanguageModel {
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>>>
{
// OpenAI crate appends /chat/completions to api_url, so we pass base + "/v1"
- let base_url = self.base_api_url(cx);
+ let base_url = self.base_api_url(config, subscription, cx);
let api_url: SharedString = format!("{base_url}/v1").into();
- let api_key = self.api_key(cx);
+ let api_key = self.current_api_key(cx);
let provider_name = PROVIDER_NAME.0.to_string();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -691,6 +575,9 @@ impl OpenCodeLanguageModel {
fn stream_openai_response(
&self,
+ config: &DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ request_limiter: &RateLimiter,
request: open_ai::responses::Request,
http_client: Arc<dyn HttpClient>,
extra_headers: CustomHeaders,
@@ -700,12 +587,12 @@ impl OpenCodeLanguageModel {
Result<futures::stream::BoxStream<'static, Result<open_ai::responses::StreamEvent>>>,
> {
// Responses crate appends /responses to api_url, so we pass base + "/v1"
- let base_url = self.base_api_url(cx);
+ let base_url = self.base_api_url(config, subscription, cx);
let api_url: SharedString = format!("{base_url}/v1").into();
- let api_key = self.api_key(cx);
+ let api_key = self.current_api_key(cx);
let provider_name = PROVIDER_NAME.0.to_string();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -728,6 +615,9 @@ impl OpenCodeLanguageModel {
fn stream_google(
&self,
+ config: &DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ request_limiter: &RateLimiter,
request: google_ai::GenerateContentRequest,
http_client: Arc<dyn HttpClient>,
extra_headers: CustomHeaders,
@@ -736,10 +626,10 @@ impl OpenCodeLanguageModel {
'static,
Result<futures::stream::BoxStream<'static, Result<google_ai::GenerateContentResponse>>>,
> {
- let api_url = self.base_api_url(cx);
- let api_key = self.api_key(cx);
+ let api_url = self.base_api_url(config, subscription, cx);
+ let api_key = self.current_api_key(cx);
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let Some(api_key) = api_key else {
return Err(LanguageModelCompletionError::NoApiKey {
provider: PROVIDER_NAME,
@@ -758,128 +648,98 @@ impl OpenCodeLanguageModel {
async move { Ok(future.await?.boxed()) }.boxed()
}
-}
-impl LanguageModel for OpenCodeLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
+ pub fn settings(cx: &App) -> &OpenCodeSettings {
+ &crate::AllLanguageModelSettings::get_global(cx).opencode
}
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(format!(
- "{}: {}",
- self.subscription.display_name(),
- self.model.display_name()
- ))
+ fn subscription_enabled(subscription: OpenCodeSubscription, cx: &App) -> bool {
+ let settings = Self::settings(cx);
+ match subscription {
+ OpenCodeSubscription::Zen => settings.show_zen_models,
+ OpenCodeSubscription::Go => settings.show_go_models,
+ }
}
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
+ fn api_url(cx: &App) -> SharedString {
+ let api_url = &Self::settings(cx).api_url;
+ if api_url.is_empty() {
+ OPENCODE_API_URL.into()
+ } else {
+ SharedString::new(api_url.as_str())
+ }
}
+}
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
+impl LanguageModelProviderState for OpenCodeLanguageModelProvider {
+ type ObservableEntity = State;
+
+ fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
+ Some(self.state.clone())
}
+}
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
+impl LanguageModelProvider for OpenCodeLanguageModelProvider {
+ fn id(&self) -> LanguageModelProviderId {
+ PROVIDER_ID
}
- fn supports_images(&self) -> bool {
- self.supports_images
+ fn name(&self) -> LanguageModelProviderName {
+ PROVIDER_NAME
}
- fn supports_thinking(&self) -> bool {
- self.supports_thinking
+ fn icon(&self) -> IconOrSvg {
+ IconOrSvg::Icon(IconName::AiOpenCode)
}
- fn supports_disabling_thinking(&self) -> bool {
- self.model
- .supported_reasoning_effort_levels()
- .is_some_and(|levels| levels.contains(&ReasoningEffort::None))
+ fn default_model(&self, _cx: &App) -> Option<LanguageModel> {
+ None
}
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- self.model
- .supported_reasoning_effort_levels()
- .map(|levels| {
- let levels = levels
- .iter()
- .copied()
- .filter(|effort| *effort != ReasoningEffort::None)
- .collect::<Vec<_>>();
- if levels.is_empty() {
- return Vec::new();
- }
- let default_index = if levels.contains(&ReasoningEffort::Max) {
- [
- ReasoningEffort::High,
- ReasoningEffort::Medium,
- ReasoningEffort::Low,
- ReasoningEffort::Minimal,
- ]
- .into_iter()
- .find_map(|effort| levels.iter().position(|level| *level == effort))
- .unwrap_or(levels.len() - 1)
- } else {
- levels.len() - 1
- };
- levels
- .into_iter()
- .enumerate()
- .map(|(i, effort)| {
- let (name, value) = reasoning_effort_display(effort);
- LanguageModelEffortLevel {
- name: name.into(),
- value: value.into(),
- is_default: i == default_index,
- }
- })
- .collect()
- })
- .unwrap_or_default()
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ None
}
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::Any => true,
- LanguageModelToolChoice::None => {
- // Google models don't support None tool choice
- self.model.protocol() != ApiProtocol::Google
- }
- }
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.opencode_models(cx)
+ .values()
+ .map(|(model, subscription)| language_model(model, *subscription))
+ .collect()
}
- fn telemetry_id(&self) -> String {
- format!(
- "opencode/{}/{}",
- self.subscription.id_prefix(),
- self.model.id()
- )
+ fn is_authenticated(&self, cx: &App) -> bool {
+ self.state.read(cx).is_authenticated()
}
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
+ fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
+ self.state.update(cx, |state, cx| state.authenticate(cx))
}
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
+ fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
+ let state = self.state.clone();
+ Some(ProviderSettingsView::SubPage(
+ SubPageProviderSettings::new(move |window, cx| {
+ cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))
+ .into()
+ })
+ .description(InlineDescription::Text(
+ "To use OpenCode models in Zed, you need an API key.".into(),
+ )),
+ ))
}
fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let (config, subscription) = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
let http_client: Arc<dyn HttpClient> = Arc::new(InjectHeaderClient {
inner: self.http_client.clone(),
name: http::HeaderName::from_static(OPENCODE_SESSION_HEADER_NAME),
@@ -887,18 +747,18 @@ impl LanguageModel for OpenCodeLanguageModel {
});
let extra_headers = self.custom_headers(cx);
- match self.model.protocol() {
+ match config.model.protocol() {
ApiProtocol::Anthropic => {
- let mode = if self.supports_thinking() && request.thinking_allowed {
+ let mode = if config.supports_thinking && request.thinking_allowed {
anthropic::AnthropicModelMode::AdaptiveThinking
} else {
anthropic::AnthropicModelMode::Default
};
let anthropic_request = match into_anthropic(
request,
- self.model.id().to_string(),
+ config.model.id().to_string(),
1.0,
- self.model.max_output_tokens().unwrap_or(8192),
+ config.model.max_output_tokens().unwrap_or(8192),
mode,
anthropic::completion::AnthropicPromptCacheMode::Automatic,
&PROVIDER_ID,
@@ -906,8 +766,15 @@ impl LanguageModel for OpenCodeLanguageModel {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let stream =
- self.stream_anthropic(anthropic_request, http_client, extra_headers, cx);
+ let stream = self.stream_anthropic(
+ &config,
+ subscription,
+ &request_limiter,
+ anthropic_request,
+ http_client,
+ extra_headers,
+ cx,
+ );
let executor = cx.background_executor().clone();
async move {
let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);
@@ -929,19 +796,26 @@ impl LanguageModel for OpenCodeLanguageModel {
};
let openai_request = match into_open_ai(
request,
- self.model.id(),
+ config.model.id(),
true,
false,
- self.model.max_output_tokens(),
+ config.model.max_output_tokens(),
ChatCompletionMaxTokensParameter::MaxCompletionTokens,
reasoning_effort,
- self.model.interleaved_reasoning(),
+ config.model.interleaved_reasoning(),
) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let stream =
- self.stream_openai_chat(openai_request, http_client, extra_headers, cx);
+ let stream = self.stream_openai_chat(
+ &config,
+ subscription,
+ &request_limiter,
+ openai_request,
+ http_client,
+ extra_headers,
+ cx,
+ );
let executor = cx.background_executor().clone();
async move {
let mapper = ChatCompletionEventMapper::new();
@@ -953,16 +827,16 @@ impl LanguageModel for OpenCodeLanguageModel {
.boxed()
}
ApiProtocol::OpenAiResponses => {
- let supports_none_reasoning_effort = self
+ let supports_none_reasoning_effort = config
.model
.supported_reasoning_effort_levels()
.is_some_and(|levels| levels.contains(&ReasoningEffort::None));
let response_request = match into_open_ai_response(
request,
- self.model.id(),
+ config.model.id(),
true,
false,
- self.model.max_output_tokens(),
+ config.model.max_output_tokens(),
None,
supports_none_reasoning_effort,
&PROVIDER_ID,
@@ -970,8 +844,15 @@ impl LanguageModel for OpenCodeLanguageModel {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let stream =
- self.stream_openai_response(response_request, http_client, extra_headers, cx);
+ let stream = self.stream_openai_response(
+ &config,
+ subscription,
+ &request_limiter,
+ response_request,
+ http_client,
+ extra_headers,
+ cx,
+ );
let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
@@ -986,20 +867,29 @@ impl LanguageModel for OpenCodeLanguageModel {
let mut request = request;
if request.max_output_tokens.is_some() {
request.max_output_tokens =
- request.effective_max_output_tokens(self.max_output_tokens());
+ request.effective_max_output_tokens(config.model.max_output_tokens());
}
- let mode = if self.supports_thinking() && request.thinking_allowed {
+ let mode = if config.supports_thinking && request.thinking_allowed {
google_ai::GoogleModelMode::Thinking {
budget_tokens: None,
}
} else {
google_ai::GoogleModelMode::Default
};
- let google_request = match into_google(request, self.model.id().to_string(), mode) {
+ let google_request = match into_google(request, config.model.id().to_string(), mode)
+ {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let stream = self.stream_google(google_request, http_client, extra_headers, cx);
+ let stream = self.stream_google(
+ &config,
+ subscription,
+ &request_limiter,
+ google_request,
+ http_client,
+ extra_headers,
+ cx,
+ );
async move {
let mapper = GoogleEventMapper::new();
Ok(mapper.map_stream(stream.await?.boxed()).boxed())
@@ -1010,6 +900,118 @@ impl LanguageModel for OpenCodeLanguageModel {
}
}
+struct InjectHeaderClient {
+ inner: Arc<dyn HttpClient>,
+ name: http::HeaderName,
+ value: http::HeaderValue,
+}
+
+impl HttpClient for InjectHeaderClient {
+ fn user_agent(&self) -> Option<&http::HeaderValue> {
+ self.inner.user_agent()
+ }
+
+ fn proxy(&self) -> Option<&http_client::Url> {
+ self.inner.proxy()
+ }
+
+ fn send(
+ &self,
+ mut req: http::Request<AsyncBody>,
+ ) -> futures::future::BoxFuture<'static, anyhow::Result<http::Response<AsyncBody>>> {
+ req.headers_mut()
+ .insert(self.name.clone(), self.value.clone());
+ self.inner.send(req)
+ }
+}
+
+// Standalone requests do not have a conversation ID, but OpenCode requires a
+// non-empty session header.
+fn opencode_session_header_value(thread_id: Option<&str>) -> http::HeaderValue {
+ thread_id
+ .filter(|thread_id| !thread_id.is_empty())
+ .and_then(|thread_id| http::HeaderValue::from_str(thread_id).ok())
+ .unwrap_or_else(|| http::HeaderValue::from(rand::random::<u64>()))
+}
+
+fn model_id(model: &opencode::Model, subscription: OpenCodeSubscription) -> String {
+ format!("{}/{}", subscription.id_prefix(), model.id())
+}
+
+fn language_model(model: &DiscoveredModel, subscription: OpenCodeSubscription) -> LanguageModel {
+ let DiscoveredModel {
+ model,
+ supports_images,
+ supports_thinking,
+ } = model;
+ let supported_effort_levels = model
+ .supported_reasoning_effort_levels()
+ .map(|levels| {
+ let levels = levels
+ .iter()
+ .copied()
+ .filter(|effort| *effort != ReasoningEffort::None)
+ .collect::<Vec<_>>();
+ if levels.is_empty() {
+ return Vec::new();
+ }
+ let default_index = if levels.contains(&ReasoningEffort::Max) {
+ [
+ ReasoningEffort::High,
+ ReasoningEffort::Medium,
+ ReasoningEffort::Low,
+ ReasoningEffort::Minimal,
+ ]
+ .into_iter()
+ .find_map(|effort| levels.iter().position(|level| *level == effort))
+ .unwrap_or(levels.len() - 1)
+ } else {
+ levels.len() - 1
+ };
+ levels
+ .into_iter()
+ .enumerate()
+ .map(|(i, effort)| {
+ let (name, value) = reasoning_effort_display(effort);
+ LanguageModelEffortLevel {
+ name: name.into(),
+ value: value.into(),
+ is_default: i == default_index,
+ }
+ })
+ .collect()
+ })
+ .unwrap_or_default();
+ LanguageModel {
+ supports_tools: model.supports_tools(),
+ supports_images: *supports_images,
+ supports_thinking: *supports_thinking,
+ supports_disabling_thinking: model
+ .supported_reasoning_effort_levels()
+ .is_some_and(|levels| levels.contains(&ReasoningEffort::None)),
+ supported_effort_levels: supported_effort_levels.into(),
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: true,
+ any: true,
+ // Google models don't support None tool choice
+ none: model.protocol() != ApiProtocol::Google,
+ },
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model_id(model, subscription)),
+ LanguageModelName::from(format!(
+ "{}: {}",
+ subscription.display_name(),
+ model.display_name()
+ )),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("opencode/{}/{}", subscription.id_prefix(), model.id()),
+ model.max_token_count(),
+ )
+ }
+}
+
#[derive(Deserialize)]
struct RichCatalogProvider {
npm: Option<String>,
@@ -1692,34 +1694,25 @@ mod tests {
assert!(!errors.contains_key(&OpenCodeSubscription::Go));
}
- #[gpui::test]
- fn test_default_reasoning_effort(cx: &mut gpui::TestAppContext) {
+ #[test]
+ fn test_default_reasoning_effort() {
use ReasoningEffort::{High, Low, Max, Medium, Minimal, XHigh};
- cx.update(|cx| {
- let settings_store = SettingsStore::test(cx);
- cx.set_global(settings_store);
- let provider = OpenCodeLanguageModelProvider::new_with_cache(
- FakeHttpClient::with_404_response(),
- Arc::new(TestCredentialsProvider),
- FakeFs::new(cx.background_executor().clone()),
- PathBuf::from("/cache/models.json"),
- cx,
- );
- for (levels, expected) in [
- (vec![Low, Medium, High, XHigh, Max], Some("high")),
- (vec![Max, High, Medium], Some("high")),
- (vec![Low, Medium, Max], Some("medium")),
- (vec![Minimal, Low, Max], Some("low")),
- (vec![Minimal, Max], Some("minimal")),
- (vec![Low, High, XHigh], Some("xhigh")),
- (vec![High, Low], Some("low")),
- (vec![Max], Some("max")),
- (vec![ReasoningEffort::None], None),
- (vec![], None),
- ] {
- let model = provider.create_language_model_with_capabilities(
- opencode::Model::new(
+ for (levels, expected) in [
+ (vec![Low, Medium, High, XHigh, Max], Some("high")),
+ (vec![Max, High, Medium], Some("high")),
+ (vec![Low, Medium, Max], Some("medium")),
+ (vec![Minimal, Low, Max], Some("low")),
+ (vec![Minimal, Max], Some("minimal")),
+ (vec![Low, High, XHigh], Some("xhigh")),
+ (vec![High, Low], Some("low")),
+ (vec![Max], Some("max")),
+ (vec![ReasoningEffort::None], None),
+ (vec![], None),
+ ] {
+ let model = language_model(
+ &DiscoveredModel {
+ model: opencode::Model::new(
"test-model".to_string(),
None,
1000,
@@ -1729,19 +1722,19 @@ mod tests {
None,
false,
),
- OpenCodeSubscription::Zen,
- false,
- true,
- );
- assert_eq!(
- model
- .default_effort_level()
- .map(|effort| effort.value.to_string())
- .as_deref(),
- expected,
- );
- }
- });
+ supports_images: false,
+ supports_thinking: true,
+ },
+ OpenCodeSubscription::Zen,
+ );
+ assert_eq!(
+ model
+ .default_effort_level()
+ .map(|effort| effort.value.to_string())
+ .as_deref(),
+ expected,
+ );
+ }
}
#[test]
@@ -1959,11 +1952,15 @@ mod tests {
state.set_api_key(Some("test-key".to_string()), cx)
});
store_key.await.unwrap();
- let model = provider.create_language_model_with_capabilities(
- test_model("test-model", ApiProtocol::Anthropic),
+ let model = add_discovered_model(
+ &provider,
+ DiscoveredModel {
+ model: test_model("test-model", ApiProtocol::Anthropic),
+ supports_images: true,
+ supports_thinking: false,
+ },
OpenCodeSubscription::Go,
- true,
- false,
+ cx,
);
let request = LanguageModelRequest {
thread_id: None,
@@ -1976,8 +1973,8 @@ mod tests {
..Default::default()
};
- let stream = model
- .stream_completion(request, &cx.to_async())
+ let stream = provider
+ .stream_completion(&model, request, &cx.to_async())
.await
.unwrap();
drop(stream);
@@ -1998,6 +1995,25 @@ mod tests {
.expect("generated session id should be a u64");
}
+ /// Adds a model to the provider's discovered list, as a successful catalog
+ /// fetch would, and returns it.
+ fn add_discovered_model(
+ provider: &OpenCodeLanguageModelProvider,
+ model: DiscoveredModel,
+ subscription: OpenCodeSubscription,
+ cx: &mut gpui::TestAppContext,
+ ) -> LanguageModel {
+ let language_model = language_model(&model, subscription);
+ provider.state.update(cx, |state, _| {
+ state
+ .discovered_models
+ .entry(subscription)
+ .or_default()
+ .push(model)
+ });
+ language_model
+ }
+
fn test_model(name: &str, protocol: ApiProtocol) -> opencode::Model {
opencode::Model::new(
name.to_string(),
diff --git a/crates/language_models/src/provider/vercel_ai_gateway.rs b/crates/language_models/src/provider/vercel_ai_gateway.rs
index b22c7d3b4a4..cc643a04f9b 100644
--- a/crates/language_models/src/provider/vercel_ai_gateway.rs
+++ b/crates/language_models/src/provider/vercel_ai_gateway.rs
@@ -9,10 +9,10 @@ use http_client::{
use language_model::chat_completion::{ChatCompletionEventMapper, ResponseStreamEvent};
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
- LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
- LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
- ProviderSettingsView, RateLimiter, env_var,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelId,
+ LanguageModelName, LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
+ LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
+ ModelRateLimiters, ProviderSettingsView, RateLimiter, env_var, unavailable_error,
};
use serde::Deserialize;
pub use settings::OpenAiCompatibleModelCapabilities as ModelCapabilities;
@@ -39,6 +39,7 @@ pub struct VercelAiGatewaySettings {
pub struct VercelAiGatewayLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -149,7 +150,11 @@ impl VercelAiGatewayLanguageModelProvider {
}
});
- Self { http_client, state }
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
fn settings(cx: &App) -> &VercelAiGatewaySettings {
@@ -176,14 +181,75 @@ impl VercelAiGatewayLanguageModelProvider {
}
}
- fn create_language_model(&self, model: AvailableModel) -> Arc<dyn LanguageModel> {
- Arc::new(VercelAiGatewayLanguageModel {
- id: LanguageModelId::from(model.name.clone()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
+ /// Every model this provider offers, keyed by id: the default model, then
+ /// the fetched list, then settings entries, each overriding the last.
+ fn vercel_ai_gateway_models(&self, cx: &App) -> BTreeMap<String, AvailableModel> {
+ let mut models = BTreeMap::default();
+
+ let default_model = Self::default_available_model();
+ models.insert(default_model.name.clone(), default_model);
+
+ for model in self.state.read(cx).available_models.clone() {
+ models.insert(model.name.clone(), model);
+ }
+
+ for model in &Self::settings(cx).available_models {
+ models.insert(model.name.clone(), model.clone());
+ }
+
+ models
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<AvailableModel, LanguageModelCompletionError> {
+ self.vercel_ai_gateway_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
+ }
+
+ fn stream_open_ai(
+ &self,
+ request_limiter: &RateLimiter,
+ request: open_ai::Request,
+ cx: &AsyncApp,
+ ) -> BoxFuture<
+ 'static,
+ Result<
+ futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
+ LanguageModelCompletionError,
+ >,
+ > {
+ let http_client = self.http_client.clone();
+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
+ let api_url = VercelAiGatewayLanguageModelProvider::api_url(cx);
+ let extra_headers = VercelAiGatewayLanguageModelProvider::settings(cx)
+ .custom_headers
+ .clone();
+ (state.api_key_state.key(&api_url), api_url, extra_headers)
+ });
+
+ let future = request_limiter.stream(async move {
+ let provider = PROVIDER_NAME;
+ let Some(api_key) = api_key else {
+ return Err(LanguageModelCompletionError::NoApiKey { provider });
+ };
+ let request = open_ai::stream_completion(
+ http_client.as_ref(),
+ provider.0.as_str(),
+ &api_url,
+ &api_key,
+ request,
+ &extra_headers,
+ );
+ let response = request.await.map_err(map_open_ai_error)?;
+ Ok(response)
+ });
+
+ async move { Ok(future.await?.boxed()) }.boxed()
}
}
@@ -208,31 +274,20 @@ impl LanguageModelProvider for VercelAiGatewayLanguageModelProvider {
IconOrSvg::Icon(IconName::AiVercel)
}
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(Self::default_available_model()))
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.vercel_ai_gateway_models(cx)
+ .get(&Self::default_available_model().name)
+ .map(language_model)
}
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
None
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let mut models = BTreeMap::default();
-
- let default_model = Self::default_available_model();
- models.insert(default_model.name.clone(), default_model);
-
- for model in self.state.read(cx).available_models.clone() {
- models.insert(model.name.clone(), model);
- }
-
- for model in &Self::settings(cx).available_models {
- models.insert(model.name.clone(), model.clone());
- }
-
- models
- .into_values()
- .map(|model| self.create_language_model(model))
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.vercel_ai_gateway_models(cx)
+ .values()
+ .map(language_model)
.collect()
}
@@ -259,55 +314,42 @@ impl LanguageModelProvider for VercelAiGatewayLanguageModelProvider {
self.state
.update(cx, |state, cx| state.set_api_key(api_key, cx))
}
-}
-
-pub struct VercelAiGatewayLanguageModel {
- id: LanguageModelId,
- model: AvailableModel,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-impl VercelAiGatewayLanguageModel {
- fn stream_open_ai(
+ fn stream_completion(
&self,
- request: open_ai::Request,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
- LanguageModelCompletionError,
- >,
- > {
- let http_client = self.http_client.clone();
- let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
- let api_url = VercelAiGatewayLanguageModelProvider::api_url(cx);
- let extra_headers = VercelAiGatewayLanguageModelProvider::settings(cx)
- .custom_headers
- .clone();
- (state.api_key_state.key(&api_url), api_url, extra_headers)
- });
-
- let future = self.request_limiter.stream(async move {
- let provider = PROVIDER_NAME;
- let Some(api_key) = api_key else {
- return Err(LanguageModelCompletionError::NoApiKey { provider });
- };
- let request = open_ai::stream_completion(
- http_client.as_ref(),
- provider.0.as_str(),
- &api_url,
- &api_key,
- request,
- &extra_headers,
- );
- let response = request.await.map_err(map_open_ai_error)?;
- Ok(response)
- });
-
- async move { Ok(future.await?.boxed()) }.boxed()
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let request = match crate::provider::open_ai::into_open_ai(
+ request,
+ &config.name,
+ config.capabilities.parallel_tool_calls,
+ config.capabilities.prompt_cache_key,
+ config.max_output_tokens,
+ crate::provider::open_ai::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ None,
+ false,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_open_ai(&request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
}
}
@@ -375,102 +417,32 @@ fn has_tag(tags: &[String], expected: &str) -> bool {
.any(|tag| tag.trim().eq_ignore_ascii_case(expected))
}
-impl LanguageModel for VercelAiGatewayLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(
- self.model
- .display_name
- .clone()
- .unwrap_or_else(|| self.model.name.clone()),
+fn language_model(model: &AvailableModel) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.capabilities.tools,
+ supports_images: model.capabilities.images,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: model.capabilities.tools,
+ any: model.capabilities.tools,
+ none: true,
+ },
+ supports_streaming_tools: true,
+ supports_split_token_display: true,
+ max_output_tokens: model.max_output_tokens,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.name.clone()),
+ LanguageModelName::from(
+ model
+ .display_name
+ .clone()
+ .unwrap_or_else(|| model.name.clone()),
+ ),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("vercel_ai_gateway/{}", model.name),
+ model.max_tokens,
)
}
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.capabilities.tools
- }
-
- fn supports_images(&self) -> bool {
- self.model.capabilities.images
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto => self.model.capabilities.tools,
- LanguageModelToolChoice::Any => self.model.capabilities.tools,
- LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn telemetry_id(&self) -> String {
- format!("vercel_ai_gateway/{}", self.model.name)
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_tokens
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- let request = match crate::provider::open_ai::into_open_ai(
- request,
- &self.model.name,
- self.model.capabilities.parallel_tool_calls,
- self.model.capabilities.prompt_cache_key,
- self.max_output_tokens(),
- crate::provider::open_ai::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
- None,
- false,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_open_ai(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
- }
- .boxed()
- }
}
#[derive(Deserialize)]
diff --git a/crates/language_models/src/provider/x_ai.rs b/crates/language_models/src/provider/x_ai.rs
index 708d05f3830..13c77f1f2d4 100644
--- a/crates/language_models/src/provider/x_ai.rs
+++ b/crates/language_models/src/provider/x_ai.rs
@@ -7,10 +7,11 @@ use http_client::{CustomHeaders, HttpClient};
use language_model::chat_completion::{ChatCompletionEventMapper, ResponseStreamEvent};
use language_model::{
ApiKeyConfiguration, ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel,
- LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelEffortLevel,
+ LanguageModelCompletionError, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
- LanguageModelToolChoice, ProviderSettingsView, RateLimiter, env_var,
+ LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView, RateLimiter, env_var,
+ unavailable_error,
};
pub use settings::XaiAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -35,6 +36,7 @@ pub struct XAiSettings {
pub struct XAiLanguageModelProvider {
http_client: Arc<dyn HttpClient>,
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
pub struct State {
@@ -96,17 +98,94 @@ impl XAiLanguageModelProvider {
}
});
- Self { http_client, state }
+ Self {
+ http_client,
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
- fn create_language_model(&self, model: x_ai::Model) -> Arc<dyn LanguageModel> {
- Arc::new(XAiLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- model,
- state: self.state.clone(),
- http_client: self.http_client.clone(),
- request_limiter: RateLimiter::new(4),
- })
+ /// Every model this provider offers, keyed by id: the built-in models,
+ /// with settings entries added or overriding built-in ones.
+ fn x_ai_models(&self, cx: &App) -> BTreeMap<String, x_ai::Model> {
+ let mut models = BTreeMap::default();
+
+ for model in x_ai::Model::iter() {
+ if !matches!(model, x_ai::Model::Custom { .. }) {
+ models.insert(model.id().to_string(), model);
+ }
+ }
+
+ for model in &Self::settings(cx).available_models {
+ models.insert(
+ model.name.clone(),
+ x_ai::Model::Custom {
+ name: model.name.clone(),
+ display_name: model.display_name.clone(),
+ max_tokens: model.max_tokens,
+ max_output_tokens: model.max_output_tokens,
+ max_completion_tokens: model.max_completion_tokens,
+ supports_images: model.supports_images,
+ supports_tools: model.supports_tools,
+ parallel_tool_calls: model.parallel_tool_calls,
+ },
+ );
+ }
+
+ models
+ }
+
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ cx: &App,
+ ) -> Result<x_ai::Model, LanguageModelCompletionError> {
+ self.x_ai_models(cx)
+ .remove(model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
+ }
+
+ fn stream_chat_completion(
+ &self,
+ request_limiter: &RateLimiter,
+ request: open_ai::Request,
+ cx: &AsyncApp,
+ ) -> BoxFuture<
+ 'static,
+ Result<
+ futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
+ LanguageModelCompletionError,
+ >,
+ > {
+ let http_client = self.http_client.clone();
+
+ let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
+ let api_url = XAiLanguageModelProvider::api_url(cx);
+ let extra_headers = XAiLanguageModelProvider::settings(cx)
+ .custom_headers
+ .clone();
+ (state.api_key_state.key(&api_url), api_url, extra_headers)
+ });
+
+ let future = request_limiter.stream(async move {
+ let provider = PROVIDER_NAME;
+ let Some(api_key) = api_key else {
+ return Err(LanguageModelCompletionError::NoApiKey { provider });
+ };
+ let request = open_ai::stream_completion(
+ http_client.as_ref(),
+ provider.0.as_str(),
+ &api_url,
+ &api_key,
+ request,
+ &extra_headers,
+ );
+ let response = request.await?;
+ Ok(response)
+ });
+
+ async move { Ok(future.await?.boxed()) }.boxed()
}
fn settings(cx: &App) -> &XAiSettings {
@@ -144,43 +223,20 @@ impl LanguageModelProvider for XAiLanguageModelProvider {
IconOrSvg::Icon(IconName::AiXAi)
}
- fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(x_ai::Model::default()))
+ fn default_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.x_ai_models(cx)
+ .get(x_ai::Model::default().id())
+ .map(language_model)
}
- fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_language_model(x_ai::Model::default_fast()))
+ fn default_fast_model(&self, cx: &App) -> Option<LanguageModel> {
+ self.x_ai_models(cx)
+ .get(x_ai::Model::default_fast().id())
+ .map(language_model)
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
- let mut models = BTreeMap::default();
-
- for model in x_ai::Model::iter() {
- if !matches!(model, x_ai::Model::Custom { .. }) {
- models.insert(model.id().to_string(), model);
- }
- }
-
- for model in &Self::settings(cx).available_models {
- models.insert(
- model.name.clone(),
- x_ai::Model::Custom {
- name: model.name.clone(),
- display_name: model.display_name.clone(),
- max_tokens: model.max_tokens,
- max_output_tokens: model.max_output_tokens,
- max_completion_tokens: model.max_completion_tokens,
- supports_images: model.supports_images,
- supports_tools: model.supports_tools,
- parallel_tool_calls: model.parallel_tool_calls,
- },
- );
- }
-
- models
- .into_values()
- .map(|model| self.create_language_model(model))
- .collect()
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
+ self.x_ai_models(cx).values().map(language_model).collect()
}
fn is_authenticated(&self, cx: &App) -> bool {
@@ -205,56 +261,43 @@ impl LanguageModelProvider for XAiLanguageModelProvider {
self.state
.update(cx, |state, cx| state.set_api_key(api_key, cx))
}
-}
-pub struct XAiLanguageModel {
- id: LanguageModelId,
- model: x_ai::Model,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-impl XAiLanguageModel {
fn stream_completion(
&self,
- request: open_ai::Request,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
- LanguageModelCompletionError,
- >,
- > {
- let http_client = self.http_client.clone();
-
- let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
- let api_url = XAiLanguageModelProvider::api_url(cx);
- let extra_headers = XAiLanguageModelProvider::settings(cx)
- .custom_headers
- .clone();
- (state.api_key_state.key(&api_url), api_url, extra_headers)
- });
-
- let future = self.request_limiter.stream(async move {
- let provider = PROVIDER_NAME;
- let Some(api_key) = api_key else {
- return Err(LanguageModelCompletionError::NoApiKey { provider });
- };
- let request = open_ai::stream_completion(
- http_client.as_ref(),
- provider.0.as_str(),
- &api_url,
- &api_key,
- request,
- &extra_headers,
- );
- let response = request.await?;
- Ok(response)
- });
-
- async move { Ok(future.await?.boxed()) }.boxed()
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match cx.update(|cx| self.config(model, cx)) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ let reasoning_effort = reasoning_effort_for_request(&request, &config);
+ let request = match crate::provider::open_ai::into_open_ai(
+ request,
+ config.id(),
+ config.supports_parallel_tool_calls(),
+ config.supports_prompt_cache_key(),
+ config.max_output_tokens(),
+ crate::provider::open_ai::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ reasoning_effort,
+ false,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = self.stream_chat_completion(&request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
}
}
@@ -340,105 +383,24 @@ fn supported_thinking_effort_levels(model: &x_ai::Model) -> Vec<LanguageModelEff
.collect()
}
-impl LanguageModel for XAiLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tool()
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto
- | LanguageModelToolChoice::Any
- | LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.supports_reasoning_effort()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- supported_thinking_effort_levels(&self.model)
- }
-
- fn telemetry_id(&self) -> String {
- format!("x_ai/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- let reasoning_effort = reasoning_effort_for_request(&request, &self.model);
- let request = match crate::provider::open_ai::into_open_ai(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- self.max_output_tokens(),
- crate::provider::open_ai::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
- reasoning_effort,
- false,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_completion(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
- }
- .boxed()
+fn language_model(model: &x_ai::Model) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.supports_tool(),
+ supports_images: model.supports_images(),
+ supports_streaming_tools: true,
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_thinking: model.supports_reasoning_effort(),
+ supported_effort_levels: supported_thinking_effort_levels(model).into(),
+ max_output_tokens: model.max_output_tokens(),
+ supports_split_token_display: true,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("x_ai/{}", model.id()),
+ model.max_token_count(),
+ )
}
}
diff --git a/crates/language_models/src/provider/x_ai_subscribed.rs b/crates/language_models/src/provider/x_ai_subscribed.rs
index a9362c3d304..8b99d7c42ef 100644
--- a/crates/language_models/src/provider/x_ai_subscribed.rs
+++ b/crates/language_models/src/provider/x_ai_subscribed.rs
@@ -1,22 +1,26 @@
use anyhow::{Result, anyhow};
use credentials_provider::CredentialsProvider;
-use futures::future::Shared;
-use gpui::{App, Context, Entity, SharedString, Task, Window};
+use futures::FutureExt as _;
+use futures::future::{BoxFuture, Shared};
+
+use gpui::{App, AsyncApp, Context, Entity, SharedString, Task, Window};
use http_client::HttpClient;
use language_model::{
- AuthenticateError, IconOrSvg, InlineDescription, LanguageModel, LanguageModelProvider,
- LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
- ProviderSettingsView,
+ AuthenticateError, IconOrSvg, InlineDescription, LanguageModel, LanguageModelCompletionError,
+ LanguageModelCompletionStream, LanguageModelProvider, LanguageModelProviderId,
+ LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest, ModelRateLimiters,
+ ProviderSettingsView, unavailable_error,
};
use std::sync::Arc;
use ui::{ConfiguredApiCard, prelude::*};
-use x_ai_subscribed::{PROVIDER_ID, PROVIDER_NAME, State, SuperGrokModel, create_language_model};
+use x_ai_subscribed::{PROVIDER_ID, PROVIDER_NAME, State, SuperGrokModel, language_model};
const SUBSCRIPTION_DESCRIPTION: &str =
"Sign in with your SuperGrok subscription to use Grok models in Zed's agent.";
pub struct XAiSubscribedProvider {
state: Entity<State>,
+ request_limiters: ModelRateLimiters,
}
impl XAiSubscribedProvider {
@@ -26,11 +30,21 @@ impl XAiSubscribedProvider {
cx: &mut App,
) -> Self {
let state = cx.new(|cx| State::new(http_client, credentials_provider, cx));
- Self { state }
+ Self {
+ state,
+ request_limiters: ModelRateLimiters::default(),
+ }
}
- fn create_model(&self, model: SuperGrokModel, cx: &App) -> Arc<dyn LanguageModel> {
- create_language_model(model, &self.state, cx)
+ /// The current configuration of `model`, if this provider still offers it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ ) -> Result<SuperGrokModel, LanguageModelCompletionError> {
+ SuperGrokModel::all()
+ .into_iter()
+ .find(|config| config.id() == model.id.0.as_ref())
+ .ok_or_else(|| unavailable_error(model))
}
}
@@ -55,25 +69,22 @@ impl LanguageModelProvider for XAiSubscribedProvider {
IconOrSvg::Icon(IconName::AiXAi)
}
- fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_model(SuperGrokModel::Grok47, cx))
+ fn default_model(&self, _cx: &App) -> Option<LanguageModel> {
+ Some(language_model(&SuperGrokModel::Grok47))
}
- fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
- Some(self.create_model(SuperGrokModel::GrokBuild01, cx))
+ fn default_fast_model(&self, _cx: &App) -> Option<LanguageModel> {
+ Some(language_model(&SuperGrokModel::GrokBuild01))
}
- fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn provided_models(&self, cx: &App) -> Vec<LanguageModel> {
if !self.is_authenticated(cx) {
return Vec::new();
}
- SuperGrokModel::all()
- .into_iter()
- .map(|model| self.create_model(model, cx))
- .collect()
+ SuperGrokModel::all().iter().map(language_model).collect()
}
- fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
+ fn recommended_models(&self, cx: &App) -> Vec<LanguageModel> {
self.default_model(cx).into_iter().collect()
}
@@ -148,6 +159,21 @@ impl LanguageModelProvider for XAiSubscribedProvider {
Sign in via Settings > AI > LLM Providers to continue."
.into()
}
+
+ fn stream_completion(
+ &self,
+ model: &LanguageModel,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match self.config(model) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ x_ai_subscribed::stream_completion(&config, &self.state, &request_limiter, request, cx)
+ }
}
struct ConfigurationView {
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 6aa0fc2c164..0a0dc1ead51 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -7,13 +7,11 @@ use cloud_llm_client::{
SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, ZED_VERSION_HEADER_NAME,
};
use futures::{
- AsyncBufReadExt, AsyncReadExt as _, FutureExt, Stream, StreamExt,
- future::BoxFuture,
- io::BufReader,
- stream::{self, BoxStream},
+ AsyncBufReadExt, AsyncReadExt as _, FutureExt, Stream, StreamExt, future::BoxFuture,
+ io::BufReader, stream,
};
use google_ai::GoogleModelMode;
-use gpui::{AppContext, AsyncApp, Context, Task};
+use gpui::{App, AppContext, Context, Task};
use http_client::http::{HeaderMap, HeaderValue};
use http_client::{
AsyncBody, HttpClient, HttpClientWithUrl, HttpRequestExt, Method, Response, StatusCode,
@@ -21,11 +19,11 @@ use http_client::{
use language_model::{
ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, CompactionResult, DisabledReason,
GOOGLE_PROVIDER_ID, GOOGLE_PROVIDER_NAME, LanguageModel, LanguageModelCompletionError,
- LanguageModelCompletionEvent, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
- LanguageModelProviderId, LanguageModelProviderName, LanguageModelRequest,
- LanguageModelToolChoice, OPEN_AI_PROVIDER_ID, OPEN_AI_PROVIDER_NAME, ProviderErrorCategory,
- RateLimiter, X_AI_PROVIDER_ID, X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID,
- ZED_CLOUD_PROVIDER_NAME,
+ LanguageModelCompletionEvent, LanguageModelCompletionStream, LanguageModelEffortLevel,
+ LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
+ LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID,
+ OPEN_AI_PROVIDER_NAME, ProviderErrorCategory, RateLimiter, X_AI_PROVIDER_ID,
+ X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME, unavailable_error,
};
use schemars::JsonSchema;
@@ -106,21 +104,12 @@ impl From<ModelMode> for AnthropicModelMode {
}
}
-pub struct CloudLanguageModel<TP: CloudLlmTokenProvider> {
- pub id: LanguageModelId,
- pub model: Arc<cloud_llm_client::LanguageModel>,
- pub token_provider: Arc<TP>,
- pub http_client: Arc<HttpClientWithUrl>,
- pub app_version: Option<Version>,
- pub request_limiter: RateLimiter,
-}
-
pub struct PerformLlmCompletionResponse {
pub response: Response<AsyncBody>,
pub includes_status_messages: bool,
}
-impl<TP: CloudLlmTokenProvider> CloudLanguageModel<TP> {
+impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
pub async fn perform_llm_completion(
http_client: &HttpClientWithUrl,
token_provider: &TP,
@@ -247,148 +236,69 @@ impl<TP: CloudLlmTokenProvider> CloudLanguageModel<TP> {
}
}
-impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
- fn check_data_retention_consent(
- &self,
- cx: &AsyncApp,
- ) -> Result<(), LanguageModelCompletionError> {
- if self.requires_data_retention() && !self.token_provider.has_data_retention_consent(cx) {
- return Err(LanguageModelCompletionError::DataRetentionConsentRequired {
- model_name: self.model.display_name.clone(),
- });
- }
- Ok(())
- }
-
- fn anthropic_request(&self, request: LanguageModelRequest) -> Result<anthropic::Request> {
- let enable_thinking = request.thinking_allowed && self.model.supports_thinking;
- let effort = request
- .thinking_effort
- .as_ref()
- .and_then(|effort| anthropic::Effort::from_str(effort).ok());
- let mut request = into_anthropic(
- request,
- self.model.id.to_string(),
- 1.0,
- self.model.max_output_tokens as u64,
- if enable_thinking && effort.is_some() {
- AnthropicModelMode::AdaptiveThinking
- } else if enable_thinking {
- AnthropicModelMode::Thinking {
- budget_tokens: Some(4_096),
- }
- } else {
- AnthropicModelMode::Default
- },
- AnthropicPromptCacheMode::Automatic,
- &ANTHROPIC_PROVIDER_ID,
- )?;
- if !self.model.supports_fast_mode {
- request.speed = None;
- }
- Ok(request)
- }
-
- fn open_ai_request(
- &self,
- request: LanguageModelRequest,
- ) -> Result<open_ai::responses::Request> {
- let enable_thinking = request.thinking_allowed && self.model.supports_thinking;
- let effort = request
- .thinking_effort
- .as_ref()
- .and_then(|effort| open_ai::ReasoningEffort::from_str(effort).ok())
- .filter(|effort| *effort != open_ai::ReasoningEffort::None);
- let supports_none_reasoning_effort =
- self.model.supported_effort_levels.iter().any(|effort| {
- open_ai::ReasoningEffort::from_str(&effort.value)
- .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
- });
- let mut request = into_open_ai_response(
- request,
- &self.model.id.0,
- self.model.supports_parallel_tool_calls,
- true,
- None,
- None,
- supports_none_reasoning_effort,
- &OPEN_AI_PROVIDER_ID,
- )?;
- if enable_thinking && let Some(effort) = effort {
- request.reasoning = Some(open_ai::responses::ReasoningConfig {
- effort,
- summary: Some(open_ai::responses::ReasoningSummaryMode::Auto),
- });
- }
- Ok(request)
+fn anthropic_request(
+ config: &cloud_llm_client::LanguageModel,
+ request: LanguageModelRequest,
+) -> Result<anthropic::Request> {
+ let enable_thinking = request.thinking_allowed && config.supports_thinking;
+ let effort = request
+ .thinking_effort
+ .as_ref()
+ .and_then(|effort| anthropic::Effort::from_str(effort).ok());
+ let mut request = into_anthropic(
+ request,
+ config.id.to_string(),
+ 1.0,
+ config.max_output_tokens as u64,
+ if enable_thinking && effort.is_some() {
+ AnthropicModelMode::AdaptiveThinking
+ } else if enable_thinking {
+ AnthropicModelMode::Thinking {
+ budget_tokens: Some(4_096),
+ }
+ } else {
+ AnthropicModelMode::Default
+ },
+ AnthropicPromptCacheMode::Automatic,
+ &ANTHROPIC_PROVIDER_ID,
+ )?;
+ if !config.supports_fast_mode {
+ request.speed = None;
}
+ Ok(request)
+}
- fn compact_anthropic(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- let thread_id = request.thread_id.clone();
- let prompt_id = request.prompt_id.clone();
- let app_version = self.app_version.clone();
- let mut request = match into_anthropic(
- request,
- self.model.id.to_string(),
- 1.0,
- self.model.max_output_tokens as u64,
- AnthropicModelMode::Default,
- AnthropicPromptCacheMode::Automatic,
- &ANTHROPIC_PROVIDER_ID,
- ) {
- Ok(request) => request.into_compact_request(),
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- if !self.model.supports_fast_mode {
- request.speed = None;
- }
-
- let http_client = self.http_client.clone();
- let token_provider = self.token_provider.clone();
- let auth_context = token_provider.auth_context(cx);
- let executor = cx.background_executor().clone();
- let future = self.request_limiter.run(async move {
- let PerformLlmCompletionResponse {
- response,
- includes_status_messages,
- } = Self::perform_llm_completion(
- &http_client,
- &*token_provider,
- auth_context,
- app_version,
- CompletionBody {
- thread_id,
- prompt_id,
- provider: cloud_llm_client::LanguageModelProvider::Anthropic,
- model: request.model.clone(),
- provider_request: serde_json::to_value(&request).map_err(|error| {
- LanguageModelCompletionError::SerializeRequest {
- provider: ANTHROPIC_PROVIDER_NAME,
- error,
- }
- })?,
- },
- )
- .await?;
-
- let mut mapper =
- AnthropicEventMapper::new(ANTHROPIC_PROVIDER_NAME, ANTHROPIC_PROVIDER_ID);
- let stream = map_cloud_completion_events(
- Box::pin(response_lines(response, includes_status_messages)),
- &ANTHROPIC_PROVIDER_NAME,
- move |event| mapper.map_event(event),
- );
- let stream = language_model::stream_in_background(stream.boxed(), executor);
- let (context, usage) =
- collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;
- Ok(CompactionResult { context, usage })
+fn open_ai_request(
+ config: &cloud_llm_client::LanguageModel,
+ request: LanguageModelRequest,
+) -> Result<open_ai::responses::Request> {
+ let enable_thinking = request.thinking_allowed && config.supports_thinking;
+ let effort = request
+ .thinking_effort
+ .as_ref()
+ .and_then(|effort| open_ai::ReasoningEffort::from_str(effort).ok())
+ .filter(|effort| *effort != open_ai::ReasoningEffort::None);
+ let supports_none_reasoning_effort = config.supported_effort_levels.iter().any(|effort| {
+ open_ai::ReasoningEffort::from_str(&effort.value)
+ .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
+ });
+ let mut request = into_open_ai_response(
+ request,
+ &config.id.0,
+ config.supports_parallel_tool_calls,
+ true,
+ None,
+ None,
+ supports_none_reasoning_effort,
+ &OPEN_AI_PROVIDER_ID,
+ )?;
+ if enable_thinking && let Some(effort) = effort {
+ request.reasoning = Some(open_ai::responses::ReasoningConfig {
+ effort,
+ summary: Some(open_ai::responses::ReasoningSummaryMode::Auto),
});
- future.boxed()
}
+ Ok(request)
}
fn needs_llm_token_refresh(response: &Response<AsyncBody>) -> bool {
@@ -492,49 +402,260 @@ impl From<ApiError> for LanguageModelCompletionError {
}
}
-impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
- /// Explicit compaction via OpenAI's dedicated compact operation, proxied
- /// through the cloud's `/completions/compact` endpoint.
- fn compact_open_ai(
+/// Describes a model from the Zed cloud model list.
+pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel {
+ use cloud_llm_client::LanguageModelProvider::*;
+ let supports_explicit_compaction = supports_explicit_compaction(model);
+ LanguageModel {
+ upstream_provider_id: Some(match model.provider {
+ Anthropic => ANTHROPIC_PROVIDER_ID,
+ OpenAi => OPEN_AI_PROVIDER_ID,
+ Google => GOOGLE_PROVIDER_ID,
+ XAi => X_AI_PROVIDER_ID,
+ }),
+ upstream_provider_name: Some(match model.provider {
+ Anthropic => ANTHROPIC_PROVIDER_NAME,
+ OpenAi => OPEN_AI_PROVIDER_NAME,
+ Google => GOOGLE_PROVIDER_NAME,
+ XAi => X_AI_PROVIDER_NAME,
+ }),
+ is_latest: model.is_latest,
+ disabled_reason: if model.is_disabled {
+ model.disabled_reason.clone().map(DisabledReason::new)
+ } else {
+ None
+ },
+ requires_data_retention: requires_data_retention(model),
+ refusal_fallback_model_id: if model.id.0.starts_with(anthropic::FABLE_MODEL_ID_PREFIX) {
+ Some(anthropic::FABLE_FALLBACK_MODEL_ID)
+ } else {
+ None
+ },
+ supports_tools: model.supports_tools,
+ supports_images: model.supports_images,
+ supports_thinking: model.supports_thinking,
+ supports_disabling_thinking: model.supports_disabling_thinking,
+ supports_fast_mode: model.supports_fast_mode,
+ supports_server_side_compaction: model.supports_server_side_compaction,
+ supports_explicit_compaction,
+ supports_explicit_compaction_output_limit: model.provider == Anthropic
+ && supports_explicit_compaction,
+ minimum_explicit_compaction_input_tokens: (model.provider == Anthropic
+ && supports_explicit_compaction)
+ .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS),
+ supported_effort_levels: model
+ .supported_effort_levels
+ .iter()
+ .map(|effort_level| LanguageModelEffortLevel {
+ name: effort_level.name.clone().into(),
+ value: effort_level.value.clone().into(),
+ is_default: effort_level.is_default.unwrap_or(false),
+ })
+ .collect(),
+ supports_streaming_tools: model.supports_streaming_tools,
+ tool_choice_support: LanguageModelToolChoiceSupport {
+ auto: true,
+ none: true,
+ any: model.provider != Anthropic || anthropic::supports_forced_tool_use(&model.id.0),
+ },
+ supports_split_token_display: matches!(model.provider, OpenAi | XAi),
+ max_output_tokens: Some(model.max_output_tokens as u64),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id.0.to_string()),
+ LanguageModelName::from(model.display_name.clone()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("zed.dev/{}", model.id),
+ model.max_token_count as u64,
+ )
+ }
+}
+
+fn requires_data_retention(model: &cloud_llm_client::LanguageModel) -> bool {
+ // Anthropic cannot offer Fable models with Zero Data Retention
+ model.id.0.starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
+}
+
+fn supports_explicit_compaction(model: &cloud_llm_client::LanguageModel) -> bool {
+ matches!(
+ model.provider,
+ cloud_llm_client::LanguageModelProvider::OpenAi
+ | cloud_llm_client::LanguageModelProvider::Anthropic
+ ) && model.supports_server_side_compaction
+}
+
+pub struct CloudModelProvider<TP: CloudLlmTokenProvider> {
+ token_provider: Arc<TP>,
+ http_client: Arc<HttpClientWithUrl>,
+ app_version: Option<Version>,
+ models: Vec<Arc<cloud_llm_client::LanguageModel>>,
+ default_model: Option<Arc<cloud_llm_client::LanguageModel>>,
+ default_fast_model: Option<Arc<cloud_llm_client::LanguageModel>>,
+ recommended_models: Vec<Arc<cloud_llm_client::LanguageModel>>,
+ request_limiters: ModelRateLimiters,
+}
+
+impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
+ pub fn new(
+ token_provider: Arc<TP>,
+ http_client: Arc<HttpClientWithUrl>,
+ app_version: Option<Version>,
+ ) -> Self {
+ Self {
+ token_provider,
+ http_client,
+ app_version,
+ models: Vec::new(),
+ default_model: None,
+ default_fast_model: None,
+ recommended_models: Vec::new(),
+ request_limiters: ModelRateLimiters::default(),
+ }
+ }
+
+ pub fn refresh_models(&self, cx: &mut Context<Self>) -> Task<Result<()>> {
+ let http_client = self.http_client.clone();
+ let token_provider = self.token_provider.clone();
+ cx.spawn(async move |this, cx| {
+ let auth_context = token_provider.auth_context(cx);
+ let response =
+ Self::fetch_models_request(&http_client, &*token_provider, auth_context).await?;
+ this.update(cx, |this, cx| {
+ this.update_models(response);
+ cx.notify();
+ })
+ })
+ }
+
+ async fn fetch_models_request(
+ http_client: &HttpClientWithUrl,
+ token_provider: &TP,
+ auth_context: TP::AuthContext,
+ ) -> Result<ListModelsResponse> {
+ let url = http_client.build_zed_llm_url("/models", &[])?;
+ let mut response =
+ authenticated_llm_request(http_client, token_provider, auth_context, |token| {
+ Ok(http_client::Request::builder()
+ .method(Method::GET)
+ .header(CLIENT_SUPPORTS_X_AI_HEADER_NAME, "true")
+ .uri(url.as_ref())
+ .header("Authorization", format!("Bearer {token}"))
+ .body(AsyncBody::empty())?)
+ })
+ .await
+ .context("failed to send list models request")?;
+
+ if response.status().is_success() {
+ let mut body = String::new();
+ response.body_mut().read_to_string(&mut body).await?;
+ Ok(serde_json::from_str(&body)?)
+ } else {
+ let mut body = String::new();
+ response.body_mut().read_to_string(&mut body).await?;
+ anyhow::bail!(
+ "error listing models.\nStatus: {:?}\nBody: {body}",
+ response.status(),
+ );
+ }
+ }
+
+ pub fn update_models(&mut self, response: ListModelsResponse) {
+ let models: Vec<_> = response.models.into_iter().map(Arc::new).collect();
+
+ self.default_model = models
+ .iter()
+ .find(|model| {
+ response
+ .default_model
+ .as_ref()
+ .is_some_and(|default_model_id| &model.id == default_model_id)
+ })
+ .cloned();
+ self.default_fast_model = models
+ .iter()
+ .find(|model| {
+ response
+ .default_fast_model
+ .as_ref()
+ .is_some_and(|default_fast_model_id| &model.id == default_fast_model_id)
+ })
+ .cloned();
+ self.recommended_models = response
+ .recommended_models
+ .iter()
+ .filter_map(|id| models.iter().find(|model| &model.id == id))
+ .cloned()
+ .collect();
+ self.models = models;
+ }
+
+ pub fn clear_models(&mut self) {
+ self.models.clear();
+ self.default_model = None;
+ self.default_fast_model = None;
+ self.recommended_models.clear();
+ }
+
+ /// The current configuration of `model`, if the model list still contains it.
+ fn config(
+ &self,
+ model: &LanguageModel,
+ ) -> Result<Arc<cloud_llm_client::LanguageModel>, LanguageModelCompletionError> {
+ self.models
+ .iter()
+ .find(|config| config.id.0.as_ref() == model.id.0.as_ref())
+ .cloned()
+ .ok_or_else(|| unavailable_error(model))
+ }
+
+ fn check_data_retention_consent(
+ &self,
+ config: &cloud_llm_client::LanguageModel,
+ cx: &App,
+ ) -> Result<(), LanguageModelCompletionError> {
+ if requires_data_retention(config) && !self.token_provider.has_data_retention_consent(cx) {
+ return Err(LanguageModelCompletionError::DataRetentionConsentRequired {
+ model_name: config.display_name.clone(),
+ });
+ }
+ Ok(())
+ }
+
+ fn compact_anthropic(
&self,
+ config: &cloud_llm_client::LanguageModel,
+ request_limiter: &RateLimiter,
request: LanguageModelRequest,
- cx: &AsyncApp,
+ cx: &App,
) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
- let model_provider = self.model.provider;
- let provider_name = provider_name(&self.model.provider);
- let supports_none_reasoning_effort =
- self.model.supported_effort_levels.iter().any(|effort| {
- open_ai::ReasoningEffort::from_str(&effort.value)
- .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
- });
- // Cloud proxies to OpenAI's own infrastructure, so the resulting
- // compaction state is owned by (and interchangeable with) OpenAI
- // proper, not by the cloud transport.
- let request = match into_open_ai_response(
+ let mut request = match into_anthropic(
request,
- &self.model.id.0,
- self.model.supports_parallel_tool_calls,
- true,
- None,
- None,
- supports_none_reasoning_effort,
- &OPEN_AI_PROVIDER_ID,
+ config.id.to_string(),
+ 1.0,
+ config.max_output_tokens as u64,
+ AnthropicModelMode::Default,
+ AnthropicPromptCacheMode::Automatic,
+ &ANTHROPIC_PROVIDER_ID,
) {
- Ok(request) => request,
+ Ok(request) => request.into_compact_request(),
Err(error) => return async move { Err(error.into()) }.boxed(),
};
- let compact_request = request.into_compact_request();
+ if !config.supports_fast_mode {
+ request.speed = None;
+ }
+
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
- let future = self.request_limiter.run(async move {
+ let executor = cx.background_executor().clone();
+ let future = request_limiter.run(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
- } = Self::perform_llm_compaction(
+ } = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
@@ -542,11 +663,11 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
CompletionBody {
thread_id,
prompt_id,
- provider: model_provider,
- model: compact_request.model.clone(),
- provider_request: serde_json::to_value(compact_request).map_err(|error| {
+ provider: cloud_llm_client::LanguageModelProvider::Anthropic,
+ model: request.model.clone(),
+ provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
- provider: provider_name.clone(),
+ provider: ANTHROPIC_PROVIDER_NAME,
error,
}
})?,
@@ -554,154 +675,122 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
)
.await?;
- let events = response_lines::<open_ai::responses::CompactedResponse>(
- response,
- includes_status_messages,
+ let mut mapper =
+ AnthropicEventMapper::new(ANTHROPIC_PROVIDER_NAME, ANTHROPIC_PROVIDER_ID);
+ let stream = map_cloud_completion_events(
+ Box::pin(response_lines(response, includes_status_messages)),
+ &ANTHROPIC_PROVIDER_NAME,
+ move |event| mapper.map_event(event),
);
- futures::pin_mut!(events);
- while let Some(event) = events.next().await {
- match event.map_err(|error| error.into_completion_error(provider_name.clone()))? {
- CompletionEvent::Event(response) => {
- let usage = token_usage_from_response_usage(&response.usage);
- let context = response
- .into_compacted_context(OPEN_AI_PROVIDER_ID)
- .map_err(LanguageModelCompletionError::Other)?;
- return Ok(CompactionResult { context, usage });
- }
- CompletionEvent::Status(_) => {}
- }
- }
-
- Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
- provider: provider_name,
- })
+ let stream = language_model::stream_in_background(stream.boxed(), executor);
+ let (context, usage) =
+ collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;
+ Ok(CompactionResult { context, usage })
});
future.boxed()
}
-}
-
-impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<TP> {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name.clone())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn upstream_provider_id(&self) -> LanguageModelProviderId {
- use cloud_llm_client::LanguageModelProvider::*;
- match self.model.provider {
- Anthropic => ANTHROPIC_PROVIDER_ID,
- OpenAi => OPEN_AI_PROVIDER_ID,
- Google => GOOGLE_PROVIDER_ID,
- XAi => X_AI_PROVIDER_ID,
- }
- }
-
- fn upstream_provider_name(&self) -> LanguageModelProviderName {
- use cloud_llm_client::LanguageModelProvider::*;
- match self.model.provider {
- Anthropic => ANTHROPIC_PROVIDER_NAME,
- OpenAi => OPEN_AI_PROVIDER_NAME,
- Google => GOOGLE_PROVIDER_NAME,
- XAi => X_AI_PROVIDER_NAME,
- }
- }
-
- fn is_latest(&self) -> bool {
- self.model.is_latest
- }
-
- fn is_disabled(&self) -> Option<DisabledReason> {
- if self.model.is_disabled {
- self.model.disabled_reason.clone().map(DisabledReason::new)
- } else {
- None
- }
- }
-
- fn requires_data_retention(&self) -> bool {
- // Anthropic cannot offer Fable models with Zero Data Retention
- self.id
- .0
- .as_ref()
- .starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
- }
-
- fn refusal_fallback_model_id(&self) -> Option<&'static str> {
- if self
- .id
- .0
- .as_ref()
- .starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
- {
- Some(anthropic::FABLE_FALLBACK_MODEL_ID)
- } else {
- None
- }
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tools
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.supports_thinking
- }
-
- fn supports_disabling_thinking(&self) -> bool {
- self.model.supports_disabling_thinking
- }
-
- fn supports_fast_mode(&self) -> bool {
- self.model.supports_fast_mode
- }
-
- fn supports_server_side_compaction(&self) -> bool {
- self.model.supports_server_side_compaction
- }
-
- fn supports_explicit_compaction(&self) -> bool {
- matches!(
- self.model.provider,
- cloud_llm_client::LanguageModelProvider::OpenAi
- | cloud_llm_client::LanguageModelProvider::Anthropic
- ) && self.model.supports_server_side_compaction
- }
+ /// Explicit compaction via OpenAI's dedicated compact operation, proxied
+ /// through the cloud's `/completions/compact` endpoint.
+ fn compact_open_ai(
+ &self,
+ config: &cloud_llm_client::LanguageModel,
+ request_limiter: &RateLimiter,
+ request: LanguageModelRequest,
+ cx: &App,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let thread_id = request.thread_id.clone();
+ let prompt_id = request.prompt_id.clone();
+ let app_version = self.app_version.clone();
+ let model_provider = config.provider;
+ let provider_name = provider_name(&config.provider);
+ let supports_none_reasoning_effort = config.supported_effort_levels.iter().any(|effort| {
+ open_ai::ReasoningEffort::from_str(&effort.value)
+ .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
+ });
+ // Cloud proxies to OpenAI's own infrastructure, so the resulting
+ // compaction state is owned by (and interchangeable with) OpenAI
+ // proper, not by the cloud transport.
+ let request = match into_open_ai_response(
+ request,
+ &config.id.0,
+ config.supports_parallel_tool_calls,
+ true,
+ None,
+ None,
+ supports_none_reasoning_effort,
+ &OPEN_AI_PROVIDER_ID,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let compact_request = request.into_compact_request();
+ let http_client = self.http_client.clone();
+ let token_provider = self.token_provider.clone();
+ let auth_context = token_provider.auth_context(cx);
+ let future = request_limiter.run(async move {
+ let PerformLlmCompletionResponse {
+ response,
+ includes_status_messages,
+ } = Self::perform_llm_compaction(
+ &http_client,
+ &*token_provider,
+ auth_context,
+ app_version,
+ CompletionBody {
+ thread_id,
+ prompt_id,
+ provider: model_provider,
+ model: compact_request.model.clone(),
+ provider_request: serde_json::to_value(compact_request).map_err(|error| {
+ LanguageModelCompletionError::SerializeRequest {
+ provider: provider_name.clone(),
+ error,
+ }
+ })?,
+ },
+ )
+ .await?;
- fn supports_explicit_compaction_output_limit(&self) -> bool {
- self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic
- && self.supports_explicit_compaction()
- }
+ let events = response_lines::<open_ai::responses::CompactedResponse>(
+ response,
+ includes_status_messages,
+ );
+ futures::pin_mut!(events);
+ while let Some(event) = events.next().await {
+ match event.map_err(|error| error.into_completion_error(provider_name.clone()))? {
+ CompletionEvent::Event(response) => {
+ let usage = token_usage_from_response_usage(&response.usage);
+ let context = response
+ .into_compacted_context(OPEN_AI_PROVIDER_ID)
+ .map_err(LanguageModelCompletionError::Other)?;
+ return Ok(CompactionResult { context, usage });
+ }
+ CompletionEvent::Status(_) => {}
+ }
+ }
- fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
- (self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic
- && self.supports_explicit_compaction())
- .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
+ Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
+ provider: provider_name,
+ })
+ });
+ future.boxed()
}
-
- fn compact(
+ /// Compacts `request`'s conversation with the listed model `model`.
+ pub fn compact(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
- cx: &AsyncApp,
+ cx: &App,
) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- if let Err(error) = self.check_data_retention_consent(cx) {
+ let config = match self.config(model) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
- if !self.supports_explicit_compaction() {
+ if !supports_explicit_compaction(&config) {
return async {
Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
"this cloud model does not support explicit compaction"
@@ -710,10 +799,12 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.boxed();
}
- match self.model.provider {
- cloud_llm_client::LanguageModelProvider::OpenAi => self.compact_open_ai(request, cx),
+ match config.provider {
+ cloud_llm_client::LanguageModelProvider::OpenAi => {
+ self.compact_open_ai(&config, &request_limiter, request, cx)
+ }
cloud_llm_client::LanguageModelProvider::Anthropic => {
- self.compact_anthropic(request, cx)
+ self.compact_anthropic(&config, &request_limiter, request, cx)
}
// Unreachable while the `supports_explicit_compaction` guard
// above holds, but a provider mismatch should degrade to the
@@ -728,64 +819,27 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
}
}
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- self.model
- .supported_effort_levels
- .iter()
- .map(|effort_level| LanguageModelEffortLevel {
- name: effort_level.name.clone().into(),
- value: effort_level.value.clone().into(),
- is_default: effort_level.is_default.unwrap_or(false),
- })
- .collect()
- }
-
- fn supports_streaming_tools(&self) -> bool {
- self.model.supports_streaming_tools
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto | LanguageModelToolChoice::None => true,
- LanguageModelToolChoice::Any => {
- self.model.provider != cloud_llm_client::LanguageModelProvider::Anthropic
- || anthropic::supports_forced_tool_use(self.id.0.as_ref())
- }
- }
- }
-
- fn supports_split_token_display(&self) -> bool {
- use cloud_llm_client::LanguageModelProvider::*;
- matches!(self.model.provider, OpenAi | XAi)
- }
-
- fn telemetry_id(&self) -> String {
- format!("zed.dev/{}", self.model.id)
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count as u64
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- Some(self.model.max_output_tokens as u64)
- }
-
- fn count_input_tokens(
+ /// Counts `request`'s input tokens for the listed model `model`, or
+ /// returns `None` when its upstream provider cannot count them.
+ pub fn count_input_tokens(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
- cx: &AsyncApp,
+ cx: &App,
) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
- if let Err(error) = self.check_data_retention_consent(cx) {
+ let config = match self.config(model) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
use cloud_llm_client::LanguageModelProvider;
- let provider_request = match self.model.provider {
- LanguageModelProvider::Anthropic => self
- .anthropic_request(request)
+ let provider_request = match config.provider {
+ LanguageModelProvider::Anthropic => anthropic_request(&config, request)
.and_then(|request| Ok(serde_json::to_value(request)?)),
- LanguageModelProvider::OpenAi => self
- .open_ai_request(request)
+ LanguageModelProvider::OpenAi => open_ai_request(&config, request)
.and_then(|request| Ok(serde_json::to_value(request)?)),
_ => return async { Ok(None) }.boxed(),
};
@@ -793,8 +847,8 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
Ok(provider_request) => CompletionBody {
thread_id: None,
prompt_id: None,
- provider: self.model.provider,
- model: self.model.id.to_string(),
+ provider: config.provider,
+ model: config.id.to_string(),
provider_request,
},
Err(error) => return async move { Err(error.into()) }.boxed(),
@@ -803,7 +857,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let app_version = self.app_version.clone();
- self.request_limiter
+ request_limiter
.run(async move {
let mut response = Self::perform_llm_request(
"/count_tokens",
@@ -840,33 +894,35 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.boxed()
}
- fn stream_completion(
+ /// Streams a completion of `request` from the listed model `model`.
+ pub fn stream_completion(
&self,
+ model: &LanguageModel,
request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
- LanguageModelCompletionError,
- >,
- > {
- if let Err(error) = self.check_data_retention_consent(cx) {
+ cx: &App,
+ ) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
+ {
+ let config = match self.config(model) {
+ Ok(config) => config,
+ Err(error) => return async move { Err(error) }.boxed(),
+ };
+ let request_limiter = self.request_limiters.for_model(&model.id);
+ if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
let mut request = request;
if request.max_output_tokens.is_some() {
request.max_output_tokens =
- request.effective_max_output_tokens(self.max_output_tokens());
+ request.effective_max_output_tokens(Some(config.max_output_tokens as u64));
}
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
- let provider_name = provider_name(&self.model.provider);
- match self.model.provider {
+ let provider_name = provider_name(&config.provider);
+ match config.provider {
cloud_llm_client::LanguageModelProvider::Anthropic => {
- let request = match self.anthropic_request(request) {
+ let request = match anthropic_request(&config, request) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
@@ -875,7 +931,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
@@ -916,14 +972,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
cloud_llm_client::LanguageModelProvider::OpenAi => {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
- let request = match self.open_ai_request(request) {
+ let request = match open_ai_request(&config, request) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
@@ -965,8 +1021,8 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let token_provider = self.token_provider.clone();
let request = match into_open_ai(
request,
- &self.model.id.0,
- self.model.supports_parallel_tool_calls,
+ &config.id.0,
+ config.supports_parallel_tool_calls,
false,
None,
ChatCompletionMaxTokensParameter::MaxCompletionTokens,
@@ -978,7 +1034,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
};
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
@@ -1019,13 +1075,12 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let request =
- match into_google(request, self.model.id.to_string(), GoogleModelMode::Default)
- {
+ match into_google(request, config.id.to_string(), GoogleModelMode::Default) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
- let future = self.request_limiter.stream(async move {
+ let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
@@ -1060,131 +1115,6 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
}
}
}
-}
-
-pub struct CloudModelProvider<TP: CloudLlmTokenProvider> {
- token_provider: Arc<TP>,
- http_client: Arc<HttpClientWithUrl>,
- app_version: Option<Version>,
- models: Vec<Arc<cloud_llm_client::LanguageModel>>,
- default_model: Option<Arc<cloud_llm_client::LanguageModel>>,
- default_fast_model: Option<Arc<cloud_llm_client::LanguageModel>>,
- recommended_models: Vec<Arc<cloud_llm_client::LanguageModel>>,
-}
-
-impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
- pub fn new(
- token_provider: Arc<TP>,
- http_client: Arc<HttpClientWithUrl>,
- app_version: Option<Version>,
- ) -> Self {
- Self {
- token_provider,
- http_client,
- app_version,
- models: Vec::new(),
- default_model: None,
- default_fast_model: None,
- recommended_models: Vec::new(),
- }
- }
-
- pub fn refresh_models(&self, cx: &mut Context<Self>) -> Task<Result<()>> {
- let http_client = self.http_client.clone();
- let token_provider = self.token_provider.clone();
- cx.spawn(async move |this, cx| {
- let auth_context = token_provider.auth_context(cx);
- let response =
- Self::fetch_models_request(&http_client, &*token_provider, auth_context).await?;
- this.update(cx, |this, cx| {
- this.update_models(response);
- cx.notify();
- })
- })
- }
-
- async fn fetch_models_request(
- http_client: &HttpClientWithUrl,
- token_provider: &TP,
- auth_context: TP::AuthContext,
- ) -> Result<ListModelsResponse> {
- let url = http_client.build_zed_llm_url("/models", &[])?;
- let mut response =
- authenticated_llm_request(http_client, token_provider, auth_context, |token| {
- Ok(http_client::Request::builder()
- .method(Method::GET)
- .header(CLIENT_SUPPORTS_X_AI_HEADER_NAME, "true")
- .uri(url.as_ref())
- .header("Authorization", format!("Bearer {token}"))
- .body(AsyncBody::empty())?)
- })
- .await
- .context("failed to send list models request")?;
-
- if response.status().is_success() {
- let mut body = String::new();
- response.body_mut().read_to_string(&mut body).await?;
- Ok(serde_json::from_str(&body)?)
- } else {
- let mut body = String::new();
- response.body_mut().read_to_string(&mut body).await?;
- anyhow::bail!(
- "error listing models.\nStatus: {:?}\nBody: {body}",
- response.status(),
- );
- }
- }
-
- pub fn update_models(&mut self, response: ListModelsResponse) {
- let models: Vec<_> = response.models.into_iter().map(Arc::new).collect();
-
- self.default_model = models
- .iter()
- .find(|model| {
- response
- .default_model
- .as_ref()
- .is_some_and(|default_model_id| &model.id == default_model_id)
- })
- .cloned();
- self.default_fast_model = models
- .iter()
- .find(|model| {
- response
- .default_fast_model
- .as_ref()
- .is_some_and(|default_fast_model_id| &model.id == default_fast_model_id)
- })
- .cloned();
- self.recommended_models = response
- .recommended_models
- .iter()
- .filter_map(|id| models.iter().find(|model| &model.id == id))
- .cloned()
- .collect();
- self.models = models;
- }
-
- pub fn clear_models(&mut self) {
- self.models.clear();
- self.default_model = None;
- self.default_fast_model = None;
- self.recommended_models.clear();
- }
-
- pub fn create_model(
- &self,
- model: &Arc<cloud_llm_client::LanguageModel>,
- ) -> Arc<dyn LanguageModel> {
- Arc::new(CloudLanguageModel::<TP> {
- id: LanguageModelId::from(model.id.0.to_string()),
- model: model.clone(),
- token_provider: self.token_provider.clone(),
- http_client: self.http_client.clone(),
- app_version: self.app_version.clone(),
- request_limiter: RateLimiter::new(4),
- })
- }
pub fn models(&self) -> &[Arc<cloud_llm_client::LanguageModel>] {
&self.models
@@ -1207,7 +1137,7 @@ pub fn map_cloud_completion_events<T, F>(
stream: Pin<Box<dyn Stream<Item = Result<CompletionEvent<T>, ResponseStreamError>> + Send>>,
provider: &LanguageModelProviderName,
mut map_callback: F,
-) -> BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
+) -> LanguageModelCompletionStream
where
T: DeserializeOwned + 'static,
F: FnMut(T) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
@@ -1350,6 +1280,7 @@ mod tests {
};
use serde_json::json;
use std::sync::Mutex;
+ use std::sync::atomic::{AtomicBool, Ordering};
#[gpui::test]
async fn cloud_google_completion_emits_stop(cx: &mut gpui::TestAppContext) {
@@ -1358,7 +1289,7 @@ mod tests {
("MAX_TOKENS", StopReason::MaxTokens),
("SAFETY", StopReason::Refusal),
] {
- let model = cloud_google_test_model(vec![
+ let (provider, model) = cloud_google_test_model(vec![
json!({"event": {"candidates": [{
"content": {"role": "model", "parts": [{"text": "Hello"}]}
}]}}),
@@ -1371,8 +1302,10 @@ mod tests {
}}),
json!({"status": "stream_ended"}),
]);
- let mut stream = model
- .stream_completion(LanguageModelRequest::default(), &cx.to_async())
+ let mut stream = cx
+ .update(|cx| {
+ provider.stream_completion(&model, LanguageModelRequest::default(), cx)
+ })
.await
.unwrap();
@@ -1411,9 +1344,11 @@ mod tests {
if stream_ended {
events.push(json!({"status": "stream_ended"}));
}
- let model = cloud_google_test_model(events);
- let mut stream = model
- .stream_completion(LanguageModelRequest::default(), &cx.to_async())
+ let (provider, model) = cloud_google_test_model(events);
+ let mut stream = cx
+ .update(|cx| {
+ provider.stream_completion(&model, LanguageModelRequest::default(), cx)
+ })
.await
.unwrap();
@@ -1447,9 +1382,11 @@ mod tests {
if stream_ended {
events.push(json!({"status": "stream_ended"}));
}
- let model = cloud_google_test_model(events);
- let mut stream = model
- .stream_completion(LanguageModelRequest::default(), &cx.to_async())
+ let (provider, model) = cloud_google_test_model(events);
+ let mut stream = cx
+ .update(|cx| {
+ provider.stream_completion(&model, LanguageModelRequest::default(), cx)
+ })
.await
.unwrap();
@@ -1527,12 +1464,16 @@ mod tests {
)))?)
}
});
- let model = cloud_test_model(http_client);
- assert!(model.supports_explicit_compaction());
- assert!(!model.supports_explicit_compaction_output_limit());
+ let language_model = language_model(&cloud_test_config());
+ assert!(language_model.supports_explicit_compaction());
+ assert!(!language_model.supports_explicit_compaction_output_limit());
+ let (provider, model) = cloud_test_model(http_client);
let request = compact_test_request();
- let result = model.compact(request, &cx.to_async()).await.unwrap();
+ let result = cx
+ .update(|cx| provider.compact(&model, request, cx))
+ .await
+ .unwrap();
assert_eq!(
result.usage,
@@ -1675,11 +1616,14 @@ mod tests {
.body(AsyncBody::from(format!("{response_lines}\n")))?)
}
});
- let model = cloud_anthropic_test_model(http_client);
+ let (provider, model) = cloud_anthropic_test_model(http_client);
let mut request = compact_test_request();
request.max_output_tokens = Some(8192);
- let result = model.compact(request, &cx.to_async()).await.unwrap();
+ let result = cx
+ .update(|cx| provider.compact(&model, request, cx))
+ .await
+ .unwrap();
assert_eq!(
result.usage,
@@ -1756,10 +1700,10 @@ mod tests {
})
)))?)
});
- let model = cloud_test_model(http_client);
+ let (provider, model) = cloud_test_model(http_client);
- let error = model
- .compact(compact_test_request(), &cx.to_async())
+ let error = cx
+ .update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
.unwrap_err();
@@ -1957,10 +1901,10 @@ mod tests {
async fn cloud_transport_errors_include_hostname(cx: &mut gpui::TestAppContext) {
let http_client =
FakeHttpClient::create(|_| async move { Err(anyhow::anyhow!("DNS lookup failed")) });
- let model = cloud_test_model(http_client);
+ let (provider, model) = cloud_test_model(http_client);
- let error = model
- .compact(compact_test_request(), &cx.to_async())
+ let error = cx
+ .update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
.expect_err("request should fail");
@@ -2004,17 +1948,43 @@ mod tests {
#[test]
fn cloud_anthropic_supports_explicit_compaction_after_minimum_input() {
- let model = cloud_anthropic_test_model(FakeHttpClient::with_404_response());
+ let model = language_model(&cloud_anthropic_test_config());
- assert!(model.supports_explicit_compaction());
- assert!(model.supports_explicit_compaction_output_limit());
- assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));
+ assert!(model.supports_explicit_compaction);
+ assert!(model.supports_explicit_compaction_output_limit);
assert_eq!(
model.minimum_explicit_compaction_input_tokens(),
Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
);
}
+ #[test]
+ fn config_resolves_current_model_list_and_rejects_unlisted_models() {
+ let listed = language_model(&cloud_test_config());
+ let mut provider = test_provider(
+ FakeHttpClient::with_404_response(),
+ vec![cloud_test_config(), cloud_anthropic_test_config()],
+ );
+ assert_eq!(provider.config(&listed).unwrap().id.0.as_ref(), "gpt-5.4");
+
+ let mut refreshed = cloud_test_config();
+ refreshed.max_output_tokens = 64_000;
+ provider.update_models(ListModelsResponse {
+ models: vec![refreshed],
+ default_model: None,
+ default_fast_model: None,
+ recommended_models: Vec::new(),
+ });
+ assert_eq!(provider.config(&listed).unwrap().max_output_tokens, 64_000);
+
+ let unlisted = language_model(&cloud_anthropic_test_config());
+ assert!(matches!(
+ provider.config(&unlisted),
+ Err(LanguageModelCompletionError::ModelUnavailable { provider, model })
+ if provider == PROVIDER_NAME && model == unlisted.id
+ ));
+ }
+
#[gpui::test]
async fn hosted_input_counts_use_authenticated_count_endpoint(cx: &mut gpui::TestAppContext) {
for anthropic in [true, false] {
@@ -2044,7 +2014,7 @@ mod tests {
}
}
});
- let model = if anthropic {
+ let (provider, model) = if anthropic {
cloud_anthropic_test_model(http_client)
} else {
cloud_test_model(http_client)
@@ -2056,8 +2026,7 @@ mod tests {
},
));
assert_eq!(
- model
- .count_input_tokens(request, &cx.to_async())
+ cx.update(|cx| provider.count_input_tokens(&model, request, cx))
.await
.unwrap(),
Some(731)
@@ -2071,13 +2040,13 @@ mod tests {
cx: &mut gpui::TestAppContext,
) {
for (status, body) in [(429, "rate limit"), (200, r#"{"tokens":-1}"#), (200, "{}")] {
- let model = cloud_test_model(FakeHttpClient::create(move |_| async move {
+ let (provider, model) = cloud_test_model(FakeHttpClient::create(move |_| async move {
Ok(Response::builder()
.status(status)
.body(AsyncBody::from(body))?)
}));
- let error = model
- .count_input_tokens(compact_test_request(), &cx.to_async())
+ let error = cx
+ .update(|cx| provider.count_input_tokens(&model, compact_test_request(), cx))
.await
.unwrap_err();
if status == 200 {
@@ -2095,13 +2064,16 @@ mod tests {
));
}
}
- let mut model = cloud_test_model(FakeHttpClient::create(|_| async {
- panic!("unsupported provider must not send a counting request");
- }));
- Arc::make_mut(&mut model.model).provider = cloud_llm_client::LanguageModelProvider::XAi;
+ let mut config = cloud_test_config();
+ config.provider = cloud_llm_client::LanguageModelProvider::XAi;
+ let (provider, model) = bind_test_model(
+ FakeHttpClient::create(|_| async {
+ panic!("unsupported provider must not send a counting request");
+ }),
+ config,
+ );
assert_eq!(
- model
- .count_input_tokens(compact_test_request(), &cx.to_async())
+ cx.update(|cx| provider.count_input_tokens(&model, compact_test_request(), cx))
.await
.unwrap(),
None
@@ -2110,16 +2082,18 @@ mod tests {
#[test]
fn opus_55_hosted_requests_use_adaptive_thinking_with_prefix_binding() -> Result<()> {
- let mut model = cloud_anthropic_test_model(FakeHttpClient::with_404_response());
- Arc::make_mut(&mut model.model).id =
- cloud_llm_client::LanguageModelId("claude-opus-5-5".into());
+ let mut config = cloud_anthropic_test_config();
+ config.id = cloud_llm_client::LanguageModelId("claude-opus-5-5".into());
for effort in [None, Some("high")] {
- let request = model.anthropic_request(LanguageModelRequest {
- thinking_allowed: true,
- thinking_effort: effort.map(str::to_string),
- temperature: Some(0.25),
- ..Default::default()
- })?;
+ let request = anthropic_request(
+ &config,
+ LanguageModelRequest {
+ thinking_allowed: true,
+ thinking_effort: effort.map(str::to_string),
+ temperature: Some(0.25),
+ ..Default::default()
+ },
+ )?;
let body = serde_json::to_value(request)?;
assert_eq!(body["thinking"]["type"], "adaptive");
assert_eq!(
@@ -2137,22 +2111,24 @@ mod tests {
cx: &mut gpui::TestAppContext,
) {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
- let mut model = cloud_anthropic_test_model(FakeHttpClient::create({
- let calls = calls.clone();
- move |request| {
- calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
- async move {
- assert_eq!(request.uri().path(), "/count_tokens");
- Ok(Response::builder()
- .status(200)
- .body(AsyncBody::from(r#"{"tokens":731}"#))?)
+ let mut config = cloud_anthropic_test_config();
+ config.id = cloud_llm_client::LanguageModelId("claude-fable-5-1".into());
+ assert!(language_model(&config).requires_data_retention());
+ let (provider, model) = bind_test_model(
+ FakeHttpClient::create({
+ let calls = calls.clone();
+ move |request| {
+ calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ async move {
+ assert_eq!(request.uri().path(), "/count_tokens");
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from(r#"{"tokens":731}"#))?)
+ }
}
- }
- }));
- model.id = LanguageModelId::from("claude-fable-5-1".to_string());
- Arc::make_mut(&mut model.model).id =
- cloud_llm_client::LanguageModelId("claude-fable-5-1".into());
- assert!(model.requires_data_retention());
+ }),
+ config,
+ );
let mut request = compact_test_request();
request.messages[0].content.push(MessageContent::Image(
language_model::LanguageModelImage {
@@ -2160,29 +2136,28 @@ mod tests {
},
));
assert!(matches!(
- model
- .count_input_tokens(request.clone(), &cx.to_async())
+ cx.update(|cx| provider.count_input_tokens(&model, request.clone(), cx))
.await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert!(matches!(
- model.compact(request.clone(), &cx.to_async()).await,
+ cx.update(|cx| provider.compact(&model, request.clone(), cx))
+ .await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert!(matches!(
- model
- .stream_completion(request.clone(), &cx.to_async())
+ cx.update(|cx| provider.stream_completion(&model, request.clone(), cx))
.await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
- Arc::get_mut(&mut model.token_provider)
- .unwrap()
- .data_retention_consent = true;
+ provider
+ .token_provider
+ .data_retention_consent
+ .store(true, Ordering::SeqCst);
assert_eq!(
- model
- .count_input_tokens(request, &cx.to_async())
+ cx.update(|cx| provider.count_input_tokens(&model, request, cx))
.await
.unwrap(),
Some(731)
@@ -2199,38 +2174,39 @@ mod tests {
(Some(8192), Some(8192)),
(Some(u64::MAX), Some(128_000)),
] {
- let model = cloud_test_model(FakeHttpClient::create(move |mut request| async move {
- assert_eq!(request.uri().path(), "/completions");
- let mut body = String::new();
- request.body_mut().read_to_string(&mut body).await?;
- let body: serde_json::Value = serde_json::from_str(&body)?;
- assert_eq!(
- body["provider_request"].get("max_output_tokens").cloned(),
- expected.map(|value| json!(value))
- );
- let completed = CompletionEvent::Event(json!({
- "type": "response.completed",
- "response": {
- "id": "response-1", "status": "completed", "output": [],
- "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}
- }
+ let (provider, model) =
+ cloud_test_model(FakeHttpClient::create(move |mut request| async move {
+ assert_eq!(request.uri().path(), "/completions");
+ let mut body = String::new();
+ request.body_mut().read_to_string(&mut body).await?;
+ let body: serde_json::Value = serde_json::from_str(&body)?;
+ assert_eq!(
+ body["provider_request"].get("max_output_tokens").cloned(),
+ expected.map(|value| json!(value))
+ );
+ let completed = CompletionEvent::Event(json!({
+ "type": "response.completed",
+ "response": {
+ "id": "response-1", "status": "completed", "output": [],
+ "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}
+ }
+ }));
+ let ended = CompletionEvent::<serde_json::Value>::Status(
+ CompletionRequestStatus::StreamEnded,
+ );
+ Ok(Response::builder()
+ .status(200)
+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
+ .body(AsyncBody::from(format!(
+ "{}\n{}\n",
+ serde_json::to_string(&completed)?,
+ serde_json::to_string(&ended)?,
+ )))?)
}));
- let ended = CompletionEvent::<serde_json::Value>::Status(
- CompletionRequestStatus::StreamEnded,
- );
- Ok(Response::builder()
- .status(200)
- .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
- .body(AsyncBody::from(format!(
- "{}\n{}\n",
- serde_json::to_string(&completed)?,
- serde_json::to_string(&ended)?,
- )))?)
- }));
let mut request = compact_test_request();
request.max_output_tokens = limit;
- let mut stream = model
- .stream_completion(request, &cx.to_async())
+ let mut stream = cx
+ .update(|cx| provider.stream_completion(&model, request, cx))
.await
.unwrap();
while let Some(event) = stream.next().await {
@@ -2241,7 +2217,7 @@ mod tests {
fn cloud_google_test_model(
events: Vec<serde_json::Value>,
- ) -> CloudLanguageModel<TestTokenProvider> {
+ ) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
let body = events
.into_iter()
.map(|event| format!("{event}\n"))
@@ -2255,12 +2231,10 @@ mod tests {
.body(AsyncBody::from(body))?)
}
});
- let mut model = cloud_test_model(http_client);
- model.id = LanguageModelId::from("gemini-3.1-pro-preview".to_string());
- let metadata = Arc::make_mut(&mut model.model);
- metadata.provider = cloud_llm_client::LanguageModelProvider::Google;
- metadata.id = cloud_llm_client::LanguageModelId(Arc::from("gemini-3.1-pro-preview"));
- model
+ let mut config = cloud_test_config();
+ config.provider = cloud_llm_client::LanguageModelProvider::Google;
+ config.id = cloud_llm_client::LanguageModelId(Arc::from("gemini-3.1-pro-preview"));
+ bind_test_model(http_client, config)
}
fn compact_test_request() -> LanguageModelRequest {
@@ -2279,71 +2253,91 @@ mod tests {
fn cloud_anthropic_test_model(
http_client: Arc<HttpClientWithUrl>,
- ) -> CloudLanguageModel<TestTokenProvider> {
- CloudLanguageModel {
- id: LanguageModelId::from("claude-opus-4-6".to_string()),
- model: Arc::new(cloud_llm_client::LanguageModel {
- provider: cloud_llm_client::LanguageModelProvider::Anthropic,
- id: cloud_llm_client::LanguageModelId(Arc::from("claude-opus-4-6")),
- display_name: "Claude Opus 4.6".to_string(),
- is_latest: true,
- max_token_count: 1_000_000,
- max_token_count_in_max_mode: None,
- max_output_tokens: 128_000,
- supports_tools: true,
- supports_images: true,
- supports_thinking: true,
- supports_disabling_thinking: true,
- supports_fast_mode: true,
- supports_server_side_compaction: true,
- supported_effort_levels: Vec::new(),
- supports_streaming_tools: true,
- supports_parallel_tool_calls: false,
- is_disabled: false,
- disabled_reason: None,
- }),
- token_provider: Arc::new(TestTokenProvider::default()),
- http_client,
- app_version: None,
- request_limiter: RateLimiter::new(4),
- }
+ ) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
+ bind_test_model(http_client, cloud_anthropic_test_config())
}
fn cloud_test_model(
http_client: Arc<HttpClientWithUrl>,
- ) -> CloudLanguageModel<TestTokenProvider> {
- CloudLanguageModel {
- id: LanguageModelId::from("gpt-5.4".to_string()),
- model: Arc::new(cloud_llm_client::LanguageModel {
- provider: cloud_llm_client::LanguageModelProvider::OpenAi,
- id: cloud_llm_client::LanguageModelId(Arc::from("gpt-5.4")),
- display_name: "GPT-5.4".to_string(),
- is_latest: true,
- max_token_count: 1_000_000,
- max_token_count_in_max_mode: None,
- max_output_tokens: 128_000,
- supports_tools: true,
- supports_images: true,
- supports_thinking: true,
- supports_disabling_thinking: true,
- supports_fast_mode: true,
- supports_server_side_compaction: true,
- supported_effort_levels: Vec::new(),
- supports_streaming_tools: true,
- supports_parallel_tool_calls: true,
- is_disabled: false,
- disabled_reason: None,
- }),
- token_provider: Arc::new(TestTokenProvider::default()),
- http_client,
- app_version: None,
- request_limiter: RateLimiter::new(4),
+ ) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
+ bind_test_model(http_client, cloud_test_config())
+ }
+
+ /// Lists `configs` in a provider's model list, as a successful `/models`
+ /// fetch would.
+ fn test_provider(
+ http_client: Arc<HttpClientWithUrl>,
+ configs: Vec<cloud_llm_client::LanguageModel>,
+ ) -> CloudModelProvider<TestTokenProvider> {
+ let mut provider =
+ CloudModelProvider::new(Arc::new(TestTokenProvider::default()), http_client, None);
+ provider.update_models(ListModelsResponse {
+ models: configs,
+ default_model: None,
+ default_fast_model: None,
+ recommended_models: Vec::new(),
+ });
+ provider
+ }
+
+ /// Lists `config` in a provider, and describes the model it lists.
+ fn bind_test_model(
+ http_client: Arc<HttpClientWithUrl>,
+ config: cloud_llm_client::LanguageModel,
+ ) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
+ let model = language_model(&config);
+ (test_provider(http_client, vec![config]), model)
+ }
+
+ fn cloud_anthropic_test_config() -> cloud_llm_client::LanguageModel {
+ cloud_llm_client::LanguageModel {
+ provider: cloud_llm_client::LanguageModelProvider::Anthropic,
+ id: cloud_llm_client::LanguageModelId(Arc::from("claude-opus-4-6")),
+ display_name: "Claude Opus 4.6".to_string(),
+ is_latest: true,
+ max_token_count: 1_000_000,
+ max_token_count_in_max_mode: None,
+ max_output_tokens: 128_000,
+ supports_tools: true,
+ supports_images: true,
+ supports_thinking: true,
+ supports_disabling_thinking: true,
+ supports_fast_mode: true,
+ supports_server_side_compaction: true,
+ supported_effort_levels: Vec::new(),
+ supports_streaming_tools: true,
+ supports_parallel_tool_calls: false,
+ is_disabled: false,
+ disabled_reason: None,
+ }
+ }
+
+ fn cloud_test_config() -> cloud_llm_client::LanguageModel {
+ cloud_llm_client::LanguageModel {
+ provider: cloud_llm_client::LanguageModelProvider::OpenAi,
+ id: cloud_llm_client::LanguageModelId(Arc::from("gpt-5.4")),
+ display_name: "GPT-5.4".to_string(),
+ is_latest: true,
+ max_token_count: 1_000_000,
+ max_token_count_in_max_mode: None,
+ max_output_tokens: 128_000,
+ supports_tools: true,
+ supports_images: true,
+ supports_thinking: true,
+ supports_disabling_thinking: true,
+ supports_fast_mode: true,
+ supports_server_side_compaction: true,
+ supported_effort_levels: Vec::new(),
+ supports_streaming_tools: true,
+ supports_parallel_tool_calls: true,
+ is_disabled: false,
+ disabled_reason: None,
}
}
#[derive(Default)]
struct TestTokenProvider {
- data_retention_consent: bool,
+ data_retention_consent: AtomicBool,
}
impl CloudLlmTokenProvider for TestTokenProvider {
@@ -2366,7 +2360,7 @@ mod tests {
}
fn has_data_retention_consent(&self, _cx: &impl AppContext) -> bool {
- self.data_retention_consent
+ self.data_retention_consent.load(Ordering::SeqCst)
}
}
}
diff --git a/crates/openai_subscribed/src/openai_subscribed.rs b/crates/openai_subscribed/src/openai_subscribed.rs
index ca77c61c49d..86ee184018d 100644
--- a/crates/openai_subscribed/src/openai_subscribed.rs
+++ b/crates/openai_subscribed/src/openai_subscribed.rs
@@ -3,15 +3,16 @@ use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use credentials_provider::CredentialsProvider;
use futures::{FutureExt, StreamExt, future::BoxFuture, future::Shared};
-use gpui::{App, AsyncApp, Context, Entity, SharedString, Task, WeakEntity};
+use gpui::{AsyncApp, Context, Entity, SharedString, Task, WeakEntity};
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt as _,
http::{HeaderName, HeaderValue},
};
use language_model::{
CompactionResult, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
- LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelRequest, LanguageModelToolChoice, RateLimiter,
+ LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
+ LanguageModelProviderId, LanguageModelProviderName, LanguageModelRequest,
+ LanguageModelToolChoiceSupport, RateLimiter,
};
use open_ai::{
ReasoningEffort,
@@ -518,54 +519,28 @@ impl ChatGptModel {
}
}
-/// Creates a [`LanguageModel`] for `model` that authenticates through
-/// `state`'s credentials, refreshing them as needed.
-pub fn create_language_model(
- model: ChatGptModel,
- state: &Entity<State>,
- cx: &App,
-) -> Arc<dyn LanguageModel> {
- Arc::new(OpenAiSubscribedLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- http_client: state.read(cx).http_client.clone(),
- model,
- state: state.clone(),
- request_limiter: RateLimiter::new(4),
- })
-}
-
-struct OpenAiSubscribedLanguageModel {
- id: LanguageModelId,
- model: ChatGptModel,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- request_limiter: RateLimiter,
-}
-
-impl OpenAiSubscribedLanguageModel {
- fn codex_responses_request(
- &self,
- mut request: LanguageModelRequest,
- ) -> Result<open_ai::responses::Request> {
- if !self.model.supports_priority() {
- request.speed = None;
- }
- let mut responses_request = into_open_ai_response(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- self.model.supports_prompt_cache_key(),
- None,
- self.model.default_reasoning_effort(),
- self.model
- .supported_reasoning_efforts()
- .contains(&ReasoningEffort::None),
- &PROVIDER_ID,
- )?;
- responses_request.store = Some(false);
- responses_request.instructions.get_or_insert_default();
- Ok(responses_request)
- }
+fn codex_responses_request(
+ model: &ChatGptModel,
+ mut request: LanguageModelRequest,
+) -> Result<open_ai::responses::Request> {
+ if !model.supports_priority() {
+ request.speed = None;
+ }
+ let mut responses_request = into_open_ai_response(
+ request,
+ model.id(),
+ model.supports_parallel_tool_calls(),
+ model.supports_prompt_cache_key(),
+ None,
+ model.default_reasoning_effort(),
+ model
+ .supported_reasoning_efforts()
+ .contains(&ReasoningEffort::None),
+ &PROVIDER_ID,
+ )?;
+ responses_request.store = Some(false);
+ responses_request.instructions.get_or_insert_default();
+ Ok(responses_request)
}
fn codex_extra_headers(
@@ -741,223 +716,180 @@ async fn list_models(
Ok(models.into_iter().map(ChatGptModel::from).collect())
}
-impl LanguageModel for OpenAiSubscribedLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- true
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
-
- fn supports_tool_choice(&self, _choice: LanguageModelToolChoice) -> bool {
- true
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_thinking(&self) -> bool {
- true
- }
-
- fn supports_fast_mode(&self) -> bool {
- self.model.supports_priority()
- }
-
- fn supports_server_side_compaction(&self) -> bool {
- true
- }
-
- fn supports_explicit_compaction(&self) -> bool {
- true
+/// Describes a ChatGPT subscription model as a [`LanguageModel`].
+pub fn language_model(model: &ChatGptModel) -> LanguageModel {
+ let default_effort = model.default_reasoning_effort();
+ let supported_effort_levels = model
+ .supported_reasoning_efforts()
+ .iter()
+ .copied()
+ .filter_map(|effort| {
+ let (name, value) = match effort {
+ ReasoningEffort::None => return None,
+ ReasoningEffort::Minimal => ("Minimal", "minimal"),
+ ReasoningEffort::Low => ("Low", "low"),
+ ReasoningEffort::Medium => ("Medium", "medium"),
+ ReasoningEffort::High => ("High", "high"),
+ ReasoningEffort::XHigh => ("Extra High", "xhigh"),
+ ReasoningEffort::Max => ("Max", "max"),
+ };
+
+ Some(LanguageModelEffortLevel {
+ name: name.into(),
+ value: value.into(),
+ is_default: Some(effort) == default_effort,
+ })
+ })
+ .collect();
+ LanguageModel {
+ supports_tools: true,
+ supports_images: model.supports_images(),
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_streaming_tools: true,
+ supports_thinking: true,
+ supports_fast_mode: model.supports_priority(),
+ supports_server_side_compaction: true,
+ supports_explicit_compaction: true,
+ supported_effort_levels,
+ max_output_tokens: model.max_output_tokens(),
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("openai-subscribed/{}", model.id()),
+ model.max_token_count(),
+ )
}
+}
- fn compact(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
- let mut responses_request = match self.codex_responses_request(request) {
- Ok(responses_request) => responses_request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- responses_request.context_management = None;
- responses_request
- .input
- .push(ResponseInputItem::CompactionTrigger);
-
- let state = self.state.downgrade();
- let http_client = self.http_client.clone();
- let request_limiter = self.request_limiter.clone();
-
- cx.spawn(async move |cx| {
- let creds = get_fresh_credentials(&state, &http_client, cx).await?;
- let extra_headers =
- codex_extra_headers(&creds, responses_request.prompt_cache_key.as_deref());
- let access_token = creds.access_token.clone();
- let response_stream = request_limiter
- .stream(async move {
- stream_response(
- http_client.as_ref(),
- PROVIDER_NAME.0.as_str(),
- CODEX_BASE_URL,
- &access_token,
- responses_request,
- &extra_headers,
- )
- .await
- .map_err(LanguageModelCompletionError::from)
- })
- .await?;
- let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
- let mut event_stream = language_model::stream_in_background(
- mapper.map_stream(response_stream.boxed()).boxed(),
- cx.background_executor().clone(),
- );
- let mut compacted_context = None;
- let mut usage = language_model::TokenUsage::default();
-
- while let Some(event) = event_stream.next().await {
- match event? {
- LanguageModelCompletionEvent::Compaction(
- language_model::CompactionUpdate::Finished(context),
- ) => {
- if compacted_context.replace(context).is_some() {
- return Err(LanguageModelCompletionError::Other(anyhow!(
- "ChatGPT subscription compaction returned multiple replacement contexts"
- )));
- }
- }
- LanguageModelCompletionEvent::UsageUpdate(updated_usage) => {
- usage = updated_usage;
+/// Compacts `request`'s conversation with the ChatGPT model `model`, using
+/// `state`'s credentials, which are refreshed as needed.
+pub fn compact(
+ model: &ChatGptModel,
+ state: &Entity<State>,
+ request_limiter: &RateLimiter,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let mut responses_request = match codex_responses_request(model, request) {
+ Ok(responses_request) => responses_request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ responses_request.context_management = None;
+ responses_request
+ .input
+ .push(ResponseInputItem::CompactionTrigger);
+
+ let http_client = state.read_with(cx, |state, _| state.http_client.clone());
+ let state = state.downgrade();
+ let request_limiter = request_limiter.clone();
+
+ cx.spawn(async move |cx| {
+ let creds = get_fresh_credentials(&state, &http_client, cx).await?;
+ let extra_headers =
+ codex_extra_headers(&creds, responses_request.prompt_cache_key.as_deref());
+ let access_token = creds.access_token.clone();
+ let response_stream = request_limiter
+ .stream(async move {
+ stream_response(
+ http_client.as_ref(),
+ PROVIDER_NAME.0.as_str(),
+ CODEX_BASE_URL,
+ &access_token,
+ responses_request,
+ &extra_headers,
+ )
+ .await
+ .map_err(LanguageModelCompletionError::from)
+ })
+ .await?;
+ let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
+ let mut event_stream = language_model::stream_in_background(
+ mapper.map_stream(response_stream.boxed()).boxed(),
+ cx.background_executor().clone(),
+ );
+ let mut compacted_context = None;
+ let mut usage = language_model::TokenUsage::default();
+
+ while let Some(event) = event_stream.next().await {
+ match event? {
+ LanguageModelCompletionEvent::Compaction(
+ language_model::CompactionUpdate::Finished(context),
+ ) => {
+ if compacted_context.replace(context).is_some() {
+ return Err(LanguageModelCompletionError::Other(anyhow!(
+ "ChatGPT subscription compaction returned multiple replacement contexts"
+ )));
}
- _ => {}
}
+ LanguageModelCompletionEvent::UsageUpdate(updated_usage) => {
+ usage = updated_usage;
+ }
+ _ => {}
}
+ }
- let context = compacted_context.ok_or_else(|| {
- LanguageModelCompletionError::Other(anyhow!(
- "ChatGPT subscription compaction returned no replacement context"
- ))
- })?;
- Ok(CompactionResult { context, usage })
- })
- .boxed()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- let default_effort = self.model.default_reasoning_effort();
- self.model
- .supported_reasoning_efforts()
- .iter()
- .copied()
- .filter_map(|effort| {
- let (name, value) = match effort {
- ReasoningEffort::None => return None,
- ReasoningEffort::Minimal => ("Minimal", "minimal"),
- ReasoningEffort::Low => ("Low", "low"),
- ReasoningEffort::Medium => ("Medium", "medium"),
- ReasoningEffort::High => ("High", "high"),
- ReasoningEffort::XHigh => ("Extra High", "xhigh"),
- ReasoningEffort::Max => ("Max", "max"),
- };
-
- Some(LanguageModelEffortLevel {
- name: name.into(),
- value: value.into(),
- is_default: Some(effort) == default_effort,
- })
- })
- .collect()
- }
-
- fn telemetry_id(&self) -> String {
- format!("openai-subscribed/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
+ let context = compacted_context.ok_or_else(|| {
+ LanguageModelCompletionError::Other(anyhow!(
+ "ChatGPT subscription compaction returned no replacement context"
+ ))
+ })?;
+ Ok(CompactionResult { context, usage })
+ })
+ .boxed()
+}
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- let responses_request = match self.codex_responses_request(request) {
- Ok(responses_request) => responses_request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
+/// Streams a completion of `request` from the ChatGPT model `model`, using
+/// `state`'s credentials, which are refreshed as needed.
+pub fn stream_completion(
+ model: &ChatGptModel,
+ state: &Entity<State>,
+ request_limiter: &RateLimiter,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>> {
+ let responses_request = match codex_responses_request(model, request) {
+ Ok(responses_request) => responses_request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
- let state = self.state.downgrade();
- let http_client = self.http_client.clone();
- let request_limiter = self.request_limiter.clone();
- let executor = cx.background_executor().clone();
-
- let future = cx.spawn(async move |cx| {
- let creds = get_fresh_credentials(&state, &http_client, cx).await?;
- let extra_headers =
- codex_extra_headers(&creds, responses_request.prompt_cache_key.as_deref());
-
- let access_token = creds.access_token.clone();
- request_limiter
- .stream(async move {
- stream_response(
- http_client.as_ref(),
- PROVIDER_NAME.0.as_str(),
- CODEX_BASE_URL,
- &access_token,
- responses_request,
- &extra_headers,
- )
- .await
- .map_err(LanguageModelCompletionError::from)
- })
+ let http_client = state.read_with(cx, |state, _| state.http_client.clone());
+ let state = state.downgrade();
+ let request_limiter = request_limiter.clone();
+ let executor = cx.background_executor().clone();
+
+ let future = cx.spawn(async move |cx| {
+ let creds = get_fresh_credentials(&state, &http_client, cx).await?;
+ let extra_headers =
+ codex_extra_headers(&creds, responses_request.prompt_cache_key.as_deref());
+
+ let access_token = creds.access_token.clone();
+ request_limiter
+ .stream(async move {
+ stream_response(
+ http_client.as_ref(),
+ PROVIDER_NAME.0.as_str(),
+ CODEX_BASE_URL,
+ &access_token,
+ responses_request,
+ &extra_headers,
+ )
.await
- });
+ .map_err(LanguageModelCompletionError::from)
+ })
+ .await
+ });
- async move {
- let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
- Ok(language_model::stream_in_background(
- mapper.map_stream(future.await?.boxed()).boxed(),
- executor,
- ))
- }
- .boxed()
+ async move {
+ let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(future.await?.boxed()).boxed(),
+ executor,
+ ))
}
+ .boxed()
}
-
async fn get_fresh_credentials(
state: &WeakEntity<State>,
http_client: &Arc<dyn HttpClient>,
@@ -1357,6 +1289,7 @@ mod tests {
use super::*;
use gpui::{AppContext as _, TestAppContext};
use http_client::FakeHttpClient;
+ use language_model::DEFAULT_MODEL_CONCURRENCY;
use parking_lot::Mutex;
use std::future::Future;
use std::pin::Pin;
@@ -2107,9 +2040,10 @@ mod tests {
let mut credentials = make_fresh_credentials();
credentials.account_id = Some("account-123".to_string());
let state = make_state(http, Some(credentials), cx);
- let model = cx.read(|cx| create_language_model(ChatGptModel::Gpt55, &state, cx));
- assert!(model.supports_server_side_compaction());
- assert!(model.supports_explicit_compaction());
+ let language_model = language_model(&ChatGptModel::Gpt55);
+ assert!(language_model.supports_server_side_compaction());
+ assert!(language_model.supports_explicit_compaction());
+ let request_limiter = RateLimiter::new(DEFAULT_MODEL_CONCURRENCY);
let request = LanguageModelRequest {
messages: vec![language_model::LanguageModelRequestMessage {
@@ -2123,12 +2057,17 @@ mod tests {
..Default::default()
};
let async_cx = cx.to_async();
- let events = model
- .stream_completion(request, &async_cx)
- .await
- .expect("the response stream should start")
- .collect::<Vec<_>>()
- .await;
+ let events = stream_completion(
+ &ChatGptModel::Gpt55,
+ &state,
+ &request_limiter,
+ request,
+ &async_cx,
+ )
+ .await
+ .expect("the response stream should start")
+ .collect::<Vec<_>>()
+ .await;
assert_eq!(compaction_request_count.load(Ordering::SeqCst), 1);
assert!(matches!(
@@ -2200,7 +2139,7 @@ mod tests {
let http: Arc<dyn HttpClient> = http_client;
let state = make_state(http, Some(make_fresh_credentials()), cx);
- let model = cx.read(|cx| create_language_model(ChatGptModel::Gpt55, &state, cx));
+ let request_limiter = RateLimiter::new(DEFAULT_MODEL_CONCURRENCY);
let request = LanguageModelRequest {
messages: vec![language_model::LanguageModelRequestMessage {
role: language_model::Role::User,
@@ -2213,10 +2152,15 @@ mod tests {
..Default::default()
};
- let result = model
- .compact(request, &cx.to_async())
- .await
- .expect("manual compaction should succeed");
+ let result = compact(
+ &ChatGptModel::Gpt55,
+ &state,
+ &request_limiter,
+ request,
+ &cx.to_async(),
+ )
+ .await
+ .expect("manual compaction should succeed");
let language_model::CompactedContext::ProviderState(compaction_state) = result.context
else {
panic!("expected provider compaction state");
diff --git a/crates/sidebar/src/sidebar.rs b/crates/sidebar/src/sidebar.rs
index 37491eb69e3..8e934d82162 100644
--- a/crates/sidebar/src/sidebar.rs
+++ b/crates/sidebar/src/sidebar.rs
@@ -3841,9 +3841,7 @@ impl Sidebar {
}
}
- let Some(configured_model) =
- LanguageModelRegistry::read_global(cx).thread_summary_model(cx)
- else {
+ let Some(model) = LanguageModelRegistry::read_global(cx).thread_summary_model(cx) else {
if let Some(workspace) = self.active_workspace(cx) {
Self::show_no_thread_summary_model_toast(workspace, cx);
}
@@ -3854,7 +3852,6 @@ impl Sidebar {
return;
}
- let model = configured_model.model;
let temperature = AgentSettings::temperature_for_model(&model, cx);
let thread_store = ThreadStore::global(cx);
diff --git a/crates/x_ai_subscribed/src/x_ai_subscribed.rs b/crates/x_ai_subscribed/src/x_ai_subscribed.rs
index a830b130e70..09a3d267f6c 100644
--- a/crates/x_ai_subscribed/src/x_ai_subscribed.rs
+++ b/crates/x_ai_subscribed/src/x_ai_subscribed.rs
@@ -3,13 +3,13 @@ use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use credentials_provider::CredentialsProvider;
use futures::{FutureExt, StreamExt, future::BoxFuture, future::Shared};
-use gpui::{App, AsyncApp, Context, Entity, SharedString, Task, WeakEntity};
+use gpui::{AsyncApp, Context, Entity, SharedString, Task, WeakEntity};
use http_client::{AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest};
use language_model::chat_completion::ChatCompletionEventMapper;
use language_model::{
- LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
+ LanguageModel, LanguageModelCompletionError, LanguageModelCompletionStream,
LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProviderId,
- LanguageModelProviderName, LanguageModelRequest, LanguageModelToolChoice,
+ LanguageModelProviderName, LanguageModelRequest, LanguageModelToolChoiceSupport,
ProviderErrorCategory, RateLimiter,
};
use open_ai::{ReasoningEffort, ResponseStreamEvent};
@@ -317,32 +317,6 @@ impl SuperGrokModel {
}
}
-pub fn create_language_model(
- model: SuperGrokModel,
- state: &Entity<State>,
- cx: &App,
-) -> Arc<dyn LanguageModel> {
- Arc::new(SuperGrokLanguageModel {
- id: LanguageModelId::from(model.id().to_string()),
- http_client: state.read(cx).http_client(),
- model,
- state: state.clone(),
- api_url: XAI_API_URL.into(),
- extra_headers: CustomHeaders::default(),
- request_limiter: RateLimiter::new(4),
- })
-}
-
-struct SuperGrokLanguageModel {
- id: LanguageModelId,
- model: SuperGrokModel,
- state: Entity<State>,
- http_client: Arc<dyn HttpClient>,
- api_url: SharedString,
- extra_headers: CustomHeaders,
- request_limiter: RateLimiter,
-}
-
fn advertised_reasoning_efforts(model: &SuperGrokModel) -> &'static [ReasoningEffort] {
// xAI rejects `reasoning_effort: "none"` on grok-4.5 and newer. Compact and
// title requests disable thinking, so we omit the field instead of sending none.
@@ -439,151 +413,101 @@ fn map_completion_error(error: LanguageModelCompletionError) -> LanguageModelCom
}
}
-impl SuperGrokLanguageModel {
- fn stream_open_ai_completion(
- &self,
- request: open_ai::Request,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
- LanguageModelCompletionError,
- >,
- > {
- let http_client = self.http_client.clone();
- let api_url = self.api_url.clone();
- let extra_headers = self.extra_headers.clone();
- let state = self.state.downgrade();
- let request_limiter = self.request_limiter.clone();
-
- let future = cx.spawn(async move |cx| {
- let credentials = get_fresh_credentials(&state, &http_client, cx).await?;
- let access_token = credentials.access_token.clone();
- request_limiter
- .stream(async move {
- open_ai::stream_completion(
- http_client.as_ref(),
- PROVIDER_NAME.0.as_str(),
- api_url.as_ref(),
- &access_token,
- request,
- &extra_headers,
- )
- .await
- .map_err(|error| {
- map_completion_error(LanguageModelCompletionError::from(error))
- })
- })
+fn stream_open_ai_completion(
+ state: &Entity<State>,
+ request_limiter: &RateLimiter,
+ request: open_ai::Request,
+ cx: &AsyncApp,
+) -> BoxFuture<
+ 'static,
+ Result<
+ futures::stream::BoxStream<'static, Result<ResponseStreamEvent>>,
+ LanguageModelCompletionError,
+ >,
+> {
+ let http_client = state.read_with(cx, |state, _| state.http_client());
+ let extra_headers = CustomHeaders::default();
+ let state = state.downgrade();
+ let request_limiter = request_limiter.clone();
+
+ let future = cx.spawn(async move |cx| {
+ let credentials = get_fresh_credentials(&state, &http_client, cx).await?;
+ let access_token = credentials.access_token.clone();
+ request_limiter
+ .stream(async move {
+ open_ai::stream_completion(
+ http_client.as_ref(),
+ PROVIDER_NAME.0.as_str(),
+ XAI_API_URL,
+ &access_token,
+ request,
+ &extra_headers,
+ )
.await
- });
+ .map_err(|error| map_completion_error(LanguageModelCompletionError::from(error)))
+ })
+ .await
+ });
- async move { Ok(future.await?.boxed()) }.boxed()
- }
+ async move { Ok(future.await?.boxed()) }.boxed()
}
-impl LanguageModel for SuperGrokLanguageModel {
- fn id(&self) -> LanguageModelId {
- self.id.clone()
- }
-
- fn name(&self) -> LanguageModelName {
- LanguageModelName::from(self.model.display_name().to_string())
- }
-
- fn provider_id(&self) -> LanguageModelProviderId {
- PROVIDER_ID
- }
-
- fn provider_name(&self) -> LanguageModelProviderName {
- PROVIDER_NAME
- }
-
- fn supports_tools(&self) -> bool {
- self.model.supports_tools()
- }
-
- fn supports_images(&self) -> bool {
- self.model.supports_images()
- }
-
- fn supports_streaming_tools(&self) -> bool {
- true
- }
-
- fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
- match choice {
- LanguageModelToolChoice::Auto
- | LanguageModelToolChoice::Any
- | LanguageModelToolChoice::None => true,
- }
- }
-
- fn supports_thinking(&self) -> bool {
- self.model.supports_reasoning_effort()
- }
-
- fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
- supported_thinking_effort_levels(&self.model)
- }
-
- fn telemetry_id(&self) -> String {
- format!("x_ai_subscribed/{}", self.model.id())
- }
-
- fn max_token_count(&self) -> u64 {
- self.model.max_token_count()
- }
-
- fn max_output_tokens(&self) -> Option<u64> {
- self.model.max_output_tokens()
- }
-
- fn supports_split_token_display(&self) -> bool {
- true
- }
-
- fn stream_completion(
- &self,
- request: LanguageModelRequest,
- cx: &AsyncApp,
- ) -> BoxFuture<
- 'static,
- Result<
- futures::stream::BoxStream<
- 'static,
- Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
- >,
- LanguageModelCompletionError,
- >,
- > {
- let reasoning_effort = reasoning_effort_for_request(&request, &self.model);
- let request = match open_ai::completion::into_open_ai(
- request,
- self.model.id(),
- self.model.supports_parallel_tool_calls(),
- false,
- self.max_output_tokens(),
- open_ai::completion::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
- reasoning_effort,
- false,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let completions = self.stream_open_ai_completion(request, cx);
- let executor = cx.background_executor().clone();
- async move {
- let mapper = ChatCompletionEventMapper::new();
- Ok(language_model::stream_in_background(
- mapper.map_stream(completions.await?).boxed(),
- executor,
- ))
- }
- .boxed()
+/// Describes a SuperGrok model as a [`LanguageModel`].
+pub fn language_model(model: &SuperGrokModel) -> LanguageModel {
+ LanguageModel {
+ supports_tools: model.supports_tools(),
+ supports_images: model.supports_images(),
+ supports_streaming_tools: true,
+ tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ supports_thinking: model.supports_reasoning_effort(),
+ supported_effort_levels: supported_thinking_effort_levels(model).into(),
+ max_output_tokens: model.max_output_tokens(),
+ supports_split_token_display: true,
+ ..LanguageModel::new(
+ LanguageModelId::from(model.id().to_string()),
+ LanguageModelName::from(model.display_name().to_string()),
+ PROVIDER_ID,
+ PROVIDER_NAME,
+ format!("x_ai_subscribed/{}", model.id()),
+ model.max_token_count(),
+ )
}
}
+/// Streams a completion of `request` from the SuperGrok model `model`, using
+/// `state`'s credentials, which are refreshed as needed.
+pub fn stream_completion(
+ model: &SuperGrokModel,
+ state: &Entity<State>,
+ request_limiter: &RateLimiter,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>> {
+ let reasoning_effort = reasoning_effort_for_request(&request, model);
+ let request = match open_ai::completion::into_open_ai(
+ request,
+ model.id(),
+ model.supports_parallel_tool_calls(),
+ false,
+ model.max_output_tokens(),
+ open_ai::completion::ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ reasoning_effort,
+ false,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let completions = stream_open_ai_completion(state, request_limiter, request, cx);
+ let executor = cx.background_executor().clone();
+ async move {
+ let mapper = ChatCompletionEventMapper::new();
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
+ }
+ .boxed()
+}
async fn get_fresh_credentials(
state: &WeakEntity<State>,
http_client: &Arc<dyn HttpClient>,