Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

ollama: Make tool call image output accessible to the model (#64121)

marius851000fe1dd2d375f1
diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rs
index a6426235dfc..6dedf9e93cd 100644
--- a/crates/language_model_core/src/request.rs
+++ b/crates/language_model_core/src/request.rs
@@ -91,6 +91,14 @@ impl LanguageModelToolResult {
pub fn is_content_empty(&self) -> bool {
self.content.iter().all(|part| part.is_empty())
}
+
+ /// Returns an iterator over all the images presents in the content parts.
+ pub fn images(&self) -> impl Iterator<Item = &LanguageModelImage> {
+ self.content.iter().filter_map(|part| match part {
+ LanguageModelToolResultContent::Image(image) => Some(image),
+ _ => None,
+ })
+ }
}
/// Serde helper that accepts both the legacy single-value shape and the new
diff --git a/crates/language_models/src/provider/ollama.rs b/crates/language_models/src/provider/ollama.rs
index 6dc0175c1b1..486111cadca 100644
--- a/crates/language_models/src/provider/ollama.rs
+++ b/crates/language_models/src/provider/ollama.rs
@@ -390,9 +390,18 @@ impl OllamaLanguageModel {
{
match tool_result {
MessageContent::ToolResult(tool_result) => {
+ let images = tool_result
+ .images()
+ .map(|image| image.source.to_string())
+ .collect::<Vec<_>>();
messages.push(ChatMessage::Tool {
tool_name: tool_result.tool_name.to_string(),
content: tool_result.text_contents(),
+ images: if images.is_empty() {
+ None
+ } else {
+ Some(images)
+ },
})
}
_ => unreachable!("Only tool result should be extracted"),
diff --git a/crates/ollama/src/ollama.rs b/crates/ollama/src/ollama.rs
index 0bbb210b126..046a71286f2 100644
--- a/crates/ollama/src/ollama.rs
+++ b/crates/ollama/src/ollama.rs
@@ -96,6 +96,8 @@ pub enum ChatMessage {
Tool {
tool_name: String,
content: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ images: Option<Vec<String>>,
},
}