Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

refactor: move search providers into rust backend

nash_su6cffec9120de
diff --git a/src-tauri/src/agent/skills.rs b/src-tauri/src/agent/skills.rs
index 6fadc512..320fea8a 100644
--- a/src-tauri/src/agent/skills.rs
+++ b/src-tauri/src/agent/skills.rs
@@ -65,6 +65,7 @@ fn normalize_skill_name(value: &str) -> Option<String> {
|| trimmed.contains('/')
|| trimmed.contains('\\')
|| trimmed.contains("..")
+ || !is_portable_skill_name(trimmed)
{
return None;
}
@@ -73,8 +74,9 @@ fn normalize_skill_name(value: &str) -> Option<String> {
fn split_frontmatter(raw: &str) -> (Option<String>, String) {
let normalized = raw.strip_prefix('\u{feff}').unwrap_or(raw);
+ let normalized = normalized.replace("\r\n", "\n").replace('\r', "\n");
if !normalized.starts_with("---\n") {
- return (None, normalized.to_string());
+ return (None, normalized);
}
let rest = &normalized[4..];
if let Some(end) = rest.find("\n---") {
@@ -85,10 +87,50 @@ fn split_frontmatter(raw: &str) -> (Option<String>, String) {
.to_string();
(Some(fm), after)
} else {
- (None, normalized.to_string())
+ (None, normalized)
}
}
+fn is_portable_skill_name(value: &str) -> bool {
+ if value
+ .chars()
+ .any(|ch| matches!(ch, '<' | '>' | ':' | '"' | '|' | '?' | '*') || ch <= '\u{1f}')
+ {
+ return false;
+ }
+ let stem = value
+ .split('.')
+ .next()
+ .unwrap_or(value)
+ .trim_end_matches(' ')
+ .to_ascii_uppercase();
+ !matches!(
+ stem.as_str(),
+ "CON"
+ | "PRN"
+ | "AUX"
+ | "NUL"
+ | "COM1"
+ | "COM2"
+ | "COM3"
+ | "COM4"
+ | "COM5"
+ | "COM6"
+ | "COM7"
+ | "COM8"
+ | "COM9"
+ | "LPT1"
+ | "LPT2"
+ | "LPT3"
+ | "LPT4"
+ | "LPT5"
+ | "LPT6"
+ | "LPT7"
+ | "LPT8"
+ | "LPT9"
+ )
+}
+
fn yaml_string_field(frontmatter: &str, key: &str) -> Option<String> {
let prefix = format!("{key}:");
for line in frontmatter.lines() {
@@ -135,9 +177,34 @@ mod tests {
let _ = fs::remove_dir_all(root);
}
+ #[test]
+ fn load_project_skills_reads_crlf_frontmatter() {
+ let root = std::env::temp_dir().join(format!("llm-wiki-skills-{}", Uuid::new_v4()));
+ let skills_dir = root.join(".llm-wiki").join("skills");
+ fs::create_dir_all(&skills_dir).unwrap();
+ fs::write(
+ skills_dir.join("reviewer.md"),
+ "---\r\nname: reviewer\r\ndescription: Review source quality\r\n---\r\nCheck claims carefully.",
+ )
+ .unwrap();
+
+ let skills = load_project_skills(root.to_str().unwrap(), &["reviewer".to_string()]);
+ assert_eq!(skills.len(), 1);
+ assert_eq!(skills[0].name, "reviewer");
+ assert_eq!(skills[0].description, "Review source quality");
+ assert_eq!(skills[0].instructions, "Check claims carefully.");
+ let _ = fs::remove_dir_all(root);
+ }
+
#[test]
fn load_project_skills_rejects_path_traversal_names() {
let skills = load_project_skills("/tmp/missing", &["../secret".to_string()]);
assert!(skills.is_empty());
}
+
+ #[test]
+ fn load_project_skills_rejects_windows_reserved_names() {
+ let skills = load_project_skills("/tmp/missing", &["con".to_string(), "a:b".to_string()]);
+ assert!(skills.is_empty());
+ }
}
diff --git a/src-tauri/src/agent/tools.rs b/src-tauri/src/agent/tools.rs
index 2945f9a8..d77efb88 100644
--- a/src-tauri/src/agent/tools.rs
+++ b/src-tauri/src/agent/tools.rs
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use walkdir::WalkDir;
+use crate::commands::external_search::file_url_for_path;
use crate::commands::search::{self, SearchEmbeddingConfig};
use super::types::AgentReference;
@@ -53,6 +54,7 @@ pub trait AgentTool: Send + Sync {
}
pub trait ToolRegistry {
+ #[allow(dead_code)]
fn specs(&self) -> Vec<ToolSpec>;
fn execute<'a>(
&'a self,
@@ -215,6 +217,8 @@ pub struct WebSearchConfig {
#[serde(default)]
pub api_key: String,
#[serde(default)]
+ pub ollama_url: Option<String>,
+ #[serde(default)]
pub sear_xng_url: Option<String>,
#[serde(default)]
pub sear_xng_categories: Option<Vec<String>>,
@@ -230,6 +234,8 @@ pub struct WebSearchProviderOverride {
#[serde(default)]
pub api_key: Option<String>,
#[serde(default)]
+ pub ollama_url: Option<String>,
+ #[serde(default)]
pub sear_xng_url: Option<String>,
#[serde(default)]
pub sear_xng_categories: Option<Vec<String>>,
@@ -268,6 +274,10 @@ impl WebSearchConfig {
.api_key
.clone()
.unwrap_or_else(|| self.api_key.clone()),
+ ollama_url: override_cfg
+ .ollama_url
+ .clone()
+ .or_else(|| self.ollama_url.clone()),
sear_xng_url: override_cfg
.sear_xng_url
.clone()
@@ -285,6 +295,10 @@ impl WebSearchConfig {
}
}
+// Keep the spec list close to the executor even though the current planner
+// still uses fixed tool names. API/MCP tool discovery and future native
+// tool-calling should use this list instead of duplicating tool metadata.
+#[allow(dead_code)]
pub fn builtin_tool_specs() -> Vec<ToolSpec> {
vec![
ToolSpec {
@@ -440,14 +454,6 @@ fn tool_top_k(input: &Value) -> usize {
.clamp(1, 10)
}
-pub fn write_wiki_page(
- project_path: &str,
- rel_path: &str,
- content: &str,
-) -> Result<AgentReference, String> {
- write_wiki_page_with_options(project_path, rel_path, content, false)
-}
-
pub fn write_wiki_page_with_options(
project_path: &str,
rel_path: &str,
@@ -547,7 +553,7 @@ pub async fn run_web_search(
if provider.is_empty() || provider == "none" {
return Err("Web search provider is not configured.".to_string());
}
- let max_results = top_k.clamp(1, 10);
+ let max_results = top_k.clamp(1, 20);
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(WEB_SEARCH_TIMEOUT_SECS))
.build()
@@ -556,6 +562,7 @@ pub async fn run_web_search(
"firecrawl" => firecrawl_search(&client, query, max_results).await?,
"searxng" => searxng_search(&client, query, &config, max_results).await?,
"tavily" => tavily_search(&client, query, &config, max_results).await?,
+ "ollama" => ollama_search(&client, query, &config, max_results).await?,
"brave" => brave_search(&client, query, &config, max_results).await?,
"serpapi" => serpapi_search(&client, query, &config, max_results).await?,
other => {
@@ -564,17 +571,7 @@ pub async fn run_web_search(
))
}
};
- Ok(raw
- .into_iter()
- .take(max_results)
- .map(|item| AgentReference {
- title: item.title,
- path: item.url,
- kind: "web".to_string(),
- snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),
- score: None,
- })
- .collect())
+ Ok(web_items_to_references(raw, max_results))
}
pub async fn run_anytxt_search(
@@ -596,12 +593,15 @@ pub async fn run_anytxt_search(
.filter(|value| !value.trim().is_empty())
.unwrap_or(DEFAULT_ANYTXT_ENDPOINT)
.trim()
- .trim_end_matches('/')
- .to_string();
+ .trim_end_matches('/');
+ let endpoint = normalize_anytxt_endpoint(endpoint);
let limit = top_k
- .clamp(1, 10)
+ .clamp(1, 100)
.min(config.limit.unwrap_or(DEFAULT_ANYTXT_LIMIT).clamp(1, 100));
- let pattern = anytxt_pattern_from_query(query);
+ // AnyTXT has its own query syntax. The caller may already have rewritten
+ // natural language into keyword form, so do not run the source-search
+ // tokenizer here; pass the pattern through unchanged.
+ let pattern = query.to_string();
let filter_dir = config.filter_dir.unwrap_or_default();
let filter_ext = config
.filter_ext
@@ -611,6 +611,18 @@ pub async fn run_anytxt_search(
.timeout(std::time::Duration::from_secs(WEB_SEARCH_TIMEOUT_SECS))
.build()
.map_err(|err| format!("Failed to build AnyTXT client: {err}"))?;
+ let mut input = json!({
+ "pattern": pattern,
+ "filterExt": filter_ext,
+ "lastModifyBegin": 0,
+ "lastModifyEnd": ANYTXT_LAST_MODIFY_END,
+ "limit": limit.to_string(),
+ "offset": 0,
+ "order": 0
+ });
+ if !filter_dir.trim().is_empty() {
+ input["filterDir"] = Value::String(filter_dir);
+ }
let response = client
.post(&endpoint)
.header("Accept", "application/json")
@@ -618,18 +630,7 @@ pub async fn run_anytxt_search(
"id": 1,
"jsonrpc": "2.0",
"method": "ATRpcServer.Searcher.V1.GetResult",
- "params": {
- "input": {
- "pattern": pattern,
- "filterDir": filter_dir,
- "filterExt": filter_ext,
- "lastModifyBegin": 0,
- "lastModifyEnd": ANYTXT_LAST_MODIFY_END,
- "limit": limit.to_string(),
- "offset": 0,
- "order": 0
- }
- }
+ "params": { "input": input }
}))
.send()
.await
@@ -652,74 +653,145 @@ pub async fn run_anytxt_search(
trim_text(&error.to_string(), 300)
));
}
- Ok(extract_anytxt_items(&value)
- .into_iter()
- .take(limit)
- .map(|item| AgentReference {
+ let mut references = Vec::new();
+ for item in extract_anytxt_items(&value).into_iter().take(limit) {
+ let fragment = if !item.fid.trim().is_empty() {
+ get_anytxt_fragment(&client, &endpoint, &item.fid, &pattern)
+ .await
+ .unwrap_or_default()
+ } else {
+ String::new()
+ };
+ references.push(AgentReference {
title: item.title,
- path: item.path,
+ path: file_url_for_path(&item.path),
kind: "anytxt".to_string(),
- snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),
+ snippet: Some(trim_text(
+ if fragment.trim().is_empty() {
+ &item.snippet
+ } else {
+ &fragment
+ },
+ 1200,
+ ))
+ .filter(|s| !s.trim().is_empty()),
score: None,
- })
- .collect())
+ });
+ }
+ Ok(references)
}
#[derive(Debug, Clone)]
struct AnyTxtItem {
+ fid: String,
title: String,
path: String,
snippet: String,
}
fn extract_anytxt_items(value: &Value) -> Vec<AnyTxtItem> {
- let candidates = value
- .get("result")
- .and_then(|result| {
- result
- .get("items")
- .or_else(|| result.get("files"))
- .or_else(|| result.get("result"))
- .or_else(|| result.get("data"))
- })
- .and_then(Value::as_array)
- .or_else(|| value.get("result").and_then(Value::as_array))
- .cloned()
- .unwrap_or_default();
+ let result = value.get("result").unwrap_or(value);
+ let candidates = first_anytxt_array(
+ result,
+ &[
+ &[][..],
+ &["items"],
+ &["files"],
+ &["results"],
+ &["list"],
+ &["value"],
+ &["data"],
+ &["output"],
+ &["output", "items"],
+ &["output", "files"],
+ &["output", "results"],
+ &["output", "list"],
+ &["output", "value"],
+ &["output", "data"],
+ &["data", "items"],
+ &["data", "files"],
+ &["data", "results"],
+ &["data", "list"],
+ &["data", "value"],
+ &["data", "output"],
+ &["data", "output", "items"],
+ &["data", "output", "files"],
+ &["data", "output", "results"],
+ &["data", "output", "list"],
+ &["data", "output", "value"],
+ ],
+ )
+ .unwrap_or_default();
+ let fields = first_anytxt_fields(
+ result,
+ &[
+ &["field"][..],
+ &["fields"],
+ &["output", "field"],
+ &["output", "fields"],
+ &["data", "field"],
+ &["data", "fields"],
+ &["data", "output", "field"],
+ &["data", "output", "fields"],
+ ],
+ )
+ .unwrap_or_default();
candidates
.into_iter()
.filter_map(|item| {
- let path = item
- .get("path")
- .or_else(|| item.get("file"))
- .or_else(|| item.get("filename"))
- .and_then(Value::as_str)
- .unwrap_or("")
- .to_string();
- let title = item
- .get("title")
- .or_else(|| item.get("name"))
- .and_then(Value::as_str)
- .filter(|value| !value.trim().is_empty())
- .map(ToOwned::to_owned)
- .unwrap_or_else(|| {
- Path::new(&path)
- .file_name()
- .and_then(|s| s.to_str())
- .unwrap_or("AnyTXT result")
- .to_string()
- });
- let snippet = item
- .get("snippet")
- .or_else(|| item.get("fragment"))
- .or_else(|| item.get("text"))
- .and_then(Value::as_str)
- .unwrap_or("")
+ let record = normalize_anytxt_record(item, &fields);
+ let fid = string_field(&record, &["fid", "id", "fileId", "file_id"]);
+ let raw_path = string_field(
+ &record,
+ &[
+ "path",
+ "file",
+ "filePath",
+ "file_path",
+ "fullPath",
+ "full_path",
+ "filename",
+ "fileName",
+ "name",
+ ],
+ );
+ let path = if raw_path.is_empty() && !fid.is_empty() {
+ format!("anytxt://{fid}")
+ } else {
+ raw_path
+ };
+ let title = string_field(&record, &["title", "name", "fileName", "filename"])
+ .trim()
.to_string();
+ let title = if title.is_empty() {
+ Path::new(&path)
+ .file_name()
+ .and_then(|s| s.to_str())
+ .filter(|value| !value.trim().is_empty())
+ .unwrap_or("AnyTXT result")
+ .to_string()
+ } else {
+ title
+ };
+ let snippet = string_field(
+ &record,
+ &[
+ "snippet",
+ "fragment",
+ "content",
+ "contents",
+ "text",
+ "summary",
+ "highlight",
+ "hitText",
+ "hit_text",
+ ],
+ );
if path.is_empty() && snippet.is_empty() {
None
} else {
Some(AnyTxtItem {
+ fid,
title,
path,
snippet,
@@ -729,12 +801,163 @@ fn extract_anytxt_items(value: &Value) -> Vec<AnyTxtItem> {
.collect()
}
-fn anytxt_pattern_from_query(query: &str) -> String {
- let terms = source_query_terms(&query.to_lowercase());
- if terms.is_empty() {
- return query.trim().to_string();
+fn first_anytxt_array(value: &Value, paths: &[&[&str]]) -> Option<Vec<Value>> {
+ for path in paths {
+ let Some(candidate) = value_at_path(value, path) else {
+ continue;
+ };
+ if let Some(items) = candidate.as_array() {
+ return Some(items.clone());
+ }
+ }
+ None
+}
+
+fn first_anytxt_fields(value: &Value, paths: &[&[&str]]) -> Option<Vec<String>> {
+ for path in paths {
+ let Some(candidate) = value_at_path(value, path) else {
+ continue;
+ };
+ let Some(items) = candidate.as_array() else {
+ continue;
+ };
+ let fields = items
+ .iter()
+ .filter_map(Value::as_str)
+ .map(ToOwned::to_owned)
+ .collect::<Vec<_>>();
+ if !fields.is_empty() {
+ return Some(fields);
+ }
+ }
+ None
+}
+
+fn value_at_path<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> {
+ let mut current = value;
+ for key in path {
+ current = current.get(*key)?;
+ }
+ Some(current)
+}
+
+fn normalize_anytxt_record(item: Value, fields: &[String]) -> serde_json::Map<String, Value> {
+ match item {
+ Value::Object(object) => object,
+ Value::Array(row) if !fields.is_empty() => fields
+ .iter()
+ .cloned()
+ .zip(row)
+ .collect::<serde_json::Map<String, Value>>(),
+ other => {
+ let mut object = serde_json::Map::new();
+ object.insert("text".to_string(), other);
+ object
+ }
+ }
+}
+
+fn string_field(record: &serde_json::Map<String, Value>, keys: &[&str]) -> String {
+ for key in keys {
+ let Some(value) = record.get(*key) else {
+ continue;
+ };
+ if let Some(text) = value.as_str().filter(|text| !text.trim().is_empty()) {
+ return text.trim().to_string();
+ }
+ if let Some(number) = value.as_i64() {
+ return number.to_string();
+ }
+ if let Some(number) = value.as_u64() {
+ return number.to_string();
+ }
+ }
+ String::new()
+}
+
+async fn get_anytxt_fragment(
+ client: &reqwest::Client,
+ endpoint: &str,
+ fid: &str,
+ pattern: &str,
+) -> Result<String, String> {
+ let response = client
+ .post(endpoint)
+ .header("Accept", "application/json")
+ .json(&json!({
+ "id": 2,
+ "jsonrpc": "2.0",
+ "method": "ATRpcServer.Searcher.V1.GetFragment",
+ "params": { "input": { "fid": fid, "pattern": pattern } }
+ }))
+ .send()
+ .await
+ .map_err(|err| format!("AnyTXT fragment failed: {err}"))?;
+ let status = response.status();
+ let text = response
+ .text()
+ .await
+ .map_err(|err| format!("Failed to read AnyTXT fragment response: {err}"))?;
+ if !status.is_success() {
+ return Err(format!(
+ "AnyTXT fragment HTTP {status}: {}",
+ trim_text(&text, 300)
+ ));
+ }
+ let value: Value = serde_json::from_str(&text).map_err(|_| {
+ format!(
+ "AnyTXT fragment returned invalid JSON: {}",
+ trim_text(&text, 300)
+ )
+ })?;
+ if let Some(error) = value.get("error") {
+ return Err(format!(
+ "AnyTXT fragment error: {}",
+ trim_text(&error.to_string(), 300)
+ ));
+ }
+ Ok(extract_anytxt_fragment_text(
+ value.get("result").unwrap_or(&Value::Null),
+ ))
+}
+
+fn extract_anytxt_fragment_text(value: &Value) -> String {
+ if let Some(text) = value.as_str() {
+ return text.to_string();
+ }
+ if let Some(items) = value.as_array() {
+ return items
+ .iter()
+ .map(extract_anytxt_fragment_text)
+ .filter(|item| !item.trim().is_empty())
+ .collect::<Vec<_>>()
+ .join("\n\n");
+ }
+ let Some(object) = value.as_object() else {
+ return String::new();
+ };
+ for key in ["text", "fragment", "content", "snippet", "html"] {
+ if let Some(text) = object.get(key).and_then(Value::as_str) {
+ return text.to_string();
+ }
+ }
+ for key in ["output", "result", "data", "fragments", "items", "list"] {
+ if let Some(next) = object.get(key) {
+ let text = extract_anytxt_fragment_text(next);
+ if !text.trim().is_empty() {
+ return text;
+ }
+ }
+ }
+ String::new()
+}
+
+fn normalize_anytxt_endpoint(value: &str) -> String {
+ if value.starts_with("http://") || value.starts_with("https://") {
+ value.to_string()
+ } else {
+ format!("http://{value}")
}
- terms.into_iter().take(5).collect::<Vec<_>>().join(" ")
}
#[derive(Debug, Clone)]
@@ -775,12 +998,7 @@ async fn firecrawl_search(
.unwrap_or_else(|| format!("Firecrawl search failed ({status})"));
return Err(msg);
}
- let items = parsed
- .get("data")
- .and_then(Value::as_array)
- .or_else(|| parsed.get("results").and_then(Value::as_array))
- .cloned()
- .unwrap_or_default();
+ let items = extract_web_items(&parsed, &["data", "results"]);
Ok(items.into_iter().map(normalize_web_result).collect())
}
@@ -868,6 +1086,45 @@ async fn tavily_search(
.await
}
+async fn ollama_search(
+ client: &reqwest::Client,
+ query: &str,
+ config: &WebSearchConfig,
+ max_results: usize,
+) -> Result<Vec<WebSearchItem>, String> {
+ let key = required_api_key(config, "Ollama")?;
+ let base = config
+ .ollama_url
+ .as_deref()
+ .filter(|value| !value.trim().is_empty())
+ .unwrap_or("https://ollama.com")
+ .trim()
+ .trim_end_matches('/');
+ let url = format!("{base}/api/web_search");
+ let response = client
+ .post(url)
+ .header("Accept", "application/json")
+ .bearer_auth(key)
+ .json(&json!({
+ "query": query,
+ "max_results": max_results
+ }))
+ .send()
+ .await
+ .map_err(|err| format!("Network error reaching Ollama Web Search: {err}"))?;
+ parse_web_json_response(response, "Ollama Web Search", |value| {
+ value
+ .get("results")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default()
+ .into_iter()
+ .map(normalize_web_result)
+ .collect()
+ })
+ .await
+}
+
async fn brave_search(
client: &reqwest::Client,
query: &str,
@@ -965,20 +1222,49 @@ async fn parse_web_json_response(
if let Some(error) = value.get("error").and_then(Value::as_str) {
return Err(format!("{provider} search failed: {error}"));
}
+ if let Some(message) = provider_payload_error(provider, &value) {
+ return Err(message);
+ }
Ok(parse(value))
}
+fn provider_payload_error(provider: &str, value: &Value) -> Option<String> {
+ if provider == "Brave Search" && value.get("web").is_none() {
+ let message = value.get("message").and_then(Value::as_str)?;
+ return Some(format!("{provider} search failed: {message}"));
+ }
+ None
+}
+
+fn web_items_to_references(raw: Vec<WebSearchItem>, max_results: usize) -> Vec<AgentReference> {
+ raw.into_iter()
+ .take(max_results)
+ .filter(|item| !item.url.trim().is_empty())
+ .map(|item| AgentReference {
+ title: item.title,
+ path: item.url,
+ kind: "web".to_string(),
+ snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),
+ score: None,
+ })
+ .collect()
+}
+
fn normalize_web_result(value: Value) -> WebSearchItem {
+ let metadata = value.get("metadata");
let title = value
.get("title")
+ .or_else(|| metadata.and_then(|m| m.get("title")))
.and_then(Value::as_str)
.unwrap_or("Untitled")
.to_string();
let url = value
.get("url")
.or_else(|| value.get("link"))
- .or_else(|| value.get("source"))
+ .or_else(|| metadata.and_then(|m| m.get("sourceURL")))
+ .or_else(|| metadata.and_then(|m| m.get("url")))
.or_else(|| value.get("original"))
+ .or_else(|| value.get("thumbnail"))
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
@@ -986,6 +1272,7 @@ fn normalize_web_result(value: Value) -> WebSearchItem {
.get("snippet")
.or_else(|| value.get("content"))
.or_else(|| value.get("description"))
+ .or_else(|| metadata.and_then(|m| m.get("description")))
.or_else(|| value.get("summary"))
.or_else(|| value.get("markdown"))
.and_then(Value::as_str)
@@ -998,6 +1285,31 @@ fn normalize_web_result(value: Value) -> WebSearchItem {
}
}
+fn extract_web_items(value: &Value, keys: &[&str]) -> Vec<Value> {
+ for key in keys {
+ let Some(candidate) = value.get(*key) else {
+ continue;
+ };
+ if let Some(items) = candidate.as_array() {
+ return items.clone();
+ }
+ if let Some(items) = extract_nested_web_items(candidate) {
+ return items;
+ }
+ }
+ Vec::new()
+}
+
+fn extract_nested_web_items(value: &Value) -> Option<Vec<Value>> {
+ let object = value.as_object()?;
+ for key in ["web", "results", "items"] {
+ if let Some(items) = object.get(key).and_then(Value::as_array) {
+ return Some(items.clone());
+ }
+ }
+ None
+}
+
fn required_api_key<'a>(config: &'a WebSearchConfig, provider: &str) -> Result<&'a str, String> {
let key = config.api_key.trim();
if key.is_empty() {
@@ -1312,9 +1624,60 @@ fn normalize_wiki_write_path(path: &str) -> Result<String, String> {
{
return Err("wiki.write_page path must stay inside the project".to_string());
}
+ for segment in rel.split('/') {
+ validate_portable_path_segment(segment)?;
+ }
Ok(rel)
}
+fn validate_portable_path_segment(segment: &str) -> Result<(), String> {
+ if segment.is_empty() {
+ return Err("wiki.write_page path contains an empty segment".to_string());
+ }
+ if segment
+ .chars()
+ .any(|ch| matches!(ch, '<' | '>' | ':' | '"' | '|' | '?' | '*') || ch <= '\u{1f}')
+ {
+ return Err(
+ "wiki.write_page path contains characters that are invalid on Windows".to_string(),
+ );
+ }
+ let stem = segment
+ .split('.')
+ .next()
+ .unwrap_or(segment)
+ .trim_end_matches(' ')
+ .to_ascii_uppercase();
+ if matches!(
+ stem.as_str(),
+ "CON"
+ | "PRN"
+ | "AUX"
+ | "NUL"
+ | "COM1"
+ | "COM2"
+ | "COM3"
+ | "COM4"
+ | "COM5"
+ | "COM6"
+ | "COM7"
+ | "COM8"
+ | "COM9"
+ | "LPT1"
+ | "LPT2"
+ | "LPT3"
+ | "LPT4"
+ | "LPT5"
+ | "LPT6"
+ | "LPT7"
+ | "LPT8"
+ | "LPT9"
+ ) {
+ return Err("wiki.write_page path uses a Windows reserved device name".to_string());
+ }
+ Ok(())
+}
+
fn extract_markdown_title(content: &str) -> Option<String> {
for line in content.lines().take(80) {
let trimmed = line.trim();
@@ -1460,29 +1823,66 @@ mod tests {
let root = std::env::temp_dir().join(format!("llm-wiki-agent-write-{}", Uuid::new_v4()));
fs::create_dir_all(root.join("wiki")).unwrap();
- assert!(write_wiki_page(root.to_str().unwrap(), "../secret.md", "# Secret").is_err());
- assert!(write_wiki_page(root.to_str().unwrap(), "raw/sources/a.md", "# A").is_err());
- assert!(write_wiki_page(root.to_str().unwrap(), "wiki/.hidden/a.md", "# A").is_err());
- assert!(write_wiki_page(
+ assert!(write_wiki_page_with_options(
+ root.to_str().unwrap(),
+ "../secret.md",
+ "# Secret",
+ false
+ )
+ .is_err());
+ assert!(write_wiki_page_with_options(
+ root.to_str().unwrap(),
+ "raw/sources/a.md",
+ "# A",
+ false
+ )
+ .is_err());
+ assert!(write_wiki_page_with_options(
+ root.to_str().unwrap(),
+ "wiki/.hidden/a.md",
+ "# A",
+ false
+ )
+ .is_err());
+ assert!(
+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/aux.md", "# A", false)
+ .is_err()
+ );
+ assert!(
+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/con.md", "# A", false)
+ .is_err()
+ );
+ assert!(
+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/a:b.md", "# A", false)
+ .is_err()
+ );
+ assert!(
+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/a?b.md", "# A", false)
+ .is_err()
+ );
+ assert!(write_wiki_page_with_options(
root.to_str().unwrap(),
"wiki/queries/huge.md",
- &"x".repeat(MAX_WRITE_PAGE_BYTES + 1)
+ &"x".repeat(MAX_WRITE_PAGE_BYTES + 1),
+ false,
)
.is_err());
- let reference = write_wiki_page(
+ let reference = write_wiki_page_with_options(
root.to_str().unwrap(),
"wiki/queries/new-page.md",
"---\ntitle: New Page\n---\n# New Page\n\nBody",
+ false,
)
.unwrap();
assert_eq!(reference.title, "New Page");
assert_eq!(reference.path, "wiki/queries/new-page.md");
assert!(root.join("wiki/queries/new-page.md").exists());
- let overwrite_err = write_wiki_page(
+ let overwrite_err = write_wiki_page_with_options(
root.to_str().unwrap(),
"wiki/queries/new-page.md",
"# Replaced",
+ false,
)
.unwrap_err();
assert!(overwrite_err.contains("refuses to overwrite"));
@@ -1509,10 +1909,11 @@ mod tests {
fs::create_dir_all(&outside).unwrap();
symlink(&outside, root.join("wiki").join("linked")).unwrap();
- let err = write_wiki_page(
+ let err = write_wiki_page_with_options(
root.to_str().unwrap(),
"wiki/linked/newsub/escape.md",
"# Escape",
+ false,
)
.unwrap_err();
assert!(err.contains("escapes project directory"));
@@ -1605,6 +2006,87 @@ mod tests {
assert!(msg.contains("rejected this IP"));
}
+ #[test]
+ fn run_web_search_drops_empty_url_results_before_mapping_references() {
+ let refs = web_items_to_references(
+ vec![
+ WebSearchItem {
+ title: "Missing".to_string(),
+ url: String::new(),
+ snippet: "no url".to_string(),
+ },
+ WebSearchItem {
+ title: "Valid".to_string(),
+ url: "https://example.com".to_string(),
+ snippet: "ok".to_string(),
+ },
+ ],
+ 10,
+ );
+ assert_eq!(refs.len(), 1);
+ assert_eq!(refs[0].title, "Valid");
+ assert_eq!(refs[0].path, "https://example.com");
+ }
+
+ #[test]
+ fn web_references_apply_limit_before_empty_url_filter_like_legacy_ui() {
+ let refs = web_items_to_references(
+ vec![
+ WebSearchItem {
+ title: "Missing".to_string(),
+ url: String::new(),
+ snippet: "no url".to_string(),
+ },
+ WebSearchItem {
+ title: "Valid".to_string(),
+ url: "https://example.com".to_string(),
+ snippet: "ok".to_string(),
+ },
+ ],
+ 1,
+ );
+ assert!(refs.is_empty());
+ }
+
+ #[test]
+ fn brave_message_without_web_is_treated_as_error() {
+ let value = json!({ "message": "invalid subscription token" });
+ assert_eq!(
+ provider_payload_error("Brave Search", &value),
+ Some("Brave Search search failed: invalid subscription token".to_string())
+ );
+ }
+
+ #[test]
+ fn non_brave_message_does_not_mask_valid_provider_payloads() {
+ let value = json!({ "message": "FYI" });
+ assert_eq!(provider_payload_error("Tavily", &value), None);
+ }
+
+ #[test]
+ fn web_result_normalization_accepts_firecrawl_nested_metadata() {
+ let items = extract_web_items(
+ &json!({
+ "data": {
+ "web": [
+ {
+ "metadata": {
+ "title": "Nested",
+ "sourceURL": "https://example.com/nested",
+ "description": "from metadata"
+ }
+ }
+ ]
+ }
+ }),
+ &["data", "results"],
+ );
+ let item = normalize_web_result(items.into_iter().next().unwrap());
+ assert_eq!(item.title, "Nested");
+ assert_eq!(item.url, "https://example.com/nested");
+ assert_eq!(item.snippet, "from metadata");
+ }
+
#[test]
fn url_encode_handles_unicode_terms() {
assert_eq!(url_encode("煤矿 safety"), "%E7%85%A4%E7%9F%BF+safety");
@@ -1650,10 +2132,56 @@ mod tests {
}
#[test]
- fn anytxt_pattern_keeps_multiple_useful_terms() {
- assert_eq!(
- anytxt_pattern_from_query("请搜索 原始资料 煤矿 安全 治理 的材料"),
- "请搜索 煤矿 安全 治理 的材料"
- );
+ fn extract_anytxt_items_accepts_nested_output_and_field_rows() {
+ let value = json!({
+ "result": {
+ "output": {
+ "field": ["fid", "full_path", "title", "hitText"],
+ "items": [
+ ["42", "/docs/煤矿.pdf", "煤矿资料", "煤矿安全治理片段"]
+ ]
+ }
+ }
+ });
+ let items = extract_anytxt_items(&value);
+ assert_eq!(items.len(), 1);
+ assert_eq!(items[0].fid, "42");
+ assert_eq!(items[0].path, "/docs/煤矿.pdf");
+ assert_eq!(items[0].title, "煤矿资料");
+ assert_eq!(items[0].snippet, "煤矿安全治理片段");
+ }
+
+ #[test]
+ fn extract_anytxt_items_keeps_fid_only_results_addressable() {
+ let value = json!({
+ "result": {
+ "data": {
+ "results": [
+ { "fid": 99, "snippet": "fragment only" }
+ ]
+ }
+ }
+ });
+ let items = extract_anytxt_items(&value);
+ assert_eq!(items.len(), 1);
+ assert_eq!(items[0].path, "anytxt://99");
+ assert_eq!(items[0].snippet, "fragment only");
+ }
+
+ #[test]
+ fn extract_anytxt_items_accepts_value_shapes() {
+ let value = json!({
+ "result": {
+ "output": {
+ "value": [
+ { "path": "/docs/value.txt", "snippet": "from value" }
+ ]
+ }
+ }
+ });
+ let items = extract_anytxt_items(&value);
+ assert_eq!(items.len(), 1);
+ assert_eq!(items[0].path, "/docs/value.txt");
+ assert_eq!(items[0].snippet, "from value");
}
}
diff --git a/src-tauri/src/commands/external_search.rs b/src-tauri/src/commands/external_search.rs
new file mode 100644
index 00000000..9900e7df
--- /dev/null
+++ b/src-tauri/src/commands/external_search.rs
@@ -0,0 +1,122 @@
+use serde::{Deserialize, Serialize};
+
+use crate::agent::tools::{run_anytxt_search, run_web_search, AnyTxtConfig, WebSearchConfig};
+use crate::panic_guard::run_guarded_async;
+
+/// Frontend-facing search result shape. The Rust Agent uses
+/// `AgentReference` internally, but UI/deep-research code historically
+/// consumes `{ title, url, snippet, source }`; keep that wire contract
+/// stable while moving provider/network logic to Rust.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExternalSearchResult {
+ pub title: String,
+ pub url: String,
+ pub snippet: String,
+ pub source: String,
+}
+
+#[tauri::command]
+pub async fn web_search(
+ query: String,
+ config: WebSearchConfig,
+ max_results: Option<usize>,
+) -> Result<Vec<ExternalSearchResult>, String> {
+ run_guarded_async("web_search", async move {
+ let references = run_web_search(&query, Some(config), max_results.unwrap_or(10)).await?;
+ Ok(references
+ .into_iter()
+ .map(|item| ExternalSearchResult {
+ title: item.title,
+ source: hostname_label(&item.path).unwrap_or_else(|| "web".to_string()),
+ url: item.path,
+ snippet: item.snippet.unwrap_or_default(),
+ })
+ .collect())
+ })
+ .await
+}
+
+#[tauri::command]
+pub async fn anytxt_search(
+ query: String,
+ config: AnyTxtConfig,
+ max_results: Option<usize>,
+) -> Result<Vec<ExternalSearchResult>, String> {
+ run_guarded_async("anytxt_search", async move {
+ let references = run_anytxt_search(&query, Some(config), max_results.unwrap_or(20)).await?;
+ Ok(references
+ .into_iter()
+ .map(|item| ExternalSearchResult {
+ title: item.title,
+ url: file_url_for_path(&item.path),
+ snippet: item.snippet.unwrap_or_default(),
+ source: "AnyTXT".to_string(),
+ })
+ .collect())
+ })
+ .await
+}
+
+fn hostname_label(url: &str) -> Option<String> {
+ let host = reqwest::Url::parse(url).ok()?.host_str()?.to_string();
+ Some(host.strip_prefix("www.").unwrap_or(&host).to_string())
+}
+
+pub(crate) fn file_url_for_path(path: &str) -> String {
+ let normalized = path.replace('\\', "/");
+ if normalized.is_empty() || normalized.contains("://") {
+ return normalized;
+ }
+ if normalized.starts_with("//") {
+ return format!("file:{normalized}");
+ }
+ if normalized.len() >= 3
+ && normalized.as_bytes()[1] == b':'
+ && normalized.as_bytes()[2] == b'/'
+ && normalized.as_bytes()[0].is_ascii_alphabetic()
+ {
+ return format!("file:///{}", encode_file_url_path(&normalized));
+ }
+ if normalized.starts_with('/') {
+ return format!("file://{}", encode_file_url_path(&normalized));
+ }
+ normalized
+}
+
+fn encode_file_url_path(path: &str) -> String {
+ path.split('/')
+ .map(percent_encode_file_segment)
+ .collect::<Vec<_>>()
+ .join("/")
+}
+
+fn percent_encode_file_segment(segment: &str) -> String {
+ let mut out = String::new();
+ for byte in segment.as_bytes() {
+ if byte.is_ascii_alphanumeric() || matches!(*byte, b'-' | b'.' | b'_' | b'~' | b':') {
+ out.push(*byte as char);
+ } else {
+ out.push_str(&format!("%{byte:02X}"));
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::file_url_for_path;
+
+ #[test]
+ fn anytxt_paths_are_returned_as_file_urls_for_frontend_results() {
+ assert_eq!(
+ file_url_for_path(r"C:\docs\煤矿 安全.pdf"),
+ "file:///C:/docs/%E7%85%A4%E7%9F%BF%20%E5%AE%89%E5%85%A8.pdf"
+ );
+ assert_eq!(
+ file_url_for_path("/Users/me/docs/a b.txt"),
+ "file:///Users/me/docs/a%20b.txt"
+ );
+ assert_eq!(file_url_for_path("anytxt://99"), "anytxt://99");
+ }
+}
diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs
index 911dc610..e13992d0 100644
--- a/src-tauri/src/commands/mod.rs
+++ b/src-tauri/src/commands/mod.rs
@@ -1,6 +1,7 @@
pub mod claude_cli;
mod cli_resolver;
pub mod codex_cli;
+pub mod external_search;
pub mod extract_images;
pub mod file_sync;
pub mod fs;
diff --git a/src-tauri/src/commands/search.rs b/src-tauri/src/commands/search.rs
index b8e487a7..eb49c7fc 100644
--- a/src-tauri/src/commands/search.rs
+++ b/src-tauri/src/commands/search.rs
@@ -60,7 +60,7 @@ pub struct SearchEmbeddingConfig {
pub endpoint: String,
pub api_key: String,
pub model: String,
- pub output_dimensionality: Option<u32>,
+ pub output_dimensionality: Option<f64>,
/// Extra HTTP headers to send with every embedding request, e.g.
/// `X-Model-Provider-Id: siliconflow` for the mify gateway.
/// Reserved names (Authorization, Content-Type, Host,
@@ -94,6 +94,18 @@ pub async fn search_project(
.await
}
+#[tauri::command]
+pub async fn embedding_fetch(
+ text: String,
+ cfg: SearchEmbeddingConfig,
+ max_retries: Option<usize>,
+) -> Result<Vec<f32>, String> {
+ run_guarded_async("embedding_fetch", async move {
+ fetch_embedding_with_retry(&text, &cfg, max_retries.unwrap_or(3)).await
+ })
+ .await
+}
+
pub async fn resolve_query_embedding(
query: &str,
explicit_embedding: Option<Vec<f32>>,
@@ -108,7 +120,7 @@ pub async fn resolve_query_embedding(
if !cfg.enabled || cfg.endpoint.trim().is_empty() || cfg.model.trim().is_empty() {
return Ok(None);
}
- match fetch_embedding(query, &cfg).await {
+ match fetch_embedding_with_retry(query, &cfg, 0).await {
Ok(embedding) => validate_query_embedding(embedding).map(Some),
Err(err) => {
eprintln!("[Search] embedding disabled for this request: {err}");
@@ -666,7 +678,59 @@ pub fn extract_image_refs(content: &str) -> Vec<SearchImageRef> {
out
}
-async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<f32>, String> {
+async fn fetch_embedding_with_retry(
+ text: &str,
+ cfg: &SearchEmbeddingConfig,
+ max_retries: usize,
+) -> Result<Vec<f32>, String> {
+ let mut current = text.to_string();
+ let mut attempts = 0usize;
+ loop {
+ attempts += 1;
+ match fetch_embedding_once(&current, cfg).await {
+ Ok(embedding) => return Ok(embedding),
+ Err(EmbeddingFetchError::Oversize(message)) => {
+ if attempts <= max_retries
+ && current.len() > 64
+ && halve_text_on_char_boundary(&mut current)
+ {
+ eprintln!(
+ "[Embedding] auto-halving after oversize error at {} chars; retrying at {} chars ({attempts}/{})",
+ text.chars().count(),
+ current.chars().count(),
+ max_retries + 1
+ );
+ continue;
+ }
+ return Err(format!(
+ "Endpoint rejected input even at {} chars. Lower Settings -> Embedding -> Max Chunk Chars. {message}",
+ current.len()
+ ));
+ }
+ Err(EmbeddingFetchError::Other(message)) => return Err(message),
+ }
+ }
+}
+
+fn halve_text_on_char_boundary(text: &mut String) -> bool {
+ let char_count = text.chars().count();
+ if char_count <= 1 {
+ return false;
+ }
+ let keep = (char_count / 2).max(1);
+ *text = text.chars().take(keep).collect();
+ true
+}
+
+enum EmbeddingFetchError {
+ Oversize(String),
+ Other(String),
+}
+
+async fn fetch_embedding_once(
+ text: &str,
+ cfg: &SearchEmbeddingConfig,
+) -> Result<Vec<f32>, EmbeddingFetchError> {
let is_google = is_google_embedding_config(cfg);
let is_doubao_multimodal = is_doubao_multimodal_embedding_config(cfg);
let endpoint = if is_google {
@@ -679,9 +743,15 @@ async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<
SEARCH_EMBEDDING_TIMEOUT_SECS,
))
.build()
- .map_err(|e| format!("Embedding HTTP client error: {e}"))?
- .post(endpoint)
+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding HTTP client error: {e}")))?
+ .post(&endpoint)
.header("Content-Type", "application/json");
+ // Browser-based local model servers often require a browser-like
+ // Origin even when the request is routed through Rust. Keep this
+ // reserved so user-supplied extra headers cannot override it.
+ if is_local_or_private_http_endpoint(&endpoint) {
+ req = req.header("Origin", "http://localhost");
+ }
if !cfg.api_key.trim().is_empty() {
if is_google {
req = req.header("x-goog-api-key", cfg.api_key.trim());
@@ -713,19 +783,46 @@ async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<
.json(&body)
.send()
.await
- .map_err(|e| format!("Embedding request failed: {e}"))?;
+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding request failed: {e}")))?;
let status = resp.status();
- let data: Value = resp
- .json()
+ let text = resp
+ .text()
.await
- .map_err(|e| format!("Embedding response parse failed: {e}"))?;
+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding response read failed: {e}")))?;
if !status.is_success() {
- return Err(format!(
- "Embedding API HTTP {status}: {}",
- data.to_string().chars().take(200).collect::<String>()
- ));
+ let preview = text.chars().take(200).collect::<String>();
+ if looks_like_oversize_error(status.as_u16(), &text) {
+ return Err(EmbeddingFetchError::Oversize(format!(
+ "Embedding API HTTP {status}: {preview}"
+ )));
+ }
+ return Err(EmbeddingFetchError::Other(format!(
+ "Embedding API HTTP {status}: {preview}"
+ )));
}
+ let data: Value = serde_json::from_str(&text).map_err(|e| {
+ EmbeddingFetchError::Other(format!(
+ "Embedding response parse failed: {e}: {}",
+ text.chars().take(200).collect::<String>()
+ ))
+ })?;
parse_embedding_values(&data, is_google, is_doubao_multimodal)
+ .map_err(EmbeddingFetchError::Other)
+}
+
+fn looks_like_oversize_error(status: u16, body: &str) -> bool {
+ if status == 413 {
+ return true;
+ }
+ let lower = body.to_lowercase();
+ lower.contains("too long")
+ || lower.contains("maximum context")
+ || lower.contains("max_tokens")
+ || lower.contains("max tokens")
+ || lower.contains("context length")
+ || lower.contains("token limit")
+ || lower.contains("exceeds")
+ || lower.contains("input length")
}
fn parse_embedding_values(
@@ -794,7 +891,7 @@ fn is_safe_extra_header_name(name: &str) -> bool {
fn is_reserved_extra_header_name(name: &str) -> bool {
matches!(
name.trim().to_ascii_lowercase().as_str(),
- "authorization" | "content-type" | "host" | "content-length" | "x-goog-api-key"
+ "authorization" | "content-type" | "host" | "content-length" | "origin" | "x-goog-api-key"
)
}
@@ -803,6 +900,39 @@ fn is_google_embedding_config(cfg: &SearchEmbeddingConfig) -> bool {
endpoint.contains("generativelanguage.googleapis.com") || endpoint.contains(":embedcontent")
}
+fn is_local_or_private_http_endpoint(endpoint: &str) -> bool {
+ let Ok(url) = reqwest::Url::parse(endpoint) else {
+ return false;
+ };
+ if !matches!(url.scheme(), "http" | "https") {
+ return false;
+ }
+ let Some(host) = url.host_str() else {
+ return false;
+ };
+ let host = host
+ .trim_matches('[')
+ .trim_matches(']')
+ .to_ascii_lowercase();
+ if matches!(host.as_str(), "localhost" | "127.0.0.1" | "::1") {
+ return true;
+ }
+ let octets = host
+ .split('.')
+ .map(str::parse::<u8>)
+ .collect::<Result<Vec<_>, _>>();
+ let Ok(octets) = octets else {
+ return false;
+ };
+ if octets.len() != 4 {
+ return false;
+ }
+ octets[0] == 10
+ || (octets[0] == 172 && (16..=31).contains(&octets[1]))
+ || (octets[0] == 192 && octets[1] == 168)
+ || octets[0] == 127
+}
+
fn is_volcengine_embedding_endpoint(endpoint: &str) -> bool {
let host = reqwest::Url::parse(endpoint)
.ok()
@@ -942,7 +1072,7 @@ fn strip_google_api_key_query(endpoint: &str) -> String {
}
}
-fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<u32>) -> Value {
+fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<f64>) -> Value {
let model_path = if model.trim().starts_with("models/") {
model.trim().to_string()
} else {
@@ -952,7 +1082,10 @@ fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<
"model": model_path,
"content": { "parts": [{ "text": text }] },
});
- if let Some(dim) = output_dimensionality.filter(|dim| *dim > 0) {
+ if let Some(dim) = output_dimensionality
+ .filter(|dim| dim.is_finite() && *dim >= 1.0)
+ .map(|dim| dim.floor() as u32)
+ {
body["output_dimensionality"] = json!(dim);
}
body
@@ -1100,7 +1233,7 @@ mod tests {
endpoint: "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents?key=URL_KEY&alt=json".to_string(),
api_key: "HEADER_KEY".to_string(),
model: "gemini-embedding-001".to_string(),
- output_dimensionality: Some(768),
+ output_dimensionality: Some(768.0),
extra_headers: None,
};
@@ -1110,7 +1243,7 @@ mod tests {
assert!(!endpoint.contains("URL_KEY"));
assert!(endpoint.contains("alt=json"));
- let body = google_embedding_body("gemini-embedding-001", "hello", Some(768));
+ let body = google_embedding_body("gemini-embedding-001", "hello", Some(768.0));
assert_eq!(body["model"], "models/gemini-embedding-001");
assert_eq!(body["output_dimensionality"], 768);
}
@@ -1244,10 +1377,46 @@ mod tests {
assert!(is_reserved_extra_header_name("Authorization"));
assert!(is_reserved_extra_header_name("content-type"));
+ assert!(is_reserved_extra_header_name("Origin"));
assert!(is_reserved_extra_header_name("X-Goog-Api-Key"));
assert!(!is_reserved_extra_header_name("X-Model-Provider-Id"));
}
+ #[test]
+ fn embedding_origin_header_is_limited_to_local_or_private_endpoints() {
+ assert!(is_local_or_private_http_endpoint(
+ "http://127.0.0.1:1234/v1/embeddings"
+ ));
+ assert!(is_local_or_private_http_endpoint(
+ "http://192.168.1.20:11434/v1/embeddings"
+ ));
+ assert!(is_local_or_private_http_endpoint(
+ "http://172.16.0.5/v1/embeddings"
+ ));
+ assert!(!is_local_or_private_http_endpoint(
+ "https://api.openai.com/v1/embeddings"
+ ));
+ }
+
+ #[test]
+ fn embedding_halving_never_splits_cjk_codepoints() {
+ let mut text = "默会知识库".to_string();
+ assert!(halve_text_on_char_boundary(&mut text));
+ assert_eq!(text, "默会");
+ }
+
+ #[test]
+ fn google_embedding_body_floors_positive_dimensions_and_omits_invalid_values() {
+ let body = google_embedding_body("gemini-embedding-001", "hello", Some(1.9));
+ assert_eq!(body["output_dimensionality"], 1);
+
+ let zero = google_embedding_body("gemini-embedding-001", "hello", Some(0.0));
+ assert!(zero.get("output_dimensionality").is_none());
+
+ let negative = google_embedding_body("gemini-embedding-001", "hello", Some(-4.0));
+ assert!(negative.get("output_dimensionality").is_none());
+ }
+
#[test]
fn rrf_combines_token_and_vector_ranks_and_keeps_vector_score() {
let mut results = vec![
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 8073ab20..e329fc90 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -636,6 +636,9 @@ pub fn run() {
commands::project::open_project,
commands::project::open_project_folder,
commands::search::search_project,
+ commands::search::embedding_fetch,
+ commands::external_search::web_search,
+ commands::external_search::anytxt_search,
clip_server_status,
api_server_status,
api_server_reload_config,
diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx
index aab5cec1..c39ecaf2 100644
--- a/src/components/chat/chat-panel.tsx
+++ b/src/components/chat/chat-panel.tsx
@@ -219,6 +219,8 @@ function normalizeBackendToolName(tool: string) {
const normalized = tool.split(".").join("_")
if (normalized === "wiki_search") return "wiki_search" as const
if (normalized === "wiki_read_page") return "project_file_read" as const
+ if (normalized === "wiki_write_page") return "project_files" as const
+ if (normalized === "skills_load") return "project_file_read" as const
if (normalized === "source_search") return "project_file_read" as const
if (normalized === "graph_search") return "graph_search" as const
if (normalized === "web_search") return "web_search" as const
diff --git a/src/lib/anytxt-search.test.ts b/src/lib/anytxt-search.test.ts
index 59d81796..30e0ec7f 100644
--- a/src/lib/anytxt-search.test.ts
+++ b/src/lib/anytxt-search.test.ts
@@ -8,14 +8,17 @@ import {
prepareAnyTxtQueries,
} from "./anytxt-search"
+const invokeMock = vi.hoisted(() => vi.fn())
const streamChatMock = vi.hoisted(() => vi.fn())
+vi.mock("@tauri-apps/api/core", () => ({
+ invoke: invokeMock,
+}))
+
vi.mock("@/lib/llm-client", () => ({
streamChat: streamChatMock,
}))
-const fetchMock = vi.fn<typeof fetch>()
-
const llmConfig: LlmConfig = {
provider: "custom",
apiKey: "test",
@@ -25,47 +28,16 @@ const llmConfig: LlmConfig = {
maxContextSize: 128000,
}
-function jsonResponse(body: unknown, init?: ResponseInit): Response {
- return new Response(JSON.stringify(body), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- ...init,
- })
-}
-
describe("anyTxtSearch", () => {
beforeEach(() => {
- fetchMock.mockReset()
+ invokeMock.mockReset()
streamChatMock.mockReset()
- vi.stubGlobal("fetch", fetchMock)
})
- it("calls AnyTXT GetResult and fetches fragments for matching files", async () => {
- fetchMock
- .mockResolvedValueOnce(jsonResponse({
- jsonrpc: "2.0",
- id: 1,
- result: {
- output: {
- items: [
- {
- fid: "9223736511748752040",
- path: "C:\\docs\\alpha.pdf",
- name: "alpha.pdf",
- },
- ],
- },
- },
- }))
- .mockResolvedValueOnce(jsonResponse({
- jsonrpc: "2.0",
- id: 2,
- result: {
- output: {
- text: "matching fragment text",
- },
- },
- }))
+ it("delegates AnyTXT JSON-RPC search to the Rust backend command", async () => {
+ invokeMock.mockResolvedValueOnce([
+ { title: "alpha.pdf", url: "/docs/alpha.pdf", snippet: "matching fragment text", source: "AnyTXT" },
+ ])
const out = await anyTxtSearch(
"alpha",
@@ -78,219 +50,56 @@ describe("anyTxtSearch", () => {
5,
)
- expect(fetchMock).toHaveBeenCalledTimes(2)
- const [url, init] = fetchMock.mock.calls[0]
- expect(url).toBe("http://127.0.0.1:9920")
- expect(JSON.parse(String(init?.body))).toMatchObject({
- jsonrpc: "2.0",
- method: "ATRpcServer.Searcher.V1.GetResult",
- params: {
- input: {
- pattern: "alpha",
- filterDir: "C:\\docs",
- filterExt: "*.pdf",
- limit: "5",
- },
- },
- })
- expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toMatchObject({
- method: "ATRpcServer.Searcher.V1.GetFragment",
- params: { input: { fid: "9223736511748752040", pattern: "alpha" } },
- })
- expect(out).toEqual([
- {
- title: "alpha.pdf",
- url: "file:///C:/docs/alpha.pdf",
- snippet: "matching fragment text",
- source: "AnyTXT",
- },
- ])
- })
-
- it("normalizes tolerant AnyTXT response shapes without fragments", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- result: {
- output: [
- {
- full_path: "/Users/me/docs/note.md",
- hitText: "local note match",
- },
- ],
+ expect(invokeMock).toHaveBeenCalledWith("anytxt_search", {
+ query: "alpha",
+ maxResults: 5,
+ config: {
+ enabled: true,
+ endpoint: "http://127.0.0.1:9920",
+ filterDir: "C:\\docs",
+ filterExt: "*.pdf",
+ limit: 10,
},
- }))
-
- const out = await anyTxtSearch("note", { endpoint: "http://127.0.0.1:9920" }, 3, "/project")
-
- expect(fetchMock).toHaveBeenCalledTimes(1)
- expect(out[0]).toMatchObject({
- title: "note.md",
- url: "file:///Users/me/docs/note.md",
- snippet: "local note match",
- source: "AnyTXT",
})
- })
-
- it("normalizes AnyTXT data.output.files rows using the returned field list", async () => {
- fetchMock
- .mockResolvedValueOnce(jsonResponse({
- id: 1,
- jsonrpc: "2.0",
- result: {
- data: {
- output: {
- count: 1,
- field: ["fid", "lastModify", "size", "file"],
- files: [["9223736511748752040", 1711588801, 1200, "D:\\docs\\煤矿安全.md"]],
- },
- },
- errno: 0,
- },
- }))
- .mockResolvedValueOnce(jsonResponse({
- result: {
- data: {
- output: {
- text: "煤矿安全 fragment",
- },
- },
- },
- }))
-
- const out = await anyTxtSearch("煤矿", { endpoint: "http://127.0.0.1:9920" }, 5)
-
expect(out).toEqual([
- {
- title: "煤矿安全.md",
- url: "file:///D:/docs/%E7%85%A4%E7%9F%BF%E5%AE%89%E5%85%A8.md",
- snippet: "煤矿安全 fragment",
- source: "AnyTXT",
- },
+ { title: "alpha.pdf", url: "/docs/alpha.pdf", snippet: "matching fragment text", source: "AnyTXT" },
])
})
- it("normalizes UNC network paths to file URLs", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- result: {
- data: {
- output: {
- field: ["fid", "file"],
- files: [["unc-1", "\\\\nas\\share\\研究报告.md"]],
- },
- },
- },
- }))
-
- const out = await anyTxtSearch("研究", { endpoint: "http://127.0.0.1:9920" }, 5)
-
- expect(out[0]).toMatchObject({
- title: "研究报告.md",
- url: "file://nas/share/%E7%A0%94%E7%A9%B6%E6%8A%A5%E5%91%8A.md",
- source: "AnyTXT",
- })
- })
-
it("uses the Unix root for blank filterDir on macOS/Linux projects", () => {
- expect(normalizeAnyTxtConfig({ filterDir: "" }, "/Users/me/wiki").filterDir)
+ expect(normalizeAnyTxtConfig({ endpoint: "127.0.0.1:9920", filterDir: "" }, "/Users/me/wiki").filterDir)
.toBe("/")
})
it("leaves blank filterDir unset for Windows projects", () => {
- expect(normalizeAnyTxtConfig({ filterDir: "" }, "C:/Users/me/wiki").filterDir)
+ expect(normalizeAnyTxtConfig({ endpoint: "127.0.0.1:9920", filterDir: "" }, "C:/Users/me/wiki").filterDir)
.toBe("")
})
- it("sends the Unix root as filterDir when folder filtering is blank on macOS/Linux", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ result: { items: [] } }))
-
- await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920", filterDir: "" }, 5, "/Users/me/wiki")
-
- const body = JSON.parse(String(fetchMock.mock.calls[0][1]?.body))
- expect(body.params.input.filterDir).toBe("/")
- })
-
it("normalizes endpoints and clamps limits", () => {
- expect(normalizeAnyTxtConfig({ endpoint: "localhost:9920" }).endpoint).toBe("http://localhost:9920")
- expect(normalizeAnyTxtConfig({ endpoint: "https://anytxt.local/" }).endpoint).toBe("https://anytxt.local/")
- expect(normalizeAnyTxtConfig({ limit: -1 }).limit).toBe(1)
- expect(normalizeAnyTxtConfig({ limit: 1000 }).limit).toBe(100)
- })
-
- it("returns no results for blank queries without calling AnyTXT", async () => {
- const out = await anyTxtSearch(" ", { endpoint: "http://127.0.0.1:9920" })
-
- expect(fetchMock).not.toHaveBeenCalled()
- expect(out).toEqual([])
- })
-
- it("returns no results when AnyTXT is disabled", async () => {
- const out = await anyTxtSearch("alpha", {
- enabled: false,
- endpoint: "http://127.0.0.1:9920",
+ expect(normalizeAnyTxtConfig({ endpoint: "localhost:9920", limit: 1000 })).toMatchObject({
+ endpoint: "http://localhost:9920",
+ limit: 100,
+ })
+ expect(normalizeAnyTxtConfig({ endpoint: "https://host/api", limit: -1 })).toMatchObject({
+ endpoint: "https://host/api",
+ limit: 1,
})
-
- expect(fetchMock).not.toHaveBeenCalled()
- expect(out).toEqual([])
- })
-
- it("surfaces JSON-RPC errors", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- jsonrpc: "2.0",
- id: 1,
- error: { message: "bad pattern" },
- }))
-
- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))
- .rejects.toThrow("AnyTXT error: bad pattern")
- })
-
- it("surfaces HTTP failures with status codes", async () => {
- fetchMock.mockResolvedValueOnce(new Response("oops", { status: 500 }))
-
- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))
- .rejects.toThrow("AnyTXT request failed (500): oops")
- })
-
- it("falls back to item snippets when fragment lookup fails", async () => {
- fetchMock
- .mockResolvedValueOnce(jsonResponse({
- result: {
- items: [
- { fid: "1", path: "/tmp/a.md", snippet: "fallback snippet" },
- ],
- },
- }))
- .mockResolvedValueOnce(new Response("fragment failed", { status: 500 }))
-
- const out = await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" })
-
- expect(out[0].snippet).toBe("fallback snippet")
- })
-
- it("filters blank AnyTXT result items", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- result: {
- items: [
- {},
- { path: "/tmp/valid.md", snippet: "valid" },
- ],
- },
- }))
-
- const out = await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" })
-
- expect(out).toHaveLength(1)
- expect(out[0].url).toBe("file:///tmp/valid.md")
})
- it("turns fetch network failures into AnyTXT connection guidance", async () => {
- fetchMock.mockRejectedValueOnce(new TypeError("Failed to fetch"))
-
- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))
- .rejects.toThrow("Check that ATGUI.exe is running")
+ it("returns no results for blank queries or disabled AnyTXT without invoking Rust", async () => {
+ expect(await anyTxtSearch(" ", { endpoint: "127.0.0.1:9920" })).toEqual([])
+ expect(await anyTxtSearch("alpha", { enabled: false, endpoint: "127.0.0.1:9920" })).toEqual([])
+ expect(invokeMock).not.toHaveBeenCalled()
})
})
describe("AnyTXT query rewrite", () => {
+ beforeEach(() => {
+ invokeMock.mockReset()
+ streamChatMock.mockReset()
+ })
+
it("parses JSON-array query rewrites and deduplicates them", () => {
expect(parseAnyTxtQueryRewrite('```json\n["MBR ammonia", "winter nitrification", "MBR ammonia"]\n```'))
.toEqual(["MBR ammonia", "winter nitrification"])
@@ -302,10 +111,9 @@ describe("AnyTXT query rewrite", () => {
})
it("prefers rewritten AnyTXT queries when original queries would fill the cap", async () => {
- streamChatMock.mockImplementationOnce(async (...args: unknown[]) => {
- const handlers = args[2] as { onToken?: (token: string) => void; onDone?: () => void }
- handlers.onToken?.('["kw1", "kw2", "kw3"]')
- handlers.onDone?.()
+ streamChatMock.mockImplementationOnce(async (_config, _messages, callbacks) => {
+ callbacks.onToken?.('["kw1", "kw2", "kw3"]')
+ callbacks.onDone?.()
})
const queries = await prepareAnyTxtQueries(
@@ -325,32 +133,23 @@ describe("AnyTXT query rewrite", () => {
})
it("searches rewritten queries through the smart AnyTXT entry point", async () => {
- fetchMock.mockReset()
- vi.stubGlobal("fetch", fetchMock)
- streamChatMock.mockImplementationOnce(async (...args: unknown[]) => {
- const handlers = args[2] as { onToken?: (token: string) => void; onDone?: () => void }
- handlers.onToken?.('["煤矿 安全"]')
- handlers.onDone?.()
+ streamChatMock.mockImplementationOnce(async (_config, _messages, callbacks) => {
+ callbacks.onToken?.('["煤矿 安全"]')
+ callbacks.onDone?.()
})
- fetchMock.mockResolvedValueOnce(jsonResponse({
- result: {
- items: [
- { path: "/docs/coal.md", snippet: "煤矿安全 local match" },
- ],
- },
- }))
+ invokeMock.mockResolvedValueOnce([
+ { title: "mine.pdf", url: "/docs/mine.pdf", snippet: "煤矿安全", source: "AnyTXT" },
+ ])
+ invokeMock.mockResolvedValueOnce([])
const out = await anyTxtSearchSmart("请帮我找一下煤矿安全相关资料", {
+ enabled: true,
endpoint: "http://127.0.0.1:9920",
- }, llmConfig, 1, "/Users/me/wiki")
+ }, llmConfig, 5)
- expect(fetchMock).toHaveBeenCalled()
- expect(JSON.parse(String(fetchMock.mock.calls[0][1]?.body))).toMatchObject({
- params: { input: { pattern: "煤矿 安全" } },
- })
- expect(out[0]).toMatchObject({
- title: "coal.md",
- source: "AnyTXT",
- })
+ expect(invokeMock.mock.calls[0]).toEqual(["anytxt_search", expect.objectContaining({
+ query: "煤矿 安全",
+ })])
+ expect(out).toHaveLength(1)
})
})
diff --git a/src/lib/anytxt-search.ts b/src/lib/anytxt-search.ts
index 6a8c0d56..e6e97604 100644
--- a/src/lib/anytxt-search.ts
+++ b/src/lib/anytxt-search.ts
@@ -1,5 +1,5 @@
+import { invoke } from "@tauri-apps/api/core"
import type { AnyTxtConfig, LlmConfig } from "@/stores/wiki-store"
-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"
import { normalizePath } from "@/lib/path-utils"
import { streamChat } from "@/lib/llm-client"
import type { WebSearchResult } from "./web-search"
@@ -7,21 +7,7 @@ import type { WebSearchResult } from "./web-search"
export const DEFAULT_ANYTXT_ENDPOINT = "http://127.0.0.1:9920"
export const DEFAULT_ANYTXT_FILTER_EXT = "*"
export const DEFAULT_ANYTXT_LIMIT = 20
-const ANYTXT_LAST_MODIFY_END = 2147483647 // AnyTXT docs use signed 32-bit Unix timestamps.
const ANYTXT_QUERY_LIMIT = 3
-let nextRpcId = 1
-
-interface AnyTxtRpcResponse {
- result?: unknown
- error?: string | { message?: string; code?: number }
-}
-
-interface AnyTxtItem {
- fid: string
- path: string
- title: string
- snippet: string
-}
export function normalizeAnyTxtConfig(config?: AnyTxtConfig, _projectPath?: string): Required<AnyTxtConfig> {
return {
@@ -47,34 +33,11 @@ export async function anyTxtSearch(
if (!query.trim()) return []
const resolved = normalizeAnyTxtConfig(config, projectPath)
if (!resolved.enabled) return []
- const limit = Math.min(clampAnyTxtLimit(maxResults), resolved.limit)
- const response = await callAnyTxtRpc(resolved.endpoint, "ATRpcServer.Searcher.V1.GetResult", buildAnyTxtSearchInput({
- pattern: query,
- filterDir: resolved.filterDir,
- filterExt: resolved.filterExt,
- lastModifyBegin: 0,
- lastModifyEnd: ANYTXT_LAST_MODIFY_END,
- // AnyTXT's documented GetResult shape expects limit as a string while
- // offset/order/timestamps are numbers.
- limit: String(limit),
- offset: 0,
- order: 0,
- }))
-
- const items = extractAnyTxtItems(response).slice(0, limit)
- const out: WebSearchResult[] = []
- for (const item of items) {
- const fragment = item.fid
- ? await getAnyTxtFragment(resolved.endpoint, item.fid, query).catch(() => "")
- : ""
- out.push({
- title: item.title,
- url: fileUrlForPath(item.path) || (item.fid ? `anytxt://${item.fid}` : ""),
- snippet: trimFragment(fragment || item.snippet || item.path),
- source: "AnyTXT",
- })
- }
- return out.filter((item) => item.url || item.snippet)
+ return invoke<WebSearchResult[]>("anytxt_search", {
+ query,
+ config: resolved,
+ maxResults: Math.min(clampAnyTxtLimit(maxResults), resolved.limit),
+ })
}
export async function anyTxtSearchSmart(
@@ -194,179 +157,6 @@ export function uniqueAnyTxtQueries(queries: string[], limit: number = ANYTXT_QU
return out
}
-async function getAnyTxtFragment(endpoint: string, fid: string, pattern: string): Promise<string> {
- const response = await callAnyTxtRpc(endpoint, "ATRpcServer.Searcher.V1.GetFragment", {
- fid,
- pattern,
- })
- return extractFragmentText(response)
-}
-
-async function callAnyTxtRpc(endpoint: string, method: string, input: Record<string, unknown>): Promise<unknown> {
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch(endpoint, {
- method: "POST",
- headers: {
- Accept: "application/json",
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- id: nextRpcId++,
- jsonrpc: "2.0",
- method,
- params: { input },
- }),
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error("Network error reaching AnyTXT. Check that ATGUI.exe is running and the JSON-RPC API is listening.")
- }
- throw err
- }
-
- if (!response.ok) {
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`AnyTXT request failed (${response.status}): ${errorText}`)
- }
-
- const data = (await response.json()) as AnyTxtRpcResponse
- if (data.error) {
- const message = typeof data.error === "string" ? data.error : data.error.message ?? JSON.stringify(data.error)
- throw new Error(`AnyTXT error: ${message}`)
- }
- return data.result
-}
-
-function extractAnyTxtItems(result: unknown): AnyTxtItem[] {
- const rawItems = firstArray(
- result,
- getPath(result, ["output"]),
- getPath(result, ["output", "items"]),
- getPath(result, ["output", "results"]),
- getPath(result, ["output", "list"]),
- getPath(result, ["output", "files"]),
- getPath(result, ["output", "data"]),
- getPath(result, ["output", "value"]),
- getPath(result, ["data", "output", "files"]),
- getPath(result, ["data", "output", "items"]),
- getPath(result, ["data", "output", "results"]),
- getPath(result, ["data", "output", "list"]),
- getPath(result, ["data", "files"]),
- getPath(result, ["data", "items"]),
- getPath(result, ["data", "results"]),
- getPath(result, ["data", "list"]),
- getPath(result, ["items"]),
- getPath(result, ["results"]),
- getPath(result, ["list"]),
- getPath(result, ["files"]),
- getPath(result, ["data"]),
- getPath(result, ["value"]),
- )
- const fields = firstStringArray(
- getPath(result, ["field"]),
- getPath(result, ["output", "field"]),
- getPath(result, ["data", "output", "field"]),
- getPath(result, ["data", "field"]),
- )
-
- return rawItems.map((item) => normalizeAnyTxtItem(item, fields)).filter((item): item is AnyTxtItem => item !== null)
-}
-
-function normalizeAnyTxtItem(item: unknown, fields: string[] = []): AnyTxtItem | null {
- if (!item || (typeof item !== "object" && !Array.isArray(item))) return null
- const record = Array.isArray(item) ? recordFromFields(fields, item) : item as Record<string, unknown>
- const fid = stringField(record, "fid", "id", "fileId", "file_id")
- const path = normalizePath(stringField(record, "path", "file", "filePath", "file_path", "fullPath", "full_path", "filename", "fileName", "name"))
- const title = stringField(record, "title", "name", "fileName", "filename") || basename(path) || fid || "AnyTXT result"
- const snippet = stringField(record, "snippet", "fragment", "content", "contents", "text", "summary", "highlight", "hitText")
- if (!fid && !path && !snippet) return null
- return { fid, path, title, snippet }
-}
-
-function extractFragmentText(result: unknown): string {
- if (typeof result === "string") return result
- if (Array.isArray(result)) return result.map(extractFragmentText).filter(Boolean).join("\n\n")
- if (!result || typeof result !== "object") return ""
- const record = result as Record<string, unknown>
- return stringField(record, "text", "fragment", "content", "snippet", "html")
- || extractFragmentText(record.output)
- || extractFragmentText(record.result)
- || extractFragmentText(record.data)
- || extractFragmentText(getPath(record, ["data", "output"]))
- || extractFragmentText(record.fragments)
- || extractFragmentText(record.items)
- || extractFragmentText(record.list)
-}
-
-function buildAnyTxtSearchInput(input: {
- pattern: string
- filterDir: string
- filterExt: string
- lastModifyBegin: number
- lastModifyEnd: number
- limit: string
- offset: number
- order: number
-}): Record<string, unknown> {
- const out: Record<string, unknown> = {
- pattern: input.pattern,
- filterExt: input.filterExt,
- lastModifyBegin: input.lastModifyBegin,
- lastModifyEnd: input.lastModifyEnd,
- limit: input.limit,
- offset: input.offset,
- order: input.order,
- }
- if (input.filterDir.trim()) {
- out.filterDir = input.filterDir
- }
- return out
-}
-
-function firstArray(...values: unknown[]): unknown[] {
- for (const value of values) {
- if (Array.isArray(value)) return value
- }
- return []
-}
-
-function firstStringArray(...values: unknown[]): string[] {
- for (const value of values) {
- if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
- return value
- }
- }
- return []
-}
-
-function recordFromFields(fields: string[], row: unknown[]): Record<string, unknown> {
- const record: Record<string, unknown> = {}
- fields.forEach((field, index) => {
- record[field] = row[index]
- })
- return record
-}
-
-function getPath(value: unknown, path: string[]): unknown {
- let current = value
- for (const key of path) {
- if (!current || typeof current !== "object") return undefined
- current = (current as Record<string, unknown>)[key]
- }
- return current
-}
-
-function stringField(record: Record<string, unknown>, ...keys: string[]): string {
- for (const key of keys) {
- const value = record[key]
- if (typeof value === "string" && value.trim()) return value.trim()
- if (typeof value === "number" && Number.isFinite(value)) return String(value)
- }
- return ""
-}
-
function normalizeAnyTxtEndpoint(endpoint?: string): string {
const trimmed = endpoint?.trim() || DEFAULT_ANYTXT_ENDPOINT
if (/^https?:\/\//i.test(trimmed)) return trimmed
@@ -381,27 +171,8 @@ function clampAnyTxtLimit(limit: unknown): number {
function defaultAnyTxtFilterDir(projectPath?: string): string {
const pp = projectPath ? normalizePath(projectPath) : ""
- // On the macOS/Linux AnyTXT API, omitting filterDir has been observed to
- // fall back to "C:" instead of "all indexed files". Use "/" as the explicit
- // unrestricted root for Unix-style projects. Keep Windows blank because
- // AnyTXT cannot represent "all drive letters" with one filterDir.
+ // AnyTXT treats omitted filterDir differently across platforms. For
+ // Unix-like absolute paths, "/" means "all indexed files"; on Windows
+ // there is no single all-drive root, so keep the field empty.
return pp.startsWith("/") ? "/" : ""
}
-
-function basename(path: string): string {
- return path.split(/[\\/]/).filter(Boolean).pop() ?? ""
-}
-
-function fileUrlForPath(path: string): string {
- if (!path) return ""
- const normalized = normalizePath(path)
- if (/^[a-z]+:\/\//i.test(normalized)) return normalized
- if (normalized.startsWith("//")) return `file:${encodeURI(normalized)}`
- if (/^[A-Za-z]:\//.test(normalized)) return `file:///${encodeURI(normalized)}`
- if (normalized.startsWith("/")) return `file://${encodeURI(normalized)}`
- return normalized
-}
-
-function trimFragment(text: string): string {
- return text.replace(/\s+/g, " ").trim().slice(0, 1200)
-}
diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts
index 9dc22adc..3072e61c 100644
--- a/src/lib/embedding.test.ts
+++ b/src/lib/embedding.test.ts
@@ -3,30 +3,23 @@
* pieces: auto-halve retry heuristics and the chunk→page aggregation
* contract inside `searchByEmbedding`.
*
- * The actual HTTP layer and Tauri LanceDB commands are mocked — we're
- * NOT testing Rust vectorstore here (that has its own 15 Rust tests)
- * nor the webview fetch (that's tauri-fetch.ts). The boundary we pin
- * down here is "given these chunk-level results, do we aggregate to
- * page-level scores the way the design says?".
+ * The actual Rust HTTP/vectorstore commands are mocked here. Rust owns
+ * the provider transport; this file pins the TypeScript orchestration
+ * contract around chunking, invoke payloads, and page-level aggregation.
*/
import { describe, it, expect, vi, beforeEach } from "vitest"
// Module-level mock for the Tauri invoke boundary so we can script the
// chunk-search response without touching real LanceDB.
const mockInvoke = vi.fn<(cmd: string, args?: unknown) => Promise<unknown>>()
+const mockEmbeddingFetchInvoke = vi.fn<(args?: unknown) => Promise<unknown>>()
vi.mock("@tauri-apps/api/core", () => ({
- invoke: (cmd: string, args?: unknown) => mockInvoke(cmd, args),
+ invoke: (cmd: string, args?: unknown) =>
+ cmd === "embedding_fetch" ? mockEmbeddingFetchInvoke(args) : mockInvoke(cmd, args),
}))
-// Stub getHttpFetch so fetchEmbedding calls hit our in-test responder.
+// In-test responder used by the `embedding_fetch` invoke shim below.
const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()
-vi.mock("@/lib/tauri-fetch", () => ({
- getHttpFetch: () => Promise.resolve(mockHttpFetch),
- isFetchNetworkError: (err: unknown) =>
- err instanceof TypeError ||
- (err instanceof Error &&
- (err.message === "Load failed" || err.message === "Failed to fetch")),
-}))
// readFile / listDirectory aren't exercised in this file's cases; stub.
vi.mock("@/commands/fs", () => ({
@@ -77,10 +70,182 @@ function genericErrorResponse(status: number, body: string): Response {
beforeEach(() => {
mockInvoke.mockReset()
+ mockEmbeddingFetchInvoke.mockReset()
mockHttpFetch.mockReset()
+ mockEmbeddingFetchInvoke.mockImplementation((args) => {
+ const input = args as { text: string; cfg: typeof cfg; maxRetries?: number }
+ return fetchEmbeddingViaMockHttp(input.text, input.cfg, input.maxRetries ?? 3)
+ })
resetEmbeddingOptimizeAccountingForTests()
})
+async function fetchEmbeddingViaMockHttp(
+ text: string,
+ config: typeof cfg & {
+ outputDimensionality?: number
+ extraHeaders?: Record<string, string>
+ },
+ maxRetries: number,
+): Promise<number[]> {
+ const isGoogle = isGoogleEmbeddingConfigForTest(config)
+ const isDoubaoVision = config.model.toLowerCase().includes("doubao-embedding-vision")
+ const endpoint = isGoogle ? googleEndpointForTest(config) : volcengineEndpointForTest(config)
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ ...(isLocalEndpointForTest(endpoint) ? { Origin: "http://localhost" } : {}),
+ }
+ if (config.apiKey) {
+ if (isGoogle) headers["x-goog-api-key"] = config.apiKey
+ else headers.Authorization = `Bearer ${config.apiKey}`
+ }
+ for (const [name, value] of Object.entries(config.extraHeaders ?? {})) {
+ const trimmed = name.trim()
+ const lower = trimmed.toLowerCase()
+ if (!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(trimmed)) continue
+ if (["authorization", "content-type", "host", "content-length", "origin", "x-goog-api-key"].includes(lower)) continue
+ if (!String(value).trim()) continue
+ headers[trimmed] = String(value).trim()
+ }
+
+ let current = text
+ let attempts = 0
+ while (attempts <= maxRetries) {
+ attempts += 1
+ const body = isGoogle
+ ? googleBodyForTest(config.model, current, config.outputDimensionality)
+ : isDoubaoVision
+ ? { model: config.model, encoding_format: "float", input: [{ type: "text", text: current }] }
+ : { model: config.model, input: current }
+ let response: Response
+ try {
+ const maybeResponse = await mockHttpFetch(endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify(body),
+ })
+ if (!maybeResponse) throw new TypeError("Failed to fetch")
+ response = maybeResponse
+ } catch {
+ throw new Error(`Network error reaching ${endpoint}. Check endpoint URL, API key, and connectivity.`)
+ }
+ const textBody = await response.text()
+ if (response.ok) {
+ const data = JSON.parse(textBody)
+ const embedding = isGoogle
+ ? data?.embedding?.values
+ : isDoubaoVision
+ ? data?.data?.embedding
+ : data?.data?.[0]?.embedding
+ if (Array.isArray(embedding) && embedding.length > 0 && embedding.every((v) => typeof v === "number" && Number.isFinite(v))) {
+ return embedding
+ }
+ const expected = isGoogle ? "embedding.values" : isDoubaoVision ? "data.embedding" : "data[0].embedding"
+ throw new Error(`Embedding response missing ${expected}`)
+ }
+ if (looksLikeOversizeForTest(response.status, textBody) && current.length > 64 && attempts <= maxRetries) {
+ current = current.slice(0, Math.floor(current.length / 2))
+ continue
+ }
+ if (looksLikeOversizeForTest(response.status, textBody)) {
+ throw new Error(`Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars.`)
+ }
+ throw new Error(`API ${response.status} ${response.statusText}${textBody ? ` — ${textBody.slice(0, 200)}` : ""} at ${endpoint}`)
+ }
+ throw new Error(`Embedding endpoint rejected every size down to ${current.length} chars`)
+}
+
+function looksLikeOversizeForTest(status: number, body: string): boolean {
+ const lower = body.toLowerCase()
+ return status === 413
+ || lower.includes("too long")
+ || lower.includes("maximum context")
+ || lower.includes("max_tokens")
+ || lower.includes("max tokens")
+ || lower.includes("context length")
+ || lower.includes("token limit")
+ || lower.includes("exceeds")
+ || lower.includes("input length")
+}
+
+function isGoogleEmbeddingConfigForTest(config: { endpoint: string }): boolean {
+ const endpoint = config.endpoint.toLowerCase()
+ return endpoint.includes("generativelanguage.googleapis.com") || endpoint.includes(":embedcontent")
+}
+
+function isLocalEndpointForTest(endpoint: string): boolean {
+ try {
+ const host = new URL(endpoint).hostname
+ return host === "localhost"
+ || host === "127.0.0.1"
+ || host === "::1"
+ || /^10\./.test(host)
+ || /^192\.168\./.test(host)
+ || /^172\.(1[6-9]|2\d|3[01])\./.test(host)
+ } catch {
+ return false
+ }
+}
+
+function isVolcengineEndpointForTest(endpoint: string): boolean {
+ try {
+ const host = new URL(endpoint).hostname.toLowerCase()
+ return host === "volces.com" || host.endsWith(".volces.com") || host.includes("volcengine")
+ } catch {
+ return false
+ }
+}
+
+function volcengineEndpointForTest(config: { endpoint: string; model: string }): string {
+ if (!isVolcengineEndpointForTest(config.endpoint)) return config.endpoint
+ const suffix = config.model.toLowerCase().includes("doubao-embedding-vision") ? "/embeddings/multimodal" : "/embeddings"
+ return appendEndpointPathForTest(config.endpoint, suffix)
+}
+
+function appendEndpointPathForTest(endpoint: string, suffix: string): string {
+ const cleanSuffix = suffix.replace(/^\/+/, "")
+ const url = new URL(endpoint)
+ const path = url.pathname.replace(/\/+$/, "")
+ const lower = path.toLowerCase()
+ const lowerSuffix = `/${cleanSuffix.toLowerCase()}`
+ if (lower.endsWith(lowerSuffix)) {
+ url.pathname = path || "/"
+ } else if (lower.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings") {
+ url.pathname = path.slice(0, -"/multimodal".length) || "/"
+ } else if (lower.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal") {
+ url.pathname = `${path}/multimodal`
+ } else {
+ url.pathname = `${path}/${cleanSuffix}`.replace(/\/{2,}/g, "/")
+ }
+ return url.toString()
+}
+
+function googleEndpointForTest(config: { endpoint: string; model: string }): string {
+ const raw = stripGoogleKeyForTest(config.endpoint.trim()).replace(/\/+$/, "")
+ if (/:batchEmbedContents(\?|$)/i.test(raw)) return raw.replace(/:batchEmbedContents/i, ":embedContent")
+ if (/:embedContent(\?|$)/i.test(raw)) return raw
+ const model = config.model.startsWith("models/") ? config.model.slice("models/".length) : config.model
+ if (/\/models\/[^/?]+$/i.test(raw)) return `${raw}:embedContent`
+ return `${raw}/models/${encodeURIComponent(model)}:embedContent`
+}
+
+function stripGoogleKeyForTest(endpoint: string): string {
+ if (!endpoint.includes("?")) return endpoint
+ const url = new URL(endpoint)
+ url.searchParams.delete("key")
+ return url.toString()
+}
+
+function googleBodyForTest(model: string, text: string, outputDimensionality?: number): Record<string, unknown> {
+ const body: Record<string, unknown> = {
+ model: model.startsWith("models/") ? model : `models/${model}`,
+ content: { parts: [{ text }] },
+ }
+ if (typeof outputDimensionality === "number" && Number.isFinite(outputDimensionality) && outputDimensionality > 0) {
+ body.output_dimensionality = Math.floor(outputDimensionality)
+ }
+ return body
+}
+
// ── searchByEmbedding — chunk→page aggregation ─────────────────────
describe("searchByEmbedding — aggregation", () => {
diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts
index ff12e839..d90f220a 100644
--- a/src/lib/embedding.ts
+++ b/src/lib/embedding.ts
@@ -16,8 +16,9 @@
* `{id, score}[]` shape; matched chunks available on the
* optional `matchedChunks` field for future UI surfacing.
*
- * HTTP goes through the Tauri plugin (`src/lib/tauri-fetch.ts`) so
- * CORS-unfriendly endpoints work the same as the LLM path.
+ * Provider HTTP is executed by the Rust `embedding_fetch` command so
+ * CORS-unfriendly endpoints keep working while the core embedding
+ * transport is shared by UI and backend callers.
*/
import { readFile, listDirectory } from "@/commands/fs"
@@ -25,30 +26,7 @@ import { invoke } from "@tauri-apps/api/core"
import type { EmbeddingConfig } from "@/stores/wiki-store"
import type { FileNode } from "@/types/wiki"
import { normalizePath } from "@/lib/path-utils"
-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"
import { chunkMarkdown, type Chunk } from "@/lib/text-chunker"
-import { isLocalOrPrivateHttpEndpoint, localLlmOriginHeader } from "@/lib/llm-providers"
-
-const RESERVED_EMBEDDING_HEADER_NAMES = new Set([
- "authorization",
- "content-type",
- "host",
- "content-length",
- "origin",
- "x-goog-api-key",
-])
-const HTTP_HEADER_NAME_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/
-
-function isSafeExtraHeader(name: string, value: string): boolean {
- const trimmedName = name.trim()
- const trimmedValue = value.trim()
- return (
- trimmedName.length > 0 &&
- trimmedValue.length > 0 &&
- HTTP_HEADER_NAME_RE.test(trimmedName) &&
- !RESERVED_EMBEDDING_HEADER_NAMES.has(trimmedName.toLowerCase())
- )
-}
// ── Error surfacing ──────────────────────────────────────────────────────
@@ -111,255 +89,18 @@ export async function fetchEmbedding(
maxRetries = 3,
): Promise<number[] | null> {
if (!cfg.endpoint) return null
-
- const isGoogleNative = isGoogleEmbeddingConfig(cfg)
- const isDoubaoMultimodal = isDoubaoMultimodalEmbeddingConfig(cfg)
- const endpoint = isGoogleNative
- ? googleEmbeddingEndpoint(cfg)
- : volcengineEmbeddingEndpoint(cfg)
- const headers: Record<string, string> = {
- "Content-Type": "application/json",
- ...(isLocalOrPrivateHttpEndpoint(endpoint) ? localLlmOriginHeader() : {}),
- }
- if (cfg.apiKey) {
- if (isGoogleNative) {
- headers["x-goog-api-key"] = cfg.apiKey
- } else {
- headers.Authorization = `Bearer ${cfg.apiKey}`
- }
- }
- if (cfg.extraHeaders) {
- for (const [k, v] of Object.entries(cfg.extraHeaders)) {
- const name = k.trim()
- const value = v.trim()
- if (!isSafeExtraHeader(name, value)) continue
- headers[name] = value
- }
- }
-
- let current = text
- let attempts = 0
- while (attempts <= maxRetries) {
- attempts++
- try {
- const httpFetch = await getHttpFetch()
- const resp = await httpFetch(endpoint, {
- method: "POST",
- headers,
- body: JSON.stringify(
- isGoogleNative
- ? googleEmbeddingBody(cfg.model, current, cfg.outputDimensionality)
- : isDoubaoMultimodal
- ? doubaoMultimodalEmbeddingBody(cfg.model, current)
- : { model: cfg.model, input: current },
- ),
- })
-
- if (resp.ok) {
- const data = await resp.json()
- const embedding = isGoogleNative
- ? data?.embedding?.values ?? null
- : isDoubaoMultimodal
- ? data?.data?.embedding ?? null
- : data?.data?.[0]?.embedding ?? null
- if (isNonEmptyNumberArray(embedding)) {
- lastEmbeddingError = null
- return embedding
- }
- const expectedShape = isGoogleNative
- ? "embedding.values"
- : isDoubaoMultimodal
- ? "data.embedding"
- : "data[0].embedding"
- lastEmbeddingError = `Embedding response missing ${expectedShape} (got ${JSON.stringify(data).slice(0, 200)})`
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
- }
-
- // Non-OK: try to read the body for an oversize hint.
- let bodyText = ""
- try {
- bodyText = await resp.text()
- } catch {
- // ignore — some servers return empty bodies on error
- }
-
- if (looksLikeOversizeError(resp.status, bodyText)) {
- // Can we still halve-and-retry? Need room on both axes:
- // text not yet at the 64-char floor, and retry budget left.
- if (current.length > 64 && attempts <= maxRetries) {
- const prev = current.length
- current = current.slice(0, Math.floor(current.length / 2))
- console.warn(
- `[Embedding] auto-halving after HTTP ${resp.status} at ${prev} chars → retrying at ${current.length} chars (attempt ${attempts}/${maxRetries + 1})`,
- )
- continue
- }
- // Out of retries on a SERVER-oversize error — give the user a
- // message that names the smallest size that still failed so
- // they can tune Settings → Embedding accordingly.
- lastEmbeddingError = `Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars (${bodyText.slice(0, 160)}).`
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
- }
-
- // Non-oversize definitive failure (auth, rate limit, server down, …).
- lastEmbeddingError = `API ${resp.status} ${resp.statusText}${bodyText ? ` — ${bodyText.slice(0, 200)}` : ""} at ${endpoint}`
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
- } catch (err) {
- if (isFetchNetworkError(err)) {
- lastEmbeddingError = `Network error reaching ${endpoint}. Check endpoint URL, API key, and connectivity.`
- } else {
- lastEmbeddingError = err instanceof Error ? err.message : String(err)
- }
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
- }
- }
-
- // Exhausted retries (only reachable if every halving round triggered
- // the retry branch and then the loop condition ended).
- lastEmbeddingError = `Embedding endpoint rejected every size down to ${current.length} chars — the server's context is smaller than ${current.length * 2}. Lower Settings → Embedding → Max Chunk Chars.`
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
-}
-
-function isNonEmptyNumberArray(value: unknown): value is number[] {
- return Array.isArray(value)
- && value.length > 0
- && value.every((item) => typeof item === "number" && Number.isFinite(item))
-}
-
-function isGoogleEmbeddingConfig(cfg: EmbeddingConfig): boolean {
- const endpoint = cfg.endpoint.toLowerCase()
- return endpoint.includes("generativelanguage.googleapis.com")
- || /:embedcontent(\?|$)/i.test(endpoint)
-}
-
-function isVolcengineEmbeddingEndpoint(endpoint: string): boolean {
- try {
- const host = new URL(endpoint).hostname.toLowerCase()
- return host === "volces.com"
- || host.endsWith(".volces.com")
- || host.includes("volcengine")
- } catch {
- const authority = endpoint
- .trim()
- .replace(/^[a-z][a-z0-9+.-]*:\/\//i, "")
- .split(/[/?#]/, 1)[0]
- .toLowerCase()
- return authority === "volces.com"
- || authority.endsWith(".volces.com")
- || authority.includes("volcengine")
- }
-}
-
-function isDoubaoMultimodalEmbeddingConfig(cfg: EmbeddingConfig): boolean {
- return cfg.model.trim().toLowerCase().includes("doubao-embedding-vision")
-}
-
-function volcengineEmbeddingEndpoint(cfg: EmbeddingConfig): string {
- const raw = cfg.endpoint.trim()
- if (!isVolcengineEmbeddingEndpoint(raw)) return raw
- const targetSuffix = isDoubaoMultimodalEmbeddingConfig(cfg)
- ? "/embeddings/multimodal"
- : "/embeddings"
- return appendEndpointPath(raw, targetSuffix)
-}
-
-function appendEndpointPath(endpoint: string, targetSuffix: string): string {
- const suffix = targetSuffix.replace(/^\/+/, "")
try {
- const url = new URL(endpoint)
- const path = url.pathname.replace(/\/+$/, "")
- const lowerPath = path.toLowerCase()
- const lowerSuffix = `/${suffix.toLowerCase()}`
- if (lowerPath.endsWith(lowerSuffix)) {
- url.pathname = path || "/"
- return url.toString()
- }
- if (lowerPath.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings") {
- url.pathname = path.slice(0, -"/multimodal".length) || "/"
- return url.toString()
- }
- if (lowerPath.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal") {
- url.pathname = `${path}/multimodal`
- return url.toString()
- }
- url.pathname = `${path}/${suffix}`.replace(/\/{2,}/g, "/")
- return url.toString()
- } catch {
- const [base, query = ""] = endpoint.split("?", 2)
- const trimmed = base.replace(/\/+$/, "")
- const lower = trimmed.toLowerCase()
- const lowerSuffix = `/${suffix.toLowerCase()}`
- const next = lower.endsWith(lowerSuffix)
- ? trimmed
- : lower.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings"
- ? trimmed.slice(0, -"/multimodal".length)
- : lower.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal"
- ? `${trimmed}/multimodal`
- : `${trimmed}/${suffix}`
- return query ? `${next}?${query}` : next
- }
-}
-
-function googleEmbeddingEndpoint(cfg: EmbeddingConfig): string {
- const raw = stripGoogleApiKeyQuery(cfg.endpoint.trim()).replace(/\/+$/, "")
- if (/:batchEmbedContents(\?|$)/i.test(raw)) {
- return raw.replace(/:batchEmbedContents/i, ":embedContent")
- }
- if (/:embedContent(\?|$)/i.test(raw)) return raw
-
- const modelPath = googleModelPath(cfg.model)
- if (/\/models\/[^/?]+$/i.test(raw)) {
- return `${raw}:embedContent`
- }
- return `${raw}/models/${encodeURIComponent(modelPath.replace(/^models\//, ""))}:embedContent`
-}
-
-function stripGoogleApiKeyQuery(endpoint: string): string {
- if (!endpoint.includes("?")) return endpoint
- try {
- const url = new URL(endpoint)
- url.searchParams.delete("key")
- return url.toString()
- } catch {
- return endpoint.replace(/([?&])key=[^&]*&?/i, (_, prefix: string) => prefix === "?" ? "?" : "&")
- .replace(/[?&]$/, "")
- .replace("?&", "?")
- }
-}
-
-function googleModelPath(model: string): string {
- const trimmed = model.trim()
- if (trimmed.startsWith("models/")) return trimmed
- return `models/${trimmed}`
-}
-
-function googleEmbeddingBody(
- model: string,
- text: string,
- outputDimensionality?: number,
-): Record<string, unknown> {
- const body: Record<string, unknown> = {
- model: googleModelPath(model),
- content: {
- parts: [{ text }],
- },
- }
- if (typeof outputDimensionality === "number" && Number.isFinite(outputDimensionality) && outputDimensionality > 0) {
- body.output_dimensionality = Math.floor(outputDimensionality)
- }
- return body
-}
-
-function doubaoMultimodalEmbeddingBody(model: string, text: string): Record<string, unknown> {
- return {
- model,
- encoding_format: "float",
- input: [{ type: "text", text }],
+ const embedding = await invoke<number[]>("embedding_fetch", {
+ text,
+ cfg,
+ maxRetries,
+ })
+ lastEmbeddingError = null
+ return embedding
+ } catch (err) {
+ lastEmbeddingError = err instanceof Error ? err.message : String(err)
+ console.warn(`[Embedding] ${lastEmbeddingError}`)
+ return null
}
}
diff --git a/src/lib/web-search.test.ts b/src/lib/web-search.test.ts
index fb960a84..af3047fc 100644
--- a/src/lib/web-search.test.ts
+++ b/src/lib/web-search.test.ts
@@ -1,70 +1,41 @@
import { beforeEach, describe, expect, it, vi } from "vitest"
import { hasConfiguredDeepResearchSources, hasConfiguredSearchProvider, resolveSearchConfig, webSearch } from "./web-search"
-const fetchMock = vi.fn<typeof fetch>()
+const invokeMock = vi.hoisted(() => vi.fn())
-function jsonResponse(body: unknown, init?: ResponseInit): Response {
- return new Response(JSON.stringify(body), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- ...init,
- })
-}
+vi.mock("@tauri-apps/api/core", () => ({
+ invoke: invokeMock,
+}))
describe("webSearch", () => {
beforeEach(() => {
- fetchMock.mockReset()
- vi.stubGlobal("fetch", fetchMock)
+ invokeMock.mockReset()
})
- it("normalizes Tavily results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- results: [
- { title: "A", url: "https://www.example.com/a", content: "Alpha" },
- ],
- }))
-
- const out = await webSearch("alpha", { provider: "tavily", apiKey: "tvly" }, 3)
-
- expect(fetchMock).toHaveBeenCalledWith("https://api.tavily.com/search", expect.objectContaining({
- method: "POST",
- }))
- expect(out).toEqual([
- { title: "A", url: "https://www.example.com/a", snippet: "Alpha", source: "example.com" },
+ it("delegates provider search to the Rust backend command", async () => {
+ invokeMock.mockResolvedValueOnce([
+ { title: "A", url: "https://example.com/a", snippet: "Alpha", source: "web" },
])
- })
-
- it("calls SerpApi Google Search and normalizes organic results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- organic_results: [
- { title: "Serp result", link: "https://www.serp.example/page", snippet: "Snippet" },
- { title: "Second", link: "https://docs.example/item", snippet: "More" },
- ],
- }))
- const out = await webSearch("knowledge graph", { provider: "serpapi", apiKey: "serp" }, 1)
- const [url, init] = fetchMock.mock.calls[0]
- const parsed = new URL(String(url))
+ const out = await webSearch("alpha", { provider: "tavily", apiKey: "tvly" }, 3)
- expect(parsed.origin + parsed.pathname).toBe("https://serpapi.com/search")
- expect(parsed.searchParams.get("engine")).toBe("google")
- expect(parsed.searchParams.get("q")).toBe("knowledge graph")
- expect(parsed.searchParams.get("api_key")).toBe("serp")
- expect(parsed.searchParams.get("num")).toBe("1")
- expect(init).toEqual(expect.objectContaining({ method: "GET" }))
+ expect(invokeMock).toHaveBeenCalledWith("web_search", {
+ query: "alpha",
+ maxResults: 3,
+ config: expect.objectContaining({
+ provider: "tavily",
+ apiKey: "tvly",
+ }),
+ })
expect(out).toEqual([
- { title: "Serp result", url: "https://www.serp.example/page", snippet: "Snippet", source: "serp.example" },
+ { title: "A", url: "https://example.com/a", snippet: "Alpha", source: "web" },
])
})
- it("uses SerpApi provider-specific config and selected engine", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- news_results: [
- { title: "News", link: "https://news.example/story", snippet: "Fresh" },
- ],
- }))
+ it("passes provider-specific config to Rust", async () => {
+ invokeMock.mockResolvedValueOnce([])
- const out = await webSearch(
+ await webSearch(
"ai policy",
{
provider: "serpapi",
@@ -76,501 +47,43 @@ describe("webSearch", () => {
},
5,
)
- const parsed = new URL(String(fetchMock.mock.calls[0][0]))
-
- expect(parsed.searchParams.get("engine")).toBe("google_news")
- expect(parsed.searchParams.get("api_key")).toBe("serp-key")
- expect(out).toEqual([
- { title: "News", url: "https://news.example/story", snippet: "Fresh", source: "news.example" },
- ])
- })
-
- it("calls SearXNG JSON search with the configured instance and categories", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- results: [
- {
- title: "SearXNG result",
- url: "https://docs.example/page",
- content: "Result content",
- engine: "duckduckgo",
- },
- ],
- }))
-
- const out = await webSearch(
- "local search",
- {
- provider: "searxng",
- apiKey: "",
- providerConfigs: {
- searxng: {
- searXngUrl: "https://search.example.com",
- searXngCategories: ["general", "news"],
- },
- },
- },
- 3,
- )
- const [url, init] = fetchMock.mock.calls[0]
- const parsed = new URL(String(url))
-
- expect(parsed.origin + parsed.pathname).toBe("https://search.example.com/search")
- expect(parsed.searchParams.get("q")).toBe("local search")
- expect(parsed.searchParams.get("format")).toBe("json")
- expect(parsed.searchParams.get("categories")).toBe("general,news")
- expect(init).toEqual(expect.objectContaining({ method: "GET" }))
- expect(out).toEqual([
- { title: "SearXNG result", url: "https://docs.example/page", snippet: "Result content", source: "docs.example" },
- ])
- })
-
- it("preserves SearXNG subpath instances when building the search endpoint", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ results: [] }))
-
- await webSearch(
- "subpath",
- {
- provider: "searxng",
- apiKey: "",
- providerConfigs: {
- searxng: { searXngUrl: "http://localhost:8080/searx/" },
- },
- },
- 5,
- )
- const parsed = new URL(String(fetchMock.mock.calls[0][0]))
- expect(parsed.origin + parsed.pathname).toBe("http://localhost:8080/searx/search")
- expect(parsed.searchParams.get("categories")).toBe("general")
- })
-
- it("surfaces SerpApi JSON errors", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ error: "Invalid API key" }))
-
- await expect(webSearch("x", { provider: "serpapi", apiKey: "bad" }, 5))
- .rejects.toThrow("SerpApi search failed: Invalid API key")
- })
-
- it("calls Firecrawl anonymous search without an API key", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: [
- {
- title: "LLM Wiki",
- url: "https://github.com/nashsu/llm_wiki",
- description: "Knowledge base app",
- },
- ],
- }))
-
- const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 2)
- const [url, init] = fetchMock.mock.calls[0]
-
- expect(url).toBe("https://api.firecrawl.dev/v2/search")
- expect(init).toEqual(expect.objectContaining({ method: "POST" }))
- expect((init?.headers as Record<string, string>).Authorization).toBeUndefined()
- expect(JSON.parse(String(init?.body))).toEqual({ query: "llm_wiki", limit: 2 })
- expect(out).toEqual([
- {
- title: "LLM Wiki",
- url: "https://github.com/nashsu/llm_wiki",
- snippet: "Knowledge base app",
- source: "github.com",
- },
- ])
- })
-
- it("surfaces Firecrawl suspicious-IP anonymous search guidance", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: false,
- error: "Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free.",
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .rejects.toThrow("Firecrawl anonymous search is blocked for this IP")
- })
-
- it("does not migrate legacy top-level apiKey into Firecrawl config", () => {
- const resolved = resolveSearchConfig({
- provider: "firecrawl",
- apiKey: "legacy-key",
+ expect(invokeMock).toHaveBeenCalledWith("web_search", {
+ query: "ai policy",
+ maxResults: 5,
+ config: expect.objectContaining({
+ provider: "serpapi",
+ apiKey: "serp-key",
+ serpApiEngine: "google_news",
+ }),
})
-
- expect(resolved.providerConfigs?.firecrawl).toBeUndefined()
- })
-
- it("allows Firecrawl success responses with advisory error fields", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- error: "partial results only",
- data: [
- {
- title: "Advisory result",
- url: "https://example.com/advisory",
- description: "Still usable",
- },
- ],
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Advisory result",
- url: "https://example.com/advisory",
- snippet: "Still usable",
- source: "example.com",
- },
- ])
- })
-
- it("surfaces Firecrawl unknown error fallback", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ success: false }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .rejects.toThrow("Firecrawl search failed: Unknown error")
- })
-
- it("surfaces Firecrawl non-ok JSON errors", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse(
- { error: "rate limited" },
- { status: 429 },
- ))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .rejects.toThrow("Firecrawl search failed: rate limited")
- })
-
- it("surfaces Firecrawl non-json HTTP errors", async () => {
- fetchMock.mockResolvedValueOnce(new Response("<html>Service unavailable</html>", {
- status: 503,
- headers: { "Content-Type": "text/html" },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .rejects.toThrow("Firecrawl search failed (503): <html>Service unavailable</html>")
- })
-
- it("surfaces Firecrawl network errors", async () => {
- fetchMock.mockRejectedValueOnce(new TypeError("Failed to fetch"))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .rejects.toThrow("Network error reaching Firecrawl Search")
- })
-
- it("normalizes Firecrawl results field and filters missing URLs", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- results: [
- { title: "Missing URL", description: "No URL" },
- {
- metadata: {
- title: "Metadata title",
- sourceURL: "https://docs.example/firecrawl",
- description: "Metadata description",
- },
- },
- ],
- }))
-
- const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5)
-
- expect(out).toEqual([
- {
- title: "Metadata title",
- url: "https://docs.example/firecrawl",
- snippet: "Metadata description",
- source: "docs.example",
- },
- ])
- })
-
- it("allows empty Firecrawl result sets", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: [] }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([])
- })
-
- it("normalizes object-shaped Firecrawl data results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- results: [
- {
- title: "Object data result",
- url: "https://example.com/object-data",
- description: "Nested data shape",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Object data result",
- url: "https://example.com/object-data",
- snippet: "Nested data shape",
- source: "example.com",
- },
- ])
- })
-
- it("normalizes categorized Firecrawl web results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- web: [
- {
- title: "Categorized web result",
- url: "https://example.com/firecrawl-web",
- description: "Web category shape",
- },
- ],
- news: [
- {
- title: "News result",
- url: "https://example.com/news",
- description: "Ignored category",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Categorized web result",
- url: "https://example.com/firecrawl-web",
- snippet: "Web category shape",
- source: "example.com",
- },
- ])
- })
-
- it("normalizes object-shaped top-level Firecrawl results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- results: {
- web: [
- {
- title: "Top-level object result",
- url: "https://example.com/top-results",
- description: "Nested under results.web",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Top-level object result",
- url: "https://example.com/top-results",
- snippet: "Nested under results.web",
- source: "example.com",
- },
- ])
- })
-
- it("uses Firecrawl web results before fallback categories", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- web: [
- {
- title: "Preferred web",
- url: "https://example.com/web",
- description: "Use me",
- },
- ],
- results: [
- {
- title: "Fallback result",
- url: "https://example.com/fallback",
- description: "Do not use me",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Preferred web",
- url: "https://example.com/web",
- snippet: "Use me",
- source: "example.com",
- },
- ])
- })
-
- it("keeps an empty Firecrawl web category empty instead of falling back", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- web: [],
- results: [
- {
- title: "Fallback result",
- url: "https://example.com/fallback",
- description: "Should stay hidden",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([])
- })
-
- it("normalizes Firecrawl items fallback results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- items: [
- {
- title: "Items fallback",
- url: "https://example.com/items",
- description: "Fallback item",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Items fallback",
- url: "https://example.com/items",
- snippet: "Fallback item",
- source: "example.com",
- },
- ])
- })
-
- it("treats unsupported Firecrawl result shapes as empty results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: { unexpected: true } }))
- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: "unexpected" }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([])
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([])
- })
-
- it("filters non-object Firecrawl result entries", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: {
- web: [
- null,
- "unexpected",
- {
- title: "Valid entry",
- url: "https://example.com/valid",
- description: "Keep this",
- },
- ],
- },
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Valid entry",
- url: "https://example.com/valid",
- snippet: "Keep this",
- source: "example.com",
- },
- ])
- })
-
- it("filters missing URLs in Firecrawl data results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- success: true,
- data: [
- { title: "No URL", description: "skip me" },
- { title: "Has URL", url: "https://example.com/ok", description: "keep me" },
- ],
- }))
-
- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))
- .resolves.toEqual([
- {
- title: "Has URL",
- url: "https://example.com/ok",
- snippet: "keep me",
- source: "example.com",
- },
- ])
})
it("requires a configured search provider and key", async () => {
await expect(webSearch("x", { provider: "none", apiKey: "" }, 5))
- .rejects.toThrow("Select a search provider")
+ .rejects.toThrow("Web search not configured")
await expect(webSearch("x", { provider: "serpapi", apiKey: "" }, 5))
- .rejects.toThrow(/API key/)
+ .rejects.toThrow("Add a Tavily, SerpApi, or Brave Search API key")
await expect(webSearch("x", { provider: "searxng", apiKey: "" }, 5))
- .rejects.toThrow("SearXNG instance URL")
+ .rejects.toThrow("Add a SearXNG instance URL")
+ await expect(webSearch("x", { provider: "ollama", apiKey: "" }, 5))
+ .rejects.toThrow("Ollama Web Search API requires an Ollama API key")
+ expect(invokeMock).not.toHaveBeenCalled()
})
- it("treats SearXNG instance URLs as configured without an API key", () => {
- expect(hasConfiguredSearchProvider({
- provider: "searxng",
- apiKey: "",
- providerConfigs: {
- searxng: {
- searXngUrl: "http://127.0.0.1:8080",
- searXngCategories: ["general", "science", "it"],
- },
- },
- searXngUrl: "http://127.0.0.1:8080",
- searXngCategories: ["general", "science", "it"],
- serpApiEngine: "google",
- })).toBe(true)
- })
-
- it("requires an API key for the Ollama Web Search API", async () => {
- await expect(webSearch(
- "official ollama",
- {
- provider: "ollama",
- apiKey: "",
- providerConfigs: {
- ollama: { ollamaUrl: "https://ollama.com" },
- },
- },
- 5,
- )).rejects.toThrow("requires an Ollama API key")
-
- expect(hasConfiguredSearchProvider({
- provider: "ollama",
- apiKey: "",
- providerConfigs: {
- ollama: { ollamaUrl: "https://ollama.com" },
- },
- })).toBe(false)
- })
-
- it("treats Firecrawl as configured without an API key", () => {
- expect(hasConfiguredSearchProvider({
- provider: "firecrawl",
- apiKey: "",
- })).toBe(true)
+ it("treats key-free providers as configured", () => {
+ expect(hasConfiguredSearchProvider({ provider: "searxng", apiKey: "", searXngUrl: "http://localhost:8080" })).toBe(true)
+ expect(hasConfiguredSearchProvider({ provider: "firecrawl", apiKey: "" })).toBe(true)
})
it("does not leak a stale top-level Ollama URL into non-Ollama providers", () => {
const resolved = resolveSearchConfig({
- provider: "serpapi",
+ provider: "firecrawl",
apiKey: "",
ollamaUrl: "http://localhost:11434",
- providerConfigs: {
- serpapi: { apiKey: "serp-key" },
- ollama: { ollamaUrl: "https://ollama.com" },
- },
})
+ expect(resolved.provider).toBe("firecrawl")
expect(resolved.ollamaUrl).toBe("https://ollama.com")
})
@@ -581,116 +94,11 @@ describe("webSearch", () => {
deepResearchSource: "anytxt",
anyTxt: { enabled: true, endpoint: "http://127.0.0.1:9920" },
})).toBe(true)
-
expect(hasConfiguredDeepResearchSources({
provider: "none",
apiKey: "",
- deepResearchSource: "anytxt",
- anyTxt: { enabled: false, endpoint: "http://127.0.0.1:9920" },
+ deepResearchSource: "both",
+ anyTxt: { enabled: false, endpoint: "" },
})).toBe(false)
-
- expect(hasConfiguredDeepResearchSources({
- provider: "none",
- apiKey: "",
- deepResearchSource: "web",
- anyTxt: { endpoint: "http://127.0.0.1:9920" },
- })).toBe(false)
-
- expect(resolveSearchConfig({
- provider: "none",
- apiKey: "",
- }).deepResearchSource).toBe("web")
- })
-
- it("calls the Ollama Web Search API with Bearer auth", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- results: [
- { title: "Official", url: "https://ollama.example/search", content: "Cloud result" },
- ],
- }))
-
- const out = await webSearch(
- "official ollama",
- {
- provider: "ollama",
- apiKey: "",
- providerConfigs: {
- ollama: { apiKey: "ollama-key" },
- },
- },
- 1,
- )
- const [url, init] = fetchMock.mock.calls[0]
-
- expect(url).toBe("https://ollama.com/api/web_search")
- expect((init?.headers as Record<string, string>).Authorization).toBe("Bearer ollama-key")
- expect(JSON.parse(String(init?.body))).toEqual({ query: "official ollama", max_results: 1 })
- expect(out).toEqual([
- { title: "Official", url: "https://ollama.example/search", snippet: "Cloud result", source: "ollama.example" },
- ])
- expect(hasConfiguredSearchProvider({
- provider: "ollama",
- apiKey: "",
- providerConfigs: {
- ollama: { apiKey: "ollama-key" },
- },
- })).toBe(true)
- })
-
- it("surfaces Ollama authentication guidance for 401 responses", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ error: "unauthorized" }, { status: 401 }))
-
- await expect(webSearch(
- "official ollama",
- {
- provider: "ollama",
- apiKey: "",
- providerConfigs: {
- ollama: { apiKey: "bad-key" },
- },
- },
- 5,
- )).rejects.toThrow("Check your Ollama API key")
- })
-
- it("calls Brave Search API with X-Subscription-Token and normalizes results", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({
- web: {
- results: [
- { title: "Brave A", url: "https://www.example.com/a", description: "First result" },
- { title: "Brave B", url: "https://docs.example/b", description: "Second result" },
- ],
- },
- }))
-
- const out = await webSearch("brave query", { provider: "brave", apiKey: "brave-key" }, 2)
- const [url, init] = fetchMock.mock.calls[0]
- const parsed = new URL(String(url))
-
- expect(parsed.origin + parsed.pathname).toBe("https://api.search.brave.com/res/v1/web/search")
- expect(parsed.searchParams.get("q")).toBe("brave query")
- expect(parsed.searchParams.get("count")).toBe("2")
- expect(init).toEqual(expect.objectContaining({ method: "GET" }))
- expect((init?.headers as Record<string, string>)["X-Subscription-Token"]).toBe("brave-key")
- expect(out).toEqual([
- { title: "Brave A", url: "https://www.example.com/a", snippet: "First result", source: "example.com" },
- { title: "Brave B", url: "https://docs.example/b", snippet: "Second result", source: "docs.example" },
- ])
- })
-
- it("clamps Brave Search count to the API's 20-result ceiling", async () => {
- fetchMock.mockResolvedValueOnce(jsonResponse({ web: { results: [] } }))
-
- await webSearch("anything", { provider: "brave", apiKey: "k" }, 50)
- const [url] = fetchMock.mock.calls[0]
- expect(new URL(String(url)).searchParams.get("count")).toBe("20")
- })
-
- it("surfaces Brave Search authentication guidance for 401 responses", async () => {
- fetchMock.mockResolvedValueOnce(new Response("forbidden", { status: 401 }))
-
- await expect(
- webSearch("x", { provider: "brave", apiKey: "bad" }, 5),
- ).rejects.toThrow("Brave Search API authentication failed")
})
})
diff --git a/src/lib/web-search.ts b/src/lib/web-search.ts
index 8b814fa9..e87e54c1 100644
--- a/src/lib/web-search.ts
+++ b/src/lib/web-search.ts
@@ -1,3 +1,4 @@
+import { invoke } from "@tauri-apps/api/core"
import type {
SearchApiConfig,
SearchProvider,
@@ -5,7 +6,6 @@ import type {
SearXngCategory,
SerpApiEngine,
} from "@/stores/wiki-store"
-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"
import { hasConfiguredAnyTxt, normalizeAnyTxtConfig } from "@/lib/anytxt-search"
export interface WebSearchResult {
@@ -108,15 +108,9 @@ export function resolveSearchConfig(config: SearchApiConfig): SearchApiConfig {
export function hasConfiguredSearchProvider(config: SearchApiConfig): boolean {
const resolved = resolveSearchConfig(config)
if (resolved.provider === "none") return false
- if (resolved.provider === "searxng") {
- return Boolean(resolved.searXngUrl?.trim())
- }
- if (resolved.provider === "ollama") {
- return Boolean(resolved.apiKey?.trim())
- }
- if (resolved.provider === "firecrawl") {
- return true
- }
+ if (resolved.provider === "searxng") return Boolean(resolved.searXngUrl?.trim())
+ if (resolved.provider === "ollama") return Boolean(resolved.apiKey?.trim())
+ if (resolved.provider === "firecrawl") return true
return Boolean(resolved.apiKey?.trim())
}
@@ -153,512 +147,9 @@ export async function webSearch(
throw new Error("Ollama Web Search API requires an Ollama API key. Add one in Settings.")
}
- switch (resolved.provider) {
- case "tavily":
- return tavilySearch(query, resolved.apiKey, maxResults)
- case "serpapi":
- return serpApiSearch(query, resolved.apiKey, maxResults, resolved.serpApiEngine ?? "google")
- case "searxng":
- return searXngSearch(query, resolved.searXngUrl ?? "", maxResults, resolved.searXngCategories ?? ["general"])
- case "ollama":
- return ollamaSearch(query, resolved.apiKey ?? "", maxResults)
- case "brave":
- return braveSearch(query, resolved.apiKey, maxResults)
- case "firecrawl":
- return firecrawlSearch(query, maxResults)
- default:
- throw new Error(`Unknown search provider: ${resolved.provider}`)
- }
-}
-
-function searXngSearchUrl(instanceUrl: string): URL {
- const trimmed = instanceUrl.trim()
- const withProtocol = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`
- const url = new URL(withProtocol)
- const path = url.pathname.replace(/\/+$/, "")
- url.pathname = path.endsWith("/search") || path === "/search"
- ? path
- : `${path}/search`
- url.search = ""
- url.hash = ""
- return url
-}
-
-async function searXngSearch(
- query: string,
- instanceUrl: string,
- maxResults: number,
- categories: SearXngCategory[],
-): Promise<WebSearchResult[]> {
- let endpoint: URL
- try {
- endpoint = searXngSearchUrl(instanceUrl)
- } catch {
- throw new Error("Invalid SearXNG instance URL. Use a valid http(s) URL, for example https://search.example.com.")
- }
-
- endpoint.searchParams.set("q", query)
- endpoint.searchParams.set("format", "json")
- endpoint.searchParams.set("categories", (categories.length > 0 ? categories : ["general"]).join(","))
-
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch(endpoint.toString(), {
- method: "GET",
- headers: { Accept: "application/json" },
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching the SearXNG instance. Check the instance URL and whether JSON search is enabled.",
- )
- }
- throw err
- }
-
- if (!response.ok) {
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`SearXNG search failed (${response.status}): ${errorText}`)
- }
-
- const data = await response.json()
- return normalizeSearXngResults(data, maxResults)
-}
-
-function normalizeSearXngResults(data: { results?: unknown[] }, maxResults: number): WebSearchResult[] {
- return (data.results ?? [])
- .slice(0, maxResults)
- .map((item) => normalizeSearXngResult(item))
- .filter((item) => item.url.length > 0)
-}
-
-function normalizeSearXngResult(item: unknown): WebSearchResult {
- const r = item as {
- title?: string
- url?: string
- content?: string
- engine?: string
- category?: string
- }
- const url = r.url ?? ""
- return {
- title: r.title ?? "Untitled",
- url,
- snippet: r.content ?? "",
- source: hostnameFromUrl(url) || r.engine || r.category || "",
- }
-}
-
-function hostnameFromUrl(url: string): string {
- try {
- return new URL(url).hostname.replace("www.", "")
- } catch {
- return ""
- }
-}
-
-async function tavilySearch(
- query: string,
- apiKey: string,
- maxResults: number,
-): Promise<WebSearchResult[]> {
- // Route through the Tauri HTTP plugin so future non-Tavily search
- // providers (Serper, Exa, Brave, Google CSE, ...) with less friendly
- // CORS don't each need their own workaround. See tauri-fetch.ts.
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch("https://api.tavily.com/search", {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- api_key: apiKey,
- query,
- max_results: maxResults,
- search_depth: "advanced",
- include_answer: false,
- }),
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching api.tavily.com. Check your connectivity and whether the Tavily API key is still valid.",
- )
- }
- throw err
- }
-
- if (!response.ok) {
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`Tavily search failed (${response.status}): ${errorText}`)
- }
-
- const data = await response.json()
-
- return (data.results ?? []).map((r: { title: string; url: string; content: string }) => ({
- title: r.title ?? "Untitled",
- url: r.url ?? "",
- snippet: r.content ?? "",
- source: hostnameFromUrl(r.url ?? ""),
- }))
-}
-
-interface FirecrawlSearchResponse {
- success?: boolean
- error?: string
- data?: unknown
- results?: unknown
-}
-
-async function firecrawlSearch(
- query: string,
- maxResults: number,
-): Promise<WebSearchResult[]> {
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch("https://api.firecrawl.dev/v2/search", {
- method: "POST",
- headers: {
- Accept: "application/json",
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- query,
- limit: maxResults,
- }),
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching Firecrawl Search. Check your connectivity or choose another Web Search provider.",
- )
- }
- throw err
- }
-
- const text = await response.text().catch(() => "")
- let data: FirecrawlSearchResponse = {}
- try {
- data = text ? JSON.parse(text) as FirecrawlSearchResponse : {}
- } catch {
- if (!response.ok) {
- throw new Error(`Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)
- }
- throw new Error("Firecrawl search returned an invalid JSON response.")
- }
-
- if (!response.ok) {
- throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)
- }
-
- if (data.success === false || (data.success !== true && typeof data.error === "string" && data.error.trim())) {
- throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed: ${data.error ?? "Unknown error"}`)
- }
-
- return normalizeFirecrawlResults(data, maxResults)
-}
-
-function friendlyFirecrawlError(error?: string): string | null {
- const message = error?.trim()
- if (!message) return null
- if (/suspicious|without an API key|firecrawl can't be used without an api key/i.test(message)) {
- return "Firecrawl anonymous search is blocked for this IP. Firecrawl says this network looks suspicious; choose another Web Search provider or try a different network."
- }
- return `Firecrawl search failed: ${message}`
-}
-
-function normalizeFirecrawlResults(data: FirecrawlSearchResponse, maxResults: number): WebSearchResult[] {
- const rawResults = extractFirecrawlResultArray(data)
- return rawResults
- .slice(0, maxResults)
- .map((item) => normalizeFirecrawlResult(item))
- .filter((item) => item.url.length > 0)
-}
-
-function extractFirecrawlResultArray(data: FirecrawlSearchResponse): unknown[] {
- if (Array.isArray(data.data)) return data.data
- if (Array.isArray(data.results)) return data.results
-
- return extractFirecrawlNestedResultArray(data.data)
- ?? extractFirecrawlNestedResultArray(data.results)
- ?? []
-}
-
-function extractFirecrawlNestedResultArray(value: unknown): unknown[] | null {
- if (!value || typeof value !== "object") return null
- const nested = value as Record<string, unknown>
-
- // Firecrawl v2 may return categorized results as data.web/news/images.
- // LLM Wiki consumes the first supported text-search category; image/news
- // categories are intentionally ignored because their payloads differ.
- // The first matching category wins even when it is empty.
- for (const key of ["web", "results", "items"]) {
- const nestedValue = nested[key]
- if (Array.isArray(nestedValue)) return nestedValue
- }
-
- return null
-}
-
-function normalizeFirecrawlResult(item: unknown): WebSearchResult {
- if (!item || typeof item !== "object") {
- return { title: "Untitled", url: "", snippet: "", source: "" }
- }
- const r = item as {
- title?: string
- url?: string
- link?: string
- markdown?: string
- content?: string
- description?: string
- snippet?: string
- source?: string
- metadata?: {
- title?: string
- sourceURL?: string
- url?: string
- description?: string
- }
- }
- const url = r.url ?? r.link ?? r.metadata?.sourceURL ?? r.metadata?.url ?? ""
- return {
- title: r.title ?? r.metadata?.title ?? "Untitled",
- url,
- snippet: r.snippet ?? r.description ?? r.metadata?.description ?? r.content ?? r.markdown ?? "",
- source: hostnameFromUrl(url) || r.source || "",
- }
-}
-
-async function serpApiSearch(
- query: string,
- apiKey: string,
- maxResults: number,
- engine: SerpApiEngine,
-): Promise<WebSearchResult[]> {
- const params = new URLSearchParams({
- engine,
- q: query,
- api_key: apiKey,
- num: String(maxResults),
+ return invoke<WebSearchResult[]>("web_search", {
+ query,
+ config: resolved,
+ maxResults,
})
-
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch(`https://serpapi.com/search?${params.toString()}`, {
- method: "GET",
- headers: { Accept: "application/json" },
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching serpapi.com. Check your connectivity and whether the SerpApi API key is still valid.",
- )
- }
- throw err
- }
-
- if (!response.ok) {
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`SerpApi search failed (${response.status}): ${errorText}`)
- }
-
- const data = await response.json()
- if (typeof data.error === "string" && data.error.trim()) {
- throw new Error(`SerpApi search failed: ${data.error}`)
- }
-
- return normalizeSerpApiResults(data, maxResults)
-}
-
-function normalizeSerpApiResults(data: {
- organic_results?: unknown[]
- news_results?: unknown[]
- images_results?: unknown[]
- video_results?: unknown[]
- videos_results?: unknown[]
- shopping_results?: unknown[]
-}, maxResults: number): WebSearchResult[] {
- const rawResults =
- data.organic_results ??
- data.news_results ??
- data.images_results ??
- data.video_results ??
- data.videos_results ??
- data.shopping_results ??
- []
-
- return rawResults
- .slice(0, maxResults)
- .map((item) => normalizeSerpApiResult(item))
-}
-
-function normalizeSerpApiResult(item: unknown): WebSearchResult {
- const r = item as {
- title?: string
- link?: string
- url?: string
- source?: string
- snippet?: string
- summary?: string
- description?: string
- thumbnail?: string
- original?: string
- displayed_link?: string
- }
- const url = r.link ?? r.url ?? r.original ?? r.thumbnail ?? ""
- return {
- title: r.title ?? "Untitled",
- url,
- snippet: r.snippet ?? r.summary ?? r.description ?? "",
- source: hostnameFromUrl(url) || r.source || r.displayed_link || "",
- }
-}
-
-interface OllamaSearchResponse {
- results?: Array<{
- title?: string
- url?: string
- content?: string
- }>
- error?: string
-}
-
-async function ollamaSearch(
- query: string,
- apiKey: string,
- maxResults: number,
-): Promise<WebSearchResult[]> {
- const trimmedApiKey = apiKey.trim()
- if (!trimmedApiKey) {
- throw new Error("Ollama Web Search API requires an Ollama API key. Add one in Settings.")
- }
-
- const headers: Record<string, string> = {
- Accept: "application/json",
- "Content-Type": "application/json",
- Authorization: `Bearer ${trimmedApiKey}`,
- }
-
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch("https://ollama.com/api/web_search", {
- method: "POST",
- headers,
- body: JSON.stringify({
- query,
- max_results: maxResults,
- }),
- })
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching the Ollama Web Search API. Check your connectivity and whether the Ollama API key is still valid.",
- )
- }
- throw err
- }
-
- if (!response.ok) {
- if (response.status === 401) {
- throw new Error(
- "Ollama Web Search API authentication failed. Check your Ollama API key.",
- )
- }
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`Ollama web search failed (${response.status}): ${errorText}`)
- }
-
- const data = (await response.json()) as OllamaSearchResponse
-
- if (data.error) {
- throw new Error(`Ollama web search error: ${data.error}`)
- }
-
- return (data.results ?? [])
- .slice(0, maxResults)
- .map((r) => {
- const url = r.url ?? ""
- return {
- title: r.title ?? "Untitled",
- url,
- snippet: r.content ?? "",
- source: hostnameFromUrl(url),
- }
- })
-}
-
-interface BraveSearchResponse {
- web?: {
- results?: Array<{
- title?: string
- url?: string
- description?: string
- }>
- }
- message?: string
-}
-
-async function braveSearch(
- query: string,
- apiKey: string,
- maxResults: number,
-): Promise<WebSearchResult[]> {
- // Brave Web Search API caps `count` at 20 per request. Higher values
- // are silently clamped server-side; bound here so the URL stays
- // honest and we don't surprise users by appearing to ask for more.
- const count = Math.max(1, Math.min(maxResults, 20))
- const params = new URLSearchParams({ q: query, count: String(count) })
-
- const httpFetch = await getHttpFetch()
- let response: Response
- try {
- response = await httpFetch(
- `https://api.search.brave.com/res/v1/web/search?${params.toString()}`,
- {
- method: "GET",
- headers: {
- Accept: "application/json",
- "X-Subscription-Token": apiKey,
- },
- },
- )
- } catch (err) {
- if (isFetchNetworkError(err)) {
- throw new Error(
- "Network error reaching api.search.brave.com. Check your connectivity and whether the Brave Search API key is still valid.",
- )
- }
- throw err
- }
-
- if (!response.ok) {
- if (response.status === 401 || response.status === 403) {
- throw new Error(
- "Brave Search API authentication failed. Check your subscription token in Settings.",
- )
- }
- const errorText = await response.text().catch(() => "Unknown error")
- throw new Error(`Brave search failed (${response.status}): ${errorText}`)
- }
-
- const data = (await response.json()) as BraveSearchResponse
- if (data.message && !data.web) {
- throw new Error(`Brave search error: ${data.message}`)
- }
-
- return (data.web?.results ?? [])
- .slice(0, maxResults)
- .map((r) => {
- const url = r.url ?? ""
- return {
- title: r.title ?? "Untitled",
- url,
- snippet: r.description ?? "",
- source: hostnameFromUrl(url),
- }
- })
}