LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
refactor: move search providers into rust backend
nash_su6cffec9120de
diff --git a/src-tauri/src/agent/skills.rs b/src-tauri/src/agent/skills.rsindex 6fadc512..320fea8a 100644--- a/src-tauri/src/agent/skills.rs+++ b/src-tauri/src/agent/skills.rs@@ -65,6 +65,7 @@ fn normalize_skill_name(value: &str) -> Option<String> {|| trimmed.contains('/')|| trimmed.contains('\\')|| trimmed.contains("..")+ || !is_portable_skill_name(trimmed){return None;}@@ -73,8 +74,9 @@ fn normalize_skill_name(value: &str) -> Option<String> {fn split_frontmatter(raw: &str) -> (Option<String>, String) {let normalized = raw.strip_prefix('\u{feff}').unwrap_or(raw);+ let normalized = normalized.replace("\r\n", "\n").replace('\r', "\n");if !normalized.starts_with("---\n") {- return (None, normalized.to_string());+ return (None, normalized);}let rest = &normalized[4..];if let Some(end) = rest.find("\n---") {@@ -85,10 +87,50 @@ fn split_frontmatter(raw: &str) -> (Option<String>, String) {.to_string();(Some(fm), after)} else {- (None, normalized.to_string())+ (None, normalized)}}+fn is_portable_skill_name(value: &str) -> bool {+ if value+ .chars()+ .any(|ch| matches!(ch, '<' | '>' | ':' | '"' | '|' | '?' | '*') || ch <= '\u{1f}')+ {+ return false;+ }+ let stem = value+ .split('.')+ .next()+ .unwrap_or(value)+ .trim_end_matches(' ')+ .to_ascii_uppercase();+ !matches!(+ stem.as_str(),+ "CON"+ | "PRN"+ | "AUX"+ | "NUL"+ | "COM1"+ | "COM2"+ | "COM3"+ | "COM4"+ | "COM5"+ | "COM6"+ | "COM7"+ | "COM8"+ | "COM9"+ | "LPT1"+ | "LPT2"+ | "LPT3"+ | "LPT4"+ | "LPT5"+ | "LPT6"+ | "LPT7"+ | "LPT8"+ | "LPT9"+ )+}+fn yaml_string_field(frontmatter: &str, key: &str) -> Option<String> {let prefix = format!("{key}:");for line in frontmatter.lines() {@@ -135,9 +177,34 @@ mod tests {let _ = fs::remove_dir_all(root);}+ #[test]+ fn load_project_skills_reads_crlf_frontmatter() {+ let root = std::env::temp_dir().join(format!("llm-wiki-skills-{}", Uuid::new_v4()));+ let skills_dir = root.join(".llm-wiki").join("skills");+ fs::create_dir_all(&skills_dir).unwrap();+ fs::write(+ skills_dir.join("reviewer.md"),+ "---\r\nname: reviewer\r\ndescription: Review source quality\r\n---\r\nCheck claims carefully.",+ )+ .unwrap();++ let skills = load_project_skills(root.to_str().unwrap(), &["reviewer".to_string()]);+ assert_eq!(skills.len(), 1);+ assert_eq!(skills[0].name, "reviewer");+ assert_eq!(skills[0].description, "Review source quality");+ assert_eq!(skills[0].instructions, "Check claims carefully.");+ let _ = fs::remove_dir_all(root);+ }+#[test]fn load_project_skills_rejects_path_traversal_names() {let skills = load_project_skills("/tmp/missing", &["../secret".to_string()]);assert!(skills.is_empty());}++ #[test]+ fn load_project_skills_rejects_windows_reserved_names() {+ let skills = load_project_skills("/tmp/missing", &["con".to_string(), "a:b".to_string()]);+ assert!(skills.is_empty());+ }}diff --git a/src-tauri/src/agent/tools.rs b/src-tauri/src/agent/tools.rsindex 2945f9a8..d77efb88 100644--- a/src-tauri/src/agent/tools.rs+++ b/src-tauri/src/agent/tools.rs@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};use serde_json::{json, Value};use walkdir::WalkDir;+use crate::commands::external_search::file_url_for_path;use crate::commands::search::{self, SearchEmbeddingConfig};use super::types::AgentReference;@@ -53,6 +54,7 @@ pub trait AgentTool: Send + Sync {}pub trait ToolRegistry {+ #[allow(dead_code)]fn specs(&self) -> Vec<ToolSpec>;fn execute<'a>(&'a self,@@ -215,6 +217,8 @@ pub struct WebSearchConfig {#[serde(default)]pub api_key: String,#[serde(default)]+ pub ollama_url: Option<String>,+ #[serde(default)]pub sear_xng_url: Option<String>,#[serde(default)]pub sear_xng_categories: Option<Vec<String>>,@@ -230,6 +234,8 @@ pub struct WebSearchProviderOverride {#[serde(default)]pub api_key: Option<String>,#[serde(default)]+ pub ollama_url: Option<String>,+ #[serde(default)]pub sear_xng_url: Option<String>,#[serde(default)]pub sear_xng_categories: Option<Vec<String>>,@@ -268,6 +274,10 @@ impl WebSearchConfig {.api_key.clone().unwrap_or_else(|| self.api_key.clone()),+ ollama_url: override_cfg+ .ollama_url+ .clone()+ .or_else(|| self.ollama_url.clone()),sear_xng_url: override_cfg.sear_xng_url.clone()@@ -285,6 +295,10 @@ impl WebSearchConfig {}}+// Keep the spec list close to the executor even though the current planner+// still uses fixed tool names. API/MCP tool discovery and future native+// tool-calling should use this list instead of duplicating tool metadata.+#[allow(dead_code)]pub fn builtin_tool_specs() -> Vec<ToolSpec> {vec![ToolSpec {@@ -440,14 +454,6 @@ fn tool_top_k(input: &Value) -> usize {.clamp(1, 10)}-pub fn write_wiki_page(- project_path: &str,- rel_path: &str,- content: &str,-) -> Result<AgentReference, String> {- write_wiki_page_with_options(project_path, rel_path, content, false)-}-pub fn write_wiki_page_with_options(project_path: &str,rel_path: &str,@@ -547,7 +553,7 @@ pub async fn run_web_search(if provider.is_empty() || provider == "none" {return Err("Web search provider is not configured.".to_string());}- let max_results = top_k.clamp(1, 10);+ let max_results = top_k.clamp(1, 20);let client = reqwest::Client::builder().timeout(std::time::Duration::from_secs(WEB_SEARCH_TIMEOUT_SECS)).build()@@ -556,6 +562,7 @@ pub async fn run_web_search("firecrawl" => firecrawl_search(&client, query, max_results).await?,"searxng" => searxng_search(&client, query, &config, max_results).await?,"tavily" => tavily_search(&client, query, &config, max_results).await?,+ "ollama" => ollama_search(&client, query, &config, max_results).await?,"brave" => brave_search(&client, query, &config, max_results).await?,"serpapi" => serpapi_search(&client, query, &config, max_results).await?,other => {@@ -564,17 +571,7 @@ pub async fn run_web_search())}};- Ok(raw- .into_iter()- .take(max_results)- .map(|item| AgentReference {- title: item.title,- path: item.url,- kind: "web".to_string(),- snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),- score: None,- })- .collect())+ Ok(web_items_to_references(raw, max_results))}pub async fn run_anytxt_search(@@ -596,12 +593,15 @@ pub async fn run_anytxt_search(.filter(|value| !value.trim().is_empty()).unwrap_or(DEFAULT_ANYTXT_ENDPOINT).trim()- .trim_end_matches('/')- .to_string();+ .trim_end_matches('/');+ let endpoint = normalize_anytxt_endpoint(endpoint);let limit = top_k- .clamp(1, 10)+ .clamp(1, 100).min(config.limit.unwrap_or(DEFAULT_ANYTXT_LIMIT).clamp(1, 100));- let pattern = anytxt_pattern_from_query(query);+ // AnyTXT has its own query syntax. The caller may already have rewritten+ // natural language into keyword form, so do not run the source-search+ // tokenizer here; pass the pattern through unchanged.+ let pattern = query.to_string();let filter_dir = config.filter_dir.unwrap_or_default();let filter_ext = config.filter_ext@@ -611,6 +611,18 @@ pub async fn run_anytxt_search(.timeout(std::time::Duration::from_secs(WEB_SEARCH_TIMEOUT_SECS)).build().map_err(|err| format!("Failed to build AnyTXT client: {err}"))?;+ let mut input = json!({+ "pattern": pattern,+ "filterExt": filter_ext,+ "lastModifyBegin": 0,+ "lastModifyEnd": ANYTXT_LAST_MODIFY_END,+ "limit": limit.to_string(),+ "offset": 0,+ "order": 0+ });+ if !filter_dir.trim().is_empty() {+ input["filterDir"] = Value::String(filter_dir);+ }let response = client.post(&endpoint).header("Accept", "application/json")@@ -618,18 +630,7 @@ pub async fn run_anytxt_search("id": 1,"jsonrpc": "2.0","method": "ATRpcServer.Searcher.V1.GetResult",- "params": {- "input": {- "pattern": pattern,- "filterDir": filter_dir,- "filterExt": filter_ext,- "lastModifyBegin": 0,- "lastModifyEnd": ANYTXT_LAST_MODIFY_END,- "limit": limit.to_string(),- "offset": 0,- "order": 0- }- }+ "params": { "input": input }})).send().await@@ -652,74 +653,145 @@ pub async fn run_anytxt_search(trim_text(&error.to_string(), 300)));}- Ok(extract_anytxt_items(&value)- .into_iter()- .take(limit)- .map(|item| AgentReference {+ let mut references = Vec::new();+ for item in extract_anytxt_items(&value).into_iter().take(limit) {+ let fragment = if !item.fid.trim().is_empty() {+ get_anytxt_fragment(&client, &endpoint, &item.fid, &pattern)+ .await+ .unwrap_or_default()+ } else {+ String::new()+ };+ references.push(AgentReference {title: item.title,- path: item.path,+ path: file_url_for_path(&item.path),kind: "anytxt".to_string(),- snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),+ snippet: Some(trim_text(+ if fragment.trim().is_empty() {+ &item.snippet+ } else {+ &fragment+ },+ 1200,+ ))+ .filter(|s| !s.trim().is_empty()),score: None,- })- .collect())+ });+ }+ Ok(references)}#[derive(Debug, Clone)]struct AnyTxtItem {+ fid: String,title: String,path: String,snippet: String,}fn extract_anytxt_items(value: &Value) -> Vec<AnyTxtItem> {- let candidates = value- .get("result")- .and_then(|result| {- result- .get("items")- .or_else(|| result.get("files"))- .or_else(|| result.get("result"))- .or_else(|| result.get("data"))- })- .and_then(Value::as_array)- .or_else(|| value.get("result").and_then(Value::as_array))- .cloned()- .unwrap_or_default();+ let result = value.get("result").unwrap_or(value);+ let candidates = first_anytxt_array(+ result,+ &[+ &[][..],+ &["items"],+ &["files"],+ &["results"],+ &["list"],+ &["value"],+ &["data"],+ &["output"],+ &["output", "items"],+ &["output", "files"],+ &["output", "results"],+ &["output", "list"],+ &["output", "value"],+ &["output", "data"],+ &["data", "items"],+ &["data", "files"],+ &["data", "results"],+ &["data", "list"],+ &["data", "value"],+ &["data", "output"],+ &["data", "output", "items"],+ &["data", "output", "files"],+ &["data", "output", "results"],+ &["data", "output", "list"],+ &["data", "output", "value"],+ ],+ )+ .unwrap_or_default();+ let fields = first_anytxt_fields(+ result,+ &[+ &["field"][..],+ &["fields"],+ &["output", "field"],+ &["output", "fields"],+ &["data", "field"],+ &["data", "fields"],+ &["data", "output", "field"],+ &["data", "output", "fields"],+ ],+ )+ .unwrap_or_default();candidates.into_iter().filter_map(|item| {- let path = item- .get("path")- .or_else(|| item.get("file"))- .or_else(|| item.get("filename"))- .and_then(Value::as_str)- .unwrap_or("")- .to_string();- let title = item- .get("title")- .or_else(|| item.get("name"))- .and_then(Value::as_str)- .filter(|value| !value.trim().is_empty())- .map(ToOwned::to_owned)- .unwrap_or_else(|| {- Path::new(&path)- .file_name()- .and_then(|s| s.to_str())- .unwrap_or("AnyTXT result")- .to_string()- });- let snippet = item- .get("snippet")- .or_else(|| item.get("fragment"))- .or_else(|| item.get("text"))- .and_then(Value::as_str)- .unwrap_or("")+ let record = normalize_anytxt_record(item, &fields);+ let fid = string_field(&record, &["fid", "id", "fileId", "file_id"]);+ let raw_path = string_field(+ &record,+ &[+ "path",+ "file",+ "filePath",+ "file_path",+ "fullPath",+ "full_path",+ "filename",+ "fileName",+ "name",+ ],+ );+ let path = if raw_path.is_empty() && !fid.is_empty() {+ format!("anytxt://{fid}")+ } else {+ raw_path+ };+ let title = string_field(&record, &["title", "name", "fileName", "filename"])+ .trim().to_string();+ let title = if title.is_empty() {+ Path::new(&path)+ .file_name()+ .and_then(|s| s.to_str())+ .filter(|value| !value.trim().is_empty())+ .unwrap_or("AnyTXT result")+ .to_string()+ } else {+ title+ };+ let snippet = string_field(+ &record,+ &[+ "snippet",+ "fragment",+ "content",+ "contents",+ "text",+ "summary",+ "highlight",+ "hitText",+ "hit_text",+ ],+ );if path.is_empty() && snippet.is_empty() {None} else {Some(AnyTxtItem {+ fid,title,path,snippet,@@ -729,12 +801,163 @@ fn extract_anytxt_items(value: &Value) -> Vec<AnyTxtItem> {.collect()}-fn anytxt_pattern_from_query(query: &str) -> String {- let terms = source_query_terms(&query.to_lowercase());- if terms.is_empty() {- return query.trim().to_string();+fn first_anytxt_array(value: &Value, paths: &[&[&str]]) -> Option<Vec<Value>> {+ for path in paths {+ let Some(candidate) = value_at_path(value, path) else {+ continue;+ };+ if let Some(items) = candidate.as_array() {+ return Some(items.clone());+ }+ }+ None+}++fn first_anytxt_fields(value: &Value, paths: &[&[&str]]) -> Option<Vec<String>> {+ for path in paths {+ let Some(candidate) = value_at_path(value, path) else {+ continue;+ };+ let Some(items) = candidate.as_array() else {+ continue;+ };+ let fields = items+ .iter()+ .filter_map(Value::as_str)+ .map(ToOwned::to_owned)+ .collect::<Vec<_>>();+ if !fields.is_empty() {+ return Some(fields);+ }+ }+ None+}++fn value_at_path<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> {+ let mut current = value;+ for key in path {+ current = current.get(*key)?;+ }+ Some(current)+}++fn normalize_anytxt_record(item: Value, fields: &[String]) -> serde_json::Map<String, Value> {+ match item {+ Value::Object(object) => object,+ Value::Array(row) if !fields.is_empty() => fields+ .iter()+ .cloned()+ .zip(row)+ .collect::<serde_json::Map<String, Value>>(),+ other => {+ let mut object = serde_json::Map::new();+ object.insert("text".to_string(), other);+ object+ }+ }+}++fn string_field(record: &serde_json::Map<String, Value>, keys: &[&str]) -> String {+ for key in keys {+ let Some(value) = record.get(*key) else {+ continue;+ };+ if let Some(text) = value.as_str().filter(|text| !text.trim().is_empty()) {+ return text.trim().to_string();+ }+ if let Some(number) = value.as_i64() {+ return number.to_string();+ }+ if let Some(number) = value.as_u64() {+ return number.to_string();+ }+ }+ String::new()+}++async fn get_anytxt_fragment(+ client: &reqwest::Client,+ endpoint: &str,+ fid: &str,+ pattern: &str,+) -> Result<String, String> {+ let response = client+ .post(endpoint)+ .header("Accept", "application/json")+ .json(&json!({+ "id": 2,+ "jsonrpc": "2.0",+ "method": "ATRpcServer.Searcher.V1.GetFragment",+ "params": { "input": { "fid": fid, "pattern": pattern } }+ }))+ .send()+ .await+ .map_err(|err| format!("AnyTXT fragment failed: {err}"))?;+ let status = response.status();+ let text = response+ .text()+ .await+ .map_err(|err| format!("Failed to read AnyTXT fragment response: {err}"))?;+ if !status.is_success() {+ return Err(format!(+ "AnyTXT fragment HTTP {status}: {}",+ trim_text(&text, 300)+ ));+ }+ let value: Value = serde_json::from_str(&text).map_err(|_| {+ format!(+ "AnyTXT fragment returned invalid JSON: {}",+ trim_text(&text, 300)+ )+ })?;+ if let Some(error) = value.get("error") {+ return Err(format!(+ "AnyTXT fragment error: {}",+ trim_text(&error.to_string(), 300)+ ));+ }+ Ok(extract_anytxt_fragment_text(+ value.get("result").unwrap_or(&Value::Null),+ ))+}++fn extract_anytxt_fragment_text(value: &Value) -> String {+ if let Some(text) = value.as_str() {+ return text.to_string();+ }+ if let Some(items) = value.as_array() {+ return items+ .iter()+ .map(extract_anytxt_fragment_text)+ .filter(|item| !item.trim().is_empty())+ .collect::<Vec<_>>()+ .join("\n\n");+ }+ let Some(object) = value.as_object() else {+ return String::new();+ };+ for key in ["text", "fragment", "content", "snippet", "html"] {+ if let Some(text) = object.get(key).and_then(Value::as_str) {+ return text.to_string();+ }+ }+ for key in ["output", "result", "data", "fragments", "items", "list"] {+ if let Some(next) = object.get(key) {+ let text = extract_anytxt_fragment_text(next);+ if !text.trim().is_empty() {+ return text;+ }+ }+ }+ String::new()+}++fn normalize_anytxt_endpoint(value: &str) -> String {+ if value.starts_with("http://") || value.starts_with("https://") {+ value.to_string()+ } else {+ format!("http://{value}")}- terms.into_iter().take(5).collect::<Vec<_>>().join(" ")}#[derive(Debug, Clone)]@@ -775,12 +998,7 @@ async fn firecrawl_search(.unwrap_or_else(|| format!("Firecrawl search failed ({status})"));return Err(msg);}- let items = parsed- .get("data")- .and_then(Value::as_array)- .or_else(|| parsed.get("results").and_then(Value::as_array))- .cloned()- .unwrap_or_default();+ let items = extract_web_items(&parsed, &["data", "results"]);Ok(items.into_iter().map(normalize_web_result).collect())}@@ -868,6 +1086,45 @@ async fn tavily_search(.await}+async fn ollama_search(+ client: &reqwest::Client,+ query: &str,+ config: &WebSearchConfig,+ max_results: usize,+) -> Result<Vec<WebSearchItem>, String> {+ let key = required_api_key(config, "Ollama")?;+ let base = config+ .ollama_url+ .as_deref()+ .filter(|value| !value.trim().is_empty())+ .unwrap_or("https://ollama.com")+ .trim()+ .trim_end_matches('/');+ let url = format!("{base}/api/web_search");+ let response = client+ .post(url)+ .header("Accept", "application/json")+ .bearer_auth(key)+ .json(&json!({+ "query": query,+ "max_results": max_results+ }))+ .send()+ .await+ .map_err(|err| format!("Network error reaching Ollama Web Search: {err}"))?;+ parse_web_json_response(response, "Ollama Web Search", |value| {+ value+ .get("results")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default()+ .into_iter()+ .map(normalize_web_result)+ .collect()+ })+ .await+}+async fn brave_search(client: &reqwest::Client,query: &str,@@ -965,20 +1222,49 @@ async fn parse_web_json_response(if let Some(error) = value.get("error").and_then(Value::as_str) {return Err(format!("{provider} search failed: {error}"));}+ if let Some(message) = provider_payload_error(provider, &value) {+ return Err(message);+ }Ok(parse(value))}+fn provider_payload_error(provider: &str, value: &Value) -> Option<String> {+ if provider == "Brave Search" && value.get("web").is_none() {+ let message = value.get("message").and_then(Value::as_str)?;+ return Some(format!("{provider} search failed: {message}"));+ }+ None+}++fn web_items_to_references(raw: Vec<WebSearchItem>, max_results: usize) -> Vec<AgentReference> {+ raw.into_iter()+ .take(max_results)+ .filter(|item| !item.url.trim().is_empty())+ .map(|item| AgentReference {+ title: item.title,+ path: item.url,+ kind: "web".to_string(),+ snippet: Some(item.snippet).filter(|s| !s.trim().is_empty()),+ score: None,+ })+ .collect()+}+fn normalize_web_result(value: Value) -> WebSearchItem {+ let metadata = value.get("metadata");let title = value.get("title")+ .or_else(|| metadata.and_then(|m| m.get("title"))).and_then(Value::as_str).unwrap_or("Untitled").to_string();let url = value.get("url").or_else(|| value.get("link"))- .or_else(|| value.get("source"))+ .or_else(|| metadata.and_then(|m| m.get("sourceURL")))+ .or_else(|| metadata.and_then(|m| m.get("url"))).or_else(|| value.get("original"))+ .or_else(|| value.get("thumbnail")).and_then(Value::as_str).unwrap_or("").to_string();@@ -986,6 +1272,7 @@ fn normalize_web_result(value: Value) -> WebSearchItem {.get("snippet").or_else(|| value.get("content")).or_else(|| value.get("description"))+ .or_else(|| metadata.and_then(|m| m.get("description"))).or_else(|| value.get("summary")).or_else(|| value.get("markdown")).and_then(Value::as_str)@@ -998,6 +1285,31 @@ fn normalize_web_result(value: Value) -> WebSearchItem {}}+fn extract_web_items(value: &Value, keys: &[&str]) -> Vec<Value> {+ for key in keys {+ let Some(candidate) = value.get(*key) else {+ continue;+ };+ if let Some(items) = candidate.as_array() {+ return items.clone();+ }+ if let Some(items) = extract_nested_web_items(candidate) {+ return items;+ }+ }+ Vec::new()+}++fn extract_nested_web_items(value: &Value) -> Option<Vec<Value>> {+ let object = value.as_object()?;+ for key in ["web", "results", "items"] {+ if let Some(items) = object.get(key).and_then(Value::as_array) {+ return Some(items.clone());+ }+ }+ None+}+fn required_api_key<'a>(config: &'a WebSearchConfig, provider: &str) -> Result<&'a str, String> {let key = config.api_key.trim();if key.is_empty() {@@ -1312,9 +1624,60 @@ fn normalize_wiki_write_path(path: &str) -> Result<String, String> {{return Err("wiki.write_page path must stay inside the project".to_string());}+ for segment in rel.split('/') {+ validate_portable_path_segment(segment)?;+ }Ok(rel)}+fn validate_portable_path_segment(segment: &str) -> Result<(), String> {+ if segment.is_empty() {+ return Err("wiki.write_page path contains an empty segment".to_string());+ }+ if segment+ .chars()+ .any(|ch| matches!(ch, '<' | '>' | ':' | '"' | '|' | '?' | '*') || ch <= '\u{1f}')+ {+ return Err(+ "wiki.write_page path contains characters that are invalid on Windows".to_string(),+ );+ }+ let stem = segment+ .split('.')+ .next()+ .unwrap_or(segment)+ .trim_end_matches(' ')+ .to_ascii_uppercase();+ if matches!(+ stem.as_str(),+ "CON"+ | "PRN"+ | "AUX"+ | "NUL"+ | "COM1"+ | "COM2"+ | "COM3"+ | "COM4"+ | "COM5"+ | "COM6"+ | "COM7"+ | "COM8"+ | "COM9"+ | "LPT1"+ | "LPT2"+ | "LPT3"+ | "LPT4"+ | "LPT5"+ | "LPT6"+ | "LPT7"+ | "LPT8"+ | "LPT9"+ ) {+ return Err("wiki.write_page path uses a Windows reserved device name".to_string());+ }+ Ok(())+}+fn extract_markdown_title(content: &str) -> Option<String> {for line in content.lines().take(80) {let trimmed = line.trim();@@ -1460,29 +1823,66 @@ mod tests {let root = std::env::temp_dir().join(format!("llm-wiki-agent-write-{}", Uuid::new_v4()));fs::create_dir_all(root.join("wiki")).unwrap();- assert!(write_wiki_page(root.to_str().unwrap(), "../secret.md", "# Secret").is_err());- assert!(write_wiki_page(root.to_str().unwrap(), "raw/sources/a.md", "# A").is_err());- assert!(write_wiki_page(root.to_str().unwrap(), "wiki/.hidden/a.md", "# A").is_err());- assert!(write_wiki_page(+ assert!(write_wiki_page_with_options(+ root.to_str().unwrap(),+ "../secret.md",+ "# Secret",+ false+ )+ .is_err());+ assert!(write_wiki_page_with_options(+ root.to_str().unwrap(),+ "raw/sources/a.md",+ "# A",+ false+ )+ .is_err());+ assert!(write_wiki_page_with_options(+ root.to_str().unwrap(),+ "wiki/.hidden/a.md",+ "# A",+ false+ )+ .is_err());+ assert!(+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/aux.md", "# A", false)+ .is_err()+ );+ assert!(+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/con.md", "# A", false)+ .is_err()+ );+ assert!(+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/a:b.md", "# A", false)+ .is_err()+ );+ assert!(+ write_wiki_page_with_options(root.to_str().unwrap(), "wiki/a?b.md", "# A", false)+ .is_err()+ );+ assert!(write_wiki_page_with_options(root.to_str().unwrap(),"wiki/queries/huge.md",- &"x".repeat(MAX_WRITE_PAGE_BYTES + 1)+ &"x".repeat(MAX_WRITE_PAGE_BYTES + 1),+ false,).is_err());- let reference = write_wiki_page(+ let reference = write_wiki_page_with_options(root.to_str().unwrap(),"wiki/queries/new-page.md","---\ntitle: New Page\n---\n# New Page\n\nBody",+ false,).unwrap();assert_eq!(reference.title, "New Page");assert_eq!(reference.path, "wiki/queries/new-page.md");assert!(root.join("wiki/queries/new-page.md").exists());- let overwrite_err = write_wiki_page(+ let overwrite_err = write_wiki_page_with_options(root.to_str().unwrap(),"wiki/queries/new-page.md","# Replaced",+ false,).unwrap_err();assert!(overwrite_err.contains("refuses to overwrite"));@@ -1509,10 +1909,11 @@ mod tests {fs::create_dir_all(&outside).unwrap();symlink(&outside, root.join("wiki").join("linked")).unwrap();- let err = write_wiki_page(+ let err = write_wiki_page_with_options(root.to_str().unwrap(),"wiki/linked/newsub/escape.md","# Escape",+ false,).unwrap_err();assert!(err.contains("escapes project directory"));@@ -1605,6 +2006,87 @@ mod tests {assert!(msg.contains("rejected this IP"));}+ #[test]+ fn run_web_search_drops_empty_url_results_before_mapping_references() {+ let refs = web_items_to_references(+ vec![+ WebSearchItem {+ title: "Missing".to_string(),+ url: String::new(),+ snippet: "no url".to_string(),+ },+ WebSearchItem {+ title: "Valid".to_string(),+ url: "https://example.com".to_string(),+ snippet: "ok".to_string(),+ },+ ],+ 10,+ );+ assert_eq!(refs.len(), 1);+ assert_eq!(refs[0].title, "Valid");+ assert_eq!(refs[0].path, "https://example.com");+ }++ #[test]+ fn web_references_apply_limit_before_empty_url_filter_like_legacy_ui() {+ let refs = web_items_to_references(+ vec![+ WebSearchItem {+ title: "Missing".to_string(),+ url: String::new(),+ snippet: "no url".to_string(),+ },+ WebSearchItem {+ title: "Valid".to_string(),+ url: "https://example.com".to_string(),+ snippet: "ok".to_string(),+ },+ ],+ 1,+ );+ assert!(refs.is_empty());+ }++ #[test]+ fn brave_message_without_web_is_treated_as_error() {+ let value = json!({ "message": "invalid subscription token" });+ assert_eq!(+ provider_payload_error("Brave Search", &value),+ Some("Brave Search search failed: invalid subscription token".to_string())+ );+ }++ #[test]+ fn non_brave_message_does_not_mask_valid_provider_payloads() {+ let value = json!({ "message": "FYI" });+ assert_eq!(provider_payload_error("Tavily", &value), None);+ }++ #[test]+ fn web_result_normalization_accepts_firecrawl_nested_metadata() {+ let items = extract_web_items(+ &json!({+ "data": {+ "web": [+ {+ "metadata": {+ "title": "Nested",+ "sourceURL": "https://example.com/nested",+ "description": "from metadata"+ }+ }+ ]+ }+ }),+ &["data", "results"],+ );+ let item = normalize_web_result(items.into_iter().next().unwrap());+ assert_eq!(item.title, "Nested");+ assert_eq!(item.url, "https://example.com/nested");+ assert_eq!(item.snippet, "from metadata");+ }+#[test]fn url_encode_handles_unicode_terms() {assert_eq!(url_encode("煤矿 safety"), "%E7%85%A4%E7%9F%BF+safety");@@ -1650,10 +2132,56 @@ mod tests {}#[test]- fn anytxt_pattern_keeps_multiple_useful_terms() {- assert_eq!(- anytxt_pattern_from_query("请搜索 原始资料 煤矿 安全 治理 的材料"),- "请搜索 煤矿 安全 治理 的材料"- );+ fn extract_anytxt_items_accepts_nested_output_and_field_rows() {+ let value = json!({+ "result": {+ "output": {+ "field": ["fid", "full_path", "title", "hitText"],+ "items": [+ ["42", "/docs/煤矿.pdf", "煤矿资料", "煤矿安全治理片段"]+ ]+ }+ }+ });+ let items = extract_anytxt_items(&value);+ assert_eq!(items.len(), 1);+ assert_eq!(items[0].fid, "42");+ assert_eq!(items[0].path, "/docs/煤矿.pdf");+ assert_eq!(items[0].title, "煤矿资料");+ assert_eq!(items[0].snippet, "煤矿安全治理片段");+ }++ #[test]+ fn extract_anytxt_items_keeps_fid_only_results_addressable() {+ let value = json!({+ "result": {+ "data": {+ "results": [+ { "fid": 99, "snippet": "fragment only" }+ ]+ }+ }+ });+ let items = extract_anytxt_items(&value);+ assert_eq!(items.len(), 1);+ assert_eq!(items[0].path, "anytxt://99");+ assert_eq!(items[0].snippet, "fragment only");+ }++ #[test]+ fn extract_anytxt_items_accepts_value_shapes() {+ let value = json!({+ "result": {+ "output": {+ "value": [+ { "path": "/docs/value.txt", "snippet": "from value" }+ ]+ }+ }+ });+ let items = extract_anytxt_items(&value);+ assert_eq!(items.len(), 1);+ assert_eq!(items[0].path, "/docs/value.txt");+ assert_eq!(items[0].snippet, "from value");}}diff --git a/src-tauri/src/commands/external_search.rs b/src-tauri/src/commands/external_search.rsnew file mode 100644index 00000000..9900e7df--- /dev/null+++ b/src-tauri/src/commands/external_search.rs@@ -0,0 +1,122 @@+use serde::{Deserialize, Serialize};++use crate::agent::tools::{run_anytxt_search, run_web_search, AnyTxtConfig, WebSearchConfig};+use crate::panic_guard::run_guarded_async;++/// Frontend-facing search result shape. The Rust Agent uses+/// `AgentReference` internally, but UI/deep-research code historically+/// consumes `{ title, url, snippet, source }`; keep that wire contract+/// stable while moving provider/network logic to Rust.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ExternalSearchResult {+ pub title: String,+ pub url: String,+ pub snippet: String,+ pub source: String,+}++#[tauri::command]+pub async fn web_search(+ query: String,+ config: WebSearchConfig,+ max_results: Option<usize>,+) -> Result<Vec<ExternalSearchResult>, String> {+ run_guarded_async("web_search", async move {+ let references = run_web_search(&query, Some(config), max_results.unwrap_or(10)).await?;+ Ok(references+ .into_iter()+ .map(|item| ExternalSearchResult {+ title: item.title,+ source: hostname_label(&item.path).unwrap_or_else(|| "web".to_string()),+ url: item.path,+ snippet: item.snippet.unwrap_or_default(),+ })+ .collect())+ })+ .await+}++#[tauri::command]+pub async fn anytxt_search(+ query: String,+ config: AnyTxtConfig,+ max_results: Option<usize>,+) -> Result<Vec<ExternalSearchResult>, String> {+ run_guarded_async("anytxt_search", async move {+ let references = run_anytxt_search(&query, Some(config), max_results.unwrap_or(20)).await?;+ Ok(references+ .into_iter()+ .map(|item| ExternalSearchResult {+ title: item.title,+ url: file_url_for_path(&item.path),+ snippet: item.snippet.unwrap_or_default(),+ source: "AnyTXT".to_string(),+ })+ .collect())+ })+ .await+}++fn hostname_label(url: &str) -> Option<String> {+ let host = reqwest::Url::parse(url).ok()?.host_str()?.to_string();+ Some(host.strip_prefix("www.").unwrap_or(&host).to_string())+}++pub(crate) fn file_url_for_path(path: &str) -> String {+ let normalized = path.replace('\\', "/");+ if normalized.is_empty() || normalized.contains("://") {+ return normalized;+ }+ if normalized.starts_with("//") {+ return format!("file:{normalized}");+ }+ if normalized.len() >= 3+ && normalized.as_bytes()[1] == b':'+ && normalized.as_bytes()[2] == b'/'+ && normalized.as_bytes()[0].is_ascii_alphabetic()+ {+ return format!("file:///{}", encode_file_url_path(&normalized));+ }+ if normalized.starts_with('/') {+ return format!("file://{}", encode_file_url_path(&normalized));+ }+ normalized+}++fn encode_file_url_path(path: &str) -> String {+ path.split('/')+ .map(percent_encode_file_segment)+ .collect::<Vec<_>>()+ .join("/")+}++fn percent_encode_file_segment(segment: &str) -> String {+ let mut out = String::new();+ for byte in segment.as_bytes() {+ if byte.is_ascii_alphanumeric() || matches!(*byte, b'-' | b'.' | b'_' | b'~' | b':') {+ out.push(*byte as char);+ } else {+ out.push_str(&format!("%{byte:02X}"));+ }+ }+ out+}++#[cfg(test)]+mod tests {+ use super::file_url_for_path;++ #[test]+ fn anytxt_paths_are_returned_as_file_urls_for_frontend_results() {+ assert_eq!(+ file_url_for_path(r"C:\docs\煤矿 安全.pdf"),+ "file:///C:/docs/%E7%85%A4%E7%9F%BF%20%E5%AE%89%E5%85%A8.pdf"+ );+ assert_eq!(+ file_url_for_path("/Users/me/docs/a b.txt"),+ "file:///Users/me/docs/a%20b.txt"+ );+ assert_eq!(file_url_for_path("anytxt://99"), "anytxt://99");+ }+}diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rsindex 911dc610..e13992d0 100644--- a/src-tauri/src/commands/mod.rs+++ b/src-tauri/src/commands/mod.rs@@ -1,6 +1,7 @@pub mod claude_cli;mod cli_resolver;pub mod codex_cli;+pub mod external_search;pub mod extract_images;pub mod file_sync;pub mod fs;diff --git a/src-tauri/src/commands/search.rs b/src-tauri/src/commands/search.rsindex b8e487a7..eb49c7fc 100644--- a/src-tauri/src/commands/search.rs+++ b/src-tauri/src/commands/search.rs@@ -60,7 +60,7 @@ pub struct SearchEmbeddingConfig {pub endpoint: String,pub api_key: String,pub model: String,- pub output_dimensionality: Option<u32>,+ pub output_dimensionality: Option<f64>,/// Extra HTTP headers to send with every embedding request, e.g./// `X-Model-Provider-Id: siliconflow` for the mify gateway./// Reserved names (Authorization, Content-Type, Host,@@ -94,6 +94,18 @@ pub async fn search_project(.await}+#[tauri::command]+pub async fn embedding_fetch(+ text: String,+ cfg: SearchEmbeddingConfig,+ max_retries: Option<usize>,+) -> Result<Vec<f32>, String> {+ run_guarded_async("embedding_fetch", async move {+ fetch_embedding_with_retry(&text, &cfg, max_retries.unwrap_or(3)).await+ })+ .await+}+pub async fn resolve_query_embedding(query: &str,explicit_embedding: Option<Vec<f32>>,@@ -108,7 +120,7 @@ pub async fn resolve_query_embedding(if !cfg.enabled || cfg.endpoint.trim().is_empty() || cfg.model.trim().is_empty() {return Ok(None);}- match fetch_embedding(query, &cfg).await {+ match fetch_embedding_with_retry(query, &cfg, 0).await {Ok(embedding) => validate_query_embedding(embedding).map(Some),Err(err) => {eprintln!("[Search] embedding disabled for this request: {err}");@@ -666,7 +678,59 @@ pub fn extract_image_refs(content: &str) -> Vec<SearchImageRef> {out}-async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<f32>, String> {+async fn fetch_embedding_with_retry(+ text: &str,+ cfg: &SearchEmbeddingConfig,+ max_retries: usize,+) -> Result<Vec<f32>, String> {+ let mut current = text.to_string();+ let mut attempts = 0usize;+ loop {+ attempts += 1;+ match fetch_embedding_once(¤t, cfg).await {+ Ok(embedding) => return Ok(embedding),+ Err(EmbeddingFetchError::Oversize(message)) => {+ if attempts <= max_retries+ && current.len() > 64+ && halve_text_on_char_boundary(&mut current)+ {+ eprintln!(+ "[Embedding] auto-halving after oversize error at {} chars; retrying at {} chars ({attempts}/{})",+ text.chars().count(),+ current.chars().count(),+ max_retries + 1+ );+ continue;+ }+ return Err(format!(+ "Endpoint rejected input even at {} chars. Lower Settings -> Embedding -> Max Chunk Chars. {message}",+ current.len()+ ));+ }+ Err(EmbeddingFetchError::Other(message)) => return Err(message),+ }+ }+}++fn halve_text_on_char_boundary(text: &mut String) -> bool {+ let char_count = text.chars().count();+ if char_count <= 1 {+ return false;+ }+ let keep = (char_count / 2).max(1);+ *text = text.chars().take(keep).collect();+ true+}++enum EmbeddingFetchError {+ Oversize(String),+ Other(String),+}++async fn fetch_embedding_once(+ text: &str,+ cfg: &SearchEmbeddingConfig,+) -> Result<Vec<f32>, EmbeddingFetchError> {let is_google = is_google_embedding_config(cfg);let is_doubao_multimodal = is_doubao_multimodal_embedding_config(cfg);let endpoint = if is_google {@@ -679,9 +743,15 @@ async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<SEARCH_EMBEDDING_TIMEOUT_SECS,)).build()- .map_err(|e| format!("Embedding HTTP client error: {e}"))?- .post(endpoint)+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding HTTP client error: {e}")))?+ .post(&endpoint).header("Content-Type", "application/json");+ // Browser-based local model servers often require a browser-like+ // Origin even when the request is routed through Rust. Keep this+ // reserved so user-supplied extra headers cannot override it.+ if is_local_or_private_http_endpoint(&endpoint) {+ req = req.header("Origin", "http://localhost");+ }if !cfg.api_key.trim().is_empty() {if is_google {req = req.header("x-goog-api-key", cfg.api_key.trim());@@ -713,19 +783,46 @@ async fn fetch_embedding(text: &str, cfg: &SearchEmbeddingConfig) -> Result<Vec<.json(&body).send().await- .map_err(|e| format!("Embedding request failed: {e}"))?;+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding request failed: {e}")))?;let status = resp.status();- let data: Value = resp- .json()+ let text = resp+ .text().await- .map_err(|e| format!("Embedding response parse failed: {e}"))?;+ .map_err(|e| EmbeddingFetchError::Other(format!("Embedding response read failed: {e}")))?;if !status.is_success() {- return Err(format!(- "Embedding API HTTP {status}: {}",- data.to_string().chars().take(200).collect::<String>()- ));+ let preview = text.chars().take(200).collect::<String>();+ if looks_like_oversize_error(status.as_u16(), &text) {+ return Err(EmbeddingFetchError::Oversize(format!(+ "Embedding API HTTP {status}: {preview}"+ )));+ }+ return Err(EmbeddingFetchError::Other(format!(+ "Embedding API HTTP {status}: {preview}"+ )));}+ let data: Value = serde_json::from_str(&text).map_err(|e| {+ EmbeddingFetchError::Other(format!(+ "Embedding response parse failed: {e}: {}",+ text.chars().take(200).collect::<String>()+ ))+ })?;parse_embedding_values(&data, is_google, is_doubao_multimodal)+ .map_err(EmbeddingFetchError::Other)+}++fn looks_like_oversize_error(status: u16, body: &str) -> bool {+ if status == 413 {+ return true;+ }+ let lower = body.to_lowercase();+ lower.contains("too long")+ || lower.contains("maximum context")+ || lower.contains("max_tokens")+ || lower.contains("max tokens")+ || lower.contains("context length")+ || lower.contains("token limit")+ || lower.contains("exceeds")+ || lower.contains("input length")}fn parse_embedding_values(@@ -794,7 +891,7 @@ fn is_safe_extra_header_name(name: &str) -> bool {fn is_reserved_extra_header_name(name: &str) -> bool {matches!(name.trim().to_ascii_lowercase().as_str(),- "authorization" | "content-type" | "host" | "content-length" | "x-goog-api-key"+ "authorization" | "content-type" | "host" | "content-length" | "origin" | "x-goog-api-key")}@@ -803,6 +900,39 @@ fn is_google_embedding_config(cfg: &SearchEmbeddingConfig) -> bool {endpoint.contains("generativelanguage.googleapis.com") || endpoint.contains(":embedcontent")}+fn is_local_or_private_http_endpoint(endpoint: &str) -> bool {+ let Ok(url) = reqwest::Url::parse(endpoint) else {+ return false;+ };+ if !matches!(url.scheme(), "http" | "https") {+ return false;+ }+ let Some(host) = url.host_str() else {+ return false;+ };+ let host = host+ .trim_matches('[')+ .trim_matches(']')+ .to_ascii_lowercase();+ if matches!(host.as_str(), "localhost" | "127.0.0.1" | "::1") {+ return true;+ }+ let octets = host+ .split('.')+ .map(str::parse::<u8>)+ .collect::<Result<Vec<_>, _>>();+ let Ok(octets) = octets else {+ return false;+ };+ if octets.len() != 4 {+ return false;+ }+ octets[0] == 10+ || (octets[0] == 172 && (16..=31).contains(&octets[1]))+ || (octets[0] == 192 && octets[1] == 168)+ || octets[0] == 127+}+fn is_volcengine_embedding_endpoint(endpoint: &str) -> bool {let host = reqwest::Url::parse(endpoint).ok()@@ -942,7 +1072,7 @@ fn strip_google_api_key_query(endpoint: &str) -> String {}}-fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<u32>) -> Value {+fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<f64>) -> Value {let model_path = if model.trim().starts_with("models/") {model.trim().to_string()} else {@@ -952,7 +1082,10 @@ fn google_embedding_body(model: &str, text: &str, output_dimensionality: Option<"model": model_path,"content": { "parts": [{ "text": text }] },});- if let Some(dim) = output_dimensionality.filter(|dim| *dim > 0) {+ if let Some(dim) = output_dimensionality+ .filter(|dim| dim.is_finite() && *dim >= 1.0)+ .map(|dim| dim.floor() as u32)+ {body["output_dimensionality"] = json!(dim);}body@@ -1100,7 +1233,7 @@ mod tests {endpoint: "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents?key=URL_KEY&alt=json".to_string(),api_key: "HEADER_KEY".to_string(),model: "gemini-embedding-001".to_string(),- output_dimensionality: Some(768),+ output_dimensionality: Some(768.0),extra_headers: None,};@@ -1110,7 +1243,7 @@ mod tests {assert!(!endpoint.contains("URL_KEY"));assert!(endpoint.contains("alt=json"));- let body = google_embedding_body("gemini-embedding-001", "hello", Some(768));+ let body = google_embedding_body("gemini-embedding-001", "hello", Some(768.0));assert_eq!(body["model"], "models/gemini-embedding-001");assert_eq!(body["output_dimensionality"], 768);}@@ -1244,10 +1377,46 @@ mod tests {assert!(is_reserved_extra_header_name("Authorization"));assert!(is_reserved_extra_header_name("content-type"));+ assert!(is_reserved_extra_header_name("Origin"));assert!(is_reserved_extra_header_name("X-Goog-Api-Key"));assert!(!is_reserved_extra_header_name("X-Model-Provider-Id"));}+ #[test]+ fn embedding_origin_header_is_limited_to_local_or_private_endpoints() {+ assert!(is_local_or_private_http_endpoint(+ "http://127.0.0.1:1234/v1/embeddings"+ ));+ assert!(is_local_or_private_http_endpoint(+ "http://192.168.1.20:11434/v1/embeddings"+ ));+ assert!(is_local_or_private_http_endpoint(+ "http://172.16.0.5/v1/embeddings"+ ));+ assert!(!is_local_or_private_http_endpoint(+ "https://api.openai.com/v1/embeddings"+ ));+ }++ #[test]+ fn embedding_halving_never_splits_cjk_codepoints() {+ let mut text = "默会知识库".to_string();+ assert!(halve_text_on_char_boundary(&mut text));+ assert_eq!(text, "默会");+ }++ #[test]+ fn google_embedding_body_floors_positive_dimensions_and_omits_invalid_values() {+ let body = google_embedding_body("gemini-embedding-001", "hello", Some(1.9));+ assert_eq!(body["output_dimensionality"], 1);++ let zero = google_embedding_body("gemini-embedding-001", "hello", Some(0.0));+ assert!(zero.get("output_dimensionality").is_none());++ let negative = google_embedding_body("gemini-embedding-001", "hello", Some(-4.0));+ assert!(negative.get("output_dimensionality").is_none());+ }+#[test]fn rrf_combines_token_and_vector_ranks_and_keeps_vector_score() {let mut results = vec![diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex 8073ab20..e329fc90 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -636,6 +636,9 @@ pub fn run() {commands::project::open_project,commands::project::open_project_folder,commands::search::search_project,+ commands::search::embedding_fetch,+ commands::external_search::web_search,+ commands::external_search::anytxt_search,clip_server_status,api_server_status,api_server_reload_config,diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsxindex aab5cec1..c39ecaf2 100644--- a/src/components/chat/chat-panel.tsx+++ b/src/components/chat/chat-panel.tsx@@ -219,6 +219,8 @@ function normalizeBackendToolName(tool: string) {const normalized = tool.split(".").join("_")if (normalized === "wiki_search") return "wiki_search" as constif (normalized === "wiki_read_page") return "project_file_read" as const+ if (normalized === "wiki_write_page") return "project_files" as const+ if (normalized === "skills_load") return "project_file_read" as constif (normalized === "source_search") return "project_file_read" as constif (normalized === "graph_search") return "graph_search" as constif (normalized === "web_search") return "web_search" as constdiff --git a/src/lib/anytxt-search.test.ts b/src/lib/anytxt-search.test.tsindex 59d81796..30e0ec7f 100644--- a/src/lib/anytxt-search.test.ts+++ b/src/lib/anytxt-search.test.ts@@ -8,14 +8,17 @@ import {prepareAnyTxtQueries,} from "./anytxt-search"+const invokeMock = vi.hoisted(() => vi.fn())const streamChatMock = vi.hoisted(() => vi.fn())+vi.mock("@tauri-apps/api/core", () => ({+ invoke: invokeMock,+}))+vi.mock("@/lib/llm-client", () => ({streamChat: streamChatMock,}))-const fetchMock = vi.fn<typeof fetch>()-const llmConfig: LlmConfig = {provider: "custom",apiKey: "test",@@ -25,47 +28,16 @@ const llmConfig: LlmConfig = {maxContextSize: 128000,}-function jsonResponse(body: unknown, init?: ResponseInit): Response {- return new Response(JSON.stringify(body), {- status: 200,- headers: { "Content-Type": "application/json" },- ...init,- })-}-describe("anyTxtSearch", () => {beforeEach(() => {- fetchMock.mockReset()+ invokeMock.mockReset()streamChatMock.mockReset()- vi.stubGlobal("fetch", fetchMock)})- it("calls AnyTXT GetResult and fetches fragments for matching files", async () => {- fetchMock- .mockResolvedValueOnce(jsonResponse({- jsonrpc: "2.0",- id: 1,- result: {- output: {- items: [- {- fid: "9223736511748752040",- path: "C:\\docs\\alpha.pdf",- name: "alpha.pdf",- },- ],- },- },- }))- .mockResolvedValueOnce(jsonResponse({- jsonrpc: "2.0",- id: 2,- result: {- output: {- text: "matching fragment text",- },- },- }))+ it("delegates AnyTXT JSON-RPC search to the Rust backend command", async () => {+ invokeMock.mockResolvedValueOnce([+ { title: "alpha.pdf", url: "/docs/alpha.pdf", snippet: "matching fragment text", source: "AnyTXT" },+ ])const out = await anyTxtSearch("alpha",@@ -78,219 +50,56 @@ describe("anyTxtSearch", () => {5,)- expect(fetchMock).toHaveBeenCalledTimes(2)- const [url, init] = fetchMock.mock.calls[0]- expect(url).toBe("http://127.0.0.1:9920")- expect(JSON.parse(String(init?.body))).toMatchObject({- jsonrpc: "2.0",- method: "ATRpcServer.Searcher.V1.GetResult",- params: {- input: {- pattern: "alpha",- filterDir: "C:\\docs",- filterExt: "*.pdf",- limit: "5",- },- },- })- expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toMatchObject({- method: "ATRpcServer.Searcher.V1.GetFragment",- params: { input: { fid: "9223736511748752040", pattern: "alpha" } },- })- expect(out).toEqual([- {- title: "alpha.pdf",- url: "file:///C:/docs/alpha.pdf",- snippet: "matching fragment text",- source: "AnyTXT",- },- ])- })-- it("normalizes tolerant AnyTXT response shapes without fragments", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- result: {- output: [- {- full_path: "/Users/me/docs/note.md",- hitText: "local note match",- },- ],+ expect(invokeMock).toHaveBeenCalledWith("anytxt_search", {+ query: "alpha",+ maxResults: 5,+ config: {+ enabled: true,+ endpoint: "http://127.0.0.1:9920",+ filterDir: "C:\\docs",+ filterExt: "*.pdf",+ limit: 10,},- }))-- const out = await anyTxtSearch("note", { endpoint: "http://127.0.0.1:9920" }, 3, "/project")-- expect(fetchMock).toHaveBeenCalledTimes(1)- expect(out[0]).toMatchObject({- title: "note.md",- url: "file:///Users/me/docs/note.md",- snippet: "local note match",- source: "AnyTXT",})- })-- it("normalizes AnyTXT data.output.files rows using the returned field list", async () => {- fetchMock- .mockResolvedValueOnce(jsonResponse({- id: 1,- jsonrpc: "2.0",- result: {- data: {- output: {- count: 1,- field: ["fid", "lastModify", "size", "file"],- files: [["9223736511748752040", 1711588801, 1200, "D:\\docs\\煤矿安全.md"]],- },- },- errno: 0,- },- }))- .mockResolvedValueOnce(jsonResponse({- result: {- data: {- output: {- text: "煤矿安全 fragment",- },- },- },- }))-- const out = await anyTxtSearch("煤矿", { endpoint: "http://127.0.0.1:9920" }, 5)-expect(out).toEqual([- {- title: "煤矿安全.md",- url: "file:///D:/docs/%E7%85%A4%E7%9F%BF%E5%AE%89%E5%85%A8.md",- snippet: "煤矿安全 fragment",- source: "AnyTXT",- },+ { title: "alpha.pdf", url: "/docs/alpha.pdf", snippet: "matching fragment text", source: "AnyTXT" },])})- it("normalizes UNC network paths to file URLs", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- result: {- data: {- output: {- field: ["fid", "file"],- files: [["unc-1", "\\\\nas\\share\\研究报告.md"]],- },- },- },- }))-- const out = await anyTxtSearch("研究", { endpoint: "http://127.0.0.1:9920" }, 5)-- expect(out[0]).toMatchObject({- title: "研究报告.md",- url: "file://nas/share/%E7%A0%94%E7%A9%B6%E6%8A%A5%E5%91%8A.md",- source: "AnyTXT",- })- })-it("uses the Unix root for blank filterDir on macOS/Linux projects", () => {- expect(normalizeAnyTxtConfig({ filterDir: "" }, "/Users/me/wiki").filterDir)+ expect(normalizeAnyTxtConfig({ endpoint: "127.0.0.1:9920", filterDir: "" }, "/Users/me/wiki").filterDir).toBe("/")})it("leaves blank filterDir unset for Windows projects", () => {- expect(normalizeAnyTxtConfig({ filterDir: "" }, "C:/Users/me/wiki").filterDir)+ expect(normalizeAnyTxtConfig({ endpoint: "127.0.0.1:9920", filterDir: "" }, "C:/Users/me/wiki").filterDir).toBe("")})- it("sends the Unix root as filterDir when folder filtering is blank on macOS/Linux", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ result: { items: [] } }))-- await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920", filterDir: "" }, 5, "/Users/me/wiki")-- const body = JSON.parse(String(fetchMock.mock.calls[0][1]?.body))- expect(body.params.input.filterDir).toBe("/")- })-it("normalizes endpoints and clamps limits", () => {- expect(normalizeAnyTxtConfig({ endpoint: "localhost:9920" }).endpoint).toBe("http://localhost:9920")- expect(normalizeAnyTxtConfig({ endpoint: "https://anytxt.local/" }).endpoint).toBe("https://anytxt.local/")- expect(normalizeAnyTxtConfig({ limit: -1 }).limit).toBe(1)- expect(normalizeAnyTxtConfig({ limit: 1000 }).limit).toBe(100)- })-- it("returns no results for blank queries without calling AnyTXT", async () => {- const out = await anyTxtSearch(" ", { endpoint: "http://127.0.0.1:9920" })-- expect(fetchMock).not.toHaveBeenCalled()- expect(out).toEqual([])- })-- it("returns no results when AnyTXT is disabled", async () => {- const out = await anyTxtSearch("alpha", {- enabled: false,- endpoint: "http://127.0.0.1:9920",+ expect(normalizeAnyTxtConfig({ endpoint: "localhost:9920", limit: 1000 })).toMatchObject({+ endpoint: "http://localhost:9920",+ limit: 100,+ })+ expect(normalizeAnyTxtConfig({ endpoint: "https://host/api", limit: -1 })).toMatchObject({+ endpoint: "https://host/api",+ limit: 1,})-- expect(fetchMock).not.toHaveBeenCalled()- expect(out).toEqual([])- })-- it("surfaces JSON-RPC errors", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- jsonrpc: "2.0",- id: 1,- error: { message: "bad pattern" },- }))-- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))- .rejects.toThrow("AnyTXT error: bad pattern")- })-- it("surfaces HTTP failures with status codes", async () => {- fetchMock.mockResolvedValueOnce(new Response("oops", { status: 500 }))-- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))- .rejects.toThrow("AnyTXT request failed (500): oops")- })-- it("falls back to item snippets when fragment lookup fails", async () => {- fetchMock- .mockResolvedValueOnce(jsonResponse({- result: {- items: [- { fid: "1", path: "/tmp/a.md", snippet: "fallback snippet" },- ],- },- }))- .mockResolvedValueOnce(new Response("fragment failed", { status: 500 }))-- const out = await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" })-- expect(out[0].snippet).toBe("fallback snippet")- })-- it("filters blank AnyTXT result items", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- result: {- items: [- {},- { path: "/tmp/valid.md", snippet: "valid" },- ],- },- }))-- const out = await anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" })-- expect(out).toHaveLength(1)- expect(out[0].url).toBe("file:///tmp/valid.md")})- it("turns fetch network failures into AnyTXT connection guidance", async () => {- fetchMock.mockRejectedValueOnce(new TypeError("Failed to fetch"))-- await expect(anyTxtSearch("alpha", { endpoint: "http://127.0.0.1:9920" }))- .rejects.toThrow("Check that ATGUI.exe is running")+ it("returns no results for blank queries or disabled AnyTXT without invoking Rust", async () => {+ expect(await anyTxtSearch(" ", { endpoint: "127.0.0.1:9920" })).toEqual([])+ expect(await anyTxtSearch("alpha", { enabled: false, endpoint: "127.0.0.1:9920" })).toEqual([])+ expect(invokeMock).not.toHaveBeenCalled()})})describe("AnyTXT query rewrite", () => {+ beforeEach(() => {+ invokeMock.mockReset()+ streamChatMock.mockReset()+ })+it("parses JSON-array query rewrites and deduplicates them", () => {expect(parseAnyTxtQueryRewrite('```json\n["MBR ammonia", "winter nitrification", "MBR ammonia"]\n```')).toEqual(["MBR ammonia", "winter nitrification"])@@ -302,10 +111,9 @@ describe("AnyTXT query rewrite", () => {})it("prefers rewritten AnyTXT queries when original queries would fill the cap", async () => {- streamChatMock.mockImplementationOnce(async (...args: unknown[]) => {- const handlers = args[2] as { onToken?: (token: string) => void; onDone?: () => void }- handlers.onToken?.('["kw1", "kw2", "kw3"]')- handlers.onDone?.()+ streamChatMock.mockImplementationOnce(async (_config, _messages, callbacks) => {+ callbacks.onToken?.('["kw1", "kw2", "kw3"]')+ callbacks.onDone?.()})const queries = await prepareAnyTxtQueries(@@ -325,32 +133,23 @@ describe("AnyTXT query rewrite", () => {})it("searches rewritten queries through the smart AnyTXT entry point", async () => {- fetchMock.mockReset()- vi.stubGlobal("fetch", fetchMock)- streamChatMock.mockImplementationOnce(async (...args: unknown[]) => {- const handlers = args[2] as { onToken?: (token: string) => void; onDone?: () => void }- handlers.onToken?.('["煤矿 安全"]')- handlers.onDone?.()+ streamChatMock.mockImplementationOnce(async (_config, _messages, callbacks) => {+ callbacks.onToken?.('["煤矿 安全"]')+ callbacks.onDone?.()})- fetchMock.mockResolvedValueOnce(jsonResponse({- result: {- items: [- { path: "/docs/coal.md", snippet: "煤矿安全 local match" },- ],- },- }))+ invokeMock.mockResolvedValueOnce([+ { title: "mine.pdf", url: "/docs/mine.pdf", snippet: "煤矿安全", source: "AnyTXT" },+ ])+ invokeMock.mockResolvedValueOnce([])const out = await anyTxtSearchSmart("请帮我找一下煤矿安全相关资料", {+ enabled: true,endpoint: "http://127.0.0.1:9920",- }, llmConfig, 1, "/Users/me/wiki")+ }, llmConfig, 5)- expect(fetchMock).toHaveBeenCalled()- expect(JSON.parse(String(fetchMock.mock.calls[0][1]?.body))).toMatchObject({- params: { input: { pattern: "煤矿 安全" } },- })- expect(out[0]).toMatchObject({- title: "coal.md",- source: "AnyTXT",- })+ expect(invokeMock.mock.calls[0]).toEqual(["anytxt_search", expect.objectContaining({+ query: "煤矿 安全",+ })])+ expect(out).toHaveLength(1)})})diff --git a/src/lib/anytxt-search.ts b/src/lib/anytxt-search.tsindex 6a8c0d56..e6e97604 100644--- a/src/lib/anytxt-search.ts+++ b/src/lib/anytxt-search.ts@@ -1,5 +1,5 @@+import { invoke } from "@tauri-apps/api/core"import type { AnyTxtConfig, LlmConfig } from "@/stores/wiki-store"-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"import { normalizePath } from "@/lib/path-utils"import { streamChat } from "@/lib/llm-client"import type { WebSearchResult } from "./web-search"@@ -7,21 +7,7 @@ import type { WebSearchResult } from "./web-search"export const DEFAULT_ANYTXT_ENDPOINT = "http://127.0.0.1:9920"export const DEFAULT_ANYTXT_FILTER_EXT = "*"export const DEFAULT_ANYTXT_LIMIT = 20-const ANYTXT_LAST_MODIFY_END = 2147483647 // AnyTXT docs use signed 32-bit Unix timestamps.const ANYTXT_QUERY_LIMIT = 3-let nextRpcId = 1--interface AnyTxtRpcResponse {- result?: unknown- error?: string | { message?: string; code?: number }-}--interface AnyTxtItem {- fid: string- path: string- title: string- snippet: string-}export function normalizeAnyTxtConfig(config?: AnyTxtConfig, _projectPath?: string): Required<AnyTxtConfig> {return {@@ -47,34 +33,11 @@ export async function anyTxtSearch(if (!query.trim()) return []const resolved = normalizeAnyTxtConfig(config, projectPath)if (!resolved.enabled) return []- const limit = Math.min(clampAnyTxtLimit(maxResults), resolved.limit)- const response = await callAnyTxtRpc(resolved.endpoint, "ATRpcServer.Searcher.V1.GetResult", buildAnyTxtSearchInput({- pattern: query,- filterDir: resolved.filterDir,- filterExt: resolved.filterExt,- lastModifyBegin: 0,- lastModifyEnd: ANYTXT_LAST_MODIFY_END,- // AnyTXT's documented GetResult shape expects limit as a string while- // offset/order/timestamps are numbers.- limit: String(limit),- offset: 0,- order: 0,- }))-- const items = extractAnyTxtItems(response).slice(0, limit)- const out: WebSearchResult[] = []- for (const item of items) {- const fragment = item.fid- ? await getAnyTxtFragment(resolved.endpoint, item.fid, query).catch(() => "")- : ""- out.push({- title: item.title,- url: fileUrlForPath(item.path) || (item.fid ? `anytxt://${item.fid}` : ""),- snippet: trimFragment(fragment || item.snippet || item.path),- source: "AnyTXT",- })- }- return out.filter((item) => item.url || item.snippet)+ return invoke<WebSearchResult[]>("anytxt_search", {+ query,+ config: resolved,+ maxResults: Math.min(clampAnyTxtLimit(maxResults), resolved.limit),+ })}export async function anyTxtSearchSmart(@@ -194,179 +157,6 @@ export function uniqueAnyTxtQueries(queries: string[], limit: number = ANYTXT_QUreturn out}-async function getAnyTxtFragment(endpoint: string, fid: string, pattern: string): Promise<string> {- const response = await callAnyTxtRpc(endpoint, "ATRpcServer.Searcher.V1.GetFragment", {- fid,- pattern,- })- return extractFragmentText(response)-}--async function callAnyTxtRpc(endpoint: string, method: string, input: Record<string, unknown>): Promise<unknown> {- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch(endpoint, {- method: "POST",- headers: {- Accept: "application/json",- "Content-Type": "application/json",- },- body: JSON.stringify({- id: nextRpcId++,- jsonrpc: "2.0",- method,- params: { input },- }),- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error("Network error reaching AnyTXT. Check that ATGUI.exe is running and the JSON-RPC API is listening.")- }- throw err- }-- if (!response.ok) {- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`AnyTXT request failed (${response.status}): ${errorText}`)- }-- const data = (await response.json()) as AnyTxtRpcResponse- if (data.error) {- const message = typeof data.error === "string" ? data.error : data.error.message ?? JSON.stringify(data.error)- throw new Error(`AnyTXT error: ${message}`)- }- return data.result-}--function extractAnyTxtItems(result: unknown): AnyTxtItem[] {- const rawItems = firstArray(- result,- getPath(result, ["output"]),- getPath(result, ["output", "items"]),- getPath(result, ["output", "results"]),- getPath(result, ["output", "list"]),- getPath(result, ["output", "files"]),- getPath(result, ["output", "data"]),- getPath(result, ["output", "value"]),- getPath(result, ["data", "output", "files"]),- getPath(result, ["data", "output", "items"]),- getPath(result, ["data", "output", "results"]),- getPath(result, ["data", "output", "list"]),- getPath(result, ["data", "files"]),- getPath(result, ["data", "items"]),- getPath(result, ["data", "results"]),- getPath(result, ["data", "list"]),- getPath(result, ["items"]),- getPath(result, ["results"]),- getPath(result, ["list"]),- getPath(result, ["files"]),- getPath(result, ["data"]),- getPath(result, ["value"]),- )- const fields = firstStringArray(- getPath(result, ["field"]),- getPath(result, ["output", "field"]),- getPath(result, ["data", "output", "field"]),- getPath(result, ["data", "field"]),- )-- return rawItems.map((item) => normalizeAnyTxtItem(item, fields)).filter((item): item is AnyTxtItem => item !== null)-}--function normalizeAnyTxtItem(item: unknown, fields: string[] = []): AnyTxtItem | null {- if (!item || (typeof item !== "object" && !Array.isArray(item))) return null- const record = Array.isArray(item) ? recordFromFields(fields, item) : item as Record<string, unknown>- const fid = stringField(record, "fid", "id", "fileId", "file_id")- const path = normalizePath(stringField(record, "path", "file", "filePath", "file_path", "fullPath", "full_path", "filename", "fileName", "name"))- const title = stringField(record, "title", "name", "fileName", "filename") || basename(path) || fid || "AnyTXT result"- const snippet = stringField(record, "snippet", "fragment", "content", "contents", "text", "summary", "highlight", "hitText")- if (!fid && !path && !snippet) return null- return { fid, path, title, snippet }-}--function extractFragmentText(result: unknown): string {- if (typeof result === "string") return result- if (Array.isArray(result)) return result.map(extractFragmentText).filter(Boolean).join("\n\n")- if (!result || typeof result !== "object") return ""- const record = result as Record<string, unknown>- return stringField(record, "text", "fragment", "content", "snippet", "html")- || extractFragmentText(record.output)- || extractFragmentText(record.result)- || extractFragmentText(record.data)- || extractFragmentText(getPath(record, ["data", "output"]))- || extractFragmentText(record.fragments)- || extractFragmentText(record.items)- || extractFragmentText(record.list)-}--function buildAnyTxtSearchInput(input: {- pattern: string- filterDir: string- filterExt: string- lastModifyBegin: number- lastModifyEnd: number- limit: string- offset: number- order: number-}): Record<string, unknown> {- const out: Record<string, unknown> = {- pattern: input.pattern,- filterExt: input.filterExt,- lastModifyBegin: input.lastModifyBegin,- lastModifyEnd: input.lastModifyEnd,- limit: input.limit,- offset: input.offset,- order: input.order,- }- if (input.filterDir.trim()) {- out.filterDir = input.filterDir- }- return out-}--function firstArray(...values: unknown[]): unknown[] {- for (const value of values) {- if (Array.isArray(value)) return value- }- return []-}--function firstStringArray(...values: unknown[]): string[] {- for (const value of values) {- if (Array.isArray(value) && value.every((item) => typeof item === "string")) {- return value- }- }- return []-}--function recordFromFields(fields: string[], row: unknown[]): Record<string, unknown> {- const record: Record<string, unknown> = {}- fields.forEach((field, index) => {- record[field] = row[index]- })- return record-}--function getPath(value: unknown, path: string[]): unknown {- let current = value- for (const key of path) {- if (!current || typeof current !== "object") return undefined- current = (current as Record<string, unknown>)[key]- }- return current-}--function stringField(record: Record<string, unknown>, ...keys: string[]): string {- for (const key of keys) {- const value = record[key]- if (typeof value === "string" && value.trim()) return value.trim()- if (typeof value === "number" && Number.isFinite(value)) return String(value)- }- return ""-}-function normalizeAnyTxtEndpoint(endpoint?: string): string {const trimmed = endpoint?.trim() || DEFAULT_ANYTXT_ENDPOINTif (/^https?:\/\//i.test(trimmed)) return trimmed@@ -381,27 +171,8 @@ function clampAnyTxtLimit(limit: unknown): number {function defaultAnyTxtFilterDir(projectPath?: string): string {const pp = projectPath ? normalizePath(projectPath) : ""- // On the macOS/Linux AnyTXT API, omitting filterDir has been observed to- // fall back to "C:" instead of "all indexed files". Use "/" as the explicit- // unrestricted root for Unix-style projects. Keep Windows blank because- // AnyTXT cannot represent "all drive letters" with one filterDir.+ // AnyTXT treats omitted filterDir differently across platforms. For+ // Unix-like absolute paths, "/" means "all indexed files"; on Windows+ // there is no single all-drive root, so keep the field empty.return pp.startsWith("/") ? "/" : ""}--function basename(path: string): string {- return path.split(/[\\/]/).filter(Boolean).pop() ?? ""-}--function fileUrlForPath(path: string): string {- if (!path) return ""- const normalized = normalizePath(path)- if (/^[a-z]+:\/\//i.test(normalized)) return normalized- if (normalized.startsWith("//")) return `file:${encodeURI(normalized)}`- if (/^[A-Za-z]:\//.test(normalized)) return `file:///${encodeURI(normalized)}`- if (normalized.startsWith("/")) return `file://${encodeURI(normalized)}`- return normalized-}--function trimFragment(text: string): string {- return text.replace(/\s+/g, " ").trim().slice(0, 1200)-}diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.tsindex 9dc22adc..3072e61c 100644--- a/src/lib/embedding.test.ts+++ b/src/lib/embedding.test.ts@@ -3,30 +3,23 @@* pieces: auto-halve retry heuristics and the chunk→page aggregation* contract inside `searchByEmbedding`.*- * The actual HTTP layer and Tauri LanceDB commands are mocked — we're- * NOT testing Rust vectorstore here (that has its own 15 Rust tests)- * nor the webview fetch (that's tauri-fetch.ts). The boundary we pin- * down here is "given these chunk-level results, do we aggregate to- * page-level scores the way the design says?".+ * The actual Rust HTTP/vectorstore commands are mocked here. Rust owns+ * the provider transport; this file pins the TypeScript orchestration+ * contract around chunking, invoke payloads, and page-level aggregation.*/import { describe, it, expect, vi, beforeEach } from "vitest"// Module-level mock for the Tauri invoke boundary so we can script the// chunk-search response without touching real LanceDB.const mockInvoke = vi.fn<(cmd: string, args?: unknown) => Promise<unknown>>()+const mockEmbeddingFetchInvoke = vi.fn<(args?: unknown) => Promise<unknown>>()vi.mock("@tauri-apps/api/core", () => ({- invoke: (cmd: string, args?: unknown) => mockInvoke(cmd, args),+ invoke: (cmd: string, args?: unknown) =>+ cmd === "embedding_fetch" ? mockEmbeddingFetchInvoke(args) : mockInvoke(cmd, args),}))-// Stub getHttpFetch so fetchEmbedding calls hit our in-test responder.+// In-test responder used by the `embedding_fetch` invoke shim below.const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()-vi.mock("@/lib/tauri-fetch", () => ({- getHttpFetch: () => Promise.resolve(mockHttpFetch),- isFetchNetworkError: (err: unknown) =>- err instanceof TypeError ||- (err instanceof Error &&- (err.message === "Load failed" || err.message === "Failed to fetch")),-}))// readFile / listDirectory aren't exercised in this file's cases; stub.vi.mock("@/commands/fs", () => ({@@ -77,10 +70,182 @@ function genericErrorResponse(status: number, body: string): Response {beforeEach(() => {mockInvoke.mockReset()+ mockEmbeddingFetchInvoke.mockReset()mockHttpFetch.mockReset()+ mockEmbeddingFetchInvoke.mockImplementation((args) => {+ const input = args as { text: string; cfg: typeof cfg; maxRetries?: number }+ return fetchEmbeddingViaMockHttp(input.text, input.cfg, input.maxRetries ?? 3)+ })resetEmbeddingOptimizeAccountingForTests()})+async function fetchEmbeddingViaMockHttp(+ text: string,+ config: typeof cfg & {+ outputDimensionality?: number+ extraHeaders?: Record<string, string>+ },+ maxRetries: number,+): Promise<number[]> {+ const isGoogle = isGoogleEmbeddingConfigForTest(config)+ const isDoubaoVision = config.model.toLowerCase().includes("doubao-embedding-vision")+ const endpoint = isGoogle ? googleEndpointForTest(config) : volcengineEndpointForTest(config)+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ ...(isLocalEndpointForTest(endpoint) ? { Origin: "http://localhost" } : {}),+ }+ if (config.apiKey) {+ if (isGoogle) headers["x-goog-api-key"] = config.apiKey+ else headers.Authorization = `Bearer ${config.apiKey}`+ }+ for (const [name, value] of Object.entries(config.extraHeaders ?? {})) {+ const trimmed = name.trim()+ const lower = trimmed.toLowerCase()+ if (!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(trimmed)) continue+ if (["authorization", "content-type", "host", "content-length", "origin", "x-goog-api-key"].includes(lower)) continue+ if (!String(value).trim()) continue+ headers[trimmed] = String(value).trim()+ }++ let current = text+ let attempts = 0+ while (attempts <= maxRetries) {+ attempts += 1+ const body = isGoogle+ ? googleBodyForTest(config.model, current, config.outputDimensionality)+ : isDoubaoVision+ ? { model: config.model, encoding_format: "float", input: [{ type: "text", text: current }] }+ : { model: config.model, input: current }+ let response: Response+ try {+ const maybeResponse = await mockHttpFetch(endpoint, {+ method: "POST",+ headers,+ body: JSON.stringify(body),+ })+ if (!maybeResponse) throw new TypeError("Failed to fetch")+ response = maybeResponse+ } catch {+ throw new Error(`Network error reaching ${endpoint}. Check endpoint URL, API key, and connectivity.`)+ }+ const textBody = await response.text()+ if (response.ok) {+ const data = JSON.parse(textBody)+ const embedding = isGoogle+ ? data?.embedding?.values+ : isDoubaoVision+ ? data?.data?.embedding+ : data?.data?.[0]?.embedding+ if (Array.isArray(embedding) && embedding.length > 0 && embedding.every((v) => typeof v === "number" && Number.isFinite(v))) {+ return embedding+ }+ const expected = isGoogle ? "embedding.values" : isDoubaoVision ? "data.embedding" : "data[0].embedding"+ throw new Error(`Embedding response missing ${expected}`)+ }+ if (looksLikeOversizeForTest(response.status, textBody) && current.length > 64 && attempts <= maxRetries) {+ current = current.slice(0, Math.floor(current.length / 2))+ continue+ }+ if (looksLikeOversizeForTest(response.status, textBody)) {+ throw new Error(`Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars.`)+ }+ throw new Error(`API ${response.status} ${response.statusText}${textBody ? ` — ${textBody.slice(0, 200)}` : ""} at ${endpoint}`)+ }+ throw new Error(`Embedding endpoint rejected every size down to ${current.length} chars`)+}++function looksLikeOversizeForTest(status: number, body: string): boolean {+ const lower = body.toLowerCase()+ return status === 413+ || lower.includes("too long")+ || lower.includes("maximum context")+ || lower.includes("max_tokens")+ || lower.includes("max tokens")+ || lower.includes("context length")+ || lower.includes("token limit")+ || lower.includes("exceeds")+ || lower.includes("input length")+}++function isGoogleEmbeddingConfigForTest(config: { endpoint: string }): boolean {+ const endpoint = config.endpoint.toLowerCase()+ return endpoint.includes("generativelanguage.googleapis.com") || endpoint.includes(":embedcontent")+}++function isLocalEndpointForTest(endpoint: string): boolean {+ try {+ const host = new URL(endpoint).hostname+ return host === "localhost"+ || host === "127.0.0.1"+ || host === "::1"+ || /^10\./.test(host)+ || /^192\.168\./.test(host)+ || /^172\.(1[6-9]|2\d|3[01])\./.test(host)+ } catch {+ return false+ }+}++function isVolcengineEndpointForTest(endpoint: string): boolean {+ try {+ const host = new URL(endpoint).hostname.toLowerCase()+ return host === "volces.com" || host.endsWith(".volces.com") || host.includes("volcengine")+ } catch {+ return false+ }+}++function volcengineEndpointForTest(config: { endpoint: string; model: string }): string {+ if (!isVolcengineEndpointForTest(config.endpoint)) return config.endpoint+ const suffix = config.model.toLowerCase().includes("doubao-embedding-vision") ? "/embeddings/multimodal" : "/embeddings"+ return appendEndpointPathForTest(config.endpoint, suffix)+}++function appendEndpointPathForTest(endpoint: string, suffix: string): string {+ const cleanSuffix = suffix.replace(/^\/+/, "")+ const url = new URL(endpoint)+ const path = url.pathname.replace(/\/+$/, "")+ const lower = path.toLowerCase()+ const lowerSuffix = `/${cleanSuffix.toLowerCase()}`+ if (lower.endsWith(lowerSuffix)) {+ url.pathname = path || "/"+ } else if (lower.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings") {+ url.pathname = path.slice(0, -"/multimodal".length) || "/"+ } else if (lower.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal") {+ url.pathname = `${path}/multimodal`+ } else {+ url.pathname = `${path}/${cleanSuffix}`.replace(/\/{2,}/g, "/")+ }+ return url.toString()+}++function googleEndpointForTest(config: { endpoint: string; model: string }): string {+ const raw = stripGoogleKeyForTest(config.endpoint.trim()).replace(/\/+$/, "")+ if (/:batchEmbedContents(\?|$)/i.test(raw)) return raw.replace(/:batchEmbedContents/i, ":embedContent")+ if (/:embedContent(\?|$)/i.test(raw)) return raw+ const model = config.model.startsWith("models/") ? config.model.slice("models/".length) : config.model+ if (/\/models\/[^/?]+$/i.test(raw)) return `${raw}:embedContent`+ return `${raw}/models/${encodeURIComponent(model)}:embedContent`+}++function stripGoogleKeyForTest(endpoint: string): string {+ if (!endpoint.includes("?")) return endpoint+ const url = new URL(endpoint)+ url.searchParams.delete("key")+ return url.toString()+}++function googleBodyForTest(model: string, text: string, outputDimensionality?: number): Record<string, unknown> {+ const body: Record<string, unknown> = {+ model: model.startsWith("models/") ? model : `models/${model}`,+ content: { parts: [{ text }] },+ }+ if (typeof outputDimensionality === "number" && Number.isFinite(outputDimensionality) && outputDimensionality > 0) {+ body.output_dimensionality = Math.floor(outputDimensionality)+ }+ return body+}+// ── searchByEmbedding — chunk→page aggregation ─────────────────────describe("searchByEmbedding — aggregation", () => {diff --git a/src/lib/embedding.ts b/src/lib/embedding.tsindex ff12e839..d90f220a 100644--- a/src/lib/embedding.ts+++ b/src/lib/embedding.ts@@ -16,8 +16,9 @@* `{id, score}[]` shape; matched chunks available on the* optional `matchedChunks` field for future UI surfacing.*- * HTTP goes through the Tauri plugin (`src/lib/tauri-fetch.ts`) so- * CORS-unfriendly endpoints work the same as the LLM path.+ * Provider HTTP is executed by the Rust `embedding_fetch` command so+ * CORS-unfriendly endpoints keep working while the core embedding+ * transport is shared by UI and backend callers.*/import { readFile, listDirectory } from "@/commands/fs"@@ -25,30 +26,7 @@ import { invoke } from "@tauri-apps/api/core"import type { EmbeddingConfig } from "@/stores/wiki-store"import type { FileNode } from "@/types/wiki"import { normalizePath } from "@/lib/path-utils"-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"import { chunkMarkdown, type Chunk } from "@/lib/text-chunker"-import { isLocalOrPrivateHttpEndpoint, localLlmOriginHeader } from "@/lib/llm-providers"--const RESERVED_EMBEDDING_HEADER_NAMES = new Set([- "authorization",- "content-type",- "host",- "content-length",- "origin",- "x-goog-api-key",-])-const HTTP_HEADER_NAME_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/--function isSafeExtraHeader(name: string, value: string): boolean {- const trimmedName = name.trim()- const trimmedValue = value.trim()- return (- trimmedName.length > 0 &&- trimmedValue.length > 0 &&- HTTP_HEADER_NAME_RE.test(trimmedName) &&- !RESERVED_EMBEDDING_HEADER_NAMES.has(trimmedName.toLowerCase())- )-}// ── Error surfacing ──────────────────────────────────────────────────────@@ -111,255 +89,18 @@ export async function fetchEmbedding(maxRetries = 3,): Promise<number[] | null> {if (!cfg.endpoint) return null-- const isGoogleNative = isGoogleEmbeddingConfig(cfg)- const isDoubaoMultimodal = isDoubaoMultimodalEmbeddingConfig(cfg)- const endpoint = isGoogleNative- ? googleEmbeddingEndpoint(cfg)- : volcengineEmbeddingEndpoint(cfg)- const headers: Record<string, string> = {- "Content-Type": "application/json",- ...(isLocalOrPrivateHttpEndpoint(endpoint) ? localLlmOriginHeader() : {}),- }- if (cfg.apiKey) {- if (isGoogleNative) {- headers["x-goog-api-key"] = cfg.apiKey- } else {- headers.Authorization = `Bearer ${cfg.apiKey}`- }- }- if (cfg.extraHeaders) {- for (const [k, v] of Object.entries(cfg.extraHeaders)) {- const name = k.trim()- const value = v.trim()- if (!isSafeExtraHeader(name, value)) continue- headers[name] = value- }- }-- let current = text- let attempts = 0- while (attempts <= maxRetries) {- attempts++- try {- const httpFetch = await getHttpFetch()- const resp = await httpFetch(endpoint, {- method: "POST",- headers,- body: JSON.stringify(- isGoogleNative- ? googleEmbeddingBody(cfg.model, current, cfg.outputDimensionality)- : isDoubaoMultimodal- ? doubaoMultimodalEmbeddingBody(cfg.model, current)- : { model: cfg.model, input: current },- ),- })-- if (resp.ok) {- const data = await resp.json()- const embedding = isGoogleNative- ? data?.embedding?.values ?? null- : isDoubaoMultimodal- ? data?.data?.embedding ?? null- : data?.data?.[0]?.embedding ?? null- if (isNonEmptyNumberArray(embedding)) {- lastEmbeddingError = null- return embedding- }- const expectedShape = isGoogleNative- ? "embedding.values"- : isDoubaoMultimodal- ? "data.embedding"- : "data[0].embedding"- lastEmbeddingError = `Embedding response missing ${expectedShape} (got ${JSON.stringify(data).slice(0, 200)})`- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null- }-- // Non-OK: try to read the body for an oversize hint.- let bodyText = ""- try {- bodyText = await resp.text()- } catch {- // ignore — some servers return empty bodies on error- }-- if (looksLikeOversizeError(resp.status, bodyText)) {- // Can we still halve-and-retry? Need room on both axes:- // text not yet at the 64-char floor, and retry budget left.- if (current.length > 64 && attempts <= maxRetries) {- const prev = current.length- current = current.slice(0, Math.floor(current.length / 2))- console.warn(- `[Embedding] auto-halving after HTTP ${resp.status} at ${prev} chars → retrying at ${current.length} chars (attempt ${attempts}/${maxRetries + 1})`,- )- continue- }- // Out of retries on a SERVER-oversize error — give the user a- // message that names the smallest size that still failed so- // they can tune Settings → Embedding accordingly.- lastEmbeddingError = `Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars (${bodyText.slice(0, 160)}).`- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null- }-- // Non-oversize definitive failure (auth, rate limit, server down, …).- lastEmbeddingError = `API ${resp.status} ${resp.statusText}${bodyText ? ` — ${bodyText.slice(0, 200)}` : ""} at ${endpoint}`- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null- } catch (err) {- if (isFetchNetworkError(err)) {- lastEmbeddingError = `Network error reaching ${endpoint}. Check endpoint URL, API key, and connectivity.`- } else {- lastEmbeddingError = err instanceof Error ? err.message : String(err)- }- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null- }- }-- // Exhausted retries (only reachable if every halving round triggered- // the retry branch and then the loop condition ended).- lastEmbeddingError = `Embedding endpoint rejected every size down to ${current.length} chars — the server's context is smaller than ${current.length * 2}. Lower Settings → Embedding → Max Chunk Chars.`- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null-}--function isNonEmptyNumberArray(value: unknown): value is number[] {- return Array.isArray(value)- && value.length > 0- && value.every((item) => typeof item === "number" && Number.isFinite(item))-}--function isGoogleEmbeddingConfig(cfg: EmbeddingConfig): boolean {- const endpoint = cfg.endpoint.toLowerCase()- return endpoint.includes("generativelanguage.googleapis.com")- || /:embedcontent(\?|$)/i.test(endpoint)-}--function isVolcengineEmbeddingEndpoint(endpoint: string): boolean {- try {- const host = new URL(endpoint).hostname.toLowerCase()- return host === "volces.com"- || host.endsWith(".volces.com")- || host.includes("volcengine")- } catch {- const authority = endpoint- .trim()- .replace(/^[a-z][a-z0-9+.-]*:\/\//i, "")- .split(/[/?#]/, 1)[0]- .toLowerCase()- return authority === "volces.com"- || authority.endsWith(".volces.com")- || authority.includes("volcengine")- }-}--function isDoubaoMultimodalEmbeddingConfig(cfg: EmbeddingConfig): boolean {- return cfg.model.trim().toLowerCase().includes("doubao-embedding-vision")-}--function volcengineEmbeddingEndpoint(cfg: EmbeddingConfig): string {- const raw = cfg.endpoint.trim()- if (!isVolcengineEmbeddingEndpoint(raw)) return raw- const targetSuffix = isDoubaoMultimodalEmbeddingConfig(cfg)- ? "/embeddings/multimodal"- : "/embeddings"- return appendEndpointPath(raw, targetSuffix)-}--function appendEndpointPath(endpoint: string, targetSuffix: string): string {- const suffix = targetSuffix.replace(/^\/+/, "")try {- const url = new URL(endpoint)- const path = url.pathname.replace(/\/+$/, "")- const lowerPath = path.toLowerCase()- const lowerSuffix = `/${suffix.toLowerCase()}`- if (lowerPath.endsWith(lowerSuffix)) {- url.pathname = path || "/"- return url.toString()- }- if (lowerPath.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings") {- url.pathname = path.slice(0, -"/multimodal".length) || "/"- return url.toString()- }- if (lowerPath.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal") {- url.pathname = `${path}/multimodal`- return url.toString()- }- url.pathname = `${path}/${suffix}`.replace(/\/{2,}/g, "/")- return url.toString()- } catch {- const [base, query = ""] = endpoint.split("?", 2)- const trimmed = base.replace(/\/+$/, "")- const lower = trimmed.toLowerCase()- const lowerSuffix = `/${suffix.toLowerCase()}`- const next = lower.endsWith(lowerSuffix)- ? trimmed- : lower.endsWith("/embeddings/multimodal") && lowerSuffix === "/embeddings"- ? trimmed.slice(0, -"/multimodal".length)- : lower.endsWith("/embeddings") && lowerSuffix === "/embeddings/multimodal"- ? `${trimmed}/multimodal`- : `${trimmed}/${suffix}`- return query ? `${next}?${query}` : next- }-}--function googleEmbeddingEndpoint(cfg: EmbeddingConfig): string {- const raw = stripGoogleApiKeyQuery(cfg.endpoint.trim()).replace(/\/+$/, "")- if (/:batchEmbedContents(\?|$)/i.test(raw)) {- return raw.replace(/:batchEmbedContents/i, ":embedContent")- }- if (/:embedContent(\?|$)/i.test(raw)) return raw-- const modelPath = googleModelPath(cfg.model)- if (/\/models\/[^/?]+$/i.test(raw)) {- return `${raw}:embedContent`- }- return `${raw}/models/${encodeURIComponent(modelPath.replace(/^models\//, ""))}:embedContent`-}--function stripGoogleApiKeyQuery(endpoint: string): string {- if (!endpoint.includes("?")) return endpoint- try {- const url = new URL(endpoint)- url.searchParams.delete("key")- return url.toString()- } catch {- return endpoint.replace(/([?&])key=[^&]*&?/i, (_, prefix: string) => prefix === "?" ? "?" : "&")- .replace(/[?&]$/, "")- .replace("?&", "?")- }-}--function googleModelPath(model: string): string {- const trimmed = model.trim()- if (trimmed.startsWith("models/")) return trimmed- return `models/${trimmed}`-}--function googleEmbeddingBody(- model: string,- text: string,- outputDimensionality?: number,-): Record<string, unknown> {- const body: Record<string, unknown> = {- model: googleModelPath(model),- content: {- parts: [{ text }],- },- }- if (typeof outputDimensionality === "number" && Number.isFinite(outputDimensionality) && outputDimensionality > 0) {- body.output_dimensionality = Math.floor(outputDimensionality)- }- return body-}--function doubaoMultimodalEmbeddingBody(model: string, text: string): Record<string, unknown> {- return {- model,- encoding_format: "float",- input: [{ type: "text", text }],+ const embedding = await invoke<number[]>("embedding_fetch", {+ text,+ cfg,+ maxRetries,+ })+ lastEmbeddingError = null+ return embedding+ } catch (err) {+ lastEmbeddingError = err instanceof Error ? err.message : String(err)+ console.warn(`[Embedding] ${lastEmbeddingError}`)+ return null}}diff --git a/src/lib/web-search.test.ts b/src/lib/web-search.test.tsindex fb960a84..af3047fc 100644--- a/src/lib/web-search.test.ts+++ b/src/lib/web-search.test.ts@@ -1,70 +1,41 @@import { beforeEach, describe, expect, it, vi } from "vitest"import { hasConfiguredDeepResearchSources, hasConfiguredSearchProvider, resolveSearchConfig, webSearch } from "./web-search"-const fetchMock = vi.fn<typeof fetch>()+const invokeMock = vi.hoisted(() => vi.fn())-function jsonResponse(body: unknown, init?: ResponseInit): Response {- return new Response(JSON.stringify(body), {- status: 200,- headers: { "Content-Type": "application/json" },- ...init,- })-}+vi.mock("@tauri-apps/api/core", () => ({+ invoke: invokeMock,+}))describe("webSearch", () => {beforeEach(() => {- fetchMock.mockReset()- vi.stubGlobal("fetch", fetchMock)+ invokeMock.mockReset()})- it("normalizes Tavily results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- results: [- { title: "A", url: "https://www.example.com/a", content: "Alpha" },- ],- }))-- const out = await webSearch("alpha", { provider: "tavily", apiKey: "tvly" }, 3)-- expect(fetchMock).toHaveBeenCalledWith("https://api.tavily.com/search", expect.objectContaining({- method: "POST",- }))- expect(out).toEqual([- { title: "A", url: "https://www.example.com/a", snippet: "Alpha", source: "example.com" },+ it("delegates provider search to the Rust backend command", async () => {+ invokeMock.mockResolvedValueOnce([+ { title: "A", url: "https://example.com/a", snippet: "Alpha", source: "web" },])- })-- it("calls SerpApi Google Search and normalizes organic results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- organic_results: [- { title: "Serp result", link: "https://www.serp.example/page", snippet: "Snippet" },- { title: "Second", link: "https://docs.example/item", snippet: "More" },- ],- }))- const out = await webSearch("knowledge graph", { provider: "serpapi", apiKey: "serp" }, 1)- const [url, init] = fetchMock.mock.calls[0]- const parsed = new URL(String(url))+ const out = await webSearch("alpha", { provider: "tavily", apiKey: "tvly" }, 3)- expect(parsed.origin + parsed.pathname).toBe("https://serpapi.com/search")- expect(parsed.searchParams.get("engine")).toBe("google")- expect(parsed.searchParams.get("q")).toBe("knowledge graph")- expect(parsed.searchParams.get("api_key")).toBe("serp")- expect(parsed.searchParams.get("num")).toBe("1")- expect(init).toEqual(expect.objectContaining({ method: "GET" }))+ expect(invokeMock).toHaveBeenCalledWith("web_search", {+ query: "alpha",+ maxResults: 3,+ config: expect.objectContaining({+ provider: "tavily",+ apiKey: "tvly",+ }),+ })expect(out).toEqual([- { title: "Serp result", url: "https://www.serp.example/page", snippet: "Snippet", source: "serp.example" },+ { title: "A", url: "https://example.com/a", snippet: "Alpha", source: "web" },])})- it("uses SerpApi provider-specific config and selected engine", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- news_results: [- { title: "News", link: "https://news.example/story", snippet: "Fresh" },- ],- }))+ it("passes provider-specific config to Rust", async () => {+ invokeMock.mockResolvedValueOnce([])- const out = await webSearch(+ await webSearch("ai policy",{provider: "serpapi",@@ -76,501 +47,43 @@ describe("webSearch", () => {},5,)- const parsed = new URL(String(fetchMock.mock.calls[0][0]))-- expect(parsed.searchParams.get("engine")).toBe("google_news")- expect(parsed.searchParams.get("api_key")).toBe("serp-key")- expect(out).toEqual([- { title: "News", url: "https://news.example/story", snippet: "Fresh", source: "news.example" },- ])- })-- it("calls SearXNG JSON search with the configured instance and categories", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- results: [- {- title: "SearXNG result",- url: "https://docs.example/page",- content: "Result content",- engine: "duckduckgo",- },- ],- }))-- const out = await webSearch(- "local search",- {- provider: "searxng",- apiKey: "",- providerConfigs: {- searxng: {- searXngUrl: "https://search.example.com",- searXngCategories: ["general", "news"],- },- },- },- 3,- )- const [url, init] = fetchMock.mock.calls[0]- const parsed = new URL(String(url))-- expect(parsed.origin + parsed.pathname).toBe("https://search.example.com/search")- expect(parsed.searchParams.get("q")).toBe("local search")- expect(parsed.searchParams.get("format")).toBe("json")- expect(parsed.searchParams.get("categories")).toBe("general,news")- expect(init).toEqual(expect.objectContaining({ method: "GET" }))- expect(out).toEqual([- { title: "SearXNG result", url: "https://docs.example/page", snippet: "Result content", source: "docs.example" },- ])- })-- it("preserves SearXNG subpath instances when building the search endpoint", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ results: [] }))-- await webSearch(- "subpath",- {- provider: "searxng",- apiKey: "",- providerConfigs: {- searxng: { searXngUrl: "http://localhost:8080/searx/" },- },- },- 5,- )- const parsed = new URL(String(fetchMock.mock.calls[0][0]))- expect(parsed.origin + parsed.pathname).toBe("http://localhost:8080/searx/search")- expect(parsed.searchParams.get("categories")).toBe("general")- })-- it("surfaces SerpApi JSON errors", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ error: "Invalid API key" }))-- await expect(webSearch("x", { provider: "serpapi", apiKey: "bad" }, 5))- .rejects.toThrow("SerpApi search failed: Invalid API key")- })-- it("calls Firecrawl anonymous search without an API key", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: [- {- title: "LLM Wiki",- url: "https://github.com/nashsu/llm_wiki",- description: "Knowledge base app",- },- ],- }))-- const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 2)- const [url, init] = fetchMock.mock.calls[0]-- expect(url).toBe("https://api.firecrawl.dev/v2/search")- expect(init).toEqual(expect.objectContaining({ method: "POST" }))- expect((init?.headers as Record<string, string>).Authorization).toBeUndefined()- expect(JSON.parse(String(init?.body))).toEqual({ query: "llm_wiki", limit: 2 })- expect(out).toEqual([- {- title: "LLM Wiki",- url: "https://github.com/nashsu/llm_wiki",- snippet: "Knowledge base app",- source: "github.com",- },- ])- })-- it("surfaces Firecrawl suspicious-IP anonymous search guidance", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: false,- error: "Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free.",- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .rejects.toThrow("Firecrawl anonymous search is blocked for this IP")- })-- it("does not migrate legacy top-level apiKey into Firecrawl config", () => {- const resolved = resolveSearchConfig({- provider: "firecrawl",- apiKey: "legacy-key",+ expect(invokeMock).toHaveBeenCalledWith("web_search", {+ query: "ai policy",+ maxResults: 5,+ config: expect.objectContaining({+ provider: "serpapi",+ apiKey: "serp-key",+ serpApiEngine: "google_news",+ }),})-- expect(resolved.providerConfigs?.firecrawl).toBeUndefined()- })-- it("allows Firecrawl success responses with advisory error fields", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- error: "partial results only",- data: [- {- title: "Advisory result",- url: "https://example.com/advisory",- description: "Still usable",- },- ],- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Advisory result",- url: "https://example.com/advisory",- snippet: "Still usable",- source: "example.com",- },- ])- })-- it("surfaces Firecrawl unknown error fallback", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ success: false }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .rejects.toThrow("Firecrawl search failed: Unknown error")- })-- it("surfaces Firecrawl non-ok JSON errors", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse(- { error: "rate limited" },- { status: 429 },- ))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .rejects.toThrow("Firecrawl search failed: rate limited")- })-- it("surfaces Firecrawl non-json HTTP errors", async () => {- fetchMock.mockResolvedValueOnce(new Response("<html>Service unavailable</html>", {- status: 503,- headers: { "Content-Type": "text/html" },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .rejects.toThrow("Firecrawl search failed (503): <html>Service unavailable</html>")- })-- it("surfaces Firecrawl network errors", async () => {- fetchMock.mockRejectedValueOnce(new TypeError("Failed to fetch"))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .rejects.toThrow("Network error reaching Firecrawl Search")- })-- it("normalizes Firecrawl results field and filters missing URLs", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- results: [- { title: "Missing URL", description: "No URL" },- {- metadata: {- title: "Metadata title",- sourceURL: "https://docs.example/firecrawl",- description: "Metadata description",- },- },- ],- }))-- const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5)-- expect(out).toEqual([- {- title: "Metadata title",- url: "https://docs.example/firecrawl",- snippet: "Metadata description",- source: "docs.example",- },- ])- })-- it("allows empty Firecrawl result sets", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: [] }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([])- })-- it("normalizes object-shaped Firecrawl data results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- results: [- {- title: "Object data result",- url: "https://example.com/object-data",- description: "Nested data shape",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Object data result",- url: "https://example.com/object-data",- snippet: "Nested data shape",- source: "example.com",- },- ])- })-- it("normalizes categorized Firecrawl web results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- web: [- {- title: "Categorized web result",- url: "https://example.com/firecrawl-web",- description: "Web category shape",- },- ],- news: [- {- title: "News result",- url: "https://example.com/news",- description: "Ignored category",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Categorized web result",- url: "https://example.com/firecrawl-web",- snippet: "Web category shape",- source: "example.com",- },- ])- })-- it("normalizes object-shaped top-level Firecrawl results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- results: {- web: [- {- title: "Top-level object result",- url: "https://example.com/top-results",- description: "Nested under results.web",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Top-level object result",- url: "https://example.com/top-results",- snippet: "Nested under results.web",- source: "example.com",- },- ])- })-- it("uses Firecrawl web results before fallback categories", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- web: [- {- title: "Preferred web",- url: "https://example.com/web",- description: "Use me",- },- ],- results: [- {- title: "Fallback result",- url: "https://example.com/fallback",- description: "Do not use me",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Preferred web",- url: "https://example.com/web",- snippet: "Use me",- source: "example.com",- },- ])- })-- it("keeps an empty Firecrawl web category empty instead of falling back", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- web: [],- results: [- {- title: "Fallback result",- url: "https://example.com/fallback",- description: "Should stay hidden",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([])- })-- it("normalizes Firecrawl items fallback results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- items: [- {- title: "Items fallback",- url: "https://example.com/items",- description: "Fallback item",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Items fallback",- url: "https://example.com/items",- snippet: "Fallback item",- source: "example.com",- },- ])- })-- it("treats unsupported Firecrawl result shapes as empty results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: { unexpected: true } }))- fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: "unexpected" }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([])- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([])- })-- it("filters non-object Firecrawl result entries", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: {- web: [- null,- "unexpected",- {- title: "Valid entry",- url: "https://example.com/valid",- description: "Keep this",- },- ],- },- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Valid entry",- url: "https://example.com/valid",- snippet: "Keep this",- source: "example.com",- },- ])- })-- it("filters missing URLs in Firecrawl data results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- success: true,- data: [- { title: "No URL", description: "skip me" },- { title: "Has URL", url: "https://example.com/ok", description: "keep me" },- ],- }))-- await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))- .resolves.toEqual([- {- title: "Has URL",- url: "https://example.com/ok",- snippet: "keep me",- source: "example.com",- },- ])})it("requires a configured search provider and key", async () => {await expect(webSearch("x", { provider: "none", apiKey: "" }, 5))- .rejects.toThrow("Select a search provider")+ .rejects.toThrow("Web search not configured")await expect(webSearch("x", { provider: "serpapi", apiKey: "" }, 5))- .rejects.toThrow(/API key/)+ .rejects.toThrow("Add a Tavily, SerpApi, or Brave Search API key")await expect(webSearch("x", { provider: "searxng", apiKey: "" }, 5))- .rejects.toThrow("SearXNG instance URL")+ .rejects.toThrow("Add a SearXNG instance URL")+ await expect(webSearch("x", { provider: "ollama", apiKey: "" }, 5))+ .rejects.toThrow("Ollama Web Search API requires an Ollama API key")+ expect(invokeMock).not.toHaveBeenCalled()})- it("treats SearXNG instance URLs as configured without an API key", () => {- expect(hasConfiguredSearchProvider({- provider: "searxng",- apiKey: "",- providerConfigs: {- searxng: {- searXngUrl: "http://127.0.0.1:8080",- searXngCategories: ["general", "science", "it"],- },- },- searXngUrl: "http://127.0.0.1:8080",- searXngCategories: ["general", "science", "it"],- serpApiEngine: "google",- })).toBe(true)- })-- it("requires an API key for the Ollama Web Search API", async () => {- await expect(webSearch(- "official ollama",- {- provider: "ollama",- apiKey: "",- providerConfigs: {- ollama: { ollamaUrl: "https://ollama.com" },- },- },- 5,- )).rejects.toThrow("requires an Ollama API key")-- expect(hasConfiguredSearchProvider({- provider: "ollama",- apiKey: "",- providerConfigs: {- ollama: { ollamaUrl: "https://ollama.com" },- },- })).toBe(false)- })-- it("treats Firecrawl as configured without an API key", () => {- expect(hasConfiguredSearchProvider({- provider: "firecrawl",- apiKey: "",- })).toBe(true)+ it("treats key-free providers as configured", () => {+ expect(hasConfiguredSearchProvider({ provider: "searxng", apiKey: "", searXngUrl: "http://localhost:8080" })).toBe(true)+ expect(hasConfiguredSearchProvider({ provider: "firecrawl", apiKey: "" })).toBe(true)})it("does not leak a stale top-level Ollama URL into non-Ollama providers", () => {const resolved = resolveSearchConfig({- provider: "serpapi",+ provider: "firecrawl",apiKey: "",ollamaUrl: "http://localhost:11434",- providerConfigs: {- serpapi: { apiKey: "serp-key" },- ollama: { ollamaUrl: "https://ollama.com" },- },})+ expect(resolved.provider).toBe("firecrawl")expect(resolved.ollamaUrl).toBe("https://ollama.com")})@@ -581,116 +94,11 @@ describe("webSearch", () => {deepResearchSource: "anytxt",anyTxt: { enabled: true, endpoint: "http://127.0.0.1:9920" },})).toBe(true)-expect(hasConfiguredDeepResearchSources({provider: "none",apiKey: "",- deepResearchSource: "anytxt",- anyTxt: { enabled: false, endpoint: "http://127.0.0.1:9920" },+ deepResearchSource: "both",+ anyTxt: { enabled: false, endpoint: "" },})).toBe(false)-- expect(hasConfiguredDeepResearchSources({- provider: "none",- apiKey: "",- deepResearchSource: "web",- anyTxt: { endpoint: "http://127.0.0.1:9920" },- })).toBe(false)-- expect(resolveSearchConfig({- provider: "none",- apiKey: "",- }).deepResearchSource).toBe("web")- })-- it("calls the Ollama Web Search API with Bearer auth", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- results: [- { title: "Official", url: "https://ollama.example/search", content: "Cloud result" },- ],- }))-- const out = await webSearch(- "official ollama",- {- provider: "ollama",- apiKey: "",- providerConfigs: {- ollama: { apiKey: "ollama-key" },- },- },- 1,- )- const [url, init] = fetchMock.mock.calls[0]-- expect(url).toBe("https://ollama.com/api/web_search")- expect((init?.headers as Record<string, string>).Authorization).toBe("Bearer ollama-key")- expect(JSON.parse(String(init?.body))).toEqual({ query: "official ollama", max_results: 1 })- expect(out).toEqual([- { title: "Official", url: "https://ollama.example/search", snippet: "Cloud result", source: "ollama.example" },- ])- expect(hasConfiguredSearchProvider({- provider: "ollama",- apiKey: "",- providerConfigs: {- ollama: { apiKey: "ollama-key" },- },- })).toBe(true)- })-- it("surfaces Ollama authentication guidance for 401 responses", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ error: "unauthorized" }, { status: 401 }))-- await expect(webSearch(- "official ollama",- {- provider: "ollama",- apiKey: "",- providerConfigs: {- ollama: { apiKey: "bad-key" },- },- },- 5,- )).rejects.toThrow("Check your Ollama API key")- })-- it("calls Brave Search API with X-Subscription-Token and normalizes results", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({- web: {- results: [- { title: "Brave A", url: "https://www.example.com/a", description: "First result" },- { title: "Brave B", url: "https://docs.example/b", description: "Second result" },- ],- },- }))-- const out = await webSearch("brave query", { provider: "brave", apiKey: "brave-key" }, 2)- const [url, init] = fetchMock.mock.calls[0]- const parsed = new URL(String(url))-- expect(parsed.origin + parsed.pathname).toBe("https://api.search.brave.com/res/v1/web/search")- expect(parsed.searchParams.get("q")).toBe("brave query")- expect(parsed.searchParams.get("count")).toBe("2")- expect(init).toEqual(expect.objectContaining({ method: "GET" }))- expect((init?.headers as Record<string, string>)["X-Subscription-Token"]).toBe("brave-key")- expect(out).toEqual([- { title: "Brave A", url: "https://www.example.com/a", snippet: "First result", source: "example.com" },- { title: "Brave B", url: "https://docs.example/b", snippet: "Second result", source: "docs.example" },- ])- })-- it("clamps Brave Search count to the API's 20-result ceiling", async () => {- fetchMock.mockResolvedValueOnce(jsonResponse({ web: { results: [] } }))-- await webSearch("anything", { provider: "brave", apiKey: "k" }, 50)- const [url] = fetchMock.mock.calls[0]- expect(new URL(String(url)).searchParams.get("count")).toBe("20")- })-- it("surfaces Brave Search authentication guidance for 401 responses", async () => {- fetchMock.mockResolvedValueOnce(new Response("forbidden", { status: 401 }))-- await expect(- webSearch("x", { provider: "brave", apiKey: "bad" }, 5),- ).rejects.toThrow("Brave Search API authentication failed")})})diff --git a/src/lib/web-search.ts b/src/lib/web-search.tsindex 8b814fa9..e87e54c1 100644--- a/src/lib/web-search.ts+++ b/src/lib/web-search.ts@@ -1,3 +1,4 @@+import { invoke } from "@tauri-apps/api/core"import type {SearchApiConfig,SearchProvider,@@ -5,7 +6,6 @@ import type {SearXngCategory,SerpApiEngine,} from "@/stores/wiki-store"-import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"import { hasConfiguredAnyTxt, normalizeAnyTxtConfig } from "@/lib/anytxt-search"export interface WebSearchResult {@@ -108,15 +108,9 @@ export function resolveSearchConfig(config: SearchApiConfig): SearchApiConfig {export function hasConfiguredSearchProvider(config: SearchApiConfig): boolean {const resolved = resolveSearchConfig(config)if (resolved.provider === "none") return false- if (resolved.provider === "searxng") {- return Boolean(resolved.searXngUrl?.trim())- }- if (resolved.provider === "ollama") {- return Boolean(resolved.apiKey?.trim())- }- if (resolved.provider === "firecrawl") {- return true- }+ if (resolved.provider === "searxng") return Boolean(resolved.searXngUrl?.trim())+ if (resolved.provider === "ollama") return Boolean(resolved.apiKey?.trim())+ if (resolved.provider === "firecrawl") return truereturn Boolean(resolved.apiKey?.trim())}@@ -153,512 +147,9 @@ export async function webSearch(throw new Error("Ollama Web Search API requires an Ollama API key. Add one in Settings.")}- switch (resolved.provider) {- case "tavily":- return tavilySearch(query, resolved.apiKey, maxResults)- case "serpapi":- return serpApiSearch(query, resolved.apiKey, maxResults, resolved.serpApiEngine ?? "google")- case "searxng":- return searXngSearch(query, resolved.searXngUrl ?? "", maxResults, resolved.searXngCategories ?? ["general"])- case "ollama":- return ollamaSearch(query, resolved.apiKey ?? "", maxResults)- case "brave":- return braveSearch(query, resolved.apiKey, maxResults)- case "firecrawl":- return firecrawlSearch(query, maxResults)- default:- throw new Error(`Unknown search provider: ${resolved.provider}`)- }-}--function searXngSearchUrl(instanceUrl: string): URL {- const trimmed = instanceUrl.trim()- const withProtocol = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`- const url = new URL(withProtocol)- const path = url.pathname.replace(/\/+$/, "")- url.pathname = path.endsWith("/search") || path === "/search"- ? path- : `${path}/search`- url.search = ""- url.hash = ""- return url-}--async function searXngSearch(- query: string,- instanceUrl: string,- maxResults: number,- categories: SearXngCategory[],-): Promise<WebSearchResult[]> {- let endpoint: URL- try {- endpoint = searXngSearchUrl(instanceUrl)- } catch {- throw new Error("Invalid SearXNG instance URL. Use a valid http(s) URL, for example https://search.example.com.")- }-- endpoint.searchParams.set("q", query)- endpoint.searchParams.set("format", "json")- endpoint.searchParams.set("categories", (categories.length > 0 ? categories : ["general"]).join(","))-- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch(endpoint.toString(), {- method: "GET",- headers: { Accept: "application/json" },- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching the SearXNG instance. Check the instance URL and whether JSON search is enabled.",- )- }- throw err- }-- if (!response.ok) {- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`SearXNG search failed (${response.status}): ${errorText}`)- }-- const data = await response.json()- return normalizeSearXngResults(data, maxResults)-}--function normalizeSearXngResults(data: { results?: unknown[] }, maxResults: number): WebSearchResult[] {- return (data.results ?? [])- .slice(0, maxResults)- .map((item) => normalizeSearXngResult(item))- .filter((item) => item.url.length > 0)-}--function normalizeSearXngResult(item: unknown): WebSearchResult {- const r = item as {- title?: string- url?: string- content?: string- engine?: string- category?: string- }- const url = r.url ?? ""- return {- title: r.title ?? "Untitled",- url,- snippet: r.content ?? "",- source: hostnameFromUrl(url) || r.engine || r.category || "",- }-}--function hostnameFromUrl(url: string): string {- try {- return new URL(url).hostname.replace("www.", "")- } catch {- return ""- }-}--async function tavilySearch(- query: string,- apiKey: string,- maxResults: number,-): Promise<WebSearchResult[]> {- // Route through the Tauri HTTP plugin so future non-Tavily search- // providers (Serper, Exa, Brave, Google CSE, ...) with less friendly- // CORS don't each need their own workaround. See tauri-fetch.ts.- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch("https://api.tavily.com/search", {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- api_key: apiKey,- query,- max_results: maxResults,- search_depth: "advanced",- include_answer: false,- }),- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching api.tavily.com. Check your connectivity and whether the Tavily API key is still valid.",- )- }- throw err- }-- if (!response.ok) {- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`Tavily search failed (${response.status}): ${errorText}`)- }-- const data = await response.json()-- return (data.results ?? []).map((r: { title: string; url: string; content: string }) => ({- title: r.title ?? "Untitled",- url: r.url ?? "",- snippet: r.content ?? "",- source: hostnameFromUrl(r.url ?? ""),- }))-}--interface FirecrawlSearchResponse {- success?: boolean- error?: string- data?: unknown- results?: unknown-}--async function firecrawlSearch(- query: string,- maxResults: number,-): Promise<WebSearchResult[]> {- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch("https://api.firecrawl.dev/v2/search", {- method: "POST",- headers: {- Accept: "application/json",- "Content-Type": "application/json",- },- body: JSON.stringify({- query,- limit: maxResults,- }),- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching Firecrawl Search. Check your connectivity or choose another Web Search provider.",- )- }- throw err- }-- const text = await response.text().catch(() => "")- let data: FirecrawlSearchResponse = {}- try {- data = text ? JSON.parse(text) as FirecrawlSearchResponse : {}- } catch {- if (!response.ok) {- throw new Error(`Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)- }- throw new Error("Firecrawl search returned an invalid JSON response.")- }-- if (!response.ok) {- throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)- }-- if (data.success === false || (data.success !== true && typeof data.error === "string" && data.error.trim())) {- throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed: ${data.error ?? "Unknown error"}`)- }-- return normalizeFirecrawlResults(data, maxResults)-}--function friendlyFirecrawlError(error?: string): string | null {- const message = error?.trim()- if (!message) return null- if (/suspicious|without an API key|firecrawl can't be used without an api key/i.test(message)) {- return "Firecrawl anonymous search is blocked for this IP. Firecrawl says this network looks suspicious; choose another Web Search provider or try a different network."- }- return `Firecrawl search failed: ${message}`-}--function normalizeFirecrawlResults(data: FirecrawlSearchResponse, maxResults: number): WebSearchResult[] {- const rawResults = extractFirecrawlResultArray(data)- return rawResults- .slice(0, maxResults)- .map((item) => normalizeFirecrawlResult(item))- .filter((item) => item.url.length > 0)-}--function extractFirecrawlResultArray(data: FirecrawlSearchResponse): unknown[] {- if (Array.isArray(data.data)) return data.data- if (Array.isArray(data.results)) return data.results-- return extractFirecrawlNestedResultArray(data.data)- ?? extractFirecrawlNestedResultArray(data.results)- ?? []-}--function extractFirecrawlNestedResultArray(value: unknown): unknown[] | null {- if (!value || typeof value !== "object") return null- const nested = value as Record<string, unknown>-- // Firecrawl v2 may return categorized results as data.web/news/images.- // LLM Wiki consumes the first supported text-search category; image/news- // categories are intentionally ignored because their payloads differ.- // The first matching category wins even when it is empty.- for (const key of ["web", "results", "items"]) {- const nestedValue = nested[key]- if (Array.isArray(nestedValue)) return nestedValue- }-- return null-}--function normalizeFirecrawlResult(item: unknown): WebSearchResult {- if (!item || typeof item !== "object") {- return { title: "Untitled", url: "", snippet: "", source: "" }- }- const r = item as {- title?: string- url?: string- link?: string- markdown?: string- content?: string- description?: string- snippet?: string- source?: string- metadata?: {- title?: string- sourceURL?: string- url?: string- description?: string- }- }- const url = r.url ?? r.link ?? r.metadata?.sourceURL ?? r.metadata?.url ?? ""- return {- title: r.title ?? r.metadata?.title ?? "Untitled",- url,- snippet: r.snippet ?? r.description ?? r.metadata?.description ?? r.content ?? r.markdown ?? "",- source: hostnameFromUrl(url) || r.source || "",- }-}--async function serpApiSearch(- query: string,- apiKey: string,- maxResults: number,- engine: SerpApiEngine,-): Promise<WebSearchResult[]> {- const params = new URLSearchParams({- engine,- q: query,- api_key: apiKey,- num: String(maxResults),+ return invoke<WebSearchResult[]>("web_search", {+ query,+ config: resolved,+ maxResults,})-- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch(`https://serpapi.com/search?${params.toString()}`, {- method: "GET",- headers: { Accept: "application/json" },- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching serpapi.com. Check your connectivity and whether the SerpApi API key is still valid.",- )- }- throw err- }-- if (!response.ok) {- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`SerpApi search failed (${response.status}): ${errorText}`)- }-- const data = await response.json()- if (typeof data.error === "string" && data.error.trim()) {- throw new Error(`SerpApi search failed: ${data.error}`)- }-- return normalizeSerpApiResults(data, maxResults)-}--function normalizeSerpApiResults(data: {- organic_results?: unknown[]- news_results?: unknown[]- images_results?: unknown[]- video_results?: unknown[]- videos_results?: unknown[]- shopping_results?: unknown[]-}, maxResults: number): WebSearchResult[] {- const rawResults =- data.organic_results ??- data.news_results ??- data.images_results ??- data.video_results ??- data.videos_results ??- data.shopping_results ??- []-- return rawResults- .slice(0, maxResults)- .map((item) => normalizeSerpApiResult(item))-}--function normalizeSerpApiResult(item: unknown): WebSearchResult {- const r = item as {- title?: string- link?: string- url?: string- source?: string- snippet?: string- summary?: string- description?: string- thumbnail?: string- original?: string- displayed_link?: string- }- const url = r.link ?? r.url ?? r.original ?? r.thumbnail ?? ""- return {- title: r.title ?? "Untitled",- url,- snippet: r.snippet ?? r.summary ?? r.description ?? "",- source: hostnameFromUrl(url) || r.source || r.displayed_link || "",- }-}--interface OllamaSearchResponse {- results?: Array<{- title?: string- url?: string- content?: string- }>- error?: string-}--async function ollamaSearch(- query: string,- apiKey: string,- maxResults: number,-): Promise<WebSearchResult[]> {- const trimmedApiKey = apiKey.trim()- if (!trimmedApiKey) {- throw new Error("Ollama Web Search API requires an Ollama API key. Add one in Settings.")- }-- const headers: Record<string, string> = {- Accept: "application/json",- "Content-Type": "application/json",- Authorization: `Bearer ${trimmedApiKey}`,- }-- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch("https://ollama.com/api/web_search", {- method: "POST",- headers,- body: JSON.stringify({- query,- max_results: maxResults,- }),- })- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching the Ollama Web Search API. Check your connectivity and whether the Ollama API key is still valid.",- )- }- throw err- }-- if (!response.ok) {- if (response.status === 401) {- throw new Error(- "Ollama Web Search API authentication failed. Check your Ollama API key.",- )- }- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`Ollama web search failed (${response.status}): ${errorText}`)- }-- const data = (await response.json()) as OllamaSearchResponse-- if (data.error) {- throw new Error(`Ollama web search error: ${data.error}`)- }-- return (data.results ?? [])- .slice(0, maxResults)- .map((r) => {- const url = r.url ?? ""- return {- title: r.title ?? "Untitled",- url,- snippet: r.content ?? "",- source: hostnameFromUrl(url),- }- })-}--interface BraveSearchResponse {- web?: {- results?: Array<{- title?: string- url?: string- description?: string- }>- }- message?: string-}--async function braveSearch(- query: string,- apiKey: string,- maxResults: number,-): Promise<WebSearchResult[]> {- // Brave Web Search API caps `count` at 20 per request. Higher values- // are silently clamped server-side; bound here so the URL stays- // honest and we don't surprise users by appearing to ask for more.- const count = Math.max(1, Math.min(maxResults, 20))- const params = new URLSearchParams({ q: query, count: String(count) })-- const httpFetch = await getHttpFetch()- let response: Response- try {- response = await httpFetch(- `https://api.search.brave.com/res/v1/web/search?${params.toString()}`,- {- method: "GET",- headers: {- Accept: "application/json",- "X-Subscription-Token": apiKey,- },- },- )- } catch (err) {- if (isFetchNetworkError(err)) {- throw new Error(- "Network error reaching api.search.brave.com. Check your connectivity and whether the Brave Search API key is still valid.",- )- }- throw err- }-- if (!response.ok) {- if (response.status === 401 || response.status === 403) {- throw new Error(- "Brave Search API authentication failed. Check your subscription token in Settings.",- )- }- const errorText = await response.text().catch(() => "Unknown error")- throw new Error(`Brave search failed (${response.status}): ${errorText}`)- }-- const data = (await response.json()) as BraveSearchResponse- if (data.message && !data.web) {- throw new Error(`Brave search error: ${data.message}`)- }-- return (data.web?.results ?? [])- .slice(0, maxResults)- .map((r) => {- const url = r.url ?? ""- return {- title: r.title ?? "Untitled",- url,- snippet: r.description ?? "",- source: hostnameFromUrl(url),- }- })}