LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: add Firecrawl web search provider
nash_suc5f90049bf40
diff --git a/src/components/settings/sections/web-search-section.tsx b/src/components/settings/sections/web-search-section.tsxindex a4d1a02e..6ef8e50f 100644--- a/src/components/settings/sections/web-search-section.tsx+++ b/src/components/settings/sections/web-search-section.tsx@@ -1,4 +1,4 @@-import { useState } from "react"+import { useRef, useState } from "react"import { ChevronDown, ChevronRight } from "lucide-react"import { useTranslation } from "react-i18next"import { Input } from "@/components/ui/input"@@ -16,6 +16,7 @@ import {SEARXNG_CATEGORY_OPTIONS,SERPAPI_ENGINE_OPTIONS,resolveSearchConfig,+ webSearch,} from "@/lib/web-search"const SEARCH_PROVIDERS = [@@ -24,28 +25,34 @@ const SEARCH_PROVIDERS = [label: "Ollama",hint: "Ollama Web Search API",keyPlaceholder: "Enter your Ollama API key (ollama.com)",- needsApiKey: true,+ configKind: "key",},{id: "tavily",label: "Tavily",hint: "General web search for Deep Research",keyPlaceholder: "Enter your Tavily API key (tavily.com)",- needsApiKey: true,+ configKind: "key",},{id: "serpapi",label: "SerpApi",hint: "Google, Bing, DuckDuckGo, Scholar, News, Images, Videos, YouTube",keyPlaceholder: "Enter your SerpApi API key (serpapi.com)",- needsApiKey: true,+ configKind: "key",},{id: "searxng",label: "SearXNG",hint: "Self-hosted metasearch via the SearXNG JSON API",urlPlaceholder: "https://search.example.com",- needsApiKey: false,+ configKind: "url",+ },+ {+ id: "firecrawl",+ label: "Firecrawl",+ hint: "Anonymous Firecrawl Search API",+ configKind: "none",},] as const@@ -59,6 +66,8 @@ export function WebSearchSection() {const showBroadAnyTxtWarning = isBroadAnyTxtFilterDir(anyTxtFilterDir)const [expanded, setExpanded] = useState<Record<string, boolean>>({})const [savedId, setSavedId] = useState<string | null>(null)+ const [testStatus, setTestStatus] = useState<Record<string, { state: "testing" | "ok" | "warning" | "error"; message: string }>>({})+ const testRunRef = useRef<Record<string, number>>({})async function persist(next: SearchApiConfig) {const { saveSearchApiConfig } = await import("@/lib/project-store")@@ -67,6 +76,12 @@ export function WebSearchSection() {}function updateProvider(id: Exclude<SearchProvider, "none">, patch: SearchProviderOverride) {+ setTestStatus((prev) => {+ if (!prev[id]) return prev+ const next = { ...prev }+ delete next[id]+ return next+ })const currentConfigs = resolvedConfig.providerConfigs ?? {}const merged = { ...(currentConfigs[id] ?? {}), ...patch }const nextConfigs = { ...currentConfigs, [id]: merged }@@ -84,6 +99,38 @@ export function WebSearchSection() {persist(resolveSearchConfig({ ...resolvedConfig, provider: nextProvider })).catch(() => {})}+ async function testProvider(id: Exclude<SearchProvider, "none">) {+ const runId = (testRunRef.current[id] ?? 0) + 1+ testRunRef.current[id] = runId+ const testConfig = resolveSearchConfig({ ...resolvedConfig, provider: id })+ setTestStatus((prev) => ({+ ...prev,+ [id]: { state: "testing", message: t("settings.sections.webSearch.testRunning") },+ }))+ try {+ const results = await webSearch("wikipedia", testConfig, 1)+ if (testRunRef.current[id] !== runId) return+ setTestStatus((prev) => ({+ ...prev,+ [id]: {+ state: results.length > 0 ? "ok" : "warning",+ message: results.length > 0+ ? t("settings.sections.webSearch.testSuccess", { count: results.length })+ : t("settings.sections.webSearch.testNoResults"),+ },+ }))+ } catch (err) {+ if (testRunRef.current[id] !== runId) return+ setTestStatus((prev) => ({+ ...prev,+ [id]: {+ state: "error",+ message: localizeSearchTestError(err, t),+ },+ }))+ }+ }+function updateDeepResearchSource(deepResearchSource: DeepResearchSource) {persist(resolveSearchConfig({ ...resolvedConfig, deepResearchSource })).catch(() => {})}@@ -231,9 +278,11 @@ export function WebSearchSection() {{SEARCH_PROVIDERS.map((provider) => {const override = resolvedConfig.providerConfigs?.[provider.id]const isActive = resolvedConfig.provider === provider.id- const hasConfig = provider.id === "searxng"- ? !!override?.searXngUrl- : !!override?.apiKey+ const hasConfig = provider.configKind === "none"+ ? true+ : provider.id === "searxng"+ ? !!override?.searXngUrl+ : !!override?.apiKeyconst isExpanded = !!expanded[provider.id]return (<div@@ -300,7 +349,19 @@ export function WebSearchSection() {{isExpanded && (<div className="space-y-4 border-t bg-background/50 px-4 py-3">- {provider.needsApiKey ? (+ {provider.configKind === "url" ? (+ <div className="space-y-2">+ <Label>{t("settings.sections.webSearch.instanceUrl")}</Label>+ <Input+ value={override?.searXngUrl ?? resolvedConfig.searXngUrl ?? ""}+ onChange={(e) => updateProvider(provider.id, { searXngUrl: e.target.value })}+ placeholder={provider.urlPlaceholder}+ />+ <p className="text-xs text-muted-foreground">+ {t("settings.sections.webSearch.searxngJsonHint")}+ </p>+ </div>+ ) : provider.configKind !== "none" ? (<div className="space-y-2"><Label>{t("settings.apiKey")}</Label><Input@@ -316,17 +377,9 @@ export function WebSearchSection() {)}</div>) : (- <div className="space-y-2">- <Label>{t("settings.sections.webSearch.instanceUrl")}</Label>- <Input- value={override?.searXngUrl ?? resolvedConfig.searXngUrl ?? ""}- onChange={(e) => updateProvider("searxng", { searXngUrl: e.target.value })}- placeholder={provider.urlPlaceholder}- />- <p className="text-xs text-muted-foreground">- {t("settings.sections.webSearch.searxngJsonHint")}- </p>- </div>+ <p className="text-xs text-muted-foreground">+ {t("settings.sections.webSearch.firecrawlHint")}+ </p>)}{provider.id === "serpapi" && (@@ -342,6 +395,36 @@ export function WebSearchSection() {onChange={(searXngCategories) => updateProvider("searxng", { searXngCategories })}/>)}++ <div className="space-y-2">+ <button+ type="button"+ onClick={() => testProvider(provider.id)}+ disabled={!hasConfig || testStatus[provider.id]?.state === "testing"}+ className="rounded-md border px-3 py-1.5 text-xs font-medium transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-60"+ >+ {!hasConfig+ ? t("settings.sections.webSearch.configureBeforeTesting")+ : testStatus[provider.id]?.state === "testing"+ ? t("settings.sections.webSearch.testRunning")+ : t("settings.sections.webSearch.testProvider")}+ </button>+ {testStatus[provider.id] && (+ <p+ className={`text-xs ${+ testStatus[provider.id].state === "ok"+ ? "text-emerald-600"+ : testStatus[provider.id].state === "warning"+ ? "text-amber-600"+ : testStatus[provider.id].state === "error"+ ? "text-destructive"+ : "text-muted-foreground"+ }`}+ >+ {testStatus[provider.id].message}+ </p>+ )}+ </div></div>)}</div>@@ -352,6 +435,20 @@ export function WebSearchSection() {)}+function localizeSearchTestError(err: unknown, t: ReturnType<typeof useTranslation>["t"]): string {+ const message = err instanceof Error ? err.message : String(err)+ if (/Firecrawl anonymous search is blocked for this IP/i.test(message)) {+ return t("settings.sections.webSearch.firecrawlIpBlocked")+ }+ if (/Network error reaching Firecrawl Search/i.test(message)) {+ return t("settings.sections.webSearch.firecrawlNetworkError")+ }+ if (/Firecrawl search returned an invalid JSON response/i.test(message)) {+ return t("settings.sections.webSearch.firecrawlInvalidJson")+ }+ return t("settings.sections.webSearch.testFailed", { message })+}+function isBroadAnyTxtFilterDir(value: string): boolean {const trimmed = value.trim().replace(/\\/g, "/")if (!trimmed) return falsediff --git a/src/i18n/en.json b/src/i18n/en.jsonindex a2df8dd3..4899a17a 100644--- a/src/i18n/en.json+++ b/src/i18n/en.json@@ -408,12 +408,23 @@"savedBadge": "Saved","instanceUrl": "Instance URL","ollamaHint": "Uses the official Ollama Web Search API (`https://ollama.com/api/web_search`). An Ollama API key is required.",+ "firecrawlHint": "Uses Firecrawl's anonymous Search API (`https://api.firecrawl.dev/v2/search`). No API key is required here, but Firecrawl may block suspicious IP addresses.","searxngJsonHint": "The instance must allow JSON search responses (`format=json`).","searchCategories": "Search categories","searxngCategoriesHint": "Categories are sent as the SearXNG `categories` parameter.","searchEngine": "Search engine / category","customSerpApiPlaceholder": "Custom SerpApi engine, e.g. google_finance",- "customSerpApiHint": "Custom engine will be sent as the SerpApi `engine` parameter."+ "customSerpApiHint": "Custom engine will be sent as the SerpApi `engine` parameter.",+ "testProvider": "Test search",+ "testRunning": "Testing...",+ "testSuccess": "Search test succeeded ({{count}} result).",+ "testSuccess_plural": "Search test succeeded ({{count}} results).",+ "testNoResults": "Connection worked, but this test query returned no results.",+ "testFailed": "Search test failed: {{message}}",+ "firecrawlIpBlocked": "Firecrawl anonymous search is blocked for this IP. Choose another Web Search provider or try a different network.",+ "firecrawlNetworkError": "Could not reach Firecrawl Search. Check your network or choose another Web Search provider.",+ "firecrawlInvalidJson": "Firecrawl returned an invalid response. Try again later or choose another Web Search provider.",+ "configureBeforeTesting": "Configure first"},"network": {"title": "Network",diff --git a/src/i18n/zh.json b/src/i18n/zh.jsonindex d4d948ad..07ed25dd 100644--- a/src/i18n/zh.json+++ b/src/i18n/zh.json@@ -408,12 +408,23 @@"savedBadge": "已保存","instanceUrl": "实例 URL","ollamaHint": "使用官方 Ollama Web Search API(`https://ollama.com/api/web_search`),需要填写 Ollama API Key。",+ "firecrawlHint": "使用 Firecrawl 的匿名搜索 API(`https://api.firecrawl.dev/v2/search`),这里不需要 API Key;但 Firecrawl 可能会拦截可疑 IP。","searxngJsonHint": "实例必须允许 JSON 搜索响应(`format=json`)。","searchCategories": "搜索分类","searxngCategoriesHint": "分类会作为 SearXNG 的 `categories` 参数发送。","searchEngine": "搜索引擎 / 分类","customSerpApiPlaceholder": "自定义 SerpApi engine,例如 google_finance",- "customSerpApiHint": "自定义 engine 会作为 SerpApi 的 `engine` 参数发送。"+ "customSerpApiHint": "自定义 engine 会作为 SerpApi 的 `engine` 参数发送。",+ "testProvider": "测试搜索",+ "testRunning": "测试中...",+ "testSuccess": "搜索测试成功({{count}} 条结果)。",+ "testSuccess_plural": "搜索测试成功({{count}} 条结果)。",+ "testNoResults": "连接可用,但测试查询没有返回结果。",+ "testFailed": "搜索测试失败:{{message}}",+ "firecrawlIpBlocked": "Firecrawl 匿名搜索已被当前 IP 拦截。请切换其他网页搜索 Provider,或尝试更换网络。",+ "firecrawlNetworkError": "无法连接 Firecrawl Search。请检查网络,或切换其他网页搜索 Provider。",+ "firecrawlInvalidJson": "Firecrawl 返回了无效响应。请稍后重试,或切换其他网页搜索 Provider。",+ "configureBeforeTesting": "请先配置"},"network": {"title": "网络",diff --git a/src/lib/web-search.test.ts b/src/lib/web-search.test.tsindex 9f219e97..7b111012 100644--- a/src/lib/web-search.test.ts+++ b/src/lib/web-search.test.ts@@ -151,6 +151,166 @@ describe("webSearch", () => {.rejects.toThrow("SerpApi search failed: Invalid API key")})+ it("calls Firecrawl anonymous search without an API key", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({+ success: true,+ data: [+ {+ title: "LLM Wiki",+ url: "https://github.com/nashsu/llm_wiki",+ description: "Knowledge base app",+ },+ ],+ }))++ const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 2)+ const [url, init] = fetchMock.mock.calls[0]++ expect(url).toBe("https://api.firecrawl.dev/v2/search")+ expect(init).toEqual(expect.objectContaining({ method: "POST" }))+ expect((init?.headers as Record<string, string>).Authorization).toBeUndefined()+ expect(JSON.parse(String(init?.body))).toEqual({ query: "llm_wiki", limit: 2 })+ expect(out).toEqual([+ {+ title: "LLM Wiki",+ url: "https://github.com/nashsu/llm_wiki",+ snippet: "Knowledge base app",+ source: "github.com",+ },+ ])+ })++ it("surfaces Firecrawl suspicious-IP anonymous search guidance", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({+ success: false,+ error: "Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free.",+ }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .rejects.toThrow("Firecrawl anonymous search is blocked for this IP")+ })++ it("does not migrate legacy top-level apiKey into Firecrawl config", () => {+ const resolved = resolveSearchConfig({+ provider: "firecrawl",+ apiKey: "legacy-key",+ })++ expect(resolved.providerConfigs?.firecrawl).toBeUndefined()+ })++ it("allows Firecrawl success responses with advisory error fields", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({+ success: true,+ error: "partial results only",+ data: [+ {+ title: "Advisory result",+ url: "https://example.com/advisory",+ description: "Still usable",+ },+ ],+ }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .resolves.toEqual([+ {+ title: "Advisory result",+ url: "https://example.com/advisory",+ snippet: "Still usable",+ source: "example.com",+ },+ ])+ })++ it("surfaces Firecrawl unknown error fallback", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({ success: false }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .rejects.toThrow("Firecrawl search failed: Unknown error")+ })++ it("surfaces Firecrawl non-ok JSON errors", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse(+ { error: "rate limited" },+ { status: 429 },+ ))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .rejects.toThrow("Firecrawl search failed: rate limited")+ })++ it("surfaces Firecrawl non-json HTTP errors", async () => {+ fetchMock.mockResolvedValueOnce(new Response("<html>Service unavailable</html>", {+ status: 503,+ headers: { "Content-Type": "text/html" },+ }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .rejects.toThrow("Firecrawl search failed (503): <html>Service unavailable</html>")+ })++ it("surfaces Firecrawl network errors", async () => {+ fetchMock.mockRejectedValueOnce(new TypeError("Failed to fetch"))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .rejects.toThrow("Network error reaching Firecrawl Search")+ })++ it("normalizes Firecrawl results field and filters missing URLs", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({+ success: true,+ results: [+ { title: "Missing URL", description: "No URL" },+ {+ metadata: {+ title: "Metadata title",+ sourceURL: "https://docs.example/firecrawl",+ description: "Metadata description",+ },+ },+ ],+ }))++ const out = await webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5)++ expect(out).toEqual([+ {+ title: "Metadata title",+ url: "https://docs.example/firecrawl",+ snippet: "Metadata description",+ source: "docs.example",+ },+ ])+ })++ it("allows empty Firecrawl result sets", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({ success: true, data: [] }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .resolves.toEqual([])+ })++ it("filters missing URLs in Firecrawl data results", async () => {+ fetchMock.mockResolvedValueOnce(jsonResponse({+ success: true,+ data: [+ { title: "No URL", description: "skip me" },+ { title: "Has URL", url: "https://example.com/ok", description: "keep me" },+ ],+ }))++ await expect(webSearch("llm_wiki", { provider: "firecrawl", apiKey: "" }, 5))+ .resolves.toEqual([+ {+ title: "Has URL",+ url: "https://example.com/ok",+ snippet: "keep me",+ source: "example.com",+ },+ ])+ })+it("requires a configured search provider and key", async () => {await expect(webSearch("x", { provider: "none", apiKey: "" }, 5)).rejects.toThrow("Select a search provider")@@ -198,6 +358,13 @@ describe("webSearch", () => {})).toBe(false)})+ it("treats Firecrawl as configured without an API key", () => {+ expect(hasConfiguredSearchProvider({+ provider: "firecrawl",+ apiKey: "",+ })).toBe(true)+ })+it("does not leak a stale top-level Ollama URL into non-Ollama providers", () => {const resolved = resolveSearchConfig({provider: "serpapi",diff --git a/src/lib/web-search.ts b/src/lib/web-search.tsindex 8761b0b7..53c41357 100644--- a/src/lib/web-search.ts+++ b/src/lib/web-search.ts@@ -42,7 +42,7 @@ export const SEARXNG_CATEGORY_OPTIONS: { value: SearXngCategory; label: string;export function resolveSearchConfig(config: SearchApiConfig): SearchApiConfig {const providerConfigs: SearchProviderConfigs = config.providerConfigs ?? {- ...(config.provider !== "none" && config.provider !== "ollama" && config.apiKey+ ...(config.provider !== "none" && config.provider !== "ollama" && config.provider !== "firecrawl" && config.apiKey? {[config.provider]: {apiKey: config.apiKey,@@ -114,6 +114,9 @@ export function hasConfiguredSearchProvider(config: SearchApiConfig): boolean {if (resolved.provider === "ollama") {return Boolean(resolved.apiKey?.trim())}+ if (resolved.provider === "firecrawl") {+ return true+ }return Boolean(resolved.apiKey?.trim())}@@ -138,7 +141,7 @@ export async function webSearch(throw new Error("Web search not configured. Select a search provider in Settings.")}if ((resolved.provider === "tavily" || resolved.provider === "serpapi") && !resolved.apiKey) {- throw new Error("Web search not configured. Add a Tavily or SerpApi API key in Settings, or select a different provider.")+ throw new Error("Web search not configured. Add a Tavily or SerpApi API key in Settings, or select a key-free provider such as Firecrawl or SearXNG.")}if (resolved.provider === "searxng" && !resolved.searXngUrl?.trim()) {throw new Error("Web search not configured. Add a SearXNG instance URL in Settings.")@@ -156,6 +159,8 @@ export async function webSearch(return searXngSearch(query, resolved.searXngUrl ?? "", maxResults, resolved.searXngCategories ?? ["general"])case "ollama":return ollamaSearch(query, resolved.apiKey ?? "", maxResults)+ case "firecrawl":+ return firecrawlSearch(query, maxResults)default:throw new Error(`Unknown search provider: ${resolved.provider}`)}@@ -294,6 +299,105 @@ async function tavilySearch(}))}+interface FirecrawlSearchResponse {+ success?: boolean+ error?: string+ data?: unknown[]+ results?: unknown[]+}++async function firecrawlSearch(+ query: string,+ maxResults: number,+): Promise<WebSearchResult[]> {+ const httpFetch = await getHttpFetch()+ let response: Response+ try {+ response = await httpFetch("https://api.firecrawl.dev/v2/search", {+ method: "POST",+ headers: {+ Accept: "application/json",+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ query,+ limit: maxResults,+ }),+ })+ } catch (err) {+ if (isFetchNetworkError(err)) {+ throw new Error(+ "Network error reaching Firecrawl Search. Check your connectivity or choose another Web Search provider.",+ )+ }+ throw err+ }++ const text = await response.text().catch(() => "")+ let data: FirecrawlSearchResponse = {}+ try {+ data = text ? JSON.parse(text) as FirecrawlSearchResponse : {}+ } catch {+ if (!response.ok) {+ throw new Error(`Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)+ }+ throw new Error("Firecrawl search returned an invalid JSON response.")+ }++ if (!response.ok) {+ throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed (${response.status}): ${text || "Unknown error"}`)+ }++ if (data.success === false || (data.success !== true && typeof data.error === "string" && data.error.trim())) {+ throw new Error(friendlyFirecrawlError(data.error) ?? `Firecrawl search failed: ${data.error ?? "Unknown error"}`)+ }++ return normalizeFirecrawlResults(data, maxResults)+}++function friendlyFirecrawlError(error?: string): string | null {+ const message = error?.trim()+ if (!message) return null+ if (/suspicious|without an API key|firecrawl can't be used without an api key/i.test(message)) {+ return "Firecrawl anonymous search is blocked for this IP. Firecrawl says this network looks suspicious; choose another Web Search provider or try a different network."+ }+ return `Firecrawl search failed: ${message}`+}++function normalizeFirecrawlResults(data: FirecrawlSearchResponse, maxResults: number): WebSearchResult[] {+ const rawResults = data.data ?? data.results ?? []+ return rawResults+ .slice(0, maxResults)+ .map((item) => normalizeFirecrawlResult(item))+ .filter((item) => item.url.length > 0)+}++function normalizeFirecrawlResult(item: unknown): WebSearchResult {+ const r = item as {+ title?: string+ url?: string+ link?: string+ markdown?: string+ content?: string+ description?: string+ snippet?: string+ source?: string+ metadata?: {+ title?: string+ sourceURL?: string+ url?: string+ description?: string+ }+ }+ const url = r.url ?? r.link ?? r.metadata?.sourceURL ?? r.metadata?.url ?? ""+ return {+ title: r.title ?? r.metadata?.title ?? "Untitled",+ url,+ snippet: r.snippet ?? r.description ?? r.metadata?.description ?? r.content ?? r.markdown ?? "",+ source: hostnameFromUrl(url) || r.source || "",+ }+}+async function serpApiSearch(query: string,apiKey: string,diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.tsindex 56165e00..a381495d 100644--- a/src/stores/wiki-store.ts+++ b/src/stores/wiki-store.ts@@ -38,7 +38,7 @@ interface LlmConfig {codexCliTimeoutMinutes?: number}-export type SearchProvider = "tavily" | "serpapi" | "searxng" | "ollama" | "none"+export type SearchProvider = "tavily" | "serpapi" | "searxng" | "ollama" | "firecrawl" | "none"export type DeepResearchSource = "web" | "anytxt" | "both"export type SerpApiEngine =| "google"