RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(chat): add SearchApi web search provider (#21563)
Gabs Ferreirabc9d7accd93c
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.mdindex 637971cb94..02e8476eb5 100644--- a/docs/references/http_api_reference.md+++ b/docs/references/http_api_reference.md@@ -2647,7 +2647,7 @@ curl --request POST \- `"refine_multiturn"`: `boolean`- `"reasoning"`: `boolean`- `"cross_languages"`: `list[string]`- - `"web_search_provider"`: `string` The web search service to use. Supported values are `"brave"`, `"exa"`, `"firecrawl"`, `"linkup"`, `"parallel"`, `"querit"`, `"search1api"`, `"serply"`, `"tavily"`, and `"youcom"`. If omitted, Tavily is selected only when `"tavily_api_key"` is configured; otherwise web search is disabled.+ - `"web_search_provider"`: `string` The web search service to use. Supported values are `"brave"`, `"exa"`, `"firecrawl"`, `"linkup"`, `"parallel"`, `"querit"`, `"search1api"`, `"searchapi"`, `"serply"`, `"tavily"`, and `"youcom"`. If omitted, Tavily is selected only when `"tavily_api_key"` is configured; otherwise web search is disabled.- `"tavily_api_key"`: `string`- `"brave_api_key"`: `string` The Brave Search API key. Set `web_search_provider` to `"brave"` when using this field.- `"exa_api_key"`: `string` The Exa API key. Set `web_search_provider` to `"exa"` when using this field.@@ -2656,6 +2656,7 @@ curl --request POST \- `"parallel_api_key"`: `string` The Parallel API key. Set `web_search_provider` to `"parallel"` when using this field.- `"querit_api_key"`: `string` The Querit API key. Set `web_search_provider` to `"querit"` when using this field.- `"search1api_api_key"`: `string` The [Search1API](https://s1.dev) API key. Set `web_search_provider` to `"search1api"` when using this field. See the [Search1API documentation](https://s1.dev/docs/basic/search) for details.+ - `"searchapi_api_key"`: `string` The [SearchApi](https://www.searchapi.io) API key. Set `web_search_provider` to `"searchapi"` when using this field. Results come from its Google engine. See the [SearchApi Google documentation](https://www.searchapi.io/docs/google) for details.- `"serply_api_key"`: `string` The [Serply](https://serply.io) API key. Set `web_search_provider` to `"serply"` when using this field. See the [Serply documentation](https://serply.io/docs) for details.- `"youcom_api_key"`: `string` The You.com API key. Set `web_search_provider` to `"youcom"` when using this field. Optional: You.com serves a rate-limited keyless endpoint, so `"youcom"` works with this field omitted, and a key lifts those limits.- `"similarity_threshold"`: `float`diff --git a/internal/service/web_search_provider.go b/internal/service/web_search_provider.goindex d2ebb87995..29248f99de 100644--- a/internal/service/web_search_provider.go+++ b/internal/service/web_search_provider.go@@ -33,8 +33,8 @@ import ()// Providers, endpoints and clients are listed alphabetically by provider id-// (brave, exa, firecrawl, linkup, parallel, querit, search1api, serply, tavily,-// youcom) so a new provider has exactly one obvious place in each list.+// (brave, exa, firecrawl, linkup, parallel, querit, search1api, searchapi, serply,+// tavily, youcom) so a new provider has exactly one obvious place in each list.// Tavily's endpoint lives with its retrieval code in chat_pipeline.go, which is// why it has no entry here.const (@@ -45,6 +45,7 @@ const (webSearchProviderParallel = "parallel"webSearchProviderQuerit = "querit"webSearchProviderSearch1API = "search1api"+ webSearchProviderSearchAPI = "searchapi"webSearchProviderSerply = "serply"webSearchProviderTavily = "tavily"webSearchProviderYouCom = "youcom"@@ -57,6 +58,7 @@ const (parallelWebSearchEndpoint = "https://api.parallel.ai/v1/search"queritWebSearchEndpoint = "https://api.querit.ai/v1/search"search1apiWebSearchEndpoint = "https://api.search1api.com/search"+ searchapiWebSearchEndpoint = "https://www.searchapi.io/api/v1/search"serplyWebSearchEndpoint = "https://api.serply.io/v1/search/"// webSearchResultCount is how many hits every provider is asked for. Six// keeps the web block in a prompt the size of one corpus chunk set — the@@ -79,6 +81,7 @@ var (parallelWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}queritWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}search1apiWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}+ searchapiWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}serplyWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}youComWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}// Tavily is reached from two call sites (the chat pipeline and the deep@@ -132,6 +135,8 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchProapiKeyField = "querit_api_key"case webSearchProviderSearch1API:apiKeyField = "search1api_api_key"+ case webSearchProviderSearchAPI:+ apiKeyField = "searchapi_api_key"case webSearchProviderSerply:apiKeyField = "serply_api_key"case webSearchProviderTavily:@@ -245,6 +250,14 @@ func retrieveWebSearchWithTavily(provider.APIKey,question,)+ case webSearchProviderSearchAPI:+ return retrieveSearchAPIWebSearch(+ ctx,+ searchapiWebSearchHTTPClient,+ searchapiWebSearchEndpoint,+ provider.APIKey,+ question,+ )case webSearchProviderSerply:return retrieveSerplyWebSearch(ctx,@@ -898,6 +911,78 @@ func decodeSearch1APIWebSearchResults(responseBody []byte) ([]search1apiWebSearcreturn results, nil}+type searchapiWebSearchResult struct {+ Title string `json:"title"`+ Link string `json:"link"`+ Snippet string `json:"snippet"`+}++func retrieveSearchAPIWebSearch(+ ctx context.Context,+ client *http.Client,+ endpoint string,+ apiKey string,+ query string,+) (map[string]interface{}, error) {+ parameters := url.Values{}+ // SearchApi fronts several engines behind one endpoint; google is the web+ // search one, and the parameter is mandatory.+ parameters.Set("engine", "google")+ parameters.Set("q", query)+ parameters.Set("num", strconv.Itoa(webSearchResultCount))++ responseBody, err := webSearchRequest(ctx, client, http.MethodGet,+ endpoint+"?"+parameters.Encode(), map[string]string{+ "Accept": "application/json",+ "Authorization": "Bearer " + apiKey,+ "User-Agent": "ragflow-web-search",+ }, nil)+ if err != nil {+ return nil, fmt.Errorf("searchapi: %w", err)+ }+ results, err := decodeSearchAPIWebSearchResults(responseBody)+ if err != nil {+ return nil, err+ }++ hits := make([]webSearchHit, 0, len(results))+ for _, result := range results {+ hits = append(hits, webSearchHit{+ Title: result.Title,+ URL: result.Link,+ Content: result.Snippet,+ })+ }+ return webSearchPayload("searchapi", hits), nil+}++// decodeSearchAPIWebSearchResults reads the organic hits. The response also+// carries knowledge graph, answer box and ads sections; only organic_results is+// a plain list of pages with a snippet the model can quote.+func decodeSearchAPIWebSearchResults(responseBody []byte) ([]searchapiWebSearchResult, error) {+ var envelope map[string]json.RawMessage+ if err := json.Unmarshal(responseBody, &envelope); err != nil {+ return nil, fmt.Errorf("searchapi: decode response: %w", err)+ }+ if envelope == nil {+ return nil, fmt.Errorf("searchapi: response must be an object")+ }++ resultsValue, exists := envelope["organic_results"]+ if !exists {+ return []searchapiWebSearchResult{}, nil+ }+ if strings.TrimSpace(string(resultsValue)) == "null" {+ return nil, fmt.Errorf("searchapi: response field organic_results must be an array")+ }++ var results []searchapiWebSearchResult+ if err := json.Unmarshal(resultsValue, &results); err != nil {+ return nil, fmt.Errorf("searchapi: response field organic_results must be an array: %w", err)+ }+ return results, nil+}+type serplyWebSearchResult struct {Title string `json:"title"`Link string `json:"link"`diff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.goindex 25545b88c9..847ae94eed 100644--- a/internal/service/web_search_provider_test.go+++ b/internal/service/web_search_provider_test.go@@ -626,6 +626,7 @@ func TestResolveWebSearchProviderSelectsKeyedProviders(t *testing.T) {{provider: "linkup", apiKeyName: "linkup_api_key", apiKey: "linkup-test"},{provider: "parallel", apiKeyName: "parallel_api_key", apiKey: "parallel-test"},{provider: "search1api", apiKeyName: "search1api_api_key", apiKey: "search1api-test"},+ {provider: "searchapi", apiKeyName: "searchapi_api_key", apiKey: "searchapi-test"},}for _, tc := range cases {@@ -946,6 +947,99 @@ func TestDecodeSearch1APIWebSearchResultsAcceptsMissingResults(t *testing.T) {}}+func TestRetrieveSearchAPIWebSearchSendsBearerKeyAndUsesSnippets(t *testing.T) {+ var requestQuery url.Values+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ t.Errorf("method = %q, want GET", r.Method)+ }+ if got := r.Header.Get("Authorization"); got != "Bearer searchapi-test" {+ t.Errorf("Authorization = %q, want %q", got, "Bearer searchapi-test")+ }+ if got := r.Header.Get("User-Agent"); got == "" {+ t.Error("User-Agent is empty")+ }+ requestQuery = r.URL.Query()+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{+ "search_metadata": {"id": "search_test", "status": "Success"},+ "search_parameters": {"engine": "google", "q": "What is RAGFlow?", "num": 6},+ "search_information": {"query_displayed": "What is RAGFlow?"},+ "organic_results": [+ {"position": 1, "title": "No snippet", "link": "https://example.com/empty", "snippet": " \t\n"},+ {"position": 2, "title": "RAGFlow", "link": "https://example.com/ragflow", "source": "RAGFlow", "domain": "example.com", "snippet": "An open-source RAG engine."}+ ],+ "pagination": {"current": 1}+ }`))+ }))+ defer server.Close()++ result, err := retrieveSearchAPIWebSearch(t.Context(), server.Client(), server.URL, "searchapi-test", "What is RAGFlow?")+ if err != nil {+ t.Fatalf("retrieve SearchApi web search: %v", err)+ }+ if got := requestQuery.Get("engine"); got != "google" {+ t.Fatalf("engine = %q, want %q", got, "google")+ }+ if got := requestQuery.Get("q"); got != "What is RAGFlow?" {+ t.Fatalf("q = %q, want %q", got, "What is RAGFlow?")+ }+ if got := requestQuery.Get("num"); got != "6" {+ t.Fatalf("num = %q, want 6", got)+ }+ // The key travels in the header only; SearchApi also accepts it as a query+ // parameter, which would put it in access logs.+ if requestQuery.Has("api_key") {+ t.Fatal("api_key must not be sent as a query parameter")+ }+ assertWebSearchChunk(t, result, "searchapi-https://example.com/ragflow", "RAGFlow", "An open-source RAG engine.")+}++func TestRetrieveSearchAPIWebSearchRejectsErrorStatuses(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusUnauthorized)+ _, _ = w.Write([]byte(`{"error": "Invalid API key."}`))+ }))+ defer server.Close()++ if _, err := retrieveSearchAPIWebSearch(t.Context(), server.Client(), server.URL, "bad-key", "ragflow"); err == nil {+ t.Fatal("error is nil")+ }+}++func TestDecodeSearchAPIWebSearchResultsRejectsMalformedContainers(t *testing.T) {+ cases := []struct {+ name string+ body string+ }{+ {name: "null response", body: `null`},+ {name: "null results", body: `{"organic_results":null}`},+ {name: "object results", body: `{"organic_results":{}}`},+ {name: "string results", body: `{"organic_results":"nope"}`},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if _, err := decodeSearchAPIWebSearchResults([]byte(tc.body)); err == nil {+ t.Fatal("error is nil")+ }+ })+ }+}++// A query with no organic hits still answers with the metadata sections, just+// without organic_results.+func TestDecodeSearchAPIWebSearchResultsAcceptsMissingResults(t *testing.T) {+ results, err := decodeSearchAPIWebSearchResults([]byte(`{"search_metadata": {"status": "Success"}, "search_information": {"total_results": 0}}`))+ if err != nil {+ t.Fatalf("decode: %v", err)+ }+ if len(results) != 0 {+ t.Fatalf("results = %#v, want empty", results)+ }+}+// A hit with no text is dropped rather than shipped as an empty citation.func TestWebSearchPayloadSkipsHitsWithoutContent(t *testing.T) {payload := webSearchPayload("exa", []webSearchHit{diff --git a/web/src/assets/searchapi.png b/web/src/assets/searchapi.pngnew file mode 100644index 0000000000..725bb0c33fBinary files /dev/null and b/web/src/assets/searchapi.png differdiff --git a/web/src/components/web-search-form-field.tsx b/web/src/components/web-search-form-field.tsxindex 276efe1d5d..40314d274b 100644--- a/web/src/components/web-search-form-field.tsx+++ b/web/src/components/web-search-form-field.tsx@@ -21,6 +21,7 @@ import linkupLogo from '@/assets/linkup.png';import parallelLogo from '@/assets/svg/parallel.svg';import queritLogo from '@/assets/querit.png';import search1apiLogo from '@/assets/search1api.png';+import searchapiLogo from '@/assets/searchapi.png';import serplyLogo from '@/assets/serply.png';import tavilyLogo from '@/assets/svg/tavily.svg';import youcomLogo from '@/assets/svg/youcom.svg';@@ -75,6 +76,11 @@ const webSearchProviderCatalog = [logo: search1apiLogo,value: WebSearchProvider.Search1API,},+ {+ name: 'SearchApi',+ logo: searchapiLogo,+ value: WebSearchProvider.SearchApi,+ },{name: 'Serply',logo: serplyLogo,@@ -160,6 +166,12 @@ const providerKeyConfig = {placeholder: 'search1apiApiKeyMessage',helpUrl: 'https://app.s1.dev',},+ [WebSearchProvider.SearchApi]: {+ name: 'prompt_config.searchapi_api_key',+ tip: 'searchapiApiKeyTip',+ placeholder: 'searchapiApiKeyMessage',+ helpUrl: 'https://www.searchapi.io',+ },[WebSearchProvider.Serply]: {name: 'prompt_config.serply_api_key',tip: 'serplyApiKeyTip',diff --git a/web/src/constants/chat.ts b/web/src/constants/chat.tsindex e809028aed..bc0c40ee60 100644--- a/web/src/constants/chat.ts+++ b/web/src/constants/chat.ts@@ -64,6 +64,7 @@ export enum WebSearchProvider {Parallel = 'parallel',Querit = 'querit',Search1API = 'search1api',+ SearchApi = 'searchapi',Serply = 'serply',Tavily = 'tavily',YouCom = 'youcom',diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.tsindex d7bfa389b0..a498446e5d 100644--- a/web/src/interfaces/database/chat.ts+++ b/web/src/interfaces/database/chat.ts@@ -52,6 +52,7 @@ export interface PromptConfig {parallel_api_key?: string;querit_api_key?: string;search1api_api_key?: string;+ searchapi_api_key?: string;serply_api_key?: string;tavily_api_key?: string;youcom_api_key?: string;diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex b8a43b6c3d..7b0dea0f20 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1305,6 +1305,9 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-ssearch1apiApiKeyTip:'When Search1API is selected, its web search results supplement dataset retrieval.',search1apiApiKeyMessage: 'Please enter your Search1API API Key',+ searchapiApiKeyTip:+ 'When SearchApi is selected, its Google web search results supplement dataset retrieval.',+ searchapiApiKeyMessage: 'Please enter your SearchApi API Key',serplyApiKeyTip:'When Serply is selected, its web search results supplement dataset retrieval.',serplyApiKeyMessage: 'Please enter your Serply API Key',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 3ef8c233a2..9f79b7f2a5 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1192,6 +1192,9 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 Rsearch1apiApiKeyTip:'选择 Search1API 后,将使用 Search1API 的网络搜索结果补充知识库检索。',search1apiApiKeyMessage: '请输入 Search1API API Key',+ searchapiApiKeyTip:+ '选择 SearchApi 后,将使用 SearchApi 的 Google 网络搜索结果补充知识库检索。',+ searchapiApiKeyMessage: '请输入 SearchApi API Key',serplyApiKeyTip:'选择 Serply 后,将使用 Serply 的网络搜索结果补充知识库检索。',serplyApiKeyMessage: '请输入 Serply API Key',diff --git a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsxindex aa4553f7ef..7b41fda552 100644--- a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx+++ b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx@@ -44,6 +44,7 @@ export function buildChatSettingFormSchema(t: (key: string) => string) {parallel_api_key: z.string().optional(),querit_api_key: z.string().optional(),search1api_api_key: z.string().optional(),+ searchapi_api_key: z.string().optional(),serply_api_key: z.string().optional(),tavily_api_key: z.string().optional(),youcom_api_key: z.string().optional(),@@ -56,6 +57,7 @@ export function buildChatSettingFormSchema(t: (key: string) => string) {WebSearchProvider.Parallel,WebSearchProvider.Querit,WebSearchProvider.Search1API,+ WebSearchProvider.SearchApi,WebSearchProvider.Serply,WebSearchProvider.Tavily,WebSearchProvider.YouCom,diff --git a/web/src/pages/next-chats/chat/web-search-api-key.test.ts b/web/src/pages/next-chats/chat/web-search-api-key.test.tsindex 3d6aee6a5c..41456c819b 100644--- a/web/src/pages/next-chats/chat/web-search-api-key.test.ts+++ b/web/src/pages/next-chats/chat/web-search-api-key.test.ts@@ -61,6 +61,25 @@ describe('getWebSearchApiKey', () => {expect(getWebSearchApiKey(promptConfig)).toBeUndefined();});+ it('uses only the selected SearchApi key', () => {+ const promptConfig = {+ web_search_provider: WebSearchProvider.SearchApi,+ searchapi_api_key: 'searchapi-test',+ tavily_api_key: 'tvly-test',+ } as PromptConfig;++ expect(getWebSearchApiKey(promptConfig)).toBe('searchapi-test');+ });++ it('does not fall back to Tavily when SearchApi is selected without a key', () => {+ const promptConfig = {+ web_search_provider: WebSearchProvider.SearchApi,+ tavily_api_key: 'tvly-test',+ } as PromptConfig;++ expect(getWebSearchApiKey(promptConfig)).toBeUndefined();+ });+it('uses only the selected Serply key', () => {const promptConfig = {web_search_provider: WebSearchProvider.Serply,@@ -249,6 +268,9 @@ describe('provider key field mapping', () => {expect(getWebSearchApiKeyField(WebSearchProvider.Search1API)).toBe('search1api_api_key',);+ expect(getWebSearchApiKeyField(WebSearchProvider.SearchApi)).toBe(+ 'searchapi_api_key',+ );expect(getWebSearchApiKeyField(WebSearchProvider.Serply)).toBe('serply_api_key',);diff --git a/web/src/pages/next-chats/chat/web-search-api-key.ts b/web/src/pages/next-chats/chat/web-search-api-key.tsindex 5da795b044..99016469f2 100644--- a/web/src/pages/next-chats/chat/web-search-api-key.ts+++ b/web/src/pages/next-chats/chat/web-search-api-key.ts@@ -60,6 +60,7 @@ const webSearchApiKeyFields: Record<WebSearchProvider, string> = {[WebSearchProvider.Parallel]: 'parallel_api_key',[WebSearchProvider.Querit]: 'querit_api_key',[WebSearchProvider.Search1API]: 'search1api_api_key',+ [WebSearchProvider.SearchApi]: 'searchapi_api_key',[WebSearchProvider.Serply]: 'serply_api_key',[WebSearchProvider.Tavily]: 'tavily_api_key',[WebSearchProvider.YouCom]: 'youcom_api_key',