Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(chat): add SearchApi web search provider (#21563)

Gabs Ferreirabc9d7accd93c
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md
index 637971cb94..02e8476eb5 100644
--- a/docs/references/http_api_reference.md
+++ b/docs/references/http_api_reference.md
@@ -2647,7 +2647,7 @@ curl --request POST \
- `"refine_multiturn"`: `boolean`
- `"reasoning"`: `boolean`
- `"cross_languages"`: `list[string]`
- - `"web_search_provider"`: `string` The web search service to use. Supported values are `"brave"`, `"exa"`, `"firecrawl"`, `"linkup"`, `"parallel"`, `"querit"`, `"search1api"`, `"serply"`, `"tavily"`, and `"youcom"`. If omitted, Tavily is selected only when `"tavily_api_key"` is configured; otherwise web search is disabled.
+ - `"web_search_provider"`: `string` The web search service to use. Supported values are `"brave"`, `"exa"`, `"firecrawl"`, `"linkup"`, `"parallel"`, `"querit"`, `"search1api"`, `"searchapi"`, `"serply"`, `"tavily"`, and `"youcom"`. If omitted, Tavily is selected only when `"tavily_api_key"` is configured; otherwise web search is disabled.
- `"tavily_api_key"`: `string`
- `"brave_api_key"`: `string` The Brave Search API key. Set `web_search_provider` to `"brave"` when using this field.
- `"exa_api_key"`: `string` The Exa API key. Set `web_search_provider` to `"exa"` when using this field.
@@ -2656,6 +2656,7 @@ curl --request POST \
- `"parallel_api_key"`: `string` The Parallel API key. Set `web_search_provider` to `"parallel"` when using this field.
- `"querit_api_key"`: `string` The Querit API key. Set `web_search_provider` to `"querit"` when using this field.
- `"search1api_api_key"`: `string` The [Search1API](https://s1.dev) API key. Set `web_search_provider` to `"search1api"` when using this field. See the [Search1API documentation](https://s1.dev/docs/basic/search) for details.
+ - `"searchapi_api_key"`: `string` The [SearchApi](https://www.searchapi.io) API key. Set `web_search_provider` to `"searchapi"` when using this field. Results come from its Google engine. See the [SearchApi Google documentation](https://www.searchapi.io/docs/google) for details.
- `"serply_api_key"`: `string` The [Serply](https://serply.io) API key. Set `web_search_provider` to `"serply"` when using this field. See the [Serply documentation](https://serply.io/docs) for details.
- `"youcom_api_key"`: `string` The You.com API key. Set `web_search_provider` to `"youcom"` when using this field. Optional: You.com serves a rate-limited keyless endpoint, so `"youcom"` works with this field omitted, and a key lifts those limits.
- `"similarity_threshold"`: `float`
diff --git a/internal/service/web_search_provider.go b/internal/service/web_search_provider.go
index d2ebb87995..29248f99de 100644
--- a/internal/service/web_search_provider.go
+++ b/internal/service/web_search_provider.go
@@ -33,8 +33,8 @@ import (
)
// Providers, endpoints and clients are listed alphabetically by provider id
-// (brave, exa, firecrawl, linkup, parallel, querit, search1api, serply, tavily,
-// youcom) so a new provider has exactly one obvious place in each list.
+// (brave, exa, firecrawl, linkup, parallel, querit, search1api, searchapi, serply,
+// tavily, youcom) so a new provider has exactly one obvious place in each list.
// Tavily's endpoint lives with its retrieval code in chat_pipeline.go, which is
// why it has no entry here.
const (
@@ -45,6 +45,7 @@ const (
webSearchProviderParallel = "parallel"
webSearchProviderQuerit = "querit"
webSearchProviderSearch1API = "search1api"
+ webSearchProviderSearchAPI = "searchapi"
webSearchProviderSerply = "serply"
webSearchProviderTavily = "tavily"
webSearchProviderYouCom = "youcom"
@@ -57,6 +58,7 @@ const (
parallelWebSearchEndpoint = "https://api.parallel.ai/v1/search"
queritWebSearchEndpoint = "https://api.querit.ai/v1/search"
search1apiWebSearchEndpoint = "https://api.search1api.com/search"
+ searchapiWebSearchEndpoint = "https://www.searchapi.io/api/v1/search"
serplyWebSearchEndpoint = "https://api.serply.io/v1/search/"
// webSearchResultCount is how many hits every provider is asked for. Six
// keeps the web block in a prompt the size of one corpus chunk set — the
@@ -79,6 +81,7 @@ var (
parallelWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
queritWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
search1apiWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
+ searchapiWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
serplyWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
youComWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second}
// Tavily is reached from two call sites (the chat pipeline and the deep
@@ -132,6 +135,8 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchPro
apiKeyField = "querit_api_key"
case webSearchProviderSearch1API:
apiKeyField = "search1api_api_key"
+ case webSearchProviderSearchAPI:
+ apiKeyField = "searchapi_api_key"
case webSearchProviderSerply:
apiKeyField = "serply_api_key"
case webSearchProviderTavily:
@@ -245,6 +250,14 @@ func retrieveWebSearchWithTavily(
provider.APIKey,
question,
)
+ case webSearchProviderSearchAPI:
+ return retrieveSearchAPIWebSearch(
+ ctx,
+ searchapiWebSearchHTTPClient,
+ searchapiWebSearchEndpoint,
+ provider.APIKey,
+ question,
+ )
case webSearchProviderSerply:
return retrieveSerplyWebSearch(
ctx,
@@ -898,6 +911,78 @@ func decodeSearch1APIWebSearchResults(responseBody []byte) ([]search1apiWebSearc
return results, nil
}
+type searchapiWebSearchResult struct {
+ Title string `json:"title"`
+ Link string `json:"link"`
+ Snippet string `json:"snippet"`
+}
+
+func retrieveSearchAPIWebSearch(
+ ctx context.Context,
+ client *http.Client,
+ endpoint string,
+ apiKey string,
+ query string,
+) (map[string]interface{}, error) {
+ parameters := url.Values{}
+ // SearchApi fronts several engines behind one endpoint; google is the web
+ // search one, and the parameter is mandatory.
+ parameters.Set("engine", "google")
+ parameters.Set("q", query)
+ parameters.Set("num", strconv.Itoa(webSearchResultCount))
+
+ responseBody, err := webSearchRequest(ctx, client, http.MethodGet,
+ endpoint+"?"+parameters.Encode(), map[string]string{
+ "Accept": "application/json",
+ "Authorization": "Bearer " + apiKey,
+ "User-Agent": "ragflow-web-search",
+ }, nil)
+ if err != nil {
+ return nil, fmt.Errorf("searchapi: %w", err)
+ }
+ results, err := decodeSearchAPIWebSearchResults(responseBody)
+ if err != nil {
+ return nil, err
+ }
+
+ hits := make([]webSearchHit, 0, len(results))
+ for _, result := range results {
+ hits = append(hits, webSearchHit{
+ Title: result.Title,
+ URL: result.Link,
+ Content: result.Snippet,
+ })
+ }
+ return webSearchPayload("searchapi", hits), nil
+}
+
+// decodeSearchAPIWebSearchResults reads the organic hits. The response also
+// carries knowledge graph, answer box and ads sections; only organic_results is
+// a plain list of pages with a snippet the model can quote.
+func decodeSearchAPIWebSearchResults(responseBody []byte) ([]searchapiWebSearchResult, error) {
+ var envelope map[string]json.RawMessage
+ if err := json.Unmarshal(responseBody, &envelope); err != nil {
+ return nil, fmt.Errorf("searchapi: decode response: %w", err)
+ }
+ if envelope == nil {
+ return nil, fmt.Errorf("searchapi: response must be an object")
+ }
+
+ resultsValue, exists := envelope["organic_results"]
+ if !exists {
+ return []searchapiWebSearchResult{}, nil
+ }
+ if strings.TrimSpace(string(resultsValue)) == "null" {
+ return nil, fmt.Errorf("searchapi: response field organic_results must be an array")
+ }
+
+ var results []searchapiWebSearchResult
+ if err := json.Unmarshal(resultsValue, &results); err != nil {
+ return nil, fmt.Errorf("searchapi: response field organic_results must be an array: %w", err)
+ }
+ return results, nil
+}
+
type serplyWebSearchResult struct {
Title string `json:"title"`
Link string `json:"link"`
diff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.go
index 25545b88c9..847ae94eed 100644
--- a/internal/service/web_search_provider_test.go
+++ b/internal/service/web_search_provider_test.go
@@ -626,6 +626,7 @@ func TestResolveWebSearchProviderSelectsKeyedProviders(t *testing.T) {
{provider: "linkup", apiKeyName: "linkup_api_key", apiKey: "linkup-test"},
{provider: "parallel", apiKeyName: "parallel_api_key", apiKey: "parallel-test"},
{provider: "search1api", apiKeyName: "search1api_api_key", apiKey: "search1api-test"},
+ {provider: "searchapi", apiKeyName: "searchapi_api_key", apiKey: "searchapi-test"},
}
for _, tc := range cases {
@@ -946,6 +947,99 @@ func TestDecodeSearch1APIWebSearchResultsAcceptsMissingResults(t *testing.T) {
}
}
+func TestRetrieveSearchAPIWebSearchSendsBearerKeyAndUsesSnippets(t *testing.T) {
+ var requestQuery url.Values
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ t.Errorf("method = %q, want GET", r.Method)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer searchapi-test" {
+ t.Errorf("Authorization = %q, want %q", got, "Bearer searchapi-test")
+ }
+ if got := r.Header.Get("User-Agent"); got == "" {
+ t.Error("User-Agent is empty")
+ }
+ requestQuery = r.URL.Query()
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{
+ "search_metadata": {"id": "search_test", "status": "Success"},
+ "search_parameters": {"engine": "google", "q": "What is RAGFlow?", "num": 6},
+ "search_information": {"query_displayed": "What is RAGFlow?"},
+ "organic_results": [
+ {"position": 1, "title": "No snippet", "link": "https://example.com/empty", "snippet": " \t\n"},
+ {"position": 2, "title": "RAGFlow", "link": "https://example.com/ragflow", "source": "RAGFlow", "domain": "example.com", "snippet": "An open-source RAG engine."}
+ ],
+ "pagination": {"current": 1}
+ }`))
+ }))
+ defer server.Close()
+
+ result, err := retrieveSearchAPIWebSearch(t.Context(), server.Client(), server.URL, "searchapi-test", "What is RAGFlow?")
+ if err != nil {
+ t.Fatalf("retrieve SearchApi web search: %v", err)
+ }
+ if got := requestQuery.Get("engine"); got != "google" {
+ t.Fatalf("engine = %q, want %q", got, "google")
+ }
+ if got := requestQuery.Get("q"); got != "What is RAGFlow?" {
+ t.Fatalf("q = %q, want %q", got, "What is RAGFlow?")
+ }
+ if got := requestQuery.Get("num"); got != "6" {
+ t.Fatalf("num = %q, want 6", got)
+ }
+ // The key travels in the header only; SearchApi also accepts it as a query
+ // parameter, which would put it in access logs.
+ if requestQuery.Has("api_key") {
+ t.Fatal("api_key must not be sent as a query parameter")
+ }
+ assertWebSearchChunk(t, result, "searchapi-https://example.com/ragflow", "RAGFlow", "An open-source RAG engine.")
+}
+
+func TestRetrieveSearchAPIWebSearchRejectsErrorStatuses(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error": "Invalid API key."}`))
+ }))
+ defer server.Close()
+
+ if _, err := retrieveSearchAPIWebSearch(t.Context(), server.Client(), server.URL, "bad-key", "ragflow"); err == nil {
+ t.Fatal("error is nil")
+ }
+}
+
+func TestDecodeSearchAPIWebSearchResultsRejectsMalformedContainers(t *testing.T) {
+ cases := []struct {
+ name string
+ body string
+ }{
+ {name: "null response", body: `null`},
+ {name: "null results", body: `{"organic_results":null}`},
+ {name: "object results", body: `{"organic_results":{}}`},
+ {name: "string results", body: `{"organic_results":"nope"}`},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if _, err := decodeSearchAPIWebSearchResults([]byte(tc.body)); err == nil {
+ t.Fatal("error is nil")
+ }
+ })
+ }
+}
+
+// A query with no organic hits still answers with the metadata sections, just
+// without organic_results.
+func TestDecodeSearchAPIWebSearchResultsAcceptsMissingResults(t *testing.T) {
+ results, err := decodeSearchAPIWebSearchResults([]byte(`{"search_metadata": {"status": "Success"}, "search_information": {"total_results": 0}}`))
+ if err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if len(results) != 0 {
+ t.Fatalf("results = %#v, want empty", results)
+ }
+}
+
// A hit with no text is dropped rather than shipped as an empty citation.
func TestWebSearchPayloadSkipsHitsWithoutContent(t *testing.T) {
payload := webSearchPayload("exa", []webSearchHit{
diff --git a/web/src/assets/searchapi.png b/web/src/assets/searchapi.png
new file mode 100644
index 0000000000..725bb0c33f
Binary files /dev/null and b/web/src/assets/searchapi.png differ
diff --git a/web/src/components/web-search-form-field.tsx b/web/src/components/web-search-form-field.tsx
index 276efe1d5d..40314d274b 100644
--- a/web/src/components/web-search-form-field.tsx
+++ b/web/src/components/web-search-form-field.tsx
@@ -21,6 +21,7 @@ import linkupLogo from '@/assets/linkup.png';
import parallelLogo from '@/assets/svg/parallel.svg';
import queritLogo from '@/assets/querit.png';
import search1apiLogo from '@/assets/search1api.png';
+import searchapiLogo from '@/assets/searchapi.png';
import serplyLogo from '@/assets/serply.png';
import tavilyLogo from '@/assets/svg/tavily.svg';
import youcomLogo from '@/assets/svg/youcom.svg';
@@ -75,6 +76,11 @@ const webSearchProviderCatalog = [
logo: search1apiLogo,
value: WebSearchProvider.Search1API,
},
+ {
+ name: 'SearchApi',
+ logo: searchapiLogo,
+ value: WebSearchProvider.SearchApi,
+ },
{
name: 'Serply',
logo: serplyLogo,
@@ -160,6 +166,12 @@ const providerKeyConfig = {
placeholder: 'search1apiApiKeyMessage',
helpUrl: 'https://app.s1.dev',
},
+ [WebSearchProvider.SearchApi]: {
+ name: 'prompt_config.searchapi_api_key',
+ tip: 'searchapiApiKeyTip',
+ placeholder: 'searchapiApiKeyMessage',
+ helpUrl: 'https://www.searchapi.io',
+ },
[WebSearchProvider.Serply]: {
name: 'prompt_config.serply_api_key',
tip: 'serplyApiKeyTip',
diff --git a/web/src/constants/chat.ts b/web/src/constants/chat.ts
index e809028aed..bc0c40ee60 100644
--- a/web/src/constants/chat.ts
+++ b/web/src/constants/chat.ts
@@ -64,6 +64,7 @@ export enum WebSearchProvider {
Parallel = 'parallel',
Querit = 'querit',
Search1API = 'search1api',
+ SearchApi = 'searchapi',
Serply = 'serply',
Tavily = 'tavily',
YouCom = 'youcom',
diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.ts
index d7bfa389b0..a498446e5d 100644
--- a/web/src/interfaces/database/chat.ts
+++ b/web/src/interfaces/database/chat.ts
@@ -52,6 +52,7 @@ export interface PromptConfig {
parallel_api_key?: string;
querit_api_key?: string;
search1api_api_key?: string;
+ searchapi_api_key?: string;
serply_api_key?: string;
tavily_api_key?: string;
youcom_api_key?: string;
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index b8a43b6c3d..7b0dea0f20 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1305,6 +1305,9 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
search1apiApiKeyTip:
'When Search1API is selected, its web search results supplement dataset retrieval.',
search1apiApiKeyMessage: 'Please enter your Search1API API Key',
+ searchapiApiKeyTip:
+ 'When SearchApi is selected, its Google web search results supplement dataset retrieval.',
+ searchapiApiKeyMessage: 'Please enter your SearchApi API Key',
serplyApiKeyTip:
'When Serply is selected, its web search results supplement dataset retrieval.',
serplyApiKeyMessage: 'Please enter your Serply API Key',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 3ef8c233a2..9f79b7f2a5 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1192,6 +1192,9 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
search1apiApiKeyTip:
'选择 Search1API 后,将使用 Search1API 的网络搜索结果补充知识库检索。',
search1apiApiKeyMessage: '请输入 Search1API API Key',
+ searchapiApiKeyTip:
+ '选择 SearchApi 后,将使用 SearchApi 的 Google 网络搜索结果补充知识库检索。',
+ searchapiApiKeyMessage: '请输入 SearchApi API Key',
serplyApiKeyTip:
'选择 Serply 后,将使用 Serply 的网络搜索结果补充知识库检索。',
serplyApiKeyMessage: '请输入 Serply API Key',
diff --git a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx
index aa4553f7ef..7b41fda552 100644
--- a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx
+++ b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx
@@ -44,6 +44,7 @@ export function buildChatSettingFormSchema(t: (key: string) => string) {
parallel_api_key: z.string().optional(),
querit_api_key: z.string().optional(),
search1api_api_key: z.string().optional(),
+ searchapi_api_key: z.string().optional(),
serply_api_key: z.string().optional(),
tavily_api_key: z.string().optional(),
youcom_api_key: z.string().optional(),
@@ -56,6 +57,7 @@ export function buildChatSettingFormSchema(t: (key: string) => string) {
WebSearchProvider.Parallel,
WebSearchProvider.Querit,
WebSearchProvider.Search1API,
+ WebSearchProvider.SearchApi,
WebSearchProvider.Serply,
WebSearchProvider.Tavily,
WebSearchProvider.YouCom,
diff --git a/web/src/pages/next-chats/chat/web-search-api-key.test.ts b/web/src/pages/next-chats/chat/web-search-api-key.test.ts
index 3d6aee6a5c..41456c819b 100644
--- a/web/src/pages/next-chats/chat/web-search-api-key.test.ts
+++ b/web/src/pages/next-chats/chat/web-search-api-key.test.ts
@@ -61,6 +61,25 @@ describe('getWebSearchApiKey', () => {
expect(getWebSearchApiKey(promptConfig)).toBeUndefined();
});
+ it('uses only the selected SearchApi key', () => {
+ const promptConfig = {
+ web_search_provider: WebSearchProvider.SearchApi,
+ searchapi_api_key: 'searchapi-test',
+ tavily_api_key: 'tvly-test',
+ } as PromptConfig;
+
+ expect(getWebSearchApiKey(promptConfig)).toBe('searchapi-test');
+ });
+
+ it('does not fall back to Tavily when SearchApi is selected without a key', () => {
+ const promptConfig = {
+ web_search_provider: WebSearchProvider.SearchApi,
+ tavily_api_key: 'tvly-test',
+ } as PromptConfig;
+
+ expect(getWebSearchApiKey(promptConfig)).toBeUndefined();
+ });
+
it('uses only the selected Serply key', () => {
const promptConfig = {
web_search_provider: WebSearchProvider.Serply,
@@ -249,6 +268,9 @@ describe('provider key field mapping', () => {
expect(getWebSearchApiKeyField(WebSearchProvider.Search1API)).toBe(
'search1api_api_key',
);
+ expect(getWebSearchApiKeyField(WebSearchProvider.SearchApi)).toBe(
+ 'searchapi_api_key',
+ );
expect(getWebSearchApiKeyField(WebSearchProvider.Serply)).toBe(
'serply_api_key',
);
diff --git a/web/src/pages/next-chats/chat/web-search-api-key.ts b/web/src/pages/next-chats/chat/web-search-api-key.ts
index 5da795b044..99016469f2 100644
--- a/web/src/pages/next-chats/chat/web-search-api-key.ts
+++ b/web/src/pages/next-chats/chat/web-search-api-key.ts
@@ -60,6 +60,7 @@ const webSearchApiKeyFields: Record<WebSearchProvider, string> = {
[WebSearchProvider.Parallel]: 'parallel_api_key',
[WebSearchProvider.Querit]: 'querit_api_key',
[WebSearchProvider.Search1API]: 'search1api_api_key',
+ [WebSearchProvider.SearchApi]: 'searchapi_api_key',
[WebSearchProvider.Serply]: 'serply_api_key',
[WebSearchProvider.Tavily]: 'tavily_api_key',
[WebSearchProvider.YouCom]: 'youcom_api_key',