Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔍 feat: Add Serper as Scraper Provider and Firecrawl Version Support (#9984)
Danny Avila31a283a4fe8f
diff --git a/api/package.json b/api/package.jsonindex b856fc0925..01c931a652 100644--- a/api/package.json+++ b/api/package.json@@ -48,7 +48,7 @@"@langchain/google-genai": "^0.2.13","@langchain/google-vertexai": "^0.2.13","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.4.83",+ "@librechat/agents": "^2.4.84","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",diff --git a/api/server/routes/config.js b/api/server/routes/config.jsindex ec38593fe8..8674769643 100644--- a/api/server/routes/config.js+++ b/api/server/routes/config.js@@ -156,7 +156,7 @@ router.get('/', async function (req, res) {if (webSearchConfig != null &&(webSearchConfig.searchProvider ||- webSearchConfig.scraperType ||+ webSearchConfig.scraperProvider ||webSearchConfig.rerankerType)) {payload.webSearch = {};@@ -165,8 +165,8 @@ router.get('/', async function (req, res) {if (webSearchConfig?.searchProvider) {payload.webSearch.searchProvider = webSearchConfig.searchProvider;}- if (webSearchConfig?.scraperType) {- payload.webSearch.scraperType = webSearchConfig.scraperType;+ if (webSearchConfig?.scraperProvider) {+ payload.webSearch.scraperProvider = webSearchConfig.scraperProvider;}if (webSearchConfig?.rerankerType) {payload.webSearch.rerankerType = webSearchConfig.rerankerType;diff --git a/api/typedefs.js b/api/typedefs.jsindex 260eb84bfe..a51e0ebaff 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -1054,7 +1054,7 @@/*** @exports TWebSearchKeys- * @typedef {import('librechat-data-provider').TWebSearchKeys} TWebSearchKeys+ * @typedef {import('@librechat/data-schemas').TWebSearchKeys} TWebSearchKeys* @memberof typedefs*/diff --git a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsxindex 84d0a30a9b..63b7e9af7b 100644--- a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx+++ b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx@@ -2,10 +2,10 @@ import { useState } from 'react';import { Button, OGDialog, OGDialogTemplate } from '@librechat/client';import {AuthType,- SearchCategories,RerankerTypes,SearchProviders,- ScraperTypes,+ ScraperProviders,+ SearchCategories,} from 'librechat-data-provider';import type { SearchApiKeyFormData } from '~/hooks/Plugins/useAuthSearchTool';import type { UseFormRegister, UseFormHandleSubmit } from 'react-hook-form';@@ -45,7 +45,9 @@ export default function ApiKeyDialog({const [selectedReranker, setSelectedReranker] = useState(config?.webSearch?.rerankerType || RerankerTypes.JINA,);- const [selectedScraper, setSelectedScraper] = useState(ScraperTypes.FIRECRAWL);+ const [selectedScraper, setSelectedScraper] = useState(+ config?.webSearch?.scraperProvider || ScraperProviders.FIRECRAWL,+ );const providerOptions: DropdownOption[] = [{@@ -119,7 +121,7 @@ export default function ApiKeyDialog({const scraperOptions: DropdownOption[] = [{- key: ScraperTypes.FIRECRAWL,+ key: ScraperProviders.FIRECRAWL,label: localize('com_ui_web_search_scraper_firecrawl'),inputs: {firecrawlApiUrl: {@@ -136,6 +138,20 @@ export default function ApiKeyDialog({},},},+ {+ key: ScraperProviders.SERPER,+ label: localize('com_ui_web_search_scraper_serper'),+ inputs: {+ serperApiKey: {+ placeholder: localize('com_ui_enter_api_key'),+ type: 'password' as const,+ link: {+ url: 'https://serper.dev/api-keys',+ text: localize('com_ui_web_search_scraper_serper_key'),+ },+ },+ },+ },];const [dropdownOpen, setDropdownOpen] = useState({@@ -157,7 +173,7 @@ export default function ApiKeyDialog({};const handleScraperChange = (key: string) => {- setSelectedScraper(key as ScraperTypes);+ setSelectedScraper(key as ScraperProviders);};return (@@ -198,7 +214,7 @@ export default function ApiKeyDialog({selectedKey={selectedScraper}onSelectionChange={handleScraperChange}dropdownOptions={scraperOptions}- showDropdown={!config?.webSearch?.scraperType}+ showDropdown={!config?.webSearch?.scraperProvider}register={register}dropdownOpen={dropdownOpen.scraper}setDropdownOpen={(open) =>diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.jsonindex b1bc9ba96e..0facdb493a 100644--- a/client/src/locales/en/translation.json+++ b/client/src/locales/en/translation.json@@ -1268,6 +1268,8 @@"com_ui_web_search_scraper": "Scraper","com_ui_web_search_scraper_firecrawl": "Firecrawl API","com_ui_web_search_scraper_firecrawl_key": "Get your Firecrawl API key",+ "com_ui_web_search_scraper_serper": "Serper Scrape API",+ "com_ui_web_search_scraper_serper_key": "Get your Serper API key","com_ui_web_search_searxng_api_key": "Enter SearXNG API Key (optional)","com_ui_web_search_searxng_instance_url": "SearXNG Instance URL","com_ui_web_searching": "Searching the web",diff --git a/package-lock.json b/package-lock.jsonindex 06ad223183..019317a2ea 100644--- a/package-lock.json+++ b/package-lock.json@@ -64,7 +64,7 @@"@langchain/google-genai": "^0.2.13","@langchain/google-vertexai": "^0.2.13","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.4.83",+ "@librechat/agents": "^2.4.84","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",@@ -21522,9 +21522,9 @@}},"node_modules/@librechat/agents": {- "version": "2.4.83",- "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.4.83.tgz",- "integrity": "sha512-xOspD4jegd7wpjWQhOieOso2LrXsNRyHNYEIIuCuk2eUZkxJU+1Rny0XzukhenTOG8P4bpne9XoVxxxZ0W5duA==",+ "version": "2.4.84",+ "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.4.84.tgz",+ "integrity": "sha512-wOPqv5yQfhkuBZ29FrJGUdDMCIvcnqUAigFeoPU8QOeqi+S9rRobx2+2D3+JbbgSsDL5yO7SyxGEHkQ7A6xZDQ==","license": "MIT","dependencies": {"@langchain/anthropic": "^0.3.26",@@ -51336,7 +51336,7 @@"@azure/storage-blob": "^12.27.0","@keyv/redis": "^4.3.3","@langchain/core": "^0.3.62",- "@librechat/agents": "^2.4.83",+ "@librechat/agents": "^2.4.84","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.17.1","axios": "^1.12.1",diff --git a/package.json b/package.jsonindex 693de111de..458f4d2985 100644--- a/package.json+++ b/package.json@@ -42,6 +42,7 @@"build:api": "cd packages/api && npm run build","build:data-schemas": "cd packages/data-schemas && npm run build","build:client-package": "cd packages/client && npm run build",+ "build:packages": "npm run build:data-provider && npm run build:data-schemas && npm run build:api && npm run build:client-package","frontend": "npm run build:data-provider && npm run build:data-schemas && npm run build:api && npm run build:client-package && cd client && npm run build","frontend:ci": "npm run build:data-provider && npm run build:client-package && cd client && npm run build:ci","frontend:dev": "cd client && npm run dev",diff --git a/packages/api/package.json b/packages/api/package.jsonindex a983aca87a..3ea7d343e4 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -80,7 +80,7 @@"@azure/storage-blob": "^12.27.0","@keyv/redis": "^4.3.3","@langchain/core": "^0.3.62",- "@librechat/agents": "^2.4.83",+ "@librechat/agents": "^2.4.84","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.17.1","axios": "^1.12.1",diff --git a/packages/api/src/web/web.spec.ts b/packages/api/src/web/web.spec.tsindex e0c66047c1..c7bb3f4962 100644--- a/packages/api/src/web/web.spec.ts+++ b/packages/api/src/web/web.spec.ts@@ -1,11 +1,11 @@import { webSearchAuth } from '@librechat/data-schemas';import { SafeSearchTypes, AuthType } from 'librechat-data-provider';import type {- ScraperTypes,+ ScraperProviders,+ TWebSearchConfig,+ SearchProviders,TCustomConfig,RerankerTypes,- SearchProviders,- TWebSearchConfig,} from 'librechat-data-provider';import { loadWebSearchAuth, extractWebSearchEnvVars } from './web';@@ -119,7 +119,7 @@ describe('web.ts', () => {}expect(result.authResult).toHaveProperty('searchProvider', 'serper');- expect(result.authResult).toHaveProperty('scraperType', 'firecrawl');+ expect(result.authResult).toHaveProperty('scraperProvider', 'firecrawl');expect(['jina', 'cohere']).toContain(result.authResult.rerankerType as string);});@@ -288,7 +288,7 @@ describe('web.ts', () => {// Check that the correct service types are setexpect(result.authResult.searchProvider).toBe('serper' as SearchProviders);- expect(result.authResult.scraperType).toBe('firecrawl' as ScraperTypes);+ expect(result.authResult.scraperProvider).toBe('firecrawl' as ScraperProviders);// One of the rerankers should be setexpect(['jina', 'cohere']).toContain(result.authResult.rerankerType as string);});@@ -330,7 +330,7 @@ describe('web.ts', () => {// Should have set values for all categoriesexpect(result.authResult.searchProvider).toBeDefined();- expect(result.authResult.scraperType).toBeDefined();+ expect(result.authResult.scraperProvider).toBeDefined();expect(result.authResult.rerankerType).toBeDefined();});@@ -359,7 +359,7 @@ describe('web.ts', () => {safeSearch: SafeSearchTypes.MODERATE,// Specify which services to usesearchProvider: 'serper' as SearchProviders,- scraperType: 'firecrawl' as ScraperTypes,+ scraperProvider: 'firecrawl' as ScraperProviders,rerankerType: 'jina' as RerankerTypes,};@@ -394,7 +394,7 @@ describe('web.ts', () => {expect(result.authResult).toHaveProperty('firecrawlApiUrl');expect(result.authResult).toHaveProperty('jinaApiKey');expect(result.authResult).toHaveProperty('searchProvider');- expect(result.authResult).toHaveProperty('scraperType');+ expect(result.authResult).toHaveProperty('scraperProvider');expect(result.authResult).toHaveProperty('rerankerType');expect(result.authenticated).toBe(true);@@ -419,7 +419,7 @@ describe('web.ts', () => {expect(result.authResult).toHaveProperty('firecrawlApiUrl', 'https://api.firecrawl.dev');expect(result.authResult).toHaveProperty('jinaApiKey', 'system-jina-key');expect(result.authResult).toHaveProperty('searchProvider', 'serper');- expect(result.authResult).toHaveProperty('scraperType', 'firecrawl');+ expect(result.authResult).toHaveProperty('scraperProvider', 'firecrawl');expect(result.authResult).toHaveProperty('rerankerType', 'jina');// Restore original env@@ -452,7 +452,7 @@ describe('web.ts', () => {safeSearch: SafeSearchTypes.MODERATE,// Specify which services to usesearchProvider: 'serper' as SearchProviders,- scraperType: 'firecrawl' as ScraperTypes,+ scraperProvider: 'firecrawl' as ScraperProviders,rerankerType: 'jina' as RerankerTypes, // Only Jina will be checked};@@ -492,7 +492,7 @@ describe('web.ts', () => {// Verify the service types are set correctlyexpect(result.authResult).toHaveProperty('searchProvider', 'serper');- expect(result.authResult).toHaveProperty('scraperType', 'firecrawl');+ expect(result.authResult).toHaveProperty('scraperProvider', 'firecrawl');expect(result.authResult).toHaveProperty('rerankerType', 'jina');// Restore original env@@ -722,8 +722,8 @@ describe('web.ts', () => {expect(providerCalls.length).toBe(1);});- it('should only check the specified scraperType', async () => {- // Initialize a webSearchConfig with a specific scraperType+ it('should only check the specified scraperProvider', async () => {+ // Initialize a webSearchConfig with a specific scraperProviderconst webSearchConfig: TCustomConfig['webSearch'] = {serperApiKey: '${SERPER_API_KEY}',searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}',@@ -734,7 +734,7 @@ describe('web.ts', () => {jinaApiUrl: '${JINA_API_URL}',cohereApiKey: '${COHERE_API_KEY}',safeSearch: SafeSearchTypes.MODERATE,- scraperType: 'firecrawl' as ScraperTypes,+ scraperProvider: 'firecrawl' as ScraperProviders,};// Mock successful authentication@@ -754,7 +754,7 @@ describe('web.ts', () => {});expect(result.authenticated).toBe(true);- expect(result.authResult.scraperType).toBe('firecrawl');+ expect(result.authResult.scraperProvider).toBe('firecrawl');// Verify that only FIRECRAWL_API_KEY and FIRECRAWL_API_URL were requested for the scrapers categoryconst scraperCalls = mockLoadAuthValues.mock.calls.filter((call) =>@@ -933,7 +933,7 @@ describe('web.ts', () => {// Should have set values for all categoriesexpect(result.authResult.searchProvider).toBeDefined();- expect(result.authResult.scraperType).toBeDefined();+ expect(result.authResult.scraperProvider).toBeDefined();expect(result.authResult.rerankerType).toBeDefined();});diff --git a/packages/api/src/web/web.ts b/packages/api/src/web/web.tsindex da34114bc5..ad172e187f 100644--- a/packages/api/src/web/web.ts+++ b/packages/api/src/web/web.ts@@ -4,14 +4,14 @@ import {SearchCategories,extractVariableName,} from 'librechat-data-provider';+import { webSearchAuth } from '@librechat/data-schemas';import type {- ScraperTypes,RerankerTypes,TCustomConfig,SearchProviders,+ ScraperProviders,TWebSearchConfig,} from 'librechat-data-provider';-import { webSearchAuth } from '@librechat/data-schemas';import type { TWebSearchKeys, TWebSearchCategories } from '@librechat/data-schemas';export function extractWebSearchEnvVars({@@ -88,8 +88,8 @@ export async function loadWebSearchAuth({let specificService: ServiceType | undefined;if (category === SearchCategories.PROVIDERS && webSearchConfig?.searchProvider) {specificService = webSearchConfig.searchProvider as unknown as ServiceType;- } else if (category === SearchCategories.SCRAPERS && webSearchConfig?.scraperType) {- specificService = webSearchConfig.scraperType as unknown as ServiceType;+ } else if (category === SearchCategories.SCRAPERS && webSearchConfig?.scraperProvider) {+ specificService = webSearchConfig.scraperProvider as unknown as ServiceType;} else if (category === SearchCategories.RERANKERS && webSearchConfig?.rerankerType) {specificService = webSearchConfig.rerankerType as unknown as ServiceType;}@@ -165,7 +165,7 @@ export async function loadWebSearchAuth({if (category === SearchCategories.PROVIDERS) {authResult.searchProvider = service as SearchProviders;} else if (category === SearchCategories.SCRAPERS) {- authResult.scraperType = service as ScraperTypes;+ authResult.scraperProvider = service as ScraperProviders;} else if (category === SearchCategories.RERANKERS) {authResult.rerankerType = service as RerankerTypes;}diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 4e318dc709..d212f559fe 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -650,7 +650,7 @@ export type TStartupConfig = {minPasswordLength?: number;webSearch?: {searchProvider?: SearchProviders;- scraperType?: ScraperTypes;+ scraperProvider?: ScraperProviders;rerankerType?: RerankerTypes;};mcpServers?: Record<@@ -689,7 +689,7 @@ export enum SearchProviders {SEARXNG = 'searxng',}-export enum ScraperTypes {+export enum ScraperProviders {FIRECRAWL = 'firecrawl',SERPER = 'serper',}@@ -711,11 +711,12 @@ export const webSearchSchema = z.object({searxngApiKey: z.string().optional().default('${SEARXNG_API_KEY}'),firecrawlApiKey: z.string().optional().default('${FIRECRAWL_API_KEY}'),firecrawlApiUrl: z.string().optional().default('${FIRECRAWL_API_URL}'),+ firecrawlVersion: z.string().optional().default('${FIRECRAWL_VERSION}'),jinaApiKey: z.string().optional().default('${JINA_API_KEY}'),jinaApiUrl: z.string().optional().default('${JINA_API_URL}'),cohereApiKey: z.string().optional().default('${COHERE_API_KEY}'),searchProvider: z.nativeEnum(SearchProviders).optional(),- scraperType: z.nativeEnum(ScraperTypes).optional(),+ scraperProvider: z.nativeEnum(ScraperProviders).optional(),rerankerType: z.nativeEnum(RerankerTypes).optional(),scraperTimeout: z.number().optional(),safeSearch: z.nativeEnum(SafeSearchTypes).default(SafeSearchTypes.MODERATE),diff --git a/packages/data-schemas/src/app/web.spec.ts b/packages/data-schemas/src/app/web.spec.tsnew file mode 100644index 0000000000..787d7809a3--- /dev/null+++ b/packages/data-schemas/src/app/web.spec.ts@@ -0,0 +1,173 @@+import { SafeSearchTypes, SearchProviders, ScraperProviders } from 'librechat-data-provider';+import type { TCustomConfig } from 'librechat-data-provider';+import { loadWebSearchConfig } from './web';++describe('loadWebSearchConfig', () => {+ describe('firecrawlVersion', () => {+ it('should use provided firecrawlVersion when specified', () => {+ const config: TCustomConfig['webSearch'] = {+ firecrawlVersion: 'v2',+ };++ const result = loadWebSearchConfig(config);++ expect(result?.firecrawlVersion).toBe('v2');+ });++ it('should default to ${FIRECRAWL_VERSION} when not provided', () => {+ const config: TCustomConfig['webSearch'] = {};++ const result = loadWebSearchConfig(config);++ expect(result?.firecrawlVersion).toBe('${FIRECRAWL_VERSION}');+ });++ it('should default to ${FIRECRAWL_VERSION} when config is undefined', () => {+ const result = loadWebSearchConfig(undefined);++ expect(result?.firecrawlVersion).toBe('${FIRECRAWL_VERSION}');+ });++ it('should preserve custom firecrawlVersion value', () => {+ const config: TCustomConfig['webSearch'] = {+ firecrawlVersion: 'v1',+ };++ const result = loadWebSearchConfig(config);++ expect(result?.firecrawlVersion).toBe('v1');+ });+ });++ describe('all config fields', () => {+ it('should apply defaults for all fields when config is empty', () => {+ const config: TCustomConfig['webSearch'] = {};++ const result = loadWebSearchConfig(config);++ expect(result).toEqual({+ serperApiKey: '${SERPER_API_KEY}',+ searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}',+ searxngApiKey: '${SEARXNG_API_KEY}',+ firecrawlApiKey: '${FIRECRAWL_API_KEY}',+ firecrawlApiUrl: '${FIRECRAWL_API_URL}',+ firecrawlVersion: '${FIRECRAWL_VERSION}',+ jinaApiKey: '${JINA_API_KEY}',+ jinaApiUrl: '${JINA_API_URL}',+ cohereApiKey: '${COHERE_API_KEY}',+ safeSearch: SafeSearchTypes.MODERATE,+ });+ });++ it('should preserve provided config values and merge with defaults', () => {+ const config: TCustomConfig['webSearch'] = {+ serperApiKey: 'custom-serper-key',+ firecrawlApiKey: 'custom-firecrawl-key',+ firecrawlVersion: 'v2',+ safeSearch: SafeSearchTypes.STRICT,+ };++ const result = loadWebSearchConfig(config);++ expect(result?.serperApiKey).toBe('custom-serper-key');+ expect(result?.firecrawlApiKey).toBe('custom-firecrawl-key');+ expect(result?.firecrawlVersion).toBe('v2');+ expect(result?.safeSearch).toBe(SafeSearchTypes.STRICT);+ expect(result?.jinaApiKey).toBe('${JINA_API_KEY}');+ });++ it('should preserve additional fields from input config', () => {+ const config: TCustomConfig['webSearch'] = {+ serperApiKey: 'test-key',+ scraperProvider: ScraperProviders.SERPER,+ searchProvider: SearchProviders.SERPER,+ };++ const result = loadWebSearchConfig(config);++ expect(result?.scraperProvider).toBe('serper');+ expect(result?.searchProvider).toBe('serper');+ expect(result?.serperApiKey).toBe('test-key');+ });+ });++ describe('safeSearch', () => {+ it('should default to MODERATE when not provided', () => {+ const config: TCustomConfig['webSearch'] = {};++ const result = loadWebSearchConfig(config);++ expect(result?.safeSearch).toBe(SafeSearchTypes.MODERATE);+ });++ it('should preserve OFF value', () => {+ const config: TCustomConfig['webSearch'] = {+ safeSearch: SafeSearchTypes.OFF,+ };++ const result = loadWebSearchConfig(config);++ expect(result?.safeSearch).toBe(SafeSearchTypes.OFF);+ });++ it('should preserve STRICT value', () => {+ const config: TCustomConfig['webSearch'] = {+ safeSearch: SafeSearchTypes.STRICT,+ };++ const result = loadWebSearchConfig(config);++ expect(result?.safeSearch).toBe(SafeSearchTypes.STRICT);+ });+ });++ describe('API keys', () => {+ it('should apply default placeholders for all API keys', () => {+ const result = loadWebSearchConfig({});++ expect(result?.serperApiKey).toBe('${SERPER_API_KEY}');+ expect(result?.searxngApiKey).toBe('${SEARXNG_API_KEY}');+ expect(result?.firecrawlApiKey).toBe('${FIRECRAWL_API_KEY}');+ expect(result?.jinaApiKey).toBe('${JINA_API_KEY}');+ expect(result?.cohereApiKey).toBe('${COHERE_API_KEY}');+ });++ it('should preserve custom API keys', () => {+ const config: TCustomConfig['webSearch'] = {+ serperApiKey: 'actual-serper-key',+ jinaApiKey: 'actual-jina-key',+ cohereApiKey: 'actual-cohere-key',+ };++ const result = loadWebSearchConfig(config);++ expect(result?.serperApiKey).toBe('actual-serper-key');+ expect(result?.jinaApiKey).toBe('actual-jina-key');+ expect(result?.cohereApiKey).toBe('actual-cohere-key');+ });+ });++ describe('URLs', () => {+ it('should apply default placeholders for URLs', () => {+ const result = loadWebSearchConfig({});++ expect(result?.searxngInstanceUrl).toBe('${SEARXNG_INSTANCE_URL}');+ expect(result?.firecrawlApiUrl).toBe('${FIRECRAWL_API_URL}');+ expect(result?.jinaApiUrl).toBe('${JINA_API_URL}');+ });++ it('should preserve custom URLs', () => {+ const config: TCustomConfig['webSearch'] = {+ searxngInstanceUrl: 'https://custom-searxng.com',+ firecrawlApiUrl: 'https://custom-firecrawl.com',+ jinaApiUrl: 'https://custom-jina.com',+ };++ const result = loadWebSearchConfig(config);++ expect(result?.searxngInstanceUrl).toBe('https://custom-searxng.com');+ expect(result?.firecrawlApiUrl).toBe('https://custom-firecrawl.com');+ expect(result?.jinaApiUrl).toBe('https://custom-jina.com');+ });+ });+});diff --git a/packages/data-schemas/src/app/web.ts b/packages/data-schemas/src/app/web.tsindex 5fbf2c674e..a61e1f1611 100644--- a/packages/data-schemas/src/app/web.ts+++ b/packages/data-schemas/src/app/web.ts@@ -18,6 +18,10 @@ export const webSearchAuth = {firecrawlApiKey: 1 as const,/** Optional (0) */firecrawlApiUrl: 0 as const,+ firecrawlVersion: 0 as const,+ },+ serper: {+ serperApiKey: 1 as const,},},rerankers: {@@ -31,10 +35,10 @@ export const webSearchAuth = {};/**- * Extracts all API keys from the webSearchAuth configuration object+ * Extracts all unique API keys from the webSearchAuth configuration object*/export function getWebSearchKeys(): TWebSearchKeys[] {- const keys: TWebSearchKeys[] = [];+ const keysSet = new Set<TWebSearchKeys>();// Iterate through each category (providers, scrapers, rerankers)for (const category of Object.keys(webSearchAuth)) {@@ -44,14 +48,14 @@ export function getWebSearchKeys(): TWebSearchKeys[] {for (const service of Object.keys(categoryObj)) {const serviceObj = categoryObj[service as keyof typeof categoryObj];- // Extract the API keys from the service+ // Extract the API keys from the service and add to set for deduplicationfor (const key of Object.keys(serviceObj)) {- keys.push(key as TWebSearchKeys);+ keysSet.add(key as TWebSearchKeys);}}}- return keys;+ return Array.from(keysSet);}export const webSearchKeys: TWebSearchKeys[] = getWebSearchKeys();@@ -64,6 +68,7 @@ export function loadWebSearchConfig(const searxngApiKey = config?.searxngApiKey ?? '${SEARXNG_API_KEY}';const firecrawlApiKey = config?.firecrawlApiKey ?? '${FIRECRAWL_API_KEY}';const firecrawlApiUrl = config?.firecrawlApiUrl ?? '${FIRECRAWL_API_URL}';+ const firecrawlVersion = config?.firecrawlVersion ?? '${FIRECRAWL_VERSION}';const jinaApiKey = config?.jinaApiKey ?? '${JINA_API_KEY}';const jinaApiUrl = config?.jinaApiUrl ?? '${JINA_API_URL}';const cohereApiKey = config?.cohereApiKey ?? '${COHERE_API_KEY}';@@ -76,9 +81,10 @@ export function loadWebSearchConfig(jinaApiUrl,cohereApiKey,serperApiKey,- searxngInstanceUrl,searxngApiKey,firecrawlApiKey,firecrawlApiUrl,+ firecrawlVersion,+ searxngInstanceUrl,};}diff --git a/packages/data-schemas/src/types/web.ts b/packages/data-schemas/src/types/web.tsindex c424be48d2..a9cc1f0cc6 100644--- a/packages/data-schemas/src/types/web.ts+++ b/packages/data-schemas/src/types/web.ts@@ -6,6 +6,7 @@ export type TWebSearchKeys =| 'searxngApiKey'| 'firecrawlApiKey'| 'firecrawlApiUrl'+ | 'firecrawlVersion'| 'jinaApiKey'| 'jinaApiUrl'| 'cohereApiKey';