Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: search formats fix
Nicolas166ce2c7d304
diff --git a/apps/api/requests/v2/search.requests.http b/apps/api/requests/v2/search.requests.httpindex c8f9f2acb..401aa28eb 100644--- a/apps/api/requests/v2/search.requests.http+++ b/apps/api/requests/v2/search.requests.http@@ -1,15 +1,35 @@# Pick your baseUrl here:-# @baseUrl = http://localhost:3002-@baseUrl = https://api.firecrawl.dev+@baseUrl = http://localhost:3002+# @baseUrl = https://api.firecrawl.dev-### Search-# @name search-POST {{baseUrl}}/v1/search HTTP/1.1+### Search - Simple string array format+# @name searchSimple+POST {{baseUrl}}/v2/search HTTP/1.1Authorization: Bearer {{$dotenv TEST_API_KEY}}content-type: application/json{"query": "firecrawl",- "limit": 50+ "sources": ["web", "images", "news"],+ "limit": 5+}++### Search - Object array format (with custom parameters per source)+# @name searchAdvanced+POST {{baseUrl}}/v2/search HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "query": "firecrawl",+ "sources": [+ {+ "type": "web"+ },+ {+ "type": "images"+ }+ ],+ "limit": 5}diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 2a628733a..90ddf4407 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -191,6 +191,10 @@ export async function searchController(logger.info("Searching for results");+ // Extract unique types from sources for the search function+ // After transformation, sources is always an array of objects+ const searchTypes = [...new Set(req.body.sources.map((s: any) => s.type))];+const searchResponse = await search({query: req.body.query,advanced: false,@@ -200,7 +204,7 @@ export async function searchController(lang: req.body.lang,country: req.body.country,location: req.body.location,- type: req.body.type,+ type: searchTypes,}) as SearchV2Response;// Apply URL filtering if neededdiff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 4d687863c..5d5031443 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1202,6 +1202,31 @@ export function toLegacyDocument(};}+// Search source type definitions+// These allow fine-grained control over each search source type+// Similar to how scrape formats work with jsonFormatWithOptions, etc.++export const webSearchSourceOptions = z.object({+ type: z.literal("web"),+ tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)+ filter: z.string().optional(), // Search filter+ lang: z.string().optional(), // Language override for this source+ country: z.string().optional(), // Country override for this source+ location: z.string().optional(), // Location override for this source+}).strict();++export const imagesSearchSourceOptions = z.object({+ type: z.literal("images"),+}).strict();++export const newsSearchSourceOptions = z.object({+ type: z.literal("news"),+}).strict();++export type WebSearchSourceOptions = z.infer<typeof webSearchSourceOptions>;+export type ImagesSearchSourceOptions = z.infer<typeof imagesSearchSourceOptions>;+export type NewsSearchSourceOptions = z.infer<typeof newsSearchSourceOptions>;+export const searchRequestSchema = z.object({query: z.string(),@@ -1216,10 +1241,21 @@ export const searchRequestSchema = z.default(5),tbs: z.string().optional(),filter: z.string().optional(),- type: z.union([- z.enum(["web", "images", "news"]),- z.array(z.enum(["web", "images", "news"]))- ]).optional().default("web"),+ sources: z+ .union([+ // Array of strings (simple format)+ z.array(z.enum(["web", "images", "news"])),+ // Array of objects (advanced format)+ z.array(+ z.union([+ webSearchSourceOptions,+ imagesSearchSourceOptions,+ newsSearchSourceOptions,+ ])+ ),+ ])+ .optional()+ .default(["web"]),lang: z.string().optional().default("en"),country: z.string().optional().default("us"),location: z.string().optional(),@@ -1251,10 +1287,51 @@ export const searchRequestSchema = z"Unrecognized key in body -- please review the v1 API documentation for request body changes",).refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)- .transform((x) => ({- ...x,- scrapeOptions: extractTransform(x.scrapeOptions),- }));+ .transform((x) => {+ // Transform string array sources to object format+ let sources = x.sources;+ if (sources && Array.isArray(sources) && sources.length > 0) {+ // Check if it's a string array by checking the first element+ if (typeof sources[0] === 'string') {+ // It's a string array, transform to object array+ sources = (sources as string[]).map((s) => {+ switch (s) {+ case 'web':+ return {+ type: 'web' as const,+ tbs: x.tbs,+ filter: x.filter,+ lang: x.lang,+ country: x.country,+ location: x.location,+ };+ case 'images':+ return {+ type: 'images' as const,+ // Images don't inherit global params in the simple format+ };+ case 'news':+ return {+ type: 'news' as const,+ tbs: x.tbs,+ lang: x.lang,+ country: x.country,+ location: x.location,+ };+ default:+ return { type: s as any };+ }+ });+ }+ // Otherwise it's already an object array, keep as is+ }++ return {+ ...x,+ sources,+ scrapeOptions: extractTransform(x.scrapeOptions),+ };+ });export type SearchRequest = z.infer<typeof searchRequestSchema>;export type SearchRequestInput = z.input<typeof searchRequestSchema>;