Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: urls optional on extract
Nicolasf1206e487003
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 1e4625494..d52c55c55 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -314,7 +314,8 @@ export const extractV1Options = z.object({urls: url.array()- .max(10, "Maximum of 10 URLs allowed per request while in beta."),+ .max(10, "Maximum of 10 URLs allowed per request while in beta.")+ .optional(),prompt: z.string().max(10000).optional(),systemPrompt: z.string().max(10000).optional(),schema: z@@ -354,6 +355,12 @@ export const extractV1Options = z.optional(),}).strict(strictMessage)+ .refine(+ (obj) => obj.urls || obj.prompt,+ {+ message: "Either 'urls' or 'prompt' must be provided.",+ },+ ).transform((obj) => ({...obj,allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.tsindex 247106602..f7dd4e320 100644--- a/apps/api/src/lib/extract/build-prompts.ts+++ b/apps/api/src/lib/extract/build-prompts.ts@@ -105,3 +105,10 @@ export function buildBatchExtractSystemPrompt(export function buildBatchExtractPrompt(prompt: string): string {return `Today is: ${new Date().toISOString()}\n${prompt}`;}+++export function buildRephraseToSerpPrompt(prompt: string): string {+ return `Rephrase the following prompt to be suitable for a search engine results page (SERP) query. Make sure the rephrased prompt is concise and focused on retrieving relevant search results:++Original Prompt: "${prompt}"`;+}diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 45d18fe62..acd9f8b0f 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -6,7 +6,7 @@ import {} from "../../controllers/v1/types";import { PlanType } from "../../types";import { logger as _logger } from "../logger";-import { processUrl } from "./url-processor";+import { generateBasicCompletion, processUrl } from "./url-processor";import { scrapeDocument } from "./document-scraper";import {generateCompletions,@@ -38,6 +38,8 @@ import { singleAnswerCompletion } from "./completions/singleAnswer";import { SourceTracker } from "./helpers/source-tracker";import { getCachedDocs, saveCachedDocs } from "./helpers/cached-docs";import { normalizeUrl } from "../canonical-url";+import { search } from "../../search";+import { buildRephraseToSerpPrompt } from "./build-prompts";interface ExtractServiceOptions {request: ExtractRequest;@@ -84,16 +86,43 @@ export async function performExtraction(let totalUrlsScraped = 0;let sources: Record<string, string[]> = {};+const logger = _logger.child({module: "extract",method: "performExtraction",extractId,});- if (request.__experimental_cacheMode == "load" && request.__experimental_cacheKey) {+ // If no URLs are provided, generate URLs from the prompt+ if ((!request.urls || request.urls.length === 0) && request.prompt) {+ logger.debug("Generating URLs from prompt...", {+ prompt: request.prompt,+ });+ const rephrasedPrompt = await generateBasicCompletion(buildRephraseToSerpPrompt(request.prompt));+ const searchResults = await search({+ query: rephrasedPrompt.replace('"', "").replace("'", ""),+ num_results: 10,+ });++ request.urls = searchResults.map(result => result.url) as string[];+ }+ if (request.urls && request.urls.length === 0) {+ logger.error("No search results found", {+ query: request.prompt,+ });+ return {+ success: false,+ error: "No search results found",+ extractId,+ };+ }++ const urls = request.urls || ([] as string[]);++ if (request.__experimental_cacheMode == "load" && request.__experimental_cacheKey && urls) {logger.debug("Loading cached docs...");try {- const cache = await getCachedDocs(request.urls, request.__experimental_cacheKey);+ const cache = await getCachedDocs(urls, request.__experimental_cacheKey);for (const doc of cache) {if (doc.metadata.url) {docsMap.set(normalizeUrl(doc.metadata.url), doc);@@ -122,11 +151,10 @@ export async function performExtraction(let startMap = Date.now();let aggMapLinks: string[] = [];logger.debug("Processing URLs...", {- urlCount: request.urls.length,+ urlCount: request.urls?.length || 0,});- // Process URLs- const urlPromises = request.urls.map((url) =>+ const urlPromises = urls.map((url) =>processUrl({url,@@ -746,7 +774,7 @@ export async function performExtraction(time_taken: (new Date().getTime() - Date.now()) / 1000,team_id: teamId,mode: "extract",- url: request.urls.join(", "),+ url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",num_tokens: totalTokensUsed,diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 3175615dd..98fdf696d 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -1119,14 +1119,14 @@ export default class FirecrawlApp {/*** Extracts information from URLs using the Firecrawl API.* Currently in Beta. Expect breaking changes on future minor versions.- * @param url - The URL to extract information from.+ * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.* @param params - Additional parameters for the extract request.* @returns The response from the extract operation.*/- async extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {+ async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: { urls: string[] } & ExtractParams<T> = { urls, ...params };+ let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls || [], ...params };let jsonSchema: any;try {if (!params?.schema) {diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex d79b174c9..4108868e1 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -646,12 +646,12 @@ class FirecrawlApp:else:self._handle_error(response, "check batch scrape errors")- def extract(self, urls: List[str], params: Optional[ExtractParams] = None) -> Any:+ def extract(self, urls: Optional[List[str]] = None, params: Optional[ExtractParams] = None) -> Any:"""Extracts information from a URL using the Firecrawl API.Args:- urls (List[str]): The URLs to extract information from.+ urls (Optional[List[str]]): The URLs to extract information from.params (Optional[ExtractParams]): Additional parameters for the extract request.Returns:@@ -662,6 +662,9 @@ class FirecrawlApp:if not params or (not params.get('prompt') and not params.get('schema')):raise ValueError("Either prompt or schema is required")+ if not urls and not params.get('prompt'):+ raise ValueError("Either urls or prompt is required")+schema = params.get('schema')if schema:if hasattr(schema, 'model_json_schema'):