Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: correlate smart scrape
Gergő Móricz0d813b628bd4
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex d2ad2a069..cee1d5a67 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -185,11 +185,13 @@ export async function extractData({urls,useAgent,extractId,+ scrapeId,}: {extractOptions: GenerateCompletionsOptions;urls: string[];useAgent: boolean;extractId?: string;+ scrapeId?: string;}): Promise<{extractedDataArray: any[];warning: any;@@ -272,7 +274,7 @@ export async function extractData({let smartscrapeResults: SmartScrapeResult[];if (isSingleUrl) {smartscrapeResults = [- await smartScrape(urls[0], extract?.smartscrape_prompt, undefined, extractId),+ await smartScrape(urls[0], extract?.smartscrape_prompt, undefined, extractId, scrapeId),];smartScrapeCost += smartscrapeResults[0].tokenUsage;smartScrapeCallCount++;@@ -286,6 +288,7 @@ export async function extractData({page.smartscrape_prompt,undefined,extractId,+ scrapeId,);}),);diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.tsindex 4e4cbb20c..046a7b5eb 100644--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts@@ -50,6 +50,7 @@ export async function smartScrape(prompt: string,sessionId?: string,extractId?: string,+ scrapeId?: string,): Promise<SmartScrapeResult> {let logger = _logger.child({method: "smartScrape",@@ -58,6 +59,7 @@ export async function smartScrape(url,prompt,sessionId,+ scrapeId,});try {logger.info("Initiating smart scrape request");@@ -71,6 +73,7 @@ export async function smartScrape(prompt,userProvidedId: sessionId ?? undefined,extractId,+ scrapeId,models: {thinkingModel: {model: "gemini-2.5-pro-preview-03-25",diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.tsindex 6ab32862e..30a0f46fc 100644--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts@@ -25,7 +25,7 @@ export async function performAgent(let smartscrapeResults: SmartScrapeResult;try {- smartscrapeResults = await smartScrape(url, prompt, sessionId)+ smartscrapeResults = await smartScrape(url, prompt, sessionId, undefined, meta.id)} catch (error) {if (error instanceof Error && error.message === "Cost limit exceeded") {logger.error("Cost limit exceeded", { error })diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex ebdb7a6b9..804ead2e3 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -601,6 +601,7 @@ export async function performLLMExtract(extractOptions: generationOptions,urls: [meta.url],useAgent: isAgentExtractModelValid(meta.options.extract?.agent?.model),+ scrapeId: meta.id,});if (warning) {