Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: correlate smart scrape

Gergő Móricz0d813b628bd4
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index d2ad2a069..cee1d5a67 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -185,11 +185,13 @@ export async function extractData({
urls,
useAgent,
extractId,
+ scrapeId,
}: {
extractOptions: GenerateCompletionsOptions;
urls: string[];
useAgent: boolean;
extractId?: string;
+ scrapeId?: string;
}): Promise<{
extractedDataArray: any[];
warning: any;
@@ -272,7 +274,7 @@ export async function extractData({
let smartscrapeResults: SmartScrapeResult[];
if (isSingleUrl) {
smartscrapeResults = [
- await smartScrape(urls[0], extract?.smartscrape_prompt, undefined, extractId),
+ await smartScrape(urls[0], extract?.smartscrape_prompt, undefined, extractId, scrapeId),
];
smartScrapeCost += smartscrapeResults[0].tokenUsage;
smartScrapeCallCount++;
@@ -286,6 +288,7 @@ export async function extractData({
page.smartscrape_prompt,
undefined,
extractId,
+ scrapeId,
);
}),
);
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
index 4e4cbb20c..046a7b5eb 100644
--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
@@ -50,6 +50,7 @@ export async function smartScrape(
prompt: string,
sessionId?: string,
extractId?: string,
+ scrapeId?: string,
): Promise<SmartScrapeResult> {
let logger = _logger.child({
method: "smartScrape",
@@ -58,6 +59,7 @@ export async function smartScrape(
url,
prompt,
sessionId,
+ scrapeId,
});
try {
logger.info("Initiating smart scrape request");
@@ -71,6 +73,7 @@ export async function smartScrape(
prompt,
userProvidedId: sessionId ?? undefined,
extractId,
+ scrapeId,
models: {
thinkingModel: {
model: "gemini-2.5-pro-preview-03-25",
diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
index 6ab32862e..30a0f46fc 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
@@ -25,7 +25,7 @@ export async function performAgent(
let smartscrapeResults: SmartScrapeResult;
try {
- smartscrapeResults = await smartScrape(url, prompt, sessionId)
+ smartscrapeResults = await smartScrape(url, prompt, sessionId, undefined, meta.id)
} catch (error) {
if (error instanceof Error && error.message === "Cost limit exceeded") {
logger.error("Cost limit exceeded", { error })
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index ebdb7a6b9..804ead2e3 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -601,6 +601,7 @@ export async function performLLMExtract(
extractOptions: generationOptions,
urls: [meta.url],
useAgent: isAgentExtractModelValid(meta.options.extract?.agent?.model),
+ scrapeId: meta.id,
});
if (warning) {