Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: llm extract in batch scrape

Nicolas726430c2e641
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 7c68341bd..cde4bd767 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -4,6 +4,7 @@ import {
BatchScrapeRequest,
batchScrapeRequestSchema,
CrawlResponse,
+ legacyExtractorOptions,
legacyScrapeOptions,
RequestWithAuth,
} from "./types";
@@ -34,6 +35,8 @@ export async function batchScrapeController(
}
const pageOptions = legacyScrapeOptions(req.body);
+ const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
+
const sc: StoredCrawl = {
crawlerOptions: null,
@@ -65,6 +68,7 @@ export async function batchScrapeController(
plan: req.auth.plan,
crawlerOptions: null,
pageOptions,
+ extractorOptions,
origin: "api",
crawl_id: id,
sitemapped: true,