Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(v1/batch/scrape): horrid memory usage

Gergő Móriczcce94289eef2
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index dd8ab2fb0..db029d40f 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -59,6 +59,10 @@ export async function batchScrapeController(
jobPriority = await getJobPriority({plan: req.auth.plan, team_id: req.auth.team_id, basePriority: 21})
}
+ const scrapeOptions: ScrapeOptions = { ...req.body };
+ delete (scrapeOptions as any).urls;
+ delete (scrapeOptions as any).appendToId;
+
const jobs = req.body.urls.map((x) => {
return {
data: {
@@ -67,7 +71,7 @@ export async function batchScrapeController(
team_id: req.auth.team_id,
plan: req.auth.plan!,
crawlerOptions: null,
- scrapeOptions: req.body,
+ scrapeOptions,
origin: "api",
crawl_id: id,
sitemapped: true,