Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(v1/batch/scrape): horrid memory usage
Gergő Móriczcce94289eef2
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex dd8ab2fb0..db029d40f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -59,6 +59,10 @@ export async function batchScrapeController(jobPriority = await getJobPriority({plan: req.auth.plan, team_id: req.auth.team_id, basePriority: 21})}+ const scrapeOptions: ScrapeOptions = { ...req.body };+ delete (scrapeOptions as any).urls;+ delete (scrapeOptions as any).appendToId;+const jobs = req.body.urls.map((x) => {return {data: {@@ -67,7 +71,7 @@ export async function batchScrapeController(team_id: req.auth.team_id,plan: req.auth.plan!,crawlerOptions: null,- scrapeOptions: req.body,+ scrapeOptions,origin: "api",crawl_id: id,sitemapped: true,