Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): don't push rawhtml to db unless requested
Móricz Gergőb96b97ed721c
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex ce3831f28..f5fcf6b68 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -196,6 +196,7 @@ export async function crawlStatusController(nextURL.searchParams.set("limit", req.query.limit);}+ // deprecated: this is done on queue-worker side now. if you see this after january 8, 2025, remove thisif (data.length > 0) {if (!doneJobs[0].data.scrapeOptions.formats.includes("rawHtml")) {for (let ii = 0; ii < doneJobs.length; ii++) {@@ -205,6 +206,7 @@ export async function crawlStatusController(}}}+ // remove until hereres.status(200).json({success: true,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex a48c798b4..90c96cf64 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -674,6 +674,10 @@ async function processJob(job: Job & { id: string }, token: string) {const rawHtml = doc.rawHtml ?? "";+ if (!job.data.scrapeOptions.formats.includes("rawHtml")) {+ delete doc.rawHtml;+ }+const data = {success: true,result: {