Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-status): keep 10 megabyte pages if they're the only thing in the output
Gergő Móriczd5e2a80e4ac9
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 2ee0638c6..ec427a9ca 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -84,8 +84,8 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara}}- // if we ran over the bytes limit, remove the last document- if (bytes > bytesLimit) {+ // if we ran over the bytes limit, remove the last document, except if it's the only document+ if (bytes > bytesLimit && doneJobs.length !== 1) {doneJobs.splice(doneJobs.length - 1, 1);}} else {