Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl-status): keep 10 megabyte pages if they're the only thing in the output

Gergő Móriczd5e2a80e4ac9
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 2ee0638c6..ec427a9ca 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -84,8 +84,8 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
}
}
- // if we ran over the bytes limit, remove the last document
- if (bytes > bytesLimit) {
+ // if we ran over the bytes limit, remove the last document, except if it's the only document
+ if (bytes > bytesLimit && doneJobs.length !== 1) {
doneJobs.splice(doneJobs.length - 1, 1);
}
} else {