Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Comments
Marek Trunkata052f03af5b7
diff --git a/src/request_queue.js b/src/request_queue.jsindex e5705fe7b..59b1ee773 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -543,7 +543,8 @@ export class RequestQueue {// - hadMultipleClients=false and this.assumedTotalCount<=this.assumedHandledCountconst isDatabaseConsistent = queryStartedAt - queueModifiedAt >= API_PROCESSED_REQUESTS_DELAY_MILLIS;const isLocallyConsistent = !hadMultipleClients && this.assumedTotalCount <= this.assumedHandledCount;- const shouldRepeatForConsistency = checkModifiedAt && (!isDatabaseConsistent && !isLocallyConsistent);+ // Consistent information from one source is enough to consider request queue finished.+ const shouldRepeatForConsistency = checkModifiedAt && !isDatabaseConsistent && !isLocallyConsistent;// If both are false then head is consistent and we may exit.if (!shouldRepeatWithHigherLimit && !shouldRepeatForConsistency) return true;