Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Puting request at last the end of head
Marek Trunkat2e44ad68d06d
diff --git a/src/request_queue.js b/src/request_queue.jsindex 704ee22b0..08287a5e2 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -252,10 +252,7 @@ export class RequestQueue {// immediately after adding to the queue.if (!request) {this._removeFromInProgress(nextId);-- // TODO: Consider skipping this line so that the request doesn't get retried immediately- // but after next head fetch instead.- this.queueHeadDict.add(nextId, nextId, true);+ this.queueHeadDict.add(nextId, nextId, false);}return request;