Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Modify request adding parameter validation for RequestQueue
Michael Huynh6c0fc283eb94
diff --git a/src/request_queue.js b/src/request_queue.jsindex ed32c4ada..801b37cd7 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -40,13 +40,12 @@ const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queu* @ignore*/const validateAddRequestParams = (request, opts) => {- let newRequest;- try {- checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');- } catch (_) {- // Delegates request constructor object validation to the Request constructor- newRequest = new Request(request);+ checkParamOrThrow(request, 'request', 'Object');++ if (!(request instanceof Request)) {+ request = new Request(request);}+checkParamOrThrow(opts, 'opts', 'Object');const { forefront = false } = opts;@@ -55,7 +54,7 @@ const validateAddRequestParams = (request, opts) => {if (request.id) throw new Error('Request has already "id" so it cannot be added to the queue!');- return { forefront, newRequest };+ return { forefront, request };};/**@@ -216,7 +215,7 @@ export class RequestQueue {* @return {RequestOperationInfo}*/addRequest(request, opts = {}) {- const { forefront, newRequest } = validateAddRequestParams(request, opts);+ const { forefront, request: newRequest } = validateAddRequestParams(request, opts);if (newRequest) {request = newRequest;@@ -637,7 +636,7 @@ export class RequestQueueLocal {}addRequest(request, opts = {}) {- const { forefront, newRequest } = validateAddRequestParams(request, opts);+ const { forefront, request: newRequest } = validateAddRequestParams(request, opts);if (newRequest) {request = newRequest;