Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updated docs
Jan Curn8d3ccbbf7f08
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex 7594b56de..f68c6e4bf 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -51,10 +51,13 @@ const DEFAULT_OPTIONS = {* // Run some resource-intensive asynchronous operation here.* },* isTaskReadyFunction: async () => {- * // Tell the pool whether more tasks are ready to be processed. (true / false)+ * // Tell the pool whether more tasks are ready to be processed.+ * // Return true or false* },* isFinishedFunction: async () => {- * // Tell the pool whether it should finish or wait for more tasks to become available. (true / false)+ * // Tell the pool whether it should finish+ * // or wait for more tasks to become available.+ * // Return true or false* }* });*diff --git a/src/request_queue.js b/src/request_queue.jsindex 7510348f7..808f434e5 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -163,9 +163,9 @@ const getRequestId = (uniqueKey) => {* const queueWithName = await Apify.openRequestQueue('some-name');** // Enqueue few requests- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'}));- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'}));- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}), { forefront: true });+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa' }));+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb' }));+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar' }), { forefront: true });** // Get requests from queue* const request1 = await queue.fetchNextRequest();