Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Update docs
Ondra Urban12c09d775af2
diff --git a/README.md b/README.mdindex b4433fafb..1a7bf453e 100644--- a/README.md+++ b/README.md@@ -144,7 +144,7 @@ Apify.main(async () => {handlePageFunction: async ({ request, page }) => {const title = await page.title();console.log(`Title of ${request.url}: ${title}`);- await Apify.utils.puppeteer.enqueueLinks(page, 'a', pseudoUrls, requestQueue);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue });},maxRequestsPerCrawl: 100,maxConcurrency: 10,diff --git a/docs/api/RequestQueue.md b/docs/api/RequestQueue.mdindex a19f8b777..9af889de4 100644--- a/docs/api/RequestQueue.md+++ b/docs/api/RequestQueue.md@@ -51,9 +51,9 @@ const queue = await Apify.openRequestQueue();const queueWithName = await Apify.openRequestQueue('some-name');// Enqueue few requests-await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa' }));-await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb' }));-await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar' }), { forefront: true });+await queue.addRequest({ url: 'http://example.com/aaa' });+await queue.addRequest({ url: 'http://example.com/bbb' });+await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });// Get requests from queueconst request1 = await queue.fetchNextRequest();diff --git a/docs/examples/HelloWorld.md b/docs/examples/HelloWorld.mdindex d52678b9a..edd26f905 100644--- a/docs/examples/HelloWorld.md+++ b/docs/examples/HelloWorld.md@@ -20,7 +20,7 @@ Apify.main(async () => {handlePageFunction: async ({ request, page }) => {const title = await page.title();console.log(`Title of ${request.url}: ${title}`);- await Apify.utils.puppeteer.enqueueLinks(page, 'a', pseudoUrls, requestQueue);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue });},maxRequestsPerCrawl: 100,maxConcurrency: 10,diff --git a/docs/guides/data_storage.md b/docs/guides/data_storage.mdindex 3c63a6266..b1caf79fe 100644--- a/docs/guides/data_storage.md+++ b/docs/guides/data_storage.md@@ -170,9 +170,9 @@ const queue = await Apify.openRequestQueue();const queueWithName = await Apify.openRequestQueue('some-name');// Enqueue few requests-await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'}));-await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'}));-await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}), { forefront: true });+await queue.addRequest({ url: 'http://example.com/aaa'});+await queue.addRequest({ url: 'http://example.com/bbb'});+await queue.addRequest({ url: 'http://example.com/foo/bar'}, { forefront: true });// Get requests from queueconst request1 = await queue.fetchNextRequest();diff --git a/docs/guides/getting_started_quick.md b/docs/guides/getting_started_quick.mdindex db1e05dcc..85277fe5f 100644--- a/docs/guides/getting_started_quick.md+++ b/docs/guides/getting_started_quick.md@@ -29,7 +29,7 @@ Apify.main(async () => {handlePageFunction: async ({ request, page }) => {const title = await page.title();console.log(`Title of ${request.url}: ${title}`);- await Apify.utils.puppeteer.enqueueLinks(page, 'a', pseudoUrls, requestQueue);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue });},maxRequestsPerCrawl: 100,maxConcurrency: 10,diff --git a/src/request_queue.js b/src/request_queue.jsindex 18f820c1d..14749b1f5 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -173,9 +173,9 @@ const getRequestId = (uniqueKey) => {* const queueWithName = await Apify.openRequestQueue('some-name');** // Enqueue few requests- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa' }));- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb' }));- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar' }), { forefront: true });+ * await queue.addRequest({ url: 'http://example.com/aaa' });+ * await queue.addRequest({ url: 'http://example.com/bbb' });+ * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });** // Get requests from queue* const request1 = await queue.fetchNextRequest();diff --git a/website/pages/en/index.js b/website/pages/en/index.jsindex 72cd46068..8101cc117 100755--- a/website/pages/en/index.js+++ b/website/pages/en/index.js@@ -216,7 +216,7 @@ const TryOut = () => (' handlePageFunction: async ({ request, page }) => {\n' +' const title = await page.title();\n' +' console.log(`Title of ${request.url}: ${title}`);\n' +- ' await Apify.utils.puppeteer.enqueueLinks(page, \'a\', pseudoUrls, requestQueue);\n' ++ ' await Apify.utils.puppeteer.enqueueLinks({ page, selector: \'a\', pseudoUrls, requestQueue });\n' +' },\n' +' maxRequestsPerCrawl: 100,\n' +' maxConcurrency: 10,\n' +