Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding some examples and documentation
Marek Trunkat13b6f47e43fd
diff --git a/examples/example_autoscaled_pool.js b/examples/example_autoscaled_pool.jsindex c8d2b27b7..c4d4b9168 100644--- a/examples/example_autoscaled_pool.js+++ b/examples/example_autoscaled_pool.js@@ -3,6 +3,10 @@ const Apify = require('../build');Apify.main(async () => {let counter = 0;+ // Worker function gets executed again and again in the pool.+ // Size of the pool is auto-scaled based on memory and CPU.+ // If this function returns null then all tasks are considered+ // to be done.const workerFunction = () => {const current = counter++;diff --git a/examples/example_basic_crawler.js b/examples/example_basic_crawler.jsnew file mode 100644index 000000000..88d9e8839--- /dev/null+++ b/examples/example_basic_crawler.js@@ -0,0 +1,37 @@+const Apify = require('../build');+const rp = require('request-promise');++Apify.main(async () => {+ // Create a request list.+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'http://www.example.com' },+ { url: 'http://www.example.com/page-2' },+ { url: 'http://www.example.com/page-3' },+ { url: 'http://www.example.com/page-4' },+ { url: 'http://www.example.com/page-5' },+ ],+ });++ await requestList.initialize();++ const crawler = new Apify.BasicCrawler({+ requestList,++ // This page is executed for each request.+ // If request failes then it's retried 3 times.+ handleRequestFunction: async ({ request }) => {+ const pageHtml = await rp(request.url);++ console.log(`Request ${request.url} succeeded with return html of length ${pageHtml.length}`);+ },++ // If request failed 4 times then this function is executed.+ handleFailedRequestFunction: async ({ request }) => {+ console.log(`Request ${request.url} failed 4 times`);+ },+ });++ // Run crawler for request list.+ await crawler.run();+});diff --git a/examples/example_puppeteer_crawler.js b/examples/example_puppeteer_crawler.jsnew file mode 100644index 000000000..11f52c041--- /dev/null+++ b/examples/example_puppeteer_crawler.js@@ -0,0 +1,35 @@+const Apify = require('../build');++Apify.main(async () => {+ // Create a request list.+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'http://www.example.com' },+ { url: 'http://www.some-nonexisting-domain.com' },+ ],+ });++ await requestList.initialize();++ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ disableProxy: true,++ // This page is executed for each request.+ // If request failes then it's retried 3 times.+ // Parameter page is Puppeteers page object with loaded page.+ handlePageFunction: async ({ page, request }) => {+ const title = await page.title();++ console.log(`Request ${request.url} succeeded and it's title is ${title}`);+ },++ // If request failed 4 times then this function is executed.+ handleFailedRequestFunction: async ({ request }) => {+ console.log(`Request ${request.url} failed 4 times`);+ },+ });++ // Run crawler for request list.+ await crawler.run();+});diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 6c9f93766..802fc5b71 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -21,13 +21,13 @@ const DEFAULT_OPTIONS = {* Basic usage of BasicCrawler:** ```javascript- * const request = require('request-promise');+ * const rp = require('request-promise');** const crawler = new Apify.BasicCrawler({* requestList,* handleRequestFunction: async ({ request }) => {* await Apify.pushData({- * html: await request(request.url),+ * html: await rp(request.url),* url: request.url,* })* },