Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: add log about starting the crawl
Martin Adámekcb85fec202f8
diff --git a/docs/upgrading/upgrading_v3.md b/docs/upgrading/upgrading_v3.mdindex 8585f895a..b3282c180 100644--- a/docs/upgrading/upgrading_v3.md+++ b/docs/upgrading/upgrading_v3.md@@ -192,7 +192,7 @@ Moreover, we can specify patterns the URL should match via globs:const crawler = new PlaywrightCrawler({async requestHandler({ enqueueLinks }) {await enqueueLinks({- globs: ['https://apify.com/*/*'],+ globs: ['https://crawlee.dev/*/*'],// we can also use `regexps` and `pseudoUrls` keys here});},diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1e99b1ad5..82fef8879 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -563,6 +563,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this.stats.startCapturing();try {+ this.log.info('Starting the crawl');await this.autoscaledPool!.run();} finally {await this.teardown();@@ -576,7 +577,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextretryHistogram: this.stats.requestRetryHistogram,...finalStats,};- this.log.info('Final request statistics:', stats);+ this.log.info('Crawl finished. Final request statistics:', stats);return stats;}diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.jsindex 189a945bd..2d4a3c534 100644--- a/packages/templates/templates/cheerio-js/src/main.js+++ b/packages/templates/templates/cheerio-js/src/main.js@@ -2,7 +2,7 @@import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -15,8 +15,4 @@ const crawler = new CheerioCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/cheerio-js/src/routes.js b/packages/templates/templates/cheerio-js/src/routes.jsindex cf0e76508..b4e575067 100644--- a/packages/templates/templates/cheerio-js/src/routes.js+++ b/packages/templates/templates/cheerio-js/src/routes.js@@ -5,7 +5,7 @@ export const router = createCheerioRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.tsindex 254a0aa96..118b6e654 100644--- a/packages/templates/templates/cheerio-ts/src/main.ts+++ b/packages/templates/templates/cheerio-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -20,8 +20,4 @@ const crawler = new CheerioCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/cheerio-ts/src/routes.ts b/packages/templates/templates/cheerio-ts/src/routes.tsindex 65402ed97..76f76a0fe 100644--- a/packages/templates/templates/cheerio-ts/src/routes.ts+++ b/packages/templates/templates/cheerio-ts/src/routes.ts@@ -5,7 +5,7 @@ export const router = createCheerioRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.jsindex 0bd5e149c..400321712 100644--- a/packages/templates/templates/playwright-js/src/main.js+++ b/packages/templates/templates/playwright-js/src/main.js@@ -2,7 +2,7 @@import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';import { router } from './routes.js';-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -15,8 +15,4 @@ const crawler = new PlaywrightCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/playwright-js/src/routes.js b/packages/templates/templates/playwright-js/src/routes.jsindex 28d33be2a..62fbf029f 100644--- a/packages/templates/templates/playwright-js/src/routes.js+++ b/packages/templates/templates/playwright-js/src/routes.js@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.tsindex d272162d8..4cbe72f89 100644--- a/packages/templates/templates/playwright-ts/src/main.ts+++ b/packages/templates/templates/playwright-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -20,8 +20,4 @@ const crawler = new PlaywrightCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/playwright-ts/src/routes.ts b/packages/templates/templates/playwright-ts/src/routes.tsindex 28d33be2a..62fbf029f 100644--- a/packages/templates/templates/playwright-ts/src/routes.ts+++ b/packages/templates/templates/playwright-ts/src/routes.ts@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.jsindex 41180604a..395f133e2 100644--- a/packages/templates/templates/puppeteer-js/src/main.js+++ b/packages/templates/templates/puppeteer-js/src/main.js@@ -2,7 +2,7 @@import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';import { router } from './routes.js';-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -15,8 +15,4 @@ const crawler = new PuppeteerCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/puppeteer-js/src/routes.js b/packages/templates/templates/puppeteer-js/src/routes.jsindex 4dd2fd8a7..80b3f0b43 100644--- a/packages/templates/templates/puppeteer-js/src/routes.js+++ b/packages/templates/templates/puppeteer-js/src/routes.js@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.tsindex d230e3bac..9b9086938 100644--- a/packages/templates/templates/puppeteer-ts/src/main.ts+++ b/packages/templates/templates/puppeteer-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);@@ -20,8 +20,4 @@ const crawler = new PuppeteerCrawler({requestHandler: router,});-await crawler.addRequests(startUrls);--log.info('Starting the crawl.');-await crawler.run();-log.info('Crawl finished.');+await crawler.run(startUrls);diff --git a/packages/templates/templates/puppeteer-ts/src/routes.ts b/packages/templates/templates/puppeteer-ts/src/routes.tsindex 4dd2fd8a7..80b3f0b43 100644--- a/packages/templates/templates/puppeteer-ts/src/routes.ts+++ b/packages/templates/templates/puppeteer-ts/src/routes.ts@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();router.addDefaultHandler(async ({ enqueueLinks, log }) => {log.info(`Handle Start URLs`);await enqueueLinks({- globs: ['https://apify.com/*'],+ globs: ['https://crawlee.dev/*'],label: 'DETAIL',});});