Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: add log about starting the crawl

Martin Adámekcb85fec202f8
diff --git a/docs/upgrading/upgrading_v3.md b/docs/upgrading/upgrading_v3.md
index 8585f895a..b3282c180 100644
--- a/docs/upgrading/upgrading_v3.md
+++ b/docs/upgrading/upgrading_v3.md
@@ -192,7 +192,7 @@ Moreover, we can specify patterns the URL should match via globs:
const crawler = new PlaywrightCrawler({
async requestHandler({ enqueueLinks }) {
await enqueueLinks({
- globs: ['https://apify.com/*/*'],
+ globs: ['https://crawlee.dev/*/*'],
// we can also use `regexps` and `pseudoUrls` keys here
});
},
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 1e99b1ad5..82fef8879 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -563,6 +563,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await this.stats.startCapturing();
try {
+ this.log.info('Starting the crawl');
await this.autoscaledPool!.run();
} finally {
await this.teardown();
@@ -576,7 +577,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
retryHistogram: this.stats.requestRetryHistogram,
...finalStats,
};
- this.log.info('Final request statistics:', stats);
+ this.log.info('Crawl finished. Final request statistics:', stats);
return stats;
}
diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.js
index 189a945bd..2d4a3c534 100644
--- a/packages/templates/templates/cheerio-js/src/main.js
+++ b/packages/templates/templates/cheerio-js/src/main.js
@@ -2,7 +2,7 @@
import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -15,8 +15,4 @@ const crawler = new CheerioCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/cheerio-js/src/routes.js b/packages/templates/templates/cheerio-js/src/routes.js
index cf0e76508..b4e575067 100644
--- a/packages/templates/templates/cheerio-js/src/routes.js
+++ b/packages/templates/templates/cheerio-js/src/routes.js
@@ -5,7 +5,7 @@ export const router = createCheerioRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});
diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.ts
index 254a0aa96..118b6e654 100644
--- a/packages/templates/templates/cheerio-ts/src/main.ts
+++ b/packages/templates/templates/cheerio-ts/src/main.ts
@@ -7,7 +7,7 @@ interface InputSchema {
debug?: boolean;
}
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -20,8 +20,4 @@ const crawler = new CheerioCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/cheerio-ts/src/routes.ts b/packages/templates/templates/cheerio-ts/src/routes.ts
index 65402ed97..76f76a0fe 100644
--- a/packages/templates/templates/cheerio-ts/src/routes.ts
+++ b/packages/templates/templates/cheerio-ts/src/routes.ts
@@ -5,7 +5,7 @@ export const router = createCheerioRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});
diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.js
index 0bd5e149c..400321712 100644
--- a/packages/templates/templates/playwright-js/src/main.js
+++ b/packages/templates/templates/playwright-js/src/main.js
@@ -2,7 +2,7 @@
import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -15,8 +15,4 @@ const crawler = new PlaywrightCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/playwright-js/src/routes.js b/packages/templates/templates/playwright-js/src/routes.js
index 28d33be2a..62fbf029f 100644
--- a/packages/templates/templates/playwright-js/src/routes.js
+++ b/packages/templates/templates/playwright-js/src/routes.js
@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});
diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.ts
index d272162d8..4cbe72f89 100644
--- a/packages/templates/templates/playwright-ts/src/main.ts
+++ b/packages/templates/templates/playwright-ts/src/main.ts
@@ -7,7 +7,7 @@ interface InputSchema {
debug?: boolean;
}
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -20,8 +20,4 @@ const crawler = new PlaywrightCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/playwright-ts/src/routes.ts b/packages/templates/templates/playwright-ts/src/routes.ts
index 28d33be2a..62fbf029f 100644
--- a/packages/templates/templates/playwright-ts/src/routes.ts
+++ b/packages/templates/templates/playwright-ts/src/routes.ts
@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});
diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.js
index 41180604a..395f133e2 100644
--- a/packages/templates/templates/puppeteer-js/src/main.js
+++ b/packages/templates/templates/puppeteer-js/src/main.js
@@ -2,7 +2,7 @@
import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -15,8 +15,4 @@ const crawler = new PuppeteerCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/puppeteer-js/src/routes.js b/packages/templates/templates/puppeteer-js/src/routes.js
index 4dd2fd8a7..80b3f0b43 100644
--- a/packages/templates/templates/puppeteer-js/src/routes.js
+++ b/packages/templates/templates/puppeteer-js/src/routes.js
@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});
diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.ts
index d230e3bac..9b9086938 100644
--- a/packages/templates/templates/puppeteer-ts/src/main.ts
+++ b/packages/templates/templates/puppeteer-ts/src/main.ts
@@ -7,7 +7,7 @@ interface InputSchema {
debug?: boolean;
}
-const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
if (debug) {
log.setLevel(log.LEVELS.DEBUG);
@@ -20,8 +20,4 @@ const crawler = new PuppeteerCrawler({
requestHandler: router,
});
-await crawler.addRequests(startUrls);
-
-log.info('Starting the crawl.');
-await crawler.run();
-log.info('Crawl finished.');
+await crawler.run(startUrls);
diff --git a/packages/templates/templates/puppeteer-ts/src/routes.ts b/packages/templates/templates/puppeteer-ts/src/routes.ts
index 4dd2fd8a7..80b3f0b43 100644
--- a/packages/templates/templates/puppeteer-ts/src/routes.ts
+++ b/packages/templates/templates/puppeteer-ts/src/routes.ts
@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`Handle Start URLs`);
await enqueueLinks({
- globs: ['https://apify.com/*'],
+ globs: ['https://crawlee.dev/*'],
label: 'DETAIL',
});
});