Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: simplify project templates

Martin Adámek520fe631242a
diff --git a/packages/templates/.eslintrc.json b/packages/templates/.eslintrc.json
index b2c3f2261..980589bad 100644
--- a/packages/templates/.eslintrc.json
+++ b/packages/templates/.eslintrc.json
@@ -3,7 +3,7 @@
"extends": "../../.eslintrc.json",
"parserOptions": {
"project": "../../tsconfig.eslint.json",
- "ecmaVersion": 2020
+ "ecmaVersion": 2022
},
"rules": {
"no-console": 0,
diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.js
index 2d4a3c534..354cfbf23 100644
--- a/packages/templates/templates/cheerio-js/src/main.js
+++ b/packages/templates/templates/cheerio-js/src/main.js
@@ -1,17 +1,11 @@
// For more information, see https://crawlee.dev/
-import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new CheerioCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});
diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.ts
index 118b6e654..354cfbf23 100644
--- a/packages/templates/templates/cheerio-ts/src/main.ts
+++ b/packages/templates/templates/cheerio-ts/src/main.ts
@@ -1,22 +1,11 @@
// For more information, see https://crawlee.dev/
-import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-interface InputSchema {
- startUrls: string[];
- debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new CheerioCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});
diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.js
index 400321712..8076b10b5 100644
--- a/packages/templates/templates/playwright-js/src/main.js
+++ b/packages/templates/templates/playwright-js/src/main.js
@@ -1,17 +1,11 @@
// For more information, see https://crawlee.dev/
-import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new PlaywrightCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});
diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.ts
index 4cbe72f89..8076b10b5 100644
--- a/packages/templates/templates/playwright-ts/src/main.ts
+++ b/packages/templates/templates/playwright-ts/src/main.ts
@@ -1,22 +1,11 @@
// For more information, see https://crawlee.dev/
-import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-interface InputSchema {
- startUrls: string[];
- debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new PlaywrightCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});
diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.js
index 395f133e2..a494a18c8 100644
--- a/packages/templates/templates/puppeteer-js/src/main.js
+++ b/packages/templates/templates/puppeteer-js/src/main.js
@@ -1,17 +1,11 @@
// For more information, see https://crawlee.dev/
-import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
import { router } from './routes.js';
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new PuppeteerCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});
diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.ts
index 9b9086938..5f4c7339d 100644
--- a/packages/templates/templates/puppeteer-ts/src/main.ts
+++ b/packages/templates/templates/puppeteer-ts/src/main.ts
@@ -1,22 +1,11 @@
// For more information, see https://crawlee.dev/
-import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PuppeteerCrawler, log } from 'crawlee';
import { router } from './routes.js';
-interface InputSchema {
- startUrls: string[];
- debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
- log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
const crawler = new PuppeteerCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
- // Be nice to the websites. Remove to unleash full power.
- maxConcurrency: 50,
requestHandler: router,
});