Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(templates): depend on crawlee v4 and fix TypeScript 6 builds (#4338)
Jindřich Bära135a8bc2a36
diff --git a/packages/templates/templates/camoufox-ts/package.json b/packages/templates/templates/camoufox-ts/package.jsonindex 0d4cd3316..ea1b7d1a0 100644--- a/packages/templates/templates/camoufox-ts/package.json+++ b/packages/templates/templates/camoufox-ts/package.json@@ -5,9 +5,9 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1","camoufox-js": "^0.13.0",- "crawlee": "^3.0.0",+ "crawlee": "^4.0.0-rc.1","playwright": "1.60.0"},"devDependencies": {diff --git a/packages/templates/templates/camoufox-ts/tsconfig.json b/packages/templates/templates/camoufox-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/camoufox-ts/tsconfig.json+++ b/packages/templates/templates/camoufox-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/packages/templates/templates/cheerio-js/package.json b/packages/templates/templates/cheerio-js/package.jsonindex f83230330..7c2baa958 100644--- a/packages/templates/templates/cheerio-js/package.json+++ b/packages/templates/templates/cheerio-js/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0"+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1"},"scripts": {"start": "node src/main.js",diff --git a/packages/templates/templates/cheerio-ts/package.json b/packages/templates/templates/cheerio-ts/package.jsonindex 1e6cebbad..c6e0f8f5a 100644--- a/packages/templates/templates/cheerio-ts/package.json+++ b/packages/templates/templates/cheerio-ts/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0"+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1"},"devDependencies": {"@apify/tsconfig": "^0.2.0",diff --git a/packages/templates/templates/cheerio-ts/tsconfig.json b/packages/templates/templates/cheerio-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/cheerio-ts/tsconfig.json+++ b/packages/templates/templates/cheerio-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/packages/templates/templates/empty-js/package.json b/packages/templates/templates/empty-js/package.jsonindex 34e2f3c25..4d03df3b7 100644--- a/packages/templates/templates/empty-js/package.json+++ b/packages/templates/templates/empty-js/package.json@@ -5,7 +5,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "crawlee": "^3.0.0"+ "crawlee": "^4.0.0-rc.1"},"scripts": {"start": "node src/main.js",diff --git a/packages/templates/templates/empty-ts/package.json b/packages/templates/templates/empty-ts/package.jsonindex 040bc75b7..f64931752 100644--- a/packages/templates/templates/empty-ts/package.json+++ b/packages/templates/templates/empty-ts/package.json@@ -5,7 +5,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "crawlee": "^3.0.0"+ "crawlee": "^4.0.0-rc.1"},"devDependencies": {"@apify/tsconfig": "^0.2.0",diff --git a/packages/templates/templates/empty-ts/tsconfig.json b/packages/templates/templates/empty-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/empty-ts/tsconfig.json+++ b/packages/templates/templates/empty-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/packages/templates/templates/getting-started-js/package.json b/packages/templates/templates/getting-started-js/package.jsonindex 150364d03..1d0b2ef34 100644--- a/packages/templates/templates/getting-started-js/package.json+++ b/packages/templates/templates/getting-started-js/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","playwright": "1.64.1"},"scripts": {diff --git a/packages/templates/templates/getting-started-ts/package.json b/packages/templates/templates/getting-started-ts/package.jsonindex 643188a9e..fe9547abf 100644--- a/packages/templates/templates/getting-started-ts/package.json+++ b/packages/templates/templates/getting-started-ts/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","playwright": "1.64.1"},"devDependencies": {diff --git a/packages/templates/templates/getting-started-ts/tsconfig.json b/packages/templates/templates/getting-started-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/getting-started-ts/tsconfig.json+++ b/packages/templates/templates/getting-started-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/packages/templates/templates/playwright-js/package.json b/packages/templates/templates/playwright-js/package.jsonindex 1b0cd8bd0..592e30777 100644--- a/packages/templates/templates/playwright-js/package.json+++ b/packages/templates/templates/playwright-js/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","playwright": "1.64.1"},"scripts": {diff --git a/packages/templates/templates/playwright-ts/package.json b/packages/templates/templates/playwright-ts/package.jsonindex 8096b99d9..af92770ba 100644--- a/packages/templates/templates/playwright-ts/package.json+++ b/packages/templates/templates/playwright-ts/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","playwright": "1.64.1"},"devDependencies": {diff --git a/packages/templates/templates/playwright-ts/tsconfig.json b/packages/templates/templates/playwright-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/playwright-ts/tsconfig.json+++ b/packages/templates/templates/playwright-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/packages/templates/templates/puppeteer-js/package.json b/packages/templates/templates/puppeteer-js/package.jsonindex 564661ad4..c7842f5cb 100644--- a/packages/templates/templates/puppeteer-js/package.json+++ b/packages/templates/templates/puppeteer-js/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","puppeteer": "*"},"scripts": {diff --git a/packages/templates/templates/puppeteer-ts/package.json b/packages/templates/templates/puppeteer-ts/package.jsonindex 2f5996dde..769d5c3cb 100644--- a/packages/templates/templates/puppeteer-ts/package.json+++ b/packages/templates/templates/puppeteer-ts/package.json@@ -5,8 +5,8 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "@crawlee/impit-client": "^3.0.0",- "crawlee": "^3.0.0",+ "@crawlee/impit-client": "^4.0.0-rc.1",+ "crawlee": "^4.0.0-rc.1","puppeteer": "*"},"devDependencies": {diff --git a/packages/templates/templates/puppeteer-ts/tsconfig.json b/packages/templates/templates/puppeteer-ts/tsconfig.jsonindex c1be191a6..54395d8eb 100644--- a/packages/templates/templates/puppeteer-ts/tsconfig.json+++ b/packages/templates/templates/puppeteer-ts/tsconfig.json@@ -4,9 +4,11 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",+ "rootDir": "src","outDir": "dist","noUnusedLocals": false,- "lib": ["DOM"]+ "skipLibCheck": true,+ "lib": ["ES2023", "DOM"]},"include": ["./src/**/*"]}diff --git a/test/templates/template-dependencies.test.ts b/test/templates/template-dependencies.test.tsnew file mode 100644index 000000000..25a257720--- /dev/null+++ b/test/templates/template-dependencies.test.ts@@ -0,0 +1,21 @@+import { readdirSync, readFileSync } from 'node:fs';+import { resolve } from 'node:path';++import { satisfies } from 'semver';++const readJson = (path: string) => JSON.parse(readFileSync(path, 'utf8'));++const templatesDir = resolve(import.meta.dirname, '../../packages/templates/templates');+const { version } = readJson(resolve(import.meta.dirname, '../../packages/crawlee/package.json'));++describe('project templates', () => {+ test.each(readdirSync(templatesDir))('%s depends on the current crawlee version', (template) => {+ const { dependencies } = readJson(resolve(templatesDir, template, 'package.json'));++ for (const [name, range] of Object.entries<string>(dependencies)) {+ if (name === 'crawlee' || name.startsWith('@crawlee/')) {+ expect(satisfies(version, range), `${name}@${range}`).toBe(true);+ }+ }+ });+});