Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: ES2022 build compatibility and move to NodeNext for module (#2258)
Vlad Frangu7fe1e6859046
diff --git a/packages/templates/templates/cheerio-ts/tsconfig.json b/packages/templates/templates/cheerio-ts/tsconfig.jsonindex d7bd5a048..5b76f5cd3 100644--- a/packages/templates/templates/cheerio-ts/tsconfig.json+++ b/packages/templates/templates/cheerio-ts/tsconfig.json@@ -1,7 +1,8 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ES2022","outDir": "dist","noUnusedLocals": false,diff --git a/packages/templates/templates/getting-started-ts/tsconfig.json b/packages/templates/templates/getting-started-ts/tsconfig.jsonindex d7bd5a048..5b76f5cd3 100644--- a/packages/templates/templates/getting-started-ts/tsconfig.json+++ b/packages/templates/templates/getting-started-ts/tsconfig.json@@ -1,7 +1,8 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ES2022","outDir": "dist","noUnusedLocals": false,diff --git a/packages/templates/templates/playwright-ts/tsconfig.json b/packages/templates/templates/playwright-ts/tsconfig.jsonindex d7bd5a048..5b76f5cd3 100644--- a/packages/templates/templates/playwright-ts/tsconfig.json+++ b/packages/templates/templates/playwright-ts/tsconfig.json@@ -1,7 +1,8 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ES2022","outDir": "dist","noUnusedLocals": false,diff --git a/packages/templates/templates/puppeteer-ts/tsconfig.json b/packages/templates/templates/puppeteer-ts/tsconfig.jsonindex d7bd5a048..5b76f5cd3 100644--- a/packages/templates/templates/puppeteer-ts/tsconfig.json+++ b/packages/templates/templates/puppeteer-ts/tsconfig.json@@ -1,7 +1,8 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ES2022","outDir": "dist","noUnusedLocals": false,diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 8b7e5cbff..212a692fe 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -50,6 +50,7 @@"@apify/log": "^2.4.0","@apify/ps-tree": "^1.2.0","@crawlee/types": "3.7.0",+ "@types/sax": "^1.2.7","cheerio": "^1.0.0-rc.12","got-scraping": "^4.0.3","ow": "^0.28.1",diff --git a/scripts/typescript_fixes.mjs b/scripts/typescript_fixes.mjsindex 716d87594..c5585ac0d 100644--- a/scripts/typescript_fixes.mjs+++ b/scripts/typescript_fixes.mjs@@ -1,4 +1,5 @@import { readFileSync, writeFileSync } from 'fs';+import { globby } from 'globby';const files = await globby('packages/*/dist/**/*.d.ts');@@ -17,13 +18,17 @@ for (const filepath of files) {changed = true;} else if (// playwright/puppeteer/got-scraping import- line.match(/^([^']+)'(playwright|puppeteer|got-scraping)'/) ||+ line.match(/^([^']+)'(playwright|puppeteer|got-scraping)'/)// proxy-per-page reexport of puppeteer- line.match(/: Puppeteer\.\w+/) ||+ || line.match(/: Puppeteer\.\w+/)// don't ask me why, but this one is needed too ¯\_(ツ)_/¯- line.match(/^export interface (PlaywrightHook|PuppeteerHook)/)+ || line.match(/^export interface (PlaywrightHook|PuppeteerHook)/)+ // /// <reference types="something" /> from newer nodenext resolutions+ || line.match(/^\/\/\/ <reference types="[^"]+" \/>/)+ // import("something") from compatibility with ES2022 module -.-+ || line.match(/import\("([^"]+)"(?:.*)?\)/)) {- output.push('// @ts-ignore optional peer dependency');+ output.push('// @ts-ignore optional peer dependency or compatibility with es2022');output.push(line);changed = true;} else {diff --git a/yarn.lock b/yarn.lockindex b93705399..ac3b6b7e9 100644--- a/yarn.lock+++ b/yarn.lock@@ -729,6 +729,7 @@ __metadata:"@apify/log": "npm:^2.4.0""@apify/ps-tree": "npm:^1.2.0""@crawlee/types": "npm:3.7.0"+ "@types/sax": "npm:^1.2.7"cheerio: "npm:^1.0.0-rc.12"got-scraping: "npm:^4.0.3"ow: "npm:^0.28.1"@@ -2112,7 +2113,7 @@ __metadata:languageName: nodelinkType: hard-"@types/sax@npm:^1.0.0":+"@types/sax@npm:^1.0.0, @types/sax@npm:^1.2.7":version: 1.2.7resolution: "@types/sax@npm:1.2.7"dependencies: