Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: ES2022 build compatibility and move to NodeNext for module (#2258)

Vlad Frangu7fe1e6859046
diff --git a/packages/templates/templates/cheerio-ts/tsconfig.json b/packages/templates/templates/cheerio-ts/tsconfig.json
index d7bd5a048..5b76f5cd3 100644
--- a/packages/templates/templates/cheerio-ts/tsconfig.json
+++ b/packages/templates/templates/cheerio-ts/tsconfig.json
@@ -1,7 +1,8 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "module": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
"target": "ES2022",
"outDir": "dist",
"noUnusedLocals": false,
diff --git a/packages/templates/templates/getting-started-ts/tsconfig.json b/packages/templates/templates/getting-started-ts/tsconfig.json
index d7bd5a048..5b76f5cd3 100644
--- a/packages/templates/templates/getting-started-ts/tsconfig.json
+++ b/packages/templates/templates/getting-started-ts/tsconfig.json
@@ -1,7 +1,8 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "module": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
"target": "ES2022",
"outDir": "dist",
"noUnusedLocals": false,
diff --git a/packages/templates/templates/playwright-ts/tsconfig.json b/packages/templates/templates/playwright-ts/tsconfig.json
index d7bd5a048..5b76f5cd3 100644
--- a/packages/templates/templates/playwright-ts/tsconfig.json
+++ b/packages/templates/templates/playwright-ts/tsconfig.json
@@ -1,7 +1,8 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "module": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
"target": "ES2022",
"outDir": "dist",
"noUnusedLocals": false,
diff --git a/packages/templates/templates/puppeteer-ts/tsconfig.json b/packages/templates/templates/puppeteer-ts/tsconfig.json
index d7bd5a048..5b76f5cd3 100644
--- a/packages/templates/templates/puppeteer-ts/tsconfig.json
+++ b/packages/templates/templates/puppeteer-ts/tsconfig.json
@@ -1,7 +1,8 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "module": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
"target": "ES2022",
"outDir": "dist",
"noUnusedLocals": false,
diff --git a/packages/utils/package.json b/packages/utils/package.json
index 8b7e5cbff..212a692fe 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -50,6 +50,7 @@
"@apify/log": "^2.4.0",
"@apify/ps-tree": "^1.2.0",
"@crawlee/types": "3.7.0",
+ "@types/sax": "^1.2.7",
"cheerio": "^1.0.0-rc.12",
"got-scraping": "^4.0.3",
"ow": "^0.28.1",
diff --git a/scripts/typescript_fixes.mjs b/scripts/typescript_fixes.mjs
index 716d87594..c5585ac0d 100644
--- a/scripts/typescript_fixes.mjs
+++ b/scripts/typescript_fixes.mjs
@@ -1,4 +1,5 @@
import { readFileSync, writeFileSync } from 'fs';
+
import { globby } from 'globby';
const files = await globby('packages/*/dist/**/*.d.ts');
@@ -17,13 +18,17 @@ for (const filepath of files) {
changed = true;
} else if (
// playwright/puppeteer/got-scraping import
- line.match(/^([^']+)'(playwright|puppeteer|got-scraping)'/) ||
+ line.match(/^([^']+)'(playwright|puppeteer|got-scraping)'/)
// proxy-per-page reexport of puppeteer
- line.match(/: Puppeteer\.\w+/) ||
+ || line.match(/: Puppeteer\.\w+/)
// don't ask me why, but this one is needed too ¯\_(ツ)_/¯
- line.match(/^export interface (PlaywrightHook|PuppeteerHook)/)
+ || line.match(/^export interface (PlaywrightHook|PuppeteerHook)/)
+ // /// <reference types="something" /> from newer nodenext resolutions
+ || line.match(/^\/\/\/ <reference types="[^"]+" \/>/)
+ // import("something") from compatibility with ES2022 module -.-
+ || line.match(/import\("([^"]+)"(?:.*)?\)/)
) {
- output.push('// @ts-ignore optional peer dependency');
+ output.push('// @ts-ignore optional peer dependency or compatibility with es2022');
output.push(line);
changed = true;
} else {
diff --git a/yarn.lock b/yarn.lock
index b93705399..ac3b6b7e9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -729,6 +729,7 @@ __metadata:
"@apify/log": "npm:^2.4.0"
"@apify/ps-tree": "npm:^1.2.0"
"@crawlee/types": "npm:3.7.0"
+ "@types/sax": "npm:^1.2.7"
cheerio: "npm:^1.0.0-rc.12"
got-scraping: "npm:^4.0.3"
ow: "npm:^0.28.1"
@@ -2112,7 +2113,7 @@ __metadata:
languageName: node
linkType: hard
-"@types/sax@npm:^1.0.0":
+"@types/sax@npm:^1.0.0, @types/sax@npm:^1.2.7":
version: 1.2.7
resolution: "@types/sax@npm:1.2.7"
dependencies: