Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: use JSON5 for INPUT.json (#1538)
Szymon Marczak09133ffa7444
diff --git a/package-lock.json b/package-lock.jsonindex 7dee88efd..55967fb3c 100644--- a/package-lock.json+++ b/package-lock.json@@ -10580,7 +10580,6 @@"version": "2.2.1","resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz","integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",- "dev": true,"bin": {"json5": "lib/cli.js"},@@ -16478,6 +16477,7 @@"@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",+ "json5": "^2.2.1","minimatch": "^5.1.0","ow": "^0.28.1","stream-chain": "^2.2.5",@@ -16601,6 +16601,7 @@"@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^10.1.0",+ "json5": "^2.2.1","mime-types": "^2.1.35","proper-lockfile": "^4.1.2","tslib": "^2.4.0"@@ -17814,6 +17815,7 @@"@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",+ "json5": "*","minimatch": "^5.1.0","ow": "^0.28.1","stream-chain": "^2.2.5",@@ -17885,6 +17887,7 @@"@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^10.1.0",+ "json5": "^2.2.1","mime-types": "^2.1.35","proper-lockfile": "^4.1.2","tslib": "^2.4.0"@@ -25516,8 +25519,7 @@"json5": {"version": "2.2.1","resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",- "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",- "dev": true+ "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA=="},"jsonc-parser": {"version": "3.0.0",diff --git a/packages/core/package.json b/packages/core/package.jsonindex 325dd684f..4877f486b 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -64,6 +64,7 @@"@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",+ "json5": "^2.2.1","minimatch": "^5.1.0","ow": "^0.28.1","stream-chain": "^2.2.5",diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 81153a149..5aecd0cb3 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -1,3 +1,4 @@+import JSON5 from 'json5';import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';import { jsonStringifyExtended } from '@apify/utilities';import ow, { ArgumentError } from 'ow';@@ -466,7 +467,7 @@ export class KeyValueStore {// Attempt to parse as JSON, or return the input as is otherwisetry {- return JSON.parse(input.toString()) as T;+ return JSON5.parse(input.toString()) as T;} catch {return input as unknown as T;}diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex ecdd7bd03..35eb81543 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -54,6 +54,7 @@"@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^10.1.0",+ "json5": "^2.2.1","mime-types": "^2.1.35","proper-lockfile": "^4.1.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/src/body-parser.ts b/packages/memory-storage/src/body-parser.tsindex 4b61708e0..1667a79f7 100644--- a/packages/memory-storage/src/body-parser.ts+++ b/packages/memory-storage/src/body-parser.ts@@ -1,4 +1,5 @@import contentTypeParser from 'content-type';+import JSON5 from 'json5';const CONTENT_TYPE_JSON = 'application/json';const STRINGIFIABLE_CONTENT_TYPE_RXS = [@@ -35,7 +36,7 @@ export function maybeParseBody(body: Buffer | ArrayBuffer, contentTypeHeader: stconst dataString = isomorphicBufferToString(body, charset);return contentType === CONTENT_TYPE_JSON- ? JSON.parse(dataString)+ ? JSON5.parse(dataString): dataString;}diff --git a/test/e2e/input-json5/INPUT b/test/e2e/input-json5/INPUTnew file mode 100644index 000000000..4fa5ce6e9--- /dev/null+++ b/test/e2e/input-json5/INPUT@@ -0,0 +1,4 @@+{+ // Hello, world!+ hello: 'world',+}diff --git a/test/e2e/input-json5/actor/.gitignore b/test/e2e/input-json5/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/input-json5/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfilenew file mode 100644index 000000000..26898a6e1--- /dev/null+++ b/test/e2e/input-json5/actor/Dockerfile@@ -0,0 +1,15 @@+FROM apify/actor-node:16-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/input-json5/actor/apify.json b/test/e2e/input-json5/actor/apify.jsonnew file mode 100644index 000000000..aa48ca695--- /dev/null+++ b/test/e2e/input-json5/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-input-json5",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/input-json5/actor/main.js b/test/e2e/input-json5/actor/main.jsnew file mode 100644index 000000000..c991cdc85--- /dev/null+++ b/test/e2e/input-json5/actor/main.js@@ -0,0 +1,18 @@+import { join } from 'node:path';+import { readFileSync } from 'node:fs';+import { Actor, Dataset, KeyValueStore } from 'apify';+import { ApifyStorageLocal } from '@apify/storage-local';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++await Actor.setValue('INPUT', readFileSync(join(process.cwd(), 'INPUT')).toString(), {+ contentType: 'application/json',+});++await Actor.main(async () => {+ await Dataset.pushData(await KeyValueStore.getInput());+ await Dataset.pushData(await Actor.getInput());+}, mainOptions);diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.jsonnew file mode 100644index 000000000..1a91ede4b--- /dev/null+++ b/test/e2e/input-json5/actor/package.json@@ -0,0 +1,20 @@+{+ "name": "test-input-json5",+ "version": "0.0.1",+ "description": "JSON5 input test",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.0"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/input-json5/test.mjs b/test/e2e/input-json5/test.mjsnew file mode 100644index 000000000..1734b68b1--- /dev/null+++ b/test/e2e/input-json5/test.mjs@@ -0,0 +1,16 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname);++await expect(datasetItems.length === 2, 'Number of dataset items');+await expect(JSON.stringify(datasetItems) === JSON.stringify([+ {+ hello: 'world',+ },+ {+ hello: 'world',+ },+]), 'Dataset items validation');diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex 475f2f14a..70f3fc66c 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -1,4 +1,4 @@-import { dirname } from 'node:path';+import { dirname, join } from 'node:path';import { fileURLToPath } from 'node:url';import { once } from 'node:events';import { readdir } from 'node:fs/promises';@@ -131,5 +131,9 @@ if (isMainThread) {// We want to exit with non-zero code if any of the tests failedif (failure) process.exit(1);} else {+ // Mimic CWD for proper INPUT loading+ // See packages/core/src/storages/key_value_store.ts+ process.cwd = () => join(basePath, workerData);+await import(`${basePath}/${workerData}/test.mjs`);}