Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: use JSON5 for INPUT.json (#1538)

Szymon Marczak09133ffa7444
diff --git a/package-lock.json b/package-lock.json
index 7dee88efd..55967fb3c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10580,7 +10580,6 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",
"integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",
- "dev": true,
"bin": {
"json5": "lib/cli.js"
},
@@ -16478,6 +16477,7 @@
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
"fs-extra": "^10.1.0",
+ "json5": "^2.2.1",
"minimatch": "^5.1.0",
"ow": "^0.28.1",
"stream-chain": "^2.2.5",
@@ -16601,6 +16601,7 @@
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
"fs-extra": "^10.1.0",
+ "json5": "^2.2.1",
"mime-types": "^2.1.35",
"proper-lockfile": "^4.1.2",
"tslib": "^2.4.0"
@@ -17814,6 +17815,7 @@
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
"fs-extra": "^10.1.0",
+ "json5": "*",
"minimatch": "^5.1.0",
"ow": "^0.28.1",
"stream-chain": "^2.2.5",
@@ -17885,6 +17887,7 @@
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
"fs-extra": "^10.1.0",
+ "json5": "^2.2.1",
"mime-types": "^2.1.35",
"proper-lockfile": "^4.1.2",
"tslib": "^2.4.0"
@@ -25516,8 +25519,7 @@
"json5": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",
- "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",
- "dev": true
+ "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA=="
},
"jsonc-parser": {
"version": "3.0.0",
diff --git a/packages/core/package.json b/packages/core/package.json
index 325dd684f..4877f486b 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -64,6 +64,7 @@
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
"fs-extra": "^10.1.0",
+ "json5": "^2.2.1",
"minimatch": "^5.1.0",
"ow": "^0.28.1",
"stream-chain": "^2.2.5",
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index 81153a149..5aecd0cb3 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -1,3 +1,4 @@
+import JSON5 from 'json5';
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
import { jsonStringifyExtended } from '@apify/utilities';
import ow, { ArgumentError } from 'ow';
@@ -466,7 +467,7 @@ export class KeyValueStore {
// Attempt to parse as JSON, or return the input as is otherwise
try {
- return JSON.parse(input.toString()) as T;
+ return JSON5.parse(input.toString()) as T;
} catch {
return input as unknown as T;
}
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index ecdd7bd03..35eb81543 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -54,6 +54,7 @@
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
"fs-extra": "^10.1.0",
+ "json5": "^2.2.1",
"mime-types": "^2.1.35",
"proper-lockfile": "^4.1.2",
"tslib": "^2.4.0"
diff --git a/packages/memory-storage/src/body-parser.ts b/packages/memory-storage/src/body-parser.ts
index 4b61708e0..1667a79f7 100644
--- a/packages/memory-storage/src/body-parser.ts
+++ b/packages/memory-storage/src/body-parser.ts
@@ -1,4 +1,5 @@
import contentTypeParser from 'content-type';
+import JSON5 from 'json5';
const CONTENT_TYPE_JSON = 'application/json';
const STRINGIFIABLE_CONTENT_TYPE_RXS = [
@@ -35,7 +36,7 @@ export function maybeParseBody(body: Buffer | ArrayBuffer, contentTypeHeader: st
const dataString = isomorphicBufferToString(body, charset);
return contentType === CONTENT_TYPE_JSON
- ? JSON.parse(dataString)
+ ? JSON5.parse(dataString)
: dataString;
}
diff --git a/test/e2e/input-json5/INPUT b/test/e2e/input-json5/INPUT
new file mode 100644
index 000000000..4fa5ce6e9
--- /dev/null
+++ b/test/e2e/input-json5/INPUT
@@ -0,0 +1,4 @@
+{
+ // Hello, world!
+ hello: 'world',
+}
diff --git a/test/e2e/input-json5/actor/.gitignore b/test/e2e/input-json5/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/input-json5/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfile
new file mode 100644
index 000000000..26898a6e1
--- /dev/null
+++ b/test/e2e/input-json5/actor/Dockerfile
@@ -0,0 +1,15 @@
+FROM apify/actor-node:16-beta
+
+COPY packages ./packages
+COPY package*.json ./
+
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+COPY . ./
diff --git a/test/e2e/input-json5/actor/apify.json b/test/e2e/input-json5/actor/apify.json
new file mode 100644
index 000000000..aa48ca695
--- /dev/null
+++ b/test/e2e/input-json5/actor/apify.json
@@ -0,0 +1,6 @@
+{
+ "name": "test-input-json5",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/input-json5/actor/main.js b/test/e2e/input-json5/actor/main.js
new file mode 100644
index 000000000..c991cdc85
--- /dev/null
+++ b/test/e2e/input-json5/actor/main.js
@@ -0,0 +1,18 @@
+import { join } from 'node:path';
+import { readFileSync } from 'node:fs';
+import { Actor, Dataset, KeyValueStore } from 'apify';
+import { ApifyStorageLocal } from '@apify/storage-local';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
+};
+
+await Actor.setValue('INPUT', readFileSync(join(process.cwd(), 'INPUT')).toString(), {
+ contentType: 'application/json',
+});
+
+await Actor.main(async () => {
+ await Dataset.pushData(await KeyValueStore.getInput());
+ await Dataset.pushData(await Actor.getInput());
+}, mainOptions);
diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.json
new file mode 100644
index 000000000..1a91ede4b
--- /dev/null
+++ b/test/e2e/input-json5/actor/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "test-input-json5",
+ "version": "0.0.1",
+ "description": "JSON5 input test",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.0"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/input-json5/test.mjs b/test/e2e/input-json5/test.mjs
new file mode 100644
index 000000000..1734b68b1
--- /dev/null
+++ b/test/e2e/input-json5/test.mjs
@@ -0,0 +1,16 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname);
+
+await expect(datasetItems.length === 2, 'Number of dataset items');
+await expect(JSON.stringify(datasetItems) === JSON.stringify([
+ {
+ hello: 'world',
+ },
+ {
+ hello: 'world',
+ },
+]), 'Dataset items validation');
diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs
index 475f2f14a..70f3fc66c 100644
--- a/test/e2e/run.mjs
+++ b/test/e2e/run.mjs
@@ -1,4 +1,4 @@
-import { dirname } from 'node:path';
+import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { once } from 'node:events';
import { readdir } from 'node:fs/promises';
@@ -131,5 +131,9 @@ if (isMainThread) {
// We want to exit with non-zero code if any of the tests failed
if (failure) process.exit(1);
} else {
+ // Mimic CWD for proper INPUT loading
+ // See packages/core/src/storages/key_value_store.ts
+ process.cwd = () => join(basePath, workerData);
+
await import(`${basePath}/${workerData}/test.mjs`);
}