Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(KeyValueStore): big buffers should not crash (#1734)

Vlad Frangu2f682f7ddd18
diff --git a/package-lock.json b/package-lock.json
index 2e7ec2da7..63793fee3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -40,6 +40,7 @@
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
"commitlint": "^17.1.2",
+ "cross-env": "^7.0.3",
"deep-equal": "^2.0.5",
"eslint": "^8.24.0",
"express": "^4.18.1",
@@ -6266,6 +6267,24 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true
},
+ "node_modules/cross-env": {
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
+ "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",
+ "dev": true,
+ "dependencies": {
+ "cross-spawn": "^7.0.1"
+ },
+ "bin": {
+ "cross-env": "src/bin/cross-env.js",
+ "cross-env-shell": "src/bin/cross-env-shell.js"
+ },
+ "engines": {
+ "node": ">=10.14",
+ "npm": ">=6",
+ "yarn": ">=1"
+ }
+ },
"node_modules/cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
diff --git a/package.json b/package.json
index 05b3732cc..d06f3d448 100644
--- a/package.json
+++ b/package.json
@@ -43,6 +43,7 @@
"postci:build": "node ./scripts/typescript_fixes.mjs",
"test": "jest --silent",
"test:e2e": "node test/e2e/run.mjs",
+ "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 jest --silent",
"coverage": "jest --coverage",
"publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish",
"release:next": "npm run build && npm run publish:next",
@@ -80,6 +81,7 @@
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
"commitlint": "^17.1.2",
+ "cross-env": "^7.0.3",
"deep-equal": "^2.0.5",
"eslint": "^8.24.0",
"express": "^4.18.1",
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts
index 1da291e89..5b3ae1699 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts
@@ -211,7 +211,7 @@ export class KeyValueStoreClient extends BaseClient {
async setRecord(record: storage.KeyValueStoreRecord): Promise<void> {
s.object({
key: s.string.lengthGreaterThan(0),
- value: s.union(s.null, s.string, s.number, s.object({}).passthrough),
+ value: s.union(s.null, s.string, s.number, s.instance(Buffer), s.object({}).passthrough),
contentType: s.string.lengthGreaterThan(0).optional,
}).parse(record);
diff --git a/packages/memory-storage/test/no-crash-on-big-buffers.test.ts b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts
new file mode 100644
index 000000000..84d0e8595
--- /dev/null
+++ b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts
@@ -0,0 +1,45 @@
+// https://github.com/apify/crawlee/issues/1732
+// https://github.com/apify/crawlee/issues/1710
+
+import { rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { MemoryStorage } from '@crawlee/memory-storage';
+import type { KeyValueStoreClient, KeyValueStoreInfo } from '@crawlee/types';
+
+describe('MemoryStorage should not crash when saving a big buffer', () => {
+ const tmpLocation = resolve(__dirname, './tmp/no-buffer-crash');
+ const storage = new MemoryStorage({
+ localDataDirectory: tmpLocation,
+ persistStorage: false,
+ });
+
+ let kvs: KeyValueStoreInfo;
+ let store: KeyValueStoreClient;
+
+ beforeAll(async () => {
+ kvs = await storage.keyValueStores().getOrCreate();
+ store = storage.keyValueStore(kvs.id);
+ });
+
+ afterAll(async () => {
+ await rm(tmpLocation, { force: true, recursive: true });
+ });
+
+ test('should not crash when saving a big buffer', async () => {
+ let zip: Buffer;
+
+ if (process.env.CRAWLEE_DIFFICULT_TESTS) {
+ const numbers = Array.from(([...Array(18_100_000).keys()]).map((i) => i * 3_000_000));
+
+ zip = Buffer.from([...numbers]);
+ } else {
+ zip = Buffer.from([...Array(100_000)].map((i) => i * 8));
+ }
+
+ try {
+ await store.setRecord({ key: 'owo.zip', value: zip });
+ } catch (err) {
+ expect(err).not.toBeDefined();
+ }
+ });
+});