Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(KeyValueStore): big buffers should not crash (#1734)
Vlad Frangu2f682f7ddd18
diff --git a/package-lock.json b/package-lock.jsonindex 2e7ec2da7..63793fee3 100644--- a/package-lock.json+++ b/package-lock.json@@ -40,6 +40,7 @@"basic-auth-parser": "^0.0.2","body-parser": "^1.20.0","commitlint": "^17.1.2",+ "cross-env": "^7.0.3","deep-equal": "^2.0.5","eslint": "^8.24.0","express": "^4.18.1",@@ -6266,6 +6267,24 @@"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==","dev": true},+ "node_modules/cross-env": {+ "version": "7.0.3",+ "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",+ "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",+ "dev": true,+ "dependencies": {+ "cross-spawn": "^7.0.1"+ },+ "bin": {+ "cross-env": "src/bin/cross-env.js",+ "cross-env-shell": "src/bin/cross-env-shell.js"+ },+ "engines": {+ "node": ">=10.14",+ "npm": ">=6",+ "yarn": ">=1"+ }+ },"node_modules/cross-fetch": {"version": "3.1.5","resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",diff --git a/package.json b/package.jsonindex 05b3732cc..d06f3d448 100644--- a/package.json+++ b/package.json@@ -43,6 +43,7 @@"postci:build": "node ./scripts/typescript_fixes.mjs","test": "jest --silent","test:e2e": "node test/e2e/run.mjs",+ "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 jest --silent","coverage": "jest --coverage","publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish","release:next": "npm run build && npm run publish:next",@@ -80,6 +81,7 @@"basic-auth-parser": "^0.0.2","body-parser": "^1.20.0","commitlint": "^17.1.2",+ "cross-env": "^7.0.3","deep-equal": "^2.0.5","eslint": "^8.24.0","express": "^4.18.1",diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex 1da291e89..5b3ae1699 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -211,7 +211,7 @@ export class KeyValueStoreClient extends BaseClient {async setRecord(record: storage.KeyValueStoreRecord): Promise<void> {s.object({key: s.string.lengthGreaterThan(0),- value: s.union(s.null, s.string, s.number, s.object({}).passthrough),+ value: s.union(s.null, s.string, s.number, s.instance(Buffer), s.object({}).passthrough),contentType: s.string.lengthGreaterThan(0).optional,}).parse(record);diff --git a/packages/memory-storage/test/no-crash-on-big-buffers.test.ts b/packages/memory-storage/test/no-crash-on-big-buffers.test.tsnew file mode 100644index 000000000..84d0e8595--- /dev/null+++ b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts@@ -0,0 +1,45 @@+// https://github.com/apify/crawlee/issues/1732+// https://github.com/apify/crawlee/issues/1710++import { rm } from 'node:fs/promises';+import { resolve } from 'node:path';+import { MemoryStorage } from '@crawlee/memory-storage';+import type { KeyValueStoreClient, KeyValueStoreInfo } from '@crawlee/types';++describe('MemoryStorage should not crash when saving a big buffer', () => {+ const tmpLocation = resolve(__dirname, './tmp/no-buffer-crash');+ const storage = new MemoryStorage({+ localDataDirectory: tmpLocation,+ persistStorage: false,+ });++ let kvs: KeyValueStoreInfo;+ let store: KeyValueStoreClient;++ beforeAll(async () => {+ kvs = await storage.keyValueStores().getOrCreate();+ store = storage.keyValueStore(kvs.id);+ });++ afterAll(async () => {+ await rm(tmpLocation, { force: true, recursive: true });+ });++ test('should not crash when saving a big buffer', async () => {+ let zip: Buffer;++ if (process.env.CRAWLEE_DIFFICULT_TESTS) {+ const numbers = Array.from(([...Array(18_100_000).keys()]).map((i) => i * 3_000_000));++ zip = Buffer.from([...numbers]);+ } else {+ zip = Buffer.from([...Array(100_000)].map((i) => i * 8));+ }++ try {+ await store.setRecord({ key: 'owo.zip', value: zip });+ } catch (err) {+ expect(err).not.toBeDefined();+ }+ });+});