Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(MemoryStorage): handling of readable streams for key-value stores when setting records (#1852)
Vlad Frangua5ee37d7e245
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex 3e7eec4b2..10157a469 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -217,7 +217,16 @@ export class KeyValueStoreClient extends BaseClient {async setRecord(record: storage.KeyValueStoreRecord): Promise<void> {s.object({key: s.string.lengthGreaterThan(0),- value: s.union(s.null, s.string, s.number, s.instance(Buffer), s.object({}).passthrough),+ value: s.union(+ s.null,+ s.string,+ s.number,+ s.instance(Buffer),+ s.instance(ArrayBuffer),+ s.typedArray(),+ // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array+ s.object({}).setValidationEnabled(false),+ ),contentType: s.string.lengthGreaterThan(0).optional,}).parse(record);diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.tsindex 455de00c1..29baa9962 100644--- a/packages/memory-storage/src/utils.ts+++ b/packages/memory-storage/src/utils.ts@@ -33,9 +33,9 @@ export function uniqueKeyToRequestId(uniqueKey: string): string {export function isBuffer(value: unknown): boolean {try {s.union(- s.typedArray(),- s.instance(ArrayBuffer),s.instance(Buffer),+ s.instance(ArrayBuffer),+ s.typedArray(),).parse(value);return true;@@ -44,17 +44,8 @@ export function isBuffer(value: unknown): boolean {}}-export function isStream(value: unknown): boolean {- try {- s.object({- on: s.any,- pipe: s.any,- }).passthrough.parse(value);-- return true;- } catch {- return false;- }+export function isStream(value: any): boolean {+ return typeof value === 'object' && value && ['on', 'pipe'].every((key) => key in value && typeof value[key] === 'function');}export const memoryStorageLog = defaultLog.child({ prefix: 'MemoryStorage' });diff --git a/packages/memory-storage/test/key-value-store-stream.test.ts b/packages/memory-storage/test/key-value-store-stream.test.tsnew file mode 100644index 000000000..5eb6ae51e--- /dev/null+++ b/packages/memory-storage/test/key-value-store-stream.test.ts@@ -0,0 +1,22 @@+import { MemoryStorage } from '@crawlee/memory-storage';+import { Readable } from 'node:stream';++describe('KeyValueStore should drain streams when setting records', () => {+ const storage = new MemoryStorage({+ persistStorage: false,+ });++ const fsStream = Readable.from([Buffer.from('hello'), Buffer.from('world')]);++ test('should drain stream', async () => {+ const defaultStoreInfo = await storage.keyValueStores().getOrCreate('default');+ const defaultStore = storage.keyValueStore(defaultStoreInfo.id);++ await defaultStore.setRecord({ key: 'streamz', value: fsStream, contentType: 'text/plain' });++ expect(fsStream.destroyed).toBeTruthy();++ const record = await defaultStore.getRecord('streamz');+ expect(record!.value.toString('utf8')).toEqual('helloworld');+ });+});