Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf: add lazyIterablePromise to kvs iterators (#3413)
Richard Solar2658610a62ae
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex ecf8148b4..66c6a3101 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -16,7 +16,7 @@ import { DEFAULT_API_PARAM_LIMIT, StorageTypes } from '../consts';import type { StorageImplementation } from '../fs/common';import { createKeyValueStorageImplementation } from '../fs/key-value-store';import type { MemoryStorage } from '../index';-import { createKeyList, createKeyStringList, isBuffer, isStream } from '../utils';+import { createKeyList, createKeyStringList, createLazyIterablePromise, isBuffer, isStream } from '../utils';import { BaseClient } from './common/base-client';const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';@@ -173,33 +173,32 @@ export class KeyValueStoreClient extends BaseClient {const firstPageKeysPromise = keys(options);- const firstPageValuesPromise = (async () => {+ const getFirstPageValues = async () => {const firstPageKeys = await firstPageKeysPromise;const keysToFetch = limit !== undefined ? firstPageKeys.items.slice(0, limit) : firstPageKeys.items;const limiter = pLimit(GET_RECORD_CONCURRENCY);- const results = await Promise.allSettled(keysToFetch.map((item) => limiter(() => getRecord(item.key))));- return results- .filter(- (r): r is PromiseFulfilledResult<storage.KeyValueStoreRecord> =>- r.status === 'fulfilled' && r.value !== undefined,- )- .map((r) => r.value.value);- })();+ const results = await Promise.all(keysToFetch.map((item) => limiter(() => getRecord(item.key))));+ return results.filter((r) => r !== undefined).map((r) => r.value);+ };async function* asyncGenerator(): AsyncGenerator<unknown> {- // Reuse the already-fetched first page values- const firstPageValues = await firstPageValuesPromise;+ const firstPageKeys = await firstPageKeysPromise;let yielded = 0;- for (const value of firstPageValues) {+ for (const item of firstPageKeys.items) {if (limit !== undefined && yielded >= limit) return;- yield value;- yielded++;+ const record = await getRecord(item.key);+ if (record) {+ yield record.value;+ yielded++;+ }}- const firstPageKeys = await firstPageKeysPromise;if (firstPageKeys.nextExclusiveStartKey && (limit === undefined || yielded < limit)) {- for await (const key of keys({ ...options, exclusiveStartKey: firstPageKeys.nextExclusiveStartKey })) {+ for await (const key of keys({+ ...options,+ exclusiveStartKey: firstPageKeys.nextExclusiveStartKey,+ })) {if (limit !== undefined && yielded >= limit) return;const record = await getRecord(key);if (record) {@@ -210,9 +209,7 @@ export class KeyValueStoreClient extends BaseClient {}}- return Object.defineProperty(firstPageValuesPromise, Symbol.asyncIterator, {- value: asyncGenerator,- }) as AsyncIterable<unknown> & Promise<unknown[]>;+ return createLazyIterablePromise(getFirstPageValues, asyncGenerator);}entries(@@ -224,36 +221,38 @@ export class KeyValueStoreClient extends BaseClient {const firstPageKeysPromise = keys(options);- const firstPageEntriesPromise = (async () => {+ const getFirstPageEntries = async () => {const firstPageKeys = await firstPageKeysPromise;const keysToFetch = limit !== undefined ? firstPageKeys.items.slice(0, limit) : firstPageKeys.items;const limiter = pLimit(GET_RECORD_CONCURRENCY);- const results = await Promise.allSettled(+ const results = await Promise.all(keysToFetch.map((item) =>limiter(() => getRecord(item.key).then((record) => ({ key: item.key, record }))),),);return results- .filter(- (r): r is PromiseFulfilledResult<{ key: string; record: storage.KeyValueStoreRecord }> =>- r.status === 'fulfilled' && r.value.record !== undefined,- )- .map((r) => [r.value.key, r.value.record.value] as [string, unknown]);- })();+ .filter((r) => r.record !== undefined)+ .map((r) => [r.key, r.record!.value] as [string, unknown]);+ };async function* asyncGenerator(): AsyncGenerator<[string, unknown]> {- const firstPageEntries = await firstPageEntriesPromise;+ const firstPageKeys = await firstPageKeysPromise;let yielded = 0;- for (const entry of firstPageEntries) {+ for (const item of firstPageKeys.items) {if (limit !== undefined && yielded >= limit) return;- yield entry;- yielded++;+ const record = await getRecord(item.key);+ if (record) {+ yield [item.key, record.value];+ yielded++;+ }}- const firstPageKeys = await firstPageKeysPromise;if (firstPageKeys.nextExclusiveStartKey && (limit === undefined || yielded < limit)) {- for await (const key of keys({ ...options, exclusiveStartKey: firstPageKeys.nextExclusiveStartKey })) {+ for await (const key of keys({+ ...options,+ exclusiveStartKey: firstPageKeys.nextExclusiveStartKey,+ })) {if (limit !== undefined && yielded >= limit) return;const record = await getRecord(key);if (record) {@@ -264,9 +263,7 @@ export class KeyValueStoreClient extends BaseClient {}}- return Object.defineProperty(firstPageEntriesPromise, Symbol.asyncIterator, {- value: asyncGenerator,- }) as AsyncIterable<[string, unknown]> & Promise<[string, unknown][]>;+ return createLazyIterablePromise(getFirstPageEntries, asyncGenerator);}private async listKeysPage(diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.tsindex a22fa1a9a..8a74bb33b 100644--- a/packages/memory-storage/src/utils.ts+++ b/packages/memory-storage/src/utils.ts@@ -222,3 +222,42 @@ export function createPaginatedEntryList<Data>(value: asyncGenerator,}) as AsyncIterable<[number, Data]> & Promise<storage.PaginatedList<[number, Data]>>;}++/**+ * Creates an object that acts as both a lazy Promise and an AsyncIterable.+ * - When awaited, it triggers `promiseFactory` (bulk fetch, cached after first call).+ * - When iterated with `for await...of`, it uses `iteratorFactory` (streaming, no bulk fetch).+ */+export function createLazyIterablePromise<TPromise, TElement>(+ promiseFactory: () => Promise<TPromise>,+ iteratorFactory: () => AsyncGenerator<TElement>,+): AsyncIterable<TElement> & Promise<TPromise> {+ let cached: Promise<TPromise> | null = null;+ function getOrCreate(): Promise<TPromise> {+ if (!cached) {+ cached = promiseFactory();+ }+ return cached;+ }++ const result = {+ then<TResult1 = TPromise, TResult2 = never>(+ onfulfilled?: ((value: TPromise) => TResult1 | PromiseLike<TResult1>) | null,+ onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null,+ ): Promise<TResult1 | TResult2> {+ return getOrCreate().then(onfulfilled, onrejected);+ },+ catch<TResult = never>(+ onrejected?: ((reason: any) => TResult | PromiseLike<TResult>) | null,+ ): Promise<TPromise | TResult> {+ return getOrCreate().catch(onrejected);+ },+ finally(onfinally?: (() => void) | null): Promise<TPromise> {+ return getOrCreate().finally(onfinally);+ },+ [Symbol.asyncIterator]: iteratorFactory,+ [Symbol.toStringTag]: 'Promise' as const,+ };++ return result as AsyncIterable<TElement> & Promise<TPromise>;+}diff --git a/packages/memory-storage/test/async-iteration.test.ts b/packages/memory-storage/test/async-iteration.test.tsindex c82f98dd7..8303dd61d 100644--- a/packages/memory-storage/test/async-iteration.test.ts+++ b/packages/memory-storage/test/async-iteration.test.ts@@ -1,11 +1,14 @@import { rm } from 'node:fs/promises';-import { resolve } from 'node:path';+import path from 'node:path';import { MemoryStorage } from '@crawlee/memory-storage';import type { DatasetClient, KeyValueStoreClient } from '@crawlee/types';+import { vi } from 'vitest';++import { createLazyIterablePromise } from '../src/utils';describe('Async iteration support', () => {- const localDataDirectory = resolve(__dirname, './tmp/async-iteration');+ const localDataDirectory = path.resolve(__dirname, './tmp/async-iteration');const storage = new MemoryStorage({localDataDirectory,persistStorage: false,@@ -429,4 +432,136 @@ describe('Async iteration support', () => {}});});++ describe('createLazyIterablePromise', () => {+ test('promise factory is not called until awaited', async () => {+ const promiseFactory = vi.fn(() => Promise.resolve([1, 2, 3]));+ async function* iteratorFactory() {+ yield 1;+ yield 2;+ yield 3;+ }++ const result = createLazyIterablePromise<number[], number>(promiseFactory, iteratorFactory);++ // Factory should not be called yet+ expect(promiseFactory).not.toHaveBeenCalled();++ // Now await it+ const values = await result;+ expect(promiseFactory).toHaveBeenCalledTimes(1);+ expect(values).toStrictEqual([1, 2, 3]);+ });++ test('iterating does not trigger the promise factory', async () => {+ const promiseFactory = vi.fn(() => Promise.resolve([1, 2, 3]));+ async function* iteratorFactory() {+ yield 10;+ yield 20;+ yield 30;+ }++ const result = createLazyIterablePromise<number[], number>(promiseFactory, iteratorFactory);++ const items: number[] = [];+ for await (const item of result) {+ items.push(item);+ }++ expect(items).toStrictEqual([10, 20, 30]);+ expect(promiseFactory).not.toHaveBeenCalled();+ });++ test('promise factory result is cached across multiple awaits', async () => {+ const promiseFactory = vi.fn(() => Promise.resolve([1, 2, 3]));+ async function* iteratorFactory() {+ yield 1;+ }++ const result = createLazyIterablePromise<number[], number>(promiseFactory, iteratorFactory);++ await result;+ await result;+ await result;++ expect(promiseFactory).toHaveBeenCalledTimes(1);+ });+ });++ describe('KeyValueStore.values lazy promise behavior', () => {+ let kvStore: KeyValueStoreClient;++ beforeAll(async () => {+ const { id } = await storage.keyValueStores().getOrCreate('lazy-test-kvs-values');+ kvStore = storage.keyValueStore(id);++ for (let i = 0; i < 5; i++) {+ await kvStore.setRecord({ key: `key-${i}`, value: { data: i } });+ }+ });++ test('calling values() does not immediately fetch records', async () => {+ const getRecordSpy = vi.spyOn(kvStore, 'getRecord');++ // Call values() but do not await or iterate+ const result = kvStore.values();++ // getRecord should not have been called yet (lazy)+ // Note: keys may be fetched eagerly, but record values should not+ // We need to wait a tick to ensure no async work triggered getRecord+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(getRecordSpy).not.toHaveBeenCalled();++ // Clean up: consume the result to avoid dangling promises+ await result;+ getRecordSpy.mockRestore();+ });++ test('iterating and awaiting produce the same values', async () => {+ const awaited = await kvStore.values();++ const iterated: unknown[] = [];+ for await (const value of kvStore.values()) {+ iterated.push(value);+ }++ expect(awaited).toStrictEqual(iterated);+ });+ });++ describe('KeyValueStore.entries lazy promise behavior', () => {+ let kvStore: KeyValueStoreClient;++ beforeAll(async () => {+ const { id } = await storage.keyValueStores().getOrCreate('lazy-test-kvs-entries');+ kvStore = storage.keyValueStore(id);++ for (let i = 0; i < 5; i++) {+ await kvStore.setRecord({ key: `key-${i}`, value: { data: i } });+ }+ });++ test('calling entries() does not immediately fetch records', async () => {+ const getRecordSpy = vi.spyOn(kvStore, 'getRecord');++ const result = kvStore.entries();++ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(getRecordSpy).not.toHaveBeenCalled();++ await result;+ getRecordSpy.mockRestore();+ });++ test('iterating and awaiting produce the same entries', async () => {+ const awaited = await kvStore.entries();++ const iterated: [string, unknown][] = [];+ for await (const entry of kvStore.entries()) {+ iterated.push(entry);+ }++ expect(awaited).toStrictEqual(iterated);+ });+ });});