Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

4.1 KB
Raw Blame
import { MemoryStorageBackend } from '@crawlee/core';
import type { DatasetBackend, KeyValueStoreBackend } from '@crawlee/types';

describe('Async iteration support', () => {
    const storage = new MemoryStorageBackend();

    describe('Dataset.getData', () => {
        const elements = Array.from({ length: 25 }, (_, i) => ({ index: i }));
        let dataset: DatasetBackend<{ index: number }>;

        beforeAll(async () => {
            dataset = (await storage.createDatasetBackend({ name: 'async-iteration-dataset' })) as DatasetBackend<{
                index: number;
            }>;
            await dataset.pushData(elements);
        });

        test('getData returns a paginated result', async () => {
            const result = await dataset.getData({ limit: 10 });

            expect(result.items).toHaveLength(10);
            expect(result.total).toBe(25);
            expect(result.offset).toBe(0);
            expect(result.items).toStrictEqual(elements.slice(0, 10));
        });

        test('respects limit option', async () => {
            const result = await dataset.getData({ limit: 10 });

            expect(result.items).toHaveLength(10);
            expect(result.items).toStrictEqual(elements.slice(0, 10));
        });

        test('respects offset option', async () => {
            const result = await dataset.getData({ offset: 5 });

            expect(result.items).toHaveLength(20);
            expect(result.items).toStrictEqual(elements.slice(5));
        });

        test('respects both offset and limit options', async () => {
            const result = await dataset.getData({ offset: 5, limit: 10 });

            expect(result.items).toHaveLength(10);
            expect(result.items).toStrictEqual(elements.slice(5, 15));
        });

        test('respects desc option', async () => {
            const result = await dataset.getData({ desc: true, limit: 5 });

            expect(result.items).toHaveLength(5);
            expect(result.items).toStrictEqual(elements.slice().reverse().slice(0, 5));
        });
    });

    describe('KeyValueStore.listKeys', () => {
        const keys = Array.from({ length: 25 }, (_, i) => `key-${String(i).padStart(2, '0')}`);
        let kvStore: KeyValueStoreBackend;

        beforeAll(async () => {
            kvStore = await storage.createKeyValueStoreBackend({ name: 'async-iteration-kvs' });

            for (const key of keys) {
                // The client is a byte-transport: pass serialized bytes + content type.
                await kvStore.setValue({
                    key,
                    value: JSON.stringify({ data: key }),
                    contentType: 'application/json; charset=utf-8',
                });
            }
        });

        test('returns all keys', async () => {
            const { items } = await kvStore.listKeys();

            expect(items).toHaveLength(25);
            expect(items.map((i) => i.key)).toStrictEqual(keys);
        });

        test('respects prefix option', async () => {
            // Only keys starting with 'key-0' (key-00 to key-09)
            const { items } = await kvStore.listKeys({ prefix: 'key-0' });

            expect(items).toHaveLength(10);
            expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 10));
        });

        test('respects exclusiveStartKey option', async () => {
            const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-09' });

            expect(items).toHaveLength(15);
            expect(items.map((i) => i.key)).toStrictEqual(keys.slice(10));
        });

        test('respects limit option', async () => {
            const { items } = await kvStore.listKeys({ limit: 5 });

            expect(items).toHaveLength(5);
            expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 5));
        });

        test('respects exclusiveStartKey and limit together', async () => {
            const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-04', limit: 5 });

            expect(items).toHaveLength(5);
            expect(items.map((i) => i.key)).toStrictEqual(keys.slice(5, 10));
        });
    });
});