Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

7.5 KB
Raw Blame
import { describe, expect, it } from 'vitest';

import { asyncifyIterable, chunkedAsyncIterable, peekableAsyncIterable } from '../../packages/core/src/iterables.js';

describe('asyncifyIterable', () => {
    it('should convert a regular array to async iterable', async () => {
        const asyncIterable = asyncifyIterable([1, 2, 3, 4, 5]);

        const result = [];
        for await (const item of asyncIterable) {
            result.push(item);
        }

        expect(result).toEqual([1, 2, 3, 4, 5]);
    });

    it('should handle empty arrays', async () => {
        const asyncIterable = asyncifyIterable([]);

        const result: unknown[] = [];
        for await (const item of asyncIterable) {
            result.push(item);
        }

        expect(result).toEqual([]);
    });

    it('should work with Set', async () => {
        const asyncIterable = asyncifyIterable(new Set([1, 2, 3, 2, 1]));

        const result = [];
        for await (const item of asyncIterable) {
            result.push(item);
        }

        expect(result).toEqual([1, 2, 3]);
    });

    it('should work with generator function', async () => {
        function* generator() {
            yield 1;
            yield 2;
            yield 3;
        }

        const asyncIterable = asyncifyIterable(generator());

        const result = [];
        for await (const item of asyncIterable) {
            result.push(item);
        }

        expect(result).toEqual([1, 2, 3]);
    });
});

describe('chunkedAsyncIterable', () => {
    it('should chunk an async iterable into specified sizes', async () => {
        const result = [];
        for await (const chunk of chunkedAsyncIterable([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3)) {
            result.push(chunk);
        }

        expect(result).toEqual([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]);
    });

    it('should handle chunk size of 1', async () => {
        const result = [];
        for await (const chunk of chunkedAsyncIterable([1, 2, 3], 1)) {
            result.push(chunk);
        }

        expect(result).toEqual([[1], [2], [3]]);
    });

    it('should handle chunk size larger than input', async () => {
        const chunked = chunkedAsyncIterable([1, 2, 3], 10);

        const result = [];
        for await (const chunk of chunked) {
            result.push(chunk);
        }

        expect(result).toEqual([[1, 2, 3]]);
    });

    it('should handle empty async iterable', async () => {
        const result = [];
        for await (const chunk of chunkedAsyncIterable([], 3)) {
            result.push(chunk);
        }

        expect(result).toEqual([]);
    });

    it('should accept a callback for dynamic chunk size', async () => {
        let size = 3;
        const result = [];
        for await (const chunk of chunkedAsyncIterable([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], () => size)) {
            result.push(chunk);
            size = 2; // shrink after first chunk
        }

        expect(result).toEqual([[1, 2, 3], [4, 5], [6, 7], [8, 9], [10]]);
    });

    it('should stop iterating when dynamic chunk size drops to zero', async () => {
        let size = 2;
        const result = [];
        for await (const chunk of chunkedAsyncIterable([1, 2, 3, 4, 5, 6], () => size)) {
            result.push(chunk);
            size = 0; // signal stop after first chunk
        }

        expect(result).toEqual([[1, 2]]);
    });

    it('should leave the underlying iterator drainable after partial consumption', async () => {
        async function* source() {
            yield 1;
            yield 2;
            yield 3;
            yield 4;
            yield 5;
        }

        const iterator = source();

        // Consume only the first chunk via chunkedAsyncIterable
        let size = 2;
        for await (const _ of chunkedAsyncIterable(iterator, () => size)) {
            size = 0; // stop after first chunk
        }

        // The underlying iterator should still be drainable
        const remaining: number[] = [];
        for await (const value of iterator) {
            remaining.push(value);
        }

        expect(remaining).toEqual([3, 4, 5]);
    });

    it('should throw error for invalid chunk size', async () => {
        await expect(
            (async () => {
                for await (const _ of chunkedAsyncIterable([1, 2, 3], 0)) {
                    // Empty block
                }
            })(),
        ).rejects.toThrow();
        await expect(
            (async () => {
                for await (const _ of chunkedAsyncIterable([1, 2, 3], -1)) {
                    // Empty block
                }
            })(),
        ).rejects.toThrow();
    });
});

describe('peekableAsyncIterable', () => {
    it('should allow peeking at the next value without advancing', async () => {
        const iterable = peekableAsyncIterable([1, 2, 3]);
        const iterator = iterable[Symbol.asyncIterator]();

        const peeked = await iterator.peek();
        expect(peeked).toBe(1);

        // Peeking again should return the same value
        const peekedAgain = await iterator.peek();
        expect(peekedAgain).toBe(1);

        // Now iterate and verify we get the peeked value first
        const results = [];
        for await (const value of iterable) {
            results.push(value);
        }

        expect(results).toEqual([1, 2, 3]);
    });

    it('should return undefined when peeking at empty iterable', async () => {
        const iterable = peekableAsyncIterable([]);
        const iterator = iterable[Symbol.asyncIterator]();

        const peeked = await iterator.peek();
        expect(peeked).toBeUndefined();
    });

    it('should handle peek after exhausting the iterable', async () => {
        const iterable = peekableAsyncIterable([1]);

        // Consume the iterable
        const results = [];
        for await (const value of iterable) {
            results.push(value);
        }
        expect(results).toEqual([1]);

        // Get a fresh iterator and peek should return undefined
        const iterator = iterable[Symbol.asyncIterator]();
        const peeked = await iterator.peek();
        expect(peeked).toBeUndefined();
    });

    it('should work with manual iteration', async () => {
        const iterable = peekableAsyncIterable([10, 20, 30]);
        const iterator = iterable[Symbol.asyncIterator]();

        // Peek first
        expect(await iterator.peek()).toBe(10);

        const first = await iterator.next();
        expect(first.value).toBe(10);
        expect(first.done).toBe(false);
        expect(await iterator.peek()).toBe(20);

        const second = await iterator.next();
        expect(second.value).toBe(20);
        expect(second.done).toBe(false);
        expect(await iterator.peek()).toBe(30);

        const third = await iterator.next();
        expect(third.value).toBe(30);
        expect(third.done).toBe(false);
        expect(await iterator.peek()).toBe(undefined);

        const done = await iterator.next();
        expect(done.done).toBe(true);
        expect(await iterator.peek()).toBe(undefined);
    });

    it('should handle peek on single element iterable', async () => {
        const iterable = peekableAsyncIterable([42]);
        const iterator = iterable[Symbol.asyncIterator]();

        expect(await iterator.peek()).toBe(42);

        const results = [];
        for await (const value of iterable) {
            results.push(value);
        }

        expect(results).toEqual([42]);

        // Get a fresh iterator and peek after exhaustion
        const newIterator = iterable[Symbol.asyncIterator]();
        expect(await newIterator.peek()).toBeUndefined();
    });
});