Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: validation of iterables when adding requests to the queue (#3091)
Jan Buchar529a1dd57278
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 854c8a5e9..2e7225370 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -50,7 +50,7 @@ import {validators,} from '@crawlee/core';import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';-import { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';+import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeFile, writeJSON } from 'fs-extra';import ow, { ArgumentError } from 'ow';@@ -1128,6 +1128,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextconst isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);const maxCrawlDepth = this.maxCrawlDepth;+ ow(+ requests,+ ow.object+ .is((value: unknown) => isIterable(value) || isAsyncIterable(value))+ .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`),+ );+async function* filteredRequests() {let yieldedRequestCount = 0;diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex 895061a7f..41468fc58 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -12,6 +12,7 @@ import type {import {chunkedAsyncIterable,downloadListOfUrls,+ getObjectType,isAsyncIterable,isIterable,peekableAsyncIterable,@@ -220,7 +221,9 @@ export abstract class RequestProvider implements IStorage {ow(requestsLike,- ow.object.is((value: unknown) => isIterable(value) || isAsyncIterable(value)),+ ow.object+ .is((value: unknown) => isIterable(value) || isAsyncIterable(value))+ .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`),);ow(options,@@ -335,6 +338,12 @@ export abstract class RequestProvider implements IStorage {checkStorageAccess();this.lastActivity = new Date();+ ow(+ requests,+ ow.object+ .is((value: unknown) => isIterable(value) || isAsyncIterable(value))+ .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`),+ );ow(options,diff --git a/packages/utils/src/internals/debug.ts b/packages/utils/src/internals/debug.tsindex a7ca60f5c..e10a7299c 100644--- a/packages/utils/src/internals/debug.ts+++ b/packages/utils/src/internals/debug.ts@@ -55,3 +55,20 @@ export function createRequestDebugInfo(...additionalFields,};}++export function getObjectType(value: unknown): string {+ const simple = typeof value;++ if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {+ return simple;+ }++ const objectType = Object.prototype.toString.call(value);+ const type = objectType.match(/\[object (\w+)]/)![1];++ if (type === 'Uint8Array') {+ return 'Buffer';+ }++ return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();+}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 363c0f13a..1fc2e3808 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1651,6 +1651,18 @@ describe('BasicCrawler', () => {});});+ describe('addRequests input validation', () => {+ test('should throw error when passed a non-iterable value', async () => {+ const crawler = new BasicCrawler({+ requestHandler: async () => {},+ });++ await expect(crawler.addRequests(new WeakSet() as any)).rejects.toThrow(+ 'Expected an iterable or async iterable, got weakset',+ );+ });+ });+describe('Dataset helpers, crawler parallelism', () => {const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];