Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing validation of RequestQueue parameter to also allow RequestQueueLocal (#60)
Marek Trunkát81ded9cd4e3d
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex d23e1b1e3..b471db1f8 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -4,7 +4,7 @@ import log from 'apify-shared/log';import { isPromise, checkParamPrototypeOrThrow } from './utils';import AutoscaledPool from './autoscaled_pool';import RequestList from './request_list';-import { RequestQueue } from './request_queue';+import { RequestQueue, RequestQueueLocal } from './request_queue';const DEFAULT_OPTIONS = {maxRequestRetries: 3,@@ -78,7 +78,7 @@ export default class BasicCrawler {} = _.defaults(opts, DEFAULT_OPTIONS);checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList', true);- checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', RequestQueue, 'Apify.RequestQueue', true);+ checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');diff --git a/src/utils.js b/src/utils.jsindex 60764948c..3b7402ec8 100644--- a/src/utils.js+++ b/src/utils.js@@ -2,6 +2,7 @@ import Promise from 'bluebird';import contentTypeParser from 'content-type';import os from 'os';import fs from 'fs';+import _ from 'underscore';import fsExtra from 'fs-extra';import ApifyClient from 'apify-client';import { ENV_VARS } from './constants';@@ -175,17 +176,19 @@ export const isPromise = (maybePromise) => {};/**- * Helper function for validation if parameter is an instance of given prototype.+ * Helper function for validation if parameter is an instance of given prototype or multiple prototypes.* TODO: Move this to shared package along with checkParamOrThrow** @ignore*/-export const checkParamPrototypeOrThrow = (paramVal, paramName, prototype, prototypeName, isOptional = false) => {+export const checkParamPrototypeOrThrow = (paramVal, paramName, prototypes, prototypeName, isOptional = false) => {if (isOptional && (paramVal === undefined || paramVal === null)) return;- if (paramVal && !(paramVal instanceof prototype)) {- throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);- }+ const hasCorrectPrototype = prototypes instanceof Array+ ? _.some(prototypes, prototype => paramVal instanceof prototype)+ : paramVal instanceof prototypes;++ if (!hasCorrectPrototype) throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);};/**diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 874af77c4..ef15eb354 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -5,7 +5,7 @@ import 'babel-polyfill';import sinon from 'sinon';import { delayPromise } from 'apify-shared/utilities';import * as Apify from '../build/index';-import { RequestQueue } from '../build/request_queue';+import { RequestQueue, RequestQueueLocal } from '../build/request_queue';chai.use(chaiAsPromised);@@ -137,6 +137,15 @@ describe('BasicCrawler', () => {expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue, requestList })).to.not.throw();});+ it('should also support RequestQueueLocal', () => {+ const requestQueue = new RequestQueue('xxx');+ const requestQueueLocal = new RequestQueueLocal('xxx', 'yyy');+ const handleRequestFunction = () => {};++ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue })).to.not.throw();+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue: requestQueueLocal })).to.not.throw();+ });+it('should correctly combine RequestList and RequestQueue', async () => {const sources = [{ url: 'http://example.com/0' },diff --git a/test/utils.js b/test/utils.jsindex 2bf2df268..74dfe5bce 100644--- a/test/utils.js+++ b/test/utils.js@@ -182,12 +182,19 @@ describe('utils.isPromise()', () => {});});-describe('utils.isPromise()', () => {+describe('utils.checkParamPrototypeOrThrow()', () => {it('works', () => {+ // One prototypeexpect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Date, 'Date')).to.not.throw();expect(() => utils.checkParamPrototypeOrThrow(null, 'param', Function, 'Date', true)).to.not.throw();expect(() => utils.checkParamPrototypeOrThrow(undefined, 'param', Function, 'Date', true)).to.not.throw();expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Function, 'Date')).to.throw();++ // Multiple prototypes+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Date, Function], 'Date')).to.not.throw();+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, Date], 'Date')).to.not.throw();+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, String], 'Date')).to.throw();+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [], 'Date')).to.throw();});});