Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixing validation of RequestQueue parameter to also allow RequestQueueLocal (#60)

Marek Trunkát81ded9cd4e3d
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index d23e1b1e3..b471db1f8 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -4,7 +4,7 @@ import log from 'apify-shared/log';
import { isPromise, checkParamPrototypeOrThrow } from './utils';
import AutoscaledPool from './autoscaled_pool';
import RequestList from './request_list';
-import { RequestQueue } from './request_queue';
+import { RequestQueue, RequestQueueLocal } from './request_queue';
const DEFAULT_OPTIONS = {
maxRequestRetries: 3,
@@ -78,7 +78,7 @@ export default class BasicCrawler {
} = _.defaults(opts, DEFAULT_OPTIONS);
checkParamPrototypeOrThrow(requestList, 'opts.requestList', RequestList, 'Apify.RequestList', true);
- checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', RequestQueue, 'Apify.RequestQueue', true);
+ checkParamPrototypeOrThrow(requestQueue, 'opts.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);
checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');
checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');
checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');
diff --git a/src/utils.js b/src/utils.js
index 60764948c..3b7402ec8 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -2,6 +2,7 @@ import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import os from 'os';
import fs from 'fs';
+import _ from 'underscore';
import fsExtra from 'fs-extra';
import ApifyClient from 'apify-client';
import { ENV_VARS } from './constants';
@@ -175,17 +176,19 @@ export const isPromise = (maybePromise) => {
};
/**
- * Helper function for validation if parameter is an instance of given prototype.
+ * Helper function for validation if parameter is an instance of given prototype or multiple prototypes.
* TODO: Move this to shared package along with checkParamOrThrow
*
* @ignore
*/
-export const checkParamPrototypeOrThrow = (paramVal, paramName, prototype, prototypeName, isOptional = false) => {
+export const checkParamPrototypeOrThrow = (paramVal, paramName, prototypes, prototypeName, isOptional = false) => {
if (isOptional && (paramVal === undefined || paramVal === null)) return;
- if (paramVal && !(paramVal instanceof prototype)) {
- throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);
- }
+ const hasCorrectPrototype = prototypes instanceof Array
+ ? _.some(prototypes, prototype => paramVal instanceof prototype)
+ : paramVal instanceof prototypes;
+
+ if (!hasCorrectPrototype) throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);
};
/**
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 874af77c4..ef15eb354 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -5,7 +5,7 @@ import 'babel-polyfill';
import sinon from 'sinon';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
-import { RequestQueue } from '../build/request_queue';
+import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
chai.use(chaiAsPromised);
@@ -137,6 +137,15 @@ describe('BasicCrawler', () => {
expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue, requestList })).to.not.throw();
});
+ it('should also support RequestQueueLocal', () => {
+ const requestQueue = new RequestQueue('xxx');
+ const requestQueueLocal = new RequestQueueLocal('xxx', 'yyy');
+ const handleRequestFunction = () => {};
+
+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue })).to.not.throw();
+ expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue: requestQueueLocal })).to.not.throw();
+ });
+
it('should correctly combine RequestList and RequestQueue', async () => {
const sources = [
{ url: 'http://example.com/0' },
diff --git a/test/utils.js b/test/utils.js
index 2bf2df268..74dfe5bce 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -182,12 +182,19 @@ describe('utils.isPromise()', () => {
});
});
-describe('utils.isPromise()', () => {
+describe('utils.checkParamPrototypeOrThrow()', () => {
it('works', () => {
+ // One prototype
expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Date, 'Date')).to.not.throw();
expect(() => utils.checkParamPrototypeOrThrow(null, 'param', Function, 'Date', true)).to.not.throw();
expect(() => utils.checkParamPrototypeOrThrow(undefined, 'param', Function, 'Date', true)).to.not.throw();
expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Function, 'Date')).to.throw();
+
+ // Multiple prototypes
+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Date, Function], 'Date')).to.not.throw();
+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, Date], 'Date')).to.not.throw();
+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, String], 'Date')).to.throw();
+ expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [], 'Date')).to.throw();
});
});