Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/request queue (#48)
Marek Trunkát7b41dea0a7eb
diff --git a/package.json b/package.jsonindex 669e2df71..6a7612a48 100644--- a/package.json+++ b/package.json@@ -43,8 +43,8 @@"lint": "npm run build && eslint src test"},"dependencies": {- "apify-client": "^0.2.3",- "apify-shared": "0.0.22",+ "apify-client": "^0.2.5",+ "apify-shared": "0.0.23","bluebird": "^3.5.0","content-type": "^1.0.3","fs-extra": "^5.0.0",diff --git a/src/constants.js b/src/constants.jsindex ebb894beb..4b3cfaa71 100644--- a/src/constants.js+++ b/src/constants.js@@ -30,6 +30,7 @@ consts.ENV_VARS = {TIMEOUT_AT: 'APIFY_TIMEOUT_AT',DEFAULT_KEY_VALUE_STORE_ID: 'APIFY_DEFAULT_KEY_VALUE_STORE_ID',DEFAULT_DATASET_ID: 'APIFY_DEFAULT_DATASET_ID',+ DEFAULT_REQUEST_QUEUE_ID: 'APIFY_DEFAULT_REQUEST_QUEUE_ID',LOCAL_EMULATION_DIR: 'APIFY_LOCAL_EMULATION_DIR',WATCH_FILE: 'APIFY_WATCH_FILE',API_BASE_URL: 'APIFY_API_BASE_URL',diff --git a/src/dataset.js b/src/dataset.jsindex 4954756de..092c529ec 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -106,20 +106,17 @@ export class DatasetLocal {}/**- * Helper function that first requests dataset by ID and if dataset doesn't exist then gets him by name.+ * Helper function that first requests dataset by ID and if dataset doesn't exist then gets it by name.** @ignore*/const getOrCreateDataset = (datasetIdOrName) => {- return apifyClient- .datasets+ return datasets.getDataset({ datasetId: datasetIdOrName }).then((existingDataset) => {if (existingDataset) return existingDataset;- return apifyClient- .datasets- .getOrCreateDataset({ datasetName: datasetIdOrName });+ return datasets.getOrCreateDataset({ datasetName: datasetIdOrName });});};diff --git a/src/index.js b/src/index.jsindex 825a345ae..49b98cf0d 100644--- a/src/index.js+++ b/src/index.js@@ -10,6 +10,7 @@ import PuppeteerCrawler from './puppeteer_crawler';import PuppeteerPool from './puppeteer_pool';import Request from './request';import RequestList from './request_list';+import { openRequestQueue } from './request_queue';import SettingsRotator from './settings_rotator';import { apifyClient, getMemoryInfo, isProduction } from './utils';import { browse, launchWebDriver } from './webdriver';@@ -183,7 +184,7 @@ module.exports = {pushData,openDataset,- // Key value store+ // Key-value storegetValue,setValue,openKeyValueStore,@@ -193,9 +194,10 @@ module.exports = {PuppeteerPool,PuppeteerCrawler,- // Request+ // RequestsRequest,RequestList,+ openRequestQueue,// Settings rotatorSettingsRotator,diff --git a/src/key_value_store.js b/src/key_value_store.jsindex fbf4401a7..87c639311 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -234,7 +234,7 @@ export class KeyValueStoreLocal {}/**- * Helper function that first requests key-value store by ID and if store doesn't exist then gets him by name.+ * Helper function that first requests key-value store by ID and if store doesn't exist then gets it by name.** @ignore*/diff --git a/src/request.js b/src/request.jsindex 2e14aec97..80d5d7ae3 100644--- a/src/request.js+++ b/src/request.js@@ -38,6 +38,7 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep*/export default class Request {constructor({+ id,url,uniqueKey,method = 'GET',@@ -48,6 +49,7 @@ export default class Request {userData = {},keepUrlFragment = false,}) {+ checkParamOrThrow(id, 'id', 'Maybe String');checkParamOrThrow(url, 'url', 'String');checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');checkParamOrThrow(method, 'method', 'String');@@ -59,6 +61,7 @@ export default class Request {if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');+ this.id = id;this.url = url;this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment);this.method = method;diff --git a/src/request_list.js b/src/request_list.jsindex e0867abe2..26521c1eb 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -4,21 +4,11 @@ import _ from 'underscore';import requestPromise from 'request-promise';import { sequentializePromises } from 'apify-shared/utilities';import Request from './request';+import { getFirstKey } from './utils';// TODO: better testsconst URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';-/**- * Helper function that returns the first key from plan object.- *- * @ignore- */-const getFirstKey = (dict) => {- for (const key in dict) { // eslint-disable-line guard-for-in, no-restricted-syntax- return key;- }-};-/*** Helper function that validates unique.* Throws an error if uniqueKey is not nonempty string.diff --git a/src/request_queue.js b/src/request_queue.jsindex 20a749ff4..7bb382f9e 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -2,24 +2,327 @@ import fs from 'fs';import fsExtra from 'fs-extra';import path from 'path';import { checkParamOrThrow } from 'apify-client/build/utils';+import LruCache from 'apify-shared/lru_cache';+import ListDictionary from 'apify-shared/list_dictionary';import Promise from 'bluebird';+import crypto from 'crypto';import _ from 'underscore';import Request from './request';-import { ensureDirExists, checkParamPrototypeOrThrow } from './utils';+import { ENV_VARS } from '../build/constants';+import { ensureDirExists, checkParamPrototypeOrThrow, apifyClient } from './utils';export const LOCAL_EMULATION_SUBDIR = 'request-queues';+const MAX_OPENED_QUEUES = 1000;+export const QUERY_HEAD_BUFFER = 50;const writeFilePromised = Promise.promisify(fs.writeFile);const readdirPromised = Promise.promisify(fs.readdir);const readFilePromised = Promise.promisify(fs.readFile);const moveFilePromised = Promise.promisify(fsExtra.move);+const { requestQueues } = apifyClient;+const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.+++/**+ * @ignore.+ */++/**+ * Helper function to validate params of *.addRequest().+ * @ignore+ */+const validateAddRequestParams = (request, opts) => {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ checkParamOrThrow(opts, 'opts', 'Object');++ const { forefront = false } = opts;++ checkParamOrThrow(forefront, 'opts.forefront', 'Boolean');++ if (request.id) throw new Error('Request has already "id" so it cannot be added to the queue!');++ return { forefront };+};++/**+ * Helper function to validate params of *.getRequest().+ * @ignore+ */+const validateGetRequestParams = (requestId) => {+ checkParamOrThrow(requestId, 'requestId', 'String');+};++/**+ * Helper function to validate params of *.markRequestHandled().+ * @ignore+ */+const validateMarkRequestHandledParams = (request) => {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ checkParamOrThrow(request.id, 'request.id', 'String');+};++/**+ * Helper function to validate params of *.reclaimRequest().+ * @ignore+ */+const validateReclaimRequestParams = (request) => {+ checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ checkParamOrThrow(request.id, 'request.id', 'String');+};++/**+ * @typedef {Object} RequestOperationInfo+ * @property {Boolean} wasAlreadyPresent Indicates if request was already present in the queue.+ * @property {Boolean} wasAlreadyHandled Indicates if request was already marked as handled.+ * @property {String} requestId The ID of the added request+ */++/**+ * @ignore.+ *+ * TODO: fix docs+ * Dataset class provides easy interface to Apify Dataset storage type. Dataset should be opened using+ * `Apify.openDataset()` function.+ *+ * Basic usage of Dataset:+ *+ * ```javascript+ * const dataset = await Apify.openDataset('my-dataset-id');+ * await dataset.pushData({ foo: 'bar' });+ * ```+ *+ * @param {String} queueId - ID of the request queue.+ */+export class RequestQueue {+ constructor(queueId) {+ checkParamOrThrow(queueId, 'options.queueId', 'String');++ this.queueId = queueId;+ this.queueHeadDict = new ListDictionary();+ this.requestIdsInProgress = {};+ this.inProgressCount = 0;+ this.queryQueueHeadPromise = null;+ }++ /**+ * Adds a request to the queue.+ *+ * @param {Request} request Request object+ * @param {Object} [opts]+ * @param {Boolean} [opts.forefront] If `true`, the request will be added to the foremost position in the queue.+ * @return {RequestOperationInfo}+ */+ addRequest(request, opts = {}) {+ const { forefront } = validateAddRequestParams(request, opts);++ return requestQueues+ .addRequest({+ request,+ queueId: this.queueId,+ forefront,+ })+ .then((requestOperationInfo) => {+ const { requestId } = requestOperationInfo;++ if (forefront && !this.requestIdsInProgress[requestId]) {+ this.queueHeadDict.add(requestId, requestId, true);+ }++ return requestOperationInfo;+ });+ }++ /**+ * Gets a request from the queue.+ *+ * @param {String} requestId Request ID+ * @return {Request}+ */+ getRequest(requestId) {+ validateGetRequestParams(requestId);++ return requestQueues+ .getRequest({+ requestId,+ queueId: this.queueId,+ })+ .then(obj => (obj ? new Request(obj) : obj));+ }++ /**+ * Returns next upcoming request.+ *+ * @returns {Request}+ */+ fetchNextRequest() {+ return this+ ._ensureHeadIsNonEmpty()+ .then(() => {+ const nextId = this.queueHeadDict.removeFirst();++ // We are likely done at this point.+ if (!nextId) return null;++ this._addToInProgress(nextId);++ return this.getRequest(nextId);+ });+ }++ /**+ * Marks request handled after successfull processing.+ *+ * @param {Request} request+ * @return {RequestOperationInfo}+ */+ markRequestHandled(request) {+ validateMarkRequestHandledParams(request);++ if (!this.requestIdsInProgress[request.id]) {+ throw new Error('Cannot mark handled request that is not in progress!');+ }++ if (!request.handledAt) request.handledAt = new Date();++ return requestQueues+ .updateRequest({+ request,+ queueId: this.queueId,+ })+ .then((response) => {+ this._removeFromInProgress(request.id);++ return response;+ });+ }++ /**+ * Reclaims request after unsuccessfull operation. Requests gets returned into the queue.+ *+ * @param {Request} request+ * @param {Object} [opts]+ * @param {Boolean} [opts.forefront=false] If true then requests gets returned to the begining of the queue+ * and to the back of the queue otherwise.+ * @return {RequestOperationInfo}+ */+ reclaimRequest(request, { forefront = false }) {+ validateReclaimRequestParams(request);++ return requestQueues+ .updateRequest({+ request,+ queueId: this.queueId,+ forefront,+ })+ .then((response) => {+ this._removeFromInProgress(request.id);++ if (forefront) this.queueHeadDict.add(request.id, request.id, true);++ return response;+ });+ }++ /**+ * Returns `true` if the next call to fetchNextRequest() will return null, otherwise it returns `false`.+ * Note that even if the queue is empty, there might be some pending requests currently being processed.+ *+ * @returns {boolean}+ */+ isEmpty() {+ return this+ ._queryQueueHead()+ .then(() => this.queueHeadDict.length() === 0);+ }++ /**+ * Returns `true` if all requests were already handled and there are no more left.+ *+ * @returns {boolean}+ */+ isFinished() {+ return this+ ._queryQueueHead()+ .then(() => this.inProgressCount === 0 && this.queueHeadDict.length() === 0);+ }+++ /**+ * @ignore.+ */+ _addToInProgress(requestId) {+ // Is already there.+ if (this.requestIdsInProgress[requestId]) return;++ this.requestIdsInProgress[requestId] = requestId;+ this.inProgressCount++;+ }++ /**+ * @ignore.+ */+ _removeFromInProgress(requestId) {+ // Is already removed.+ if (!this.requestIdsInProgress[requestId]) return;++ delete this.requestIdsInProgress[requestId];+ this.inProgressCount--;+ }++ /**+ * We always request more items than is in proggress to ensure that something+ * falls into head.+ *+ * @ignore.+ */+ _ensureHeadIsNonEmpty(limit = this.inProgressCount + QUERY_HEAD_BUFFER) {+ checkParamOrThrow(limit, 'limit', 'Number');++ // If is nonempty resolve immediately.+ if (this.queueHeadDict.length()) return Promise.resolve();++ if (!this.queryQueueHeadPromise) {+ this.queryQueueHeadPromise = requestQueues+ .getHead({+ limit,+ queueId: this.queueId,+ })+ .then(({ items }) => {+ items.forEach(({ id }) => {+ if (!this.requestIdsInProgress[id]) this.queueHeadDict.add(id, id, false);+ });++ // This is needed so that the next call can request queue head again.+ this.queryQueueHeadPromise = null;++ return {+ prevLimit: limit,+ limitReached: items.length === limit,+ };+ });+ }++ return this.queryQueueHeadPromise+ .then(({ prevLimit, limitReached }) => {+ // If queue is still empty then it's likely because some of the other calls waiting+ // for this promise already consumed all the returned requests or the limit was too+ // low and contained only requests in progress.+ //+ // If limit was not reached in the call then there are no more requests to be returned.+ if (!this.queueHeadDict.length() && limitReached) {+ return this._ensureHeadIsNonEmpty(prevLimit + QUERY_HEAD_BUFFER);+ }+ });+ }+}+/*** Helper function that extracts queue order number from filename.** @ignore*/-const filepathToQueueOrderNo = (filepath) => {+const filePathToQueueOrderNo = (filepath) => {const int = filepath.split(path.sep).pop() // Get filename from path.split('.')[0]; // Remove extension@@ -27,6 +330,23 @@ const filepathToQueueOrderNo = (filepath) => {return parseInt(int, 10);};+/**+ * Helper function that creates ID from uniqueKey for local usage.+ *+ * @ignore+ */+const uniqueKeyToId = (uniqueKey) => {+ const str = crypto+ .createHash('sha256')+ .update(uniqueKey)+ .digest('base64')+ .replace(/(\+|\/|=)/g, '');++ return str.length > 15+ ? str.substr(0, 15)+ : str;+};+/*** Local implementation of RequestQueue.*@@ -44,7 +364,7 @@ export class RequestQueueLocal {this.pendingCount = 0;this.inProgressCount = 0;- this.uniqueKeyToQueueOrderNo = {};+ this.requestIdToQueueOrderNo = {};this.queueOrderNoInProgress = {};this.initializationPromise = this._initialize();@@ -73,9 +393,9 @@ export class RequestQueueLocal {return readFilePromised(filepath).then((str) => {const request = JSON.parse(str);- const queueOrderNo = filepathToQueueOrderNo(filepath);+ const queueOrderNo = filePathToQueueOrderNo(filepath);- this.uniqueKeyToQueueOrderNo[request.uniqueKey] = queueOrderNo;+ this.requestIdToQueueOrderNo[request.id] = queueOrderNo;});}@@ -89,37 +409,52 @@ export class RequestQueueLocal {}addRequest(request, opts = {}) {- checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');- checkParamOrThrow(opts, 'opts', 'Object');-- const { putInFront = false } = opts;-- checkParamOrThrow(putInFront, 'opts.putInFront', 'Boolean');+ const { forefront } = validateAddRequestParams(request, opts);return this.initializationPromise.then(() => {- const sgn = (putInFront ? 1 : 2) * (10 ** 15);+ const sgn = (forefront ? 1 : 2) * (10 ** 15);const base = (10 ** (13)); // Date.now() returns int with 13 numbers.// We always add pending count for a case that two pages are insterted at the same millisecond.const now = Date.now() + this.pendingCount;- const queueOrderNo = putInFront+ const queueOrderNo = forefront? sgn + (base - now): sgn + (base + now);- const filePath = this._getFilePath(queueOrderNo);- this.pendingCount++;- this.uniqueKeyToQueueOrderNo[request.uniqueKey] = queueOrderNo;+ // Add ID as server does.+ const requestCopy = JSON.parse(JSON.stringify(request));+ requestCopy.id = uniqueKeyToId(request.uniqueKey);++ // If request already exists then don't override it!+ if (this.requestIdToQueueOrderNo[requestCopy.id]) {+ return this+ .getRequest(requestCopy.id)+ .then(existingRequest => ({+ requestId: existingRequest.id,+ wasAlreadyHandled: existingRequest && existingRequest.handledAt,+ wasAlreadyPresent: true,+ }));+ }++ this.requestIdToQueueOrderNo[requestCopy.id] = queueOrderNo;+ if (!requestCopy.handledAt) this.pendingCount++;+ const filePath = this._getFilePath(queueOrderNo, !!requestCopy.handledAt);- return writeFilePromised(filePath, JSON.stringify(request, null, 4));+ return writeFilePromised(filePath, JSON.stringify(requestCopy, null, 4))+ .then(() => ({+ requestId: requestCopy.id,+ wasAlreadyHandled: false,+ wasAlreadyPresent: false,+ }));});}- getRequest(uniqueKey) {- checkParamOrThrow(uniqueKey, 'uniqueKey', 'String');+ getRequest(requestId) {+ validateGetRequestParams(requestId);return this.initializationPromise.then(() => {- const queueOrderNo = this.uniqueKeyToQueueOrderNo[uniqueKey];+ const queueOrderNo = this.requestIdToQueueOrderNo[requestId];return this._getRequestByQueueOrderNo(queueOrderNo);});@@ -150,7 +485,7 @@ export class RequestQueueLocal {let queueOrderNo;_.find(files, (filename) => {- const no = filepathToQueueOrderNo(filename);+ const no = filePathToQueueOrderNo(filename);if (!this.queueOrderNoInProgress[no]) {queueOrderNo = no;@@ -168,11 +503,11 @@ export class RequestQueueLocal {}markRequestHandled(request) {- checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ validateMarkRequestHandledParams(request);return this.initializationPromise.then(() => {- const queueOrderNo = this.uniqueKeyToQueueOrderNo[request.uniqueKey];+ const queueOrderNo = this.requestIdToQueueOrderNo[request.id];const source = this._getFilePath(queueOrderNo, false);const dest = this._getFilePath(queueOrderNo, true);@@ -180,21 +515,28 @@ export class RequestQueueLocal {throw new Error('Cannot mark handled request that is not in progress!');}+ if (!request.handledAt) request.handledAt = new Date();+return moveFilePromised(source, dest).then(() => {this.pendingCount--;this.inProgressCount--;delete this.queueOrderNoInProgress[queueOrderNo];- });+ })+ .then(() => ({+ requestId: request.id,+ wasAlreadyHandled: false,+ wasAlreadyPresent: true,+ }));});}reclaimRequest(request) {- checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');+ validateReclaimRequestParams(request);return this.initializationPromise.then(() => {- const queueOrderNo = this.uniqueKeyToQueueOrderNo[request.uniqueKey];+ const queueOrderNo = this.requestIdToQueueOrderNo[request.id];if (!this.queueOrderNoInProgress[queueOrderNo]) {throw new Error('Cannot reclaim request that is not in progress!');@@ -203,7 +545,12 @@ export class RequestQueueLocal {this.inProgressCount--;delete this.queueOrderNoInProgress[queueOrderNo];- });+ })+ .then(() => ({+ requestId: request.id,+ wasAlreadyHandled: false,+ wasAlreadyPresent: true,+ }));}isEmpty() {@@ -217,6 +564,20 @@ export class RequestQueueLocal {}}+/**+ * Helper function that first requests queue by ID and if queue doesn't exist then gets it by name.+ *+ * @ignore+ */+const getOrCreateQueue = (queueIdOrName) => {+ return requestQueues.getQueue({ queueId: queueIdOrName })+ .then((existingQueue) => {+ if (existingQueue) return existingQueue;++ return requestQueues.getOrCreateQueue({ queueName: queueIdOrName });+ });+};+/*** Opens request queue and returns its object.</p>*@@ -257,16 +618,34 @@ export const openRequestQueue = (queueIdOrName) => {checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');- // TODO:- // should reuse instances- if (!localEmulationDir) {- return Promise.reject(new Error('We currently don\'t support remote queues! This is only temporary'- + 'implementation that stores the queue in the key-value store.'));+ let isDefault = false;+ let queuePromise;++ if (!queueIdOrName) {+ const envVar = ENV_VARS.DEFAULT_REQUEST_QUEUE_ID;++ // Env var doesn't exist.+ if (!process.env[envVar]) return Promise.reject(new Error(`The '${envVar}' environment variable is not defined.`));++ isDefault = true;+ queueIdOrName = process.env[envVar];+ }++ queuePromise = queuesCache.get(queueIdOrName);++ // Found in cache.+ if (queuePromise) return queuePromise;++ // Use local emulation?+ if (localEmulationDir) {+ queuePromise = Promise.resolve(new RequestQueueLocal(queueIdOrName, localEmulationDir));+ } else {+ queuePromise = isDefault // If true then we know that this is an ID of existing queue.+ ? Promise.resolve(new RequestQueue(queueIdOrName))+ : getOrCreateQueue(queueIdOrName).then(queue => (new RequestQueue(queue.id)));}- const queue = localEmulationDir- ? new RequestQueueLocal(queueIdOrName, localEmulationDir)- : new RequestQueueRemote('default-request-queue');+ queuesCache.add(queueIdOrName, queuePromise);- return Promise.resolve(queue);+ return queuePromise;};diff --git a/src/utils.js b/src/utils.jsindex cb5ae5251..60764948c 100644--- a/src/utils.js+++ b/src/utils.js@@ -202,6 +202,17 @@ export const isProduction = () => process.env.NODE_ENV !== 'production';*/export const ensureDirExists = path => ensureDirPromised(path);+/**+ * Helper function that returns the first key from plan object.+ *+ * @ignore+ */+export const getFirstKey = (dict) => {+ for (const key in dict) { // eslint-disable-line guard-for-in, no-restricted-syntax+ return key;+ }+};+/*** Gets a typical path to Chrome executable, depending on the current operating system.* @returns {string}diff --git a/test/request_list.js b/test/request_list.jsindex ae7f3971d..8c4a4803e 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -137,8 +137,8 @@ describe('Apify.RequestList', () => {it('should use regex parameter to parse urls', async () => {const mock = sinon.mock(request);- const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"http://google.com/a/b/c";dgg:dd';- const listArr = ['https://example.com/a/b/c?q=1#abc', 'http://google.com/a/b/c'];+ const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';+ const listArr = ['https://example.com/a/b/c?q=1#abc', 'HTTP://google.com/a/b/c'];mock.expects('get').once()diff --git a/test/request_queue.js b/test/request_queue.jsindex 219fc3d5b..e552c0d8b 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -1,13 +1,21 @@import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';+import _ from 'underscore';import 'babel-polyfill';+import sinon from 'sinon';+import { delayPromise } from 'apify-shared/utilities';import * as Apify from '../build/index';+import { ENV_VARS } from '../build/constants';import { apifyClient } from '../build/utils';-import { RequestQueueLocal, LOCAL_EMULATION_SUBDIR } from '../build/request_queue';-import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR } from './_helper';+import { RequestQueueLocal, RequestQueue, LOCAL_EMULATION_SUBDIR, QUERY_HEAD_BUFFER } from '../build/request_queue';+import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR, expectNotLocalEmulation } from './_helper';chai.use(chaiAsPromised);+const expectRequestsSame = (req1, req2) => {+ expect(_.omit(req1, 'id')).to.be.eql(_.omit(req2, 'id'));+};+describe('RequestQueue', () => {before(() => apifyClient.setOptions({ token: 'xxx' }));after(() => apifyClient.setOptions({ token: undefined }));@@ -18,24 +26,28 @@ describe('RequestQueue', () => {it('should work', async () => {const queue = new RequestQueueLocal('my-queue', LOCAL_EMULATION_DIR);- const request1 = new Apify.Request({ url: 'http://example.com/first' });- const request2 = new Apify.Request({ url: 'http://example.com/middle' });- const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });+ await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));+ await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));+ await queue.addRequest(new Apify.Request({ url: 'http://example.com/last-but-first' }), { forefront: true });- await queue.addRequest(request1);- await queue.addRequest(request2);- await queue.addRequest(request3, { putInFront: true });+ const request3 = await queue.fetchNextRequest();+ const request1 = await queue.fetchNextRequest();+ const request2 = await queue.fetchNextRequest();- expect(await queue.getRequest(request1.uniqueKey)).to.be.eql(request1);- expect(await queue.getRequest(request2.uniqueKey)).to.be.eql(request2);- expect(await queue.getRequest(request3.uniqueKey)).to.be.eql(request3);- expect(await queue.getRequest(request1.uniqueKey)).to.not.be.eql(request3);+ expect(await queue.getRequest(request1.id)).to.be.eql(request1);+ expect(await queue.getRequest(request2.id)).to.be.eql(request2);+ expect(await queue.getRequest(request3.id)).to.be.eql(request3);+ expect(await queue.getRequest(request1.id)).to.not.be.eql(request3);- expect(await queue.fetchNextRequest()).to.be.eql(request3);- expect(await queue.fetchNextRequest()).to.be.eql(request1);- expect(await queue.fetchNextRequest()).to.be.eql(request2);- expect(await queue.fetchNextRequest()).to.be.eql(null);+ expect(request3.url).to.be.eql('http://example.com/last-but-first');+ expect(request1.url).to.be.eql('http://example.com/first');+ expect(request2.url).to.be.eql('http://example.com/middle');+ expect((await queue.getRequest(request3.id)).url).to.be.eql('http://example.com/last-but-first');+ expect((await queue.getRequest(request1.id)).url).to.be.eql('http://example.com/first');+ expect((await queue.getRequest(request2.id)).url).to.be.eql('http://example.com/middle');++ expect(await queue.fetchNextRequest()).to.be.eql(null);expect(await queue.isEmpty()).to.be.eql(true);expect(await queue.isFinished()).to.be.eql(false);@@ -55,31 +67,235 @@ describe('RequestQueue', () => {});it('should get initialized from existing dir', async () => {- const queue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);-const request1 = new Apify.Request({ url: 'http://example.com/first' });const request2 = new Apify.Request({ url: 'http://example.com/middle' });const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });+ // Do something with 3 requests in one queue.+ const queue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);await queue.addRequest(request1);await queue.addRequest(request2);- await queue.addRequest(request3, { putInFront: true });-- expect(await queue.fetchNextRequest()).to.be.eql(request3);- expect(await queue.fetchNextRequest()).to.be.eql(request1);- await queue.markRequestHandled(request1);+ await queue.addRequest(request3, { forefront: true });+ const freshRequest3 = await queue.fetchNextRequest();+ const freshRequest1 = await queue.fetchNextRequest();+ expectRequestsSame(freshRequest3, request3);+ expectRequestsSame(freshRequest1, request1);+ await queue.markRequestHandled(freshRequest1);+ // Now do the same with another queue.const anotherQueue = new RequestQueueLocal('my-queue-2', LOCAL_EMULATION_DIR);expect(await anotherQueue.isEmpty()).to.be.eql(false);expect(await anotherQueue.isFinished()).to.be.eql(false);- expect(await anotherQueue.fetchNextRequest()).to.be.eql(request3);- expect(await anotherQueue.fetchNextRequest()).to.be.eql(request2);+ const request3FromAnotherQueue = await anotherQueue.fetchNextRequest();+ const request2FromAnotherQueue = await anotherQueue.fetchNextRequest();+ expectRequestsSame(request3FromAnotherQueue, request3);+ expectRequestsSame(request2FromAnotherQueue, request2);expect(await anotherQueue.isEmpty()).to.be.eql(true);expect(await anotherQueue.isFinished()).to.be.eql(false);- await anotherQueue.markRequestHandled(request2);- await anotherQueue.markRequestHandled(request3);+ await anotherQueue.markRequestHandled(request3FromAnotherQueue);+ await anotherQueue.markRequestHandled(request2FromAnotherQueue);expect(await anotherQueue.isEmpty()).to.be.eql(true);expect(await anotherQueue.isFinished()).to.be.eql(true);});});++ describe('remote', async () => {+ it('should work', async () => {+ expectNotLocalEmulation();++ const { Request } = Apify;++ const queue = new RequestQueue('some-id');+ const mock = sinon.mock(apifyClient.requestQueues);++ const requestA = new Request({ url: 'http://example.com/a' });+ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestA,+ forefront: false,+ })+ .returns(Promise.resolve({ requestId: 'a' }));+ await queue.addRequest(requestA);++ const requestB = new Request({ url: 'http://example.com/b' });+ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestB,+ forefront: true,+ })+ .returns(Promise.resolve({ requestId: 'b' }));+ await queue.addRequest(requestB, { forefront: true });+ expect(queue.queueHeadDict.length()).to.be.eql(1);+ expect(queue.inProgressCount).to.be.eql(0);++ // Forefronted request was added to the queue.+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'b',+ })+ .returns(Promise.resolve(_.extend(requestB, { id: 'b' })));+ const requestBFromQueue = await queue.fetchNextRequest();+ expect(requestBFromQueue).to.be.eql(requestB);+ expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.inProgressCount).to.be.eql(1);++ // Reclaim it.+ mock.expects('updateRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestB,+ forefront: true,+ })+ .returns(Promise.resolve({ requestId: requestB.id }));+ await queue.reclaimRequest(requestB, { forefront: true });+ expect(queue.queueHeadDict.length()).to.be.eql(1);+ expect(queue.inProgressCount).to.be.eql(0);++ // Fetch again.+ expect(queue.queueHeadDict.length()).to.be.eql(1);+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'b',+ })+ .returns(Promise.resolve(_.extend(requestB, { id: 'b' })));+ const requestBFromQueue2 = await queue.fetchNextRequest();+ expect(requestBFromQueue2).to.be.eql(requestB);+ expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.inProgressCount).to.be.eql(1);++ // Mark handled.+ mock.expects('updateRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestB,+ })+ .returns(Promise.resolve({ requestId: requestB.id }));+ await queue.markRequestHandled(requestB);+ expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.inProgressCount).to.be.eql(0);++ // Query queue head.+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_BUFFER,+ })+ .returns(Promise.resolve({+ items: [+ { id: 'a' },+ { id: 'c' },+ ],+ }));+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'a',+ })+ .returns(Promise.resolve(_.extend(requestA, { id: 'a' })));+ const requestAFromQueue = await queue.fetchNextRequest();+ expect(requestAFromQueue).to.be.eql(requestA);+ expect(queue.queueHeadDict.length()).to.be.eql(1);+ expect(queue.inProgressCount).to.be.eql(1);++ mock.verify();+ mock.restore();+ });+ });++ describe('Apify.openRequestQueue', async () => {+ it('should open a local request queue when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is set', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const queue = await Apify.openRequestQueue('some-id-2');+ expect(queue).to.be.instanceof(RequestQueueLocal);+ expect(queue).not.to.be.instanceof(RequestQueue);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ });++ it('should reuse cached request queue instances', async () => {+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const queue1 = await Apify.openRequestQueue('some-id-3');+ const queue2 = await Apify.openRequestQueue('some-id-3');+ const queue3 = new RequestQueueLocal('some-id-3', LOCAL_EMULATION_DIR);++ expect(queue1).to.be.instanceof(RequestQueueLocal);+ expect(queue2).to.be.instanceof(RequestQueueLocal);+ expect(queue3).to.be.instanceof(RequestQueueLocal);++ expect(queue1).to.be.equal(queue2);+ expect(queue1).not.to.be.equal(queue3);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];++ // Here must be some timeout to don't finish before initialization of queues finishes.+ // Otherwise we delete the directory and scandir will throw ENOENT: no such file or directory+ await delayPromise(100);+ });++ it('should open default request queue when queueIdOrName is not provided', async () => {+ process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID] = 'some-id-4';+ process.env[ENV_VARS.LOCAL_EMULATION_DIR] = LOCAL_EMULATION_DIR;++ const queue = await Apify.openRequestQueue();+ expect(queue.queueId).to.be.eql('some-id-4');+ expect(queue).to.be.instanceof(RequestQueueLocal);++ delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];+ process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID] = 'some-id-5';+ expectNotLocalEmulation();++ const queue2 = await Apify.openRequestQueue();+ expect(queue2.queueId).to.be.eql('some-id-5');+ expect(queue2).to.be.instanceof(RequestQueue);++ delete process.env[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID];+ });++ it('should open remote queue when process.env[ENV_VARS.LOCAL_EMULATION_DIR] is NOT set', async () => {+ expectNotLocalEmulation();++ const mock = sinon.mock(apifyClient.requestQueues);++ // First when used with id it only requests store object.+ mock.expects('getQueue')+ .once()+ .withArgs({ queueId: 'some-id-6' })+ .returns(Promise.resolve({ id: 'some-id-6' }));+ const queue = await Apify.openRequestQueue('some-id-6');+ expect(queue.queueId).to.be.eql('some-id-6');+ expect(queue).to.be.instanceof(RequestQueue);++ // Then used with name it requests store object, gets empty response+ // so then it creates queue.+ mock.expects('getQueue')+ .once()+ .withArgs({ queueId: 'some-name-7' })+ .returns(Promise.resolve(null));+ mock.expects('getOrCreateQueue')+ .once()+ .withArgs({ queueName: 'some-name-7' })+ .returns(Promise.resolve({ id: 'some-id-7' }));++ const queue2 = await Apify.openRequestQueue('some-name-7');+ expect(queue2.queueId).to.be.eql('some-id-7');+ expect(queue2).to.be.instanceof(RequestQueue);++ mock.verify();+ mock.restore();+ });+ });});