Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

RequestQueue improvements, more unit test

Jan Curn3c90f1ffcc78
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a9e505c2..a8bcc02f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+====================
+- Fixes in `RequestQueue` to deal with inconsistencies in the underlying data storage
+- **BREAKING CHANGE**: `RequestQueue.addRequest()` now sets the ID of the
+ newly added request to the passed `Request` object
+- **DEPRECATED**: The `RequestQueue.handledCount()` function has been deprecated,
+ please use `RequestQueue.getInfo()` instead
+
0.14.14 / 2019-05-30
====================
- Fix error where live view would crash when started with concurrency already higher than 1.
@@ -10,7 +17,6 @@
====================
- `Snapshotter` will now log critical memory overload warnings at most once per 10 seconds.
_ Live view snapshots are now made right after navigation finishes, instead of right before page close.
-- Fixes in `RequestQueue` to deal with inconsistencies in the underlying data storage
0.14.11 / 2019-05-28
====================
diff --git a/src/request_queue.js b/src/request_queue.js
index 537230e4d..afea3cb0a 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -51,20 +51,17 @@ const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queu
*/
const validateAddRequestParams = (request, opts) => {
checkParamOrThrow(request, 'request', 'Object');
-
- if (!(request instanceof Request)) {
- request = new Request(request);
- }
-
checkParamOrThrow(opts, 'opts', 'Object');
+ const newRequest = request instanceof Request ? request : new Request(request);
+
const { forefront = false } = opts;
checkParamOrThrow(forefront, 'opts.forefront', 'Boolean');
- if (request.id) throw new Error('Request has already "id" so it cannot be added to the queue!');
+ if (request.id) throw new Error('Request already has the "id" field set so it cannot be added to the queue!');
- return { forefront, request };
+ return { forefront, newRequest };
};
/**
@@ -118,9 +115,7 @@ const getRequestId = (uniqueKey) => {
.digest('base64')
.replace(/(\+|\/|=)/g, '');
- return str.length > 15
- ? str.substr(0, 15)
- : str;
+ return str.substr(0, 15);
};
/**
@@ -132,7 +127,7 @@ const getRequestId = (uniqueKey) => {
* @property {Boolean} wasAlreadyPresent Indicates if request was already present in the queue.
* @property {Boolean} wasAlreadyHandled Indicates if request was already marked as handled.
* @property {String} requestId The ID of the added request
- * @property {Request} request The original `Request` object passed to the `RequestQueue` function.
+ * @property {Request|Object} request The original `Request` object passed to the `RequestQueue` function.
*/
/**
@@ -217,8 +212,8 @@ export class RequestQueue {
this.queueHeadDict = new ListDictionary();
this.queryQueueHeadPromise = null;
- // Contains a list of all requests that are currently being handled.
- // Keys are request IDs, values are true.
+ // A set of all request IDs that are currently being handled,
+ // i.e. which were returned by fetchNextRequest() but not markRequestHandled()
this.inProgress = new Set();
// Contains a list of recently handled requests. It is used to avoid inconsistencies
@@ -233,8 +228,8 @@ export class RequestQueue {
// Caching requests to avoid redundant addRequest() calls.
// Key is computed using getRequestId() and value is { id, isHandled }.
- // TODO: We could extend the functionality of caching to improve performance
- // of other operations too, such as fetchNextRequest().
+ // TODO: We could extend the caching to improve performance
+ // of other operations such as fetchNextRequest().
this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
}
@@ -252,20 +247,22 @@ export class RequestQueue {
* it will not be updated. You can find out whether this happened from the resulting
* {@link QueueOperationInfo} object.
*
- * @param {Request|Object} request {@link Request} object, or an object to construct a `Request` instance from.
- * Note that the passed object is not modified by the function, i.e. not even the new request ID is set it.
+ * @param {Request|Object} request {@link Request} object or vanilla object with request data.
+ * Note that the function sets the `uniqueKey` and `id` fields to the passed object.
* @param {Object} [options]
* @param {Boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
* @return {QueueOperationInfo}
*/
async addRequest(request, options = {}) {
- const { forefront, request: newRequest } = validateAddRequestParams(request, options);
- request = newRequest;
+ const { newRequest, forefront } = validateAddRequestParams(request, options);
- const cacheKey = getRequestId(request.uniqueKey);
+ request.uniqueKey = newRequest.uniqueKey;
+
+ const cacheKey = getRequestId(newRequest.uniqueKey);
const cachedInfo = this.requestsCache.get(cacheKey);
if (cachedInfo) {
+ request.id = cachedInfo.id;
return {
wasAlreadyPresent: true,
// We may assume that if request is in local cache then also the information if the
@@ -277,7 +274,7 @@ export class RequestQueue {
}
const queueOperationInfo = await requestQueues.addRequest({
- request,
+ request: newRequest,
queueId: this.queueId,
forefront,
clientKey: this.clientKey,
@@ -294,6 +291,7 @@ export class RequestQueue {
this._maybeAddRequestToQueueHead(requestId, forefront);
}
+ request.id = requestId;
queueOperationInfo.request = request;
return queueOperationInfo;
@@ -321,14 +319,14 @@ export class RequestQueue {
/**
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
*
- * Once you successfully finish processing of the request, you need to call {@link RequestQueue#markRequestHandled}
+ * Once you successfully finish processing of the request, you need to call {@link RequestQueue.markRequestHandled}
* to mark the request as handled in the queue. If there was some error in processing the request,
- * call {@link RequestQueue#reclaimRequest} instead, so that the queue will give the request to some other consumer
+ * call {@link RequestQueue.reclaimRequest} instead, so that the queue will give the request to some other consumer
* in another call to the `fetchNextRequest` function.
*
* Note that the `null` return value doesn't mean the queue processing finished,
* it means there are currently no pending requests.
- * To check whether all requests in queue were finished, use {@link RequestQueue#isFinished} instead.
+ * To check whether all requests in queue were finished, use {@link RequestQueue.isFinished} instead.
*
* @returns {Promise<Request>}
* Returns the request object or `null` if there are no more pending requests.
@@ -362,15 +360,15 @@ export class RequestQueue {
throw e;
}
- // NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can happen in two situations:
+ // NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:
- // 1) Queue head index is ahead of the main table and the request is not present in the table yet.
+ // 1) Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
// In this case, keep the request marked as in progress for a short while,
// so that isFinished() doesn't return true and _ensureHeadIsNonEmpty() doesn't not load the request
// into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
// will try to fetch this request again, until it eventually appears in the main table.
if (!request) {
- log.debug('Cannot find request from the beginning of queue, will be retried later', { nextRequestId });
+ log.debug('Cannot find a request from the beginning of queue, will be retried later', { nextRequestId });
setTimeout(() => {
this.inProgress.delete(nextRequestId);
}, STORAGE_CONSISTENCY_DELAY_MILLIS);
@@ -391,8 +389,9 @@ export class RequestQueue {
}
/**
- * Marks request as handled after successful processing.
- * Handled requests will never again be returned by the {@link RequestQueue#fetchNextRequest} function.
+ * Marks a request that was previously returned by the {@link RequestQueue.fetchNextRequest} function
+ * as handled after successful processing.
+ * Handled requests will never again be returned by the `fetchNextRequest` function.
*
* @param {Request} request
* @return {Promise<QueueOperationInfo>}
@@ -429,7 +428,7 @@ export class RequestQueue {
/**
* Reclaims a failed request back to the queue, so that it can be returned for processed later again
- * by another call to {@link RequestQueue#fetchNextRequest}.
+ * by another call to {@link RequestQueue.fetchNextRequest}.
* The request record in the queue is updated using the provided `request` parameter.
* For example, this lets you store the number of retries or error messages for the request.
*
@@ -437,7 +436,7 @@ export class RequestQueue {
* @param {Object} [options]
* @param {Boolean} [options.forefront=false]
* If `true` then the request it placed to the beginning of the queue, so that it's returned
- * in the next call to {@link RequestQueue#fetchNextRequest}. By default, it's put to the end of the queue.
+ * in the next call to {@link RequestQueue.fetchNextRequest}. By default, it's put to the end of the queue.
* @return {Promise<QueueOperationInfo>}
*/
async reclaimRequest(request, options = {}) {
@@ -479,9 +478,9 @@ export class RequestQueue {
}
/**
- * Resolves to `true` if the next call to {@link RequestQueue#fetchNextRequest} would return `null`, otherwise it resolves to `false`.
+ * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest} would return `null`, otherwise it resolves to `false`.
* Note that even if the queue is empty, there might be some pending requests currently being processed.
- * If you need to ensure that there is no activity in the queue, use {@link RequestQueue#isFinished}.
+ * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
*
* @returns {Promise<Boolean>}
*/
@@ -654,9 +653,11 @@ export class RequestQueue {
* Returns the number of handled requests.
*
* @return {Promise<number>}
+ * @deprecated
*/
async handledCount() {
- // TODO: Let's deprecate this function in favor of getInfo()
+ log.deprecated('RequestQueue.handledCount() is deprecated, please use RequestQueue.getInfo() instead.');
+
const queueInfo = await requestQueues.getQueue({ queueId: this.queueId });
return queueInfo.handledRequestCount;
}
@@ -809,12 +810,10 @@ export class RequestQueueLocal {
});
}
- addRequest(request, opts = {}) {
- const { forefront, request: newRequest } = validateAddRequestParams(request, opts);
+ async addRequest(request, opts = {}) {
+ const { newRequest, forefront } = validateAddRequestParams(request, opts);
- if (newRequest) {
- request = newRequest;
- }
+ request.uniqueKey = newRequest.uniqueKey;
return this.initializationPromise
.then(() => {
@@ -822,8 +821,9 @@ export class RequestQueueLocal {
// Add ID as server does.
// TODO: This way of cloning doesn't preserve Dates!
- const requestCopy = JSON.parse(JSON.stringify(request));
+ const requestCopy = JSON.parse(JSON.stringify(newRequest));
requestCopy.id = getRequestId(request.uniqueKey);
+ request.id = requestCopy.id;
this._updateMetadata(true);
@@ -914,7 +914,7 @@ export class RequestQueueLocal {
const dest = this._getFilePath(queueOrderNo, true);
if (!this.queueOrderNoInProgress[queueOrderNo]) {
- throw new Error(`Cannot mark request ${request.id} handled request that is not in progress!`);
+ throw new Error(`Cannot mark request ${request.id} as handled, because it is not in progress!`);
}
if (!request.handledAt) request.handledAt = new Date();
@@ -952,7 +952,7 @@ export class RequestQueueLocal {
const dest = this._getFilePath(newQueueOrderNo);
if (!this.queueOrderNoInProgress[oldQueueOrderNo]) {
- throw new Error(`Cannot reclaim request ${request.id} that is not in progress!`);
+ throw new Error(`Cannot reclaim request ${request.id}, because it is not in progress!`);
}
this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;
diff --git a/test/request_queue.js b/test/request_queue.js
index 573daec10..c9c33c142 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -32,6 +32,8 @@ describe('RequestQueue', () => {
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/last-but-first' }), { forefront: true });
+ expect(req1.id).to.be.a('string');
+ expect(req1.uniqueKey).to.eql('http://example.com/first');
expect(info1).to.contain({
wasAlreadyPresent: false,
wasAlreadyHandled: false,
@@ -61,6 +63,14 @@ describe('RequestQueue', () => {
expect(await queue.isEmpty()).to.be.eql(true);
expect(await queue.isFinished()).to.be.eql(false);
+ // Test validations
+ await queue.markRequestHandled(new Apify.Request({ id: 'XXX', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Cannot mark request XXX as handled, because it is not in progress/));
+ await queue.reclaimRequest(new Apify.Request({ id: 'XXX', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Cannot reclaim request XXX, because it is not in progress/));
+ await queue.addRequest(new Apify.Request({ id: 'id-already-set', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Request already has the "id" field set so it cannot be added to the queue/));
+
// Check that changes to Requests are persisted to Queue.
request1.errorMessages = ['Hello'];
request2.retryCount = 2;
@@ -292,7 +302,33 @@ describe('RequestQueue', () => {
clientKey,
})
.returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));
- await queue.addRequest(requestA);
+ const queueOperationInfo1 = await queue.addRequest(requestA);
+
+ expect(requestA.id).to.eql('a');
+ expect(requestA.uniqueKey).to.eql('http://example.com/a');
+ expect(queueOperationInfo1).to.include({
+ wasAlreadyPresent: false,
+ wasAlreadyHandled: false,
+ requestId: 'a',
+ });
+ expect(queueOperationInfo1.request).to.include({
+ id: 'a',
+ });
+ expect(queue.queueHeadDict.length()).to.be.eql(1);
+
+ // Try to add again the a request with the same URL
+ const copyOfRequest = { url: 'http://example.com/a' };
+ const queueOperationInfo2 = await queue.addRequest(copyOfRequest);
+ expect(copyOfRequest.id).to.eql('a');
+ expect(copyOfRequest.uniqueKey).to.eql('http://example.com/a');
+ expect(queueOperationInfo2).to.include({
+ wasAlreadyPresent: true,
+ wasAlreadyHandled: false,
+ requestId: 'a',
+ });
+ expect(queueOperationInfo2.request).to.include({
+ id: 'a',
+ });
expect(queue.queueHeadDict.length()).to.be.eql(1);
const requestB = new Request({ url: 'http://example.com/b' });
@@ -306,6 +342,7 @@ describe('RequestQueue', () => {
})
.returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestB }));
await queue.addRequest(requestB, { forefront: true });
+
expect(queue.queueHeadDict.length()).to.be.eql(2);
expect(queue.inProgressCount()).to.be.eql(0);
@@ -322,6 +359,14 @@ describe('RequestQueue', () => {
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount()).to.be.eql(1);
+ // Test validations
+ await queue.markRequestHandled(new Request({ id: 'XXX', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Cannot mark request XXX as handled, because it is not in progress/));
+ await queue.reclaimRequest(new Request({ id: 'XXX', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Cannot reclaim request XXX, because it is not in progress/));
+ await queue.addRequest(new Request({ id: 'id-already-set', url: 'dummy' }))
+ .catch(err => expect(err.message).to.match(/Request already has the "id" field set so it cannot be added to the queue/));
+
// getRequest() returns null if object was not found.
mock.expects('getRequest')
.once()
@@ -345,13 +390,12 @@ describe('RequestQueue', () => {
.returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));
await queue.reclaimRequest(requestB, { forefront: true });
expect(queue.queueHeadDict.length()).to.be.eql(1);
- expect(queue.inProgressCount()).to.be.eql(0);
+ expect(queue.inProgressCount()).to.be.eql(1);
await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);
expect(queue.queueHeadDict.length()).to.be.eql(2);
expect(queue.inProgressCount()).to.be.eql(0);
// Fetch again.
- expect(queue.queueHeadDict.length()).to.be.eql(1);
mock.expects('getRequest')
.once()
.withArgs({
@@ -377,6 +421,9 @@ describe('RequestQueue', () => {
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount()).to.be.eql(0);
+ // Emulate there are no cached items in queue
+ queue.queueHeadDict.clear();
+
// Query queue head.
mock.expects('getHead')
.once()
@@ -566,6 +613,7 @@ describe('RequestQueue', () => {
request: requestA,
}));
await queue.addRequest(requestA, { forefront: true });
+ expect(queue.queueHeadDict.length()).to.be.eql(1);
// Try to get requestA which is not available yet.
mock.expects('getRequest')
@@ -577,7 +625,10 @@ describe('RequestQueue', () => {
.returns(Promise.resolve(null));
expect(await queue.fetchNextRequest()).to.be.eql(null);
- // Should try it once again.
+ // Give queue time to mark request 'a' as not in progress
+ await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);
+
+ // Should try it once again (the queue head is queried again)
mock.expects('getRequest')
.once()
.withArgs({
@@ -585,6 +636,18 @@ describe('RequestQueue', () => {
requestId: 'a',
})
.returns(Promise.resolve(requestA));
+ mock.expects('getHead')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ limit: QUERY_HEAD_MIN_LENGTH,
+ clientKey,
+ })
+ .returns(Promise.resolve({
+ items: [
+ { id: 'a', uniqueKey: 'aaa' },
+ ],
+ }));
expect(await queue.fetchNextRequest()).to.be.eql(requestA);
mock.verify();
@@ -771,7 +834,7 @@ describe('RequestQueue', () => {
.returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));
await queue.reclaimRequest(requestA, { forefront: true });
expect(queue.queueHeadDict.length()).to.be.eql(0);
- expect(queue.inProgressCount()).to.be.eql(0);
+ expect(queue.inProgressCount()).to.be.eql(1);
expect(queue.assumedTotalCount).to.be.eql(2);
expect(queue.assumedHandledCount).to.be.eql(1);
await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);