Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

RequestQueue - caching requests locally (#76)

Marek Trunkát9a9977f2b1e7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a44c3aac..d02cf77bd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- `RequestQueue` now caches known requests and their state to beware of unneeded API calls.
+
0.5.23 / 2018-04-18
===================
- **WARNING**: `disableProxy` configuration of `PuppeteerCrawler` and `PuppeteerPool` removed. By default no proxy is used.
diff --git a/src/request_queue.js b/src/request_queue.js
index 9086bda67..6049d612a 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -14,6 +14,7 @@ import { ensureDirExists, apifyClient } from './utils';
export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.requestQueues;
const MAX_OPENED_QUEUES = 1000;
+const MAX_CACHED_REQUESTS = 1000 * 1000;
// When requesting queue head we always fetch requestsInProgressCount + QUERY_HEAD_BUFFER number of requests
export const QUERY_HEAD_BUFFER = 50;
@@ -83,6 +84,29 @@ const validateReclaimRequestParams = (request, opts) => {
return { forefront };
};
+/**
+ * Helper function that creates ID from uniqueKey for local emulation of request queue.
+ * It's also used for local cache of remote request queue.
+ *
+ * This function may not exactly match how requestId is created server side.
+ * So we never pass requestId created by this to server and use it only for local cache.
+ *
+ * @ignore
+ */
+const getRequestId = (uniqueKey) => {
+ checkParamOrThrow(uniqueKey, 'uniqueKey', 'String');
+
+ const str = crypto
+ .createHash('sha256')
+ .update(uniqueKey)
+ .digest('base64')
+ .replace(/(\+|\/|=)/g, '');
+
+ return str.length > 15
+ ? str.substr(0, 15)
+ : str;
+};
+
/**
* @typedef {Object} RequestOperationInfo
* @property {Boolean} wasAlreadyPresent Indicates if request was already present in the queue.
@@ -135,6 +159,10 @@ export class RequestQueue {
this.inProgressCount = 0;
this.queryQueueHeadPromise = null;
+ // Caching requests to avoid duplicite addRequest() calls.
+ // Key is computed using getRequestId() and value is { id, isHandled }.
+ this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
+
// This contains false if we were not able to get queue head with queueModifiedAt older than
// at least API_PROCESSED_REQUESTS_DELAY_MILLIS.
this.isHeadConsistent = true;
@@ -150,6 +178,18 @@ export class RequestQueue {
*/
addRequest(request, opts = {}) {
const { forefront } = validateAddRequestParams(request, opts);
+ const cacheKey = getRequestId(request.uniqueKey);
+ const cachedInfo = this.requestsCache.get(cacheKey);
+
+ if (cachedInfo) {
+ return Promise.resolve({
+ wasAlreadyPresent: true,
+ // We may assume that if request is in local cache then also the information if the
+ // request was already handled is there because just one client should be using one queue.
+ wasAlreadyHandled: cachedInfo.isHandled,
+ requestId: cachedInfo.id,
+ });
+ }
return requestQueues
.addRequest({
@@ -160,6 +200,8 @@ export class RequestQueue {
.then((requestOperationInfo) => {
const { requestId } = requestOperationInfo;
+ this._cacheRequest(cacheKey, requestOperationInfo);
+
if (forefront && !this.requestIdsInProgress[requestId]) {
this.queueHeadDict.add(requestId, requestId, true);
}
@@ -225,10 +267,11 @@ export class RequestQueue {
request,
queueId: this.queueId,
})
- .then((response) => {
+ .then((requestOperationInfo) => {
this._removeFromInProgress(request.id);
+ this._cacheRequest(getRequestId(request.uniqueKey), requestOperationInfo);
- return response;
+ return requestOperationInfo;
});
}
@@ -250,12 +293,13 @@ export class RequestQueue {
queueId: this.queueId,
forefront,
})
- .then((response) => {
+ .then((requestOperationInfo) => {
this._removeFromInProgress(request.id);
+ this._cacheRequest(getRequestId(request.uniqueKey), requestOperationInfo);
if (forefront) this.queueHeadDict.add(request.id, request.id, true);
- return response;
+ return requestOperationInfo;
});
}
@@ -285,11 +329,29 @@ export class RequestQueue {
.then(() => this.isHeadConsistent && this.inProgressCount === 0 && this.queueHeadDict.length() === 0);
}
+ /**
+ * Caches information about request to beware of unneeded addRequest() calls.
+ *
+ * @ignore
+ */
+ _cacheRequest(cacheKey, requestOperationInfo) {
+ checkParamOrThrow(cacheKey, 'cacheKey', 'String');
+ checkParamOrThrow(requestOperationInfo, 'requestOperationInfo', 'Object');
+ checkParamOrThrow(requestOperationInfo.requestId, 'requestOperationInfo.requestId', 'String');
+ checkParamOrThrow(requestOperationInfo.wasAlreadyHandled, 'requestOperationInfo.wasAlreadyHandled', 'Boolean');
+
+ this.requestsCache.add(cacheKey, {
+ id: requestOperationInfo.requestId,
+ isHandled: requestOperationInfo.wasAlreadyHandled,
+ });
+ }
/**
* @ignore
*/
_addToInProgress(requestId) {
+ checkParamOrThrow(requestId, 'requestId', 'String');
+
// Is already there.
if (this.requestIdsInProgress[requestId]) return;
@@ -301,6 +363,8 @@ export class RequestQueue {
* @ignore
*/
_removeFromInProgress(requestId) {
+ checkParamOrThrow(requestId, 'requestId', 'String');
+
// Is already removed.
if (!this.requestIdsInProgress[requestId]) return;
@@ -315,7 +379,9 @@ export class RequestQueue {
* @ignore
*/
_ensureHeadIsNonEmpty(checkModifiedAt = false, limit = this.inProgressCount + QUERY_HEAD_BUFFER, iteration = 0) {
+ checkParamOrThrow(checkModifiedAt, 'checkModifiedAt', 'Boolean');
checkParamOrThrow(limit, 'limit', 'Number');
+ checkParamOrThrow(iteration, 'iteration', 'Number');
// If is nonempty resolve immediately.
if (this.queueHeadDict.length()) return Promise.resolve();
@@ -329,8 +395,11 @@ export class RequestQueue {
queueId: this.queueId,
})
.then(({ items, queueModifiedAt }) => {
- items.forEach(({ id }) => {
- if (!this.requestIdsInProgress[id]) this.queueHeadDict.add(id, id, false);
+ items.forEach(({ id, uniqueKey }) => {
+ if (!this.requestIdsInProgress[id]) {
+ this.queueHeadDict.add(id, id, false);
+ this._cacheRequest(getRequestId(uniqueKey), { requestId: id, wasAlreadyHandled: false });
+ }
});
// This is needed so that the next call can request queue head again.
@@ -401,23 +470,6 @@ const filePathToQueueOrderNo = (filepath) => {
return parseInt(int, 10);
};
-/**
- * Helper function that creates ID from uniqueKey for local usage.
- *
- * @ignore
- */
-const uniqueKeyToId = (uniqueKey) => {
- const str = crypto
- .createHash('sha256')
- .update(uniqueKey)
- .digest('base64')
- .replace(/(\+|\/|=)/g, '');
-
- return str.length > 15
- ? str.substr(0, 15)
- : str;
-};
-
/**
* Local implementation of the `RequestQueue` class.
*
@@ -519,7 +571,7 @@ export class RequestQueueLocal {
// Add ID as server does.
const requestCopy = JSON.parse(JSON.stringify(request));
- requestCopy.id = uniqueKeyToId(request.uniqueKey);
+ requestCopy.id = getRequestId(request.uniqueKey);
// If request already exists then don't override it!
if (this.requestIdToQueueOrderNo[requestCopy.id]) {
diff --git a/test/request_queue.js b/test/request_queue.js
index c1fc09cf5..38758f1e8 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -144,7 +144,7 @@ describe('RequestQueue', () => {
request: requestA,
forefront: false,
})
- .returns(Promise.resolve({ requestId: 'a' }));
+ .returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false }));
await queue.addRequest(requestA);
const requestB = new Request({ url: 'http://example.com/b' });
@@ -155,7 +155,7 @@ describe('RequestQueue', () => {
request: requestB,
forefront: true,
})
- .returns(Promise.resolve({ requestId: 'b' }));
+ .returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false }));
await queue.addRequest(requestB, { forefront: true });
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount).to.be.eql(0);
@@ -181,7 +181,7 @@ describe('RequestQueue', () => {
request: requestB,
forefront: true,
})
- .returns(Promise.resolve({ requestId: requestB.id }));
+ .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true }));
await queue.reclaimRequest(requestB, { forefront: true });
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount).to.be.eql(0);
@@ -207,7 +207,7 @@ describe('RequestQueue', () => {
queueId: 'some-id',
request: requestB,
})
- .returns(Promise.resolve({ requestId: requestB.id }));
+ .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true }));
await queue.markRequestHandled(requestB);
expect(queue.queueHeadDict.length()).to.be.eql(0);
expect(queue.inProgressCount).to.be.eql(0);
@@ -221,8 +221,8 @@ describe('RequestQueue', () => {
})
.returns(Promise.resolve({
items: [
- { id: 'a' },
- { id: 'c' },
+ { id: 'a', uniqueKey: 'aaa' },
+ { id: 'c', uniqueKey: 'ccc' },
],
}));
mock.expects('getRequest')
@@ -240,6 +240,117 @@ describe('RequestQueue', () => {
mock.verify();
mock.restore();
});
+
+ it('should cache requests new locally', async () => {
+ expectNotLocalEmulation();
+
+ const { Request } = Apify;
+
+ const queue = new RequestQueue('some-id');
+ const mock = sinon.mock(apifyClient.requestQueues);
+
+ const requestA = new Request({ url: 'http://example.com/a' });
+ const requestB = new Request({ url: 'http://example.com/a' }); // Has same uniqueKey as A
+
+ // Add request A
+ mock.expects('addRequest')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ request: requestA,
+ forefront: false,
+ })
+ .returns(Promise.resolve({
+ requestId: 'a',
+ wasAlreadyHandled: false,
+ wasAlreadyPresent: false,
+ }));
+ await queue.addRequest(requestA);
+
+ // Add request B that has same unique so that addRequest() is not called because it's already cached.
+ mock.expects('addRequest').never();
+ expect(await queue.addRequest(requestB)).to.be.eql({
+ requestId: 'a',
+ wasAlreadyPresent: true,
+ wasAlreadyHandled: false,
+ });
+
+ mock.verify();
+ mock.restore();
+ });
+
+ it('should cache requests locally with info if request was already handled', async () => {
+ expectNotLocalEmulation();
+
+ const { Request } = Apify;
+
+ const queue = new RequestQueue('some-id');
+ const mock = sinon.mock(apifyClient.requestQueues);
+
+ const requestX = new Request({ url: 'http://example.com/x' });
+ const requestY = new Request({ url: 'http://example.com/x' }); // Has same uniqueKey as X
+
+ // Add request X.
+ mock.expects('addRequest')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ request: requestX,
+ forefront: false,
+ })
+ .returns(Promise.resolve({
+ requestId: 'x',
+ wasAlreadyHandled: true,
+ wasAlreadyPresent: true,
+ }));
+ await queue.addRequest(requestX);
+
+ // Add request Y that has same unique so that addRequest() is not called because it's already cached.
+ mock.expects('addRequest').never();
+ expect(await queue.addRequest(requestY)).to.be.eql({
+ requestId: 'x',
+ wasAlreadyPresent: true,
+ wasAlreadyHandled: true,
+ });
+
+ mock.verify();
+ mock.restore();
+ });
+
+ it('should cache requests from queue head', async () => {
+ expectNotLocalEmulation();
+
+ const { Request } = Apify;
+
+ const queue = new RequestQueue('some-id');
+ const mock = sinon.mock(apifyClient.requestQueues);
+
+ // Query queue head with request A
+ mock.expects('getHead')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ limit: QUERY_HEAD_BUFFER,
+ })
+ .returns(Promise.resolve({
+ items: [
+ { id: 'a', uniqueKey: 'aaa' },
+ ],
+ }));
+ expect(await queue.isEmpty()).to.be.eql(false);
+
+ // Add request A and addRequest is not called because was already cached.
+ const requestA = new Request({ url: 'http://example.com/a', uniqueKey: 'aaa' });
+ mock.expects('addRequest').never();
+ expect(await queue.addRequest(requestA)).to.be.eql({
+ requestId: 'a',
+ wasAlreadyPresent: true,
+ wasAlreadyHandled: false,
+ });
+
+ mock.verify();
+ mock.restore();
+ });
});
describe('Apify.openRequestQueue', async () => {