Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix RequestQueue not updating on Local

Ondra Urbance6f546d243e
diff --git a/src/request.js b/src/request.js
index 35f32d706..51bd730de 100644
--- a/src/request.js
+++ b/src/request.js
@@ -54,6 +54,7 @@ export default class Request {
userData = {},
keepUrlFragment = false,
ignoreErrors = false,
+ handledAt = null,
} = opts;
@@ -67,6 +68,7 @@ export default class Request {
checkParamOrThrow(headers, 'headers', 'Object');
checkParamOrThrow(userData, 'userData', 'Object');
checkParamOrThrow(ignoreErrors, 'ignoreErrors', 'Boolean');
+ checkParamOrThrow(handledAt, 'handledAt', 'Maybe String');
if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');
@@ -80,6 +82,7 @@ export default class Request {
this.headers = headers;
this.userData = userData;
this.ignoreErrors = ignoreErrors;
+ this.handledAt = handledAt && new Date(handledAt);
}
/**
diff --git a/src/request_queue.js b/src/request_queue.js
index c6915c25b..8e421c0b0 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -30,7 +30,7 @@ export const MAX_QUERIES_FOR_CONSISTENCY = 6;
const writeFilePromised = Promise.promisify(fs.writeFile);
const readdirPromised = Promise.promisify(fs.readdir);
const readFilePromised = Promise.promisify(fs.readFile);
-const moveFilePromised = Promise.promisify(fsExtra.move);
+const deleteFilePromised = Promise.promisify(fs.unlink);
const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { requestQueues } = apifyClient;
@@ -665,7 +665,10 @@ export class RequestQueueLocal {
if (!request.handledAt) request.handledAt = new Date();
- return moveFilePromised(source, dest)
+ return Promise.all([
+ writeFilePromised(dest, JSON.stringify(request, null, 4)),
+ deleteFilePromised(source),
+ ])
.then(() => {
this.pendingCount--;
this.inProgressCount--;
@@ -693,13 +696,14 @@ export class RequestQueueLocal {
this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;
- return moveFilePromised(
- this._getFilePath(oldQueueOrderNo),
- this._getFilePath(newQueueOrderNo),
- ).then(() => {
- this.inProgressCount--;
- delete this.queueOrderNoInProgress[oldQueueOrderNo];
- });
+ return Promise.all([
+ writeFilePromised(this._getFilePath(newQueueOrderNo), JSON.stringify(request, null, 4)),
+ deleteFilePromised(this._getFilePath(oldQueueOrderNo)),
+ ])
+ .then(() => {
+ this.inProgressCount--;
+ delete this.queueOrderNoInProgress[oldQueueOrderNo];
+ });
})
.then(() => ({
requestId: request.id,
diff --git a/test/request_queue.js b/test/request_queue.js
index 1c96ab509..0653e0c8a 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -53,11 +53,20 @@ describe('RequestQueue', () => {
expect(await queue.isEmpty()).to.be.eql(true);
expect(await queue.isFinished()).to.be.eql(false);
+ // Check that changes to Requests are persisted to Queue.
+ request1.errorMessages = ['Hello'];
+ request2.retryCount = 2;
+ request3.retryCount = 3;
+
await queue.markRequestHandled(request3);
await queue.reclaimRequest(request1);
await queue.reclaimRequest(request2);
expect(await queue.isEmpty()).to.be.eql(false);
+ const handledRequest3 = await queue.getRequest(request3.id);
+ expect(handledRequest3.handledAt).to.be.an.instanceof(Date);
+ expect(handledRequest3).to.be.eql(request3);
+
expect(await queue.fetchNextRequest()).to.be.eql(request1);
expect(await queue.fetchNextRequest()).to.be.eql(request2);
expect(await queue.fetchNextRequest()).to.be.eql(null);