Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: ignore requests that are no longer in progress (#1258)

Martin Adámeked9c80b99245
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c579b89f1..f8f2fc85f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+2.2.1 / 2021/12/20
+====================
+- fix: ignore requests that are no longer in progress
+
2.2.0 / 2021/12/17
====================
diff --git a/src/storages/request_queue.js b/src/storages/request_queue.js
index 666e85187..12a9189b7 100644
--- a/src/storages/request_queue.js
+++ b/src/storages/request_queue.js
@@ -339,7 +339,7 @@ export class RequestQueue {
* Handled requests will never again be returned by the `fetchNextRequest` function.
*
* @param {Request} request
- * @return {Promise<QueueOperationInfo>}
+ * @return {Promise<QueueOperationInfo | null>}
*/
async markRequestHandled(request) {
ow(request, ow.object.partialShape({
@@ -349,7 +349,8 @@ export class RequestQueue {
}));
if (!this.inProgress.has(request.id)) {
- throw new Error(`Cannot mark request ${request.id} as handled, because it is not in progress!`);
+ this.log.debug(`Cannot mark request ${request.id} as handled, because it is not in progress!`, { requestId: request.id });
+ return null;
}
if (!request.handledAt) request.handledAt = new Date();
@@ -382,7 +383,7 @@ export class RequestQueue {
* If `true` then the request it placed to the beginning of the queue, so that it's returned
* in the next call to {@link RequestQueue#fetchNextRequest}.
* By default, it's put to the end of the queue.
- * @return {Promise<QueueOperationInfo>}
+ * @return {Promise<QueueOperationInfo | null>}
*/
async reclaimRequest(request, options = {}) {
ow(request, ow.object.partialShape({
@@ -396,7 +397,8 @@ export class RequestQueue {
const { forefront = false } = options;
if (!this.inProgress.has(request.id)) {
- throw new Error(`Cannot reclaim request ${request.id}, because it is not in progress!`);
+ this.log.debug(`Cannot reclaim request ${request.id}, because it is not in progress!`, { requestId: request.id });
+ return null;
}
// TODO: If request hasn't been changed since the last getRequest(),
@@ -411,7 +413,7 @@ export class RequestQueue {
tryCancel();
if (!this.inProgress.has(request.id)) {
- this.log.warning('The request is no longer marked as in progress in the queue?!', { requestId: request.id });
+ this.log.debug('The request is no longer marked as in progress in the queue?!', { requestId: request.id });
return;
}