Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixing bug where duplicate already handled requests added with forefront were returned in fetchNextRequest()

Marek Trunkat3ec109ea8479
diff --git a/src/request_queue.js b/src/request_queue.js
index 63239c0a2..372d0ca72 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -231,11 +231,11 @@ export class RequestQueue {
forefront,
})
.then((requestOperationInfo) => {
- const { requestId } = requestOperationInfo;
+ const { requestId, wasAlreadyHandled } = requestOperationInfo;
this._cacheRequest(cacheKey, requestOperationInfo);
- if (forefront && !this.requestIdsInProgress[requestId]) {
+ if (forefront && !this.requestIdsInProgress[requestId] && !wasAlreadyHandled) {
this.queueHeadDict.add(requestId, requestId, true);
}
diff --git a/test/request_queue.js b/test/request_queue.js
index 21f32fbf5..40b737cd4 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -427,6 +427,45 @@ describe('RequestQueue', () => {
mock.verify();
mock.restore();
});
+
+ it('should not add handled request to queue head dict', async () => {
+ expectNotUsingLocalStorage();
+
+ const { Request } = Apify;
+
+ const queue = new RequestQueue('some-id', 'some-name');
+ const mock = sinon.mock(apifyClient.requestQueues);
+
+ const requestA = new Request({ url: 'http://example.com/a' });
+
+ mock.expects('addRequest')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ request: requestA,
+ forefront: true,
+ })
+ .returns(Promise.resolve({
+ requestId: 'a',
+ wasAlreadyHandled: true,
+ wasAlreadyPresent: true,
+ }));
+ mock.expects('getRequest')
+ .never();
+ mock.expects('getHead')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ limit: QUERY_HEAD_MIN_LENGTH,
+ })
+ .returns(Promise.resolve({ items: [] }));
+
+ await queue.addRequest(requestA, { forefront: true });
+ expect(await queue.fetchNextRequest()).to.be.eql(null);
+
+ mock.verify();
+ mock.restore();
+ });
});
describe('Apify.openRequestQueue', async () => {