Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing bug where duplicate already handled requests added with forefront were returned in fetchNextRequest()
Marek Trunkat3ec109ea8479
diff --git a/src/request_queue.js b/src/request_queue.jsindex 63239c0a2..372d0ca72 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -231,11 +231,11 @@ export class RequestQueue {forefront,}).then((requestOperationInfo) => {- const { requestId } = requestOperationInfo;+ const { requestId, wasAlreadyHandled } = requestOperationInfo;this._cacheRequest(cacheKey, requestOperationInfo);- if (forefront && !this.requestIdsInProgress[requestId]) {+ if (forefront && !this.requestIdsInProgress[requestId] && !wasAlreadyHandled) {this.queueHeadDict.add(requestId, requestId, true);}diff --git a/test/request_queue.js b/test/request_queue.jsindex 21f32fbf5..40b737cd4 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -427,6 +427,45 @@ describe('RequestQueue', () => {mock.verify();mock.restore();});++ it('should not add handled request to queue head dict', async () => {+ expectNotUsingLocalStorage();++ const { Request } = Apify;++ const queue = new RequestQueue('some-id', 'some-name');+ const mock = sinon.mock(apifyClient.requestQueues);++ const requestA = new Request({ url: 'http://example.com/a' });++ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestA,+ forefront: true,+ })+ .returns(Promise.resolve({+ requestId: 'a',+ wasAlreadyHandled: true,+ wasAlreadyPresent: true,+ }));+ mock.expects('getRequest')+ .never();+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_MIN_LENGTH,+ })+ .returns(Promise.resolve({ items: [] }));++ await queue.addRequest(requestA, { forefront: true });+ expect(await queue.fetchNextRequest()).to.be.eql(null);++ mock.verify();+ mock.restore();+ });});describe('Apify.openRequestQueue', async () => {