Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing situation when newly added request is not available in the queue yet causing it to stuck forever
Marek Trunkat8b3b250c7ed0
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex e6a50113b..ee96a9bc5 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -241,7 +241,7 @@ export default class BasicCrawler {// rejectOnStopPromise rejects when .stop() is called or AutoscaledPool throws.// All running tasks are therefore terminated with an error to be reclaimed and retried.await Promise.race([this.handleRequestFunction({ request }), this.rejectOnStopPromise]);- source.markRequestHandled(request);+ await source.markRequestHandled(request);this.handledRequestsCount++;} catch (err) {await this._requestFunctionErrorHandler(err, request, source);diff --git a/src/request_queue.js b/src/request_queue.jsindex 358fac610..704ee22b0 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -245,7 +245,21 @@ export class RequestQueue {this._addToInProgress(nextId);- return this.getRequest(nextId);+ return this+ .getRequest(nextId)+ .then((request) => {+ // We need to handle this situation because request may not be available+ // immediately after adding to the queue.+ if (!request) {+ this._removeFromInProgress(nextId);++ // TODO: Consider skipping this line so that the request doesn't get retried immediately+ // but after next head fetch instead.+ this.queueHeadDict.add(nextId, nextId, true);+ }++ return request;+ });});}diff --git a/test/request_queue.js b/test/request_queue.jsindex ebfcea5f5..25fa68ad2 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -377,6 +377,56 @@ describe('RequestQueue', () => {mock.verify();mock.restore();});++ it('shoud handle situation when newly created request is not available yet', async () => {+ expectNotUsingLocalStorage();++ const { Request } = Apify;++ const queue = new RequestQueue('some-id');+ const mock = sinon.mock(apifyClient.requestQueues);++ const requestA = new Request({ url: 'http://example.com/a' });++ // Add request A+ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestA,+ forefront: true,+ })+ .returns(Promise.resolve({+ requestId: 'a',+ wasAlreadyHandled: false,+ wasAlreadyPresent: false,+ }));+ await queue.addRequest(requestA, { forefront: true });++ // Try to get requestA which is not available yet.+ mock.expects('getHead').never();+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'a',+ })+ .returns(Promise.resolve(null));+ expect(await queue.fetchNextRequest()).to.be.eql(null);++ // Should try it once again.+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'a',+ })+ .returns(Promise.resolve(requestA));+ expect(await queue.fetchNextRequest()).to.be.eql(requestA);++ mock.verify();+ mock.restore();+ });});describe('Apify.openRequestQueue', async () => {