Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added "request" field to RequestOperationInfo object
Jan Curne647d047f123
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 50c7909ca..ea985307d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@xxx===================- Fixed `RequestQueue.addRequest()` to gracefully handle invalid URLs+- Added `request` field to `RequestOperationInfo`0.8.17 / 2018-10-30===================diff --git a/src/request_queue.js b/src/request_queue.jsindex a7ff3d526..0ed3ddd39 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -113,14 +113,18 @@ const getRequestId = (uniqueKey) => {: str;};+// TODO: Let's rename RequestOperationInfo to QueueOperationInfo, it makes more sense, it's not used anywhere+/**- * A helper class that is used to report results from the- * [`Apify.utils.puppeteer.enqueueLinks()`](../api/puppeteer#puppeteer.enqueueLinks) function.+ * A helper class that is used to report results from various+ * [`RequestQueue`](../api/requestqueue) functions as well as+ * [`Apify.utils.puppeteer.enqueueLinks()`](../api/puppeteer#puppeteer.enqueueLinks).** @typedef {Object} RequestOperationInfo* @property {Boolean} wasAlreadyPresent Indicates if request was already present in the queue.* @property {Boolean} wasAlreadyHandled Indicates if request was already marked as handled.* @property {String} requestId The ID of the added request+ * @property {Request} request The original `Request` object passed to the `RequestQueue` function.*//**@@ -235,6 +239,8 @@ export class RequestQueue {// request was already handled is there because just one client should be using one queue.wasAlreadyHandled: cachedInfo.isHandled,requestId: cachedInfo.id,+ // TODO: Why not set request.id to cachedInfo.id???+ request,});}@@ -253,6 +259,9 @@ export class RequestQueue {this.queueHeadDict.add(requestId, requestId, true);}+ // TODO: Why not set request.id to cachedInfo.id???+ requestOperationInfo.request = request;+return requestOperationInfo;});}@@ -266,6 +275,8 @@ export class RequestQueue {getRequest(requestId) {validateGetRequestParams(requestId);+ // TODO: Could we also use requestsCache here? It would be consistent with addRequest()+return requestQueues.getRequest({requestId,@@ -329,6 +340,8 @@ export class RequestQueue {this._removeFromInProgress(request.id);this._cacheRequest(getRequestId(request.uniqueKey), requestOperationInfo);+ requestOperationInfo.request = request;+return requestOperationInfo;});}@@ -359,6 +372,8 @@ export class RequestQueue {if (forefront) this.queueHeadDict.add(request.id, request.id, true);+ requestOperationInfo.request = request;+return requestOperationInfo;});}@@ -651,6 +666,7 @@ export class RequestQueueLocal {const queueOrderNo = this._getQueueOrderNo(forefront);// Add ID as server does.+ // TODO: This way of cloning doesn't preserve Dates!const requestCopy = JSON.parse(JSON.stringify(request));requestCopy.id = getRequestId(request.uniqueKey);@@ -662,6 +678,7 @@ export class RequestQueueLocal {requestId: existingRequest.id,wasAlreadyHandled: existingRequest && existingRequest.handledAt,wasAlreadyPresent: true,+ request,}));}@@ -675,6 +692,7 @@ export class RequestQueueLocal {requestId: requestCopy.id,wasAlreadyHandled: false,wasAlreadyPresent: false,+ request,}));});}@@ -750,6 +768,7 @@ export class RequestQueueLocal {requestId: request.id,wasAlreadyHandled: false,wasAlreadyPresent: true,+ request,};});});@@ -783,6 +802,7 @@ export class RequestQueueLocal {requestId: request.id,wasAlreadyHandled: false,wasAlreadyPresent: true,+ request,};});});diff --git a/test/request_queue.js b/test/request_queue.jsindex 0a8f65692..c90953afb 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -28,10 +28,18 @@ describe('RequestQueue', () => {it('should work', async () => {const queue = new RequestQueueLocal('my-queue-0', LOCAL_STORAGE_DIR);- await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));+ const req1 = new Apify.Request({ url: 'http://example.com/first' });+ const info1 = await queue.addRequest(req1);await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));await queue.addRequest(new Apify.Request({ url: 'http://example.com/last-but-first' }), { forefront: true });+ expect(info1).to.contain({+ wasAlreadyPresent: false,+ wasAlreadyHandled: false,+ request: req1,+ });+ expect(info1.requestId).to.be.a('string');+const request3 = await queue.fetchNextRequest();const request1 = await queue.fetchNextRequest();const request2 = await queue.fetchNextRequest();@@ -60,9 +68,16 @@ describe('RequestQueue', () => {await queue.markRequestHandled(request3);await queue.reclaimRequest(request1);- await queue.reclaimRequest(request2);+ const info2 = await queue.reclaimRequest(request2);expect(await queue.isEmpty()).to.be.eql(false);+ expect(info2).to.contain({+ wasAlreadyPresent: true,+ wasAlreadyHandled: false,+ request: request2,+ });+ expect(info2.requestId).to.be.a('string');+const handledRequest3 = await queue.getRequest(request3.id);expect(handledRequest3.handledAt).to.be.an.instanceof(Date);expect(handledRequest3).to.be.eql(request3);@@ -70,9 +85,16 @@ describe('RequestQueue', () => {expect(await queue.fetchNextRequest()).to.be.eql(request1);expect(await queue.fetchNextRequest()).to.be.eql(request2);expect(await queue.fetchNextRequest()).to.be.eql(null);- await queue.markRequestHandled(request1);+ const info3 = await queue.markRequestHandled(request1);await queue.markRequestHandled(request2);+ expect(info3).to.contain({+ wasAlreadyPresent: true,+ wasAlreadyHandled: false,+ request: request1,+ });+ expect(info3.requestId).to.be.a('string');+expect(await queue.isEmpty()).to.be.eql(true);expect(await queue.isFinished()).to.be.eql(true);@@ -195,7 +217,7 @@ describe('RequestQueue', () => {request: requestA,forefront: false,})- .returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false }));+ .returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));await queue.addRequest(requestA);const requestB = new Request({ url: 'http://example.com/b' });@@ -206,7 +228,7 @@ describe('RequestQueue', () => {request: requestB,forefront: true,})- .returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false }));+ .returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestB }));await queue.addRequest(requestB, { forefront: true });expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount).to.be.eql(0);@@ -232,7 +254,7 @@ describe('RequestQueue', () => {request: requestB,forefront: true,})- .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true }));+ .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));await queue.reclaimRequest(requestB, { forefront: true });expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount).to.be.eql(0);@@ -258,7 +280,7 @@ describe('RequestQueue', () => {queueId: 'some-id',request: requestB,})- .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true }));+ .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));await queue.markRequestHandled(requestB);expect(queue.queueHeadDict.length()).to.be.eql(0);expect(queue.inProgressCount).to.be.eql(0);@@ -324,6 +346,7 @@ describe('RequestQueue', () => {requestId: 'a',wasAlreadyHandled: false,wasAlreadyPresent: false,+ request: requestA,}));await queue.addRequest(requestA);@@ -333,6 +356,7 @@ describe('RequestQueue', () => {requestId: 'a',wasAlreadyPresent: true,wasAlreadyHandled: false,+ request: requestB,});mock.verify();@@ -362,6 +386,7 @@ describe('RequestQueue', () => {requestId: 'x',wasAlreadyHandled: true,wasAlreadyPresent: true,+ request: requestX,}));await queue.addRequest(requestX);@@ -371,6 +396,7 @@ describe('RequestQueue', () => {requestId: 'x',wasAlreadyPresent: true,wasAlreadyHandled: true,+ request: requestY,});mock.verify();@@ -406,6 +432,7 @@ describe('RequestQueue', () => {requestId: 'a',wasAlreadyPresent: true,wasAlreadyHandled: false,+ request: requestA,});mock.verify();@@ -435,6 +462,7 @@ describe('RequestQueue', () => {requestId: 'a',wasAlreadyHandled: false,wasAlreadyPresent: false,+ request: requestA,}));await queue.addRequest(requestA, { forefront: true });@@ -483,6 +511,7 @@ describe('RequestQueue', () => {requestId: 'a',wasAlreadyHandled: true,wasAlreadyPresent: true,+ request: requestA,}));mock.expects('getRequest').never();