Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added RequestQueue.getInfo() function, bugfixed LocalRequestQueue.getRequest() to return null if not found, improved docs and tests
Jan Curn7582afb2be84
diff --git a/src/request_queue.js b/src/request_queue.jsindex 2d916f347..c9a70af7c 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -30,6 +30,7 @@ const writeFilePromised = Promise.promisify(fs.writeFile);const readdirPromised = Promise.promisify(fs.readdir);const readFilePromised = Promise.promisify(fs.readFile);const renamePromised = Promise.promisify(fs.rename);+const statPromised = Promise.promisify(fs.stat);const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);const { requestQueues } = apifyClient;@@ -267,25 +268,24 @@ export class RequestQueue {* Gets the request from the queue specified by ID.** @param {String} requestId Request ID- * @return {Promise<Request>}+ * @return {Promise<Request>} Returns the request object, or `null` if it was not found.*/- getRequest(requestId) {+ async getRequest(requestId) {validateGetRequestParams(requestId);// TODO: Could we also use requestsCache here? It would be consistent with addRequest()+ const obj = await requestQueues.getRequest({+ requestId,+ queueId: this.queueId,+ });- return requestQueues- .getRequest({- requestId,- queueId: this.queueId,- })- .then(obj => (obj ? new Request(obj) : obj));+ return obj ? new Request(obj) : obj;}/*** Returns next request in the queue to be processed.*- * @returns {Promise<Request>}+ * @returns {Promise<Request>} Returns the request object, or `null` if there are no more pending requests.*/fetchNextRequest() {return this@@ -550,9 +550,39 @@ export class RequestQueue {* @return {Promise<number>}*/async handledCount() {+ // TODO: We should deprectate this function in favor of getInfo()const queueInfo = await requestQueues.getQueue({ queueId: this.queueId });return queueInfo.handledRequestCount;}++ /**+ * Returns an object containing general information about the request queue.+ *+ * **Example:**+ * ```+ * {+ * "id": "WkzbQMuFYuamGv3YF",+ * "name": "my-queue",+ * "userId": "wRsJZtadYvn4mBZmm",+ * "createdAt": new Date("2015-12-12T07:34:14.202Z"),+ * "modifiedAt": new Date("2015-12-13T08:36:13.202Z"),+ * "accessedAt": new Date("2015-12-14T08:36:13.202Z"),+ * totalRequestCount: 0,+ * handledRequestCount: 0,+ * pendingRequestCount: 0,+ * }+ * ```+ * @param {Object} [options={}]+ * Additional options passed to Apify API Client's+ * [getQueue](https://www.apify.com/docs/api/apify-client-js/latest#ApifyClient-requestQueues-getQueue) function.+ * @returns {Promise<Object>}+ */+ getInfo(options = {}) {+ const { queueId } = this;+ const params = Object.assign({ queueId }, options);++ return requestQueues.getQueue(params);+ }}/**@@ -590,6 +620,10 @@ export class RequestQueueLocal {this.requestIdToQueueOrderNo = {};this.queueOrderNoInProgress = {};+ this.createdAt = null;+ this.modifiedAt = null;+ this.accessedAt = null;+this.initializationPromise = this._initialize();}@@ -610,6 +644,11 @@ export class RequestQueueLocal {const pendingPaths = pending.map(filename => path.join(this.localPendingEmulationPath, filename));const filePaths = handledPaths.concat(pendingPaths);+ const stats = await statPromised(this.localPendingEmulationPath);+ this.createdAt = stats.birthtime;+ this.modifiedAt = stats.mtime;+ this.accessedAt = stats.atime;+return Promise.mapSeries(filePaths, filepath => this._readFile(filepath));}@@ -710,6 +749,9 @@ export class RequestQueueLocal {.then(() => {const queueOrderNo = this.requestIdToQueueOrderNo[requestId];+ // TODO: We should update the last access time to files...+ if (!queueOrderNo) return null;+return this._getRequestByQueueOrderNo(queueOrderNo);});}@@ -836,6 +878,24 @@ export class RequestQueueLocal {await this.initializationPromise;return this._handledCount;}++ async getInfo() {+ await this.initializationPromise;++ const id = this.queueId;+ const name = id === ENV_VARS.DEFAULT_REQUEST_QUEUE_ID ? null : id;+ return {+ id,+ name,+ userId: process.env[ENV_VARS.USER_ID] || null,+ createdAt: this.createdAt,+ modifiedAt: this.modifiedAt,+ accessedAt: this.accessedAt,+ totalRequestCount: this._handledCount + this.pendingCount,+ handledRequestCount: this._handledCount,+ pendingRequestCount: this.pendingCount,+ };+ }}/**diff --git a/test/request_queue.js b/test/request_queue.jsindex da00d3263..df4f9ed22 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -50,6 +50,7 @@ describe('RequestQueue', () => {expect(await queue.getRequest(request2.id)).to.be.eql(request2);expect(await queue.getRequest(request3.id)).to.be.eql(request3);expect(await queue.getRequest(request1.id)).to.not.be.eql(request3);+ expect(await queue.getRequest('non-exiting')).to.be.eql(null);expect(request3.url).to.be.eql('http://example.com/last-but-first');expect(request1.url).to.be.eql('http://example.com/first');@@ -201,32 +202,76 @@ describe('RequestQueue', () => {).to.be.eql('http://example.com/a');});- it('should return correct handledCount', async () => {- const queue = new RequestQueueLocal('id', LOCAL_STORAGE_DIR);+ it('getInfo() and handledCount() should work', async () => {+ const queueName = 'stats-queue';+ const queue = new RequestQueueLocal(queueName, LOCAL_STORAGE_DIR);let count = await queue.handledCount();+ let info;expect(count).to.be.eql(0);const r1 = new Apify.Request({ url: 'http://example.com/1' });const r2 = new Apify.Request({ url: 'http://example.com/2' });const r3 = new Apify.Request({ url: 'http://example.com/3' });- await queue.addRequest(r1);+ const op1 = await queue.addRequest(r1);await queue.addRequest(r2);+count = await queue.handledCount();+ info = await queue.getInfo();expect(count).to.be.eql(0);+ expect(info).to.be.an('object');+ expect(info.id).to.be.eql(queueName);+ expect(info.name).to.be.eql(queueName);+ expect(info.userId).to.be.eql(null);+ expect(info.totalRequestCount).to.be.eql(2);+ expect(info.pendingRequestCount).to.be.eql(2);+ expect(info.handledRequestCount).to.be.eql(0);+ const cTime = info.createdAt.getTime();+ let mTime = info.modifiedAt.getTime();+ expect(cTime).to.be.below(Date.now() + 1);+ expect(cTime).to.be.eql(mTime);+const rf1 = await queue.fetchNextRequest();await queue.markRequestHandled(rf1);count = await queue.handledCount();+ info = await queue.getInfo();expect(count).to.be.eql(1);+ expect(info.totalRequestCount).to.be.eql(2);+ expect(info.pendingRequestCount).to.be.eql(1);+ expect(info.handledRequestCount).to.be.eql(1);+await queue.addRequest(r3);const rf2 = await queue.fetchNextRequest();await queue.markRequestHandled(rf2);const rf3 = await queue.fetchNextRequest();await queue.markRequestHandled(rf3);count = await queue.handledCount();+ info = await queue.getInfo();expect(count).to.be.eql(3);-- const newQueue = new RequestQueueLocal('id', LOCAL_STORAGE_DIR);+ expect(info.totalRequestCount).to.be.eql(3);+ expect(info.pendingRequestCount).to.be.eql(0);+ expect(info.handledRequestCount).to.be.eql(3);++ // Test access time+ await Apify.utils.sleep(11);+ await queue.getRequest(op1.requestId);+ await Apify.utils.sleep(11);+ const now = Date.now();+ await Apify.utils.sleep(11);+ info = await queue.getInfo();+ const cTime2 = info.createdAt.getTime();+ mTime = info.modifiedAt.getTime();+ const aTime = info.accessedAt.getTime();+ expect(cTime).to.be.eql(cTime2);+ expect(mTime).to.be.below(aTime);+ expect(mTime).to.be.below(now);+ expect(aTime).to.be.below(now);++ const newQueue = new RequestQueueLocal(queueName, LOCAL_STORAGE_DIR);count = await newQueue.handledCount();+ info = await queue.getInfo();expect(count).to.be.eql(3);+ expect(info.totalRequestCount).to.be.eql(3);+ expect(info.pendingRequestCount).to.be.eql(0);+ expect(info.handledRequestCount).to.be.eql(3);});});@@ -276,6 +321,17 @@ describe('RequestQueue', () => {expect(queue.queueHeadDict.length()).to.be.eql(0);expect(queue.inProgressCount).to.be.eql(1);+ // getRequest() returns null if object was not found.+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'non-existent',+ })+ .returns(Promise.resolve(null));+ const requestXFromQueue = await queue.getRequest('non-existent');+ expect(requestXFromQueue).to.be.eql(null);+// Reclaim it.mock.expects('updateRequest').once()@@ -626,6 +682,34 @@ describe('RequestQueue', () => {mock.verify();mock.restore();});++ it('getInfo() should work', async () => {+ const queue = new RequestQueue('some-id', 'some-name');+ const mock = sinon.mock(apifyClient.requestQueues);++ const expected = {+ id: 'WkzbQMuFYuamGv3YF',+ name: 'my-queue',+ userId: 'wRsJZtadYvn4mBZmm',+ createdAt: new Date('2015-12-12T07:34:14.202Z'),+ modifiedAt: new Date('2015-12-13T08:36:13.202Z'),+ accessedAt: new Date('2015-12-14T08:36:13.202Z'),+ totalRequestCount: 0,+ handledRequestCount: 0,+ pendingRequestCount: 0,+ };++ mock.expects('getQueue')+ .once()+ .returns(Promise.resolve(expected));++ const result = await queue.getInfo();++ expect(result).to.be.eql(expected);++ mock.verify();+ mock.restore();+ });});describe('Apify.openRequestQueue', async () => {