Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
(Dataset|RequestQueue).getInfo() no longer accept an options argument, fixed metadate updates, updated comments and tests
Jan Curndf611135ed98
diff --git a/src/dataset.js b/src/dataset.jsindex b4d8bfb0c..08f910f24 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -255,6 +255,12 @@ export class Dataset {/*** Returns an object containing general information about the dataset.*+ * The function returns the same object as the Apify API Client's+ * [getDataset](https://www.apify.com/docs/api/apify-client-js/latest#ApifyClient-datasets-getDataset)+ * function, which in turn calls the+ * [Get dataset](https://www.apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)+ * API endpoint.+ ** **Example:*** ```* {@@ -269,17 +275,10 @@ export class Dataset {* }* ```*- * @param {Object} [options={}]- * Additional options passed to Apify API Client's- * [getDataset](https://www.apify.com/docs/api/apify-client-js/latest#ApifyClient-datasets-getDataset) function.- ** @returns {Promise<Object>}*/- getInfo(options = {}) {- const { datasetId } = this;- const params = Object.assign({ datasetId }, options);-- return datasets.getDataset(params);+ async getInfo() {+ return datasets.getDataset({ datasetId: this.datasetId });}/**@@ -510,23 +509,25 @@ export class DatasetLocal {});}- getInfo() {- return this.initializationPromise- .then(() => {- const id = this.datasetId;- const name = id === ENV_VARS.DEFAULT_DATASET_ID ? null : id;- return {- id,- name,- userId: process.env[ENV_VARS.USER_ID] || null,- createdAt: this.createdAt,- modifiedAt: this.modifiedAt,- accessedAt: this.accessedAt,- itemCount: this.counter,- // TODO: This number is not counted correctly!- cleanItemCount: this.counter,- };- });+ async getInfo() {+ await this.initializationPromise;++ const id = this.datasetId;+ const name = id === ENV_VARS.DEFAULT_DATASET_ID ? null : id;+ const result = {+ id,+ name,+ userId: process.env[ENV_VARS.USER_ID] || null,+ createdAt: this.createdAt,+ modifiedAt: this.modifiedAt,+ accessedAt: this.accessedAt,+ itemCount: this.counter,+ // TODO: This number is not counted correctly!+ cleanItemCount: this.counter,+ };++ this._updateMetadata();+ return result;}forEach(iteratee) {diff --git a/src/request_queue.js b/src/request_queue.jsindex c9a70af7c..b49616f95 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -558,6 +558,12 @@ export class RequestQueue {/*** Returns an object containing general information about the request queue.*+ * The function returns the same object as the Apify API Client's+ * [getQueue](https://www.apify.com/docs/api/apify-client-js/latest#ApifyClient-requestQueues-getQueue)+ * function, which in turn calls the+ * [Get request queue](https://www.apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)+ * API endpoint.+ ** **Example:*** ```* {@@ -572,16 +578,11 @@ export class RequestQueue {* pendingRequestCount: 0,* }* ```- * @param {Object} [options={}]- * Additional options passed to Apify API Client's- * [getQueue](https://www.apify.com/docs/api/apify-client-js/latest#ApifyClient-requestQueues-getQueue) function.+ ** @returns {Promise<Object>}*/- getInfo(options = {}) {- const { queueId } = this;- const params = Object.assign({ queueId }, options);-- return requestQueues.getQueue(params);+ async getInfo() {+ return requestQueues.getQueue({ queueId: this.queueId });}}@@ -715,6 +716,8 @@ export class RequestQueueLocal {const requestCopy = JSON.parse(JSON.stringify(request));requestCopy.id = getRequestId(request.uniqueKey);+ this._updateMetadata(true);+// If request already exists then don't override it!if (this.requestIdToQueueOrderNo[requestCopy.id]) {return this@@ -749,6 +752,8 @@ export class RequestQueueLocal {.then(() => {const queueOrderNo = this.requestIdToQueueOrderNo[requestId];+ this._updateMetadata();+// TODO: We should update the last access time to files...if (!queueOrderNo) return null;@@ -761,6 +766,8 @@ export class RequestQueueLocal {const files = await readdirPromised(this.localPendingEmulationPath);+ this._updateMetadata();+let request = null;while (!request && files.length) {const filename = files.shift();@@ -803,6 +810,8 @@ export class RequestQueueLocal {if (!request.handledAt) request.handledAt = new Date();+ this._updateMetadata(true);+// NOTE: First write to old file and then rename to new one to do the operation atomically.// Situation where two files exists at the same time may cause race condition bugs.return writeFilePromised(source, JSON.stringify(request, null, 4))@@ -839,6 +848,8 @@ export class RequestQueueLocal {this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;+ this._updateMetadata(true);+// NOTE: First write to old file and then rename to new one to do the operation atomically.// Situation where two files exists at the same time may cause race condition bugs.return writeFilePromised(source, JSON.stringify(request, null, 4))@@ -857,25 +868,26 @@ export class RequestQueueLocal {});}- isEmpty() {- return this.initializationPromise- .then(() => this.pendingCount === this.inProgressCount);+ async isEmpty() {+ await this.initializationPromise;+ this._updateMetadata();+ return this.pendingCount === this.inProgressCount;}- isFinished() {- return this.initializationPromise- .then(() => this.pendingCount === 0);+ async isFinished() {+ await this.initializationPromise;+ this._updateMetadata();+ return this.pendingCount === 0;}- delete() {- return emptyDirPromised(this.localStoragePath)- .then(() => {- queuesCache.remove(this.queueId);- });+ async delete() {+ await emptyDirPromised(this.localStoragePath);+ queuesCache.remove(this.queueId);}async handledCount() {await this.initializationPromise;+ this._updateMetadata();return this._handledCount;}@@ -884,7 +896,7 @@ export class RequestQueueLocal {const id = this.queueId;const name = id === ENV_VARS.DEFAULT_REQUEST_QUEUE_ID ? null : id;- return {+ const result = {id,name,userId: process.env[ENV_VARS.USER_ID] || null,@@ -895,6 +907,15 @@ export class RequestQueueLocal {handledRequestCount: this._handledCount,pendingRequestCount: this.pendingCount,};++ this._updateMetadata();+ return result;+ }++ _updateMetadata(isModified) {+ const date = new Date();+ this.accessedAt = date;+ if (isModified) this.modifiedAt = date;}}diff --git a/test/request_queue.js b/test/request_queue.jsindex df4f9ed22..2b5996b22 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -251,11 +251,11 @@ describe('RequestQueue', () => {expect(info.handledRequestCount).to.be.eql(3);// Test access time- await Apify.utils.sleep(11);+ await Apify.utils.sleep(2);await queue.getRequest(op1.requestId);- await Apify.utils.sleep(11);+ await Apify.utils.sleep(2);const now = Date.now();- await Apify.utils.sleep(11);+ await Apify.utils.sleep(2);info = await queue.getInfo();const cTime2 = info.createdAt.getTime();mTime = info.modifiedAt.getTime();