Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add getInfo() to dataset
Ondra Urban3d2c3a5b08db
diff --git a/src/constants.js b/src/constants.jsindex e64e015d2..79c8bcda8 100644--- a/src/constants.js+++ b/src/constants.js@@ -103,4 +103,6 @@ consts.USER_AGENT_LIST = ['Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0',];+consts.LOCAL_USER_ID = 'local_user_id';+module.exports = consts;diff --git a/src/dataset.js b/src/dataset.jsindex 5f80d0599..8e0e15695 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -6,7 +6,7 @@ import Promise from 'bluebird';import { leftpad } from 'apify-shared/utilities';import LruCache from 'apify-shared/lru_cache';import { checkParamOrThrow } from 'apify-client/build/utils';-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';+import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES, LOCAL_USER_ID } from './constants';import { apifyClient, ensureDirExists } from './utils';export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.datasets;@@ -18,6 +18,7 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%const writeFilePromised = Promise.promisify(fs.writeFile);const readFilePromised = Promise.promisify(fs.readFile);const readdirPromised = Promise.promisify(fs.readdir);+const statPromised = Promise.promisify(fs.stat);const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);const getLocaleFilename = index => `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;@@ -202,6 +203,30 @@ export class Dataset {return datasets.getItems(params);}+ /**+ * Returns a dataset object containing general information about the dataset.+ *+ * @example+ * {+ * "id": "WkzbQMuFYuamGv3YF",+ * "name": "d7b9MDYsbtX5L7XAj",+ * "userId": "wRsJZtadYvn4mBZmm",+ * "createdAt": "2015-12-12T07:34:14.202Z",+ * "modifiedAt": "2015-12-13T08:36:13.202Z",+ * "accessedAt": "2015-12-14T08:36:13.202Z",+ * "itemsCount": 0+ * }+ *+ * @param opts+ * @returns {Promise}+ */+ getInfo(opts = {}) {+ const { datasetId } = this;+ const params = Object.assign({ datasetId }, opts);++ return datasets.getDataset(params);+ }+/*** Iterates over the all dataset items, yielding each in turn to an iteratee function.* Each invocation of iteratee is called with three arguments: (element, index).@@ -351,11 +376,16 @@ export class DatasetLocal {.then((files) => {if (files.length) {const lastFileNum = files.pop().split('.')[0];-this.counter = parseInt(lastFileNum, 10);} else {this.counter = 0;}+ return statPromised(this.localEmulationPath);+ })+ .then((stats) => {+ this.createdAt = stats.birthtime;+ this.modifiedAt = stats.mtime;+ this.accessedAt = stats.atime;});}@@ -375,7 +405,7 @@ export class DatasetLocal {return writeFilePromised(filePath, itemStr);});-+ this._updateMetadata(true);return Promise.all(promises);});}@@ -395,6 +425,7 @@ export class DatasetLocal {return Promise.mapSeries(indexes, index => this._readAndParseFile(index));}).then((items) => {+ this._updateMetadata();return {items,total: this.counter,@@ -405,6 +436,23 @@ export class DatasetLocal {});}+ getInfo() {+ return this.initializationPromise+ .then(() => {+ const id = this.datasetId;+ const name = id === ENV_VARS.DEFAULT_DATASET_ID ? null : id;+ return {+ id,+ name,+ userId: LOCAL_USER_ID,+ createdAt: this.createdAt,+ modifiedAt: this.modifiedAt,+ accessedAt: this.accessedAt,+ itemsCount: this.counter,+ };+ });+ }+forEach(iteratee) {return this.initializationPromise.then(() => {@@ -451,6 +499,7 @@ export class DatasetLocal {return this.initializationPromise.then(() => emptyDirPromised(this.localEmulationPath)).then(() => {+ this._updateMetadata(true);datasetsCache.remove(this.datasetId);});}@@ -474,7 +523,15 @@ export class DatasetLocal {const filePath = path.join(this.localEmulationPath, getLocaleFilename(index));return readFilePromised(filePath)- .then(json => JSON.parse(json));+ .then((json) => {+ this._updateMetadata();+ return JSON.parse(json);+ });+ }++ _updateMetadata(isModified) {+ this.accessedAt = Date.now();+ if (isModified) this.modifiedAt = Date.now();}}