Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add getInfo() to dataset

Ondra Urban3d2c3a5b08db
diff --git a/src/constants.js b/src/constants.js
index e64e015d2..79c8bcda8 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -103,4 +103,6 @@ consts.USER_AGENT_LIST = [
'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0',
];
+consts.LOCAL_USER_ID = 'local_user_id';
+
module.exports = consts;
diff --git a/src/dataset.js b/src/dataset.js
index 5f80d0599..8e0e15695 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -6,7 +6,7 @@ import Promise from 'bluebird';
import { leftpad } from 'apify-shared/utilities';
import LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';
+import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES, LOCAL_USER_ID } from './constants';
import { apifyClient, ensureDirExists } from './utils';
export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.datasets;
@@ -18,6 +18,7 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
const writeFilePromised = Promise.promisify(fs.writeFile);
const readFilePromised = Promise.promisify(fs.readFile);
const readdirPromised = Promise.promisify(fs.readdir);
+const statPromised = Promise.promisify(fs.stat);
const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const getLocaleFilename = index => `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
@@ -202,6 +203,30 @@ export class Dataset {
return datasets.getItems(params);
}
+ /**
+ * Returns a dataset object containing general information about the dataset.
+ *
+ * @example
+ * {
+ * "id": "WkzbQMuFYuamGv3YF",
+ * "name": "d7b9MDYsbtX5L7XAj",
+ * "userId": "wRsJZtadYvn4mBZmm",
+ * "createdAt": "2015-12-12T07:34:14.202Z",
+ * "modifiedAt": "2015-12-13T08:36:13.202Z",
+ * "accessedAt": "2015-12-14T08:36:13.202Z",
+ * "itemsCount": 0
+ * }
+ *
+ * @param opts
+ * @returns {Promise}
+ */
+ getInfo(opts = {}) {
+ const { datasetId } = this;
+ const params = Object.assign({ datasetId }, opts);
+
+ return datasets.getDataset(params);
+ }
+
/**
* Iterates over the all dataset items, yielding each in turn to an iteratee function.
* Each invocation of iteratee is called with three arguments: (element, index).
@@ -351,11 +376,16 @@ export class DatasetLocal {
.then((files) => {
if (files.length) {
const lastFileNum = files.pop().split('.')[0];
-
this.counter = parseInt(lastFileNum, 10);
} else {
this.counter = 0;
}
+ return statPromised(this.localEmulationPath);
+ })
+ .then((stats) => {
+ this.createdAt = stats.birthtime;
+ this.modifiedAt = stats.mtime;
+ this.accessedAt = stats.atime;
});
}
@@ -375,7 +405,7 @@ export class DatasetLocal {
return writeFilePromised(filePath, itemStr);
});
-
+ this._updateMetadata(true);
return Promise.all(promises);
});
}
@@ -395,6 +425,7 @@ export class DatasetLocal {
return Promise.mapSeries(indexes, index => this._readAndParseFile(index));
})
.then((items) => {
+ this._updateMetadata();
return {
items,
total: this.counter,
@@ -405,6 +436,23 @@ export class DatasetLocal {
});
}
+ getInfo() {
+ return this.initializationPromise
+ .then(() => {
+ const id = this.datasetId;
+ const name = id === ENV_VARS.DEFAULT_DATASET_ID ? null : id;
+ return {
+ id,
+ name,
+ userId: LOCAL_USER_ID,
+ createdAt: this.createdAt,
+ modifiedAt: this.modifiedAt,
+ accessedAt: this.accessedAt,
+ itemsCount: this.counter,
+ };
+ });
+ }
+
forEach(iteratee) {
return this.initializationPromise
.then(() => {
@@ -451,6 +499,7 @@ export class DatasetLocal {
return this.initializationPromise
.then(() => emptyDirPromised(this.localEmulationPath))
.then(() => {
+ this._updateMetadata(true);
datasetsCache.remove(this.datasetId);
});
}
@@ -474,7 +523,15 @@ export class DatasetLocal {
const filePath = path.join(this.localEmulationPath, getLocaleFilename(index));
return readFilePromised(filePath)
- .then(json => JSON.parse(json));
+ .then((json) => {
+ this._updateMetadata();
+ return JSON.parse(json);
+ });
+ }
+
+ _updateMetadata(isModified) {
+ this.accessedAt = Date.now();
+ if (isModified) this.modifiedAt = Date.now();
}
}