Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add getInfo() test

Ondra Urban89ab84ddc592
diff --git a/src/dataset.js b/src/dataset.js
index 8e0e15695..d6a7e4810 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -530,8 +530,9 @@ export class DatasetLocal {
}
_updateMetadata(isModified) {
- this.accessedAt = Date.now();
- if (isModified) this.modifiedAt = Date.now();
+ const date = new Date();
+ this.accessedAt = date;
+ if (isModified) this.modifiedAt = date;
}
}
diff --git a/test/dataset.js b/test/dataset.js
index 9e082ffbd..60796c668 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -5,7 +5,7 @@ import fs from 'fs-extra';
import path from 'path';
import sinon from 'sinon';
import { leftpad, delayPromise } from 'apify-shared/utilities';
-import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES } from '../build/constants';
+import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES, LOCAL_USER_ID } from '../build/constants';
import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR,
LOCAL_GET_ITEMS_DEFAULT_LIMIT, checkAndSerialize, chunkBySize } from '../build/dataset';
import { apifyClient } from '../build/utils';
@@ -107,6 +107,33 @@ describe('dataset', () => {
});
});
+ it('getInfo() should work', async () => {
+ const dataset = await getLocalDataset([
+ { foo: 'a' },
+ { foo: 'b' },
+ { foo: 'c' },
+ { foo: 'd' },
+ ]);
+
+ const info = await dataset.getInfo();
+ expect(info).to.be.an('object');
+ expect(info.id).to.be.eql('my-dataset');
+ expect(info.name).to.be.eql('my-dataset');
+ expect(info.userId).to.be.eql(LOCAL_USER_ID);
+ expect(info.itemsCount).to.be.eql(4);
+
+ const cTime = info.createdAt.getTime();
+ const mTime = info.modifiedAt.getTime();
+ const aTime = info.accessedAt.getTime();
+
+ expect(cTime).to.be.below(Date.now());
+ expect(cTime).to.be.below(mTime);
+ expect(mTime).to.be.eql(aTime);
+
+ await dataset.getData();
+ // TODO
+ });
+
it('forEach() should work', async () => {
const dataset = await getLocalDataset([
{ foo: 'a' },