Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add tests for deprecated storage .delete() methods

Ondra Urbanb83e16b1e168
diff --git a/package.json b/package.json
index 12481ec47..3dae78b58 100644
--- a/package.json
+++ b/package.json
@@ -77,8 +77,11 @@
"@babel/core": "^7.0.0",
"@babel/preset-env": "^7.0.0",
"@babel/register": "^7.0.0",
+ "@types/chai": "^4.1.7",
"@types/fs-extra": "^7.0.0",
- "@types/puppeteer": "^1.12.4",
+ "@types/mocha": "^5.2.7",
+ "@types/puppeteer": "^1.19.0",
+ "@types/sinon": "^7.0.13",
"@types/socket.io": "^2.1.2",
"basic-auth-parser": "^0.0.2",
"chai": "^4.2.0",
diff --git a/test/dataset.js b/test/dataset.js
index fb5e28d54..e117722e2 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -47,7 +47,7 @@ describe('dataset', () => {
await newDataset.pushData({ foo2: 'bar2' });
expect(read('my-dataset', 5)).to.be.eql({ foo2: 'bar2' });
- // Delete works.
+ // Drop works.
const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');
expectDirNonEmpty(datasetDir);
await newDataset.drop();
@@ -315,6 +315,16 @@ describe('dataset', () => {
expect(result.foo).to.be.eql(5);
expect(calledForIndexes).to.be.eql([1, 2, 3]);
});
+
+ it('deprecated delete() still works', async () => {
+ const dataset = new DatasetLocal('to-delete', LOCAL_STORAGE_DIR);
+ await dataset.pushData({ foo: 'bar' });
+
+ const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ expectDirNonEmpty(datasetDir);
+ await dataset.delete();
+ expectDirEmpty(datasetDir);
+ });
});
describe('remote', async () => {
@@ -722,6 +732,19 @@ describe('dataset', () => {
expect(result.foo).to.be.eql(5);
expect(calledForIndexes).to.be.eql([1, 2, 3]);
});
+
+ it('deprecated delete() still works', async () => {
+ const mock = sinon.mock(apifyClient.datasets);
+ const dataset = new Dataset('some-id', 'some-name');
+ mock.expects('deleteDataset')
+ .once()
+ .withArgs({ datasetId: 'some-id' })
+ .resolves();
+
+ await dataset.drop();
+
+ mock.verify();
+ });
});
describe('Apify.openDataset', async () => {
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 63b9e113f..85d2dfc51 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -93,12 +93,22 @@ describe('KeyValueStore', () => {
expect(await store2.getValue('key-ctype')).to.be.eql(buffer);
expect(await store2.getValue('key-badctype')).to.be.eql(buffer);
- // Delete works.
+ // Drop works.
const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');
expectDirNonEmpty(storeDir);
await store.drop();
expectDirEmpty(storeDir);
});
+
+ it('deprecated delete() still works', async () => {
+ const kvs = new KeyValueStoreLocal('to-delete', LOCAL_STORAGE_DIR);
+ await kvs.setValue('dummy', { foo: 'bar' });
+
+ const kvsDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ expectDirNonEmpty(kvsDir);
+ await kvs.delete();
+ expectDirEmpty(kvsDir);
+ });
});
describe('remote', async () => {
@@ -141,7 +151,7 @@ describe('KeyValueStore', () => {
.returns(Promise.resolve(null));
await store.setValue('key-1', null);
- // Delete.
+ // Drop.
mock.expects('deleteStore')
.once()
.withArgs({
@@ -153,6 +163,19 @@ describe('KeyValueStore', () => {
mock.verify();
mock.restore();
});
+
+ it('deprecated delete() still works', async () => {
+ const mock = sinon.mock(apifyClient.keyValueStores);
+ const kvs = new KeyValueStore('some-id', 'some-name');
+ mock.expects('deleteStore')
+ .once()
+ .withArgs({ storeId: 'some-id' })
+ .resolves();
+
+ await kvs.drop();
+
+ mock.verify();
+ });
});
describe('Apify.openKeyValueStore', async () => {
@@ -464,7 +487,7 @@ describe('KeyValueStore', () => {
expect(r[0]).to.be.eql(`key${i + 4}`);
});
- // Delete works.
+ // Drop works.
const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, storeId);
expectDirNonEmpty(storeDir);
await store.drop();
diff --git a/test/request_queue.js b/test/request_queue.js
index 538bba7e2..fd64a0319 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -108,7 +108,7 @@ describe('RequestQueue', () => {
expect(await queue.isEmpty()).to.be.eql(true);
expect(await queue.isFinished()).to.be.eql(true);
- // Delete it.
+ // Drop it.
const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');
expectDirNonEmpty(queueDir);
await queue.drop();
@@ -280,6 +280,16 @@ describe('RequestQueue', () => {
expect(info.pendingRequestCount).to.be.eql(0);
expect(info.handledRequestCount).to.be.eql(3);
});
+
+ it('deprecated delete() still works', async () => {
+ const rq = new RequestQueueLocal('to-delete', LOCAL_STORAGE_DIR);
+ await rq.addRequest({ url: 'https://example.com' });
+
+ const rqDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ expectDirNonEmpty(rqDir);
+ await rq.delete();
+ expectDirEmpty(rqDir);
+ });
});
describe('remote', async () => {
@@ -450,7 +460,7 @@ describe('RequestQueue', () => {
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount()).to.be.eql(1);
- // Delete queue.
+ // Drop queue.
mock.expects('deleteQueue')
.once()
.withArgs({
@@ -923,6 +933,19 @@ describe('RequestQueue', () => {
mock.verify();
mock.restore();
});
+
+ it('deprecated delete() still works', async () => {
+ const mock = sinon.mock(apifyClient.requestQueues);
+ const rq = new RequestQueue('some-id', 'some-name');
+ mock.expects('deleteQueue')
+ .once()
+ .withArgs({ queueId: 'some-id' })
+ .resolves();
+
+ await rq.drop();
+
+ mock.verify();
+ });
});
describe('Apify.openRequestQueue', async () => {