Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add tests for deprecated storage .delete() methods
Ondra Urbanb83e16b1e168
diff --git a/package.json b/package.jsonindex 12481ec47..3dae78b58 100644--- a/package.json+++ b/package.json@@ -77,8 +77,11 @@"@babel/core": "^7.0.0","@babel/preset-env": "^7.0.0","@babel/register": "^7.0.0",+ "@types/chai": "^4.1.7","@types/fs-extra": "^7.0.0",- "@types/puppeteer": "^1.12.4",+ "@types/mocha": "^5.2.7",+ "@types/puppeteer": "^1.19.0",+ "@types/sinon": "^7.0.13","@types/socket.io": "^2.1.2","basic-auth-parser": "^0.0.2","chai": "^4.2.0",diff --git a/test/dataset.js b/test/dataset.jsindex fb5e28d54..e117722e2 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -47,7 +47,7 @@ describe('dataset', () => {await newDataset.pushData({ foo2: 'bar2' });expect(read('my-dataset', 5)).to.be.eql({ foo2: 'bar2' });- // Delete works.+ // Drop works.const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');expectDirNonEmpty(datasetDir);await newDataset.drop();@@ -315,6 +315,16 @@ describe('dataset', () => {expect(result.foo).to.be.eql(5);expect(calledForIndexes).to.be.eql([1, 2, 3]);});++ it('deprecated delete() still works', async () => {+ const dataset = new DatasetLocal('to-delete', LOCAL_STORAGE_DIR);+ await dataset.pushData({ foo: 'bar' });++ const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');+ expectDirNonEmpty(datasetDir);+ await dataset.delete();+ expectDirEmpty(datasetDir);+ });});describe('remote', async () => {@@ -722,6 +732,19 @@ describe('dataset', () => {expect(result.foo).to.be.eql(5);expect(calledForIndexes).to.be.eql([1, 2, 3]);});++ it('deprecated delete() still works', async () => {+ const mock = sinon.mock(apifyClient.datasets);+ const dataset = new Dataset('some-id', 'some-name');+ mock.expects('deleteDataset')+ .once()+ .withArgs({ datasetId: 'some-id' })+ .resolves();++ await dataset.drop();++ mock.verify();+ });});describe('Apify.openDataset', async () => {diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 63b9e113f..85d2dfc51 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -93,12 +93,22 @@ describe('KeyValueStore', () => {expect(await store2.getValue('key-ctype')).to.be.eql(buffer);expect(await store2.getValue('key-badctype')).to.be.eql(buffer);- // Delete works.+ // Drop works.const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');expectDirNonEmpty(storeDir);await store.drop();expectDirEmpty(storeDir);});++ it('deprecated delete() still works', async () => {+ const kvs = new KeyValueStoreLocal('to-delete', LOCAL_STORAGE_DIR);+ await kvs.setValue('dummy', { foo: 'bar' });++ const kvsDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');+ expectDirNonEmpty(kvsDir);+ await kvs.delete();+ expectDirEmpty(kvsDir);+ });});describe('remote', async () => {@@ -141,7 +151,7 @@ describe('KeyValueStore', () => {.returns(Promise.resolve(null));await store.setValue('key-1', null);- // Delete.+ // Drop.mock.expects('deleteStore').once().withArgs({@@ -153,6 +163,19 @@ describe('KeyValueStore', () => {mock.verify();mock.restore();});++ it('deprecated delete() still works', async () => {+ const mock = sinon.mock(apifyClient.keyValueStores);+ const kvs = new KeyValueStore('some-id', 'some-name');+ mock.expects('deleteStore')+ .once()+ .withArgs({ storeId: 'some-id' })+ .resolves();++ await kvs.drop();++ mock.verify();+ });});describe('Apify.openKeyValueStore', async () => {@@ -464,7 +487,7 @@ describe('KeyValueStore', () => {expect(r[0]).to.be.eql(`key${i + 4}`);});- // Delete works.+ // Drop works.const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, storeId);expectDirNonEmpty(storeDir);await store.drop();diff --git a/test/request_queue.js b/test/request_queue.jsindex 538bba7e2..fd64a0319 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -108,7 +108,7 @@ describe('RequestQueue', () => {expect(await queue.isEmpty()).to.be.eql(true);expect(await queue.isFinished()).to.be.eql(true);- // Delete it.+ // Drop it.const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');expectDirNonEmpty(queueDir);await queue.drop();@@ -280,6 +280,16 @@ describe('RequestQueue', () => {expect(info.pendingRequestCount).to.be.eql(0);expect(info.handledRequestCount).to.be.eql(3);});++ it('deprecated delete() still works', async () => {+ const rq = new RequestQueueLocal('to-delete', LOCAL_STORAGE_DIR);+ await rq.addRequest({ url: 'https://example.com' });++ const rqDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');+ expectDirNonEmpty(rqDir);+ await rq.delete();+ expectDirEmpty(rqDir);+ });});describe('remote', async () => {@@ -450,7 +460,7 @@ describe('RequestQueue', () => {expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(1);- // Delete queue.+ // Drop queue.mock.expects('deleteQueue').once().withArgs({@@ -923,6 +933,19 @@ describe('RequestQueue', () => {mock.verify();mock.restore();});++ it('deprecated delete() still works', async () => {+ const mock = sinon.mock(apifyClient.requestQueues);+ const rq = new RequestQueue('some-id', 'some-name');+ mock.expects('deleteQueue')+ .once()+ .withArgs({ queueId: 'some-id' })+ .resolves();++ await rq.drop();++ mock.verify();+ });});describe('Apify.openRequestQueue', async () => {