Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding delete methods to stores (#106)

Marek Trunkátf50636b0fc85
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11c88833f..ebf0f4720 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- Added `delete()` method to `RequestQueue`, `Dataset` and `KeyValueStore` classes.
+
0.5.34 / 2018-05-18
===================
- Added `loggingIntervalMillis` options to `AutoscaledPool`
diff --git a/src/dataset.js b/src/dataset.js
index 4c1262e4d..f3a65fc45 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -1,4 +1,5 @@
import fs from 'fs';
+import fsExtra from 'fs-extra';
import path from 'path';
import _ from 'underscore';
import Promise from 'bluebird';
@@ -14,6 +15,7 @@ const MAX_OPENED_STORES = 1000;
const writeFilePromised = Promise.promisify(fs.writeFile);
const readdirPromised = Promise.promisify(fs.readdir);
+const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { datasets } = apifyClient;
const datasetsCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open Datasets are stored here.
@@ -52,6 +54,21 @@ export class Dataset {
data,
});
}
+
+ /**
+ * Deletes the dataset.
+ *
+ * @return {Promise}
+ */
+ delete() {
+ return datasets
+ .deleteDataset({
+ datasetId: this.datasetId,
+ })
+ .then(() => {
+ datasetsCache.remove(this.datasetId);
+ });
+ }
}
/**
@@ -103,6 +120,13 @@ export class DatasetLocal {
return Promise.all(promises);
});
}
+
+ delete() {
+ return emptyDirPromised(this.localEmulationPath)
+ .then(() => {
+ datasetsCache.remove(this.datasetId);
+ });
+ }
}
/**
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 7b45b9d57..c83688717 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -1,4 +1,5 @@
import fs from 'fs';
+import fsExtra from 'fs-extra';
import path from 'path';
import Promise from 'bluebird';
import contentTypeParser from 'content-type';
@@ -21,6 +22,7 @@ const DEFAULT_LOCAL_FILE_TYPE = LOCAL_FILE_TYPES[0];
const readFilePromised = Promise.promisify(fs.readFile);
const writeFilePromised = Promise.promisify(fs.writeFile);
const unlinkPromised = Promise.promisify(fs.unlink);
+const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.
@@ -162,6 +164,21 @@ export class KeyValueStore {
contentType: addCharsetToContentType(optionsCopy.contentType),
});
}
+
+ /**
+ * Deletes the store.
+ *
+ * @return {Promise}
+ */
+ delete() {
+ return keyValueStores
+ .deleteStore({
+ storeId: this.storeId,
+ })
+ .then(() => {
+ storesCache.remove(this.storeId);
+ });
+ }
}
/**
@@ -242,6 +259,13 @@ export class KeyValueStoreLocal {
throw new Error(`Error writing file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
});
}
+
+ delete() {
+ return emptyDirPromised(this.localEmulationPath)
+ .then(() => {
+ storesCache.remove(this.storeId);
+ });
+ }
}
/**
diff --git a/src/request_queue.js b/src/request_queue.js
index de479a8ce..e2eaae5e8 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -31,6 +31,7 @@ const writeFilePromised = Promise.promisify(fs.writeFile);
const readdirPromised = Promise.promisify(fs.readdir);
const readFilePromised = Promise.promisify(fs.readFile);
const moveFilePromised = Promise.promisify(fsExtra.move);
+const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { requestQueues } = apifyClient;
const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.
@@ -456,6 +457,21 @@ export class RequestQueue {
}
});
}
+
+ /**
+ * Deletes the queue.
+ *
+ * @return {Promise}
+ */
+ delete() {
+ return requestQueues
+ .deleteQueue({
+ queueId: this.queueId,
+ })
+ .then(() => {
+ queuesCache.remove(this.queueId);
+ });
+ }
}
/**
@@ -701,6 +717,13 @@ export class RequestQueueLocal {
return this.initializationPromise
.then(() => this.pendingCount === 0);
}
+
+ delete() {
+ return emptyDirPromised(this.localEmulationPath)
+ .then(() => {
+ queuesCache.remove(this.queueId);
+ });
+ }
}
/**
diff --git a/test/_helper.js b/test/_helper.js
index 8e269480b..38bfd62fa 100644
--- a/test/_helper.js
+++ b/test/_helper.js
@@ -23,3 +23,13 @@ export const emptyLocalEmulationSubdir = (subdir) => {
};
export const expectNotLocalEmulation = () => expect(process.env[ENV_VARS.LOCAL_EMULATION_DIR]).to.be.a('undefined');
+
+export const expectDirEmpty = (dirPath) => {
+ const content = fs.readdirSync(dirPath);
+ expect(content).to.have.lengthOf(0);
+};
+
+export const expectDirNonEmpty = (dirPath) => {
+ const content = fs.readdirSync(dirPath);
+ expect(content).to.have.lengthOf.above(0);
+};
diff --git a/test/dataset.js b/test/dataset.js
index cb88da734..843b8557d 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -9,7 +9,7 @@ import { ENV_VARS } from '../build/constants';
import { LEFTPAD_COUNT, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR } from '../build/dataset';
import { apifyClient } from '../build/utils';
import * as Apify from '../build/index';
-import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation } from './_helper';
+import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
chai.use(chaiAsPromised);
@@ -47,6 +47,12 @@ describe('dataset', () => {
const newDataset = new DatasetLocal('my-dataset', LOCAL_EMULATION_DIR);
await newDataset.pushData({ foo2: 'bar2' });
expect(read('my-dataset', 5)).to.be.eql({ foo2: 'bar2' });
+
+ // Delete works.
+ const datasetDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-dataset');
+ expectDirNonEmpty(datasetDir);
+ await newDataset.delete();
+ expectDirEmpty(datasetDir);
});
});
@@ -72,6 +78,12 @@ describe('dataset', () => {
{ foo: 'restaurant' },
]);
+ mock.expects('deleteDataset')
+ .once()
+ .withArgs({ datasetId: 'some-id' })
+ .returns(Promise.resolve());
+ await dataset.delete();
+
mock.verify();
mock.restore();
});
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 0769e3fff..ee93cfc52 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -2,11 +2,12 @@ import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import 'babel-polyfill';
import sinon from 'sinon';
+import path from 'path';
import { ENV_VARS } from '../build/constants';
import { KeyValueStoreLocal, KeyValueStore, maybeStringify, LOCAL_EMULATION_SUBDIR } from '../build/key_value_store';
import { apifyClient } from '../build/utils';
import * as Apify from '../build/index';
-import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation } from './_helper';
+import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
chai.use(chaiAsPromised);
@@ -65,6 +66,12 @@ describe('KeyValueStore', () => {
expect(await store.getValue('key-buffer')).to.be.eql(buffer);
expect(await store.getValue('key-nonexist')).to.be.eql(null);
expect(await store2.getValue('key-obj')).to.be.eql({ foo: 'hotel' });
+
+ // Delete works.
+ const storeDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-store-id');
+ expectDirNonEmpty(storeDir);
+ await store.delete();
+ expectDirEmpty(storeDir);
});
});
@@ -108,6 +115,15 @@ describe('KeyValueStore', () => {
.returns(Promise.resolve(null));
await store.setValue('key-1', null);
+ // Delete.
+ mock.expects('deleteStore')
+ .once()
+ .withArgs({
+ storeId: 'some-id-1',
+ })
+ .returns(Promise.resolve());
+ await store.delete();
+
mock.verify();
mock.restore();
});
diff --git a/test/request_queue.js b/test/request_queue.js
index c449335b6..6d284e8d3 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -3,12 +3,13 @@ import chaiAsPromised from 'chai-as-promised';
import _ from 'underscore';
import 'babel-polyfill';
import sinon from 'sinon';
+import path from 'path';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
import { apifyClient } from '../build/utils';
import { RequestQueueLocal, RequestQueue, LOCAL_EMULATION_SUBDIR, QUERY_HEAD_MIN_LENGTH } from '../build/request_queue';
-import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR, expectNotLocalEmulation } from './_helper';
+import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
chai.use(chaiAsPromised);
@@ -64,6 +65,12 @@ describe('RequestQueue', () => {
expect(await queue.isEmpty()).to.be.eql(true);
expect(await queue.isFinished()).to.be.eql(true);
+
+ // Delete it.
+ const queueDir = path.join(LOCAL_EMULATION_DIR, LOCAL_EMULATION_SUBDIR, 'my-queue-0');
+ expectDirNonEmpty(queueDir);
+ await queue.delete();
+ expectDirEmpty(queueDir);
});
it('supports forefront param in reclaimRequest()', async () => {
@@ -237,6 +244,15 @@ describe('RequestQueue', () => {
expect(queue.queueHeadDict.length()).to.be.eql(1);
expect(queue.inProgressCount).to.be.eql(1);
+ // Delete queue.
+ mock.expects('deleteQueue')
+ .once()
+ .withArgs({
+ queueId: 'some-id',
+ })
+ .returns(Promise.resolve());
+ await queue.delete();
+
mock.verify();
mock.restore();
});