Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Local storage dir emulator introduced to tests, removed settings_rotator.js and settings_rotator.test.js. Refactored code.

Petr Patekabc5eff84a53
diff --git a/src/index.js b/src/index.js
index 122ebbfee..172a32565 100644
--- a/src/index.js
+++ b/src/index.js
@@ -14,7 +14,6 @@ import PuppeteerPool from './puppeteer_pool';
import Request from './request';
import { RequestList, openRequestList } from './request_list';
import { openRequestQueue } from './request_queue';
-import SettingsRotator from './settings_rotator';
import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo, printOutdatedSdkWarning } from './utils';
import { puppeteerUtils } from './puppeteer_utils';
import { socialUtils } from './utils_social';
@@ -92,7 +91,6 @@ module.exports = {
openRequestList,
openRequestQueue,
- SettingsRotator,
openSessionPool,
LiveViewServer,
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index b549da60b..6e3634bf2 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -25,7 +25,6 @@ const DEFAULT_OPTIONS = {
instanceKillerIntervalSecs: 60,
killInstanceAfterSecs: 300,
- // TODO: use settingsRotator()
launchPuppeteerFunction: launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),
recycleDiskCache: false,
diff --git a/src/settings_rotator.js b/src/settings_rotator.js
deleted file mode 100644
index 8d8debc28..000000000
--- a/src/settings_rotator.js
+++ /dev/null
@@ -1,59 +0,0 @@
-import { checkParamOrThrow } from 'apify-client/build/utils';
-
-// TODO: randomWaitBetweenUsagesMillis
-
-/**
- * Rotates settings created by a user-provided function passed
- * via `newSettingsFunction`.
- * This is useful during web crawling to dynamically change settings and thus
- * avoid detection of the crawler.
- *
- * This class is still work in progress, more features will be added soon.
- *
- * @param {Object} options
- * @param {Function} options.newSettingsFunction
- * @param {Number} options.maxUsages
- * @ignore
- */
-export default class SettingsRotator {
- constructor(opts) {
- checkParamOrThrow(opts, 'options', 'Object');
-
- const {
- newSettingsFunction,
- maxUsages,
- } = opts;
-
- checkParamOrThrow(newSettingsFunction, 'options.newSettingsFunction', 'Function');
- checkParamOrThrow(maxUsages, 'options.maxUsages', 'Number');
-
- this.maxUsages = maxUsages;
- this.newSettingsFunction = newSettingsFunction;
-
- this.currentSettings = null;
- this.currentSettingsUsageCount = 0;
- }
-
- /**
- * Fetches a settings object.
- *
- * @return {*}
- */
- fetchSettings() {
- if (!this.currentSettings || this.currentSettingsUsageCount >= this.maxUsages) {
- this.currentSettings = this.newSettingsFunction();
- this.currentSettingsUsageCount = 0;
- }
-
- this.currentSettingsUsageCount++;
-
- return this.currentSettings;
- }
-
- /**
- * Reclaims settings after use.
- *
- * @param {*} settings
- */
- reclaimSettings(settings) {} // eslint-disable-line
-}
diff --git a/test/_helper.js b/test/_helper.js
index 3e60d8a05..3f2ac3b39 100644
--- a/test/_helper.js
+++ b/test/_helper.js
@@ -5,16 +5,14 @@ import { ENV_VARS } from 'apify-shared/consts';
export const LOCAL_STORAGE_DIR = path.join(__dirname, '..', 'tmp', 'local-emulation-dir');
// Log unhandled rejections.
-process.on('unhandledRejection', (err) => {
- console.log('----------------------------------------------------------------');
- console.log('- ERROR: Exiting tests because of unhandled promise rejection! -');
- console.log('----------------------------------------------------------------');
- console.log(err);
- process.exit(1);
-});
-
-// Immediately ensure that local emulation dir exists.
-fs.ensureDirSync(path.resolve(LOCAL_STORAGE_DIR));
+// process.on('unhandledRejection', (err) => {
+// console.log('----------------------------------------------------------------');
+// console.log('- ERROR: Exiting tests because of unhandled promise rejection! -');
+// console.log('----------------------------------------------------------------');
+// console.log(err);
+// process.exit(1);
+// });
+
export const emptyLocalStorageSubdir = (subdir) => {
const fullPath = path.resolve(path.join(LOCAL_STORAGE_DIR, subdir));
diff --git a/test/crawlers/basic_crawler.test.js b/test/crawlers/basic_crawler.test.js
index 165dd3860..b3a244afe 100644
--- a/test/crawlers/basic_crawler.test.js
+++ b/test/crawlers/basic_crawler.test.js
@@ -6,22 +6,29 @@ import * as Apify from '../../build';
import * as keyValueStore from '../../build/key_value_store';
import { RequestQueue, RequestQueueLocal } from '../../build/request_queue';
import { sleep } from '../../build/utils';
-import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
import events from '../../build/events';
+import LocalStorageDirEmulator from '../local_storage_dir_emulator';
describe('BasicCrawler', () => {
let logLevel;
+ let localStorageEmulator;
+ let LOCAL_STORAGE_DIR;
- beforeAll(() => {
+ beforeAll(async () => {
logLevel = log.getLevel();
log.setLevel(log.LEVELS.OFF);
- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
- emptyLocalStorageSubdir('key_value_stores/default');
+ localStorageEmulator = new LocalStorageDirEmulator();
+ await localStorageEmulator.init();
+ LOCAL_STORAGE_DIR = localStorageEmulator.localStorageDir;
});
- afterAll(() => {
+ beforeEach(async () => {
+ await localStorageEmulator.clean();
+ });
+
+ afterAll(async () => {
+ await localStorageEmulator.destroy();
log.setLevel(logLevel);
- emptyLocalStorageSubdir('key_value_stores/default');
});
test('should run in parallel thru all the requests', async () => {
@@ -659,7 +666,7 @@ describe('BasicCrawler', () => {
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
- it('should teardown Session pool after it is finished', async () => {
+ it('should destroy Session pool after it is finished', async () => {
const url = 'https://example.com';
const requestList = new Apify.RequestList({ sources: [{ url }] });
await requestList.initialize();
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 42ab1f9d3..3a518a984 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -11,7 +11,7 @@ import Apify from '../../build';
import { sleep } from '../../build/utils';
import { Session } from '../../build/session_pool/session';
import { STATUS_CODES_BLOCKED } from '../../build/constants';
-import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR } from '../_helper';
+import LocalStorageDirEmulator from '../local_storage_dir_emulator';
// Add common props to mocked request responses.
const responseMock = {
@@ -148,12 +148,20 @@ describe('CheerioCrawler', () => {
port = server.address().port; //eslint-disable-line
});
- afterAll(() => {
- log.setLevel(logLevel);
+ let localStorageEmulator;
+
+ beforeAll(async () => {
+ localStorageEmulator = new LocalStorageDirEmulator();
+ await localStorageEmulator.init();
});
beforeEach(async () => {
- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ await localStorageEmulator.clean();
+ });
+
+ afterAll(async () => {
+ log.setLevel(logLevel);
+ await localStorageEmulator.destroy();
});
test('should work', async () => {
@@ -795,14 +803,10 @@ describe('CheerioCrawler', () => {
let requestList;
beforeEach(async () => {
- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ await localStorageEmulator.clean();
requestList = await Apify.openRequestList('test', sources);
});
- afterEach(() => {
- emptyLocalStorageSubdir('key_value_stores/default');
- });
-
test('should work', async () => {
const crawler = new Apify.CheerioCrawler({
requestList,
diff --git a/test/dataset.test.js b/test/dataset.test.js
index 61fa57d62..e2fe1492f 100644
--- a/test/dataset.test.js
+++ b/test/dataset.test.js
@@ -7,27 +7,42 @@ import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_STORAGE_SUBDIR,
LOCAL_GET_ITEMS_DEFAULT_LIMIT, checkAndSerialize, chunkBySize } from '../build/dataset';
import * as utils from '../build/utils';
import * as Apify from '../build/index';
-import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import { expectDirEmpty, expectDirNonEmpty } from './_helper';
+import LocalStorageDirEmulator from './local_storage_dir_emulator';
const { apifyClient } = utils;
+describe('dataset', () => {
+ let localStorageEmulator;
+ let localStorageDir;
+
+ beforeAll(async () => {
+ apifyClient.setOptions({ token: 'xxx' });
+ localStorageEmulator = new LocalStorageDirEmulator();
+ await localStorageEmulator.init();
+ localStorageDir = localStorageEmulator.localStorageDir; // eslint-disable-line
+ });
-const read = (datasetName, index) => {
- const fileName = `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
- const filePath = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, datasetName, fileName);
- const str = fs.readFileSync(path.resolve(filePath));
+ afterAll(async () => {
+ apifyClient.setOptions({ token: undefined });
+ await localStorageEmulator.destroy();
+ });
- return JSON.parse(str);
-};
+ beforeEach(async () => {
+ await localStorageEmulator.clean();
+ });
+
+ const read = (datasetName, index) => {
+ const fileName = `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
+ const filePath = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, datasetName, fileName);
+ const str = fs.readFileSync(path.resolve(filePath));
+
+ return JSON.parse(str);
+ };
-describe('dataset', () => {
- beforeAll(() => apifyClient.setOptions({ token: 'xxx' }));
- afterAll(() => apifyClient.setOptions({ token: undefined }));
- beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
- afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
describe('local', () => {
test('should successfully save data', async () => {
- const dataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
+ const dataset = new DatasetLocal('my-dataset', localStorageDir);
await dataset.pushData({ foo: 'bar' });
await dataset.pushData({ foo: 'hotel' });
@@ -42,19 +57,19 @@ describe('dataset', () => {
expect(read('my-dataset', 4)).toEqual({ foo: 'from-array-1', arr: [1, 2, 3] });
// Correctly initializes the state.
- const newDataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
+ const newDataset = new DatasetLocal('my-dataset', localStorageDir);
await newDataset.pushData({ foo2: 'bar2' });
expect(read('my-dataset', 5)).toEqual({ foo2: 'bar2' });
// Drop works.
- const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');
+ const datasetDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'my-dataset');
expectDirNonEmpty(datasetDir);
await newDataset.drop();
expectDirEmpty(datasetDir);
});
const getLocalDataset = async (data) => {
- const dataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);
+ const dataset = new DatasetLocal('my-dataset', localStorageDir);
await dataset.pushData(data);
return dataset;
@@ -114,7 +129,7 @@ describe('dataset', () => {
test('getInfo() should work', async () => {
const datasetName = 'stats-dataset';
- const dataset = new DatasetLocal(datasetName, LOCAL_STORAGE_DIR);
+ const dataset = new DatasetLocal(datasetName, localStorageDir);
await Apify.utils.sleep(2);
// Save orig env var since it persists over tests.
@@ -316,10 +331,10 @@ describe('dataset', () => {
});
test('deprecated delete() still works', async () => {
- const dataset = new DatasetLocal('to-delete', LOCAL_STORAGE_DIR);
+ const dataset = new DatasetLocal('to-delete', localStorageDir);
await dataset.pushData({ foo: 'bar' });
- const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ const datasetDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'to-delete');
expectDirNonEmpty(datasetDir);
await dataset.delete();
expectDirEmpty(datasetDir);
@@ -760,7 +775,7 @@ describe('dataset', () => {
test('should work', () => {
const mock = sinon.mock(utils);
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
mock.expects('openLocalStorage').once();
Apify.openDataset();
@@ -786,7 +801,7 @@ describe('dataset', () => {
describe('pushData', () => {
test('throws on invalid args', async () => {
process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-8';
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
const dataErrMsg = 'Parameter "data" of type Array | Object must be provided';
await expect(Apify.pushData()).rejects.toThrow(dataErrMsg);
@@ -822,7 +837,7 @@ describe('dataset', () => {
);
test('correctly stores records', async () => {
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-9';
await Apify.pushData({ foo: 'bar' });
diff --git a/test/key_value_store.test.js b/test/key_value_store.test.js
index 342e717f4..d5ae1cf4a 100644
--- a/test/key_value_store.test.js
+++ b/test/key_value_store.test.js
@@ -5,23 +5,26 @@ import { KeyValueStoreLocal, KeyValueStore, maybeStringify, getFileNameRegexp, L
import * as utils from '../build/utils';
import * as Apify from '../build/index';
import { expectDirEmpty, expectDirNonEmpty } from './_helper';
-import LocalStorageEmulator from './local-storage-emulator';
+import LocalStorageDirEmulator from './local_storage_dir_emulator';
const { apifyClient } = utils;
describe('KeyValueStore', () => {
let localStorageEmulator;
- let LOCAL_STORAGE_DIR;
+ let localStorageDir;
+
beforeAll(async () => {
apifyClient.setOptions({ token: 'xxx' });
- localStorageEmulator = new LocalStorageEmulator();
+ localStorageEmulator = new LocalStorageDirEmulator();
await localStorageEmulator.init();
- LOCAL_STORAGE_DIR = localStorageEmulator.localStorageDir;
+ localStorageDir = localStorageEmulator.localStorageDir; // eslint-disable-line
});
+
afterAll(async () => {
apifyClient.setOptions({ token: undefined });
- await localStorageEmulator.teardown();
+ await localStorageEmulator.destroy();
});
+
beforeEach(async () => {
await localStorageEmulator.clean();
});
@@ -64,8 +67,8 @@ describe('KeyValueStore', () => {
describe('local', () => {
test('should work', async () => {
- const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
- const store2 = new KeyValueStoreLocal('another-store-id', LOCAL_STORAGE_DIR);
+ const store = new KeyValueStoreLocal('my-store-id', localStorageDir);
+ const store2 = new KeyValueStoreLocal('another-store-id', localStorageDir);
const buffer = Buffer.from('some text value');
await store.setValue('key-obj', { foo: 'bar' });
@@ -106,17 +109,17 @@ describe('KeyValueStore', () => {
expect(await store2.getValue('key-badctype')).toEqual(buffer);
// Drop works.
- const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');
+ const storeDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'my-store-id');
expectDirNonEmpty(storeDir);
await store.drop();
expectDirEmpty(storeDir);
});
test('deprecated delete() still works', async () => {
- const kvs = new KeyValueStoreLocal('to-delete', LOCAL_STORAGE_DIR);
+ const kvs = new KeyValueStoreLocal('to-delete', localStorageDir);
await kvs.setValue('dummy', { foo: 'bar' });
- const kvsDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ const kvsDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'to-delete');
expectDirNonEmpty(kvsDir);
await kvs.delete();
expectDirEmpty(kvsDir);
@@ -194,7 +197,7 @@ describe('KeyValueStore', () => {
test('should work', async () => {
const mock = sinon.mock(utils);
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
mock.expects('openLocalStorage').once();
await Apify.openKeyValueStore();
@@ -220,7 +223,7 @@ describe('KeyValueStore', () => {
describe('getValue', () => {
test('throws on invalid args', async () => {
process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '1234';
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
await expect(Apify.getValue()).rejects.toThrow('Parameter "key" of type String must be provided');
await expect(Apify.getValue({})).rejects.toThrow('Parameter "key" of type String must be provided');
await expect(Apify.getValue('')).rejects.toThrow('The "key" parameter cannot be empty');
@@ -246,7 +249,7 @@ describe('KeyValueStore', () => {
describe('setValue', () => {
test('throws on invalid args', async () => {
process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '12345';
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
await expect(Apify.setValue()).rejects.toThrow('Parameter "key" of type String must be provided');
await expect(Apify.setValue('', null)).rejects.toThrow('The "key" parameter cannot be empty');
@@ -290,7 +293,7 @@ describe('KeyValueStore', () => {
});
test('throws on invalid key', async () => {
- const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
+ const store = new KeyValueStoreLocal('my-store-id', localStorageDir);
const INVALID_CHARACTERS = '?|\\/"*<>%:';
let counter = 0;
@@ -408,8 +411,8 @@ describe('KeyValueStore', () => {
describe('getPublicUrl', () => {
test('should return the local url of a file', () => {
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
- const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
+ const store = new KeyValueStoreLocal('my-store-id', localStorageDir);
expect(store.getPublicUrl('file.txt')).toBe(`file://${store.localStoragePath}/file.txt`);
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
});
@@ -487,7 +490,7 @@ describe('KeyValueStore', () => {
test('should work locally', async () => {
const storeId = 'some-id-1';
- const store = new KeyValueStoreLocal(storeId, LOCAL_STORAGE_DIR);
+ const store = new KeyValueStoreLocal(storeId, localStorageDir);
for (let i = 0; i < 10; i++) {
await store.setValue(`key${i}`, {});
@@ -506,7 +509,7 @@ describe('KeyValueStore', () => {
});
// Drop works.
- const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, storeId);
+ const storeDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, storeId);
expectDirNonEmpty(storeDir);
await store.drop();
expectDirEmpty(storeDir);
@@ -515,7 +518,7 @@ describe('KeyValueStore', () => {
describe('getInput', () => {
test('should work', async () => {
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = localStorageDir;
const defaultStore = await Apify.openKeyValueStore();
// Uses default value.
const oldGet = defaultStore.getValue;
diff --git a/test/local-storage-emulator.js b/test/local_storage_dir_emulator.js
similarity index 61%
rename from test/local-storage-emulator.js
rename to test/local_storage_dir_emulator.js
index a61bbbbbc..10b4b3073 100644
--- a/test/local-storage-emulator.js
+++ b/test/local_storage_dir_emulator.js
@@ -1,12 +1,26 @@
import { cryptoRandomObjectId } from 'apify-shared/utilities';
+import { LOCAL_STORAGE_SUBDIRS, LOCAL_ENV_VARS, ENV_VARS } from 'apify-shared/consts';
import fs from 'fs-extra';
import path from 'path';
import log from 'apify-shared/log';
import { LOCAL_STORAGE_DIR } from './_helper';
-const DEFAULT_FOLDERS = ['datasets', 'key_value_stores/default', 'live_view', 'request_queues'];
-class LocalStorageEmulator {
+const DEFAULT_FOLDERS = Object.values(LOCAL_STORAGE_SUBDIRS)
+ .concat([
+ `${LOCAL_STORAGE_SUBDIRS.keyValueStores}/${LOCAL_ENV_VARS[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID]}`,
+ 'live_view',
+ ]);
+
+/**
+ * Emulates storage for testing purposes.
+ * Creates an unique folder with default structure.
+ * This class should be used in all tests that are using the storage.
+ *
+ * Basic usage: Create and initialize `LocalStorageDirEmulator` in beforeAll hook,
+ * call `clean()` in afterEach hook and finally call `destroy()` in afterAll hook.
+ */
+class LocalStorageDirEmulator {
constructor(localStorageDir = cryptoRandomObjectId(10)) {
this.localStorageDir = path.join(LOCAL_STORAGE_DIR, localStorageDir);
log.debug(`Created local storage emulation in folder ${this.localStorageDir}`);
@@ -23,15 +37,15 @@ class LocalStorageEmulator {
* Removes the folder itself
* @return {Promise<void>}
*/
- async teardown() {
+ async destroy() {
fs.removeSync(this.localStorageDir);
+ delete process.env.APIFY_LOCAL_STORAGE_DIR;
}
/**
* Removes all files/folders form it
* @return {Promise<void>}
*/
- // Maybe empty?
async clean() {
for (const folder of DEFAULT_FOLDERS) {
await fs.emptyDirSync(path.join(this.localStorageDir, folder));
@@ -46,4 +60,4 @@ class LocalStorageEmulator {
}
}
-export default LocalStorageEmulator;
+export default LocalStorageDirEmulator;
diff --git a/test/request_queue.test.js b/test/request_queue.test.js
index a70b7e71d..1d8c74267 100644
--- a/test/request_queue.test.js
+++ b/test/request_queue.test.js
@@ -6,9 +6,10 @@ import * as Apify from '../build/index';
import * as utils from '../build/utils';
import {
RequestQueueLocal, RequestQueue,
- LOCAL_STORAGE_SUBDIR, QUERY_HEAD_MIN_LENGTH, API_PROCESSED_REQUESTS_DELAY_MILLIS, STORAGE_CONSISTENCY_DELAY_MILLIS,
+ LOCAL_STORAGE_SUBDIR, LOCAL_STORAGE_DIR, QUERY_HEAD_MIN_LENGTH, API_PROCESSED_REQUESTS_DELAY_MILLIS, STORAGE_CONSISTENCY_DELAY_MILLIS,
} from '../build/request_queue';
-import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR, expectNotUsingLocalStorage, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import { expectNotUsingLocalStorage, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import LocalStorageDirEmulator from './local_storage_dir_emulator';
const { apifyClient } = utils;
@@ -19,12 +20,29 @@ const expectRequestsSame = (req1, req2) => {
describe('RequestQueue', () => {
beforeAll(() => apifyClient.setOptions({ token: 'xxx' }));
afterAll(() => apifyClient.setOptions({ token: undefined }));
- beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
- afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));
describe('local', () => {
+ let localStorageEmulator;
+ let localStorageDir;
+
+ beforeAll(async () => {
+ apifyClient.setOptions({ token: 'xxx' });
+ localStorageEmulator = new LocalStorageDirEmulator();
+ await localStorageEmulator.init();
+ localStorageDir = localStorageEmulator.localStorageDir; // eslint-disable-line
+ });
+
+ afterAll(async () => {
+ apifyClient.setOptions({ token: undefined });
+ await localStorageEmulator.destroy();
+ });
+
+ beforeEach(async () => {
+ await localStorageEmulator.clean();
+ });
+
test('should work', async () => {
- const queue = new RequestQueueLocal('my-queue-0', LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal('my-queue-0', localStorageDir);
const req1 = new Apify.Request({ url: 'http://example.com/first' });
const info1 = await queue.addRequest(req1);
@@ -110,14 +128,14 @@ describe('RequestQueue', () => {
expect(await queue.isFinished()).toBe(true);
// Drop it.
- const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');
+ const queueDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'my-queue-0');
expectDirNonEmpty(queueDir);
await queue.drop();
expectDirEmpty(queueDir);
});
test('handles invalid URLs gracefully', async () => {
- const queue = new RequestQueueLocal('my-queue-x', LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal('my-queue-x', localStorageDir);
try {
await queue.addRequest({ url: '' });
@@ -143,7 +161,7 @@ describe('RequestQueue', () => {
});
test('supports forefront param in reclaimRequest()', async () => {
- const queue = new RequestQueueLocal('my-queue-1', LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal('my-queue-1', localStorageDir);
await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
@@ -176,7 +194,7 @@ describe('RequestQueue', () => {
const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });
// Do something with 3 requests in one queue.
- const queue = new RequestQueueLocal('my-queue-2', LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal('my-queue-2', localStorageDir);
await queue.addRequest(request1);
await queue.addRequest(request2);
await queue.addRequest(request3, { forefront: true });
@@ -187,7 +205,7 @@ describe('RequestQueue', () => {
await queue.markRequestHandled(freshRequest1);
// Now do the same with another queue.
- const anotherQueue = new RequestQueueLocal('my-queue-2', LOCAL_STORAGE_DIR);
+ const anotherQueue = new RequestQueueLocal('my-queue-2', localStorageDir);
expect(await anotherQueue.isEmpty()).toBe(false);
expect(await anotherQueue.isFinished()).toBe(false);
const request3FromAnotherQueue = await anotherQueue.fetchNextRequest();
@@ -203,7 +221,7 @@ describe('RequestQueue', () => {
});
test('should accept plain object in addRequest()', async () => {
- const queue = new RequestQueueLocal('some-id', LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal('some-id', localStorageDir);
await queue.addRequest({ url: 'http://example.com/a' });
expect(
(await queue.fetchNextRequest()).url,
@@ -212,7 +230,7 @@ describe('RequestQueue', () => {
test('getInfo() and handledCount() should work', async () => {
const queueName = 'stats-queue';
- const queue = new RequestQueueLocal(queueName, LOCAL_STORAGE_DIR);
+ const queue = new RequestQueueLocal(queueName, localStorageDir);
let count = await queue.handledCount();
let info;
expect(count).toBe(0);
@@ -273,7 +291,7 @@ describe('RequestQueue', () => {
expect(mTime).toBeLessThan(now);
expect(aTime).toBeLessThan(now);
- const newQueue = new RequestQueueLocal(queueName, LOCAL_STORAGE_DIR);
+ const newQueue = new RequestQueueLocal(queueName, localStorageDir);
count = await newQueue.handledCount();
info = await queue.getInfo();
expect(count).toBe(3);
@@ -283,10 +301,10 @@ describe('RequestQueue', () => {
});
test('deprecated delete() still works', async () => {
- const rq = new RequestQueueLocal('to-delete', LOCAL_STORAGE_DIR);
+ const rq = new RequestQueueLocal('to-delete', localStorageDir);
await rq.addRequest({ url: 'https://example.com' });
- const rqDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'to-delete');
+ const rqDir = path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, 'to-delete');
expectDirNonEmpty(rqDir);
await rq.delete();
expectDirEmpty(rqDir);
diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.js
index b0f533b2e..d12c607d4 100644
--- a/test/session_pool/session_pool.test.js
+++ b/test/session_pool/session_pool.test.js
@@ -1,18 +1,17 @@
-import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
import { SessionPool, openSessionPool } from '../../build/session_pool/session_pool';
import Apify from '../../build';
import events from '../../build/events';
import { ACTOR_EVENT_NAMES_EX } from '../../build/constants';
import { Session } from '../../src/session_pool/session';
-import LocalStorageEmulator from '../local-storage-emulator';
+import LocalStorageDirEmulator from '../local_storage_dir_emulator';
describe('SessionPool - testing session pool', () => {
let sessionPool;
let localStorageEmulator;
beforeAll(async () => {
- localStorageEmulator = new LocalStorageEmulator();
+ localStorageEmulator = new LocalStorageDirEmulator();
await localStorageEmulator.init();
});
@@ -21,12 +20,13 @@ describe('SessionPool - testing session pool', () => {
sessionPool = await Apify.openSessionPool();
});
- afterEach(() => {
+ afterEach(async () => {
events.removeAllListeners(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);
+ await localStorageEmulator.clean();
});
afterAll(async () => {
- await localStorageEmulator.teardown();
+ await localStorageEmulator.destroy();
});
// eslint-disable-line
@@ -60,6 +60,8 @@ describe('SessionPool - testing session pool', () => {
Object.entries(opts).forEach(([key, value]) => {
expect(sessionPool[key]).toEqual(value);
});
+ const store = await Apify.openKeyValueStore('TEST');
+ await store.drop();
});
test('should work using openSessionPool', async () => {
@@ -84,6 +86,8 @@ describe('SessionPool - testing session pool', () => {
Object.entries(opts).forEach(([key, value]) => {
expect(sessionPool[key]).toEqual(value);
});
+ const store = await Apify.openKeyValueStore('TEST');
+ await store.drop();
});
describe('should retrieve session', () => {
@@ -199,7 +203,6 @@ describe('SessionPool - testing session pool', () => {
});
afterEach(async () => {
- await emptyLocalStorageSubdir(`key_value_stores/${KV_STORE}`);
sessionPool.teardown();
});
diff --git a/test/settings_rotator.test.js b/test/settings_rotator.test.js
deleted file mode 100644
index 9126c6470..000000000
--- a/test/settings_rotator.test.js
+++ /dev/null
@@ -1,18 +0,0 @@
-import _ from 'underscore';
-import Apify from '../build/index';
-
-describe('Apify.SettingsRotator', () => {
- test('works', async () => {
- const maxUsages = 12;
- const totalCalls = 167;
-
- const rotator = new Apify.SettingsRotator({
- newSettingsFunction: () => Math.random(),
- maxUsages,
- });
-
- const settings = _.range(0, totalCalls).map(() => rotator.fetchSettings());
-
- expect(_.uniq(settings).length).toEqual(Math.ceil(totalCalls / maxUsages));
- });
-});
diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.js
index 0bfac0366..ce5420173 100644
--- a/test/stealth/stealth.test.js
+++ b/test/stealth/stealth.test.js
@@ -1,7 +1,7 @@
import scanner from 'fpscanner';
import Apify from '../../build';
-import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
+import LocalStorageDirEmulator from '../local_storage_dir_emulator';
const fingerPrintPath = require.resolve('fpcollect/dist/fpCollect.min.js');
@@ -13,10 +13,21 @@ const getFingerPrint = async (page) => {
// we can speed up the test to make the requests to the local static html
describe('Stealth - testing headless chrome hiding tricks', () => {
+ let localStorageEmulator;
+
beforeAll(async () => {
- process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
- await emptyLocalStorageSubdir('key_value_stores/default');
+ localStorageEmulator = new LocalStorageDirEmulator();
+ await localStorageEmulator.init();
+ });
+
+ beforeEach(async () => {
+ await localStorageEmulator.clean();
});
+
+ afterAll(async () => {
+ await localStorageEmulator.destroy();
+ });
+
test('it adds plugins, mimeTypes and passes', async () => {
const browser = await Apify.launchPuppeteer({
stealth: true,