Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: use config instance in `sdk.openSessionPool()` (#1154)

Martin Adámek664bdc9a1f65
diff --git a/src/apify.js b/src/apify.js
index 3db78196f..252df9ded 100644
--- a/src/apify.js
+++ b/src/apify.js
@@ -800,7 +800,7 @@ export class Apify {
* @return {Promise<SessionPool>}
*/
async openSessionPool(sessionPoolOptions) {
- const sessionPool = new SessionPool(sessionPoolOptions);
+ const sessionPool = new SessionPool(sessionPoolOptions, this.config);
await sessionPool.initialize();
return sessionPool;
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index 21e4ab72b..d59f48feb 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -5,6 +5,7 @@ import { Session, SessionOptions } from './session'; // eslint-disable-line no-u
import events from '../events';
import defaultLog from '../utils_log';
import { ACTOR_EVENT_NAMES_EX } from '../constants';
+import { Configuration } from '../configuration';
/**
* Factory user-function which creates customized {@link Session} instances.
@@ -87,8 +88,9 @@ export class SessionPool extends EventEmitter {
/**
* Session pool configuration.
* @param {SessionPoolOptions} [options] All `SessionPool` configuration options.
+ * @param {Configuration} [config]
*/
- constructor(options = {}) {
+ constructor(options = {}, config = Configuration.getGlobalConfig()) {
ow(options, ow.object.exactShape({
maxPoolSize: ow.optional.number,
persistStateKeyValueStoreId: ow.optional.string,
@@ -112,6 +114,7 @@ export class SessionPool extends EventEmitter {
super();
+ this.config = config;
this.log = log.child({ prefix: 'SessionPool' });
// Pool Configuration
@@ -160,7 +163,7 @@ export class SessionPool extends EventEmitter {
* @return {Promise<void>}
*/
async initialize() {
- this.keyValueStore = await openKeyValueStore(this.persistStateKeyValueStoreId);
+ this.keyValueStore = await openKeyValueStore(this.persistStateKeyValueStoreId, { config: this.config });
// in case of migration happened and SessionPool state should be restored from the keyValueStore.
await this._maybeLoadSessionPool();
diff --git a/src/storages/dataset.js b/src/storages/dataset.js
index ba2c3c771..6ae24b73a 100644
--- a/src/storages/dataset.js
+++ b/src/storages/dataset.js
@@ -9,6 +9,7 @@ import log from '../utils_log';
import * as ApifyClient from 'apify-client';
// @ts-ignore
import { ApifyStorageLocal } from '@apify/storage-local';
+import { Configuration } from '../configuration';
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
@@ -412,6 +413,7 @@ export class Dataset {
* @param {boolean} [options.forceCloud=false]
* If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
* environment variable is set. This way it is possible to combine local and cloud storage.
+ * @param {Configuration} [options.config] SDK configuration instance, defaults to the static register
* @returns {Promise<Dataset>}
* @memberof module:Apify
* @name openDataset
@@ -421,9 +423,10 @@ export const openDataset = (datasetIdOrName, options = {}) => {
ow(datasetIdOrName, ow.optional.string);
ow(options, ow.object.exactShape({
forceCloud: ow.optional.boolean,
+ config: ow.optional.object.instanceOf(Configuration),
}));
- const manager = new StorageManager(Dataset);
+ const manager = new StorageManager(Dataset, options.config);
return manager.openStorage(datasetIdOrName, options);
};
diff --git a/src/storages/key_value_store.js b/src/storages/key_value_store.js
index 7d0705b0a..206e8eadc 100644
--- a/src/storages/key_value_store.js
+++ b/src/storages/key_value_store.js
@@ -339,6 +339,7 @@ export class KeyValueStore {
* @param {boolean} [options.forceCloud=false]
* If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
* environment variable is set. This way it is possible to combine local and cloud storage.
+ * @param {Configuration} [options.config] SDK configuration instance, defaults to the static register
* @returns {Promise<KeyValueStore>}
* @memberof module:Apify
* @name openKeyValueStore
@@ -348,9 +349,10 @@ export const openKeyValueStore = async (storeIdOrName, options = {}) => {
ow(storeIdOrName, ow.optional.string);
ow(options, ow.object.exactShape({
forceCloud: ow.optional.boolean,
+ config: ow.optional.object.instanceOf(Configuration),
}));
- const manager = new StorageManager(KeyValueStore);
+ const manager = new StorageManager(KeyValueStore, options.config);
return manager.openStorage(storeIdOrName, options);
};
diff --git a/test/apify.test.js b/test/apify.test.js
index cc6143b6a..c45372c5a 100644
--- a/test/apify.test.js
+++ b/test/apify.test.js
@@ -153,6 +153,25 @@ describe('new Apify({ ... })', () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
});
+ test('respects `localStorageEnableWalMode` option (gh issue #956)', async () => {
+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
+ delete process.env[ENV_VARS.TOKEN];
+
+ const sdk1 = new Apify();
+ const sessionPool1 = await sdk1.openSessionPool();
+ expect(sessionPool1).toBeInstanceOf(SessionPool);
+ const storage1 = sdk1.config.getStorageLocal();
+ expect(storage1.enableWalMode).toBe(true);
+
+ const sdk2 = new Apify({ localStorageEnableWalMode: false });
+ const sessionPool2 = await sdk2.openSessionPool();
+ expect(sessionPool2).toBeInstanceOf(SessionPool);
+ const storage2 = sdk2.config.getStorageLocal();
+ expect(storage2.enableWalMode).toBe(false);
+
+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
+ });
+
test('works with promised user function', async () => {
let called = false;
await testMain({