Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add global `useState` helper (#1551)
Matt Stephens2b0317772a2b
diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.tsindex 299a2ace8..012eaa5e9 100644--- a/packages/core/src/storages/utils.ts+++ b/packages/core/src/storages/utils.ts@@ -1,5 +1,6 @@-import type { StorageClient } from '@crawlee/types';+import type { Dictionary, StorageClient } from '@crawlee/types';import { Configuration } from '../configuration';+import { KeyValueStore } from './key_value_store';/*** Cleans up the local storage folder (defaults to `./storage`) created when running code locally.@@ -22,3 +23,30 @@ export async function purgeDefaultStorages(config = Configuration.getGlobalConfiawait client.purge?.();}}++export interface UseStateOptions {+ config?: Configuration;+ /**+ * The name of the key-value store you'd like the state to be stored in.+ * If not provided, the default store will be used.+ */+ keyValueStoreName?: string | null;+}++/**+ * Easily create and manage state values. All state values are automatically persisted.+ *+ * Values can be modified by simply using the assignment operator.+ *+ * @param name The name of the store to use.+ * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.+ * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.+ */+export async function useState<State extends Dictionary = Dictionary>(+ name?: string,+ defaultValue = {} as State,+ options?: UseStateOptions,+) {+ const kvStore = await KeyValueStore.open(options?.keyValueStoreName, { config: options?.config || Configuration.getGlobalConfig() });+ return kvStore.getAutoSavedValue<State>(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);+}diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.tsnew file mode 100644index 000000000..2927922dd--- /dev/null+++ b/test/core/storages/utils.test.ts@@ -0,0 +1,66 @@+import { useState, KeyValueStore } from '@crawlee/core';+import type { Dictionary } from '@crawlee/core';+import { Configuration } from 'apify';+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';++describe('useState', () => {+ const emulator = new MemoryStorageEmulator();++ beforeAll(async () => {+ Configuration.getGlobalConfig().set('persistStateIntervalMillis', 1e3);+ });++ beforeEach(async () => {+ await emulator.init();+ });++ afterAll(async () => {+ await emulator.destroy();+ });++ it('Should initialize with the provided value', async () => {+ const state = await useState('my-state', { hello: 'world' });++ expect(state).toHaveProperty('hello');+ expect(state).toEqual({ hello: 'world' });+ });++ it('Should auto-save the modified value', async () => {+ const state1 = await useState<Dictionary<any>>('my-state', { hello: 'world' });+ expect(state1).toEqual({ hello: 'world' });++ state1.hello = 'foo';+ state1.foo = ['fizz'];++ const state2 = await useState<Dictionary<any>>('my-state', { hello: 'world' });+ expect(state2).toEqual({ hello: 'foo', foo: ['fizz'] });++ state2.foo!.push('buzz');+ expect(state2).toEqual({ hello: 'foo', foo: ['fizz', 'buzz'] });+ });++ it('Should save the value to the default key-value store', async () => {+ const state = await useState<Dictionary<any>>('my-state', { hello: 'world' });+ expect(state).toEqual({ hello: 'world' });++ state.hello = 'foo';+ state.foo = ['fizz'];++ const manager = Configuration.globalConfig.getEventManager();++ await manager.init();++ await new Promise((resolve) => {+ manager.on('persistState', () => {+ resolve(true);+ });+ });++ const data = await KeyValueStore.getValue('my-state');++ expect(data).toHaveProperty('hello');+ expect(data).toHaveProperty('foo');++ await manager.close();+ });+});