Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor!: Remove unnecessary parameters to storage frontend constructors (#3880)

Jan Bucharf83b6f0a4430
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 6a813186c..5f81762a6 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -24,6 +24,7 @@ import type { HttpRequestOptions } from '@crawlee/types';
import type { ISession } from '@crawlee/types';
import type { ISessionPool } from '@crawlee/types';
import type { KeyValueStoreBackend } from '@crawlee/types';
+import type { KeyValueStoreInfo } from '@crawlee/types';
import { Log } from '@apify/log';
import log from '@apify/log';
import { Logger } from '@apify/log';
@@ -508,10 +509,7 @@ export interface DatasetMapper<Data, R> {
export interface DatasetOptions {
// (undocumented)
backend: DatasetBackend;
- // (undocumented)
- id: string;
- // (undocumented)
- name?: string;
+ metadata: DatasetInfo;
}
// @public
@@ -928,10 +926,7 @@ export interface KeyValueStoreIteratorOptions {
export interface KeyValueStoreOptions {
// (undocumented)
backend: KeyValueStoreBackend;
- // (undocumented)
- id: string;
- // (undocumented)
- name?: string;
+ metadata: KeyValueStoreInfo;
}
// @public
@@ -1057,11 +1052,11 @@ export interface MemorySnapshot extends LoadSnapshot {
export class MemoryStorageBackend implements storage.StorageBackend {
constructor(options?: MemoryStorageOptions);
// (undocumented)
- createDatasetBackend(options?: storage.CreateDatasetBackendOptions): Promise<storage.DatasetBackend>;
+ createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
// (undocumented)
- createKeyValueStoreBackend(options?: storage.CreateKeyValueStoreBackendOptions): Promise<storage.KeyValueStoreBackend>;
+ createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
// (undocumented)
- createRequestQueueBackend(options?: storage.CreateRequestQueueBackendOptions): Promise<RequestQueueBackend_2>;
+ createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend_2>;
// (undocumented)
readonly datasetBackendCache: DatasetBackend_2[];
getStorageBackendCacheKey(): string;
@@ -1443,8 +1438,6 @@ export class RequestQueue implements IStorage, IRequestManager {
backend: RequestQueueBackend;
protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
// (undocumented)
- clientKey: string;
- // (undocumented)
protected readonly config: Configuration;
drop(): Promise<void>;
// (undocumented)
@@ -1482,8 +1475,6 @@ export class RequestQueue implements IStorage, IRequestManager {
protected requestSeenCache: RequestDeduplicationCache;
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
get stats(): RequestQueueStats;
- // (undocumented)
- timeoutSecs: number;
}
// @internal (undocumented)
@@ -1505,10 +1496,7 @@ export interface RequestQueueOperationOptions {
export interface RequestQueueOptions {
// (undocumented)
backend: RequestQueueBackend;
- // (undocumented)
- id: string;
- // (undocumented)
- name?: string;
+ metadata: RequestQueueInfo;
proxyConfiguration?: ProxyConfiguration;
}
diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.md
index 79553b4b1..bba73cf4c 100644
--- a/docs/public-api/crawlee-fs-storage.api.md
+++ b/docs/public-api/crawlee-fs-storage.api.md
@@ -15,11 +15,11 @@ import type * as storage from '@crawlee/types';
export class FileSystemStorageBackend implements storage.StorageBackend {
constructor(options: FileSystemStorageOptions);
// (undocumented)
- createDatasetBackend(options?: storage.CreateDatasetBackendOptions): Promise<storage.DatasetBackend>;
+ createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
// (undocumented)
- createKeyValueStoreBackend(options?: storage.CreateKeyValueStoreBackendOptions): Promise<storage.KeyValueStoreBackend>;
+ createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
// (undocumented)
- createRequestQueueBackend(options?: storage.CreateRequestQueueBackendOptions): Promise<storage.RequestQueueBackend>;
+ createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
// (undocumented)
readonly datasetBackendCache: DatasetBackend[];
// (undocumented)
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index 348a3491f..b0435846c 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -77,18 +77,6 @@ export interface CrawleeLoggerOptions {
prefix?: string | null;
}
-// @public
-export type CreateDatasetBackendOptions = StorageIdentifier;
-
-// @public
-export type CreateKeyValueStoreBackendOptions = StorageIdentifier;
-
-// @public
-export type CreateRequestQueueBackendOptions = StorageIdentifier & {
- clientKey?: string;
- timeoutSecs?: number;
-};
-
// @public (undocumented)
export interface DatasetBackend<Data extends Dictionary = Dictionary> {
drop(): Promise<void>;
@@ -466,9 +454,9 @@ export interface SetStatusMessageOptions {
// @public
export interface StorageBackend {
- createDatasetBackend(options?: CreateDatasetBackendOptions): Promise<DatasetBackend>;
- createKeyValueStoreBackend(options?: CreateKeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;
- createRequestQueueBackend(options?: CreateRequestQueueBackendOptions): Promise<RequestQueueBackend>;
+ createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
+ createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
+ createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
getStorageBackendCacheKey?(): string;
// (undocumented)
purge?(): Promise<void>;
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 52f98f48f..685b57196 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -533,7 +533,7 @@ The `StorageBackend` interface changed from synchronous sub-client getters to **
| `client.datasets().getOrCreate(name)` | _(absorbed into `createDatasetBackend`)_ |
| `client.keyValueStore(id)` | `backend.createKeyValueStoreBackend({ id?, name? })` |
| `client.keyValueStores().getOrCreate(name)` | _(absorbed into `createKeyValueStoreBackend`)_ |
-| `client.requestQueue(id, opts)` | `backend.createRequestQueueBackend({ id?, name?, clientKey?, timeoutSecs? })` |
+| `client.requestQueue(id, opts)` | `backend.createRequestQueueBackend({ id?, name? })` |
| `client.requestQueues().getOrCreate(name)` | _(absorbed into `createRequestQueueBackend`)_ |
The sub-backend interfaces (`DatasetBackend`, `KeyValueStoreBackend`, `RequestQueueBackend`, formerly `DatasetClient` / `KeyValueStoreClient` / `RequestQueueClient`) have been aligned with their Python counterparts:
@@ -622,9 +622,11 @@ The `RequestQueue.internalTimeoutMillis` property and the associated "stuck queu
**Apify-specific fields removed from storage metadata.** The metadata returned by `getMetadata()` (`DatasetInfo`, `KeyValueStoreInfo`, `RequestQueueInfo`) has been trimmed to what is meaningful for any storage backend. The following platform-specific fields were dropped: `actId`, `actRunId`, `userId`, and — on `RequestQueueInfo` — `expireAt` and `hadMultipleClients`. The per-storage `stats` field (and its `DatasetStats` / `KeyValueStoreStats` / `RequestQueueStats` types) was removed as well. If you consumed any of these, read them from the Apify API client directly; a custom `StorageBackend` should simply stop returning them.
-**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`, `DatasetStats`, `KeyValueStoreStats`, `RequestQueueStats`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`.
+**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`, `DatasetStats`, `KeyValueStoreStats`, `RequestQueueStats`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`. The `CreateDatasetBackendOptions`, `CreateKeyValueStoreBackendOptions`, and `CreateRequestQueueBackendOptions` aliases were removed — the `create*Backend` methods now take `StorageIdentifier` directly.
-The high-level storage classes (`Dataset`, `KeyValueStore`, `RequestQueue`) now receive their sub-backend directly in the constructor options (via the `backend` option) instead of receiving a `StorageBackend` and calling its methods.
+The high-level storage classes (`Dataset`, `KeyValueStore`, `RequestQueue`) are now thin wrappers over a single sub-backend, which they receive directly in the constructor options. The constructor takes `{ metadata, backend }`, where `backend` is the sub-backend and `metadata` is the resolved storage info (as returned by the backend's `getMetadata()`) that the storage derives its `id` and `name` from — instead of receiving separate `id` / `name` arguments (or a `StorageBackend` and calling its methods). In practice you never call these constructors yourself; use `Dataset.open()` / `KeyValueStore.open()` / `RequestQueue.open()`, which resolve the metadata and open the backend for you.
+
+`RequestQueue` no longer accepts (or stores) `clientKey` / `timeoutSecs`. These are request-locking concerns that are now internal to the storage backend implementation (see [apify/crawlee#3328](https://github.com/apify/crawlee/issues/3328)); they are also no longer part of the `createRequestQueueBackend` options — all three `create*Backend` methods now take a plain `StorageIdentifier` (`{ id?, name?, alias? }`). The now-redundant `CreateDatasetBackendOptions`, `CreateKeyValueStoreBackendOptions`, and `CreateRequestQueueBackendOptions` type aliases have been removed from `@crawlee/types`; use `StorageIdentifier` instead.
### `RecordOptions` simplified
diff --git a/packages/core/src/memory-storage/memory-storage.ts b/packages/core/src/memory-storage/memory-storage.ts
index d321633ac..c5ec9cca1 100644
--- a/packages/core/src/memory-storage/memory-storage.ts
+++ b/packages/core/src/memory-storage/memory-storage.ts
@@ -50,7 +50,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {
return { isAlias, cacheKey };
}
- async createDatasetBackend(options: storage.CreateDatasetBackendOptions = {}): Promise<storage.DatasetBackend> {
+ async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {
const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
if (cacheKey) {
@@ -75,9 +75,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {
return newStore;
}
- async createKeyValueStoreBackend(
- options: storage.CreateKeyValueStoreBackendOptions = {},
- ): Promise<storage.KeyValueStoreBackend> {
+ async createKeyValueStoreBackend(options: storage.StorageIdentifier = {}): Promise<storage.KeyValueStoreBackend> {
const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
if (cacheKey) {
@@ -102,9 +100,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {
return newStore;
}
- async createRequestQueueBackend(
- options: storage.CreateRequestQueueBackendOptions = {},
- ): Promise<RequestQueueBackend> {
+ async createRequestQueueBackend(options: storage.StorageIdentifier = {}): Promise<RequestQueueBackend> {
const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
if (cacheKey) {
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index 174452bb3..14ef5a3ac 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -202,8 +202,8 @@ export class Dataset<Data extends Dictionary = Dictionary> {
options: DatasetOptions,
readonly config = Configuration.getGlobalConfig(),
) {
- this.id = options.id;
- this.name = options.name;
+ this.id = options.metadata.id;
+ this.name = options.metadata.name;
this.backend = options.backend;
this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
}
@@ -775,8 +775,8 @@ export interface DatasetReducer<T, Data> {
}
export interface DatasetOptions {
- id: string;
- name?: string;
+ /** Resolved metadata for the dataset, as returned by the backend's `getMetadata()`. */
+ metadata: DatasetInfo;
backend: DatasetBackend;
}
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index b329c6101..3dff671f1 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -1,4 +1,4 @@
-import type { Dictionary, KeyValueStoreBackend, KeyValueStoreItemData } from '@crawlee/types';
+import type { Dictionary, KeyValueStoreBackend, KeyValueStoreInfo, KeyValueStoreItemData } from '@crawlee/types';
import ow, { ArgumentError } from 'ow';
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
@@ -98,8 +98,8 @@ export class KeyValueStore {
options: KeyValueStoreOptions,
readonly config = Configuration.getGlobalConfig(),
) {
- this.id = options.id;
- this.name = options.name;
+ this.id = options.metadata.id;
+ this.name = options.metadata.name;
this.backend = options.backend;
}
@@ -889,8 +889,8 @@ export interface KeyConsumer {
}
export interface KeyValueStoreOptions {
- id: string;
- name?: string;
+ /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
+ metadata: KeyValueStoreInfo;
backend: KeyValueStoreBackend;
}
diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts
index de5ad5bf5..e097f561c 100644
--- a/packages/core/src/storages/request_queue.ts
+++ b/packages/core/src/storages/request_queue.ts
@@ -22,7 +22,6 @@ import ow from 'ow';
import type { ReadonlyDeep } from 'type-fest';
import { LruCache } from '@apify/datastructures';
-import { cryptoRandomObjectId } from '@apify/utilities';
import { Configuration } from '../configuration.js';
import type { Constructor } from '../typedefs.js';
@@ -92,15 +91,11 @@ const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
export class RequestQueue implements IStorage, IRequestManager {
id: string;
name?: string;
- timeoutSecs = 30;
- clientKey = cryptoRandomObjectId();
backend: RequestQueueBackend;
protected proxyConfiguration?: ProxyConfiguration;
log: CrawleeLogger;
- private isInitialized = false;
-
protected requestCache: LruCache<RequestLruItem>;
/**
@@ -145,8 +140,8 @@ export class RequestQueue implements IStorage, IRequestManager {
options: RequestQueueOptions,
protected readonly config: Configuration = Configuration.getGlobalConfig(),
) {
- this.id = options.id;
- this.name = options.name;
+ this.id = options.metadata.id;
+ this.name = options.metadata.name;
this.events = serviceLocator.getEventManager();
this.backend = options.backend;
@@ -987,21 +982,6 @@ export class RequestQueue implements IStorage, IRequestManager {
queue.proxyConfiguration = options.proxyConfiguration;
queue.httpClient = options.httpClient;
- if (!queue.isInitialized) {
- // Re-create the request queue backend with clientKey and timeoutSecs so that
- // request locking works correctly for API-backed implementations.
- // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
- // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
- // where locking is handled internally by the backend (see crawlee-python PR #1194).
- queue.backend = await storageBackend.createRequestQueueBackend({
- id: queue.id,
- clientKey: queue.clientKey,
- timeoutSecs: queue.timeoutSecs,
- });
-
- queue.isInitialized = true;
- }
-
return queue;
}
}
@@ -1016,8 +996,8 @@ interface RequestLruItem {
}
export interface RequestQueueOptions {
- id: string;
- name?: string;
+ /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */
+ metadata: RequestQueueInfo;
backend: RequestQueueBackend;
/**
diff --git a/packages/core/src/storages/storage_instance_manager.ts b/packages/core/src/storages/storage_instance_manager.ts
index ee4a14c8c..e79221614 100644
--- a/packages/core/src/storages/storage_instance_manager.ts
+++ b/packages/core/src/storages/storage_instance_manager.ts
@@ -258,11 +258,9 @@ export class StorageInstanceManager {
subBackend as DatasetBackend | KeyValueStoreBackend | RequestQueueBackend
).getMetadata();
- const instance = new cls({
- id: storageInfo.id,
- name: storageInfo.name,
- backend: subBackend,
- }) as TStorage;
+ // Storage frontends are thin wrappers over the backend. We hand them the resolved metadata
+ // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
+ const instance = new cls({ metadata: storageInfo, backend: subBackend });
// Atomic cache writes (no awaits between these).
this.cache.set(cls, instance, backendCacheKey, alias);
diff --git a/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.test.ts b/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.test.ts
new file mode 100644
index 000000000..1c74a4dcd
--- /dev/null
+++ b/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.test.ts
@@ -0,0 +1,47 @@
+import { load } from 'cheerio';
+import {
+ cheerioCrawlerEnqueueLinks,
+ EnqueueStrategy,
+ MemoryStorageBackend,
+ RequestQueue,
+ serviceLocator,
+} from 'crawlee';
+
+const HTML = `
+<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <ul>
+ <li>
+ <a class="first" href="https://menicka.cz/redirect.php?w=akce&id=f1ab8ae200bddaa17fd50150943d1e06">
+ I'm sorry, Dave. I'm afraid I can't do that.
+ </a>
+ </li>
+ </ul>
+ </body>
+</html>
+`;
+
+describe('enqueueLinks() - it should store the enqueue strategy in requests', () => {
+ const $ = load(HTML);
+
+ beforeEach(() => {
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
+ });
+
+ test('it should store the enqueue strategy in requests', async () => {
+ const requestQueue = await RequestQueue.open();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: { selector: 'a', strategy: EnqueueStrategy.SameHostname },
+ $,
+ requestManager: requestQueue,
+ originalRequestUrl: 'https://menicka.cz',
+ });
+
+ const enqueued = await requestQueue.fetchNextRequest();
+ expect(enqueued!.userData.__crawlee.enqueueStrategy).toEqual(EnqueueStrategy.SameHostname);
+ });
+});
diff --git a/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts b/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts
deleted file mode 100644
index eaeda0eb7..000000000
--- a/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import { load } from 'cheerio';
-import type { Source } from 'crawlee';
-import { cheerioCrawlerEnqueueLinks, EnqueueStrategy, RequestQueue, serviceLocator } from 'crawlee';
-
-import log from '@apify/log';
-
-const apifyClient = serviceLocator.getStorageBackend();
-
-const HTML = `
-<html>
- <head>
- <title>Example</title>
- </head>
- <body>
- <ul>
- <li>
- <a class="first" href="https://menicka.cz/redirect.php?w=akce&id=f1ab8ae200bddaa17fd50150943d1e06">
- I'm sorry, Dave. I'm afraid I can't do that.
- </a>
- </li>
- </ul>
- </body>
-</html>
-`;
-
-function createRequestQueueMock() {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', backend: apifyClient });
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
- };
-
- return { enqueued, requestQueue };
-}
-
-describe('enqueueLinks() - it should store the enqueue strategy in requests', () => {
- let ll: number;
- beforeAll(() => {
- ll = log.getLevel();
- log.setLevel(log.LEVELS.ERROR);
- });
-
- afterAll(() => {
- log.setLevel(ll);
- });
-
- const $ = load(HTML);
-
- test('it should store the enqueue strategy in requests', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
-
- await cheerioCrawlerEnqueueLinks({
- options: { selector: 'a', strategy: EnqueueStrategy.SameHostname },
- $,
- requestQueue,
- originalRequestUrl: 'https://menicka.cz',
- });
-
- expect(enqueued[0].userData!.__crawlee.enqueueStrategy).toEqual(EnqueueStrategy.SameHostname);
- });
-});
diff --git a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
index 3908e062b..fac31d208 100644
--- a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
+++ b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
@@ -1,10 +1,12 @@
import { load } from 'cheerio';
-import type { CheerioRoot, Source } from 'crawlee';
-import { cheerioCrawlerEnqueueLinks, EnqueueStrategy, RequestQueue, serviceLocator } from 'crawlee';
-
-import log from '@apify/log';
-
-const apifyClient = serviceLocator.getStorageBackend();
+import type { CheerioRoot } from 'crawlee';
+import {
+ cheerioCrawlerEnqueueLinks,
+ EnqueueStrategy,
+ MemoryStorageBackend,
+ RequestQueue,
+ serviceLocator,
+} from 'crawlee';
const HTML = `
<html>
@@ -20,37 +22,23 @@ const HTML = `
</html>
`;
-function createRequestQueueMock() {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', backend: apifyClient }, serviceLocator.getConfiguration());
+type MemoryRequestQueueBackend = Awaited<ReturnType<MemoryStorageBackend['createRequestQueueBackend']>>;
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
- };
-
- return { enqueued, requestQueue };
+/** Collect the URLs of all requests in the queue, regardless of order. */
+async function enqueuedUrls(requestQueue: RequestQueue): Promise<Set<string>> {
+ const items = await (requestQueue.backend as MemoryRequestQueueBackend).listItems();
+ return new Set(items.map((item) => item.url));
}
describe('enqueueLinks() - matching and ignoring http/https protocol differences', () => {
- let ll: number;
- beforeAll(() => {
- ll = log.getLevel();
- log.setLevel(log.LEVELS.ERROR);
- });
-
- afterAll(() => {
- log.setLevel(ll);
- });
-
let $: CheerioRoot;
beforeEach(() => {
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
$ = load(HTML);
});
test('SameHostname should ignore protocol difference', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameHostname },
@@ -59,13 +47,13 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(2);
- expect(enqueued[0].url).toBe('https://example.com/first');
- expect(enqueued[1].url).toBe('http://example.com/second');
+ expect(await enqueuedUrls(requestQueue)).toEqual(
+ new Set(['https://example.com/first', 'http://example.com/second']),
+ );
});
test('SameDomain should ignore protocol difference', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameDomain },
@@ -74,13 +62,13 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
originalRequestUrl: 'http://example.com',
});
- expect(enqueued).toHaveLength(2);
- expect(enqueued[0].url).toBe('https://example.com/first');
- expect(enqueued[1].url).toBe('http://example.com/second');
+ expect(await enqueuedUrls(requestQueue)).toEqual(
+ new Set(['https://example.com/first', 'http://example.com/second']),
+ );
});
test('SameOrigin should respect protocol', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: { selector: 'a', strategy: EnqueueStrategy.SameOrigin },
@@ -89,7 +77,6 @@ describe('enqueueLinks() - matching and ignoring http/https protocol differences
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(1);
- expect(enqueued[0].url).toBe('https://example.com/first');
+ expect(await enqueuedUrls(requestQueue)).toEqual(new Set(['https://example.com/first']));
});
});
diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
index 877f0f186..683c6f7c9 100644
--- a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
@@ -1,10 +1,12 @@
import { load } from 'cheerio';
-import type { CheerioRoot, Source } from 'crawlee';
-import { cheerioCrawlerEnqueueLinks, EnqueueStrategy, RequestQueue, serviceLocator } from 'crawlee';
-
-import log from '@apify/log';
-
-const apifyClient = serviceLocator.getStorageBackend();
+import type { CheerioRoot } from 'crawlee';
+import {
+ cheerioCrawlerEnqueueLinks,
+ EnqueueStrategy,
+ MemoryStorageBackend,
+ RequestQueue,
+ serviceLocator,
+} from 'crawlee';
const HTML = `
<html>
@@ -32,37 +34,23 @@ const HTML = `
</html>
`;
-function createRequestQueueMock() {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', backend: apifyClient }, serviceLocator.getConfiguration());
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
- };
+type MemoryRequestQueueBackend = Awaited<ReturnType<MemoryStorageBackend['createRequestQueueBackend']>>;
- return { enqueued, requestQueue };
+/** Collect the URLs of all requests in the queue, regardless of order. */
+async function enqueuedUrls(requestQueue: RequestQueue): Promise<Set<string>> {
+ const items = await (requestQueue.backend as MemoryRequestQueueBackend).listItems();
+ return new Set(items.map((item) => item.url));
}
describe('enqueueLinks() - combining user patterns with enqueue strategies', () => {
- let ll: number;
- beforeAll(() => {
- ll = log.getLevel();
- log.setLevel(log.LEVELS.ERROR);
- });
-
- afterAll(() => {
- log.setLevel(ll);
- });
-
let $: CheerioRoot;
beforeEach(() => {
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
$ = load(HTML);
});
test('works with globs and same domain strategy', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const globs = ['**/first'];
@@ -77,13 +65,11 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(1);
-
- expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(await enqueuedUrls(requestQueue)).toEqual(new Set(['https://example.com/a/b/first']));
});
test('works with globs and all domains strategy', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const globs = ['**/first'];
@@ -98,14 +84,13 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(2);
-
- expect(enqueued[0].url).toBe('https://example.com/a/b/first');
- expect(enqueued[1].url).toBe('https://another.com/a/first');
+ expect(await enqueuedUrls(requestQueue)).toEqual(
+ new Set(['https://example.com/a/b/first', 'https://another.com/a/first']),
+ );
});
test('works with no user provided patterns but with same domain strategy', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: {
@@ -117,13 +102,13 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(2);
- expect(enqueued[0].url).toBe('https://example.com/a/b/first');
- expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ expect(await enqueuedUrls(requestQueue)).toEqual(
+ new Set(['https://example.com/a/b/first', 'https://example.com/a/b/third']),
+ );
});
test('works with globs and exclude', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const globs = ['**/first'];
const exclude = ['**/first'];
@@ -139,11 +124,11 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(0);
+ expect(await enqueuedUrls(requestQueue)).toEqual(new Set());
});
test('works with exclude only', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const exclude = ['**/second', '**/third', 'https://another.com/**'];
@@ -158,8 +143,8 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(2);
- expect(enqueued[0].url).toBe('https://example.com/a/b/first');
- expect(enqueued[1].url).toBe('http://cool.com/');
+ expect(await enqueuedUrls(requestQueue)).toEqual(
+ new Set(['https://example.com/a/b/first', 'http://cool.com/']),
+ );
});
});
diff --git a/packages/core/test/enqueue_links/userData.test.ts b/packages/core/test/enqueue_links/userData.test.ts
index 0ef548816..ddd115dd4 100644
--- a/packages/core/test/enqueue_links/userData.test.ts
+++ b/packages/core/test/enqueue_links/userData.test.ts
@@ -1,12 +1,7 @@
-import type { Source } from '@crawlee/cheerio';
-import { cheerioCrawlerEnqueueLinks, RequestQueue, serviceLocator } from '@crawlee/cheerio';
+import { cheerioCrawlerEnqueueLinks, MemoryStorageBackend, RequestQueue, serviceLocator } from '@crawlee/cheerio';
import type { CheerioAPI } from 'cheerio';
import { load } from 'cheerio';
-import log from '@apify/log';
-
-const apifyClient = serviceLocator.getStorageBackend();
-
const HTML = `
<html>
<head>
@@ -21,37 +16,22 @@ const HTML = `
</html>
`;
-function createRequestQueueMock() {
- const enqueued: Source[] = [];
- const requestQueue = new RequestQueue({ id: 'xxx', backend: apifyClient }, serviceLocator.getConfiguration());
-
- // @ts-expect-error Override method for testing
- requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
- };
+type MemoryRequestQueueBackend = Awaited<ReturnType<MemoryStorageBackend['createRequestQueueBackend']>>;
- return { enqueued, requestQueue };
+/** Collect all requests currently in the queue (order not significant). */
+async function collect(requestQueue: RequestQueue) {
+ return (requestQueue.backend as MemoryRequestQueueBackend).listItems();
}
describe("enqueueLinks() - userData shouldn't be changed and outer label must take priority", () => {
- let ll: number;
- beforeAll(() => {
- ll = log.getLevel();
- log.setLevel(log.LEVELS.ERROR);
- });
-
- afterAll(() => {
- log.setLevel(ll);
- });
-
let $: CheerioAPI;
beforeEach(() => {
+ serviceLocator.setStorageBackend(new MemoryStorageBackend());
$ = load(HTML);
});
test('multiple enqueues with different labels', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const userData = { foo: 'bar' };
await cheerioCrawlerEnqueueLinks({
@@ -76,16 +56,16 @@ describe("enqueueLinks() - userData shouldn't be changed and outer label must ta
originalRequestUrl: 'https://example.com',
});
- expect(enqueued).toHaveLength(2);
-
- expect(enqueued[0].url).toBe('https://example.com/first');
- expect(enqueued[0].userData.label).toBe('first');
- expect(enqueued[1].url).toBe('https://example.com/second');
- expect(enqueued[1].userData.label).toBe('second');
+ const enqueued = await collect(requestQueue);
+ const byUrl = Object.fromEntries(enqueued.map((r) => [r.url, r.userData?.label]));
+ expect(byUrl).toEqual({
+ 'https://example.com/first': 'first',
+ 'https://example.com/second': 'second',
+ });
});
test("JSON string of userData shouldn't change, but enqueued label should be different", async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
const userData = { foo: 'bar', label: 'bogus' };
const originalUserData = JSON.stringify(userData);
@@ -101,13 +81,15 @@ describe("enqueueLinks() - userData shouldn't be changed and outer label must ta
});
const userDataAfterEnqueue = JSON.stringify(userData);
expect(userDataAfterEnqueue).toEqual(originalUserData);
+
+ const enqueued = await collect(requestQueue);
expect(enqueued).toHaveLength(1);
expect(enqueued[0].url).toBe('https://example.com/first');
- expect(enqueued[0].label).toBe('first');
+ expect(enqueued[0].userData?.label).toBe('first');
});
test('sets sessionId on all enqueued requests', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: {
@@ -118,13 +100,13 @@ describe("enqueueLinks() - userData shouldn't be changed and outer label must ta
originalRequestUrl: 'https://example.com',
});
+ const enqueued = await collect(requestQueue);
expect(enqueued).toHaveLength(2);
- expect(enqueued[0].userData?.__crawlee?.sessionId).toBe('my-session');
- expect(enqueued[1].userData?.__crawlee?.sessionId).toBe('my-session');
+ expect(enqueued.every((r) => r.userData?.__crawlee?.sessionId === 'my-session')).toBe(true);
});
test('does not set sessionId when option is not provided', async () => {
- const { enqueued, requestQueue } = createRequestQueueMock();
+ const requestQueue = await RequestQueue.open();
await cheerioCrawlerEnqueueLinks({
options: {},
@@ -133,8 +115,8 @@ describe("enqueueLinks() - userData shouldn't be changed and outer label must ta
originalRequestUrl: 'https://example.com',
});
+ const enqueued = await collect(requestQueue);
expect(enqueued).toHaveLength(2);
- expect(enqueued[0].userData?.__crawlee?.sessionId).toBeUndefined();
- expect(enqueued[1].userData?.__crawlee?.sessionId).toBeUndefined();
+ expect(enqueued.every((r) => r.userData?.__crawlee?.sessionId === undefined)).toBe(true);
});
});
diff --git a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts
index c2092ddea..8eab8ea8d 100644
--- a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts
+++ b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts
@@ -14,7 +14,7 @@ describe('RequestQueue#addRequest should not call the API if the request is alre
test('should not call the API if the request is already in the queue', async () => {
const config = serviceLocator.getConfiguration();
const rqInfo = await rqClient.getMetadata();
- const requestQueue = new RequestQueue({ id: rqInfo.id, backend: rqClient }, config);
+ const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient }, config);
const clientSpy = vitest.spyOn(requestQueue.backend, 'addBatchOfRequests');
@@ -37,7 +37,7 @@ describe('RequestQueue#addRequests should not call the API if the request is alr
test('should not call the API if the request is already in the queue', async () => {
const config = serviceLocator.getConfiguration();
const rqInfo = await rqClient.getMetadata();
- const requestQueue = new RequestQueue({ id: rqInfo.id, backend: rqClient }, config);
+ const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient }, config);
const clientSpy = vitest.spyOn(requestQueue.backend, 'addBatchOfRequests');
diff --git a/packages/core/test/request-queue/request-queue.test.ts b/packages/core/test/request-queue/request-queue.test.ts
index 9a761b5bd..cc950dc5a 100644
--- a/packages/core/test/request-queue/request-queue.test.ts
+++ b/packages/core/test/request-queue/request-queue.test.ts
@@ -8,7 +8,7 @@ async function makeQueue(name: string, numOfRequestsToAdd = 0) {
const rqClient = await storage.createRequestQueueBackend({ name });
const rqInfo = await rqClient.getMetadata();
- const queue = new RequestQueue({ id: rqInfo.id, backend: rqClient });
+ const queue = new RequestQueue({ metadata: rqInfo, backend: rqClient });
if (numOfRequestsToAdd) {
await queue.addRequests(
diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts
index 71561e6c8..eb68969c7 100644
--- a/packages/fs-storage/src/file-system-storage.ts
+++ b/packages/fs-storage/src/file-system-storage.ts
@@ -100,7 +100,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return { id: options.id, name: options.name, alias: options.alias, cacheKey };
}
- async createDatasetBackend(options: storage.CreateDatasetBackendOptions = {}): Promise<storage.DatasetBackend> {
+ async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {
const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
if (cacheKey) {
@@ -127,9 +127,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return newStore;
}
- async createKeyValueStoreBackend(
- options: storage.CreateKeyValueStoreBackendOptions = {},
- ): Promise<storage.KeyValueStoreBackend> {
+ async createKeyValueStoreBackend(options: storage.StorageIdentifier = {}): Promise<storage.KeyValueStoreBackend> {
const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
if (cacheKey) {
@@ -156,9 +154,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return newStore;
}
- async createRequestQueueBackend(
- options: storage.CreateRequestQueueBackendOptions = {},
- ): Promise<storage.RequestQueueBackend> {
+ async createRequestQueueBackend(options: storage.StorageIdentifier = {}): Promise<storage.RequestQueueBackend> {
const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
if (cacheKey) {
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 084e2902f..f54792ed7 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -404,36 +404,6 @@ export type StorageIdentifier =
| { id?: never; name?: never; alias: string }
| { id?: never; name?: never; alias?: never };
-/**
- * Options for creating a dataset backend via {@apilink StorageBackend.createDatasetBackend}.
- */
-export type CreateDatasetBackendOptions = StorageIdentifier;
-
-/**
- * Options for creating a key-value store backend via {@apilink StorageBackend.createKeyValueStoreBackend}.
- */
-export type CreateKeyValueStoreBackendOptions = StorageIdentifier;
-
-/**
- * Options for creating a request queue backend via {@apilink StorageBackend.createRequestQueueBackend}.
- */
-export type CreateRequestQueueBackendOptions = StorageIdentifier & {
- /**
- * Client key for request locking.
- * TODO: This is an Apify-platform concern and should eventually be pushed down
- * into the Apify SDK's client implementation (aligning with crawlee-python).
- * https://github.com/apify/crawlee/issues/3328
- */
- clientKey?: string;
- /**
- * Timeout in seconds for request queue operations.
- * TODO: This is an Apify-platform concern and should eventually be pushed down
- * into the Apify SDK's client implementation (aligning with crawlee-python).
- * https://github.com/apify/crawlee/issues/3328
- */
- timeoutSecs?: number;
-};
-
/**
* Represents a storage backend capable of working with datasets, key-value stores and request queues.
*
@@ -453,21 +423,21 @@ export interface StorageBackend {
* If `name` is provided, opens an existing dataset with that name or creates a new one.
* If neither is provided, opens or creates the default dataset.
*/
- createDatasetBackend(options?: CreateDatasetBackendOptions): Promise<DatasetBackend>;
+ createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
/**
* Create (or open) a key-value store backend.
* If `id` is provided, opens the key-value store with that ID.
* If `name` is provided, opens an existing store with that name or creates a new one.
* If neither is provided, opens or creates the default key-value store.
*/
- createKeyValueStoreBackend(options?: CreateKeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;
+ createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
/**
* Create (or open) a request queue backend.
* If `id` is provided, opens the request queue with that ID.
* If `name` is provided, opens an existing queue with that name or creates a new one.
* If neither is provided, opens or creates the default request queue.
*/
- createRequestQueueBackend(options?: CreateRequestQueueBackendOptions): Promise<RequestQueueBackend>;
+ createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
/**
* Check whether a storage with the given ID exists.
*
diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts
index 0c7f90d25..4225c2c76 100644
--- a/test/core/storages/dataset.test.ts
+++ b/test/core/storages/dataset.test.ts
@@ -8,11 +8,6 @@ beforeEach(async () => {
});
describe('dataset', () => {
- async function createDataset(id = 'some-id', name?: string) {
- const client = await serviceLocator.getStorageBackend().createDatasetBackend(name ? { name } : { id });
- return new Dataset({ id, name, backend: client });
- }
-
beforeEach(async () => {
vitest.clearAllMocks();
});
@@ -21,7 +16,7 @@ describe('dataset', () => {
const mockData = (bytes: number) => 'x'.repeat(bytes);
test('should work', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const pushDataSpy = vitest.spyOn(dataset.backend, 'pushData');
@@ -50,7 +45,7 @@ describe('dataset', () => {
test('should successfully save large data', async () => {
const half = mockData(MAX_PAYLOAD_SIZE_BYTES / 2);
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const mockPushData = vitest.spyOn(dataset.backend, 'pushData');
mockPushData.mockResolvedValueOnce(undefined);
@@ -67,7 +62,7 @@ describe('dataset', () => {
const chunk = { foo: string, bar: 'baz' };
const data = Array(count).fill(chunk);
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const mockPushData = vitest.spyOn(dataset.backend, 'pushData');
mockPushData.mockResolvedValueOnce(undefined);
@@ -79,7 +74,7 @@ describe('dataset', () => {
});
test('getData() should work', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const expected = {
items: [{ foo: 'bar' }, { foo: 'hotel' }],
@@ -111,7 +106,7 @@ describe('dataset', () => {
});
test('getInfo() should work', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const expected: Awaited<ReturnType<Dataset['getInfo']>> = {
id: 'WkzbQMuFYuamGv3YF',
@@ -131,7 +126,7 @@ describe('dataset', () => {
});
const getRemoteDataset = async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const firstResolve = {
items: [{ foo: 'a' }, { foo: 'b' }],
@@ -289,7 +284,7 @@ describe('dataset', () => {
});
test('reduce() uses first value as memo if no memo is provided', async () => {
- const dataset = await createDataset('some-id', 'some-name');
+ const dataset = await Dataset.open();
const mockGetData = vitest.spyOn(dataset.backend, 'getData');
mockGetData.mockResolvedValueOnce({
items: [{ foo: 4 }, { foo: 5 }],
@@ -338,7 +333,7 @@ describe('dataset', () => {
describe('pushData', () => {
test('throws on invalid args', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
// @ts-expect-error JS-side validation
await expect(dataset.pushData()).rejects.toThrow(
'Expected `data` to be of type `object` but received type `undefined`',
@@ -697,12 +692,12 @@ describe('dataset', () => {
describe('stats', () => {
test('start at zero', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
expect(dataset.stats).toEqual({ readCount: 0, writeCount: 0 });
});
test('count writes and reads per client call', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
await dataset.pushData({ foo: 'bar' });
await dataset.pushData([{ foo: 'baz' }, { foo: 'qux' }]);
@@ -713,7 +708,7 @@ describe('dataset', () => {
});
test('snapshot is a copy, not a live reference', async () => {
- const dataset = await createDataset();
+ const dataset = await Dataset.open();
const before = dataset.stats;
await dataset.pushData({ foo: 'bar' });
diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts
index a10722e62..fe0896d10 100644
--- a/test/core/storages/key_value_store.test.ts
+++ b/test/core/storages/key_value_store.test.ts
@@ -9,17 +9,12 @@ beforeEach(async () => {
});
describe('KeyValueStore', () => {
- async function createKeyValueStore(id = 'some-id-1', name?: string) {
- const client = await serviceLocator.getStorageBackend().createKeyValueStoreBackend(name ? { name } : { id });
- return new KeyValueStore({ id, name, backend: client });
- }
-
beforeEach(async () => {
vitest.clearAllMocks();
});
test('should work', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
// Record definition
const record = { foo: 'bar' };
@@ -94,7 +89,7 @@ describe('KeyValueStore', () => {
describe('getValue', () => {
test('throws on invalid args', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
// @ts-expect-error JS-side validation
await expect(store.getValue()).rejects.toThrow(
@@ -129,7 +124,7 @@ describe('KeyValueStore', () => {
describe('recordExists', () => {
test('throws on invalid args', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
// @ts-expect-error JS-side validation
await expect(store.recordExists()).rejects.toThrow(
@@ -159,7 +154,7 @@ describe('KeyValueStore', () => {
describe('setValue', () => {
test('throws on invalid args', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
// @ts-expect-error JS-side validation
await expect(store.setValue()).rejects.toThrow(
@@ -235,7 +230,7 @@ describe('KeyValueStore', () => {
});
test('throws on invalid key', async () => {
- const store = await createKeyValueStore('my-store-id');
+ const store = await KeyValueStore.open();
const INVALID_CHARACTERS = '?|\\/"*<>%:';
for (const char of INVALID_CHARACTERS) {
@@ -251,7 +246,7 @@ describe('KeyValueStore', () => {
});
test('correctly adds charset to content type', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
const mockSetValue = vitest
// @ts-expect-error Accessing private property
@@ -269,7 +264,7 @@ describe('KeyValueStore', () => {
});
test('correctly passes object values as JSON', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
const record = { foo: 'bar' };
const recordStr = JSON.stringify(record, null, 2);
@@ -290,7 +285,7 @@ describe('KeyValueStore', () => {
});
test('correctly passes raw string values', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
const mockSetValue = vitest
// @ts-expect-error Accessing private property
@@ -308,7 +303,7 @@ describe('KeyValueStore', () => {
});
test('correctly passes raw Buffer values', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
const mockSetValue = vitest
// @ts-expect-error Accessing private property
@@ -327,7 +322,7 @@ describe('KeyValueStore', () => {
});
test('correctly passes a stream', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
const mockSetValue = vitest
// @ts-expect-error Accessing private property
@@ -532,7 +527,7 @@ describe('KeyValueStore', () => {
});
test('should work remotely', async () => {
- const store = await createKeyValueStore('my-store-id-1');
+ const store = await KeyValueStore.open();
// @ts-expect-error Accessing private property
const mockListKeys = vitest.spyOn(store.backend, 'listKeys');
@@ -739,12 +734,12 @@ describe('KeyValueStore', () => {
describe('stats', () => {
test('start at zero', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
expect(store.stats).toEqual({ readCount: 0, writeCount: 0, deleteCount: 0, listCount: 0 });
});
test('count reads, writes and deletes per client call', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
await store.setValue('foo', { a: 1 });
await store.setValue('bar', { b: 2 });
@@ -759,7 +754,7 @@ describe('KeyValueStore', () => {
});
test('count list operations when iterating keys', async () => {
- const store = await createKeyValueStore();
+ const store = await KeyValueStore.open();
await store.setValue('key1', { value: 1 });
await store.setValue('key2', { value: 2 });
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index 5af889943..e01b97858 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -29,13 +29,8 @@ describe('RequestQueue remote', () => {
vitest.clearAllMocks();
});
- async function createRequestQueue(id = 'some-id', name?: string) {
- const client = await serviceLocator.getStorageBackend().createRequestQueueBackend(name ? { name } : { id });
- return new RequestQueue({ id, name, backend: client }, serviceLocator.getConfiguration());
- }
-
test('adding a request makes it fetchable; fetching again returns null while in progress', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
const info = await queue.addRequest({ url: 'http://example.com/a' });
expect(info.wasAlreadyPresent).toBe(false);
@@ -51,7 +46,7 @@ describe('RequestQueue remote', () => {
});
test('adding the same uniqueKey twice does not duplicate and is served from the local cache', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
const requestA = new Request({ url: 'http://example.com/a' });
const requestB = new Request({ url: 'http://example.com/a' }); // Has the same uniqueKey as A.
@@ -81,7 +76,7 @@ describe('RequestQueue remote', () => {
});
test('a handled request is not fetched again and isFinished() becomes true', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
await queue.addRequest({ url: 'http://example.com/a' });
@@ -95,7 +90,7 @@ describe('RequestQueue remote', () => {
});
test('a reclaimed request is fetched again; reclaim with forefront returns it to the front', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
await queue.addRequest({ url: 'http://example.com/a' });
await sleep(5);
@@ -114,7 +109,7 @@ describe('RequestQueue remote', () => {
});
test('addRequests processes requests and reports processed/unprocessed', async () => {
- const queue = await createRequestQueue('batch-requests');
+ const queue = await RequestQueue.open();
const result = await queue.addRequests([{ url: 'http://example.com/a' }, { url: 'http://example.com/b' }]);
@@ -141,10 +136,7 @@ describe('RequestQueue remote', () => {
});
test('addRequestsBatched does not retry permanently unprocessed requests forever', async () => {
- const queue = new RequestQueue({
- id: 'unprocessed-requests',
- backend: { addBatchOfRequests: async () => ({ processedRequests: [], unprocessedRequests: [] }) } as any,
- });
+ const queue = await RequestQueue.open();
const mockAddRequests = vitest.spyOn(queue.backend, 'addBatchOfRequests');
const requestOptions = { url: 'http://example.com/bad' };
@@ -168,30 +160,15 @@ describe('RequestQueue remote', () => {
});
test('addRequestsBatched does not re-submit already enqueued requests beyond the initial batch (#3120)', async () => {
- const queue = new RequestQueue({
- id: 'dedup-across-batches',
- backend: { addBatchOfRequests: async () => ({ processedRequests: [], unprocessedRequests: [] }) } as any,
- });
- const mockAddRequests = vitest.spyOn(queue.backend, 'addBatchOfRequests');
+ const queue = await RequestQueue.open();
- // Fake platform: deduplicates server-side by `uniqueKey` and counts every submitted request as a write.
- const serverSeen = new Set<string>();
+ // The real memory backend already deduplicates server-side by `uniqueKey`; we only need to count
+ // how many requests are actually submitted to it (calling through to the real implementation).
let submittedCount = 0;
- mockAddRequests.mockImplementation(async (requests) => {
+ const addBatchOfRequests = queue.backend.addBatchOfRequests.bind(queue.backend);
+ vitest.spyOn(queue.backend, 'addBatchOfRequests').mockImplementation(async (requests, options) => {
submittedCount += requests.length;
- return {
- processedRequests: requests.map((r) => {
- const wasAlreadyPresent = serverSeen.has(r.uniqueKey);
- serverSeen.add(r.uniqueKey);
- return {
- requestId: `id-${r.uniqueKey}`,
- uniqueKey: r.uniqueKey,
- wasAlreadyPresent,
- wasAlreadyHandled: false,
- };
- }),
- unprocessedRequests: [],
- };
+ return addBatchOfRequests(requests, options);
});
// More requests than a single batch, so the tail is added in background batches (the buggy path).
@@ -212,7 +189,7 @@ describe('RequestQueue remote', () => {
});
test('fetchNextRequest order respects forefront enqueues', async () => {
- const queue = await createRequestQueue('forefront-order');
+ const queue = await RequestQueue.open();
// Add some non-forefront requests (sleep between adds to keep orderNo deterministic).
await queue.addRequest({ url: 'http://example.com/1' });
@@ -247,7 +224,7 @@ describe('RequestQueue remote', () => {
});
test('isEmpty() reflects fetchable requests while isFinished() accounts for in-progress ones', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
await queue.addRequest({ url: 'http://example.com/a' });
// There is a pending request, so the queue is neither empty nor finished.
@@ -268,7 +245,7 @@ describe('RequestQueue remote', () => {
});
test('should accept plain object in addRequest()', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
const requestOpts = { url: 'http://example.com/a' };
const info = await queue.addRequest(requestOpts);
@@ -285,7 +262,7 @@ describe('RequestQueue remote', () => {
});
test('should return correct handledCount', async () => {
- const queue = await createRequestQueue('id');
+ const queue = await RequestQueue.open();
const getMock = vitest.spyOn(queue.backend, 'getMetadata');
getMock.mockResolvedValueOnce({
handledRequestCount: 33,
@@ -297,7 +274,7 @@ describe('RequestQueue remote', () => {
});
test('getInfo() should work', async () => {
- const queue = await createRequestQueue('some-id', 'some-name');
+ const queue = await RequestQueue.open();
const expected = {
id: 'WkzbQMuFYuamGv3YF',
@@ -322,7 +299,7 @@ describe('RequestQueue remote', () => {
});
test('drop() works', async () => {
- const queue = await createRequestQueue('some-id', 'some-name');
+ const queue = await RequestQueue.open();
const dropMock = vitest.spyOn(queue.backend, 'drop').mockResolvedValueOnce(undefined);
await queue.drop();
@@ -374,7 +351,7 @@ describe('RequestQueue remote', () => {
describe('setExpectedRequestProcessingTimeSecs', () => {
test('forwards the value to the client, but only ever raises it', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
// The in-memory client does not implement this optional hint (it has no request locking to
// tune), so attach a stub to verify the frontend's raise-only forwarding logic in isolation.
const spy = vitest.fn();
@@ -397,12 +374,12 @@ describe('RequestQueue remote', () => {
describe('stats', () => {
test('start at zero', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
expect(queue.stats).toEqual({ writeCount: 0, headItemReadCount: 0 });
});
test('count writes on add, handle and reclaim', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
await queue.addRequest({ url: 'http://example.com/a' });
expect(queue.stats.writeCount).toBe(1);
@@ -415,7 +392,7 @@ describe('RequestQueue remote', () => {
});
test('count head reads on fetchNextRequest', async () => {
- const queue = await createRequestQueue();
+ const queue = await RequestQueue.open();
await queue.addRequest({ url: 'http://example.com/a' });