Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Rectify inconsistencies in the StorageClient interface family (#3800)

Jan Buchara57768bfad08
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index a015d9193..db0f775ff 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -363,6 +363,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
+ get stats(): DatasetStats;
values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
}
@@ -447,6 +448,12 @@ export interface DatasetReducer<T, Data> {
(memo: T, item: Data, index: number): Awaitable_2<T>;
}
+// @public
+export interface DatasetStats {
+ readCount: number;
+ writeCount: number;
+}
+
// @public
export interface DefaultStorageIdentifier {
// (undocumented)
@@ -740,7 +747,7 @@ export interface IRequestManager extends IRequestLoader {
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
purge?(): Promise<void>;
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
- setExpectedRequestProcessingTimeSecs?(secs: number): void;
+ setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
// @internal (undocumented)
@@ -796,6 +803,7 @@ export class KeyValueStore {
static recordExists(key: string): Promise<boolean>;
setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+ get stats(): KeyValueStoreStats;
values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
}
@@ -822,6 +830,14 @@ export interface KeyValueStoreRawRecord {
value: Buffer | ArrayBuffer;
}
+// @public
+export interface KeyValueStoreStats {
+ deleteCount: number;
+ listCount: number;
+ readCount: number;
+ writeCount: number;
+}
+
// @internal (undocumented)
export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
request: LoadedRequest<Context['request']>;
@@ -1213,7 +1229,7 @@ export class RequestManagerTandem implements IRequestManager {
purge(): Promise<void>;
// (undocumented)
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
- setExpectedRequestProcessingTimeSecs(secs: number): void;
+ setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
}
// @public
@@ -1294,7 +1310,8 @@ export class RequestQueue implements IStorage, IRequestManager {
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
// (undocumented)
protected requestCache: LruCache<RequestLruItem>;
- setExpectedRequestProcessingTimeSecs(secs: number): void;
+ setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+ get stats(): RequestQueueStats;
// (undocumented)
timeoutSecs: number;
}
@@ -1325,6 +1342,12 @@ export interface RequestQueueOptions {
proxyConfiguration?: ProxyConfiguration;
}
+// @public
+export interface RequestQueueStats {
+ headItemReadCount: number;
+ writeCount: number;
+}
+
// @internal (undocumented)
export const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
@@ -1886,6 +1909,13 @@ export interface StorageOpenOptions {
storageClient?: StorageClient;
}
+// @public
+export class StorageStatsTracker<T extends Record<keyof T, number>> {
+ constructor(initial: T);
+ add(key: keyof T, by?: number): void;
+ get current(): T;
+}
+
// @public
export interface SystemInfo {
// (undocumented)
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index 76819e4fe..45400a26f 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -113,10 +113,6 @@ export interface DatasetInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
- actId?: string;
- // (undocumented)
- actRunId?: string;
- // (undocumented)
createdAt: Date;
// (undocumented)
id: string;
@@ -128,18 +124,6 @@ export interface DatasetInfo {
name?: string;
}
-// @public (undocumented)
-export interface DatasetStats {
- // (undocumented)
- deleteCount?: number;
- // (undocumented)
- readCount?: number;
- // (undocumented)
- storageBytes?: number;
- // (undocumented)
- writeCount?: number;
-}
-
// @public (undocumented)
export type Dictionary<T = any> = Record<PropertyKey, T>;
@@ -230,7 +214,7 @@ export interface KeyValueStoreClient {
getMetadata(): Promise<KeyValueStoreInfo>;
getPublicUrl(key: string): Promise<string | undefined>;
getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
- listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreItemData[]>;
+ listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
purge(): Promise<void>;
recordExists(key: string): Promise<boolean>;
setValue(record: KeyValueStoreInputRecord): Promise<void>;
@@ -241,10 +225,6 @@ export interface KeyValueStoreInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
- actId?: string;
- // (undocumented)
- actRunId?: string;
- // (undocumented)
createdAt: Date;
// (undocumented)
id: string;
@@ -252,10 +232,6 @@ export interface KeyValueStoreInfo {
modifiedAt: Date;
// (undocumented)
name?: string;
- // (undocumented)
- stats?: KeyValueStoreStats;
- // (undocumented)
- userId?: string;
}
// @public
@@ -270,6 +246,7 @@ export interface KeyValueStoreInputRecord {
// @public (undocumented)
export interface KeyValueStoreItemData {
+ contentType: string;
// (undocumented)
key: string;
// (undocumented)
@@ -283,6 +260,16 @@ export interface KeyValueStoreListKeysOptions {
prefix?: string;
}
+// @public
+export interface KeyValueStoreListKeysResult {
+ count: number;
+ exclusiveStartKey?: string;
+ isTruncated: boolean;
+ items: KeyValueStoreItemData[];
+ limit: number;
+ nextExclusiveStartKey?: string;
+}
+
// @public
export interface KeyValueStoreRecord {
// (undocumented)
@@ -296,20 +283,6 @@ export interface KeyValueStoreRecord {
// @public
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
-// @public (undocumented)
-export interface KeyValueStoreStats {
- // (undocumented)
- deleteCount?: number;
- // (undocumented)
- listCount?: number;
- // (undocumented)
- readCount?: number;
- // (undocumented)
- storageBytes?: number;
- // (undocumented)
- writeCount?: number;
-}
-
// @public
export interface NewPageOptions {
id?: string;
@@ -321,7 +294,7 @@ export interface PageState {
cookies: Cookie[];
}
-// @public (undocumented)
+// @public
export interface PaginatedList<Data> {
count: number;
desc?: boolean;
@@ -366,27 +339,19 @@ export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
headers: Headers;
}) => void;
-// @public (undocumented)
-export interface RequestOptions {
- // (undocumented)
- [k: string]: unknown;
- // (undocumented)
- forefront?: boolean;
-}
-
// @public
export interface RequestQueueClient {
- addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
+ addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
drop(): Promise<void>;
- fetchNextRequest(): Promise<RequestOptions | null>;
+ fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
getMetadata(): Promise<RequestQueueInfo>;
- getRequest(uniqueKey: string): Promise<RequestOptions | undefined>;
+ getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
isEmpty(): Promise<boolean>;
isFinished(): Promise<boolean>;
- markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | null>;
+ markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
purge(): Promise<void>;
- reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo | null>;
- setExpectedRequestProcessingTimeSecs?(secs: number): void;
+ reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
+ setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
// @public (undocumented)
@@ -394,16 +359,8 @@ export interface RequestQueueInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
- actId?: string;
- // (undocumented)
- actRunId?: string;
- // (undocumented)
createdAt: Date;
// (undocumented)
- expireAt?: string;
- // (undocumented)
- hadMultipleClients?: boolean;
- // (undocumented)
handledRequestCount: number;
// (undocumented)
id: string;
@@ -414,25 +371,12 @@ export interface RequestQueueInfo {
// (undocumented)
pendingRequestCount: number;
// (undocumented)
- stats?: RequestQueueStats;
- // (undocumented)
totalRequestCount: number;
- // (undocumented)
- userId?: string;
}
-// @public (undocumented)
-export interface RequestQueueStats {
- // (undocumented)
- deleteCount?: number;
- // (undocumented)
- headItemReadCount?: number;
- // (undocumented)
- readCount?: number;
- // (undocumented)
- storageBytes?: number;
- // (undocumented)
- writeCount?: number;
+// @public
+export interface RequestQueueOperationOptions {
+ forefront?: boolean;
}
// @public (undocumented)
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index f38f1a9c8..8468bb54e 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -557,7 +557,7 @@ The sub-client interfaces (`DatasetClient`, `KeyValueStoreClient`, `RequestQueue
| `setRecord(record, options?)` | `setValue(record)` |
| `deleteRecord(key)` | `deleteValue(key)` |
| `getRecordPublicUrl(key)` | `getPublicUrl(key)` |
-| `listKeys(options?)` → `KeyValueStoreClientListData` | `listKeys(options?)` → `KeyValueStoreItemData[]` |
+| `listKeys(options?)` → `KeyValueStoreClientListData` | `listKeys(options?)` → `KeyValueStoreListKeysResult` (a single self-describing page) |
| `keys()`, `values()`, `entries()` | Removed (handled by `KeyValueStore` frontend) |
**`RequestQueueClient`:**
@@ -580,9 +580,11 @@ The request queue client was reduced from 12 methods to 10. The distributed-lock
The lifecycle is now: `fetchNextRequest()` hands out a pending request and marks it in progress; once processed, call `markRequestAsHandled(request)`; on failure call `reclaimRequest(request, { forefront? })` to return it to the queue.
+Methods that may have "nothing" to return now consistently resolve to `undefined` rather than `null`. `fetchNextRequest()` resolves to `undefined` when there is nothing to fetch, and `markRequestAsHandled()` / `reclaimRequest()` resolve to `undefined` when the request is not something the client is currently processing (a no-op, not an error). This matches the `undefined` already returned by `getRequest()`, `KeyValueStoreClient.getValue()`, and `getPublicUrl()`, so the whole client family uses a single "absent" sentinel. If you implemented a custom client that returned `null` from these methods, return `undefined` instead.
+
`RequestQueueClient.isEmpty()` and `RequestQueueClient.isFinished()` answer two different questions:
-- `isEmpty()` is the weak check — `true` when the next `fetchNextRequest()` would return `null`, i.e. there is nothing left to fetch right now. Requests that are currently in progress (fetched but not yet handled or reclaimed) are **not** counted, because they are not fetchable. This is what drives the crawler's task scheduling.
+- `isEmpty()` is the weak check — `true` when the next `fetchNextRequest()` would return `undefined`, i.e. there is nothing left to fetch right now. Requests that are currently in progress (fetched but not yet handled or reclaimed) are **not** counted, because they are not fetchable. This is what drives the crawler's task scheduling.
- `isFinished()` is the strong check — `true` only when there are no pending requests **and** no requests currently in progress (including those locked by other clients sharing the queue). This is what determines whether crawling is actually done. An in-progress request keeps the queue *empty but not finished*, which is what stops a crawler from shutting down while a request is still being processed.
The separate `RequestQueueV1`/`RequestQueueV2` classes (and the `RequestProvider` base class) have been removed. They no longer differ in behavior — request coordination is now internal to the storage client — so they are merged into a single `RequestQueue` class. Replace any `RequestQueueV1`, `RequestQueueV2`, or `RequestProvider` imports with `RequestQueue`.
@@ -613,7 +615,9 @@ queue.setExpectedRequestProcessingTimeSecs(600);
The `RequestQueue.internalTimeoutMillis` property and the associated "stuck queue" self-recovery have been removed. In v3 the `RequestQueue` frontend kept its own copy of the queue head and in-progress set, which could drift out of sync with the backing storage (an eventual-consistency hazard on the Apify platform); `isFinished()` watched for inactivity exceeding `internalTimeoutMillis` and reset that frontend state to recover. In v4 the frontend no longer holds any such bookkeeping — the storage client is the single source of truth — so there is nothing for a reset to fix, and stuck request locks now self-heal on expiry. Any consistency-recovery logic that is genuinely specific to the Apify platform's distributed storage belongs in the Apify SDK's client implementation instead, and is tracked in [apify/crawlee#3328](https://github.com/apify/crawlee/issues/3328).
-**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`.
+**Apify-specific fields removed from storage metadata.** The metadata returned by `getMetadata()` (`DatasetInfo`, `KeyValueStoreInfo`, `RequestQueueInfo`) has been trimmed to what is meaningful for any storage backend. The following platform-specific fields were dropped: `actId`, `actRunId`, `userId`, and — on `RequestQueueInfo` — `expireAt` and `hadMultipleClients`. The per-storage `stats` field (and its `DatasetStats` / `KeyValueStoreStats` / `RequestQueueStats` types) was removed as well. If you consumed any of these, read them from the Apify API client directly; a custom `StorageClient` should simply stop returning them.
+
+**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`, `DatasetStats`, `KeyValueStoreStats`, `RequestQueueStats`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`.
The high-level storage classes (`Dataset`, `KeyValueStore`, `RequestQueue`) now receive their sub-client directly in the constructor options instead of receiving a `StorageClient` and calling its methods.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 249784cd2..05a072305 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -587,6 +587,13 @@ export class BasicCrawler<
*/
private ownedRequestManager?: IRequestManager;
+ /**
+ * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
+ * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
+ * so it only needs to be applied once, at the first async access of the manager.
+ */
+ private requestManagerTimeoutsApplied = false;
+
/**
* A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
* > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
@@ -875,11 +882,6 @@ export class BasicCrawler<
this.internalTimeoutMillis =
tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
- // override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
- if (this.requestManager !== undefined) {
- this.applyRequestManagerTimeouts(this.requestManager);
- }
-
this.maxRequestRetries = maxRequestRetries;
this.maxCrawlDepth = maxCrawlDepth;
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
@@ -1455,6 +1457,14 @@ export class BasicCrawler<
this.requestManager = await this.openOwnedRequestQueue();
}
+ // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
+ // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
+ // but guard so we do not re-issue it on every call.
+ if (!this.requestManagerTimeoutsApplied) {
+ this.requestManagerTimeoutsApplied = true;
+ await this.applyRequestManagerTimeouts(this.requestManager);
+ }
+
return this.requestManager;
}
@@ -1478,7 +1488,6 @@ export class BasicCrawler<
this.crawlerInstanceIndex === 0 ? null : { alias: `__default_${this.crawlerInstanceIndex}__` };
const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
- this.applyRequestManagerTimeouts(requestQueue);
this.ownedRequestManager = requestQueue;
return requestQueue;
}
@@ -1490,8 +1499,8 @@ export class BasicCrawler<
* request from being handed out a second time while it is still being processed — and it works
* regardless of whether the manager is a plain {@apilink RequestQueue} or a `RequestManagerTandem`.
*/
- private applyRequestManagerTimeouts(requestManager: IRequestManager): void {
- requestManager.setExpectedRequestProcessingTimeSecs?.(
+ private async applyRequestManagerTimeouts(requestManager: IRequestManager): Promise<void> {
+ await requestManager.setExpectedRequestProcessingTimeSecs?.(
Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60),
);
}
diff --git a/packages/core/src/memory-storage/resource-clients/key-value-store.ts b/packages/core/src/memory-storage/resource-clients/key-value-store.ts
index 1549ecb0a..80ea89fc0 100644
--- a/packages/core/src/memory-storage/resource-clients/key-value-store.ts
+++ b/packages/core/src/memory-storage/resource-clients/key-value-store.ts
@@ -91,7 +91,7 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
this.updateTimestamps(true);
}
- async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreItemData[]> {
+ async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreListKeysResult> {
const { prefix, exclusiveStartKey, limit } = s
.object({
prefix: s.string().optional(),
@@ -107,6 +107,7 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
items.push({
key: record.key,
size,
+ contentType: record.contentType ?? 'application/octet-stream',
});
}
@@ -126,13 +127,20 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
filteredItems = filteredItems.slice(keyPos + 1);
}
- if (limit !== undefined) {
- filteredItems = filteredItems.slice(0, limit);
- }
+ const isTruncated = limit !== undefined && filteredItems.length > limit;
+ const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+ const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
this.updateTimestamps(false);
- return filteredItems;
+ return {
+ items: pageItems,
+ count: pageItems.length,
+ limit: limit ?? pageItems.length,
+ exclusiveStartKey,
+ isTruncated,
+ nextExclusiveStartKey,
+ };
}
/**
@@ -251,7 +259,6 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
accessedAt: this.accessedAt,
createdAt: this.createdAt,
modifiedAt: this.modifiedAt,
- userId: '1',
};
}
diff --git a/packages/core/src/memory-storage/resource-clients/request-queue.ts b/packages/core/src/memory-storage/resource-clients/request-queue.ts
index 581f72ac2..0a429bb06 100644
--- a/packages/core/src/memory-storage/resource-clients/request-queue.ts
+++ b/packages/core/src/memory-storage/resource-clients/request-queue.ts
@@ -43,7 +43,7 @@ export interface InternalRequest {
orderNo: number | null;
url: string;
uniqueKey: string;
- method: Exclude<storage.RequestOptions['method'], undefined>;
+ method: storage.RequestSchema['method'];
retryCount: number;
json: string;
}
@@ -219,7 +219,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
};
}
- async fetchNextRequest(): Promise<storage.RequestOptions | null> {
+ async fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined> {
this.updateTimestamps(false);
await this.queueStateMutex.wait();
@@ -230,14 +230,14 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
} = await this.listPendingHead(1);
if (!head) {
- return null;
+ return undefined;
}
// Mark the request as in progress so it is not handed out again until it is handled or
// reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
this.inProgressRequestIds.add(head.id);
- return this._jsonToRequest(head.json) ?? null;
+ return this._jsonToRequest(head.json) ?? undefined;
} finally {
this.queueStateMutex.shift();
}
@@ -245,7 +245,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
async addBatchOfRequests(
requests: storage.RequestSchema[],
- options: storage.RequestOptions = {},
+ options: storage.RequestQueueOperationOptions = {},
): Promise<storage.BatchAddRequestsResult> {
batchRequestShapeWithoutId.parse(requests);
requestOptionsShape.parse(options);
@@ -307,7 +307,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}
}
- async getRequest(uniqueKey: string): Promise<storage.RequestOptions | undefined> {
+ async getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined> {
s.string().parse(uniqueKey);
this.updateTimestamps(false);
const id = uniqueKeyToRequestId(uniqueKey);
@@ -315,7 +315,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
return this._jsonToRequest(json);
}
- async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | null> {
+ async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {
requestShape.parse(request);
this.updateTimestamps(false);
@@ -333,7 +333,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// be in progress: marking an already-released request handled must still succeed, otherwise
// the request could be handed out again and the queue would never finish.
if (!existingRequest) {
- return null;
+ return undefined;
}
// A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
@@ -366,8 +366,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
async reclaimRequest(
request: storage.UpdateRequestSchema,
- options: storage.RequestOptions = {},
- ): Promise<storage.QueueOperationInfo | null> {
+ options: storage.RequestQueueOperationOptions = {},
+ ): Promise<storage.QueueOperationInfo | undefined> {
requestShape.parse(request);
requestOptionsShape.parse(options);
this.updateTimestamps(false);
@@ -387,7 +387,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// already-released request to the queue (e.g. to honor a `forefront` reorder) must still
// work, rather than have the reclaim silently dropped.
if (!existingRequest || existingRequest.orderNo === null) {
- return null;
+ return undefined;
}
// Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
@@ -462,15 +462,12 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
return {
accessedAt: this.accessedAt,
createdAt: this.createdAt,
- hadMultipleClients: false,
handledRequestCount: this.handledRequestCount,
id: this.id,
modifiedAt: this.modifiedAt,
name: this.name,
pendingRequestCount: this.pendingRequestCount,
- stats: {},
totalRequestCount: this.requests.size,
- userId: '1',
};
}
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index 3f04d0bc7..1396cc162 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -8,6 +8,8 @@ import { serviceLocator } from '../service_locator.js';
import type { Awaitable } from '../typedefs.js';
import { checkStorageAccess } from './access_checking.js';
import { KeyValueStore } from './key_value_store.js';
+import type { DatasetStats } from './storage_stats.js';
+import { StorageStatsTracker } from './storage_stats.js';
import type { StorageIdentifier } from './storage_instance_manager.js';
import type { StorageOpenOptions } from './utils.js';
import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -188,6 +190,11 @@ export class Dataset<Data extends Dictionary = Dictionary> {
client: DatasetClient<Data>;
log: CrawleeLogger;
+ private readonly statsTracker = new StorageStatsTracker<DatasetStats>({
+ readCount: 0,
+ writeCount: 0,
+ });
+
/**
* @internal
*/
@@ -201,6 +208,14 @@ export class Dataset<Data extends Dictionary = Dictionary> {
this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
}
+ /**
+ * Backend-independent usage counters tracked for this dataset (read / write operations issued to
+ * the underlying storage client). Counted per client call.
+ */
+ get stats(): DatasetStats {
+ return this.statsTracker.current;
+ }
+
/**
* Stores an object or an array of objects to the dataset.
* The function returns a promise that resolves when the operation finishes.
@@ -223,6 +238,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
assertJsonSerializable(items[i], i);
}
+ this.statsTracker.add('writeCount');
await this.client.pushData(items);
}
@@ -233,6 +249,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
checkStorageAccess();
try {
+ this.statsTracker.add('readCount');
return await this.client.getData(options);
} catch (e) {
const error = e as Error;
@@ -537,6 +554,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
if (fetchLimit <= 0) break;
+ this.statsTracker.add('readCount');
const page = await this.client.getData({ ...options, offset, limit: fetchLimit });
yield page;
diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts
index 04bc1130d..7d596e062 100644
--- a/packages/core/src/storages/index.ts
+++ b/packages/core/src/storages/index.ts
@@ -6,6 +6,7 @@ export type * from './request_loader.js';
export type * from './request_manager.js';
export * from './request_queue.js';
export * from './storage_instance_manager.js';
+export * from './storage_stats.js';
export * from './utils.js';
export * from './access_checking.js';
export * from './sitemap_request_loader.js';
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index c677ae3bd..596e24aa8 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -12,6 +12,8 @@ import { serviceLocator } from '../service_locator.js';
import type { Awaitable } from '../typedefs.js';
import { checkStorageAccess } from './access_checking.js';
import { parseValue, serializeValue } from './key_value_store_codec.js';
+import type { KeyValueStoreStats } from './storage_stats.js';
+import { StorageStatsTracker } from './storage_stats.js';
import type { StorageIdentifier } from './storage_instance_manager.js';
import type { StorageOpenOptions } from './utils.js';
import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -86,6 +88,13 @@ export class KeyValueStore {
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
private readonly cache = new Map<string, Dictionary>();
+ private readonly statsTracker = new StorageStatsTracker<KeyValueStoreStats>({
+ readCount: 0,
+ writeCount: 0,
+ deleteCount: 0,
+ listCount: 0,
+ });
+
/**
* @internal
*/
@@ -98,6 +107,14 @@ export class KeyValueStore {
this.client = options.client;
}
+ /**
+ * Backend-independent usage counters tracked for this key-value store (read / write / delete /
+ * list operations issued to the underlying storage client). Counted per client call.
+ */
+ get stats(): KeyValueStoreStats {
+ return this.statsTracker.current;
+ }
+
/**
* Gets a value from the key-value store.
*
@@ -198,6 +215,7 @@ export class KeyValueStore {
checkStorageAccess();
ow(key, ow.string.nonEmpty);
+ this.statsTracker.add('readCount');
const record = await this.client.getValue(key);
// A missing record falls back to the default; a record that parses to a falsy value (including
@@ -240,6 +258,7 @@ export class KeyValueStore {
checkStorageAccess();
ow(key, ow.string.nonEmpty);
+ this.statsTracker.add('readCount');
const record = await this.client.getValue(key);
if (!record) return null;
@@ -314,6 +333,7 @@ export class KeyValueStore {
for await (const page of this.fetchKeyPages(options)) {
const results: T[] = [];
for (const item of page) {
+ this.statsTracker.add('readCount');
const record = await this.client.getValue(item.key);
if (record) {
const parsed = parseValue(record.value, record.contentType ?? null);
@@ -331,10 +351,15 @@ export class KeyValueStore {
let exclusiveStartKey: string | undefined;
while (true) {
- const items = await this.client.listKeys({ ...options, exclusiveStartKey, limit });
+ this.statsTracker.add('listCount');
+ const { items, isTruncated, nextExclusiveStartKey } = await this.client.listKeys({
+ ...options,
+ exclusiveStartKey,
+ limit,
+ });
yield items;
- if (items.length < limit) break;
- exclusiveStartKey = items[items.length - 1].key;
+ if (!isTruncated) break;
+ exclusiveStartKey = nextExclusiveStartKey;
}
}
@@ -426,10 +451,14 @@ export class KeyValueStore {
}
// In this case delete the record.
- if (value === null) return this.client.deleteValue(key);
+ if (value === null) {
+ this.statsTracker.add('deleteCount');
+ return this.client.deleteValue(key);
+ }
const serialized = serializeValue(value, optionsCopy.contentType);
+ this.statsTracker.add('writeCount');
return this.client.setValue({
key,
value: serialized.value,
diff --git a/packages/core/src/storages/request_manager.ts b/packages/core/src/storages/request_manager.ts
index 63095d453..ca858db73 100644
--- a/packages/core/src/storages/request_manager.ts
+++ b/packages/core/src/storages/request_manager.ts
@@ -40,5 +40,5 @@ export interface IRequestManager extends IRequestLoader {
* the same request out again while it is still being processed. Implementations that do not need
* this hint may leave it `undefined`.
*/
- setExpectedRequestProcessingTimeSecs?(secs: number): void;
+ setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.ts
index 014f2dc5b..9ba885368 100644
--- a/packages/core/src/storages/request_manager_tandem.ts
+++ b/packages/core/src/storages/request_manager_tandem.ts
@@ -57,7 +57,9 @@ export class RequestManagerTandem implements IRequestManager {
// Apply any hint received before the manager was resolved.
if (this.expectedRequestProcessingSecs !== undefined) {
- this.resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.expectedRequestProcessingSecs);
+ await this.resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(
+ this.expectedRequestProcessingSecs,
+ );
}
}
return this.resolvedRequestManager;
@@ -238,8 +240,8 @@ export class RequestManagerTandem implements IRequestManager {
* reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.
* @inheritdoc
*/
- setExpectedRequestProcessingTimeSecs(secs: number): void {
+ async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {
this.expectedRequestProcessingSecs = secs;
- this.resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
+ await this.resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
}
}
diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts
index 783fd5189..461249eb7 100644
--- a/packages/core/src/storages/request_queue.ts
+++ b/packages/core/src/storages/request_queue.ts
@@ -35,6 +35,8 @@ import { Request } from '../request.js';
import { serviceLocator } from '../service_locator.js';
import { checkStorageAccess } from './access_checking.js';
import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { RequestQueueStats } from './storage_stats.js';
+import { StorageStatsTracker } from './storage_stats.js';
import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
import type { StorageOpenOptions } from './utils.js';
import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -115,6 +117,19 @@ export class RequestQueue implements IStorage, IRequestManager {
protected readonly events: EventManager;
+ private readonly statsTracker = new StorageStatsTracker<RequestQueueStats>({
+ writeCount: 0,
+ headItemReadCount: 0,
+ });
+
+ /**
+ * Backend-independent usage counters tracked for this request queue (write operations and
+ * queue-head reads issued to the underlying storage client). Counted per client call.
+ */
+ get stats(): RequestQueueStats {
+ return this.statsTracker.current;
+ }
+
/**
* @internal
*/
@@ -220,6 +235,7 @@ export class RequestQueue implements IStorage, IRequestManager {
};
}
+ this.statsTracker.add('writeCount');
const { processedRequests } = await this.client.addBatchOfRequests([request], { forefront });
const queueOperationInfo = {
...processedRequests[0],
@@ -326,6 +342,7 @@ export class RequestQueue implements IStorage, IRequestManager {
return results;
}
+ this.statsTracker.add('writeCount');
const apiResults = await this.client.addBatchOfRequests([...requestsToAdd.values()], { forefront });
// Report unprocessed requests
@@ -590,6 +607,7 @@ export class RequestQueue implements IStorage, IRequestManager {
return null;
}
+ this.statsTracker.add('headItemReadCount');
const requestOptions = await this.client.fetchNextRequest();
if (!requestOptions) return null;
@@ -617,6 +635,7 @@ export class RequestQueue implements IStorage, IRequestManager {
const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
const handledAt = request.handledAt ?? new Date().toISOString();
+ this.statsTracker.add('writeCount');
const processedRequest = await this.client.markRequestAsHandled({
...request,
handledAt,
@@ -668,6 +687,7 @@ export class RequestQueue implements IStorage, IRequestManager {
const { forefront = false } = options;
+ this.statsTracker.add('writeCount');
const processedRequest = await this.client.reclaimRequest(request, { forefront });
// The request was not in progress — nothing to reclaim.
@@ -730,13 +750,13 @@ export class RequestQueue implements IStorage, IRequestManager {
* ever raise the reservation duration, never lower it — otherwise a short-lived consumer could cut
* short the reservation of a long-lived one and have its in-flight request stolen.
*/
- setExpectedRequestProcessingTimeSecs(secs: number): void {
+ async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {
if (secs <= this.expectedRequestProcessingSecs) {
return;
}
this.expectedRequestProcessingSecs = secs;
- this.client.setExpectedRequestProcessingTimeSecs?.(secs);
+ await this.client.setExpectedRequestProcessingTimeSecs?.(secs);
}
/**
diff --git a/packages/core/src/storages/storage_stats.ts b/packages/core/src/storages/storage_stats.ts
new file mode 100644
index 000000000..1ecc29753
--- /dev/null
+++ b/packages/core/src/storages/storage_stats.ts
@@ -0,0 +1,61 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@apilink Dataset}, {@apilink KeyValueStore}, {@apilink RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage client, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+
+/** Usage counters for a {@apilink Dataset}. */
+export interface DatasetStats {
+ /** Number of read operations issued to the dataset client (e.g. `getData`). */
+ readCount: number;
+ /** Number of write operations issued to the dataset client (e.g. `pushData`). */
+ writeCount: number;
+}
+
+/** Usage counters for a {@apilink KeyValueStore}. */
+export interface KeyValueStoreStats {
+ /** Number of read operations issued to the key-value store client (e.g. `getValue`). */
+ readCount: number;
+ /** Number of write operations issued to the key-value store client (e.g. `setValue`). */
+ writeCount: number;
+ /** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
+ deleteCount: number;
+ /** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
+ listCount: number;
+}
+
+/** Usage counters for a {@apilink RequestQueue}. */
+export interface RequestQueueStats {
+ /** Number of write operations issued to the request queue client (add / handle / reclaim). */
+ writeCount: number;
+ /** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
+ headItemReadCount: number;
+}
+
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export class StorageStatsTracker<T extends Record<keyof T, number>> {
+ private readonly counters: T;
+
+ constructor(initial: T) {
+ this.counters = { ...initial };
+ }
+
+ /** Increment a counter bucket by `by` (default `1`). */
+ add(key: keyof T, by = 1): void {
+ (this.counters[key] as number) += by;
+ }
+
+ /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+ get current(): T {
+ return { ...this.counters };
+ }
+}
diff --git a/packages/core/test/memory-storage/async-iteration.test.ts b/packages/core/test/memory-storage/async-iteration.test.ts
index fcf6323b5..ca012ffca 100644
--- a/packages/core/test/memory-storage/async-iteration.test.ts
+++ b/packages/core/test/memory-storage/async-iteration.test.ts
@@ -71,7 +71,7 @@ describe('Async iteration support', () => {
});
test('returns all keys', async () => {
- const items = await kvStore.listKeys();
+ const { items } = await kvStore.listKeys();
expect(items).toHaveLength(25);
expect(items.map((i) => i.key)).toStrictEqual(keys);
@@ -79,28 +79,28 @@ describe('Async iteration support', () => {
test('respects prefix option', async () => {
// Only keys starting with 'key-0' (key-00 to key-09)
- const items = await kvStore.listKeys({ prefix: 'key-0' });
+ const { items } = await kvStore.listKeys({ prefix: 'key-0' });
expect(items).toHaveLength(10);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 10));
});
test('respects exclusiveStartKey option', async () => {
- const items = await kvStore.listKeys({ exclusiveStartKey: 'key-09' });
+ const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-09' });
expect(items).toHaveLength(15);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(10));
});
test('respects limit option', async () => {
- const items = await kvStore.listKeys({ limit: 5 });
+ const { items } = await kvStore.listKeys({ limit: 5 });
expect(items).toHaveLength(5);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 5));
});
test('respects exclusiveStartKey and limit together', async () => {
- const items = await kvStore.listKeys({ exclusiveStartKey: 'key-04', limit: 5 });
+ const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-04', limit: 5 });
expect(items).toHaveLength(5);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(5, 10));
diff --git a/packages/core/test/memory-storage/key-value-store/purge.test.ts b/packages/core/test/memory-storage/key-value-store/purge.test.ts
index 23de7850d..c9cfe0fb0 100644
--- a/packages/core/test/memory-storage/key-value-store/purge.test.ts
+++ b/packages/core/test/memory-storage/key-value-store/purge.test.ts
@@ -25,7 +25,7 @@ describe('MemoryStorageClient.purge preserves the default key-value store input'
// ...while every other record is removed.
expect(await store.getValue('some-other-key')).toBeUndefined();
- const keys = await store.listKeys();
+ const { items: keys } = await store.listKeys();
expect(keys.map((item) => item.key)).toEqual(['INPUT']);
});
diff --git a/packages/core/test/memory-storage/request-queue/forefront.test.ts b/packages/core/test/memory-storage/request-queue/forefront.test.ts
index c17ef0c1f..7ab2f861b 100644
--- a/packages/core/test/memory-storage/request-queue/forefront.test.ts
+++ b/packages/core/test/memory-storage/request-queue/forefront.test.ts
@@ -12,7 +12,7 @@ import type { RequestQueueClient } from '@crawlee/types';
async function fetchOrder(client: RequestQueueClient): Promise<string[]> {
const order: string[] = [];
- for (let request = await client.fetchNextRequest(); request !== null; request = await client.fetchNextRequest()) {
+ for (let request = await client.fetchNextRequest(); request != null; request = await client.fetchNextRequest()) {
order.push(new URL(request.url).pathname);
await client.markRequestAsHandled({ ...request, id: request.id! });
}
@@ -124,7 +124,7 @@ describe('RequestQueue respects `forefront` when fetching requests', () => {
await fetchOrder(requestQueue);
expect(await requestQueue.isEmpty()).toBe(true);
- expect(await requestQueue.fetchNextRequest()).toBeNull();
+ expect(await requestQueue.fetchNextRequest()).toBeUndefined();
});
test('a fetched (locked) request leaves the queue empty but unfinished until it is handled', async () => {
@@ -167,7 +167,7 @@ describe('RequestQueue holds fetched requests in progress', () => {
// While in progress, the request is not handed out again. The in-memory queue lives in a single
// process, so there is no lock expiry — the request stays in progress until it is explicitly
// reclaimed (or handled).
- expect(await requestQueue.fetchNextRequest()).toBeNull();
+ expect(await requestQueue.fetchNextRequest()).toBeUndefined();
await requestQueue.reclaimRequest({ ...first!, id: first!.id as string });
diff --git a/packages/core/test/memory-storage/request-queue/in-progress.test.ts b/packages/core/test/memory-storage/request-queue/in-progress.test.ts
index 498a3cd9c..9d2eb5cd9 100644
--- a/packages/core/test/memory-storage/request-queue/in-progress.test.ts
+++ b/packages/core/test/memory-storage/request-queue/in-progress.test.ts
@@ -15,7 +15,7 @@ describe('RequestQueue in-progress requests', () => {
// queue is empty — but the in-progress request means it is not yet finished. Unlike the previous
// disk-backed implementation, there is no lock expiry: the request never becomes fetchable again
// on its own.
- expect(await queue.fetchNextRequest()).toBeNull();
+ expect(await queue.fetchNextRequest()).toBeUndefined();
expect(await queue.isEmpty()).toBe(true);
expect(await queue.isFinished()).toBe(false);
});
@@ -55,7 +55,7 @@ describe('RequestQueue in-progress requests', () => {
expect(metadata.pendingRequestCount).toBe(0);
expect(await queue.isEmpty()).toBe(true);
expect(await queue.isFinished()).toBe(true);
- expect(await queue.fetchNextRequest()).toBeNull();
+ expect(await queue.fetchNextRequest()).toBeUndefined();
});
test('multiple requests are each handed out only once while in progress', async () => {
@@ -76,7 +76,7 @@ describe('RequestQueue in-progress requests', () => {
expect(first!.uniqueKey).not.toBe(second!.uniqueKey);
// Both are now in progress, so nothing more is fetchable.
- expect(await queue.fetchNextRequest()).toBeNull();
+ expect(await queue.fetchNextRequest()).toBeUndefined();
expect(await queue.isFinished()).toBe(false);
});
@@ -94,6 +94,6 @@ describe('RequestQueue in-progress requests', () => {
// A head scan on the dropped client must not throw and must report an empty, finished queue.
await expect(queue.isEmpty()).resolves.toBe(true);
await expect(queue.isFinished()).resolves.toBe(true);
- await expect(queue.fetchNextRequest()).resolves.toBeNull();
+ await expect(queue.fetchNextRequest()).resolves.toBeUndefined();
});
});
diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.ts
index 6fd58ac17..1b3c07d8b 100644
--- a/packages/fs-storage/src/resource-clients/key-value-store.ts
+++ b/packages/fs-storage/src/resource-clients/key-value-store.ts
@@ -86,7 +86,7 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
this.updateTimestamps(true);
}
- async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreItemData[]> {
+ async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreListKeysResult> {
const { prefix, exclusiveStartKey, limit } = s
.object({
prefix: s.string().optional(),
@@ -104,6 +104,7 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
items.push({
key: record.key,
size,
+ contentType: record.contentType ?? (mime.contentType(record.extension) || 'application/octet-stream'),
});
}
@@ -123,13 +124,20 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
filteredItems = filteredItems.slice(keyPos + 1);
}
- if (limit !== undefined) {
- filteredItems = filteredItems.slice(0, limit);
- }
+ const isTruncated = limit !== undefined && filteredItems.length > limit;
+ const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+ const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
this.updateTimestamps(false);
- return filteredItems;
+ return {
+ items: pageItems,
+ count: pageItems.length,
+ limit: limit ?? pageItems.length,
+ exclusiveStartKey,
+ isTruncated,
+ nextExclusiveStartKey,
+ };
}
/**
@@ -262,7 +270,6 @@ export class KeyValueStoreClient extends BaseClient implements storage.KeyValueS
accessedAt: this.accessedAt,
createdAt: this.createdAt,
modifiedAt: this.modifiedAt,
- userId: '1',
};
}
diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.ts
index de63231d2..e67c2fac1 100644
--- a/packages/fs-storage/src/resource-clients/request-queue.ts
+++ b/packages/fs-storage/src/resource-clients/request-queue.ts
@@ -50,7 +50,7 @@ export interface InternalRequest {
orderNo: number | null;
url: string;
uniqueKey: string;
- method: Exclude<storage.RequestOptions['method'], undefined>;
+ method: storage.RequestSchema['method'];
retryCount: number;
json: string;
}
@@ -125,7 +125,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
* caller (the `RequestQueue` frontend) owns the policy of what this value should be — this method
* just applies it.
*/
- setExpectedRequestProcessingTimeSecs(secs: number): void {
+ async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {
this.lockSecs = secs;
}
@@ -292,7 +292,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
};
}
- async fetchNextRequest(): Promise<storage.RequestOptions | null> {
+ async fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined> {
this.updateTimestamps(false);
await this.queueStateMutex.wait();
@@ -303,7 +303,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
} = await this.listPendingHead(1);
if (!head) {
- return null;
+ return undefined;
}
// Lock the request by pushing its `orderNo` beyond the lock expiry, preserving the sign so
@@ -317,7 +317,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// (see `releaseOwnLocks`) instead of leaving the request stuck until the lock expires.
this.inProgressRequestIds.add(head.id);
- return this._jsonToRequest(head.json) ?? null;
+ return this._jsonToRequest(head.json) ?? undefined;
} finally {
this.queueStateMutex.shift();
}
@@ -325,7 +325,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
async addBatchOfRequests(
requests: storage.RequestSchema[],
- options: storage.RequestOptions = {},
+ options: storage.RequestQueueOperationOptions = {},
): Promise<storage.BatchAddRequestsResult> {
batchRequestShapeWithoutId.parse(requests);
requestOptionsShape.parse(options);
@@ -396,7 +396,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}
}
- async getRequest(uniqueKey: string): Promise<storage.RequestOptions | undefined> {
+ async getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined> {
s.string().parse(uniqueKey);
this.updateTimestamps(false);
const id = uniqueKeyToRequestId(uniqueKey);
@@ -404,7 +404,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
return this._jsonToRequest(json);
}
- async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | null> {
+ async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {
requestShape.parse(request);
this.updateTimestamps(false);
@@ -424,7 +424,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// a low `setExpectedRequestProcessingTimeSecs`) must still be able to mark its request handled,
// otherwise the request would be handed out again forever and the queue would never finish.
if (!existingRequest) {
- return null;
+ return undefined;
}
// A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
@@ -462,8 +462,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
async reclaimRequest(
request: storage.UpdateRequestSchema,
- options: storage.RequestOptions = {},
- ): Promise<storage.QueueOperationInfo | null> {
+ options: storage.RequestQueueOperationOptions = {},
+ ): Promise<storage.QueueOperationInfo | undefined> {
requestShape.parse(request);
requestOptionsShape.parse(options);
this.updateTimestamps(false);
@@ -484,7 +484,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// after its lock expired must still be able to return the request to the queue (e.g. to honor
// a `forefront` reorder), rather than have the reclaim silently dropped.
if (!existingRequest || existingRequest.orderNo === null) {
- return null;
+ return undefined;
}
// Reclaiming resets the `orderNo` to a fresh timestamp, releasing the lock and restoring the
@@ -609,15 +609,12 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
return {
accessedAt: this.accessedAt,
createdAt: this.createdAt,
- hadMultipleClients: false,
handledRequestCount: this.handledRequestCount,
id: this.id,
modifiedAt: this.modifiedAt,
name: this.name,
pendingRequestCount: this.pendingRequestCount,
- stats: {},
totalRequestCount: this.requests.size,
- userId: '1',
};
}
diff --git a/packages/fs-storage/test/async-iteration.test.ts b/packages/fs-storage/test/async-iteration.test.ts
index 616427f24..7cd551641 100644
--- a/packages/fs-storage/test/async-iteration.test.ts
+++ b/packages/fs-storage/test/async-iteration.test.ts
@@ -81,7 +81,7 @@ describe('Async iteration support', () => {
});
test('returns all keys', async () => {
- const items = await kvStore.listKeys();
+ const { items } = await kvStore.listKeys();
expect(items).toHaveLength(25);
expect(items.map((i) => i.key)).toStrictEqual(keys);
@@ -89,28 +89,28 @@ describe('Async iteration support', () => {
test('respects prefix option', async () => {
// Only keys starting with 'key-0' (key-00 to key-09)
- const items = await kvStore.listKeys({ prefix: 'key-0' });
+ const { items } = await kvStore.listKeys({ prefix: 'key-0' });
expect(items).toHaveLength(10);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 10));
});
test('respects exclusiveStartKey option', async () => {
- const items = await kvStore.listKeys({ exclusiveStartKey: 'key-09' });
+ const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-09' });
expect(items).toHaveLength(15);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(10));
});
test('respects limit option', async () => {
- const items = await kvStore.listKeys({ limit: 5 });
+ const { items } = await kvStore.listKeys({ limit: 5 });
expect(items).toHaveLength(5);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(0, 5));
});
test('respects exclusiveStartKey and limit together', async () => {
- const items = await kvStore.listKeys({ exclusiveStartKey: 'key-04', limit: 5 });
+ const { items } = await kvStore.listKeys({ exclusiveStartKey: 'key-04', limit: 5 });
expect(items).toHaveLength(5);
expect(items.map((i) => i.key)).toStrictEqual(keys.slice(5, 10));
diff --git a/packages/fs-storage/test/request-queue/dangling-forefront-on-reload.test.ts b/packages/fs-storage/test/request-queue/dangling-forefront-on-reload.test.ts
index 0dec65342..032fb799e 100644
--- a/packages/fs-storage/test/request-queue/dangling-forefront-on-reload.test.ts
+++ b/packages/fs-storage/test/request-queue/dangling-forefront-on-reload.test.ts
@@ -64,7 +64,7 @@ describe('Request queue reload drops dangling forefront ids without a backing re
// None of these head scans must throw, and an empty/finished queue must be reported.
await expect(queue.isEmpty()).resolves.toBe(true);
await expect(queue.isFinished()).resolves.toBe(true);
- await expect(queue.fetchNextRequest()).resolves.toBeNull();
+ await expect(queue.fetchNextRequest()).resolves.toBeUndefined();
});
test('a valid forefront request is still served while a dangling sibling id is dropped', async () => {
@@ -81,6 +81,6 @@ describe('Request queue reload drops dangling forefront ids without a backing re
expect(first).not.toBeNull();
expect(first!.url).toEqual('http://example.com');
- expect(await queue.fetchNextRequest()).toBeNull();
+ expect(await queue.fetchNextRequest()).toBeUndefined();
});
});
diff --git a/packages/fs-storage/test/request-queue/forefront.test.ts b/packages/fs-storage/test/request-queue/forefront.test.ts
index b95ed944d..4608f6477 100644
--- a/packages/fs-storage/test/request-queue/forefront.test.ts
+++ b/packages/fs-storage/test/request-queue/forefront.test.ts
@@ -12,7 +12,7 @@ import type { RequestQueueClient } from '@crawlee/types';
async function fetchOrder(client: RequestQueueClient): Promise<string[]> {
const order: string[] = [];
- for (let request = await client.fetchNextRequest(); request !== null; request = await client.fetchNextRequest()) {
+ for (let request = await client.fetchNextRequest(); request != null; request = await client.fetchNextRequest()) {
order.push(new URL(request.url).pathname);
await client.markRequestAsHandled({ ...request, id: request.id! });
}
@@ -124,7 +124,7 @@ describe('RequestQueue respects `forefront` when fetching requests', () => {
await fetchOrder(requestQueue);
expect(await requestQueue.isEmpty()).toBe(true);
- expect(await requestQueue.fetchNextRequest()).toBeNull();
+ expect(await requestQueue.fetchNextRequest()).toBeUndefined();
});
test('a fetched (locked) request leaves the queue empty but unfinished until it is handled', async () => {
@@ -168,7 +168,7 @@ describe('RequestQueue locks fetched requests', () => {
expect(first!.uniqueKey).toBe('1');
// While locked, the request is not handed out again.
- expect(await requestQueue.fetchNextRequest()).toBeNull();
+ expect(await requestQueue.fetchNextRequest()).toBeUndefined();
// After the lock expires (default 3 minutes), the request is fetchable again — this is what
// prevents a crashed consumer from blocking its requests forever.
@@ -210,8 +210,8 @@ describe('RequestQueue locking is visible across clients sharing on-disk storage
expect(fromA!.uniqueKey).not.toBe(fromB!.uniqueKey);
// Both requests are now locked, so neither client can fetch anything more.
- expect(await clientA.fetchNextRequest()).toBeNull();
- expect(await clientB.fetchNextRequest()).toBeNull();
+ expect(await clientA.fetchNextRequest()).toBeUndefined();
+ expect(await clientB.fetchNextRequest()).toBeUndefined();
await clientA.drop();
});
@@ -227,7 +227,7 @@ describe('RequestQueue locking is visible across clients sharing on-disk storage
expect(fromA).not.toBeNull();
// Client B has nothing it can fetch right now, so from its point of view the queue is empty...
- expect(await clientB.fetchNextRequest()).toBeNull();
+ expect(await clientB.fetchNextRequest()).toBeUndefined();
expect(await clientB.isEmpty()).toBe(true);
// ...but the request still exists and is merely locked by A, so B must NOT consider the queue
// finished — otherwise the crawler driving B could shut down while A is still processing.
@@ -251,7 +251,7 @@ describe('RequestQueue locking is visible across clients sharing on-disk storage
const clientB = await storageB.createRequestQueueClient({ name: 'shared-teardown' });
// While A holds the lock, B cannot fetch the request.
- expect(await clientB.fetchNextRequest()).toBeNull();
+ expect(await clientB.fetchNextRequest()).toBeUndefined();
// Tearing down A's storage releases its locks (instead of leaving them until the 3-minute
// expiry), so B can pick the request up right away.
diff --git a/packages/fs-storage/test/request-queue/ignore-non-json-files.test.ts b/packages/fs-storage/test/request-queue/ignore-non-json-files.test.ts
index 2525b025a..bae333e81 100644
--- a/packages/fs-storage/test/request-queue/ignore-non-json-files.test.ts
+++ b/packages/fs-storage/test/request-queue/ignore-non-json-files.test.ts
@@ -64,6 +64,6 @@ describe('when falling back to fs, Request queue should ignore non-JSON files',
expect(first!.url).toEqual('http://example.com');
const second = await defaultQueue.fetchNextRequest();
- expect(second).toBeNull();
+ expect(second).toBeUndefined();
});
});
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 21e9b3bf1..ff98ffc1e 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -15,6 +15,13 @@ export interface QueueOperationInfo {
requestId: string;
}
+/**
+ * A single page of items returned by {@link DatasetClient.getData}.
+ *
+ * Datasets paginate by offset, so a page is self-describing via `total` / `offset` / `limit`: a
+ * frontend assembling all pages knows it has reached the end once `offset + items.length >= total`.
+ * The cursor-based counterpart for key-value stores is {@link KeyValueStoreListKeysResult}.
+ */
export interface PaginatedList<Data> {
/** Total count of entries in the dataset. */
total: number;
@@ -43,14 +50,6 @@ export interface DatasetInfo {
modifiedAt: Date;
accessedAt: Date;
itemCount: number;
- actId?: string;
- actRunId?: string;
-}
-export interface DatasetStats {
- readCount?: number;
- writeCount?: number;
- deleteCount?: number;
- storageBytes?: number;
}
export interface DatasetClient<Data extends Dictionary = Dictionary> {
@@ -76,24 +75,12 @@ export interface DatasetClient<Data extends Dictionary = Dictionary> {
getData(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
}
-export interface KeyValueStoreStats {
- readCount?: number;
- writeCount?: number;
- deleteCount?: number;
- listCount?: number;
- storageBytes?: number;
-}
-
export interface KeyValueStoreInfo {
id: string;
name?: string;
- userId?: string;
createdAt: Date;
modifiedAt: Date;
accessedAt: Date;
- actId?: string;
- actRunId?: string;
- stats?: KeyValueStoreStats;
}
/**
@@ -146,6 +133,33 @@ export interface KeyValueStoreListKeysOptions {
export interface KeyValueStoreItemData {
key: string;
size: number;
+ /** The MIME content type the record was stored with, if known. */
+ contentType: string;
+}
+
+/**
+ * A single page of keys returned by {@link KeyValueStoreClient.listKeys}.
+ *
+ * This mirrors {@link PaginatedList} (the shape returned by {@link DatasetClient.getData}) so that
+ * both listing operations on the storage-client layer return a self-describing page. The difference
+ * is the pagination model: datasets are offset-based (`total` / `offset`), whereas key-value stores
+ * are cursor-based. A frontend assembling all pages should therefore not guess "is this the last
+ * page?" from `items.length < limit` — it should rely on {@link isTruncated} and resume from
+ * {@link nextExclusiveStartKey}.
+ */
+export interface KeyValueStoreListKeysResult {
+ /** Keys returned on this page. */
+ items: KeyValueStoreItemData[];
+ /** Number of keys returned on this page (`items.length`). */
+ count: number;
+ /** Maximum number of keys requested for this page. */
+ limit: number;
+ /** The `exclusiveStartKey` that produced this page, if any. */
+ exclusiveStartKey?: string;
+ /** `true` if there are more keys beyond this page. When `true`, {@link nextExclusiveStartKey} is set. */
+ isTruncated: boolean;
+ /** Cursor to pass as the next call's `exclusiveStartKey`, or `undefined` when {@link isTruncated} is `false`. */
+ nextExclusiveStartKey?: string;
}
/**
@@ -181,8 +195,16 @@ export interface KeyValueStoreClient {
/** Delete a record by key. */
deleteValue(key: string): Promise<void>;
- /** List keys in the store. Returns at most `limit` keys starting after `exclusiveStartKey`. */
- listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreItemData[]>;
+ /**
+ * List a single page of keys in the store. Returns at most `limit` keys starting after
+ * `exclusiveStartKey`, wrapped in a self-describing page.
+ *
+ * Like {@link DatasetClient.getData}, this returns one page rather than the whole collection;
+ * assembling all pages (e.g. for `KeyValueStore.keys()`) is the frontend's job. The result carries
+ * a cursor (`isTruncated` / `nextExclusiveStartKey`) so the frontend can paginate deterministically
+ * instead of inferring the end from `items.length < limit`.
+ */
+ listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
/** Get the public URL for a record, or `undefined` if unavailable. */
getPublicUrl(key: string): Promise<string | undefined>;
@@ -191,34 +213,24 @@ export interface KeyValueStoreClient {
recordExists(key: string): Promise<boolean>;
}
-export interface RequestQueueStats {
- readCount?: number;
- writeCount?: number;
- deleteCount?: number;
- headItemReadCount?: number;
- storageBytes?: number;
-}
-
export interface RequestQueueInfo {
id: string;
name?: string;
- userId?: string;
createdAt: Date;
modifiedAt: Date;
accessedAt: Date;
- expireAt?: string;
totalRequestCount: number;
handledRequestCount: number;
pendingRequestCount: number;
- actId?: string;
- actRunId?: string;
- hadMultipleClients?: boolean;
- stats?: RequestQueueStats;
}
-export interface RequestOptions {
+/**
+ * Options for request-queue operations that add or return requests to the queue
+ * ({@link RequestQueueClient.addBatchOfRequests}, {@link RequestQueueClient.reclaimRequest}).
+ */
+export interface RequestQueueOperationOptions {
+ /** Place the affected request(s) at the beginning of the queue so they are processed sooner. */
forefront?: boolean;
- [k: string]: unknown;
}
export interface RequestSchema {
@@ -289,54 +301,60 @@ export interface RequestQueueClient {
* but not re-added. With `forefront`, requests are placed at the beginning of the queue so
* they are processed sooner.
*/
- addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
+ addBatchOfRequests(
+ requests: RequestSchema[],
+ options?: RequestQueueOperationOptions,
+ ): Promise<BatchAddRequestsResult>;
/**
* Retrieve a request from the queue by its `uniqueKey`, or `undefined` if it does not exist.
*/
- getRequest(uniqueKey: string): Promise<RequestOptions | undefined>;
+ getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
/**
- * Return the next request in the queue to be processed, or `null` if there are currently no
+ * Return the next request in the queue to be processed, or `undefined` if there are currently no
* pending requests.
*
* The returned request is marked as in-progress; it will not be returned again until it is
* either reclaimed via {@link reclaimRequest} or marked as handled via {@link markRequestAsHandled}.
*
- * A `null` return value does not mean processing is finished — only that there are no pending
+ * An `undefined` return value does not mean processing is finished — only that there are no pending
* requests right now. Use {@link isEmpty} (together with the frontend's knowledge of pending
* add operations) to determine whether the queue is truly finished.
*/
- fetchNextRequest(): Promise<RequestOptions | null>;
+ fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
/**
* Mark a request previously returned by {@link fetchNextRequest} as handled.
*
* Handled requests are never returned again by {@link fetchNextRequest}. Returns information
- * about the operation, or `null` if the request was not in progress.
+ * about the operation, or `undefined` if the request was not in progress.
*
- * A `null` result is a no-op, not an error: the request is simply not something this client is
+ * An `undefined` result is a no-op, not an error: the request is simply not something this client is
* currently processing, so nothing is changed and the request is never added to the queue as a side
* effect. (Marking an already-handled request is idempotent and still returns operation info with
- * `wasAlreadyHandled: true` rather than `null`.)
+ * `wasAlreadyHandled: true` rather than `undefined`.)
*/
- markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | null>;
+ markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
/**
* Reclaim a failed request back to the queue so it can be processed again by a later call to
* {@link fetchNextRequest}. With `forefront`, the request is returned to the beginning of the
- * queue. Returns information about the operation, or `null` if the request was not in progress.
+ * queue. Returns information about the operation, or `undefined` if the request was not in progress.
*
* The request is expected to already be present in the queue (it should have been obtained via
- * {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it. A `null` result
+ * {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it. An `undefined` result
* is a no-op, not an error: the request is simply not something this client is currently processing,
* so nothing is changed and the request is never added to the queue as a side effect. Use
- * {@link addRequest} to insert a new request.
+ * {@link addBatchOfRequests} to insert a new request.
*/
- reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo | null>;
+ reclaimRequest(
+ request: UpdateRequestSchema,
+ options?: RequestQueueOperationOptions,
+ ): Promise<QueueOperationInfo | undefined>;
/**
- * Resolves to `true` if the next call to {@link fetchNextRequest} would return `null` — i.e. there
+ * Resolves to `true` if the next call to {@link fetchNextRequest} would return `undefined` — i.e. there
* are no pending requests to fetch right now.
*
* Requests that are currently in progress (fetched but not yet handled or reclaimed, including
@@ -367,7 +385,7 @@ export interface RequestQueueClient {
* this long, so that a long-running consumer does not have its request handed out again while it is
* still being processed. Clients that do not lock may ignore it.
*/
- setExpectedRequestProcessingTimeSecs?(secs: number): void;
+ setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
export interface SetStatusMessageOptions {
diff --git a/test/core/request_manager_tandem.test.ts b/test/core/request_manager_tandem.test.ts
index 256bbcc1e..6ccd0a4d9 100644
--- a/test/core/request_manager_tandem.test.ts
+++ b/test/core/request_manager_tandem.test.ts
@@ -303,7 +303,7 @@ describe('RequestManagerTandem', () => {
// Resolve the manager first (the queue was passed eagerly, but make the dependency explicit).
await tandem.fetchNextRequest();
- tandem.setExpectedRequestProcessingTimeSecs(600);
+ await tandem.setExpectedRequestProcessingTimeSecs(600);
expect(hintSpy).toHaveBeenCalledWith(600);
});
@@ -316,7 +316,7 @@ describe('RequestManagerTandem', () => {
const tandem = new RequestManagerTandem(requestList, () => requestQueue);
// Hint arrives before anything resolves the manager — nothing forwarded yet.
- tandem.setExpectedRequestProcessingTimeSecs(600);
+ await tandem.setExpectedRequestProcessingTimeSecs(600);
expect(hintSpy).not.toHaveBeenCalled();
// Resolving the manager (via any operation) applies the remembered hint.
diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts
index 51b5e29ed..45dba6e70 100644
--- a/test/core/storages/dataset.test.ts
+++ b/test/core/storages/dataset.test.ts
@@ -677,5 +677,33 @@ describe('dataset', () => {
expect(items).toEqual([]);
});
});
+
+ describe('stats', () => {
+ test('start at zero', async () => {
+ const dataset = await createDataset();
+ expect(dataset.stats).toEqual({ readCount: 0, writeCount: 0 });
+ });
+
+ test('count writes and reads per client call', async () => {
+ const dataset = await createDataset();
+
+ await dataset.pushData({ foo: 'bar' });
+ await dataset.pushData([{ foo: 'baz' }, { foo: 'qux' }]);
+ expect(dataset.stats).toEqual({ readCount: 0, writeCount: 2 });
+
+ await dataset.getData();
+ expect(dataset.stats).toEqual({ readCount: 1, writeCount: 2 });
+ });
+
+ test('snapshot is a copy, not a live reference', async () => {
+ const dataset = await createDataset();
+
+ const before = dataset.stats;
+ await dataset.pushData({ foo: 'bar' });
+
+ expect(before.writeCount).toBe(0);
+ expect(dataset.stats.writeCount).toBe(1);
+ });
+ });
});
});
diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts
index 6c46efff3..c42981c6e 100644
--- a/test/core/storages/key_value_store.test.ts
+++ b/test/core/storages/key_value_store.test.ts
@@ -543,13 +543,18 @@ describe('KeyValueStore', () => {
// @ts-expect-error Accessing private property
const mockListKeys = vitest.spyOn(store.client, 'listKeys');
- mockListKeys.mockResolvedValueOnce([
- { key: 'key1', size: 1 },
- { key: 'key2', size: 2 },
- { key: 'key3', size: 3 },
- { key: 'key4', size: 4 },
- { key: 'key5', size: 5 },
- ]);
+ mockListKeys.mockResolvedValueOnce({
+ items: [
+ { key: 'key1', size: 1, contentType: 'application/octet-stream' },
+ { key: 'key2', size: 2, contentType: 'application/octet-stream' },
+ { key: 'key3', size: 3, contentType: 'application/octet-stream' },
+ { key: 'key4', size: 4, contentType: 'application/octet-stream' },
+ { key: 'key5', size: 5, contentType: 'application/octet-stream' },
+ ],
+ count: 5,
+ limit: 5,
+ isTruncated: false,
+ });
const results: [string, number, { size: number }][] = [];
await store.forEachKey(
@@ -738,4 +743,38 @@ describe('KeyValueStore', () => {
]);
});
});
+
+ describe('stats', () => {
+ test('start at zero', async () => {
+ const store = await createKeyValueStore();
+ expect(store.stats).toEqual({ readCount: 0, writeCount: 0, deleteCount: 0, listCount: 0 });
+ });
+
+ test('count reads, writes and deletes per client call', async () => {
+ const store = await createKeyValueStore();
+
+ await store.setValue('foo', { a: 1 });
+ await store.setValue('bar', { b: 2 });
+ expect(store.stats).toMatchObject({ writeCount: 2, readCount: 0, deleteCount: 0 });
+
+ await store.getValue('foo');
+ expect(store.stats).toMatchObject({ writeCount: 2, readCount: 1 });
+
+ // Setting a value to null deletes it.
+ await store.setValue('bar', null);
+ expect(store.stats).toMatchObject({ writeCount: 2, deleteCount: 1 });
+ });
+
+ test('count list operations when iterating keys', async () => {
+ const store = await createKeyValueStore();
+
+ await store.setValue('key1', { value: 1 });
+ await store.setValue('key2', { value: 2 });
+
+ const listCountBefore = store.stats.listCount;
+ await store.forEachKey(() => {});
+
+ expect(store.stats.listCount).toBeGreaterThan(listCountBefore);
+ });
+ });
});
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index f3f019164..a95f315c0 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -383,19 +383,49 @@ describe('RequestQueue remote', () => {
(queue.client as any).setExpectedRequestProcessingTimeSecs = spy;
// First hint is forwarded.
- queue.setExpectedRequestProcessingTimeSecs(60);
+ await queue.setExpectedRequestProcessingTimeSecs(60);
expect(spy).toHaveBeenLastCalledWith(60);
// A larger hint is forwarded.
- queue.setExpectedRequestProcessingTimeSecs(120);
+ await queue.setExpectedRequestProcessingTimeSecs(120);
expect(spy).toHaveBeenLastCalledWith(120);
// A smaller (or equal) hint must not shorten the reservation, so it is not forwarded.
- queue.setExpectedRequestProcessingTimeSecs(30);
- queue.setExpectedRequestProcessingTimeSecs(120);
+ await queue.setExpectedRequestProcessingTimeSecs(30);
+ await queue.setExpectedRequestProcessingTimeSecs(120);
expect(spy).toHaveBeenCalledTimes(2);
});
});
+
+ describe('stats', () => {
+ test('start at zero', async () => {
+ const queue = await createRequestQueue();
+ expect(queue.stats).toEqual({ writeCount: 0, headItemReadCount: 0 });
+ });
+
+ test('count writes on add, handle and reclaim', async () => {
+ const queue = await createRequestQueue();
+
+ await queue.addRequest({ url: 'http://example.com/a' });
+ expect(queue.stats.writeCount).toBe(1);
+
+ const request = await queue.fetchNextRequest();
+ expect(queue.stats.headItemReadCount).toBe(1);
+
+ await queue.markRequestAsHandled(request!);
+ expect(queue.stats.writeCount).toBe(2);
+ });
+
+ test('count head reads on fetchNextRequest', async () => {
+ const queue = await createRequestQueue();
+
+ await queue.addRequest({ url: 'http://example.com/a' });
+
+ const headReadsBefore = queue.stats.headItemReadCount;
+ await queue.fetchNextRequest();
+ expect(queue.stats.headItemReadCount).toBe(headReadsBefore + 1);
+ });
+ });
});
describe('RequestQueue with requestsFromUrl', () => {
diff --git a/test/e2e/storage-open-return-storage-object/test.mjs b/test/e2e/storage-open-return-storage-object/test.mjs
index a186160db..2be15ea88 100644
--- a/test/e2e/storage-open-return-storage-object/test.mjs
+++ b/test/e2e/storage-open-return-storage-object/test.mjs
@@ -24,13 +24,11 @@ await expect(
const datasetStorageObject = parsed.datasetStorageObject;
const keyValueStorageObject = parsed.keyValueStorageObject;
-await expect(datasetStorageObject.id !== null, 'datasetStorageObject contains id');
+await expect(datasetStorageObject.id != null, 'datasetStorageObject contains id');
await expect(
[null, 'default'].includes(datasetStorageObject.name),
'Default dataset\'s name is either "default" or null',
);
-await expect(datasetStorageObject.userId !== null, 'datasetStorageObject contains userId');
-await expect(keyValueStorageObject.id !== null, 'keyValueStorageObject contains id');
+await expect(keyValueStorageObject.id != null, 'keyValueStorageObject contains id');
await expect([null, 'default'].includes(keyValueStorageObject.name), 'Default KVS\'s name is either "default" or null');
-await expect(keyValueStorageObject.userId !== null, 'keyValueStorageObject contains userId');
diff --git a/test/shared/MemoryStorageEmulator.ts b/test/shared/MemoryStorageEmulator.ts
index 24d07c7fc..8247e60b9 100644
--- a/test/shared/MemoryStorageEmulator.ts
+++ b/test/shared/MemoryStorageEmulator.ts
@@ -59,7 +59,7 @@ export class MemoryStorageEmulator extends StorageEmulator {
const items = [];
for (
let request = await requestQueue.fetchNextRequest();
- request !== null;
+ request != null;
request = await requestQueue.fetchNextRequest()
) {
items.push(request);