Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): accept null fields in stored requests (#4306)

Jindřich Bärfe60af70483a
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index 6cacc61df..53a0bee9c 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -391,21 +391,21 @@ export interface RequestQueueOperationOptions {
// @public (undocumented)
export interface RequestSchema {
// (undocumented)
- errorMessages?: string[];
+ errorMessages?: string[] | null;
// (undocumented)
- handledAt?: string;
+ handledAt?: string | null;
// (undocumented)
- headers?: Dictionary<string>;
+ headers?: Dictionary<string> | null;
// (undocumented)
id?: string;
// (undocumented)
- loadedUrl?: string;
+ loadedUrl?: string | null;
// (undocumented)
method?: AllowedHttpMethods;
// (undocumented)
- noRetry?: boolean;
+ noRetry?: boolean | null;
// (undocumented)
- payload?: string;
+ payload?: string | null;
// (undocumented)
retryCount?: number;
// (undocumented)
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index c9eaea5bb..5a95bb371 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -3,6 +3,7 @@ import crypto from 'node:crypto';
import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
import type { EnqueueStrategy } from '@crawlee/utils';
+import type { IsEqual } from 'type-fest';
import { z } from 'zod';
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
@@ -49,6 +50,32 @@ const requestOptionsSchema = z.compile(
}),
);
+/** Storages return `null` for fields left unset by other clients, e.g. the Python SDK. */
+const unset = <T extends z.ZodType>(schema: T) => schema.nullish().transform((value) => value ?? undefined);
+
+/** A request as storages return it. Accepts no less than the constructor does for the same fields. */
+const storedRequestSchema = z.compile(
+ z.object({
+ id: z.string().optional(),
+ url: z.string(),
+ uniqueKey: z.string(),
+ method: z.custom<AllowedHttpMethods>((value) => typeof value === 'string').optional(),
+ payload: unset(z.custom<string>((value) => typeof value === 'string' || value instanceof Uint8Array)),
+ noRetry: unset(z.boolean()),
+ retryCount: z.number().optional(),
+ // Entries are not checked, persisted requests can hold `null`s there (#4087).
+ errorMessages: unset(z.custom<string[]>(Array.isArray)),
+ headers: unset(z.custom<Dictionary<string>>((value) => typeof value === 'object' && value !== null)),
+ userData: schemas.anyObject.optional(),
+ // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+ handledAt: unset(z.custom<string>((value) => typeof value === 'string' || value instanceof Date)),
+ loadedUrl: unset(z.string()),
+ }),
+);
+
+// `@crawlee/types` cannot depend on `zod`, so this keeps `RequestSchema` in sync with the schema above.
+true satisfies IsEqual<z.input<typeof storedRequestSchema>, RequestSchema>;
+
/**
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
* with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
@@ -255,13 +282,13 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
* {@apilink RequestOptions} object cannot carry. Subclasses get instances of themselves.
*/
static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData> {
- const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+ const stored = parseArgument(schema, storedRequestSchema, 'RequestSchema');
+ const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = stored;
const request = new this(options as RequestOptions<UserData>);
request.id = id;
request.retryCount = retryCount;
request.errorMessages = [...errorMessages];
request.loadedUrl = loadedUrl;
- // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
request.handledAt =
(handledAt as unknown) instanceof Date ? (handledAt as unknown as Date).toISOString() : handledAt;
return request;
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index e098b7329..e2f766682 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -245,14 +245,14 @@ export interface RequestSchema {
url: string;
uniqueKey: string;
method?: AllowedHttpMethods;
- payload?: string;
- noRetry?: boolean;
+ payload?: string | null;
+ noRetry?: boolean | null;
retryCount?: number;
- errorMessages?: string[];
- headers?: Dictionary<string>;
+ errorMessages?: string[] | null;
+ headers?: Dictionary<string> | null;
userData?: Dictionary;
- handledAt?: string;
- loadedUrl?: string;
+ handledAt?: string | null;
+ loadedUrl?: string | null;
}
export interface UpdateRequestSchema extends RequestSchema {
diff --git a/test/core/request-list.test.ts b/test/core/request-list.test.ts
index b94d4c552..d908dee24 100644
--- a/test/core/request-list.test.ts
+++ b/test/core/request-list.test.ts
@@ -747,4 +747,38 @@ describe('Request', () => {
test('alwaysEnqueue throws when combined with a custom uniqueKey', () => {
expect(() => new Request({ url: 'https://example.com', uniqueKey: 'custom', alwaysEnqueue: true })).toThrow();
});
+
+ test('fromSchema treats null fields of a stored request as unset', () => {
+ const request = Request.fromSchema({
+ id: 'id',
+ url: 'https://example.com',
+ uniqueKey: 'https://example.com',
+ method: 'GET',
+ payload: null,
+ noRetry: null,
+ retryCount: 0,
+ errorMessages: null,
+ headers: null,
+ userData: {},
+ handledAt: null,
+ loadedUrl: null,
+ });
+
+ expect(request).toMatchObject({ payload: undefined, noRetry: false, errorMessages: [], headers: {} });
+ expect(request).toMatchObject({ handledAt: undefined, loadedUrl: undefined });
+ });
+
+ test('fromSchema accepts what storages and the constructor accept beyond the declared types', () => {
+ const request = Request.fromSchema({
+ url: 'https://example.com',
+ uniqueKey: 'https://example.com',
+ method: 'POST',
+ payload: new Uint8Array([1]) as unknown as string,
+ errorMessages: [null] as unknown as string[],
+ handledAt: new Date('2026-01-01T00:00:00.000Z') as unknown as string,
+ });
+
+ expect(request).toMatchObject({ errorMessages: [null], handledAt: '2026-01-01T00:00:00.000Z' });
+ expect(request.payload).toBeInstanceOf(Uint8Array);
+ });
});