Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf(core): compile the per-request zod schemas (#4123)
Vlad Frangu8c8ac17d2b5d
diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 957caba82..da68e1383 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -8,7 +8,6 @@ import { z } from 'zod';import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';import { serviceLocator } from './service_locator.js';-import { keys } from './typedefs.js';import { parseArgument, schemas } from './validators.js';/** The `strategy` option accepted by {@apilink ExtractLinksOptions} and {@apilink EnqueueUrlsOptions}. */@@ -26,31 +25,28 @@ interface CrawleeRequestData {enqueueStrategy?: EnqueueStrategyOption;}-const requestUrlSchema = z.object({ url: z.string() });-// new properties on the Request object breaks serialization-const requestOptionalSchemaShapes: Record<string, z.ZodType> = {- uniqueKey: z.string().optional(),- method: z.string().optional(),- payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),- noRetry: z.boolean().optional(),- sessionId: z.string().optional(),- maxRetries: schemas.anyNumber.optional(),- headers: z.looseObject({}).optional(),- userData: z.looseObject({}).optional(),- label: z.string().optional(),- keepUrlFragment: z.boolean().optional(),- useExtendedUniqueKey: z.boolean().optional(),- alwaysEnqueue: z.boolean().optional(),- skipNavigation: z.boolean().optional(),- crawlDepth: schemas.anyNumber- .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')- .optional(),-};--// Each schema is wrapped in a single-key object so validation errors carry the property name.-const requestOptionalSchemas: Partial<Record<string, z.ZodType>> = Object.fromEntries(- Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]),+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.+const requestOptionsSchema = z.compile(+ z.looseObject({+ url: z.string(),+ uniqueKey: z.string().optional(),+ method: z.string().optional(),+ payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),+ noRetry: z.boolean().optional(),+ sessionId: z.string().optional(),+ maxRetries: schemas.anyNumber.optional(),+ headers: z.looseObject({}).optional(),+ userData: z.looseObject({}).optional(),+ label: z.string().optional(),+ keepUrlFragment: z.boolean().optional(),+ useExtendedUniqueKey: z.boolean().optional(),+ alwaysEnqueue: z.boolean().optional(),+ skipNavigation: z.boolean().optional(),+ crawlDepth: schemas.anyNumber+ .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')+ .optional(),+ }),);/**@@ -153,23 +149,7 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {}parseArgument(options, schemas.anyObject, 'RequestOptions');- parseArgument(options, requestUrlSchema, 'RequestOptions');- // Full-shape validation is slow, because it checks all predicates- // even if the validated object has only 1 property.- // This custom validation loop iterates only over existing- // properties and speeds up the validation cca 3-fold.- keys(options).forEach((prop) => {- // skip url, because it is validated above- if (prop === 'url') {- return;- }-- const schema = requestOptionalSchemas[prop as string];- const value = options[prop];- if (schema) {- parseArgument({ [prop]: value }, schema, 'RequestOptions');- }- });+ parseArgument(options, requestOptionsSchema, 'RequestOptions');const {url,diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex f213fd072..ce4ea5ef3 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -65,19 +65,26 @@ const addRequestsBatchedOptionsSchema = z.strictObject({waitBetweenBatchesMillis: schemas.anyNumber.default(1000),maxNewRequests: schemas.anyNumber.optional(),});-const newRequestLikeSchema = z.looseObject({- url: z.string(),- id: z.undefined().optional(),-});-const handledRequestSchema = z.looseObject({- id: z.string(),- uniqueKey: z.string(),- handledAt: z.string().optional(),-});-const reclaimedRequestSchema = z.looseObject({- id: z.string(),- uniqueKey: z.string(),-});+// Compiled: these run once per request.+const newRequestLikeSchema = z.compile(+ z.looseObject({+ url: z.string(),+ id: z.undefined().optional(),+ }),+);+const handledRequestSchema = z.compile(+ z.looseObject({+ id: z.string(),+ uniqueKey: z.string(),+ handledAt: z.string().optional(),+ }),+);+const reclaimedRequestSchema = z.compile(+ z.looseObject({+ id: z.string(),+ uniqueKey: z.string(),+ }),+);const uniqueKeySchema = z.string();const openOptionsSchema = z.strictObject({configuration: z.instanceof(Configuration).optional(),diff --git a/packages/utils/src/internals/schemas.ts b/packages/utils/src/internals/schemas.tsindex 3f8ea71e1..cdb6fe74f 100644--- a/packages/utils/src/internals/schemas.ts+++ b/packages/utils/src/internals/schemas.ts@@ -82,11 +82,7 @@ export const plainObject = z.custom<Record<string, unknown>>({ message: 'Invalid input: expected an object' },);-/**- * Shape of a request stored in a request queue.- * @internal- */-export const storageRequest = z.looseObject({+const storageRequestShape = z.looseObject({id: z.string(),url: z.url({ protocol: /^https?$/ }),uniqueKey: z.string(),@@ -95,11 +91,17 @@ export const storageRequest = z.looseObject({handledAt: z.union([z.string(), z.date()]).optional(),});+/**+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.+ * @internal+ */+export const storageRequest = z.compile(storageRequestShape);+/*** {@link storageRequest} before an id is assigned.* @internal*/-export const storageRequestWithoutId = storageRequest.omit({ id: true });+export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));/*** `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —@@ -118,7 +120,7 @@ export function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string):* Batch of {@link storageRequestWithoutId}.* @internal*/-export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');+export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));/*** Options of request queue add/update operations.diff --git a/scripts/benchmarks/request-validation.mjs b/scripts/benchmarks/request-validation.mjsnew file mode 100644index 000000000..2b21081d9--- /dev/null+++ b/scripts/benchmarks/request-validation.mjs@@ -0,0 +1,50 @@+// Throughput of the per-request validation paths. Run against a build: `pnpm build && node scripts/benchmarks/request-validation.mjs`.+import { createRequire } from 'node:module';+import { pathToFileURL } from 'node:url';++const req = createRequire(new URL('../../packages/crawlee/package.json', import.meta.url));+const load = (specifier) => import(pathToFileURL(req.resolve(specifier)).href);+const { Request } = await load('@crawlee/core');+const { schemas } = await load('@crawlee/utils/internal');+const { z } = await load('zod');++const options = {+ url: 'https://example.com/products/123?page=2',+ uniqueKey: 'https://example.com/products/123?page=2',+ method: 'GET',+ userData: { label: 'DETAIL', depth: 3 },+ headers: { accept: 'text/html' },+};+const stored = { id: 'abc123', ...options, retryCount: 0 };+const batch = Array.from({ length: 100 }, (_, i) => ({+ ...options,+ url: `${options.url}&i=${i}`,+ uniqueKey: `${options.uniqueKey}&i=${i}`,+}));++function opsPerSecond(fn, minMs = 500) {+ for (let i = 0; i < 2000; i++) fn();+ let n = 0;+ const t0 = performance.now();+ let t;+ do {+ for (let i = 0; i < 1000; i++) fn();+ n += 1000;+ t = performance.now();+ } while (t - t0 < minMs);+ return n / ((t - t0) / 1000);+}+const best = (fn) => Math.round(Math.max(opsPerSecond(fn), opsPerSecond(fn), opsPerSecond(fn)));++console.table([+ { case: 'new Request(options)', 'ops/s': best(() => new Request(options)) },+ { case: 'storageRequest parse', 'ops/s': best(() => z.parse(schemas.storageRequest, stored)) },+ {+ case: 'storageRequestBatch parse (100 requests)',+ 'ops/s': best(() => z.parse(schemas.storageRequestBatch, batch)),+ },+ {+ case: 'requestQueueOperationOptions parse',+ 'ops/s': best(() => z.parse(schemas.requestQueueOperationOptions, { forefront: true })),+ },+]);