Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { CrawlingContext } from '@crawlee/basic';
import { BasicCrawler, CrawlingRequest, defaultRoute, MissingRouteError, Router } from '@crawlee/basic';
import { RequestValidationError } from '@crawlee/core';
import {
CheerioCrawler,
type CheerioCrawlingContext,
createCheerioRouter,
createPlaywrightRouter,
createPuppeteerRouter,
PlaywrightCrawler,
type PlaywrightCrawlingContext,
PuppeteerCrawler,
type PuppeteerCrawlingContext,
} from 'crawlee';
import { z } from 'zod';
describe('Router', () => {
test('should be callable and route based on the label', async () => {
const router = Router.create();
const logs: string[] = [];
router.addHandler('A', async (ctx) => {
logs.push(`label A handled with url ${ctx.request.loadedUrl}`);
});
router.addHandler('B', async (ctx) => {
logs.push(`label B handled with url ${ctx.request.loadedUrl}`);
});
router.addHandler('C', async (ctx) => {
logs.push(`label C handled with url ${ctx.request.loadedUrl}`);
});
router.addDefaultHandler(async (ctx) => {
logs.push(`default handled with url ${ctx.request.loadedUrl}`);
});
router.use(({ request }) => {
logs.push(`middleware 1: ${request.loadedUrl}`);
});
router.use(({ request }) => {
logs.push(`middleware 2: ${request.loadedUrl}`);
});
router.use(({ request }) => {
logs.push(`middleware 3: ${request.loadedUrl}`);
});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/B', label: 'B' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/B', label: 'B' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/D', label: 'D' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);
expect(logs).toEqual([
'middleware 1: https://example.com/A',
'middleware 2: https://example.com/A',
'middleware 3: https://example.com/A',
'label A handled with url https://example.com/A',
'middleware 1: https://example.com/A',
'middleware 2: https://example.com/A',
'middleware 3: https://example.com/A',
'label A handled with url https://example.com/A',
'middleware 1: https://example.com/C',
'middleware 2: https://example.com/C',
'middleware 3: https://example.com/C',
'label C handled with url https://example.com/C',
'middleware 1: https://example.com/A',
'middleware 2: https://example.com/A',
'middleware 3: https://example.com/A',
'label A handled with url https://example.com/A',
'middleware 1: https://example.com/B',
'middleware 2: https://example.com/B',
'middleware 3: https://example.com/B',
'label B handled with url https://example.com/B',
'middleware 1: https://example.com/B',
'middleware 2: https://example.com/B',
'middleware 3: https://example.com/B',
'label B handled with url https://example.com/B',
'middleware 1: https://example.com/',
'middleware 2: https://example.com/',
'middleware 3: https://example.com/',
'default handled with url https://example.com/',
'middleware 1: https://example.com/A',
'middleware 2: https://example.com/A',
'middleware 3: https://example.com/A',
'label A handled with url https://example.com/A',
'middleware 1: https://example.com/C',
'middleware 2: https://example.com/C',
'middleware 3: https://example.com/C',
'label C handled with url https://example.com/C',
'middleware 1: https://example.com/C',
'middleware 2: https://example.com/C',
'middleware 3: https://example.com/C',
'label C handled with url https://example.com/C',
'middleware 1: https://example.com/D',
'middleware 2: https://example.com/D',
'middleware 3: https://example.com/D',
'default handled with url https://example.com/D',
'middleware 1: https://example.com/C',
'middleware 2: https://example.com/C',
'middleware 3: https://example.com/C',
'label C handled with url https://example.com/C',
]);
});
test('should expose the per-route timeouts, falling back the same way as getHandler', async () => {
const router = Router.create();
router.addHandler('SLOW', async () => {}, { requestHandlerTimeoutSecs: 120 });
router.addHandler('PLAIN', async () => {});
router.addDefaultHandler(async () => {}, { requestHandlerTimeoutSecs: 30 });
expect(router.getTimeoutSecs('SLOW')).toBe(120);
// a route of its own with no override means the crawler's own timeout, not the default route's
expect(router.getTimeoutSecs('PLAIN')).toBeUndefined();
// no route of its own, so it inherits the default route - handler and timeout alike
expect(router.getTimeoutSecs('UNKNOWN')).toBe(30);
expect(router.getTimeoutSecs()).toBe(30);
expect(router.getMaxTimeoutSecs()).toBe(120);
});
test('getMaxTimeoutSecs is undefined when no route overrides the timeout', async () => {
const router = Router.create();
router.addHandler('A', async () => {});
router.addDefaultHandler(async () => {});
expect(router.getMaxTimeoutSecs()).toBeUndefined();
expect(router.getTimeoutSecs('A')).toBeUndefined();
});
test('should be possible to define routes when creating router', async () => {
const logs: string[] = [];
// it should be possible to define router inline when creating router
const router = Router.create({
'A': async (ctx) => {
logs.push(`label A handled with url ${ctx.request.loadedUrl}`);
},
'B': async (ctx) => {
logs.push(`label B handled with url ${ctx.request.loadedUrl}`);
},
});
// and it's still possible to attach handlers later
router.addHandler('C', async (ctx) => {
logs.push(`label C handled with url ${ctx.request.loadedUrl}`);
});
router.addDefaultHandler(async (ctx) => {
logs.push(`default handled with url ${ctx.request.loadedUrl}`);
});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/B', label: 'B' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);
await router({ request: { loadedUrl: 'https://example.com/' }, log } as any);
expect(logs).toEqual([
'label A handled with url https://example.com/A',
'label B handled with url https://example.com/B',
'label C handled with url https://example.com/C',
'default handled with url https://example.com/',
]);
});
test('validation', async () => {
const router = Router.create();
router.addHandler('A', async (ctx) => {});
expect(() => router.addHandler('A', async (ctx) => {})).toThrow();
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
await expect(
router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any),
).rejects.toThrow(MissingRouteError);
router.addDefaultHandler(async (ctx) => {});
expect(() => router.addDefaultHandler(async (ctx) => {})).toThrow();
});
test('assignability to requestHandler', async () => {
const router = Router.create();
const crawler = new BasicCrawler({
requestHandler: router,
});
});
test('context has correct type', async () => {
const router = createPlaywrightRouter();
router.addHandler('label', async (ctx: PlaywrightCrawlingContext) => {
// just to test if this works on type level, the assertion is not actually executed
expect(ctx.page.$$).toBeDefined();
});
});
test('addHandler accepts userdata generic', async () => {
const testType = <T>(t: T): void => {};
const router: Router<CrawlingContext<{ foo: 'foo' }>> = {
addHandler: () => {},
addDefaultHandler: () => {},
} as any;
router.addHandler('1', (ctx) => {
testType<'foo'>(ctx.request.userData.foo);
});
router.addHandler<{ foo: 'bar' }>('2', (ctx) => {
testType<'bar'>(ctx.request.userData.foo);
});
router.addDefaultHandler((ctx) => {
testType<'foo'>(ctx.request.userData.foo);
});
router.addDefaultHandler<{ foo: 'bar' }>((ctx) => {
testType<'bar'>(ctx.request.userData.foo);
});
});
test('addHandler infers userData from a declared route map', async () => {
const testType = <T>(t: T): void => {};
interface Routes {
PRODUCT: { sku: string; price: number };
CATEGORY: { categoryId: string };
}
const router: Router<CrawlingContext, Routes> = {
addHandler: () => {},
addDefaultHandler: () => {},
} as any;
router.addHandler('PRODUCT', (ctx) => {
testType<string>(ctx.request.userData.sku);
testType<number>(ctx.request.userData.price);
});
router.addHandler('CATEGORY', (ctx) => {
testType<string>(ctx.request.userData.categoryId);
});
// @ts-expect-error unknown labels are rejected when a route map is declared
router.addHandler('UNKNOWN', () => {});
router.addDefaultHandler((ctx) => {
// the default handler is a fallback for any request, so userData stays loosely typed
testType<Record<string, unknown>>(ctx.request.userData);
});
});
test('factory infers userData from a route map passed as the second type argument', async () => {
const testType = <T>(t: T): void => {};
interface Routes {
PRODUCT: { sku: string; price: number };
CATEGORY: { categoryId: string };
}
// the documented two-argument form: `Routes` is the second type argument of the factory
const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
router.addHandler('PRODUCT', (ctx) => {
testType<string>(ctx.request.userData.sku);
testType<number>(ctx.request.userData.price);
});
router.addHandler('CATEGORY', (ctx) => {
testType<string>(ctx.request.userData.categoryId);
});
// @ts-expect-error unknown labels are rejected when a route map is declared
router.addHandler('UNKNOWN', () => {});
});
test('factory keeps the legacy flat-userData generic working (backwards compatibility)', async () => {
const testType = <T>(t: T): void => {};
// a flat `userData` shape (with a scalar field) resolves to the legacy open-map router,
// so any label is accepted and `userData` is typed as the passed shape
const router = createCheerioRouter<CheerioCrawlingContext, { token: string }>();
router.addHandler('anyLabel', (ctx) => {
testType<string>(ctx.request.userData.token);
});
router.addHandler('anotherLabel', (ctx) => {
testType<string>(ctx.request.userData.token);
});
});
test('schema map infers userData types and validates them at dispatch', async () => {
const testType = <T>(t: T): void => {};
const logs: string[] = [];
const router = createCheerioRouter({
PRODUCT: z.object({ sku: z.string(), price: z.coerce.number() }),
CATEGORY: z.object({ categoryId: z.string() }),
});
router.addHandler('PRODUCT', async (ctx) => {
// inferred from the schema (note: price is coerced to a number)
testType<string>(ctx.request.userData.sku);
testType<number>(ctx.request.userData.price);
logs.push(`product ${ctx.request.userData.sku} @ ${ctx.request.userData.price}`);
});
// @ts-expect-error unknown labels are still rejected when a schema map is declared
router.addHandler('UNKNOWN', () => {});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
// valid userData passes and is replaced with the parsed (coerced) value
const validRequest = {
loadedUrl: 'https://example.com/p',
label: 'PRODUCT',
userData: { sku: 'A1', price: '42' },
};
await router({ request: validRequest, log } as any);
expect(logs).toEqual(['product A1 @ 42']);
expect(validRequest.userData.price).toBe(42);
// invalid userData throws a RequestValidationError before the handler runs
await expect(
router({
request: { loadedUrl: 'https://example.com/p', label: 'PRODUCT', userData: { sku: 123 } },
log,
} as any),
).rejects.toThrow(RequestValidationError);
});
test('schema validation preserves the request label and internal metadata on a real Request', async () => {
const router = createCheerioRouter({
PRODUCT: z.object({ sku: z.string() }),
});
const handled: { label?: string; sku?: string; crawlDepth?: number } = {};
router.addHandler('PRODUCT', async ({ request }) => {
handled.label = request.label;
handled.sku = request.userData.sku;
handled.crawlDepth = request.crawlDepth;
});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
// a real Request keeps `label` inside `userData` and `crawlDepth` inside the non-enumerable `__crawlee`;
// both must survive the schema replacing `userData` with the parsed (label-less) value.
const request = new CrawlingRequest({
url: 'https://example.com/p',
label: 'PRODUCT',
userData: { sku: 'A1' },
});
request.crawlDepth = 3;
await router({ request, log } as any);
expect(handled).toEqual({ label: 'PRODUCT', sku: 'A1', crawlDepth: 3 });
expect(request.label).toBe('PRODUCT');
expect(request.crawlDepth).toBe(3);
});
test('schema map leaves requests without a registered label untouched', async () => {
const logs: string[] = [];
const router = createCheerioRouter({
PRODUCT: z.object({ sku: z.string() }),
});
router.addDefaultHandler(async (ctx) => {
logs.push(`default ${ctx.request.label ?? 'none'}`);
});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
// a label with no schema is not validated and falls through to the default handler
await router({
request: { loadedUrl: 'https://example.com/o', label: 'OTHER', userData: { anything: true } },
log,
} as any);
expect(logs).toEqual(['default OTHER']);
});
test('a defaultRoute schema validates requests that fall through to the default handler', async () => {
const seen: [string, Record<string, unknown>][] = [];
const router = createCheerioRouter({
PRODUCT: z.object({ sku: z.string() }),
[defaultRoute]: z.object({ page: z.coerce.number() }),
});
router.addHandler('PRODUCT', async ({ request }) => {
seen.push(['PRODUCT', request.userData]);
});
router.addDefaultHandler(async ({ request }) => {
seen.push(['default', request.userData]);
});
const log = { info: vitest.fn(), warn: vitest.fn(), debug: vitest.fn() };
// an unregistered label falls through to the default handler and is validated + coerced by its schema
await router({
request: { loadedUrl: 'https://example.com/l', label: 'LIST', userData: { page: '2' } },
log,
} as any);
// a registered label keeps using its own schema, not the default one
await router({
request: { loadedUrl: 'https://example.com/p', label: 'PRODUCT', userData: { sku: 'x' } },
log,
} as any);
expect(seen).toEqual([
['default', { page: 2, label: 'LIST' }],
['PRODUCT', { sku: 'x', label: 'PRODUCT' }],
]);
// a default-route request whose userData violates the default schema throws
await expect(
router({
request: { loadedUrl: 'https://example.com/x', label: 'X', userData: { page: 'not-a-number' } },
log,
} as any),
).rejects.toThrow(RequestValidationError);
});
test('a defaultRoute schema also types the default handler userData', () => {
// type-level only: never executed, it just has to type-check
const typeOnly = () => {
const testType = <T>(_v: T): void => {};
const router = createCheerioRouter({
PRODUCT: z.object({ sku: z.string() }),
[defaultRoute]: z.object({ page: z.coerce.number() }),
});
router.addDefaultHandler(({ request }) => {
// inferred from the defaultRoute schema
testType<number>(request.userData.page);
// @ts-expect-error page is a number, not a string — proves it is not typed as `any`
testType<string>(request.userData.page);
});
// without a defaultRoute schema, the default handler stays loosely typed as before
const plain = createCheerioRouter({ PRODUCT: z.object({ sku: z.string() }) });
plain.addDefaultHandler(({ request }) => {
testType<unknown>(request.userData.anything);
});
};
expect(typeof typeOnly).toBe('function');
});
test('crawler infers the route map from a typed requestHandler and types addRequests/context', () => {
// type-level only: the block is never executed, it just has to type-check
const typeOnly = async () => {
interface Routes {
PRODUCT: { sku: string; price: number };
CATEGORY: { categoryId: string };
}
const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
router.addHandler('PRODUCT', async ({ addRequests, enqueueLinks }) => {
// context methods are typed from the route map
await addRequests([{ url: 'https://e.com/c', label: 'CATEGORY', userData: { categoryId: 'c1' } }]);
await enqueueLinks({ label: 'PRODUCT', userData: { sku: 's', price: 1 } });
// @ts-expect-error wrong userData shape for the label
await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { categoryId: 'x' } }]);
// @ts-expect-error label not present in the route map
await addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
});
// the crawler infers `Routes` from the typed router passed as `requestHandler`
const crawler = new CheerioCrawler({ requestHandler: router });
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's', price: 1 } }]);
await crawler.run([
'https://e.com',
{ url: 'https://e.com/c', label: 'CATEGORY', userData: { categoryId: 'c1' } },
]);
// @ts-expect-error wrong userData shape for the label
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { categoryId: 'x' } }]);
// @ts-expect-error label not present in the route map
await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
};
expect(typeof typeOnly).toBe('function');
});
test('browser crawler also infers the route map from a typed requestHandler', () => {
// type-level only: never executed
const typeOnly = async () => {
interface Routes {
PRODUCT: { sku: string };
}
const router = createPlaywrightRouter<PlaywrightCrawlingContext, Routes>();
router.addHandler('PRODUCT', async ({ addRequests }) => {
await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
// @ts-expect-error wrong userData shape for the label
await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
// @ts-expect-error label not present in the route map
await addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
});
const crawler = new PlaywrightCrawler({ requestHandler: router });
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
// @ts-expect-error wrong userData shape for the label
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
// @ts-expect-error label not present in the route map
await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
};
expect(typeof typeOnly).toBe('function');
});
test('puppeteer crawler infers the route map too (inherited requestHandler path)', () => {
// type-level only: never executed
const typeOnly = async () => {
interface Routes {
PRODUCT: { sku: string };
}
const router = createPuppeteerRouter<PuppeteerCrawlingContext, Routes>();
const crawler = new PuppeteerCrawler({ requestHandler: router });
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
// @ts-expect-error wrong userData shape for the label
await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
// @ts-expect-error label not present in the route map
await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
};
expect(typeof typeOnly).toBe('function');
});
});