Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: Fix HttpCrawler context types (#3291)

Jan Buchar28468a38eedb
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index f6d15618c..ab9079a12 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -120,7 +120,7 @@ export type RequireContextPipeline<
export interface BasicCrawlerOptions<
Context extends CrawlingContext = CrawlingContext,
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
> {
/**
@@ -475,7 +475,7 @@ export interface CrawlerExperiments {
*/
export class BasicCrawler<
Context extends CrawlingContext = CrawlingContext,
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
> {
protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index f39a3347e..b27493957 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -104,7 +104,7 @@ export interface BrowserCrawlerOptions<
ProvidedController,
Dictionary
>,
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
__BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,
@@ -299,7 +299,7 @@ export abstract class BrowserCrawler<
ProvidedController,
Dictionary
>,
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
GoToOptions extends Dictionary = Dictionary,
> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 0bf69f5a0..a7cc15046 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -28,10 +28,11 @@ export type CheerioErrorHandler<
> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerOptions<
- ExtendedContext extends CheerioCrawlingContext,
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ExtendedContext> {}
+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
export type CheerioHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -172,12 +173,13 @@ export type CheerioRequestHandler<
* @category Crawlers
*/
export class CheerioCrawler<
- ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext,
-> extends HttpCrawler<CheerioCrawlingContext, ExtendedContext> {
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
+> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
/**
* All `CheerioCrawler` parameters are passed via an options object.
*/
- constructor(options?: CheerioCrawlerOptions<ExtendedContext>, config?: Configuration) {
+ constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>, config?: Configuration) {
super(
{
...options,
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 86b8ea88d..227284650 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -78,8 +78,9 @@ export type HttpErrorHandler<
export interface HttpCrawlerOptions<
Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,
- ExtendedContext extends Context = Context,
-> extends BasicCrawlerOptions<Context, ExtendedContext> {
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends Context = Context & ContextExtension,
+> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
/**
* Timeout in which the HTTP request to the resource needs to finish, given in seconds.
*/
@@ -329,9 +330,9 @@ export type HttpRequestHandler<
*/
export class HttpCrawler<
Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawler<Context, ExtendedContext> {
+> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
protected persistCookiesPerSession: boolean;
@@ -364,7 +365,7 @@ export class HttpCrawler<
* All `HttpCrawlerOptions` parameters are passed via an options object.
*/
constructor(
- options: HttpCrawlerOptions<Context, ExtendedContext> &
+ options: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> &
RequireContextPipeline<InternalHttpCrawlingContext, Context> = {} as any,
override readonly config = Configuration.getGlobalConfig(),
) {
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index eb625110c..7ef79c0fa 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -37,10 +37,11 @@ export type JSDOMErrorHandler<
> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
export interface JSDOMCrawlerOptions<
- ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext,
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {
+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
/**
* Download and run scripts.
*/
@@ -180,10 +181,10 @@ const resources = new ResourceLoader({
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
});
-export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<
- JSDOMCrawlingContext,
- ExtendedContext
-> {
+export class JSDOMCrawler<
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
+> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
protected static override optionsShape = {
...HttpCrawler.optionsShape,
runScripts: ow.optional.boolean,
@@ -194,7 +195,7 @@ export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCr
protected hideInternalConsole: boolean;
protected virtualConsole: VirtualConsole | null = null;
- constructor(options: JSDOMCrawlerOptions<ExtendedContext> = {}, config?: Configuration) {
+ constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}, config?: Configuration) {
const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
super(
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index d77aecf02..e65b59c44 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -32,10 +32,11 @@ export type LinkeDOMErrorHandler<
> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
export interface LinkeDOMCrawlerOptions<
- ExtendedContext extends LinkeDOMCrawlingContext,
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {}
+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
@@ -163,11 +164,12 @@ export type LinkeDOMRequestHandler<
*/
export class LinkeDOMCrawler<
- ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,
-> extends HttpCrawler<LinkeDOMCrawlingContext, ExtendedContext> {
+ ContextExtension = Dictionary<never>,
+ ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
+> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
private static parser = new DOMParser();
- constructor(options: LinkeDOMCrawlerOptions<ExtendedContext>) {
+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {
super({
...options,
contextPipelineBuilder: () =>
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 77f0a508a..1e969e6e4 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -36,7 +36,7 @@ export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, P
export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
export interface PlaywrightCrawlerOptions<
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
> extends BrowserCrawlerOptions<
Page,
@@ -179,7 +179,7 @@ export interface PlaywrightCrawlerOptions<
* @category Crawlers
*/
export class PlaywrightCrawler<
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
> extends BrowserCrawler<
Page,
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index d47ad2a56..3772f815b 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -36,7 +36,7 @@ export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, Pup
export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
export interface PuppeteerCrawlerOptions<
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
> extends BrowserCrawlerOptions<
Page,
@@ -153,7 +153,7 @@ export interface PuppeteerCrawlerOptions<
* @category Crawlers
*/
export class PuppeteerCrawler<
- ContextExtension = {},
+ ContextExtension = Dictionary<never>,
ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
> extends BrowserCrawler<
Page,