Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Fix HttpCrawler context types (#3291)
Jan Buchar28468a38eedb
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f6d15618c..ab9079a12 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -120,7 +120,7 @@ export type RequireContextPipeline<export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext,- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,> {/**@@ -475,7 +475,7 @@ export interface CrawlerExperiments {*/export class BasicCrawler<Context extends CrawlingContext = CrawlingContext,- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,> {protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex f39a3347e..b27493957 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -104,7 +104,7 @@ export interface BrowserCrawlerOptions<ProvidedController,Dictionary>,- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,__BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,@@ -299,7 +299,7 @@ export abstract class BrowserCrawler<ProvidedController,Dictionary>,- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,GoToOptions extends Dictionary = Dictionary,> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 0bf69f5a0..a7cc15046 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -28,10 +28,11 @@ export type CheerioErrorHandler<> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;export interface CheerioCrawlerOptions<- ExtendedContext extends CheerioCrawlingContext,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ExtendedContext> {}+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}export type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -172,12 +173,13 @@ export type CheerioRequestHandler<* @category Crawlers*/export class CheerioCrawler<- ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext,-> extends HttpCrawler<CheerioCrawlingContext, ExtendedContext> {+ ContextExtension = Dictionary<never>,+ ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,+> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {/*** All `CheerioCrawler` parameters are passed via an options object.*/- constructor(options?: CheerioCrawlerOptions<ExtendedContext>, config?: Configuration) {+ constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>, config?: Configuration) {super({...options,diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 86b8ea88d..227284650 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -78,8 +78,9 @@ export type HttpErrorHandler<export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,- ExtendedContext extends Context = Context,-> extends BasicCrawlerOptions<Context, ExtendedContext> {+ ContextExtension = Dictionary<never>,+ ExtendedContext extends Context = Context & ContextExtension,+> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {/*** Timeout in which the HTTP request to the resource needs to finish, given in seconds.*/@@ -329,9 +330,9 @@ export type HttpRequestHandler<*/export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,-> extends BasicCrawler<Context, ExtendedContext> {+> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];protected persistCookiesPerSession: boolean;@@ -364,7 +365,7 @@ export class HttpCrawler<* All `HttpCrawlerOptions` parameters are passed via an options object.*/constructor(- options: HttpCrawlerOptions<Context, ExtendedContext> &+ options: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> &RequireContextPipeline<InternalHttpCrawlingContext, Context> = {} as any,override readonly config = Configuration.getGlobalConfig(),) {diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex eb625110c..7ef79c0fa 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -37,10 +37,11 @@ export type JSDOMErrorHandler<> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;export interface JSDOMCrawlerOptions<- ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {/*** Download and run scripts.*/@@ -180,10 +181,10 @@ const resources = new ResourceLoader({'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',});-export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<- JSDOMCrawlingContext,- ExtendedContext-> {+export class JSDOMCrawler<+ ContextExtension = Dictionary<never>,+ ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,+> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {protected static override optionsShape = {...HttpCrawler.optionsShape,runScripts: ow.optional.boolean,@@ -194,7 +195,7 @@ export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrprotected hideInternalConsole: boolean;protected virtualConsole: VirtualConsole | null = null;- constructor(options: JSDOMCrawlerOptions<ExtendedContext> = {}, config?: Configuration) {+ constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}, config?: Configuration) {const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;super(diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex d77aecf02..e65b59c44 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -32,10 +32,11 @@ export type LinkeDOMErrorHandler<> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;export interface LinkeDOMCrawlerOptions<- ExtendedContext extends LinkeDOMCrawlingContext,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {}+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}@@ -163,11 +164,12 @@ export type LinkeDOMRequestHandler<*/export class LinkeDOMCrawler<- ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,-> extends HttpCrawler<LinkeDOMCrawlingContext, ExtendedContext> {+ ContextExtension = Dictionary<never>,+ ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,+> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {private static parser = new DOMParser();- constructor(options: LinkeDOMCrawlerOptions<ExtendedContext>) {+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {super({...options,contextPipelineBuilder: () =>diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 77f0a508a..1e969e6e4 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -36,7 +36,7 @@ export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, Pexport type PlaywrightGotoOptions = Parameters<Page['goto']>[1];export interface PlaywrightCrawlerOptions<- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,> extends BrowserCrawlerOptions<Page,@@ -179,7 +179,7 @@ export interface PlaywrightCrawlerOptions<* @category Crawlers*/export class PlaywrightCrawler<- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,> extends BrowserCrawler<Page,diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex d47ad2a56..3772f815b 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -36,7 +36,7 @@ export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, Pupexport type PuppeteerGoToOptions = Parameters<Page['goto']>[1];export interface PuppeteerCrawlerOptions<- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,> extends BrowserCrawlerOptions<Page,@@ -153,7 +153,7 @@ export interface PuppeteerCrawlerOptions<* @category Crawlers*/export class PuppeteerCrawler<- ContextExtension = {},+ ContextExtension = Dictionary<never>,ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,> extends BrowserCrawler<Page,