Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: make `CheerioCrawlerOptions` type more loose

Martin Adámekd871d8caf22b
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 08ae20404..6034ccbf2 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -18,22 +18,22 @@ import type { IncomingMessage } from 'http';
export type CheerioErrorHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerOptions<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>> {}
export type CheerioHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends InternalHttpCrawlingContext<UserData, JSONData, CheerioCrawler> {
/**
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
@@ -46,7 +46,7 @@ export interface CheerioCrawlingContext<
export type CheerioRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
diff --git a/packages/dom-crawler/src/internals/dom-crawler.ts b/packages/dom-crawler/src/internals/dom-crawler.ts
index 99f198ab5..94c89e8a3 100644
--- a/packages/dom-crawler/src/internals/dom-crawler.ts
+++ b/packages/dom-crawler/src/internals/dom-crawler.ts
@@ -16,22 +16,22 @@ import type { IncomingMessage } from 'http';
export type DOMErrorHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = ErrorHandler<DOMCrawlingContext<UserData, JSONData>>;
export interface DOMCrawlerOptions<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends HttpCrawlerOptions<DOMCrawlingContext<UserData, JSONData>> {}
export type DOMHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<DOMCrawlingContext<UserData, JSONData>>;
export interface DOMCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends InternalHttpCrawlingContext<UserData, JSONData, DOMCrawler> {
window: DOMWindow;
@@ -40,7 +40,7 @@ export interface DOMCrawlingContext<
export type DOMRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = RequestHandler<DOMCrawlingContext<UserData, JSONData>>;
export interface DOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 7218172e7..28777bcbc 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -49,7 +49,7 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS: AutoscaledPoolOptions = {
export type HttpErrorHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext> extends BasicCrawlerOptions<Context> {
@@ -155,7 +155,7 @@ export type InternalHttpHook<Context> = (
export type HttpHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
/**
@@ -163,7 +163,7 @@ export type HttpHook<
*/
export interface InternalHttpCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
Crawler = HttpCrawler<any>
> extends CrawlingContext<UserData> {
/**
@@ -193,7 +193,7 @@ export interface HttpCrawlingContext<UserData = any, JSONData = any>
export type HttpRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = Dictionary,
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
/**