Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: unify querySelectorAll/querySelectorOne across crawlers (#4271)

Jan Buchareaec7ae0b284
diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.md
index ba45d343b..f802d49a8 100644
--- a/docs/public-api/crawlee-cheerio.api.md
+++ b/docs/public-api/crawlee-cheerio.api.md
@@ -4,6 +4,8 @@
```ts
+import type { AnyNode } from 'domhandler';
+import type { Cheerio } from 'cheerio';
import type { CheerioAPI } from 'cheerio';
import type { CrawlingContext } from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
@@ -19,7 +21,7 @@ import type { RouteSchemas } from '@crawlee/http';
import type { RoutesFromSchemas } from '@crawlee/http';
// @public
-export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<CheerioParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<CheerioParseResult, Cheerio<AnyNode>, Cheerio<AnyNode>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
}
@@ -31,7 +33,7 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat
// @public (undocumented)
export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends DOMCrawlingContext<CheerioParseResult, UserData, JSONData> {
+JSONData extends Dictionary = any> extends DOMCrawlingContext<CheerioParseResult, Cheerio<AnyNode>, Cheerio<AnyNode>, UserData, JSONData> {
}
// @public (undocumented)
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 40dd5085d..d886700e5 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -5,9 +5,11 @@
```ts
import type { AddRequestsBatchedResult } from '@crawlee/basic';
+import type { AnyNode } from 'domhandler';
import type { Awaitable } from '@crawlee/types';
import { BasicCrawler } from '@crawlee/basic';
import { BasicCrawlerOptions } from '@crawlee/basic';
+import type { Cheerio } from 'cheerio';
import type { CheerioAPI } from 'cheerio';
import type { ConcurrencySystemOptions } from '@crawlee/basic';
import { ContextPipeline } from '@crawlee/basic';
@@ -53,38 +55,41 @@ export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawl
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
// @public
-export class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
- constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+export class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+ constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
// (undocumented)
- protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
}
// @public (undocumented)
-export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
- parser: DOMParser_2<Parsed>;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+ parser: DOMParser_2<Parsed, Selection, Single>;
}
// @public (undocumented)
-export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;
// @public (undocumented)
-export interface DOMCrawlingHelpers {
+export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}
// @public
-interface DOMParser_2<Parsed extends DOMParseResult> {
+interface DOMParser_2<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
cleanup?(parsed: Parsed): Awaitable<void>;
extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
readonly mutable?: boolean;
// (undocumented)
parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
readonly placeholderMembers: Record<keyof Parsed & string, true>;
- select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+ querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
+ querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
}
export { DOMParser_2 as DOMParser }
@@ -144,6 +149,10 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
// @public (undocumented)
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+ // (undocumented)
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ // (undocumented)
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
}
// @public (undocumented)
@@ -165,6 +174,8 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
};
json: JSONData;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index 1048eb280..648bcd8c3 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -87,8 +87,8 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
page: Page;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
- querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
// (undocumented)
request: LoadedRequest<CrawlingRequest<UserData>>;
response: Response;
@@ -304,6 +304,8 @@ interface PlaywrightContextUtils {
injectJQuery(): Promise<unknown>;
listDownloads(): Promise<Download[]>;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
index 4358b1fde..208d45c96 100644
--- a/docs/public-api/crawlee-puppeteer.api.md
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -4,6 +4,7 @@
```ts
+import type { AnyNode } from 'domhandler';
import type { BatchAddRequestsResult } from '@crawlee/types';
import type { Browser } from 'puppeteer';
import { BrowserCrawler } from '@crawlee/browser';
@@ -14,6 +15,7 @@ import type { BrowserLaunchContext } from '@crawlee/browser';
import type { BrowserPool } from '@crawlee/browser-pool';
import type { BrowserPoolHooks } from '@crawlee/browser-pool';
import type { BrowserPoolOptions } from '@crawlee/browser-pool';
+import type { Cheerio } from 'cheerio';
import type { CheerioAPI } from 'cheerio';
import type { ClickOptions } from 'puppeteer';
import { Configuration } from '@crawlee/browser';
@@ -154,6 +156,8 @@ interface PuppeteerContextUtils {
injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
injectJQuery(): Promise<unknown>;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index 2f34806d3..1f6f7c653 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -51,6 +51,7 @@
"@crawlee/types": "workspace:*",
"@crawlee/utils": "workspace:*",
"cheerio": "^1.0.0",
+ "domhandler": "^6.0.0",
"htmlparser2": "^10.0.0",
"tslib": "^2.8.1"
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 291e89d12..0b493c3b2 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -12,6 +12,8 @@ import type {
} from '@crawlee/http';
import { DOMCrawler, Router } from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
+import type { Cheerio } from 'cheerio';
+import type { AnyNode } from 'domhandler';
import type { CheerioParseResult } from './cheerio-parser.js';
import { cheerioParser } from './cheerio-parser.js';
@@ -43,7 +45,7 @@ export interface CheerioCrawlerOptions<
export interface CheerioCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends DOMCrawlingContext<CheerioParseResult, UserData, JSONData> {}
+> extends DOMCrawlingContext<CheerioParseResult, Cheerio<AnyNode>, Cheerio<AnyNode>, UserData, JSONData> {}
export type CheerioRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -137,7 +139,15 @@ export class CheerioCrawler<
GetUserDataFromRequest<CheerioCrawlingContext['request']>
>,
StatisticStateExtension extends object = {},
-> extends DOMCrawler<CheerioParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+> extends DOMCrawler<
+ CheerioParseResult,
+ Cheerio<AnyNode>,
+ Cheerio<AnyNode>,
+ ContextExtension,
+ ExtendedContext,
+ Routes,
+ StatisticStateExtension
+> {
/**
* All `CheerioCrawler` parameters are passed via an options object.
*/
diff --git a/packages/cheerio-crawler/src/internals/cheerio-parser.ts b/packages/cheerio-crawler/src/internals/cheerio-parser.ts
index 395c5b3f9..a0499668d 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-parser.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-parser.ts
@@ -1,6 +1,7 @@
import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';
import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
-import type { CheerioAPI, CheerioOptions } from 'cheerio';
+import type { Cheerio, CheerioAPI, CheerioOptions } from 'cheerio';
+import type { AnyNode } from 'domhandler';
// slim uses faster htmlparser2 parser and doesn't load extra libs like undici
import * as cheerio from 'cheerio/slim';
import { parseDocument } from 'htmlparser2';
@@ -14,7 +15,7 @@ export interface CheerioParseResult {
* A {@apilink DOMParser} backed by [cheerio](https://www.npmjs.com/package/cheerio). Pass it to a
* {@apilink DOMCrawler} to get the crawling context {@apilink CheerioCrawler} provides.
*/
-export function cheerioParser(): DOMParser<CheerioParseResult> {
+export function cheerioParser(): DOMParser<CheerioParseResult, Cheerio<AnyNode>, Cheerio<AnyNode>> {
return {
placeholderMembers: { $: true, body: true },
parse(context: InternalHttpCrawlingContext) {
@@ -30,7 +31,11 @@ export function cheerioParser(): DOMParser<CheerioParseResult> {
return { $, body };
},
extractLinks: ({ $ }, selector, baseUrl) => extractUrlsFromCheerio($, selector, baseUrl),
- select: ({ $ }, selector) => $(selector).get(),
+ querySelectorAll: ({ $ }, selector) => $(selector),
+ querySelectorOne: ({ $ }, selector) => {
+ const match = $(selector).first();
+ return match.length ? match : undefined;
+ },
toCheerio: ({ $ }) => $,
};
}
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index e5ba6b750..9343cab6a 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -54,6 +54,7 @@
"@crawlee/utils": "workspace:*",
"cheerio": "^1.0.0",
"content-type": "^3.1.1",
+ "domhandler": "^6.0.0",
"iconv-lite": "^0.7.2",
"mime-types": "^3.0.1",
"tslib": "^2.8.1",
diff --git a/packages/http-crawler/src/internals/dom-crawler.ts b/packages/http-crawler/src/internals/dom-crawler.ts
index 5bbb951c2..bcfd42814 100644
--- a/packages/http-crawler/src/internals/dom-crawler.ts
+++ b/packages/http-crawler/src/internals/dom-crawler.ts
@@ -44,7 +44,11 @@ export interface DOMParseResult {
* });
* ```
*/
-export interface DOMParser<Parsed extends DOMParseResult> {
+export interface DOMParser<
+ Parsed extends DOMParseResult,
+ Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
+ Single = unknown,
+> {
/**
* The context members {@apilink DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
* placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
@@ -61,10 +65,16 @@ export interface DOMParser<Parsed extends DOMParseResult> {
extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
/**
- * Returns the current matches of `selector`. Only the count is used, by
- * {@apilink DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+ * Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
+ * {@apilink DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
+ */
+ querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
+
+ /**
+ * Returns the first match of `selector`, or `undefined` when there is none - this is what
+ * {@apilink DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
*/
- select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+ querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
/**
* Whether the parse result can change after {@apilink DOMParser.parse|`parse`} returned - the case when the DOM
@@ -87,7 +97,7 @@ export interface DOMParser<Parsed extends DOMParseResult> {
cleanup?(parsed: Parsed): Awaitable<void>;
}
-export interface DOMCrawlingHelpers {
+export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
/**
* Extracts URLs from the parsed DOM, without adding them to the request queue.
*/
@@ -115,6 +125,17 @@ export interface DOMCrawlingHelpers {
*/
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ /**
+ * Like {@apilink DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
+ * the parser produces them - a Cheerio object in {@apilink CheerioCrawler}. Throws when nothing matches.
+ */
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
+
+ /**
+ * Like {@apilink DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
+ */
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
+
/**
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
* When provided with the `selector` argument, it will throw if it's not available.
@@ -132,25 +153,42 @@ export interface DOMCrawlingHelpers {
export type DOMCrawlingContext<
Parsed extends DOMParseResult = DOMParseResult,
+ Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
+ Single = unknown,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> &
+ Parsed &
+ DOMCrawlingHelpers<Selection, Single>;
export interface DOMCrawlerOptions<
Parsed extends DOMParseResult = DOMParseResult,
+ Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
+ Single = unknown,
ContextExtension = Dictionary<never>,
- ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension,
+ ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<
+ Parsed,
+ Selection,
+ Single
+ > &
+ ContextExtension,
Routes extends Record<keyof Routes, Dictionary> = Record<string, any>,
StatisticStateExtension extends object = {},
> extends Omit<
- HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>,
+ HttpCrawlerOptions<
+ DOMCrawlingContext<Parsed, Selection, Single>,
+ ContextExtension,
+ ExtendedContext,
+ Routes,
+ StatisticStateExtension
+ >,
'contextPipelineBuilder'
> {
/**
* The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
* context, so the members the request handler receives follow from the parser you pass.
*/
- parser: DOMParser<Parsed>;
+ parser: DOMParser<Parsed, Selection, Single>;
}
/**
@@ -164,18 +202,39 @@ export interface DOMCrawlerOptions<
*/
export class DOMCrawler<
Parsed extends DOMParseResult = DOMParseResult,
+ Selection extends ArrayLike<unknown> = ArrayLike<unknown>,
+ Single = unknown,
ContextExtension = Dictionary<never>,
- ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension,
+ ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<
+ Parsed,
+ Selection,
+ Single
+ > &
+ ContextExtension,
Routes extends Record<keyof Routes, Dictionary> = Record<
string,
- GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>
+ GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>
>,
StatisticStateExtension extends object = {},
-> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
- readonly #parser: DOMParser<Parsed>;
+> extends HttpCrawler<
+ DOMCrawlingContext<Parsed, Selection, Single>,
+ ContextExtension,
+ ExtendedContext,
+ Routes,
+ StatisticStateExtension
+> {
+ readonly #parser: DOMParser<Parsed, Selection, Single>;
constructor(
- options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>,
+ options: DOMCrawlerOptions<
+ Parsed,
+ Selection,
+ Single,
+ ContextExtension,
+ ExtendedContext,
+ Routes,
+ StatisticStateExtension
+ >,
) {
const { parser, ...rest } = options;
@@ -184,7 +243,10 @@ export class DOMCrawler<
this.#parser = parser;
}
- protected override buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>> {
+ protected override buildContextPipeline(): ContextPipeline<
+ CrawlingContext,
+ DOMCrawlingContext<Parsed, Selection, Single>
+ > {
return super
.buildContextPipeline()
.compose(this.#parseContent.bind(this))
@@ -234,22 +296,52 @@ export class DOMCrawler<
options?.baseUrl ?? context.request.loadedUrl ?? context.request.url,
);
- const waitForSelector = async (selector: string, timeoutMs = 5_000): Promise<void> => {
+ // Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
+ const poll = async <T>(
+ selector: string,
+ timeoutMs: number,
+ query: () => Awaitable<T>,
+ isEmpty: (result: T) => boolean,
+ ): Promise<T> => {
let remaining = parser.mutable ? timeoutMs : 0;
+ let result = await query();
- while ((await parser.select(context, selector)).length === 0) {
+ while (isEmpty(result)) {
if (remaining <= 0) {
throw new Error(`Selector '${selector}' not found.`);
}
await sleep(50);
remaining -= 50;
+ result = await query();
}
+
+ return result;
+ };
+
+ const querySelectorAll = async (selector: string, timeoutMs = 5_000): Promise<Selection> =>
+ poll(
+ selector,
+ timeoutMs,
+ () => parser.querySelectorAll(context, selector),
+ (matches) => matches.length === 0,
+ );
+
+ const waitForSelector = async (selector: string, timeoutMs?: number): Promise<void> => {
+ await querySelectorAll(selector, timeoutMs);
};
return {
extractLinks,
waitForSelector,
+ querySelectorAll,
+ querySelectorOne: async (selector: string, timeoutMs = 5_000): Promise<Single> =>
+ poll(
+ selector,
+ timeoutMs,
+ () => parser.querySelectorOne(context, selector),
+ (match) => match === undefined,
+ ) as Promise<Single>,
enqueueLinks: async (options: EnqueueLinksOptions = {}) => {
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
enqueueStrategy: options.strategy,
@@ -266,19 +358,18 @@ export class DOMCrawler<
strategy: options.strategy ?? EnqueueStrategy.SameHostname,
});
},
- async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {
+ async parseWithCheerio(selector?: string, timeoutMs?: number) {
+ if (selector) {
+ await waitForSelector(selector, timeoutMs);
+ }
+
// Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
- const $ =
+ return (
(await parser.toCheerio?.(context)) ??
(await import('cheerio')).load(context.body, {
xmlMode: context.contentType.type.includes('xml'),
- });
-
- if (selector && $(selector).get().length === 0) {
- throw new Error(`Selector '${selector}' not found.`);
- }
-
- return $;
+ })
+ );
},
};
}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 88786ba6c..183df4b71 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -32,7 +32,8 @@ import {
import { ResponseWithUrl } from '@crawlee/http-client';
import type { Awaitable, Dictionary, ISession } from '@crawlee/types';
import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
-import type { CheerioAPI } from 'cheerio';
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
import { isTypeValid, parse as parseContentType } from 'content-type';
import iconv from 'iconv-lite';
import type { JsonValue } from 'type-fest';
@@ -242,6 +243,19 @@ export interface InternalHttpCrawlingContext<
*/
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ /**
+ * Like {@apilink InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
+ * elements - a Cheerio object here, whatever the parser produces in {@apilink DOMCrawler}. Throws when nothing
+ * matches.
+ */
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
+
+ /**
+ * Like {@apilink InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
+ * match.
+ */
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
+
/**
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
* This is here to unify the crawler API, so they all have this handy method - in {@apilink CheerioCrawler} it has
@@ -265,7 +279,10 @@ export interface InternalHttpCrawlingContext<
export interface HttpCrawlingContext<
UserData extends Dictionary = any,
JSONData extends JsonValue = any,
-> extends InternalHttpCrawlingContext<UserData, JSONData> {}
+> extends InternalHttpCrawlingContext<UserData, JSONData> {
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+}
export type HttpRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -577,6 +594,16 @@ export class HttpCrawler<
'The `parseWithCheerio` method is not available - `skipNavigation` was used',
);
},
+ get querySelectorAll(): InternalHttpCrawlingContext['querySelectorAll'] {
+ throw new NavigationSkippedError(
+ 'The `querySelectorAll` method is not available - `skipNavigation` was used',
+ );
+ },
+ get querySelectorOne(): InternalHttpCrawlingContext['querySelectorOne'] {
+ throw new NavigationSkippedError(
+ 'The `querySelectorOne` method is not available - `skipNavigation` was used',
+ );
+ },
};
}
@@ -615,12 +642,19 @@ export class HttpCrawler<
return load(parsed.body!.toString(), { xmlMode: contentType.type.includes('xml') });
};
- const waitForSelector = async (selector: string, _timeoutMs?: number) => {
- const $ = await loadBody();
+ const querySelectorAll = async (selector: string, _timeoutMs?: number) => {
+ const matches = (await loadBody())(selector);
- if ($(selector).get().length === 0) {
+ if (matches.length === 0) {
throw new Error(`Selector '${selector}' not found.`);
}
+
+ return matches;
+ };
+ const querySelectorOne = async (selector: string, timeoutMs?: number) =>
+ (await querySelectorAll(selector, timeoutMs)).first();
+ const waitForSelector = async (selector: string, timeoutMs?: number) => {
+ await querySelectorAll(selector, timeoutMs);
};
const parseWithCheerio = async (selector?: string, timeoutMs?: number) => {
const $ = await loadBody();
@@ -658,6 +692,8 @@ export class HttpCrawler<
return JSON.parse(jsonString);
},
waitForSelector,
+ querySelectorOne,
+ querySelectorAll,
parseWithCheerio,
contentType,
body: parsed.body,
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index f682dd8eb..c1cfcb26c 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -57,6 +57,7 @@
"@crawlee/types": "workspace:*",
"@crawlee/utils": "workspace:*",
"cheerio": "^1.0.0",
+ "domhandler": "^6.0.0",
"jquery": "^3.7.1",
"ml-logistic-regression": "^2.0.0",
"ml-matrix": "^6.12.1",
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 0e94a3f02..2327cbce5 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -107,16 +107,17 @@ export interface AdaptivePlaywrightCrawlerContext<
page: Page;
/**
- * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
+ * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+ * Throws when nothing matches - in the HTTP-only run, that makes the crawler retry the request in a browser.
* Timeout defaults to 5s.
*/
- querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
/**
- * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
- * Timeout defaults to 5s.
+ * Like {@apilink AdaptivePlaywrightCrawlerContext.querySelectorAll|`querySelectorAll`}, but returns only the
+ * first match.
*/
- querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
/**
* Wait for an element matching the selector to appear.
@@ -281,9 +282,9 @@ interface InnerCrawlerLifecycle {
* ```javascript
* const crawler = new AdaptivePlaywrightCrawler({
* renderingTypeDetectionRatio: 0.1,
- * async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
+ * async requestHandler({ querySelectorAll, pushData, enqueueLinks, request, log }) {
* // This function is called to extract data from a single web page
- * const $prices = await querySelector('span.price')
+ * const $prices = await querySelectorAll('span.price')
*
* await pushData({
* url: request.url,
@@ -527,12 +528,12 @@ export class AdaptivePlaywrightCrawler<
get page(): Page {
throw new Error(errorMessage('page'));
},
- get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {
- throw new Error(errorMessage('querySelector'));
- },
get querySelectorAll(): AdaptivePlaywrightCrawlerContext['querySelectorAll'] {
throw new Error(errorMessage('querySelectorAll'));
},
+ get querySelectorOne(): AdaptivePlaywrightCrawlerContext['querySelectorOne'] {
+ throw new Error(errorMessage('querySelectorOne'));
+ },
get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {
throw new Error(errorMessage('waitForSelector'));
},
@@ -550,12 +551,6 @@ export class AdaptivePlaywrightCrawler<
get page(): Page {
throw new Error('Page object was used in HTTP-only request handler');
},
- async querySelector(selector: string) {
- return cheerioContext.$(selector).first();
- },
- async querySelectorAll(selector: string) {
- return cheerioContext.$(selector);
- },
enqueueLinks: async (options: EnqueueLinksOptions = {}) => {
const urls = extractUrlsFromCheerio(
cheerioContext.$,
@@ -578,20 +573,6 @@ export class AdaptivePlaywrightCrawler<
status: originalResponse.status(),
statusText: originalResponse.statusText(),
}),
- async querySelector(selector: string, timeoutMs = 5000) {
- const locator = playwrightContext.page.locator(selector).first();
- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
- const $ = await playwrightContext.parseWithCheerio();
-
- return $(selector).first() as Cheerio<any>;
- },
- async querySelectorAll(selector: string, timeoutMs = 5000) {
- const locator = playwrightContext.page.locator(selector).first();
- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
- const $ = await playwrightContext.parseWithCheerio();
-
- return $(selector) as Cheerio<any>;
- },
enqueueLinks: async (options: EnqueueLinksOptions = {}, timeoutMs = 5000) => {
// TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
const selector = options.selector ?? 'a';
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index bb69c5960..d8b503f35 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -274,6 +274,24 @@ export class PlaywrightCrawler<
const locator = context.page.locator(selector).first();
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
};
+ const parseWithCheerioFn = async (selector?: string, timeoutMs = 5_000) => {
+ if (selector) {
+ await waitForSelector(selector, timeoutMs);
+ }
+
+ return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+ };
+ const querySelectorAll = async (selector: string, timeoutMs?: number) => {
+ const matches = (await parseWithCheerioFn(selector, timeoutMs))(selector);
+
+ // The browser located it but the Cheerio snapshot does not have it - a Playwright-only selector,
+ // or an element under a shadow root/iframe the snapshot was told to ignore.
+ if (matches.length === 0) {
+ throw new Error(`Selector '${selector}' not found in the page snapshot.`);
+ }
+
+ return matches;
+ };
const downloads: Download[] = [];
context.page.on('download', (download) => downloads.push(download));
@@ -293,13 +311,10 @@ export class PlaywrightCrawler<
},
blockRequests: async (options?: BlockRequestsOptions) => blockRequests(context.page, options),
waitForSelector,
- parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => {
- if (selector) {
- await waitForSelector(selector, timeoutMs);
- }
-
- return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
- },
+ parseWithCheerio: parseWithCheerioFn,
+ querySelectorAll,
+ querySelectorOne: async (selector: string, timeoutMs?: number) =>
+ (await querySelectorAll(selector, timeoutMs)).first(),
infiniteScroll: async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options),
listDownloads: async () => downloads,
saveSnapshot: async (options?: SaveSnapshotOptions) =>
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index 22c6b6a9b..68d433701 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -24,7 +24,8 @@ import vm from 'node:vm';
import { Configuration, KeyValueStore, type Request, serviceLocator, SessionError, validators } from '@crawlee/browser';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
import { sleep } from '@crawlee/utils';
import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
import type { Download, Page, Response, Route } from 'playwright';
@@ -881,6 +882,17 @@ export interface PlaywrightContextUtils {
*/
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ /**
+ * Like {@apilink PlaywrightContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
+ * as a Cheerio object. Throws when nothing matches.
+ */
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+
+ /**
+ * Like {@apilink PlaywrightContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
+ */
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+
/**
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
* When provided with the `selector` argument, it waits for it to be available first.
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 420a65de7..788fb58ce 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -55,6 +55,7 @@
"@crawlee/utils": "workspace:*",
"cheerio": "^1.0.0",
"devtools-protocol": "*",
+ "domhandler": "^6.0.0",
"jquery": "^3.7.1",
"tslib": "^2.8.1",
"zod": "catalog:"
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index 5c8bbf315..b2cbb8cfd 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -259,6 +259,24 @@ export class PuppeteerCrawler<
const waitForSelector = async (selector: string, timeoutMs = 5_000) => {
await context.page.waitForSelector(selector, { timeout: timeoutMs });
};
+ const parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => {
+ if (selector) {
+ await waitForSelector(selector, timeoutMs);
+ }
+
+ return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+ };
+ const querySelectorAll = async (selector: string, timeoutMs?: number) => {
+ const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
+
+ // The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
+ // or an element under a shadow root/iframe the snapshot was told to ignore.
+ if (matches.length === 0) {
+ throw new Error(`Selector '${selector}' not found in the page snapshot.`);
+ }
+
+ return matches;
+ };
return {
injectFile: async (filePath: string, options?: InjectFileOptions) =>
@@ -274,13 +292,10 @@ export class PuppeteerCrawler<
await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
},
waitForSelector,
- parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => {
- if (selector) {
- await waitForSelector(selector, timeoutMs);
- }
-
- return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
- },
+ parseWithCheerio,
+ querySelectorAll,
+ querySelectorOne: async (selector: string, timeoutMs?: number) =>
+ (await querySelectorAll(selector, timeoutMs)).first(),
enqueueLinksByClickingElements: async (
options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>,
) =>
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer-utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer-utils.ts
index 4cf6eee58..135c1e03d 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer-utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer-utils.ts
@@ -25,7 +25,8 @@ import vm from 'node:vm';
import type { Request } from '@crawlee/browser';
import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
import { sleep } from '@crawlee/utils';
import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
@@ -740,6 +741,17 @@ export interface PuppeteerContextUtils {
*/
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ /**
+ * Like {@apilink PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
+ * as a Cheerio object. Throws when nothing matches.
+ */
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+
+ /**
+ * Like {@apilink PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
+ */
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+
/**
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
* When provided with the `selector` argument, it waits for it to be available first.
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c65857ad6..bca5bbe92 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -613,6 +613,9 @@ importers:
cheerio:
specifier: ^1.0.0
version: 1.2.0
+ domhandler:
+ specifier: ^6.0.0
+ version: 6.0.1
htmlparser2:
specifier: ^10.0.0
version: 10.1.0
@@ -805,6 +808,9 @@ importers:
content-type:
specifier: ^3.1.1
version: 3.1.1
+ domhandler:
+ specifier: ^6.0.0
+ version: 6.0.1
iconv-lite:
specifier: ^0.7.2
version: 0.7.3
@@ -893,6 +899,9 @@ importers:
cheerio:
specifier: ^1.0.0
version: 1.2.0
+ domhandler:
+ specifier: ^6.0.0
+ version: 6.0.1
jquery:
specifier: ^3.7.1
version: 3.7.1
@@ -941,6 +950,9 @@ importers:
devtools-protocol:
specifier: '*'
version: 0.0.1710668
+ domhandler:
+ specifier: ^6.0.0
+ version: 6.0.1
jquery:
specifier: ^3.7.1
version: 3.7.1
diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.ts
index 5c66ed956..fd4e24616 100644
--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts
+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts
@@ -263,7 +263,7 @@ describe('AdaptivePlaywrightCrawler', () => {
});
});
- describe('querySelector and querySelectorAll', () => {
+ describe('querySelectorOne and querySelectorAll', () => {
test.each([
['/static', 'static'],
['/dynamic', 'clientOnly'],
@@ -275,14 +275,10 @@ describe('AdaptivePlaywrightCrawler', () => {
const url = new URL(`http://${HOSTNAME}:${port}${path}`);
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
- async ({ pushData, querySelector, querySelectorAll }) => {
- const first = await querySelector('a');
+ async ({ pushData, querySelectorOne, querySelectorAll }) => {
+ const first = await querySelectorOne('a');
const all = await querySelectorAll('a');
- await pushData({
- firstCount: first.length,
- firstText: first.text(),
- allCount: all.length,
- });
+ await pushData({ firstCount: first.length, firstText: first.text(), allCount: all.length });
},
);
@@ -296,9 +292,29 @@ describe('AdaptivePlaywrightCrawler', () => {
await crawler.run();
- // `querySelector` returns only the first match, `querySelectorAll` returns the whole collection.
expect((await Dataset.getData()).items).toEqual([{ firstCount: 1, firstText: 'Link 1', allCount: 5 }]);
});
+
+ test('falls back to the browser when the static run finds nothing', async () => {
+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+ detectionProbabilityRecommendation: 0,
+ renderingType: 'static',
+ });
+ const url = new URL(`http://${HOSTNAME}:${port}/dynamic`);
+
+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
+ async ({ pushData, querySelectorAll }) => {
+ await pushData({ heading: (await querySelectorAll('h1')).text() });
+ },
+ );
+
+ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor }, [url.toString()]);
+
+ await crawler.run();
+
+ expect(requestHandler).toHaveBeenCalledTimes(2);
+ expect((await Dataset.getData()).items).toEqual([{ heading: 'Heading' }]);
+ });
});
describe('fullResultComparator', () => {
@@ -421,9 +437,9 @@ describe('AdaptivePlaywrightCrawler', () => {
const url = new URL(`http://${HOSTNAME}:${port}/dynamic`);
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
- async ({ pushData, querySelector }) => {
+ async ({ pushData, parseWithCheerio }) => {
await pushData({
- heading: (await querySelector('h1')).text(),
+ heading: (await parseWithCheerio())('h1').text(),
});
},
);
diff --git a/test/core/crawlers/dom-crawler.test.ts b/test/core/crawlers/dom-crawler.test.ts
index c6602618b..6abe7dbcc 100644
--- a/test/core/crawlers/dom-crawler.test.ts
+++ b/test/core/crawlers/dom-crawler.test.ts
@@ -10,7 +10,7 @@ interface FakeParseResult {
body: string;
}
-function fakeParser(): DOMParser<FakeParseResult> {
+function fakeParser(): DOMParser<FakeParseResult, string[], string> {
return {
placeholderMembers: { title: true, body: true },
parse: (context) => {
@@ -19,7 +19,8 @@ function fakeParser(): DOMParser<FakeParseResult> {
},
extractLinks: (parsed, _selector, baseUrl) =>
[...parsed.body.matchAll(/<a href="([^"]+)"/g)].map(([, href]) => new URL(href, baseUrl).href),
- select: (parsed, selector) => (selector === 'title' && parsed.title ? [parsed.title] : []),
+ querySelectorAll: (parsed, selector) => (selector === 'title' && parsed.title ? [parsed.title] : []),
+ querySelectorOne: (parsed, selector) => (selector === 'title' && parsed.title ? parsed.title : undefined),
};
}
diff --git a/test/e2e/adaptive-playwright-default/actor/main.js b/test/e2e/adaptive-playwright-default/actor/main.js
index a0e98a30c..02be9ffbf 100644
--- a/test/e2e/adaptive-playwright-default/actor/main.js
+++ b/test/e2e/adaptive-playwright-default/actor/main.js
@@ -26,7 +26,7 @@ const crawler = new AdaptivePlaywrightCrawler({
requestHandler: async (context) => {
const { url } = context.request;
- const heading = (await context.querySelector('h1')).text();
+ const heading = (await context.querySelectorOne('h1')).text();
const requestHandlerMode = await (async () => {
try {