Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: refactor the `enqueueLinks` options type
Martin Adámek26a1a9625f4f
diff --git a/package-lock.json b/package-lock.jsonindex 441d847f1..c29166c67 100644--- a/package-lock.json+++ b/package-lock.json@@ -16251,7 +16251,8 @@"@crawlee/types": "^3.0.4","@crawlee/utils": "^3.0.4","got-scraping": "^3.2.9",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "^2.19.0"},"engines": {"node": ">=16.0.0"@@ -16364,7 +16365,8 @@"stream-chain": "^2.2.5","stream-json": "^1.7.4","tldts": "^5.7.80",- "tough-cookie": "^4.0.0"+ "tough-cookie": "^4.0.0",+ "type-fest": "^2.19.0"},"engines": {"node": ">=16.0.0"@@ -17630,7 +17632,8 @@"@crawlee/types": "^3.0.4","@crawlee/utils": "^3.0.4","got-scraping": "^3.2.9",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "^2.19.0"}},"@crawlee/browser": {@@ -17703,7 +17706,8 @@"stream-chain": "^2.2.5","stream-json": "^1.7.4","tldts": "^5.7.80",- "tough-cookie": "^4.0.0"+ "tough-cookie": "^4.0.0",+ "type-fest": "^2.19.0"},"dependencies": {"brace-expansion": {diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex ccbc29ea9..f5727964a 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -52,6 +52,7 @@"@crawlee/types": "^3.0.4","@crawlee/utils": "^3.0.4","got-scraping": "^3.2.9",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "^2.19.0"}}diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 17654dabf..b96fc211e 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -2,6 +2,7 @@ import type { Log } from '@apify/log';import defaultLog from '@apify/log';import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';+import type { SetRequired } from 'type-fest';import type {AutoscaledPoolOptions,EnqueueLinksOptions,@@ -37,13 +38,38 @@ import {} from '@crawlee/core';import type { Method, OptionsInit } from 'got-scraping';import { gotScraping } from 'got-scraping';-import type { ProcessedRequest, Dictionary, Awaitable } from '@crawlee/types';+import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';import { chunk, sleep } from '@crawlee/utils';import ow, { ArgumentError } from 'ow';export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary,-> extends CrawlingContext<BasicCrawler, UserData> {}+> extends CrawlingContext<BasicCrawler, UserData> {+ /**+ * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}+ * currently used by the crawler.+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions+ * and override settings of the enqueued {@apilink Request} objects.+ *+ * Check out the [Crawl a website with relative links](https://crawlee.dev/docs/examples/crawl-relative-links) example+ * for more details regarding its usage.+ *+ * **Example usage**+ *+ * ```ts+ * async requestHandler({ enqueueLinks }) {+ * await enqueueLinks({+ * urls: [...],+ * });+ * },+ * ```+ *+ * @param [options] All `enqueueLinks()` parameters are passed via an options object.+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.+ */+ enqueueLinks(options?: SetRequired<EnqueueLinksOptions, 'urls'>): Promise<BatchAddRequestsResult>;+}/*** Since there's no set number of seconds before the container is terminated after@@ -851,11 +877,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextlog: this.log,request,session,- enqueueLinks: async (enqueueOptions: Partial<EnqueueLinksOptions>) => {+ enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {return enqueueLinks({// specify the RQ first to allow overriding itrequestQueue: await this.getRequestQueue(),- ...enqueueOptions,+ ...options,});},sendRequest: async (overrideOptions?: OptionsInit) => {diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex e9c6d8777..af891e760 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -655,7 +655,7 @@ export abstract class BrowserCrawler</** @internal */interface EnqueueLinksInternalOptions {- options?: Partial<EnqueueLinksOptions>;+ options?: EnqueueLinksOptions;page: CommonPage;requestQueue: RequestQueue;originalRequestUrl: string;diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex c30c72056..9f6dfba83 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -178,7 +178,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {}interface EnqueueLinksInternalOptions {- options?: Partial<EnqueueLinksOptions>;+ options?: EnqueueLinksOptions;$: cheerio.CheerioAPI | null;requestQueue: RequestQueue;originalRequestUrl: string;diff --git a/packages/core/package.json b/packages/core/package.jsonindex 4877f486b..0f94c2c17 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -70,6 +70,7 @@"stream-chain": "^2.2.5","stream-json": "^1.7.4","tldts": "^5.7.80",- "tough-cookie": "^4.0.0"+ "tough-cookie": "^4.0.0",+ "type-fest": "^2.19.0"}}diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 5aea6b378..fcf3e1b07 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -45,10 +45,10 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary* },* ```*- * @param options All `enqueueLinks()` parameters are passed via an options object.+ * @param [options] All `enqueueLinks()` parameters are passed via an options object.* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.*/- enqueueLinks(options: Partial<EnqueueLinksOptions>): Promise<BatchAddRequestsResult>;+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;/*** Fires HTTP request via [`got-scraping`](https://crawlee.dev/docs/guides/got-scraping), allowing to override the requestdiff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex 15fef676d..25a0eb65f 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -1,6 +1,7 @@import { getDomain } from 'tldts';import ow from 'ow';import log from '@apify/log';+import type { SetRequired } from 'type-fest';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, UrlPatternObject } from './shared';import {@@ -22,7 +23,7 @@ export interface EnqueueLinksOptions {urls?: string[];/** A request queue to which the URLs will be enqueued. */- requestQueue: RequestQueue;+ requestQueue?: RequestQueue;/** A CSS selector matching links to be enqueued. */selector?: string;@@ -164,7 +165,7 @@ export enum EnqueueStrategy {* @param options All `enqueueLinks()` parameters are passed via an options object.* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.*/-export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult> {+export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>): Promise<BatchAddRequestsResult> {ow(options, ow.object.exactShape({urls: ow.array.ofType(ow.string),requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 567947f78..00643cb36 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -138,7 +138,7 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {}interface EnqueueLinksInternalOptions {- options?: Partial<EnqueueLinksOptions>;+ options?: EnqueueLinksOptions;window: DOMWindow | null;requestQueue: RequestQueue;originalRequestUrl: string;