Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: accessing crawler state, key-value store and named datasets via crawling context (#2283)
Jan Buchar58dd5fcc25f3
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex db0bf38ae..d966a670b 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -954,18 +954,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}/**- * Pushes data to the default crawler {@apilink Dataset} by calling {@apilink Dataset.pushData}.+ * Pushes data to the specified {@apilink Dataset}, or the default crawler {@apilink Dataset} by calling {@apilink Dataset.pushData}.*/- async pushData(...args: Parameters<Dataset['pushData']>): Promise<void> {- const dataset = await this.getDataset();- return dataset.pushData(...args);+ async pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void> {+ const dataset = await this.getDataset(datasetIdOrName);+ return dataset.pushData(data);}/**- * Retrieves the default crawler {@apilink Dataset}.+ * Retrieves the specified {@apilink Dataset}, or the default crawler {@apilink Dataset}.*/- async getDataset(): Promise<Dataset> {- return Dataset.open(undefined, { config: this.config });+ async getDataset(idOrName?: string): Promise<Dataset> {+ return Dataset.open(idOrName, { config: this.config });}/**@@ -1213,9 +1213,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext...options,});},- pushData: async (...args: Parameters<Dataset['pushData']>) => {- return this.pushData(...args);- },+ addRequests: this.addRequests.bind(this),+ pushData: this.pushData.bind(this),sendRequest: async (overrideOptions?: OptionsInit) => {const cookieJar = session ? {getCookieString: async (url: string) => session!.getCookieString(url),@@ -1239,6 +1238,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextcookieJar,});},+ getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),};this.crawlingContexts.set(crawlingContext.id, crawlingContext);diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 049c807df..b42e43933 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -5,17 +5,81 @@ import type { Response as GotResponse, OptionsInit } from 'got-scraping';import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links';import type { Log } from '../log';import type { ProxyInfo } from '../proxy_configuration';-import type { Request } from '../request';+import type { Request, Source } from '../request';import type { Session } from '../session_pool/session';-import { type Dataset } from '../storages';+import type { RequestQueueOperationOptions, Dataset, KeyValueStore } from '../storages';// eslint-disable-next-line @typescript-eslint/ban-types-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {- id: string;+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown>{/*** The original {@apilink Request} object.*/request: Request<UserData>;++ /**+ * This function allows you to push data to a {@apilink Dataset} specified by name, or the one currently used by the crawler.+ *+ * Shortcut for `crawler.pushData()`.+ *+ * @param [data] Data to be pushed to the default dataset.+ */+ pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void>;++ /**+ * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}+ * currently used by the crawler.+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions+ * and override settings of the enqueued {@apilink Request} objects.+ *+ * Check out the [Crawl a website with relative links](https://crawlee.dev/docs/examples/crawl-relative-links) example+ * for more details regarding its usage.+ *+ * **Example usage**+ *+ * ```ts+ * async requestHandler({ enqueueLinks }) {+ * await enqueueLinks({+ * globs: [+ * 'https://www.example.com/handbags/*',+ * ],+ * });+ * },+ * ```+ *+ * @param [options] All `enqueueLinks()` parameters are passed via an options object.+ */+ enqueueLinks: (options?: Omit<EnqueueLinksOptions, 'requestQueue'>) => Promise<unknown>;++ /**+ * Add requests directly to the request queue.+ *+ * @param requests The requests to add+ * @param options Options for the request queue+ */+ addRequests: (+ requestsLike: (string | Source)[],+ options?: RequestQueueOperationOptions,+ ) => Promise<void>;++ /**+ * Returns the state - a piece of mutable persistent data shared across all the request handler runs.+ */+ useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;++ /**+ * Get a key-value store with given name or id, or the default one for the crawler.+ */+ getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue'>>;++ /**+ * A preconfigured logger for the request handler.+ */+ log: Log;+}++export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {+ id: string;session?: Session;/**@@ -23,7 +87,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary* and configured by the {@apilink ProxyConfiguration} class.*/proxyInfo?: ProxyInfo;- log: Log;+crawler: Crawler;/**@@ -54,13 +118,9 @@ export interface CrawlingContext<Crawler = unknown, UserData extends DictionaryenqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;/**- * This function allows you to push data to the default {@apilink Dataset} currently used by the crawler.- *- * Shortcut for `crawler.pushData()`.- *- * @param [data] Data to be pushed to the default dataset.+ * Get a key-value store with given name or id, or the default one for the crawler.*/- pushData(...args: Parameters<Dataset['pushData']>): Promise<void>;+ getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;/*** Fires HTTP request via [`got-scraping`](https://crawlee.dev/docs/guides/got-scraping), allowing to override the request