Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(core): add default dataset helpers to `BasicCrawler` (#2057)
Jindřich Bäre2a7544ddf77
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 3ca127e19..74616eaf3 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -40,6 +40,7 @@ import {validators,RetryRequestError,SessionError,+ Dataset,} from '@crawlee/core';import type { Dictionary, Awaitable, BatchAddRequestsResult, SetStatusMessageOptions } from '@crawlee/types';import { ROTATE_PROXY_ERRORS } from '@crawlee/utils';@@ -874,6 +875,29 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn requestQueue.addRequestsBatched(requests, options);}+ /**+ * Pushes data to the default crawler {@apilink Dataset} by calling {@apilink Dataset.pushData}.+ */+ async pushData(...args: Parameters<Dataset['pushData']>): Promise<void> {+ const dataset = await Dataset.open(undefined, { config: this.config });+ return dataset.pushData(...args);+ }++ /**+ * Retrieves the default crawler {@apilink Dataset} by calling {@apilink Dataset.open}.+ */+ async getDataset(): Promise<Dataset> {+ return Dataset.open(undefined, { config: this.config });+ }++ /**+ * Retrieves data from the default crawler {@apilink Dataset} by calling {@apilink Dataset.getData}.+ */+ async getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']> {+ const dataset = await this.getDataset();+ return dataset.getData(...args);+ }+protected async _init(): Promise<void> {if (!this.events.isInitialized()) {await this.events.init();@@ -1068,6 +1092,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext...options,});},+ pushData: async (...args: Parameters<Dataset['pushData']>) => {+ return this.pushData(...args);+ },sendRequest: async (overrideOptions?: OptionsInit) => {const cookieJar = session ? {getCookieString: async (url: string) => session!.getCookieString(url),diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex dba824fe4..4a9e9c129 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -6,6 +6,7 @@ import type { Log } from '../log';import type { ProxyInfo } from '../proxy_configuration';import type { Request } from '../request';import type { Session } from '../session_pool/session';+import { type Dataset } from '../storages';// eslint-disable-next-line @typescript-eslint/ban-typesexport interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {@@ -51,6 +52,15 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary*/enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;+ /**+ * This function allows you to push data to the default {@apilink Dataset} currently used by the crawler.+ *+ * Shortcut for `crawler.pushData()`.+ *+ * @param [data] Data to be pushed to the default dataset.+ */+ pushData(...args: Parameters<Dataset['pushData']>): Promise<void>;+/*** Fires HTTP request via [`got-scraping`](https://crawlee.dev/docs/guides/got-scraping), allowing to override the request* options on the fly.diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 7d078d997..cb13ff2d7 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1381,4 +1381,56 @@ describe('BasicCrawler', () => {expect(crawler).toBeTruthy();});});++ describe('Dataset helpers, crawler paralellism', () => {+ const payload: Dictionary<any>[] = [{ foo: 'bar' }];+ const getPayload: (id: string) => Dictionary<any>[] = (id) => [{ foo: id }];++ test('should expose default Dataset methods', async () => {+ const crawler = new BasicCrawler();++ await crawler.pushData(payload);++ expect((await crawler.getData()).items)+ .toEqual(payload);+ });++ test('should expose pushData helper', async () => {+ const crawler = new BasicCrawler({+ requestHandler: ({ pushData }) => pushData(payload),+ });++ await crawler.run([{+ url: `http://${HOSTNAME}:${port}`,+ }]);++ expect((await crawler.getData()).items)+ .toEqual(payload);+ });++ test("Crawlers with different Configurations don't share Datasets", async () => {+ const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));+ const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));++ await crawlerA.pushData(getPayload('A'));+ await crawlerB.pushData(getPayload('B'));++ expect((await crawlerA.getData()).items)+ .toEqual(getPayload('A'));++ expect((await crawlerB.getData()).items)+ .toEqual(getPayload('B'));+ });++ test('Crawlers with different Configurations run separately', async () => {+ const crawlerA = new BasicCrawler({ requestHandler: () => {} }, new Configuration({ persistStorage: false }));+ const crawlerB = new BasicCrawler({ requestHandler: () => {} }, new Configuration({ persistStorage: false }));++ await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);+ await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);++ expect(crawlerA.stats.state.requestsFinished).toBe(1);+ expect(crawlerB.stats.state.requestsFinished).toBe(1);+ });+ });});diff --git a/test/core/multiple_crawlers.test.ts b/test/core/multiple_crawlers.test.tsdeleted file mode 100644index cd40f7cb7..000000000--- a/test/core/multiple_crawlers.test.ts+++ /dev/null@@ -1,31 +0,0 @@-import { MemoryStorage } from '@crawlee/memory-storage';-import { CheerioCrawler, Configuration } from 'crawlee';--describe('multiple crawlers', () => {- test('Crawler instances with different StorageClients do not affect each other', async () => {- const getCrawler = () => {- return new CheerioCrawler({- requestHandler: async () => {},- }, new Configuration({- storageClient: new MemoryStorage({- persistStorage: false,- }),- }));- };-- const a = getCrawler();-- await a.run([- { url: 'https://example.org/' },- ]);-- const b = getCrawler();-- await b.run([- { url: 'https://example.org/' },- ]);-- expect(a.stats.state.requestsFinished).toBe(1);- expect(b.stats.state.requestsFinished).toBe(1);- });-});