Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: export cheerio types in all crawler packages (#3204)
Matouš Maříkf05790b8c4e7
diff --git a/packages/basic-crawler/src/index.ts b/packages/basic-crawler/src/index.tsindex f774a1f31..ba211fc2b 100644--- a/packages/basic-crawler/src/index.ts+++ b/packages/basic-crawler/src/index.ts@@ -1,3 +1,4 @@export * from '@crawlee/core';export * from './internals/basic-crawler';export * from './internals/constants';+export { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils';diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex 6a42ec70e..2b4b79dcb 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -4,7 +4,9 @@ import * as cheerio from 'cheerio';import { tryAbsoluteURL } from './extract-urls';+/** @deprecated use CheerioAPI instead */export type CheerioRoot = ReturnType<typeof load>;+export type { CheerioAPI, Cheerio, Element } from 'cheerio';// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 9e663d13a..b73065804 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1,7 +1,16 @@import type { IncomingHttpHeaders, Server } from 'node:http';import { Readable } from 'node:stream';-import type { CheerioCrawlingContext, CheerioRequestHandler, ProxyInfo, Source } from '@crawlee/cheerio';+import type {+ Cheerio,+ CheerioAPI,+ CheerioCrawlingContext,+ CheerioRequestHandler,+ CheerioRoot,+ Element,+ ProxyInfo,+ Source,+} from '@crawlee/cheerio';import {AutoscaledPool,CheerioCrawler,@@ -1257,6 +1266,23 @@ describe('CheerioCrawler', () => {});await cheerioCrawler.run();});++ test('should have correct types in crawling context', async () => {+ const requestHandler = (crawlingContext: CheerioCrawlingContext) => {+ // Checking that types are correct+ const _cheerioRootType: CheerioRoot = crawlingContext.$;+ const _apiType: CheerioAPI = crawlingContext.$;+ const _cheerioElementType: Cheerio<Element> = crawlingContext.$('div');+ };++ const cheerioCrawler = new CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ requestHandler,+ });+ await cheerioCrawler.run();+ });});describe('use', () => {diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex d4da700cb..ec65e5697 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -2,7 +2,16 @@ import type { Server } from 'node:http';import type { AddressInfo } from 'node:net';import os from 'node:os';-import type { PlaywrightGotoOptions, PlaywrightRequestHandler, Request } from '@crawlee/playwright';+import type {+ Cheerio,+ CheerioAPI,+ CheerioRoot,+ Element,+ PlaywrightCrawlingContext,+ PlaywrightGotoOptions,+ PlaywrightRequestHandler,+ Request,+} from '@crawlee/playwright';import { PlaywrightCrawler, RequestList } from '@crawlee/playwright';import express from 'express';import playwright from 'playwright';@@ -197,4 +206,23 @@ describe('PlaywrightCrawler', () => {expect(reducedMotion).toBe(launchOptions.reducedMotion);},);++ test('should have correct types in crawling context', async () => {+ const requestHandler = async (crawlingContext: PlaywrightCrawlingContext) => {+ // Checking that types are correct+ const $ = await crawlingContext.parseWithCheerio();++ const _cheerioRootType: CheerioRoot = $;+ const _apiType: CheerioAPI = $;+ const _cheerioElementType: Cheerio<Element> = $('div');+ };++ const playwrightCrawler = new PlaywrightCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ requestHandler,+ });+ await playwrightCrawler.run();+ });});diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex 422f30bde..fc98254a0 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -5,7 +5,15 @@ import type { AddressInfo } from 'node:net';import os from 'node:os';import { promisify } from 'node:util';-import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, Request } from '@crawlee/puppeteer';+import type {+ Cheerio,+ CheerioAPI,+ CheerioRoot,+ Element,+ PuppeteerCrawlingContext,+ PuppeteerGoToOptions,+ Request,+} from '@crawlee/puppeteer';import { ProxyConfiguration, PuppeteerCrawler, RequestList, RequestQueue, Session } from '@crawlee/puppeteer';import type { Cookie } from '@crawlee/types';import { sleep } from '@crawlee/utils';@@ -436,4 +444,23 @@ describe('PuppeteerCrawler', () => {expect(count[2] + count[3] + count[4]).toBe(6);});}++ test('should have correct types in crawling context', async () => {+ const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {+ // Checking that types are correct+ const $ = await crawlingContext.parseWithCheerio();++ const _cheerioRootType: CheerioRoot = $;+ const _apiType: CheerioAPI = $;+ const _cheerioElementType: Cheerio<Element> = $('div');+ };++ const puppeteerCrawler = new PuppeteerCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ requestHandler,+ });+ await puppeteerCrawler.run();+ });});