Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(jsdom,linkedom): Expose document to crawler router context (#1950)
Pavel 'Strajk' Dolecek4536dc2900ee
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 0e6403eda..d05016e1f 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -49,6 +49,7 @@ export interface JSDOMCrawlingContext<JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {window: DOMWindow;+ document: Document;/*** Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.@@ -261,6 +262,9 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {get body() {return window.document.documentElement.outerHTML;},+ get document() {+ return window.document;+ },enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {return domCrawlerEnqueueLinks({options: enqueueOptions,diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 2a7f9a10c..ca05f81a9 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -38,6 +38,12 @@ export interface LinkeDOMCrawlingContext<JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> extends InternalHttpCrawlingContext<UserData, JSONData, LinkeDOMCrawler> {window: Window;+ // Technically the document is not of type Document but of type either HTMLDocument or XMLDocument+ // from linkedom/types/{html/xml}/document, depending on the content type of the response+ // Using union of the real types would make writing the crawlers inconvenient,+ // so we specify the type as the native Document type from lib.dom.d.ts+ // even though it's not technically 100% correct+ document: Document;/*** Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.@@ -135,13 +141,17 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {protected override async _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: LinkeDOMCrawlingContext) {const body = await concatStreamToBuffer(response);- const document = LinkeDOMCrawler.parser.parseFromString(body.toString(), isXml ? 'text/html' : 'text/html');+ const document = LinkeDOMCrawler.parser.parseFromString(body.toString(), isXml ? 'text/xml' : 'text/html');return {window: document.defaultView,get body() {return document.documentElement.outerHTML;},+ get document() {+ // See comment about typing in LinkeDOMCrawlingContext definition+ return document as unknown as Document;+ },enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {return linkedomCrawlerEnqueueLinks({options: enqueueOptions,diff --git a/test/e2e/jsdom-default-ts/actor/main.ts b/test/e2e/jsdom-default-ts/actor/main.tsindex b8839a82a..dc8b6bc76 100644--- a/test/e2e/jsdom-default-ts/actor/main.ts+++ b/test/e2e/jsdom-default-ts/actor/main.ts@@ -1,5 +1,6 @@import { Actor } from 'apify';import { JSDOMCrawler, Dataset } from '@crawlee/jsdom';+import assert from 'node:assert';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {// @ts-ignore@@ -10,13 +11,15 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {const crawler = new JSDOMCrawler();-crawler.router.addDefaultHandler(async ({ window, enqueueLinks, request, log }) => {+crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({globs: ['https://crawlee.dev/docs/**'],});const pageTitle = window.document.title;+ const { title } = document;+ assert.strictEqual(pageTitle, title);log.info(`URL: ${url} TITLE: ${pageTitle}`);await Dataset.pushData({ url, pageTitle });diff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.tsindex 12b5e45b0..7ade2f017 100644--- a/test/e2e/linkedom-default-ts/actor/main.ts+++ b/test/e2e/linkedom-default-ts/actor/main.ts@@ -1,5 +1,6 @@import { Actor } from 'apify';import { LinkeDOMCrawler, Dataset } from '@crawlee/linkedom';+import assert from 'node:assert';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {// @ts-ignore@@ -10,13 +11,15 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {const crawler = new LinkeDOMCrawler();-crawler.router.addDefaultHandler(async ({ window, enqueueLinks, request, log }) => {+crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({globs: ['https://crawlee.dev/docs/**'],});const pageTitle = window.document.title;+ const { title } = document;+ assert.strictEqual(pageTitle, title);log.info(`URL: ${url} TITLE: ${pageTitle}`);await Dataset.pushData({ url, pageTitle });