Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: expose linkedom in `crawlee` metapackage + fix e2e tests
Martin Adámek42ea9ebcc611
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 606673b6e..5ee6132b7 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -61,6 +61,7 @@"@crawlee/core": "^3.3.3","@crawlee/http": "^3.3.3","@crawlee/jsdom": "^3.3.3",+ "@crawlee/linkedom": "^3.3.3","@crawlee/playwright": "^3.3.3","@crawlee/puppeteer": "^3.3.3","@crawlee/utils": "^3.3.3",diff --git a/packages/crawlee/src/index.ts b/packages/crawlee/src/index.tsindex a7cee29be..591d4f85f 100644--- a/packages/crawlee/src/index.ts+++ b/packages/crawlee/src/index.ts@@ -11,6 +11,7 @@ export * from '@crawlee/basic';export * from '@crawlee/browser';export * from '@crawlee/http';export * from '@crawlee/jsdom';+export * from '@crawlee/linkedom';export * from '@crawlee/cheerio';export * from '@crawlee/puppeteer';export * from '@crawlee/playwright';diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 67eea55fe..2a7f9a10c 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -7,9 +7,11 @@ import type {RequestHandler,EnqueueLinksOptions,RequestQueue,+ GetUserDataFromRequest,+ RouterRoutes,} from '@crawlee/http';+import type { Dictionary } from '@crawlee/types';import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';-import type { Dictionary, GetUserDataFromRequest, RouterRoutes } from '@crawlee/core';import { concatStreamToBuffer } from '@apify/utilities';import { DOMParser } from 'linkedom/cached';import type { IncomingMessage } from 'http';diff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.tsindex fc94f7180..12b5e45b0 100644--- a/test/e2e/linkedom-default-ts/actor/main.ts+++ b/test/e2e/linkedom-default-ts/actor/main.ts@@ -3,8 +3,7 @@ import { LinkeDOMCrawler, Dataset } from '@crawlee/linkedom';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {// @ts-ignore- const { ApifyStorageLocal } = await import('@apify/storage-local');- await Actor.init({ storage: new ApifyStorageLocal() });+ await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });} else {await Actor.init();}