Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: expose linkedom in `crawlee` metapackage + fix e2e tests

Martin Adámek42ea9ebcc611
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index 606673b6e..5ee6132b7 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -61,6 +61,7 @@
"@crawlee/core": "^3.3.3",
"@crawlee/http": "^3.3.3",
"@crawlee/jsdom": "^3.3.3",
+ "@crawlee/linkedom": "^3.3.3",
"@crawlee/playwright": "^3.3.3",
"@crawlee/puppeteer": "^3.3.3",
"@crawlee/utils": "^3.3.3",
diff --git a/packages/crawlee/src/index.ts b/packages/crawlee/src/index.ts
index a7cee29be..591d4f85f 100644
--- a/packages/crawlee/src/index.ts
+++ b/packages/crawlee/src/index.ts
@@ -11,6 +11,7 @@ export * from '@crawlee/basic';
export * from '@crawlee/browser';
export * from '@crawlee/http';
export * from '@crawlee/jsdom';
+export * from '@crawlee/linkedom';
export * from '@crawlee/cheerio';
export * from '@crawlee/puppeteer';
export * from '@crawlee/playwright';
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index 67eea55fe..2a7f9a10c 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -7,9 +7,11 @@ import type {
RequestHandler,
EnqueueLinksOptions,
RequestQueue,
+ GetUserDataFromRequest,
+ RouterRoutes,
} from '@crawlee/http';
+import type { Dictionary } from '@crawlee/types';
import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';
-import type { Dictionary, GetUserDataFromRequest, RouterRoutes } from '@crawlee/core';
import { concatStreamToBuffer } from '@apify/utilities';
import { DOMParser } from 'linkedom/cached';
import type { IncomingMessage } from 'http';
diff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.ts
index fc94f7180..12b5e45b0 100644
--- a/test/e2e/linkedom-default-ts/actor/main.ts
+++ b/test/e2e/linkedom-default-ts/actor/main.ts
@@ -3,8 +3,7 @@ import { LinkeDOMCrawler, Dataset } from '@crawlee/linkedom';
if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {
// @ts-ignore
- const { ApifyStorageLocal } = await import('@apify/storage-local');
- await Actor.init({ storage: new ApifyStorageLocal() });
+ await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });
} else {
await Actor.init();
}