Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix linkedom e2e test
Martin Adámek47b7c7e3e08d
diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex 8cf52fe40..e44566e40 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.3.1",- "@crawlee/types": "^3.3.1",+ "@crawlee/http": "^3.3.3",+ "@crawlee/types": "^3.3.3","linkedom": "^0.14.25","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/test/e2e/.eslintrc.json b/test/e2e/.eslintrc.jsonindex 486906d01..a81cd4b26 100644--- a/test/e2e/.eslintrc.json+++ b/test/e2e/.eslintrc.json@@ -11,6 +11,7 @@"**/*.d.ts"],"rules": {+ "@typescript-eslint/ban-ts-comment": 0,"import/extensions": 0,"import/no-extraneous-dependencies": 0}diff --git a/test/e2e/cheerio-default-ts/actor/main.ts b/test/e2e/cheerio-default-ts/actor/main.tsindex e6325aab3..38fa35e1b 100644--- a/test/e2e/cheerio-default-ts/actor/main.ts+++ b/test/e2e/cheerio-default-ts/actor/main.ts@@ -2,6 +2,7 @@ import { Actor } from 'apify';import { CheerioCrawler, Dataset } from '@crawlee/cheerio';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ // @ts-ignoreawait Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });} else {await Actor.init();diff --git a/test/e2e/jsdom-default-ts/actor/main.ts b/test/e2e/jsdom-default-ts/actor/main.tsindex f6a8d463a..b8839a82a 100644--- a/test/e2e/jsdom-default-ts/actor/main.ts+++ b/test/e2e/jsdom-default-ts/actor/main.ts@@ -2,6 +2,7 @@ import { Actor } from 'apify';import { JSDOMCrawler, Dataset } from '@crawlee/jsdom';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ // @ts-ignoreawait Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });} else {await Actor.init();diff --git a/test/e2e/jsdom-react-ts/actor/main.ts b/test/e2e/jsdom-react-ts/actor/main.tsindex 58fc971dc..dc841cb0e 100644--- a/test/e2e/jsdom-react-ts/actor/main.ts+++ b/test/e2e/jsdom-react-ts/actor/main.ts@@ -2,6 +2,7 @@ import { Actor } from 'apify';import { JSDOMCrawler, Dataset } from '@crawlee/jsdom';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ // @ts-ignoreawait Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });} else {await Actor.init();diff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.tsindex 4a54374a1..fc94f7180 100644--- a/test/e2e/linkedom-default-ts/actor/main.ts+++ b/test/e2e/linkedom-default-ts/actor/main.ts@@ -2,6 +2,7 @@ import { Actor } from 'apify';import { LinkeDOMCrawler, Dataset } from '@crawlee/linkedom';if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ // @ts-ignoreconst { ApifyStorageLocal } = await import('@apify/storage-local');await Actor.init({ storage: new ApifyStorageLocal() });} else {@@ -10,13 +11,13 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {const crawler = new LinkeDOMCrawler();-crawler.router.addDefaultHandler(async ({ document, enqueueLinks, request, log }) => {+crawler.router.addDefaultHandler(async ({ window, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({globs: ['https://crawlee.dev/docs/**'],});- const pageTitle = document.title;+ const pageTitle = window.document.title;log.info(`URL: ${url} TITLE: ${pageTitle}`);await Dataset.pushData({ url, pageTitle });diff --git a/yarn.lock b/yarn.lockindex 9e1d2e6aa..822f5d6e3 100644--- a/yarn.lock+++ b/yarn.lock@@ -849,7 +849,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@^3.3.1, @crawlee/http@^3.3.3, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/http@^3.3.3, @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:@@ -891,8 +891,8 @@ __metadata:dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.1.4- "@crawlee/http": ^3.3.1- "@crawlee/types": ^3.3.1+ "@crawlee/http": ^3.3.3+ "@crawlee/types": ^3.3.3linkedom: ^0.14.25ow: ^0.28.2tslib: ^2.4.0@@ -1029,7 +1029,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@^3.3.0, @crawlee/types@^3.3.1, @crawlee/types@^3.3.3, @crawlee/types@workspace:packages/types":+"@crawlee/types@^3.3.0, @crawlee/types@^3.3.3, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -4970,9 +4970,9 @@ __metadata:linkType: hard"devtools-protocol@npm:*":- version: 0.0.1152884- resolution: "devtools-protocol@npm:0.0.1152884"- checksum: e670c916c26fa337e9ab4f0c047f732977c7336f5ca16b2c4a19132fa66b66aa6c6bb9ef3fd0d2d3a1ccef4871311b786aaaee92e5a95304d3b7b921dc56ff4b+ version: 0.0.1154250+ resolution: "devtools-protocol@npm:0.0.1154250"+ checksum: 5c30eb40b9b484ddb9e98f2017dbd18cb14da42719a811eca24a909457eb61d860da31b50eae399f8a24135f192a1dbdb7d0e6bd920fafae4f32e2b49334e359languageName: nodelinkType: hard