Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(e2e): fix tests after the documentation redesign (#2905)
Jindřich Bär6b7bb80da59a
diff --git a/test/e2e/cheerio-default-ts/actor/main.ts b/test/e2e/cheerio-default-ts/actor/main.tsindex 7714c7973..93c1b9332 100644--- a/test/e2e/cheerio-default-ts/actor/main.ts+++ b/test/e2e/cheerio-default-ts/actor/main.ts@@ -13,7 +13,7 @@ const crawler = new CheerioCrawler();crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = $('title').first().text();@@ -22,6 +22,6 @@ crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => {await Dataset.pushData({ url, pageTitle });});-await crawler.run(['https://crawlee.dev/docs/quick-start']);+await crawler.run(['https://crawlee.dev/js/docs/quick-start']);await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/cheerio-default/actor/main.js b/test/e2e/cheerio-default/actor/main.jsindex a1a8b9357..f881af85a 100644--- a/test/e2e/cheerio-default/actor/main.js+++ b/test/e2e/cheerio-default/actor/main.js@@ -20,7 +20,7 @@ await Actor.main(async () => {async requestHandler({ $, enqueueLinks, request, log }) {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = $('title').first().text();@@ -30,5 +30,5 @@ await Actor.main(async () => {},});- await crawler.run(['https://crawlee.dev/docs/quick-start']);+ await crawler.run(['https://crawlee.dev/js/docs/quick-start']);}, mainOptions);diff --git a/test/e2e/cheerio-max-requests/actor/main.js b/test/e2e/cheerio-max-requests/actor/main.jsindex 0b0e27bd2..dbf711ff3 100644--- a/test/e2e/cheerio-max-requests/actor/main.js+++ b/test/e2e/cheerio-max-requests/actor/main.js@@ -49,5 +49,5 @@ await Actor.main(async () => {},});- await crawler.run([{ url: 'https://crawlee.dev/docs/examples', userData: { label: 'START' } }]);+ await crawler.run([{ url: 'https://crawlee.dev/js/docs/examples', userData: { label: 'START' } }]);}, mainOptions);diff --git a/test/e2e/cheerio-page-info/actor/main.js b/test/e2e/cheerio-page-info/actor/main.jsindex 3d4eaf06f..f82298872 100644--- a/test/e2e/cheerio-page-info/actor/main.js+++ b/test/e2e/cheerio-page-info/actor/main.js@@ -40,5 +40,5 @@ await Actor.main(async () => {requestHandler: router,});- await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);+ await crawler.run([{ url: 'https://crawlee.dev/js/docs/3.0/examples', userData: { label: 'START' } }]);}, mainOptions);diff --git a/test/e2e/cheerio-request-queue-v2/actor/main.js b/test/e2e/cheerio-request-queue-v2/actor/main.jsindex fc5336197..60e9c81b5 100644--- a/test/e2e/cheerio-request-queue-v2/actor/main.js+++ b/test/e2e/cheerio-request-queue-v2/actor/main.js@@ -14,7 +14,7 @@ await Actor.main(async () => {async requestHandler({ $, enqueueLinks, request, log }) {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = $('title').first().text();@@ -32,7 +32,7 @@ await Actor.main(async () => {});try {- await crawler.run(['https://crawlee.dev/docs/quick-start']);+ await crawler.run(['https://crawlee.dev/js/docs/quick-start']);} catch (e) {console.error(e);}diff --git a/test/e2e/cheerio-stop-resume-ts/actor/main.ts b/test/e2e/cheerio-stop-resume-ts/actor/main.tsindex 8f14b3068..887f0f088 100644--- a/test/e2e/cheerio-stop-resume-ts/actor/main.ts+++ b/test/e2e/cheerio-stop-resume-ts/actor/main.ts@@ -14,7 +14,7 @@ const crawler = new CheerioCrawler();crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = $('title').first().text();@@ -24,8 +24,8 @@ crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => {if (requestCount++ > 10) crawler.stop();});-await crawler.run(['https://crawlee.dev/docs/quick-start']);+await crawler.run(['https://crawlee.dev/js/docs/quick-start']);requestCount = 0;-await crawler.run(['https://crawlee.dev/docs/quick-start'], { purgeRequestQueue: false });+await crawler.run(['https://crawlee.dev/js/docs/quick-start'], { purgeRequestQueue: false });await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/jsdom-default-ts/actor/main.ts b/test/e2e/jsdom-default-ts/actor/main.tsindex 3c56b294f..6e16608c8 100644--- a/test/e2e/jsdom-default-ts/actor/main.ts+++ b/test/e2e/jsdom-default-ts/actor/main.ts@@ -15,7 +15,7 @@ const crawler = new JSDOMCrawler();crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = window.document.title;@@ -26,6 +26,6 @@ crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, requesawait Dataset.pushData({ url, pageTitle });});-await crawler.run(['https://crawlee.dev/docs/quick-start']);+await crawler.run(['https://crawlee.dev/js/docs/quick-start']);await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/linkedom-default-ts/actor/main.ts b/test/e2e/linkedom-default-ts/actor/main.tsindex d5c07a987..de5cd4b13 100644--- a/test/e2e/linkedom-default-ts/actor/main.ts+++ b/test/e2e/linkedom-default-ts/actor/main.ts@@ -15,7 +15,7 @@ const crawler = new LinkeDOMCrawler();crawler.router.addDefaultHandler(async ({ document, enqueueLinks, request, log }) => {const { url } = request;await enqueueLinks({- globs: ['https://crawlee.dev/docs/**'],+ globs: ['https://crawlee.dev/js/docs/**'],});const pageTitle = document.querySelector('title')?.textContent ?? '';@@ -25,6 +25,6 @@ crawler.router.addDefaultHandler(async ({ document, enqueueLinks, request, log }await Dataset.pushData({ url, pageTitle });});-await crawler.run(['https://crawlee.dev/docs/quick-start']);+await crawler.run(['https://crawlee.dev/js/docs/quick-start']);await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.jsindex dc10bacf8..bdda681cf 100644--- a/test/e2e/playwright-default/actor/main.js+++ b/test/e2e/playwright-default/actor/main.js@@ -26,5 +26,5 @@ await Actor.main(async () => {},});- await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);+ await crawler.run(['https://crawlee.dev/js/docs/3.0/examples/']);}, mainOptions);diff --git a/test/e2e/playwright-multi-run/actor/main.js b/test/e2e/playwright-multi-run/actor/main.jsindex 1f8e065ce..3f10d8b58 100644--- a/test/e2e/playwright-multi-run/actor/main.js+++ b/test/e2e/playwright-multi-run/actor/main.js@@ -19,8 +19,8 @@ const crawler = new PlaywrightCrawler({},});-await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);-await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);-await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);+await crawler.run(['https://crawlee.dev/js/docs/3.0/examples/']);+await crawler.run(['https://crawlee.dev/js/docs/3.0/examples/']);+await crawler.run(['https://crawlee.dev/js/docs/3.0/examples/']);await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.jsindex fde7a900a..46d92d652 100644--- a/test/e2e/puppeteer-default/actor/main.js+++ b/test/e2e/puppeteer-default/actor/main.js@@ -27,5 +27,5 @@ await Actor.main(async () => {},});- await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);+ await crawler.run(['https://crawlee.dev/js/docs/3.0/examples/']);}, mainOptions);