Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: make enqueue-links-base e2e tests hermetic (#3674)
Martin Adámek377384514e80
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/main.js b/test/e2e/cheerio-enqueue-links-base/actor/main.jsindex ac2ad0848..0f3c22c0b 100644--- a/test/e2e/cheerio-enqueue-links-base/actor/main.js+++ b/test/e2e/cheerio-enqueue-links-base/actor/main.js@@ -1,6 +1,41 @@+import http from 'node:http';+import { Actor } from 'apify';import { CheerioCrawler, Dataset } from '@crawlee/cheerio';+// Self-contained fixture: a start page that uses <base href="/sub/"> so the+// crawler can only discover the linked pages if it honors the base href when+// resolving the otherwise-relative <a href> values.+const pages = {+ '/start': `<!doctype html>+<html><head><title>Start</title><base href="/sub/"></head>+<body>+ <a href="a">A</a>+ <a href="b">B</a>+ <a href="c">C</a>+ <a href="/elsewhere">Elsewhere (absolute)</a>+</body></html>`,+ '/sub/a': '<!doctype html><html><head><title>A</title></head><body>A</body></html>',+ '/sub/b': '<!doctype html><html><head><title>B</title></head><body>B</body></html>',+ '/sub/c': '<!doctype html><html><head><title>C</title></head><body>C</body></html>',+ '/elsewhere': '<!doctype html><html><head><title>Elsewhere</title></head><body>Elsewhere</body></html>',+};++const server = http.createServer((req, res) => {+ const body = pages[req.url];+ if (body === undefined) {+ res.statusCode = 404;+ res.end('Not Found');+ return;+ }+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });+ res.end(body);+});++await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));+const { port } = server.address();+const baseUrl = `http://127.0.0.1:${port}`;+const mainOptions = {exit: Actor.isAtHome(),storage:@@ -12,7 +47,6 @@ const mainOptions = {await Actor.main(async () => {const crawler = new CheerioCrawler({maxRequestsPerCrawl: 30,- proxyConfiguration: await Actor.createProxyConfiguration(),async requestHandler({ $, enqueueLinks, request, log }) {const { url, loadedUrl } = request;@@ -20,15 +54,16 @@ await Actor.main(async () => {log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);await Dataset.pushData({ url, loadedUrl, pageTitle });+ // Only the /sub/ links should match — the absolute /elsewhere link+ // and any unresolved relative URLs (which would land at /a, /b, /c+ // without base-href handling) are filtered out by the glob.await enqueueLinks({- globs: [- 'https://www.jamesallen.com/about-us/**',- 'https://www.jamesallen.com/terms-of-use/**',- 'https://www.jamesallen.com/guarantee/**',- ],+ globs: [`${baseUrl}/sub/**`],});},});- await crawler.run(['https://www.jamesallen.com/faq/']);+ await crawler.run([`${baseUrl}/start`]);}, mainOptions);++server.close();diff --git a/test/e2e/cheerio-enqueue-links-base/test.mjs b/test/e2e/cheerio-enqueue-links-base/test.mjsindex 502745fdd..1af2a7738 100644--- a/test/e2e/cheerio-enqueue-links-base/test.mjs+++ b/test/e2e/cheerio-enqueue-links-base/test.mjs@@ -5,4 +5,4 @@ await initialize(testActorDirname);const { datasetItems } = await runActor(testActorDirname);-await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);+await expect(datasetItems.length === 4, `Enqueueing respects <base href>`);diff --git a/test/e2e/playwright-enqueue-links-base/actor/main.js b/test/e2e/playwright-enqueue-links-base/actor/main.jsindex 7364d2bb9..bf2398293 100644--- a/test/e2e/playwright-enqueue-links-base/actor/main.js+++ b/test/e2e/playwright-enqueue-links-base/actor/main.js@@ -1,6 +1,41 @@+import http from 'node:http';+import { Actor } from 'apify';import { PlaywrightCrawler, Dataset } from '@crawlee/playwright';+// Self-contained fixture: a start page that uses <base href="/sub/"> so the+// crawler can only discover the linked pages if it honors the base href when+// resolving the otherwise-relative <a href> values.+const pages = {+ '/start': `<!doctype html>+<html><head><title>Start</title><base href="/sub/"></head>+<body>+ <a href="a">A</a>+ <a href="b">B</a>+ <a href="c">C</a>+ <a href="/elsewhere">Elsewhere (absolute)</a>+</body></html>`,+ '/sub/a': '<!doctype html><html><head><title>A</title></head><body>A</body></html>',+ '/sub/b': '<!doctype html><html><head><title>B</title></head><body>B</body></html>',+ '/sub/c': '<!doctype html><html><head><title>C</title></head><body>C</body></html>',+ '/elsewhere': '<!doctype html><html><head><title>Elsewhere</title></head><body>Elsewhere</body></html>',+};++const server = http.createServer((req, res) => {+ const body = pages[req.url];+ if (body === undefined) {+ res.statusCode = 404;+ res.end('Not Found');+ return;+ }+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });+ res.end(body);+});++await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));+const { port } = server.address();+const baseUrl = `http://127.0.0.1:${port}`;+const mainOptions = {exit: Actor.isAtHome(),storage:@@ -12,7 +47,6 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({maxRequestsPerCrawl: 30,- proxyConfiguration: await Actor.createProxyConfiguration(),async requestHandler({ parseWithCheerio, enqueueLinks, request, log }) {const { url, loadedUrl } = request;@@ -21,15 +55,16 @@ await Actor.main(async () => {log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);await Dataset.pushData({ url, loadedUrl, pageTitle });+ // Only the /sub/ links should match — the absolute /elsewhere link+ // and any unresolved relative URLs (which would land at /a, /b, /c+ // without base-href handling) are filtered out by the glob.await enqueueLinks({- globs: [- 'https://www.jamesallen.com/about-us/**',- 'https://www.jamesallen.com/terms-of-use/**',- 'https://www.jamesallen.com/guarantee/**',- ],+ globs: [`${baseUrl}/sub/**`],});},});- await crawler.run(['https://www.jamesallen.com/faq/']);+ await crawler.run([`${baseUrl}/start`]);}, mainOptions);++server.close();diff --git a/test/e2e/playwright-enqueue-links-base/test.mjs b/test/e2e/playwright-enqueue-links-base/test.mjsindex e07a7890a..1af2a7738 100644--- a/test/e2e/playwright-enqueue-links-base/test.mjs+++ b/test/e2e/playwright-enqueue-links-base/test.mjs@@ -1,10 +1,8 @@-import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';--await skipTest('too flaky');+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);const { datasetItems } = await runActor(testActorDirname);-await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);+await expect(datasetItems.length === 4, `Enqueueing respects <base href>`);