Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: keep same-domain filtering after an off-domain redirect (#4076)
Martin Adámekbd4771ece1cc
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex e74205696..dc74a4ffa 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -179,8 +179,7 @@ export function resolveBaseUrlForEnqueueLinksFiltering({}// If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match- // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything- // which is the intended behavior (since we went off domain)+ // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirelyif (enqueueStrategy === EnqueueStrategy.SameDomain) {const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false })!;const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false })!;@@ -189,7 +188,7 @@ export function resolveBaseUrlForEnqueueLinksFiltering({return finalUrlOrigin;}- return undefined;+ return originalUrlOrigin;}// Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirectdiff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex 06772447d..19be0b3c5 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -72,7 +72,7 @@ async function createRequestQueueMock(seedUrl = 'https://example.com') {// `enqueueLinks()`'s hostname/domain-based strategies see whatever start URL a test navigates to,// regardless of where the content is actually served from.class FixtureHttpClient extends BaseHttpClient {- constructor(private readonly html: string) {+ constructor(protected readonly html: string) {super();}@@ -415,6 +415,52 @@ describe('enqueueLinks()', () => {]);});+ test('keeps filtering by the original domain with the strategy of same-domain after an off-domain redirect', async () => {+ const { enqueued, requestQueue } = await createRequestQueueMock();+ // Serves the fixture HTML while reporting the load as if it had redirected off-domain.+ const redirectingHttpClient = new (class extends FixtureHttpClient {+ override async sendRequest(): Promise<Response> {+ return new ResponseWithUrl(this.html, {+ url: 'https://another.com/',+ status: 200,+ headers: { 'content-type': 'text/html; charset=utf-8' },+ });+ }+ })(HTML);++ const crawler = new CheerioCrawler({+ requestManager: requestQueue,+ httpClient: redirectingHttpClient,+ requestHandler: async ({ request, enqueueLinks }) => {+ if (request.url !== 'https://example.com') return;+ await enqueueLinks({ strategy: EnqueueStrategy.SameDomain });+ },+ });+ await crawler.run(['https://example.com']);++ expect(enqueued.map((r) => r.url)).toEqual([+ 'https://example.com/a/b/first',+ 'https://example.com/a/second',+ 'https://example.com/a/b/third',+ ]);+ });++ test('ignores an explicitly undefined baseUrl and keeps the resolved one', async () => {+ const { enqueued, requestQueue } = await createRequestQueueMock();+ await runCheerioEnqueueLinks(+ { strategy: EnqueueStrategy.SameDomain, baseUrl: undefined },+ { requestManager: requestQueue },+ );++ expect(enqueued.map((r) => r.url)).toEqual([+ 'https://example.com/a/b/first',+ 'https://example.com/a/second',+ 'https://example.com/a/b/third',+ 'https://example.com/x/absolutepath',+ 'https://example.com/y/relativepath',+ ]);+ });+test('correctly resolves relative URLs with the strategy of all', async () => {const { enqueued, requestQueue } = await createRequestQueueMock();await runCheerioEnqueueLinks(