Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: regression in resolving the base url for enqueue link filtering (#1422)
Vlad Frangu8a00cf6f35ad
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex e2943b813..17a9c61fd 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -294,6 +294,20 @@ export function resolveBaseUrlForEnqueueLinksFiltering({return finalUrlOrigin;}+ // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match+ // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything+ // which is the intended behavior (since we went off domain)+ if (enqueueStrategy === EnqueueStrategy.SameDomain) {+ const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false })!;+ const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false })!;++ if (originalHostname === finalHostname) {+ return finalUrlOrigin;+ }++ return undefined;+ }+// Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect// before actually finding the urlsreturn originalUrlOrigin;