Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: ignore invalid URLs in `enqueueLinks` in browser crawlers (#1803)

robertm-975ac336c5b83b
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index ebc25ec2a..9bdcc1360 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -23,6 +23,7 @@ import {
BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
BasicCrawler,
RequestState,
+ tryAbsoluteURL,
} from '@crawlee/basic';
import type {
BrowserController,
@@ -723,8 +724,10 @@ async function extractUrlsFromPage(page: { $$eval: Function }, selector: string,
throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+ 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
}
+
return baseUrl
- ? (new URL(href, baseUrl)).href
+ ? tryAbsoluteURL(href, baseUrl)
: href;
- });
+ })
+ .filter((href: string | undefined) => !!href);
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 9f6dfba83..7bd781772 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -8,7 +8,7 @@ import type {
RequestQueue,
Configuration,
} from '@crawlee/http';
-import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';
+import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
import type { CheerioOptions } from 'cheerio';
import * as cheerio from 'cheerio';
@@ -224,15 +224,8 @@ function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl
throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+ 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
}
- const tryAbsolute = () => {
- try {
- return (new URL(href, baseUrl)).href;
- } catch {
- return undefined;
- }
- };
return baseUrl
- ? tryAbsolute()
+ ? tryAbsoluteURL(href, baseUrl)
: href;
})
.filter((href) => !!href) as string[];
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 7ac577fc2..50ebd601c 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -211,6 +211,17 @@ export function createRequestOptions(
});
}
+/**
+ * Helper function used to validate URLs used when extracting URLs from a page
+ */
+export function tryAbsoluteURL(href: string, baseUrl: string): string | undefined {
+ try {
+ return (new URL(href, baseUrl)).href;
+ } catch {
+ return undefined;
+ }
+}
+
/**
* Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
* {@apilink enqueueLinks} to modify requests before enqueuing them.
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 977964fa5..a26c9a31c 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -9,7 +9,7 @@ import type {
RequestQueue,
Configuration,
} from '@crawlee/http';
-import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';
+import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import { concatStreamToBuffer } from '@apify/utilities';
import type { DOMWindow } from 'jsdom';
@@ -265,12 +265,7 @@ function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: str
if (href === undefined) {
return undefined;
}
-
- try {
- return (new URL(href, baseUrl)).href;
- } catch {
- return undefined;
- }
+ return tryAbsoluteURL(href, baseUrl);
})
.filter((href) => href !== undefined && href !== '') as string[];
}
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index c3d3eb16a..69fb03f0d 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -38,6 +38,7 @@ const HTML = `
<a href="/x/absolutepath">This is a relative link.</a>
<a href="y/relativepath">This is a relative link.</a>
<a href="//example.absolute.com/hello">This is a link to a different subdomain</a>
+ <a href="http://">Invalid URL link, this needs to be ignored</a>
</body>
</html>
`;