Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): support relative links in `enqueueLinks` explicitly provided via `urls` option (#2014)

Martin Adámekcbd9d0806569
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 5223a5d6f..bebf02079 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -208,19 +208,21 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatt
*/
export function createRequestOptions(
sources: (string | Record<string, unknown>)[],
- options: Pick<EnqueueLinksOptions, 'label' | 'userData'> = {},
+ options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl'> = {},
): RequestOptions[] {
return sources
.map((src) => (typeof src === 'string' ? { url: src } : src as unknown as RequestOptions))
.filter(({ url }) => {
try {
- return new URL(url).href;
+ return new URL(url, options.baseUrl).href;
} catch (err) {
return false;
}
})
.map((requestOptions) => {
+ requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
requestOptions.userData ??= options.userData ?? {};
+
if (typeof options.label === 'string') {
requestOptions.userData = {
...requestOptions.userData,
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 0d71fac02..04274d15d 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -853,6 +853,29 @@ describe('enqueueLinks()', () => {
expect(enqueued[2].userData).toEqual({});
});
+ test('correctly resolves relative URLs with `urls` option', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ baseUrl: 'http://www.absolute.com/removethis/',
+ urls: ['/relative/url1', '/relative/url2'],
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ expect(enqueued[0].url).toBe('http://www.absolute.com/relative/url1');
+ expect(enqueued[0].method).toBe('GET');
+ expect(enqueued[0].userData).toEqual({});
+
+ expect(enqueued[1].url).toBe('http://www.absolute.com/relative/url2');
+ expect(enqueued[1].method).toBe('GET');
+ expect(enqueued[1].userData).toEqual({});
+ });
+
test('correctly works with transformRequestFunction', async () => {
const { enqueued, requestQueue } = createRequestQueueMock();
const pseudoUrls = [