Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): support relative links in `enqueueLinks` explicitly provided via `urls` option (#2014)
Martin Adámekcbd9d0806569
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 5223a5d6f..bebf02079 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -208,19 +208,21 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatt*/export function createRequestOptions(sources: (string | Record<string, unknown>)[],- options: Pick<EnqueueLinksOptions, 'label' | 'userData'> = {},+ options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl'> = {},): RequestOptions[] {return sources.map((src) => (typeof src === 'string' ? { url: src } : src as unknown as RequestOptions)).filter(({ url }) => {try {- return new URL(url).href;+ return new URL(url, options.baseUrl).href;} catch (err) {return false;}}).map((requestOptions) => {+ requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;requestOptions.userData ??= options.userData ?? {};+if (typeof options.label === 'string') {requestOptions.userData = {...requestOptions.userData,diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex 0d71fac02..04274d15d 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -853,6 +853,29 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).toEqual({});});+ test('correctly resolves relative URLs with `urls` option', async () => {+ const { enqueued, requestQueue } = createRequestQueueMock();+ await cheerioCrawlerEnqueueLinks({+ options: {+ baseUrl: 'http://www.absolute.com/removethis/',+ urls: ['/relative/url1', '/relative/url2'],+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });++ expect(enqueued).toHaveLength(2);++ expect(enqueued[0].url).toBe('http://www.absolute.com/relative/url1');+ expect(enqueued[0].method).toBe('GET');+ expect(enqueued[0].userData).toEqual({});++ expect(enqueued[1].url).toBe('http://www.absolute.com/relative/url2');+ expect(enqueued[1].method).toBe('GET');+ expect(enqueued[1].userData).toEqual({});+ });+test('correctly works with transformRequestFunction', async () => {const { enqueued, requestQueue } = createRequestQueueMock();const pseudoUrls = [