Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: clone `request.userData` when creating new request object (#1728)
Matěj Volf222ef59b6467
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 088b1d8c5..7ac577fc2 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -200,7 +200,13 @@ export function createRequestOptions(}).map((requestOptions) => {requestOptions.userData ??= options.userData ?? {};- requestOptions.userData!.label ??= options.label;+ if (typeof options.label === 'string') {+ requestOptions.userData = {+ ...requestOptions.userData,+ label: options.label,+ };+ }+return requestOptions;});}diff --git a/packages/core/test/enqueue_links/userData.test.ts b/packages/core/test/enqueue_links/userData.test.tsnew file mode 100644index 000000000..2a6fe45b9--- /dev/null+++ b/packages/core/test/enqueue_links/userData.test.ts@@ -0,0 +1,106 @@+import log from '@apify/log';+import { load } from 'cheerio';+import type { CheerioRoot, Request, RequestOptions } from 'crawlee';+import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue, EnqueueStrategy } from 'crawlee';++const apifyClient = Configuration.getStorageClient();++const HTML = `+<html>+ <head>+ <title>Example</title>+ </head>+ <body>+ <ul>+ <li><a class="first" href="https://example.com/first">I'm sorry, Dave. I'm afraid I can't do that.</a></li>+ <li><a class="second" href="https://example.com/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>+ </ul>+ </body>+</html>+`;++function getMockRequestQueue() {+ const enqueued: (Request | RequestOptions)[] = [];++ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });++ // @ts-expect-error Override method for testing+ requestQueue.addRequests = (requests) => {+ enqueued.push(...requests);+ };++ return { enqueued, requestQueue };+}++describe('enqueueLinks() - userData shouldn\'t be changed and outer label must take priority', () => {+ let ll: number;+ beforeAll(() => {+ ll = log.getLevel();+ log.setLevel(log.LEVELS.ERROR);+ });++ afterAll(() => {+ log.setLevel(ll);+ });++ let $: CheerioRoot;+ beforeEach(() => {+ $ = load(HTML);+ });++ test('multiple enqueues with different labels', async () => {+ const { enqueued, requestQueue } = getMockRequestQueue();++ const userData = { foo: 'bar' };+ await cheerioCrawlerEnqueueLinks({+ options: {+ selector: 'a.first',+ userData,+ label: 'first',+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });++ await cheerioCrawlerEnqueueLinks({+ options: {+ selector: 'a.second',+ userData,+ label: 'second',+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });++ expect(enqueued).toHaveLength(2);++ expect(enqueued[0].url).toBe('https://example.com/first');+ expect(enqueued[0].userData.label).toBe('first');+ expect(enqueued[1].url).toBe('https://example.com/second');+ expect(enqueued[1].userData.label).toBe('second');+ });++ test('JSON string of userData shouldn\'t change, but enqueued label should be different', async () => {+ const { enqueued, requestQueue } = getMockRequestQueue();++ const userData = { foo: 'bar', label: 'bogus' };+ const originalUserData = JSON.stringify(userData);+ await cheerioCrawlerEnqueueLinks({+ options: {+ selector: 'a.first',+ userData,+ label: 'first',+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });+ const userDataAfterEnqueue = JSON.stringify(userData);+ expect(userDataAfterEnqueue).toEqual(originalUserData);+ expect(enqueued).toHaveLength(1);+ expect(enqueued[0].url).toBe('https://example.com/first');+ expect(enqueued[0].label).toBe('first');+ });+});