Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: clone `request.userData` when creating new request object (#1728)

Matěj Volf222ef59b6467
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 088b1d8c5..7ac577fc2 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -200,7 +200,13 @@ export function createRequestOptions(
})
.map((requestOptions) => {
requestOptions.userData ??= options.userData ?? {};
- requestOptions.userData!.label ??= options.label;
+ if (typeof options.label === 'string') {
+ requestOptions.userData = {
+ ...requestOptions.userData,
+ label: options.label,
+ };
+ }
+
return requestOptions;
});
}
diff --git a/packages/core/test/enqueue_links/userData.test.ts b/packages/core/test/enqueue_links/userData.test.ts
new file mode 100644
index 000000000..2a6fe45b9
--- /dev/null
+++ b/packages/core/test/enqueue_links/userData.test.ts
@@ -0,0 +1,106 @@
+import log from '@apify/log';
+import { load } from 'cheerio';
+import type { CheerioRoot, Request, RequestOptions } from 'crawlee';
+import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue, EnqueueStrategy } from 'crawlee';
+
+const apifyClient = Configuration.getStorageClient();
+
+const HTML = `
+<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <ul>
+ <li><a class="first" href="https://example.com/first">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="second" href="https://example.com/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ </body>
+</html>
+`;
+
+function getMockRequestQueue() {
+ const enqueued: (Request | RequestOptions)[] = [];
+
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = (requests) => {
+ enqueued.push(...requests);
+ };
+
+ return { enqueued, requestQueue };
+}
+
+describe('enqueueLinks() - userData shouldn\'t be changed and outer label must take priority', () => {
+ let ll: number;
+ beforeAll(() => {
+ ll = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ afterAll(() => {
+ log.setLevel(ll);
+ });
+
+ let $: CheerioRoot;
+ beforeEach(() => {
+ $ = load(HTML);
+ });
+
+ test('multiple enqueues with different labels', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ const userData = { foo: 'bar' };
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: 'a.first',
+ userData,
+ label: 'first',
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: 'a.second',
+ userData,
+ label: 'second',
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ expect(enqueued[0].url).toBe('https://example.com/first');
+ expect(enqueued[0].userData.label).toBe('first');
+ expect(enqueued[1].url).toBe('https://example.com/second');
+ expect(enqueued[1].userData.label).toBe('second');
+ });
+
+ test('JSON string of userData shouldn\'t change, but enqueued label should be different', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ const userData = { foo: 'bar', label: 'bogus' };
+ const originalUserData = JSON.stringify(userData);
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: 'a.first',
+ userData,
+ label: 'first',
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+ const userDataAfterEnqueue = JSON.stringify(userData);
+ expect(userDataAfterEnqueue).toEqual(originalUserData);
+ expect(enqueued).toHaveLength(1);
+ expect(enqueued[0].url).toBe('https://example.com/first');
+ expect(enqueued[0].label).toBe('first');
+ });
+});