Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: unify extended unique key format with Python, add `alwaysEnqueue` (#3840)

Jindřich Bär8eab134d92a1
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 5bf838f0a..939225bcf 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -1243,6 +1243,7 @@ export class RequestManagerTandem implements IRequestManager {
// @public
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
+ alwaysEnqueue?: boolean;
crawlDepth?: number;
// @internal (undocumented)
enqueueStrategy?: EnqueueLinksOptions['strategy'];
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 2c2db5ac9..d781f84c2 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -6,7 +6,7 @@ import type { Dictionary } from '@crawlee/types';
import type { BasePredicate } from 'ow';
import ow from 'ow';
-import { normalizeUrl } from '@apify/utilities';
+import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
import type { SkippedRequestReason } from './enqueue_links/shared.js';
@@ -32,6 +32,7 @@ const requestOptionalPredicates = {
handledAt: ow.optional.any(ow.string.date, ow.date),
keepUrlFragment: ow.optional.boolean,
useExtendedUniqueKey: ow.optional.boolean,
+ alwaysEnqueue: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
@@ -178,6 +179,7 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
handledAt,
keepUrlFragment = false,
useExtendedUniqueKey = false,
+ alwaysEnqueue = false,
skipNavigation,
enqueueStrategy,
crawlDepth,
@@ -195,12 +197,23 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');
+ if (uniqueKey && alwaysEnqueue) {
+ throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
+ }
+
this.id = id;
this.url = url;
this.loadedUrl = loadedUrl;
this.uniqueKey =
uniqueKey ||
- CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+ CrawleeRequest.computeUniqueKey({
+ url,
+ method,
+ payload,
+ keepUrlFragment,
+ useExtendedUniqueKey,
+ alwaysEnqueue,
+ });
this.method = method;
this.payload = payload;
this.noRetry = noRetry;
@@ -448,9 +461,13 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
payload,
keepUrlFragment = false,
useExtendedUniqueKey = false,
+ alwaysEnqueue = false,
}: ComputeUniqueKeyOptions) {
const normalizedMethod = method.toUpperCase();
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+
+ let uniqueKey: string;
+
if (!useExtendedUniqueKey) {
if (normalizedMethod !== 'GET' && payload) {
serviceLocator
@@ -461,10 +478,17 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.',
);
}
- return normalizedUrl;
+ uniqueKey = normalizedUrl;
+ } else {
+ const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
+ uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
+ }
+
+ if (alwaysEnqueue) {
+ uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
}
- const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
- return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+
+ return uniqueKey;
}
/** @internal */
@@ -491,7 +515,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
* The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
*
* The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
- * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+ * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
* when requests point to the same URL, but with different methods and payloads. For example: form submits.
*
* Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -547,6 +571,14 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
*/
useExtendedUniqueKey?: boolean;
+ /**
+ * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+ * is always enqueued even if a request with the same URL (and method/payload) is already present
+ * in the queue. Cannot be used together with a custom `uniqueKey`.
+ * @default false
+ */
+ alwaysEnqueue?: boolean;
+
/**
* The `true` value indicates that the request will not be automatically retried on error.
* @default false
@@ -616,6 +648,7 @@ interface ComputeUniqueKeyOptions {
payload?: string | Buffer;
keepUrlFragment?: boolean;
useExtendedUniqueKey?: boolean;
+ alwaysEnqueue?: boolean;
}
export type Source = (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | CrawleeRequest;
diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.ts
index dc22882ce..b07609864 100644
--- a/test/core/request_list.test.ts
+++ b/test/core/request_list.test.ts
@@ -725,3 +725,16 @@ describe('RequestList', () => {
// console.log(initMemory, 'MB');
// });
});
+
+describe('Request', () => {
+ test('alwaysEnqueue makes requests to the same URL have different unique keys', () => {
+ const request1 = new Request({ url: 'https://example.com', alwaysEnqueue: true });
+ const request2 = new Request({ url: 'https://example.com', alwaysEnqueue: true });
+
+ expect(request1.uniqueKey).not.toBe(request2.uniqueKey);
+ });
+
+ test('alwaysEnqueue throws when combined with a custom uniqueKey', () => {
+ expect(() => new Request({ url: 'https://example.com', uniqueKey: 'custom', alwaysEnqueue: true })).toThrow();
+ });
+});