Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: better docs for fingerprint options (#1778)

Jindřich Bärbf35e94aeade
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index 61946b56e..733f5b6fe 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -14,6 +14,7 @@ import type { LaunchContext } from './launch-context';
import { log } from './logger';
import type { InferBrowserPluginArray, UnwrapPromise } from './utils';
import { createFingerprintPreLaunchHook, createPrePageCreateHook, createPostPageCreateHook } from './fingerprinting/hooks';
+import type { FingerprintGeneratorOptions } from './fingerprinting/types';
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
@@ -25,13 +26,26 @@ export interface BrowserPoolEvents<BC extends BrowserController, Page> {
[BROWSER_POOL_EVENTS.BROWSER_LAUNCHED]: (browserController: BC) => void | Promise<void>;
}
+/**
+ * Settings for the fingerprint generator and virtual session management system.
+ *
+ * > To set the specific fingerprint generation options (operating system, device type, screen dimensions), use the `fingerprintGeneratorOptions` property.
+ */
export interface FingerprintOptions {
- fingerprintGeneratorOptions?: ConstructorParameters<typeof FingerprintGenerator>[0];
/**
+ * Customizes the fingerprint generation by setting e.g. the device type, operating system or screen size.
+ */
+ fingerprintGeneratorOptions?: FingerprintGeneratorOptions;
+ /**
+ * Enables the virtual session management system. This ties every Crawlee session with a specific browser fingerprint,
+ * so your scraping activity seems more natural to the target website.
* @default true
*/
useFingerprintCache?: boolean;
/**
+ * The maximum number of fingerprints that can be stored in the cache.
+ *
+ * Only relevant if `useFingerprintCache` is set to `true`.
* @default 10000
*/
fingerprintCacheSize?: number;
diff --git a/packages/browser-pool/src/fingerprinting/types.ts b/packages/browser-pool/src/fingerprinting/types.ts
index fa3cd77a7..64b1aaa33 100644
--- a/packages/browser-pool/src/fingerprinting/types.ts
+++ b/packages/browser-pool/src/fingerprinting/types.ts
@@ -9,31 +9,106 @@ export interface GetFingerprintReturn {
}
export interface FingerprintGeneratorOptions {
- browsers?: BrowserName[] | BrowserSpecification[];
+ /**
+ * List of `BrowserSpecification` objects
+ * or one of `chrome`, `edge`, `firefox` and `safari`.
+ */
+ browsers?: BrowserSpecification[] | BrowserName[];
+ /**
+ * Browser generation query based on the real world data.
+ * For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).
+ *
+ * > Note: If `browserListQuery` is passed, the `browsers` array is ignored.
+ */
+ browserListQuery?: string;
+ /**
+ * List of operating systems to generate the headers for.
+ */
operatingSystems?: OperatingSystemsName[];
+ /**
+ * List of device types to generate the fingerprints for.
+ */
devices?: DeviceCategory[];
+ /**
+ * List of at most 10 languages to include in the
+ * [Accept-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header
+ * in the language format accepted by that header, for example `en`, `en-US` or `de`.
+ */
locales?: string[];
+ /**
+ * Http version to be used to generate headers (the headers differ depending on the version).
+ *
+ * Can be either 1 or 2. Default value is 2.
+ */
+ httpVersion?: HttpVersion;
+ /**
+ * Defines the screen dimensions of the generated fingerprint.
+ *
+ * > Note: Using this option can lead to a substantial performance drop (from ~0.0007s/fingerprint to ~0.03s/fingerprint)
+ */
+ screen?: {
+ minWidth?: number;
+ maxWidth?: number;
+ minHeight?: number;
+ maxHeight?: number;
+ };
}
+const SUPPORTED_HTTP_VERSIONS = ['1', '2'] as const;
+
+/**
+ * String specifying the HTTP version to use.
+ */
+type HttpVersion = typeof SUPPORTED_HTTP_VERSIONS[number];
+
export const enum BrowserName {
chrome = 'chrome',
firefox = 'firefox',
safari = 'safari',
+ edge = 'edge',
}
export interface BrowserSpecification {
+ /**
+ * String representing the browser name.
+ */
name: BrowserName;
+ /**
+ * Minimum version of browser used.
+ */
minVersion?: number;
+ /**
+ * Maximum version of browser used.
+ */
maxVersion?: number;
+ /**
+ * HTTP version to be used for header generation (the headers differ depending on the version).
+ */
+ httpVersion?: HttpVersion;
}
export const enum OperatingSystemsName {
linux = 'linux',
macos = 'macos',
windows = 'windows',
+ /**
+ * `android` is (mostly) a mobile operating system. You can use this option only together with the `mobile` device category.
+ */
+ android = 'android',
+ /**
+ * `ios` is a mobile operating system. You can use this option only together with the `mobile` device category.
+ */
+ ios = 'ios',
}
export const enum DeviceCategory {
+ /**
+ * Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
+ * > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
+ */
mobile = 'mobile',
+ /**
+ * Describes desktop computers and laptops. These devices usually have larger, horizontal screens and load full-sized versions of websites.
+ */
desktop = 'desktop',
}