Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: adapt E2E suite to v4 and fix regressions it caught (#4012)
Martin Adámekea9da4c1c2ec
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex 796c87f2e..27fc79b55 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -18,7 +18,8 @@ jobs:strategy:fail-fast: falsematrix:- storage: [ LOCAL, MEMORY, PLATFORM ]+ # TODO(v4): re-enable LOCAL once @apify/storage-local supports v4, see https://github.com/apify/crawlee/issues/3406+ storage: [ MEMORY, PLATFORM ]steps:- name: Checkout repositorydiff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mdindex 1a22baf1d..b5a122e69 100644--- a/docs/public-api/crawlee-jsdom.api.md+++ b/docs/public-api/crawlee-jsdom.api.md@@ -7,6 +7,7 @@import { AnyPredicate } from 'ow';import { ArrayPredicate } from 'ow';import { BasePredicate } from 'ow';+import type { BatchAddRequestsResult } from '@crawlee/types';import { BooleanPredicate } from 'ow';import { CheerioAPI } from 'cheerio';import { CheerioRoot } from '@crawlee/utils/internal';@@ -51,7 +52,7 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContextreadonly body: string;readonly document: Document;} & {- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<CheerioAPI>;}>;@@ -123,6 +124,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,body: string;// (undocumented)document: Document;+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;// (undocumented)diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.mdindex cf1b4affa..a88c86bc1 100644--- a/docs/public-api/crawlee-linkedom.api.md+++ b/docs/public-api/crawlee-linkedom.api.md@@ -4,6 +4,7 @@```ts+import type { BatchAddRequestsResult } from '@crawlee/types';import * as cheerio from 'cheerio';import { CheerioRoot } from '@crawlee/utils/internal';import { ContextPipeline } from '@crawlee/http';@@ -33,14 +34,14 @@ export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = L// @publicexport class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);+ constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {readonly window: Window;readonly body: string;readonly document: Document;} & {- enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<unknown>;+ enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;}>;@@ -61,6 +62,7 @@ export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // wJSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {// (undocumented)document: Document;+ enqueueLinks(options?: LinkeDOMCrawlerEnqueueLinksOptions): Promise<BatchAddRequestsResult>;parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;// (undocumented)diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 41053bb7a..c4825a5f5 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -464,6 +464,27 @@ const crawler = new CheerioCrawler({Previously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.+### The default HTTP client is now `impit`++The HTTP crawlers (and `sendRequest`) no longer use `got-scraping` by default. The default client is now `ImpitHttpClient` from the optional `@crawlee/impit-client` package (a Rust-based client with TLS fingerprint impersonation); when it is not installed, the crawlers fall back to a plain `fetch`-based client with a logged warning (no proxy support or impersonation). To keep using `got-scraping`, install `@crawlee/got-scraping-client` and pass `httpClient: new GotScrapingHttpClient()` explicitly.++Note that the session's fingerprint drives the impersonation: each new session gets a randomized realistic fingerprint by default, and its `browser` hint overrides the `browser` option passed to the `ImpitHttpClient` constructor. To force a specific browser family, pin the fingerprint on the sessions instead:++```ts+const crawler = new CheerioCrawler({+ httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),+ sessionPool: new SessionPool({+ createSessionFunction: async (opts) =>+ new Session({+ ...opts?.sessionOptions,+ fingerprint: { browser: 'firefox', platform: 'linux', device: 'desktop' },+ }),+ }),+});+```++See the [Avoid getting blocked](https://crawlee.dev/js/docs/guides/avoid-blocking) guide for how the session fingerprint interacts with both HTTP and browser crawlers.+### `HttpClient` instances return `Response` objectsThe interface of `HttpClient` instances was changed to return the [native `Response` objects](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of custom `HttpResponse` objects.@@ -584,6 +605,19 @@ The mechanism can be disabled entirely with `transactionalStorage: false` on any- `checkStorageAccess` and `withCheckedStorageAccess` are superseded by the transaction mechanism; the per-call-site helper is now `withDirectStorageAccess()`.- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone — `requestsFromUrl` sources are now expanded when added, so the fetched URLs appear in `enqueuedUrls` (and are what `fullResultComparator` compares).+### `storageObject` is removed from storage classes++The `storageObject` property (the raw backend record exposed on `Dataset`, `KeyValueStore` and `RequestQueue` instances) is removed. The commonly used fields are available directly on the instance as `id` and `name`, and `Dataset.getInfo()` returns the full metadata:++```ts+// v3+const { id, name } = dataset.storageObject;++// v4+const { id, name } = dataset;+const info = await dataset.getInfo();+```+### `KeyValueStore.getPublicUrl` is now asyncThe `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public URL directly from the storage backend.@@ -632,6 +666,26 @@ await enqueueLinks({});```+#### `include` patterns no longer replace the enqueue strategy++In v3, providing any URL patterns (`globs`, `regexps`, `pseudoUrls`) disabled the default enqueue strategy - the patterns were the only filter. In v4, the `strategy` **always applies** and is combined with `include` using AND logic: a URL must match an `include` pattern *and* satisfy the strategy (default `same-hostname`) to be enqueued. This mirrors the behavior of Crawlee for Python.++The practical consequence: patterns that used to match URLs on other hostnames (e.g. subdomains) now silently enqueue nothing unless you relax the strategy explicitly:++```ts+// v3 - the glob alone allowed subdomains+await enqueueLinks({+ globs: ['https://*.example.com/'],+});++// v4 - the default same-hostname strategy would filter the subdomains out again,+// so relax it explicitly+await enqueueLinks({+ include: ['https://*.example.com/'],+ strategy: 'same-domain',+});+```+### `transformRequestFunction` precedence in `enqueueLinks`The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`include`, `exclude`) instead of before. This means it has the highest priority and can overwrite any request options set by the global `label` / `userData` options.@@ -734,6 +788,37 @@ The exported handler types were reshaped accordingly. `ErrorHandler` and `Reques`serialize` and `deserialize` now take and return values rather than strings (so v3 records will not load) and each also accept a [Standard Schema](https://standardschema.dev) — a zod codec works as `deserialize` directly — `reset()` is synchronous, no longer clears the persisted record (the new `resetStore()` does) and doubles as a way to establish the state without awaiting `initialize()`, `persistStateKvsName` and `persistStateKvsId` collapsed into a single `keyValueStore` option taking a store (or a pending `KeyValueStore.open()`), `defaultState` also accepts a factory (which you need for a state `structuredClone` cannot rebuild, as the deep copy no longer goes through `serialize`/`deserialize`), and there is a new `persistenceTimeoutMillis` option. `teardown()` is no longer terminal — `initialize()` can be called again to open another persistence window — and a write that fails during a periodic `PERSIST_STATE` or during `teardown()` is warned about rather than thrown. A direct `persistState()` still throws.+### The `log` crawler option is replaced by `logger`++The crawler constructors no longer accept a `log` option with an `@apify/log` `Log` instance. Pass a `logger` implementing the `CrawleeLogger` interface instead. To keep using an `@apify/log` instance (e.g. a `child()` with a custom prefix), wrap it in the `ApifyLogAdapter` from `@crawlee/core`:++```ts+// v3+import { CheerioCrawler, log } from 'crawlee';++const crawler = new CheerioCrawler({+ log: log.child({ prefix: 'MyCrawler' }),+});++// v4+import { ApifyLogAdapter, CheerioCrawler, log } from 'crawlee';++const crawler = new CheerioCrawler({+ logger: new ApifyLogAdapter(log.child({ prefix: 'MyCrawler' })),+});+```++Related: `crawler.log` and the crawling context `log` are typed as `CrawleeLogger`, which has no `setLevel()` method - level filtering belongs to the underlying logging library. With the default logger, set the level on the global `@apify/log` instance instead:++```ts+// v3+crawler.log.setLevel(LogLevel.DEBUG);++// v4+import log, { LogLevel } from '@apify/log';+log.setLevel(LogLevel.DEBUG);+```+### The `log` property is typed as `CrawleeLogger`The `log` property exposed throughout the public API (on the crawling context, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.@@ -1005,6 +1090,40 @@ If you don't pass a predictor, nothing changes: the crawler builds one from `renThis experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.+### `handleCloudflareChallenge` hooks must return the response++In v3, calling `handleCloudflareChallenge()` in a `postNavigationHooks` entry was enough on its own - the helper received the session and removed `403` from the session pool's blocked status codes, so the challenge page (which is served with a 403 status) did not trip the blocked-request detection.++In v4, blocked status code handling is internal to the crawler and runs *after* the post-navigation hooks, and the helper no longer touches it. Instead, `handleCloudflareChallenge()` returns the reloaded `Response` after solving the challenge, and the hook must return it as the new context response - otherwise the crawler still sees the original 403 challenge response and throws a `SessionError` before your `requestHandler` runs, even when the challenge was solved successfully.++Use the pre-wrapped `handleCloudflareChallengeHook()` (it also handles the no-challenge case), or return the response yourself:++```ts+// v3+postNavigationHooks: [+ async ({ handleCloudflareChallenge }) => {+ await handleCloudflareChallenge();+ },+],++// v4+import { handleCloudflareChallengeHook } from 'crawlee';++postNavigationHooks: [handleCloudflareChallengeHook()],++// v4 (manual equivalent)+postNavigationHooks: [+ async ({ handleCloudflareChallenge }) => {+ // Returning `{ response: undefined }` would clobber the navigation response+ // when there was no challenge, so only return it when one was solved.+ const response = await handleCloudflareChallenge();+ return response && { response };+ },+],+```++If you called the standalone `playwrightUtils.handleCloudflareChallenge(page, url, session, options)` directly, note that the `session` parameter is gone - the v4 signature is `handleCloudflareChallenge(page, url, options)`, so an options object passed in the old fourth position would be silently ignored.+## Only if you customize crawler statisticsApplies when you passed `statisticsOptions` to a crawler, or subclassed `Statistics`.@@ -1725,6 +1844,7 @@ Besides the resource-detection helpers above, several other `@crawlee/utils` exp- **Removed URL helpers:** `filterUrl(target, origin, strategy)`, `matchesEnqueueStrategy(strategy, target, origin)`, and the `UNSUPPORTED_SCHEME_MESSAGE` constant. URL filtering by enqueue strategy is now internal to `enqueueLinks`. The related `filterRequestsByPatterns(requests, patterns?, onSkippedUrl?)` function (from `@crawlee/core`) was removed for the same reason — pattern-based request filtering now happens inside `enqueueLinks`.- **Relocated enums/types:** `EnqueueStrategy` now lives in `@crawlee/core` and `SearchParams` in `@crawlee/types`. They are no longer re-exported from `@crawlee/utils`, so `import { EnqueueStrategy } from '@crawlee/utils'` breaks — import them from `crawlee` (the meta-package) or from `@crawlee/core` / `@crawlee/types` instead.- **Removed `RobotsFile` alias:** `RobotsFile` was an alias for the `RobotsTxtFile` class and is removed. Rename any usage to `RobotsTxtFile`; the class itself is unchanged apart from the signature change described below.+- **Split into public and `/internal` entry points:** the main `@crawlee/utils` entry now exposes only the user-facing helpers (`sleep`, `htmlToText`, `extractUrls`, `downloadListOfUrls`, `expandShadowRoots`, the `social` namespace, the Open Graph parser, and the robots/sitemap utilities). Helpers that primarily serve the crawler packages - e.g. `URL_NO_COMMAS_REGEX`, `URL_WITH_COMMAS_REGEX`, `extractUrlsFromCheerio`, `tryAbsoluteURL`, the `CheerioRoot` type, and the blocked-detection and iterable helpers - moved to the `@crawlee/utils/internal` entry point. They keep working, but imports need updating: `import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal'`. As the name suggests, the internal entry follows no semver guarantees.#### `RobotsTxtFile.find` signature changed; sitemap options removeddiff --git a/package.json b/package.jsonindex 9990f5f5a..b39a2f98c 100644--- a/package.json+++ b/package.json@@ -65,12 +65,14 @@"@crawlee/basic": "workspace:*","@crawlee/cheerio": "workspace:*","@crawlee/core": "workspace:*",+ "@crawlee/http-client": "workspace:*","@crawlee/impit-client": "workspace:*","@crawlee/jsdom": "workspace:*","@crawlee/linkedom": "workspace:*","@crawlee/playwright": "workspace:*","@crawlee/puppeteer": "workspace:*","@crawlee/stagehand": "workspace:*",+ "@crawlee/types": "workspace:*","@crawlee/utils": "workspace:*","@microsoft/api-extractor": "^7.58.9","@playwright/browser-chromium": "1.61.1",diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 90d881754..3afc05371 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -972,7 +972,16 @@ export class BasicCrawler<logger ||(configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {- const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageBackend, logger);+ // Inherit the ambient locator's already-set services for anything not explicitly+ // provided - e.g. only passing a `logger` must not detach the crawler from a globally+ // configured storage backend.+ const ambientServices = serviceLocator.getServicesIfSet();+ const scopedServiceLocator = new ServiceLocator(+ configuration ?? ambientServices.configuration,+ eventManager ?? ambientServices.eventManager,+ storageBackend ?? ambientServices.storageBackend,+ logger ?? ambientServices.logger,+ );serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);}diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 12d0898f4..4ec3dc7d5 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -265,7 +265,7 @@ export class CheerioCrawler<originalRequestUrl: crawlingContext.request.url,finalRequestUrl: crawlingContext.request.loadedUrl,enqueueLinks: originalEnqueueLinks,- })) as BatchAddRequestsResult; // TODO make this type safe+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024},waitForSelector: async (selector: string, _timeoutMs?: number) => {if (crawlingContext.$(selector).get().length === 0) {diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.tsindex 09832c7ec..db33580fc 100644--- a/packages/core/src/crawlers/error_snapshotter.ts+++ b/packages/core/src/crawlers/error_snapshotter.ts@@ -116,7 +116,7 @@ export class ErrorSnapshotter {}/**- * Save the HTML snapshot of the page, and return the fileName with the extension.+ * Save the HTML snapshot of the page, and return the key it was stored under.*/async saveHTMLSnapshot(html: string,@@ -125,7 +125,9 @@ export class ErrorSnapshotter {): Promise<string | undefined> {try {await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });- return `${fileName}.html`;+ // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,+ // where local storage put the extension in the key) would break `getPublicUrl`.+ return fileName;} catch {return undefined;}diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex ac1a2ac51..2b1f642ce 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -225,7 +225,10 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {userData.label = label;}- this.#userData = { __crawlee: {}, ...userData };+ // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the+ // bag is non-enumerable, so the spread alone would silently drop the internal state+ // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.+ this.#userData = { __crawlee: (userData as Dictionary).__crawlee ?? {}, ...userData };// `userData` must stay an enumerable own accessor — serialization in the storages relies on itObject.defineProperties(this, {diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.tsindex cad4b8a3f..4b2437366 100644--- a/packages/core/src/service_locator.ts+++ b/packages/core/src/service_locator.ts@@ -83,6 +83,18 @@ interface ServiceLocatorInterface {*/getStorageInstanceManager(): StorageInstanceManager;+ /**+ * Returns the currently set services without triggering the implicit creation of defaults.+ * Used to inherit already-materialized services into crawler-scoped service locators.+ * @internal+ */+ getServicesIfSet(): {+ configuration?: Configuration;+ eventManager?: EventManager;+ storageBackend?: StorageBackend;+ logger?: CrawleeLogger;+ };+/*** Resets the service locator to its initial state.* Used mainly for testing purposes.@@ -155,6 +167,21 @@ export class ServiceLocator implements ServiceLocatorInterface {this.#logger = logger;}+ /** @internal */+ getServicesIfSet(): {+ configuration?: Configuration;+ eventManager?: EventManager;+ storageBackend?: StorageBackend;+ logger?: CrawleeLogger;+ } {+ return {+ configuration: this.#configuration,+ eventManager: this.#eventManager,+ storageBackend: this.#storageBackend,+ logger: this.#logger,+ };+ }+getConfiguration(): Configuration {if (!this.#configuration) {this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.tsindex 656961a10..d0af8e27a 100644--- a/packages/fs-storage/src/file-system-storage.ts+++ b/packages/fs-storage/src/file-system-storage.ts@@ -5,15 +5,20 @@ import type * as storage from '@crawlee/types';import type { CrawleeLogger } from '@crawlee/types';import { s } from '@sapphire/shapeshift';-import {- FileSystemDatasetClient as NativeDatasetBackend,- FileSystemKeyValueStoreClient as NativeKeyValueStoreBackend,- FileSystemRequestQueueClient as NativeRequestQueueBackend,-} from '@crawlee/fs-storage-native';import { DatasetBackend } from './resource-clients/dataset.js';import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';import { RequestQueueBackend } from './resource-clients/request-queue.js';+// The native package throws at load time on platforms without a published binary (e.g.+// linux musl), and `@crawlee/core` imports this module eagerly via its service locator.+// Load it lazily so merely importing `@crawlee/fs-storage` stays safe everywhere and the+// native binding is only loaded when a file-system storage is actually used.+let nativeModule: Promise<typeof import('@crawlee/fs-storage-native')> | undefined;+async function importNativeModule() {+ nativeModule ??= import('@crawlee/fs-storage-native');+ return nativeModule;+}+/** The alias `@crawlee/core` opens the default (unnamed) storage under. */const DEFAULT_STORAGE_ALIAS = '__default__';@@ -125,7 +130,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {return found;}- const nativeBackend = await NativeDatasetBackend.open(id, name, alias, this.localDataDirectory);+ const nativeBackend = await (+ await importNativeModule()+ ).FileSystemDatasetClient.open(id, name, alias, this.localDataDirectory);const newStore = await DatasetBackend.create({name: alias ? undefined : (name ?? cacheKey),cacheKey,@@ -150,7 +157,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {return found;}- const nativeBackend = await NativeKeyValueStoreBackend.open(id, name, alias, this.localDataDirectory);+ const nativeBackend = await (+ await importNativeModule()+ ).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);const newStore = await KeyValueStoreBackend.create({name: alias ? undefined : (name ?? cacheKey),cacheKey,@@ -175,7 +184,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {return found;}- const nativeBackend = await NativeRequestQueueBackend.open(+ const nativeBackend = await (+ await importNativeModule()+ ).FileSystemRequestQueueClient.open(id,name,alias,diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 23778694a..52700f8d7 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -23,7 +23,7 @@ import {Router,tryAbsoluteURL,} from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot } from '@crawlee/utils/internal';import { type RobotsTxtFile, sleep } from '@crawlee/utils';import type { DOMWindow } from 'jsdom';@@ -97,6 +97,11 @@ export interface JSDOMCrawlingContext<* ```*/parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;++ /**+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.+ */+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;}export type JSDOMRequestHandler<@@ -360,7 +365,7 @@ export class JSDOMCrawler<return {enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {- return domCrawlerEnqueueLinks({+ return (await domCrawlerEnqueueLinks({// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;// pre-clamping it here would make the crawler log the internal limit as a user-provided oneoptions: enqueueOptions,@@ -371,7 +376,7 @@ export class JSDOMCrawler<originalRequestUrl: crawlingContext.request.url,finalRequestUrl: crawlingContext.request.loadedUrl,enqueueLinks: originalEnqueueLinks,- });+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024},async waitForSelector(selector: string, timeoutMs = 5_000) {const cheerio = await import('cheerio');diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex a558e1023..cda4dc2d2 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -23,7 +23,7 @@ import {Router,tryAbsoluteURL,} from '@crawlee/http';-import type { Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot } from '@crawlee/utils/internal';import { type RobotsTxtFile, sleep } from '@crawlee/utils';import * as cheerio from 'cheerio';@@ -90,6 +90,11 @@ export interface LinkeDOMCrawlingContext<* ```*/parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;++ /**+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.+ */+ enqueueLinks(options?: LinkeDOMCrawlerEnqueueLinksOptions): Promise<BatchAddRequestsResult>;}export type LinkeDOMRequestHandler<@@ -180,7 +185,7 @@ export class LinkeDOMCrawler<> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {static #parser = new DOMParser();- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes> = {}) {const { contextPipelineBuilder, ...rest } = options;super({@@ -244,23 +249,23 @@ export class LinkeDOMCrawler<}}- private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string }) {+ private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: Window }) {const originalEnqueueLinks = crawlingContext.enqueueLinks;return {enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {- return linkedomCrawlerEnqueueLinks({+ return (await linkedomCrawlerEnqueueLinks({// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;// pre-clamping it here would make the crawler log the internal limit as a user-provided oneoptions: enqueueOptions,- window: document.defaultView,+ window: crawlingContext.window,requestManager: await this.getRequestManager(),robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),onSkippedRequest: this.handleSkippedRequest,originalRequestUrl: crawlingContext.request.url,finalRequestUrl: crawlingContext.request.loadedUrl,enqueueLinks: originalEnqueueLinks,- });+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024},async waitForSelector(selector: string, timeoutMs = 5_000) {const $ = cheerio.load(crawlingContext.body);diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex e206172d3..c07c9e452 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -14,7 +14,7 @@ overrides:tar: ^7.5.16lerna>js-yaml: ^4.2.0lerna>minimatch: ^3.1.4- apify: 4.0.0-beta.19+ apify: ^4.0.0-beta.24apify>@crawlee/core: workspace:*apify>@crawlee/types: workspace:*apify>@crawlee/utils: workspace:*@@ -49,6 +49,9 @@ importers:'@crawlee/core':specifier: workspace:*version: link:packages/core+ '@crawlee/http-client':+ specifier: workspace:*+ version: link:packages/http-client'@crawlee/impit-client':specifier: workspace:*version: link:packages/impit-client@@ -67,6 +70,9 @@ importers:'@crawlee/stagehand':specifier: workspace:*version: link:packages/stagehand-crawler+ '@crawlee/types':+ specifier: workspace:*+ version: link:packages/types'@crawlee/utils':specifier: workspace:*version: link:packages/utils@@ -143,8 +149,8 @@ importers:specifier: ^4.0.16version: 4.1.4([email protected])apify:- specifier: 4.0.0-beta.19- version: 4.0.0-beta.19([email protected])+ specifier: ^4.0.0-beta.24+ version: 4.0.0-beta.24([email protected])apify-node-curl-impersonate:specifier: ^1.0.23version: 1.0.29@@ -266,8 +272,8 @@ importers:specifier: workspace:*version: link:../packages/stagehand-crawlerapify:- specifier: 4.0.0-beta.19- version: 4.0.0-beta.19([email protected])+ specifier: ^4.0.0-beta.24+ version: 4.0.0-beta.24([email protected])crawlee:specifier: workspace:*version: link:../packages/crawlee@@ -1307,9 +1313,6 @@ packages:engines: {node: '>= 0.10'}hasBin: true- '@apify/[email protected]':- resolution: {integrity: sha512-lyvwMXee8SJNjNyxhr+nSTNyvjyoxbxol51xikq9VytFOPNSEMz8N02mUAuLVJNqrnqCBFRybjeqZdg4Y5AZlA==}-'@apify/[email protected]':resolution: {integrity: sha512-WOjdkuNRSo09ikse47t6jIW4R55FEmZdOaoUTbSmpwfmNNHVht+iTgoqeHIVr7vvI9s+mtTmvzPoesoZa0cjVA==}@@ -5279,8 +5282,8 @@ packages:resolution: {integrity: sha512-5Oa9VE5rpDBdRgg71Vl09bqraw5RV0nyL6+rnOzIrVe+YCFCOyjj1KwsCz92gPNjGXGQG53w005la3dCTTrgdw==}- resolution: {integrity: sha512-p6fgxlqOqwtYX77u6MZiWjvNlcJwjkGGDJ9HVt56UEBC88q+n5igZgWgPpgj/WHp9yajTf7WtAdZUO16qKa75A==}+ resolution: {integrity: sha512-CPoMf/o8SJ0rq/yPtHXIabT0ldcKD59q0r9Tgs+AN47txC4JlskyG0S1ck1HNGbuhPuGyX4blWR6yib7TugYOw==}engines: {node: '>=22.0.0'}@@ -12947,8 +12950,6 @@ snapshots:dependencies:event-stream: 3.3.4- '@apify/[email protected]': {}-'@apify/[email protected]': {}'@apify/[email protected]': {}@@ -17793,12 +17794,13 @@ snapshots:dependencies:'@apify/consts': 2.52.1+ '@apify/datastructures': 2.0.4'@apify/input_secrets': 1.2.30'@apify/log': 2.5.35- '@apify/timeout': 0.3.3+ '@apify/timeout': 0.4.4'@apify/utilities': 2.27.0'@crawlee/core': link:packages/core'@crawlee/types': link:packages/typesdiff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex ae18d1f10..44166d6fa 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -29,9 +29,9 @@ overrides:# The e2e tests run the `apify` SDK against the local 4.x workspace packages.# The default `latest`/`next` tags are the 3.x SDK, which is not runtime# compatible with `@crawlee/core@4` (e.g. `Configuration.INTEGER_VARS` was- # removed), so pin the v4 SDK and force it to use the workspace `@crawlee/*`+ # removed), so keep the SDK on the latest v4 beta and force it to use the workspace `@crawlee/*`# packages (mirroring the per-actor `overrides.apify` used for PLATFORM tests).- apify: "4.0.0-beta.19"+ apify: "^4.0.0-beta.24""apify>@crawlee/core": "workspace:*""apify>@crawlee/types": "workspace:*""apify>@crawlee/utils": "workspace:*"diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 1ef9b1a25..77086ab38 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -103,6 +103,24 @@ describe('BasicCrawler', () => {serviceLocator.setStorageBackend(new MemoryStorageBackend());});+ test('crawler-scoped service locator inherits ambient services that are not explicitly overridden', async () => {+ const ambientBackend = serviceLocator.getStorageBackend();++ let seenBackend: unknown;+ // Passing only a `logger` creates a crawler-scoped service locator - it must inherit the+ // ambient storage backend rather than falling back to the default file-system one.+ const crawler = new BasicCrawler({+ logger: serviceLocator.getLogger(),+ requestHandler: async () => {+ seenBackend = serviceLocator.getStorageBackend();+ },+ });++ await crawler.run(['https://example.com']);++ expect(seenBackend).toBe(ambientBackend);+ });+test('does not leak sigint events', async () => {let count = 0;diff --git a/test/core/error_snapshotter.test.ts b/test/core/error_snapshotter.test.tsnew file mode 100644index 000000000..9263901f3--- /dev/null+++ b/test/core/error_snapshotter.test.ts@@ -0,0 +1,29 @@+import type { KeyValueStore } from '@crawlee/core';+import { ErrorSnapshotter } from '@crawlee/core';+import { describe, expect, test, vitest } from 'vitest';++describe('ErrorSnapshotter', () => {+ test('saveHTMLSnapshot returns the record key it stored the snapshot under', async () => {+ const snapshotter = new ErrorSnapshotter();+ const setValue = vitest.fn(async () => {});+ const keyValueStore = { setValue } as unknown as KeyValueStore;++ const key = await snapshotter.saveHTMLSnapshot('<html></html>', keyValueStore, 'ERROR_SNAPSHOT_foo');++ expect(setValue).toHaveBeenCalledWith('ERROR_SNAPSHOT_foo', '<html></html>', { contentType: 'text/html' });+ // The returned value must be the actual record key (no appended extension),+ // as it is fed to `keyValueStore.getPublicUrl()`.+ expect(key).toBe('ERROR_SNAPSHOT_foo');+ });++ test('saveHTMLSnapshot returns undefined when storing fails', async () => {+ const snapshotter = new ErrorSnapshotter();+ const keyValueStore = {+ setValue: async () => {+ throw new Error('nope');+ },+ } as unknown as KeyValueStore;++ await expect(snapshotter.saveHTMLSnapshot('<html></html>', keyValueStore, 'KEY')).resolves.toBeUndefined();+ });+});diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex 36977f30d..fd298f569 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -342,6 +342,13 @@ describe('RequestQueue remote', () => {bar: true,crawlDepth: 10,});+ // Re-wrapping userData that comes from another Request instance (where `__crawlee` is+ // non-enumerable) must preserve the internal state instead of dropping it via the spread.+ const r4 = new Request({ url, method, userData: r1.userData });+ expect(r4.skipNavigation).toBe(true);+ expect(r4.maxRetries).toBe(5);+ expect(r4.crawlDepth).toBe(10);+ expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, maxRetries: 5 });const desc2 = Object.getOwnPropertyDescriptor(r2.userData, '__crawlee');expect(desc2!.enumerable).toBe(false);expect(r2.maxRetries).toBeUndefined();diff --git a/test/e2e/adaptive-playwright-default/actor/Dockerfile b/test/e2e/adaptive-playwright-default/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/adaptive-playwright-default/actor/Dockerfile+++ b/test/e2e/adaptive-playwright-default/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/adaptive-playwright-default/actor/main.js b/test/e2e/adaptive-playwright-default/actor/main.jsindex 308718583..f58262268 100644--- a/test/e2e/adaptive-playwright-default/actor/main.js+++ b/test/e2e/adaptive-playwright-default/actor/main.js@@ -1,6 +1,6 @@import { Actor } from 'apify';import { AdaptivePlaywrightCrawler } from '@crawlee/playwright';-import { LogLevel } from '@apify/log';+import log, { LogLevel } from '@apify/log';await Actor.init({storage:@@ -45,7 +45,8 @@ const crawler = new AdaptivePlaywrightCrawler({},});-crawler.log.setLevel(LogLevel.DEBUG);+// CrawleeLogger has no setLevel; raise the level on the global @apify/log instead.+log.setLevel(LogLevel.DEBUG);await crawler.run(['https://crawlee.dev/js/docs/next/examples/accept-user-input']);diff --git a/test/e2e/adaptive-playwright-default/actor/package.json b/test/e2e/adaptive-playwright-default/actor/package.jsonindex 6b84deaed..2fcfe425c 100644--- a/test/e2e/adaptive-playwright-default/actor/package.json+++ b/test/e2e/adaptive-playwright-default/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Adaptive Playwright Test - Default","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfileindex 6433c507b..3fd414cf0 100644--- a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile+++ b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/adaptive-playwright-robots-file/actor/package.json b/test/e2e/adaptive-playwright-robots-file/actor/package.jsonindex 65775d421..baf52ddc2 100644--- a/test/e2e/adaptive-playwright-robots-file/actor/package.json+++ b/test/e2e/adaptive-playwright-robots-file/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Adaptive Playwright Test - Robots file","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfileindex 9dee3c33a..eb099b2ac 100644--- a/test/e2e/automatic-persist-value/actor/Dockerfile+++ b/test/e2e/automatic-persist-value/actor/Dockerfile@@ -3,12 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./-RUN rm -r node_modules+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.jsonindex f4abd9b47..cab800bfc 100644--- a/test/e2e/automatic-persist-value/actor/package.json+++ b/test/e2e/automatic-persist-value/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Key-Value Store - Automatic Persist Value Test","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types",@@ -14,9 +13,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js b/test/e2e/autoscaling-max-tasks-per-minute/actor/main.jsindex 9f25e8edf..e7362c05c 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js@@ -1,10 +1,12 @@import { Actor } from 'apify';-import { BasicCrawler, log as defaultLog, LogLevel } from '@crawlee/basic';+import { ApifyLogAdapter, BasicCrawler, log as defaultLog, LogLevel } from '@crawlee/basic';-const crawlerLogger = defaultLog.child({- prefix: 'AutoscalingTest',- level: LogLevel.INFO,-});+const crawlerLogger = new ApifyLogAdapter(+ defaultLog.child({+ prefix: 'AutoscalingTest',+ level: LogLevel.INFO,+ }),+);const mainOptions = {exit: Actor.isAtHome(),@@ -18,7 +20,7 @@ let crawlCalledAt = Date.now();await Actor.main(async () => {const crawler = new BasicCrawler({- log: crawlerLogger,+ logger: crawlerLogger,maxRequestsPerMinute: 1,requestHandler({ log }) {log.info(`Crawler requestHandler called after ${Date.now() - crawlCalledAt}ms`);diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.jsonindex 9a703cf0d..7bab52695 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Autoscaling Pool Test - Max Tasks per Minute","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types",@@ -14,9 +13,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/camoufox-cloudflare/actor/Dockerfile b/test/e2e/camoufox-cloudflare/actor/Dockerfileindex 2c6befacb..810df0f5d 100644--- a/test/e2e/camoufox-cloudflare/actor/Dockerfile+++ b/test/e2e/camoufox-cloudflare/actor/Dockerfile@@ -10,11 +10,12 @@ COPY --chown=myuser package*.json ./# `--ignore-scripts` skips the `camoufox-js fetch` postinstall; the browser is already# baked into the base image.+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit --ignore-scripts \+ && npm install --only=prod --no-audit --ignore-scripts \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/camoufox-cloudflare/actor/main.js b/test/e2e/camoufox-cloudflare/actor/main.jsindex 3667ebd6a..ac6c77954 100644--- a/test/e2e/camoufox-cloudflare/actor/main.js+++ b/test/e2e/camoufox-cloudflare/actor/main.js@@ -1,4 +1,4 @@-import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';+import { Dataset, handleCloudflareChallengeHook, PlaywrightCrawler } from '@crawlee/playwright';import { Actor } from 'apify';import { launchOptions } from 'camoufox-js';import { firefox } from 'playwright';@@ -13,7 +13,9 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({- proxyConfiguration: await Actor.createProxyConfiguration(),+ // The target hard-blocks datacenter IP ranges outright (block page, not a solvable+ // challenge), so this test needs residential exit nodes.+ proxyConfiguration: await Actor.createProxyConfiguration({ groups: ['RESIDENTIAL'] }),// Camoufox ships its own anti-detection; Crawlee's fingerprint injection conflicts with it// and keeps Cloudflare from ever clearing the challenge.browserPoolOptions: { useFingerprints: false },@@ -38,11 +40,8 @@ await Actor.main(async () => {});},],- postNavigationHooks: [- async ({ handleCloudflareChallenge }) => {- await handleCloudflareChallenge();- },- ],+ // verbose keeps the challenge detection visible in the nightly run logs+ postNavigationHooks: [handleCloudflareChallengeHook({ verbose: true })],launchContext: {launcher: firefox,launchOptions: await launchOptions({diff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.jsonindex 98886a293..be2e2bb8a 100644--- a/test/e2e/camoufox-cloudflare/actor/package.json+++ b/test/e2e/camoufox-cloudflare/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Camoufox - Solving Cloudflare Challenge","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -12,16 +11,13 @@"@crawlee/playwright": "file:./packages/playwright-crawler","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils",- "camoufox-js": "^0.11.0",+ "camoufox-js": "^0.12.0","playwright": "1.60.0"},"overrides": {"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/camoufox-cloudflare/test.mjs b/test/e2e/camoufox-cloudflare/test.mjsindex a063f66c5..a44664b71 100644--- a/test/e2e/camoufox-cloudflare/test.mjs+++ b/test/e2e/camoufox-cloudflare/test.mjs@@ -2,10 +2,12 @@ import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tool// TODO: re-enable on LOCAL/MEMORY once Camoufox supports the repo's Playwright version.// Those storage types import the actor in-process against the repo-root node_modules (currently-// Playwright 1.61), but camoufox-js 0.11 ships Firefox 150 and only launches under Playwright 1.60.-// Only the PLATFORM build isolates the actor's pinned 1.60, so the test runs there exclusively.+// Playwright 1.61), but camoufox-js 0.12 declares a playwright-core <1.61 peer range.+// Only the PLATFORM build isolates the actor's pinned playwright version, so the test runs there exclusively.if (process.env.STORAGE_IMPLEMENTATION !== 'PLATFORM') {- await skipTest('Camoufox needs Playwright 1.60; only the PLATFORM build can isolate it from the repo-root 1.61');+ await skipTest(+ 'Camoufox needs its paired Playwright version; only the PLATFORM build can isolate it from the repo-root one',+ );}const testActorDirname = getActorTestDir(import.meta.url);diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfileindex eb6de5b07..792fbb7a8 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile@@ -4,8 +4,9 @@ COPY /packages ./packagesCOPY /package*.json ./COPY /tsconfig.json ./COPY /main.ts ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update \&& npm run build@@ -13,6 +14,9 @@ FROM lwthiker/curl-impersonateCOPY --from=builder /usr/local/bin /usr/local/binCOPY --from=builder /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm+# Wipe any preinstalled node_modules so the builder tree (with file: symlinks)+# can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -20,7 +24,7 @@ COPY --from=builder /main.js ./COPY /.actor ./.actorRUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.tsindex f51d11483..fe36db3df 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts@@ -1,16 +1,6 @@-import { Readable } from 'node:stream';--import type { Dictionary } from '@crawlee/cheerio';import { CheerioCrawler } from '@crawlee/cheerio';-import type {- BaseHttpClient,- BaseHttpResponseData,- HttpRequest,- HttpResponse,- RedirectHandler,- ResponseTypes,- StreamingHttpResponse,-} from '@crawlee/core';+import { BaseHttpClient, type CustomFetchOptions, ResponseWithUrl } from '@crawlee/http-client';+import type { Dictionary } from '@crawlee/types';import { Actor } from 'apify';import { CurlImpersonate } from 'apify-node-curl-impersonate';@@ -25,157 +15,73 @@ interface CurlImpersonateHttpClientOptions {impersonate?: ConstructorParameters<typeof CurlImpersonate>[1]['impersonate'];}-type CurlResponse = Awaited<ReturnType<CurlImpersonate['makeRequest']>>;--class CurlImpersonateHttpClient implements BaseHttpClient {- constructor(private options: CurlImpersonateHttpClientOptions = {}) {}-- protected async curlOptionsForRequest(request: HttpRequest<any>) {- const result = {- method: request.method ?? 'get',- headers: request.headers ?? {},- flags: ['--compressed'],- impersonate: this.options.impersonate,- debugLogger: () => {},- };-- if (request.proxyUrl !== undefined) {- result.flags.push('--proxy', request.proxyUrl);- }-- if (request.cookieJar) {- result.headers.cookie = (await (request.cookieJar as any)?.getCookieString?.(request.url.toString())) ?? '';- }-- return result;+/**+ * A v4 `BaseHttpClient` implementation backed by `apify-node-curl-impersonate`.+ * The base class provides redirect following, proxy and cookie handling - this+ * only performs the raw request and wraps the result in a fetch `Response`.+ */+class CurlImpersonateHttpClient extends BaseHttpClient {+ constructor(private options: CurlImpersonateHttpClientOptions = {}) {+ super();}- protected shouldRedirect(request: HttpRequest, response: CurlResponse, redirectUrls: URL[]): boolean {- if (request.maxRedirects !== undefined && request.maxRedirects <= redirectUrls.length) {- return false;- }+ protected async fetch(request: Request, init?: RequestInit & CustomFetchOptions): Promise<Response> {+ const headers: Record<string, string> = {};+ request.headers.forEach((value, key) => {+ headers[key] = value;+ });- if (request.followRedirect instanceof Function) {- return request.followRedirect(response);+ const flags = ['--compressed'];+ if (init?.proxyUrl) {+ flags.push('--proxy', init.proxyUrl);}- return request.followRedirect === undefined || request.followRedirect;- }-- protected async performRequest(request: HttpRequest<any>, onRedirect?: RedirectHandler) {- let response: CurlResponse | undefined;- const redirectUrls: URL[] = [];-- const updatedRequest: Required<Pick<HttpRequest, 'url' | 'headers'>> = {- url: request.url.toString(),- headers: {},- };-- while (true) {- const impersonate = new CurlImpersonate(updatedRequest.url.toString(), {- ...(await this.curlOptionsForRequest(request)),- followRedirects: false,- });-- response = await impersonate.makeRequest();+ const curl = new CurlImpersonate(request.url, {+ method: request.method,+ headers,+ flags,+ impersonate: this.options.impersonate,+ followRedirects: false,+ debugLogger: () => {},+ });- if (response.statusCode >= 300 && response.statusCode < 400) {- if (!this.shouldRedirect(request, response, redirectUrls)) {- break;- }+ const response = await curl.makeRequest();- updatedRequest.url = response.responseHeaders.location;- updatedRequest.headers = response.responseHeaders;- redirectUrls.push(new URL(updatedRequest.url));- onRedirect?.(this.transformResponse(response, redirectUrls), updatedRequest);- } else if (request.throwHttpErrors !== false && response.statusCode >= 400) {- throw Object.assign(new Error(`Error status code encountered - ${response.statusCode}`), {- request,- response,- });- } else {- break;+ // The parsed curl headers include the status line as a key (e.g. "HTTP/2 200 "),+ // which the fetch Headers constructor rejects - keep only valid header names.+ const responseHeaders: Record<string, string> = {};+ for (const [name, value] of Object.entries(response.responseHeaders ?? {})) {+ if (/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name)) {+ responseHeaders[name] = String(value);}}- return { ...response, redirectUrls };- }-- protected transformResponse(response: CurlResponse, redirectUrls: URL[]): BaseHttpResponseData {- return {- redirectUrls,- url: response.url,- ip: response.ipAddress,- statusCode: response.statusCode,- headers: response.responseHeaders,- trailers: {}, // TODO apify-node-curl-impersonate doesn't seem to expose this- complete: true,- };- }-- async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(- request: HttpRequest<TResponseType>,- ): Promise<HttpResponse<TResponseType>> {- const response = await this.performRequest(request);-- const body = ((): ResponseTypes[TResponseType] => {- const buffer = Buffer.from(response.response, request.encoding);-- switch (request.responseType) {- case 'buffer':- return buffer as ResponseTypes[TResponseType];- case 'json':- return JSON.parse(buffer.toString());- default:- return buffer.toString() as ResponseTypes[TResponseType];- }- })();-- return {- request,- body,- ...this.transformResponse(response, response.redirectUrls),- };- }-- async stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {- const response = await this.performRequest(request, onRedirect);-- const stream = new Readable();- stream.push(response.response);- stream.push(null);-- return {- request,- url: response.url,- ip: response.ipAddress,- statusCode: response.statusCode,- stream,- complete: true,- downloadProgress: { percent: 100, transferred: response.response.length },- uploadProgress: { percent: 100, transferred: 0 },- redirectUrls: response.redirectUrls,- headers: response.responseHeaders,- trailers: {},- };+ return new ResponseWithUrl(response.response, {+ status: response.statusCode,+ headers: responseHeaders,+ url: response.url ?? request.url,+ });}}const crawler = new CheerioCrawler({async requestHandler(context) {- const { body: text } = await context.sendRequest({- url: 'https://api.apify.com/v2/browser-info',- });-- const { body: json } = await context.sendRequest<Dictionary>({- url: 'https://api.apify.com/v2/browser-info',- responseType: 'json',- });+ const text = await (+ await context.sendRequest({+ url: 'https://api.apify.com/v2/browser-info',+ })+ ).text();++ const json = (await (+ await context.sendRequest({+ url: 'https://api.apify.com/v2/browser-info',+ })+ ).json()) as Dictionary;await context.pushData({body: context.body,title: context.$('title').text(),- userAgent: json.headers['user-agent'],+ userAgent: (json.headers as Dictionary)['user-agent'],clientIpTextResponse: text,clientIpJsonResponse: json,});diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json b/test/e2e/cheerio-curl-impersonate-ts/actor/package.jsonindex 09a29a3e2..05255fdba 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json@@ -3,16 +3,15 @@"version": "0.0.1","description": "Cheerio Crawler Test - curl-impersonate HTTP client","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/cheerio": "file:./packages/cheerio-crawler","@crawlee/core": "file:./packages/core","@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/http-client": "file:./packages/http-client","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils",- "apify": "next","apify-node-curl-impersonate": "^1.0.29"},"overrides": {@@ -20,9 +19,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.jsonindex 61f6d2984..5c269847a 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json@@ -1,9 +1,10 @@{"extends": "@apify/tsconfig","compilerOptions": {- "module": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"],+ "lib": ["ES2022", "DOM"],"skipLibCheck": true,"incremental": false},diff --git a/test/e2e/cheerio-curl-impersonate-ts/test.mjs b/test/e2e/cheerio-curl-impersonate-ts/test.mjsindex 48aea4fe7..ecc0b9d90 100644--- a/test/e2e/cheerio-curl-impersonate-ts/test.mjs+++ b/test/e2e/cheerio-curl-impersonate-ts/test.mjs@@ -19,5 +19,3 @@ await expect();await expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains client IP');await expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains client IP');-await expect(result.uuidJsonResponse.uuid !== undefined, 'JSON response contains UUID');-await expect(JSON.parse(result.uuidTextResponse).uuid !== undefined, 'Text response contains UUID');diff --git a/test/e2e/cheerio-default-ts/actor/Dockerfile b/test/e2e/cheerio-default-ts/actor/Dockerfileindex 8f856bbb0..ba62560de 100644--- a/test/e2e/cheerio-default-ts/actor/Dockerfile+++ b/test/e2e/cheerio-default-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.jsonindex 9fbed6f67..18ff10310 100644--- a/test/e2e/cheerio-default-ts/actor/package.json+++ b/test/e2e/cheerio-default-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - TypeScript","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/cheerio-default-ts/actor/tsconfig.json b/test/e2e/cheerio-default-ts/actor/tsconfig.jsonindex 370e50c0a..dc194464d 100644--- a/test/e2e/cheerio-default-ts/actor/tsconfig.json+++ b/test/e2e/cheerio-default-ts/actor/tsconfig.json@@ -4,7 +4,8 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"]+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true},"include": ["./**/*.ts"]}diff --git a/test/e2e/cheerio-default/actor/Dockerfile b/test/e2e/cheerio-default/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-default/actor/Dockerfile+++ b/test/e2e/cheerio-default/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-default/actor/package.json b/test/e2e/cheerio-default/actor/package.jsonindex 2c64ec07d..e27b6bd77 100644--- a/test/e2e/cheerio-default/actor/package.json+++ b/test/e2e/cheerio-default/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Default","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile+++ b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-enqueue-links-base/actor/package.json b/test/e2e/cheerio-enqueue-links-base/actor/package.jsonindex 0aa9f8c70..ae50efae7 100644--- a/test/e2e/cheerio-enqueue-links-base/actor/package.json+++ b/test/e2e/cheerio-enqueue-links-base/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Enqueue Links with <base href>","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-enqueue-links/actor/Dockerfile b/test/e2e/cheerio-enqueue-links/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-enqueue-links/actor/Dockerfile+++ b/test/e2e/cheerio-enqueue-links/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-enqueue-links/actor/package.json b/test/e2e/cheerio-enqueue-links/actor/package.jsonindex ddea7a6d7..bdcd83b49 100644--- a/test/e2e/cheerio-enqueue-links/actor/package.json+++ b/test/e2e/cheerio-enqueue-links/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Enqueue Links","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-error-snapshot/actor/Dockerfile b/test/e2e/cheerio-error-snapshot/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-error-snapshot/actor/Dockerfile+++ b/test/e2e/cheerio-error-snapshot/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-error-snapshot/actor/package.json b/test/e2e/cheerio-error-snapshot/actor/package.jsonindex 1c989a12b..42b4a09fb 100644--- a/test/e2e/cheerio-error-snapshot/actor/package.json+++ b/test/e2e/cheerio-error-snapshot/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Should save errors snapshots","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js b/test/e2e/cheerio-ignore-ssl-errors/actor/main.jsindex c96f6347d..2160f0afc 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js@@ -22,6 +22,7 @@ await Actor.main(async () => {log.info('Bad ssl page opened!');await enqueueLinks({include: ['https://*.badssl.com/'],+ strategy: 'same-domain',label: 'DETAIL',selector: '.group a.bad',});diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json b/test/e2e/cheerio-ignore-ssl-errors/actor/package.jsonindex cf6d1dae4..ea7198ee4 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Ignore SSL Errors","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-impit-ts/actor/Dockerfile b/test/e2e/cheerio-impit-ts/actor/Dockerfileindex 9e846faee..7cd3bcbf9 100644--- a/test/e2e/cheerio-impit-ts/actor/Dockerfile+++ b/test/e2e/cheerio-impit-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,6 +15,7 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \&& npm install --only=prod --no-audit \&& npm update --no-audit \diff --git a/test/e2e/cheerio-impit-ts/actor/main.ts b/test/e2e/cheerio-impit-ts/actor/main.tsindex a4cccb647..810fd6ecc 100644--- a/test/e2e/cheerio-impit-ts/actor/main.ts+++ b/test/e2e/cheerio-impit-ts/actor/main.ts@@ -1,5 +1,5 @@-import type { Dictionary } from '@crawlee/cheerio';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Session, SessionPool } from '@crawlee/cheerio';+import type { Dictionary } from '@crawlee/types';import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { Actor } from 'apify';@@ -12,24 +12,36 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {const crawler = new CheerioCrawler({async requestHandler(context) {- const { body: text } = await context.sendRequest({- url: 'https://api.apify.com/v2/browser-info',- });+ const text = await (+ await context.sendRequest({+ url: 'https://api.apify.com/v2/browser-info',+ })+ ).text();- const { body: json } = await context.sendRequest<Dictionary>({- url: 'https://api.apify.com/v2/browser-info',- responseType: 'json',- });+ const json = (await (+ await context.sendRequest({+ url: 'https://api.apify.com/v2/browser-info',+ })+ ).json()) as Dictionary;await context.pushData({body: context.body,title: context.$('title').text(),- userAgent: json.headers['user-agent'],+ userAgent: (json.headers as Dictionary)['user-agent'],clientIpTextResponse: text,clientIpJsonResponse: json,});},httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),+ // The random default session fingerprint would override the client's browser+ // impersonation, so pin the sessions to Firefox as well.+ sessionPool: new SessionPool({+ createSessionFunction: async (opts) =>+ new Session({+ ...opts?.sessionOptions,+ fingerprint: { browser: 'firefox', platform: 'linux', device: 'desktop' },+ }),+ }),});await crawler.run(['https://crawlee.dev']);diff --git a/test/e2e/cheerio-impit-ts/actor/package.json b/test/e2e/cheerio-impit-ts/actor/package.jsonindex f862e3e2e..e9058fe1f 100644--- a/test/e2e/cheerio-impit-ts/actor/package.json+++ b/test/e2e/cheerio-impit-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Impit HTTP client","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -19,9 +18,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/cheerio-impit-ts/actor/tsconfig.json b/test/e2e/cheerio-impit-ts/actor/tsconfig.jsonindex 3d4d42c61..5c269847a 100644--- a/test/e2e/cheerio-impit-ts/actor/tsconfig.json+++ b/test/e2e/cheerio-impit-ts/actor/tsconfig.json@@ -4,7 +4,7 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"],+ "lib": ["ES2022", "DOM"],"skipLibCheck": true,"incremental": false},diff --git a/test/e2e/cheerio-initial-cookies/actor/Dockerfile b/test/e2e/cheerio-initial-cookies/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-initial-cookies/actor/Dockerfile+++ b/test/e2e/cheerio-initial-cookies/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.jsindex a1dcf6874..d309d8484 100644--- a/test/e2e/cheerio-initial-cookies/actor/main.js+++ b/test/e2e/cheerio-initial-cookies/actor/main.js@@ -14,14 +14,6 @@ const expectedCookies = [name: 'hook_request',value: 'true',},- {- name: 'got_options_upper_case',- value: 'true',- },- {- name: 'got_options_lower_case',- value: 'true',- },];const mainOptions = {@@ -36,21 +28,9 @@ await Actor.main(async () => {const crawler = new CheerioCrawler({additionalMimeTypes: ['application/json'],preNavigationHooks: [- ({ session, request }, gotOptions) => {- session.setCookies(- [- {- name: 'session',- value: 'true',- },- ],- request.url,- );+ async ({ session, request }) => {+ await session.setCookie('session=true', request.url);request.headers.cookie = 'hook_request=true';-- gotOptions.headers ??= {};- gotOptions.headers.Cookie = 'got_options_upper_case=true';- gotOptions.headers.cookie = 'got_options_lower_case=true';},],async requestHandler({ json }) {diff --git a/test/e2e/cheerio-initial-cookies/actor/package.json b/test/e2e/cheerio-initial-cookies/actor/package.jsonindex 747f78126..557deac9a 100644--- a/test/e2e/cheerio-initial-cookies/actor/package.json+++ b/test/e2e/cheerio-initial-cookies/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Initial Cookies","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjsindex 136a7d032..760a89858 100644--- a/test/e2e/cheerio-initial-cookies/test.mjs+++ b/test/e2e/cheerio-initial-cookies/test.mjs@@ -6,7 +6,7 @@ await initialize(testActorDirname);const { stats, datasetItems } = await runActor(testActorDirname);await expect(stats.requestsFinished === 1, 'All requests finished');-await expect(datasetItems[0].numberOfMatchingCookies === 5, 'Number of page cookies');+await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');await expect(datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,`Page cookies match the initial defined cookies. Number of non-matching cookies is ` +diff --git a/test/e2e/cheerio-max-requests/actor/Dockerfile b/test/e2e/cheerio-max-requests/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-max-requests/actor/Dockerfile+++ b/test/e2e/cheerio-max-requests/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-max-requests/actor/package.json b/test/e2e/cheerio-max-requests/actor/package.jsonindex 292a77daa..482e47c4e 100644--- a/test/e2e/cheerio-max-requests/actor/package.json+++ b/test/e2e/cheerio-max-requests/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Max Requests Per Crawl","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-page-info/actor/Dockerfile b/test/e2e/cheerio-page-info/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-page-info/actor/Dockerfile+++ b/test/e2e/cheerio-page-info/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-page-info/actor/package.json b/test/e2e/cheerio-page-info/actor/package.jsonindex 2dfac264f..919fa6b2c 100644--- a/test/e2e/cheerio-page-info/actor/package.json+++ b/test/e2e/cheerio-page-info/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Page Info","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile b/test/e2e/cheerio-request-queue-v2/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile+++ b/test/e2e/cheerio-request-queue-v2/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-request-queue-v2/actor/main.js b/test/e2e/cheerio-request-queue-v2/actor/main.jsindex cc2db1d09..68e6ba564 100644--- a/test/e2e/cheerio-request-queue-v2/actor/main.js+++ b/test/e2e/cheerio-request-queue-v2/actor/main.js@@ -1,5 +1,5 @@-import { Actor, LogLevel, log as Logger } from 'apify';-import { CheerioCrawler, Dataset } from '@crawlee/cheerio';+import { Actor, log as Logger } from 'apify';+import { ApifyLogAdapter, CheerioCrawler, Dataset } from '@crawlee/cheerio';const mainOptions = {exit: Actor.isAtHome(),@@ -22,10 +22,11 @@ await Actor.main(async () => {await Dataset.pushData({ url, pageTitle });},- log: Logger.child({- prefix: 'CheerioCrawler',- // level: LogLevel.DEBUG,- }),+ logger: new ApifyLogAdapter(+ Logger.child({+ prefix: 'CheerioCrawler',+ }),+ ),});try {diff --git a/test/e2e/cheerio-request-queue-v2/actor/package.json b/test/e2e/cheerio-request-queue-v2/actor/package.jsonindex 8fc0412f0..8d3395656 100644--- a/test/e2e/cheerio-request-queue-v2/actor/package.json+++ b/test/e2e/cheerio-request-queue-v2/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Request Queue V2","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-robots-file/actor/Dockerfile b/test/e2e/cheerio-robots-file/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-robots-file/actor/Dockerfile+++ b/test/e2e/cheerio-robots-file/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-robots-file/actor/package.json b/test/e2e/cheerio-robots-file/actor/package.jsonindex 44d38a2cc..c79747c55 100644--- a/test/e2e/cheerio-robots-file/actor/package.json+++ b/test/e2e/cheerio-robots-file/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Test - Robots file","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfileindex 8f856bbb0..ba62560de 100644--- a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile+++ b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-stop-resume-ts/actor/package.json b/test/e2e/cheerio-stop-resume-ts/actor/package.jsonindex 89fb8ae53..a0e00b3d5 100644--- a/test/e2e/cheerio-stop-resume-ts/actor/package.json+++ b/test/e2e/cheerio-stop-resume-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Crawler Stop-Resume Test - TypeScript","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json b/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.jsonindex 370e50c0a..dc194464d 100644--- a/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json+++ b/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json@@ -4,7 +4,8 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"]+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true},"include": ["./**/*.ts"]}diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.jsindex b926ccbcb..44f55c5a3 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js@@ -21,6 +21,7 @@ await Actor.main(async () => {log.info('Bad ssl page opened!');await enqueueLinks({include: ['https://*.badssl.com/'],+ strategy: 'same-domain',label: 'DETAIL',selector: '.group a.bad',});diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.jsonindex e1b75655f..e5ecdf60c 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Should throw on SSL Errors","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/input-json5/actor/Dockerfile+++ b/test/e2e/input-json5/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.jsonindex 785618c16..0d6e03127 100644--- a/test/e2e/input-json5/actor/package.json+++ b/test/e2e/input-json5/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "JSON5 input test","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"},@@ -12,9 +11,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfileindex 8f856bbb0..ba62560de 100644--- a/test/e2e/jsdom-default-ts/actor/Dockerfile+++ b/test/e2e/jsdom-default-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.jsonindex 7c7142b7b..895e4b9be 100644--- a/test/e2e/jsdom-default-ts/actor/package.json+++ b/test/e2e/jsdom-default-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "JSDOM Crawler Test - TypeScript","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/jsdom-default-ts/actor/tsconfig.json b/test/e2e/jsdom-default-ts/actor/tsconfig.jsonindex 370e50c0a..dc194464d 100644--- a/test/e2e/jsdom-default-ts/actor/tsconfig.json+++ b/test/e2e/jsdom-default-ts/actor/tsconfig.json@@ -4,7 +4,8 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"]+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true},"include": ["./**/*.ts"]}diff --git a/test/e2e/jsdom-react-ts/actor/Dockerfile b/test/e2e/jsdom-react-ts/actor/Dockerfileindex 8f856bbb0..ba62560de 100644--- a/test/e2e/jsdom-react-ts/actor/Dockerfile+++ b/test/e2e/jsdom-react-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/jsdom-react-ts/actor/package.json b/test/e2e/jsdom-react-ts/actor/package.jsonindex f09a8f586..4d5286746 100644--- a/test/e2e/jsdom-react-ts/actor/package.json+++ b/test/e2e/jsdom-react-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "JSDOM Crawler Test - React - TypeScript","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/jsdom-react-ts/actor/tsconfig.json b/test/e2e/jsdom-react-ts/actor/tsconfig.jsonindex 370e50c0a..dc194464d 100644--- a/test/e2e/jsdom-react-ts/actor/tsconfig.json+++ b/test/e2e/jsdom-react-ts/actor/tsconfig.json@@ -4,7 +4,8 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"]+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true},"include": ["./**/*.ts"]}diff --git a/test/e2e/linkedom-default-ts/actor/Dockerfile b/test/e2e/linkedom-default-ts/actor/Dockerfileindex 8f856bbb0..ba62560de 100644--- a/test/e2e/linkedom-default-ts/actor/Dockerfile+++ b/test/e2e/linkedom-default-ts/actor/Dockerfile@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder# copy all files, install all dependencies (including dev deps) and build the projectCOPY . ./+RUN rm -rf node_modules package-lock.jsonRUN npm install --include=dev \&& npm run build@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./COPY --from=builder /usr/src/app/main.js ./# install only prod deps+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/linkedom-default-ts/actor/package.json b/test/e2e/linkedom-default-ts/actor/package.jsonindex eafbc38ac..5311cea3b 100644--- a/test/e2e/linkedom-default-ts/actor/package.json+++ b/test/e2e/linkedom-default-ts/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "LinkeDOM Crawler Test - TypeScript","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -18,9 +17,6 @@"@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"devDependencies": {diff --git a/test/e2e/linkedom-default-ts/actor/tsconfig.json b/test/e2e/linkedom-default-ts/actor/tsconfig.jsonindex 370e50c0a..dc194464d 100644--- a/test/e2e/linkedom-default-ts/actor/tsconfig.json+++ b/test/e2e/linkedom-default-ts/actor/tsconfig.json@@ -4,7 +4,8 @@"module": "NodeNext","moduleResolution": "NodeNext","target": "ES2022",- "lib": ["DOM"]+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true},"include": ["./**/*.ts"]}diff --git a/test/e2e/migration/actor/Dockerfile b/test/e2e/migration/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/migration/actor/Dockerfile+++ b/test/e2e/migration/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/migration/actor/package.json b/test/e2e/migration/actor/package.jsonindex 007d21799..98c7a5c3f 100644--- a/test/e2e/migration/actor/package.json+++ b/test/e2e/migration/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Migration Test","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/playwright-default/actor/Dockerfile+++ b/test/e2e/playwright-default/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.jsindex 369ffa6a4..db1272731 100644--- a/test/e2e/playwright-default/actor/main.js+++ b/test/e2e/playwright-default/actor/main.js@@ -12,8 +12,8 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = 'networkidle';+ ({ gotoOptions }) => {+ gotoOptions.waitUntil = 'networkidle';},],async requestHandler({ page, enqueueLinks, request }) {diff --git a/test/e2e/playwright-default/actor/package.json b/test/e2e/playwright-default/actor/package.jsonindex 431d8c34c..89d24593b 100644--- a/test/e2e/playwright-default/actor/package.json+++ b/test/e2e/playwright-default/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Default","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile b/test/e2e/playwright-enqueue-links-base/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile+++ b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-enqueue-links-base/actor/package.json b/test/e2e/playwright-enqueue-links-base/actor/package.jsonindex b7f979514..c7effd3a8 100644--- a/test/e2e/playwright-enqueue-links-base/actor/package.json+++ b/test/e2e/playwright-enqueue-links-base/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Enqueue Links with <base href>","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -19,9 +18,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-enqueue-links/actor/package.json b/test/e2e/playwright-enqueue-links/actor/package.jsonindex 6894cb4f8..84675345d 100644--- a/test/e2e/playwright-enqueue-links/actor/package.json+++ b/test/e2e/playwright-enqueue-links/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Enqueue Links","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -20,9 +19,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-initial-cookies/actor/main.js b/test/e2e/playwright-initial-cookies/actor/main.jsindex 0c1f2b05b..2510e6470 100644--- a/test/e2e/playwright-initial-cookies/actor/main.js+++ b/test/e2e/playwright-initial-cookies/actor/main.js@@ -27,19 +27,11 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({preNavigationHooks: [- ({ session, request }, goToOptions) => {- session.setCookies(- [- {- name: 'session',- value: 'true',- },- ],- request.url,- );+ async ({ session, request, gotoOptions }) => {+ await session.setCookie('session=true', request.url);request.headers.cookie = 'hook_request=true';- goToOptions.waitUntil = 'networkidle';+ gotoOptions.waitUntil = 'networkidle';},],async requestHandler({ page }) {diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.jsonindex 5c6fbab28..67fa86db6 100644--- a/test/e2e/playwright-initial-cookies/actor/package.json+++ b/test/e2e/playwright-initial-cookies/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Initial Cookies","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-introduction-guide/actor/Dockerfile b/test/e2e/playwright-introduction-guide/actor/Dockerfileindex 9849e3409..7d570fa0c 100644--- a/test/e2e/playwright-introduction-guide/actor/Dockerfile+++ b/test/e2e/playwright-introduction-guide/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional \+ && npm install --only=prod \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-introduction-guide/actor/package.json b/test/e2e/playwright-introduction-guide/actor/package.jsonindex f7730a7ef..f5d96aa2d 100644--- a/test/e2e/playwright-introduction-guide/actor/package.json+++ b/test/e2e/playwright-introduction-guide/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Crawlee Introduction Guide (playwright + chrome)","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/playwright-multi-run/actor/Dockerfile+++ b/test/e2e/playwright-multi-run/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-multi-run/actor/package.json b/test/e2e/playwright-multi-run/actor/package.jsonindex 9394b7d0c..376373749 100644--- a/test/e2e/playwright-multi-run/actor/package.json+++ b/test/e2e/playwright-multi-run/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Multiple run calls to the same crawler","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/playwright-robots-file/actor/Dockerfile b/test/e2e/playwright-robots-file/actor/Dockerfileindex 6433c507b..3fd414cf0 100644--- a/test/e2e/playwright-robots-file/actor/Dockerfile+++ b/test/e2e/playwright-robots-file/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/playwright-robots-file/actor/package.json b/test/e2e/playwright-robots-file/actor/package.jsonindex d444b0ab3..1cb1576fa 100644--- a/test/e2e/playwright-robots-file/actor/package.json+++ b/test/e2e/playwright-robots-file/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Playwright Test - Robots file","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfileindex 2956f2058..6122e22a2 100644--- a/test/e2e/proxy-rotation/actor/Dockerfile+++ b/test/e2e/proxy-rotation/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -18,7 +21,7 @@ COPY /main.js ./ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chromeRUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/proxy-rotation/actor/main.js b/test/e2e/proxy-rotation/actor/main.jsindex db66b0ad1..29e30202c 100644--- a/test/e2e/proxy-rotation/actor/main.js+++ b/test/e2e/proxy-rotation/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { Dataset, KeyValueStore, PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, KeyValueStore, PuppeteerCrawler, Session, SessionPool } from '@crawlee/puppeteer';const mainOptions = {exit: Actor.isAtHome(),@@ -10,10 +10,19 @@ const mainOptions = {};await Actor.main(async () => {+ const proxyConfiguration = await Actor.createProxyConfiguration();+ const sessionPool = new SessionPool({+ createSessionFunction: async (opts) =>+ new Session({+ ...opts?.sessionOptions,+ maxUsageCount: 1,+ proxyInfo: await proxyConfiguration.newProxyInfo(),+ }),+ });+const crawler = new PuppeteerCrawler({- proxyConfiguration: await Actor.createProxyConfiguration(),+ sessionPool,maxConcurrency: 1,- sessionPoolOptions: { sessionOptions: { maxUsageCount: 1 } },async requestHandler({ response }) {const { clientIp } = await response.json();diff --git a/test/e2e/proxy-rotation/actor/package.json b/test/e2e/proxy-rotation/actor/package.jsonindex 88257553c..40bfb00ad 100644--- a/test/e2e/proxy-rotation/actor/package.json+++ b/test/e2e/proxy-rotation/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Proxy Test - Rotation","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfileindex 2956f2058..6122e22a2 100644--- a/test/e2e/puppeteer-default/actor/Dockerfile+++ b/test/e2e/puppeteer-default/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -18,7 +21,7 @@ COPY /main.js ./ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chromeRUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.jsindex 30c121812..83f6f588a 100644--- a/test/e2e/puppeteer-default/actor/main.js+++ b/test/e2e/puppeteer-default/actor/main.js@@ -12,8 +12,8 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PuppeteerCrawler({preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ ({ gotoOptions }) => {+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page, enqueueLinks, request, infiniteScroll }) {diff --git a/test/e2e/puppeteer-default/actor/package.json b/test/e2e/puppeteer-default/actor/package.jsonindex 7818af154..5a252a6d9 100644--- a/test/e2e/puppeteer-default/actor/package.json+++ b/test/e2e/puppeteer-default/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Default","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-enqueue-links/actor/package.json b/test/e2e/puppeteer-enqueue-links/actor/package.jsonindex 5acb45535..1703d799b 100644--- a/test/e2e/puppeteer-enqueue-links/actor/package.json+++ b/test/e2e/puppeteer-enqueue-links/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Enqueue Links","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -20,9 +19,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile b/test/e2e/puppeteer-error-snapshot/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile+++ b/test/e2e/puppeteer-error-snapshot/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-error-snapshot/actor/package.json b/test/e2e/puppeteer-error-snapshot/actor/package.jsonindex b5bfac8f3..ccfddff11 100644--- a/test/e2e/puppeteer-error-snapshot/actor/package.json+++ b/test/e2e/puppeteer-error-snapshot/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Should save errors snapshots","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.jsindex 205900a7d..4c6f3b484 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js@@ -16,8 +16,8 @@ await Actor.main(async () => {maxConcurrency: 2,launchContext: { launchOptions: { acceptInsecureCerts: true } },preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ ({ gotoOptions }) => {+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page, enqueueLinks, request, log }) {@@ -30,6 +30,7 @@ await Actor.main(async () => {log.info('Bad ssl page opened!');await enqueueLinks({include: ['https://*.badssl.com/'],+ strategy: 'same-domain',label: 'DETAIL',selector: '.group a.bad',});diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.jsonindex 21a8be8ee..a33a6bf32 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Ignore SSL Errors","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.jsindex 89ddc2319..259a945c4 100644--- a/test/e2e/puppeteer-initial-cookies/actor/main.js+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js@@ -27,19 +27,11 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PuppeteerCrawler({preNavigationHooks: [- ({ session, request }, goToOptions) => {- session.setCookies(- [- {- name: 'session',- value: 'true',- },- ],- request.url,- );+ async ({ session, request, gotoOptions }) => {+ await session.setCookie('session=true', request.url);request.headers.cookie = 'hook_request=true';- goToOptions.waitUntil = ['networkidle2'];+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page }) {diff --git a/test/e2e/puppeteer-initial-cookies/actor/package.json b/test/e2e/puppeteer-initial-cookies/actor/package.jsonindex 89258fd2a..5e296be06 100644--- a/test/e2e/puppeteer-initial-cookies/actor/package.json+++ b/test/e2e/puppeteer-initial-cookies/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Initial Cookies","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-page-info/actor/Dockerfile+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.jsindex a7837ee55..dd7140440 100644--- a/test/e2e/puppeteer-page-info/actor/main.js+++ b/test/e2e/puppeteer-page-info/actor/main.js@@ -12,8 +12,8 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PuppeteerCrawler({preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ ({ gotoOptions }) => {+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page, enqueueLinks, request }) {diff --git a/test/e2e/puppeteer-page-info/actor/package.json b/test/e2e/puppeteer-page-info/actor/package.jsonindex 1dad46ec1..684633a9e 100644--- a/test/e2e/puppeteer-page-info/actor/package.json+++ b/test/e2e/puppeteer-page-info/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Page Info","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.jsindex ae5efa02e..59a2ca48f 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js@@ -76,8 +76,11 @@ await Actor.main(async () => {const crawler = new PuppeteerCrawler({maxRequestsPerCrawl: 10,preNavigationHooks: [- (_crawlingContext, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ async ({ page, gotoOptions }) => {+ await page.evaluateOnNewDocument(() => {+ localStorage.setItem('themeExitPopup', 'true');+ });+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page, request, log, enqueueLinks, injectJQuery }) {diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json b/test/e2e/puppeteer-store-pagination-jquery/actor/package.jsonindex aa4a680f6..74a23773f 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Store Pagination with jQuery","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.jsindex 8c02ce9ee..d7a76b8d0 100644--- a/test/e2e/puppeteer-store-pagination/actor/main.js+++ b/test/e2e/puppeteer-store-pagination/actor/main.js@@ -74,8 +74,11 @@ await Actor.init({const crawler = new PuppeteerCrawler({maxRequestsPerCrawl: 10,preNavigationHooks: [- (_crawlingContext, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ async ({ page, gotoOptions }) => {+ await page.evaluateOnNewDocument(() => {+ localStorage.setItem('themeExitPopup', 'true');+ });+ gotoOptions.waitUntil = ['networkidle2'];},],});diff --git a/test/e2e/puppeteer-store-pagination/actor/package.json b/test/e2e/puppeteer-store-pagination/actor/package.jsonindex f0200f35a..3be0ebe52 100644--- a/test/e2e/puppeteer-store-pagination/actor/package.json+++ b/test/e2e/puppeteer-store-pagination/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Store Pagination","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfileindex 4d84738ff..695dedcae 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-puppeteer-chrome:24-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.jsindex 33ed50ead..8e8382e51 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js@@ -13,8 +13,8 @@ await Actor.main(async () => {const crawler = new PuppeteerCrawler({launchContext: { launchOptions: { acceptInsecureCerts: false } }, // This is the defaultpreNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = ['networkidle2'];+ ({ gotoOptions }) => {+ gotoOptions.waitUntil = ['networkidle2'];},],async requestHandler({ page, enqueueLinks, request, log }) {@@ -27,6 +27,7 @@ await Actor.main(async () => {log.info('Bad ssl page opened!');await enqueueLinks({include: ['https://*.badssl.com/'],+ strategy: 'same-domain',label: 'DETAIL',selector: '.group a.bad',});diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.jsonindex 31e5a2552..00ebde431 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Should throw on SSL Errors","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/request-queue-with-concurrency/actor/Dockerfile b/test/e2e/request-queue-with-concurrency/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/request-queue-with-concurrency/actor/Dockerfile+++ b/test/e2e/request-queue-with-concurrency/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/request-queue-with-concurrency/actor/package.json b/test/e2e/request-queue-with-concurrency/actor/package.jsonindex b41e9d076..79caeb1ba 100644--- a/test/e2e/request-queue-with-concurrency/actor/package.json+++ b/test/e2e/request-queue-with-concurrency/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Request Queue Test - Zero Concurrency","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-zero-concurrency/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile+++ b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/request-queue-zero-concurrency/actor/package.json b/test/e2e/request-queue-zero-concurrency/actor/package.jsonindex 0d47e0acc..98bc9651b 100644--- a/test/e2e/request-queue-zero-concurrency/actor/package.json+++ b/test/e2e/request-queue-zero-concurrency/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Request Queue Test - Zero Concurrency","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/request-queue-zero-concurrency/test.mjs b/test/e2e/request-queue-zero-concurrency/test.mjsindex 4c3e1eee0..0ad1ce2ef 100644--- a/test/e2e/request-queue-zero-concurrency/test.mjs+++ b/test/e2e/request-queue-zero-concurrency/test.mjs@@ -1,4 +1,8 @@-import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';+import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs';++// TODO(v4): redesign for the v4 RequestQueue. The test stages a stuck queue via the v3+// client-side `requestQueue.inProgress` set, which no longer exists in the rewritten queue.+await skipTest('The v4 RequestQueue has no client-side `inProgress` set to stage a stuck queue with');const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/request-skip-navigation/actor/Dockerfile b/test/e2e/request-skip-navigation/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/request-skip-navigation/actor/Dockerfile+++ b/test/e2e/request-skip-navigation/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/request-skip-navigation/actor/package.json b/test/e2e/request-skip-navigation/actor/package.jsonindex 136736e8d..f08f6d449 100644--- a/test/e2e/request-skip-navigation/actor/package.json+++ b/test/e2e/request-skip-navigation/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Request Test - skipNavigation","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool","@crawlee/http": "file:./packages/http-crawler",@@ -17,9 +16,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex 169e76c89..dcd0cc173 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -158,12 +158,12 @@ if (isMainThread) {for (let attempt = 0; attempt < 5; attempt++) {try {execSync(`pnpm exec camoufox-js fetch`, { stdio: 'inherit' });+ break;} catch (e) {console.error('Failed to fetch Camoufox', e);if (attempt === 4) throw e;console.log(`Retrying to fetch Camoufox (attempt ${attempt + 2}/5)...`);await new Promise((resolve) => setTimeout(resolve, 10e3));- continue;}}}diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/session-rotation/actor/Dockerfile+++ b/test/e2e/session-rotation/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.jsindex c9b3ba5e0..5d9d582f9 100644--- a/test/e2e/session-rotation/actor/main.js+++ b/test/e2e/session-rotation/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PlaywrightCrawler } from '@crawlee/playwright';+import { PlaywrightCrawler, Session, SessionPool } from '@crawlee/playwright';const mainOptions = {exit: Actor.isAtHome(),@@ -12,14 +12,20 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({maxRequestRetries: 10,- sessionPoolOptions: {- sessionOptions: {- maxErrorScore: 2,- },- },- requestHandler: async ({ session }) => {+ sessionPool: new SessionPool({+ createSessionFunction: async (opts) =>+ new Session({+ ...opts?.sessionOptions,+ maxErrorScore: 2,+ }),+ }),+ requestHandler: async ({ session, registerDeferredCleanup }) => {+ // The handler runs in a storage transaction, so a plain pushData would be rolled+ // back when the handler throws. Deferred cleanups run outside of it.const { id, usageCount, errorScore } = session;- await Actor.pushData({ id, usageCount, errorScore });+ registerDeferredCleanup(async () => {+ await Actor.pushData({ id, usageCount, errorScore });+ });throw new Error('retry');},});diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.jsonindex 9bcc4a1d7..81e5a4d7c 100644--- a/test/e2e/session-rotation/actor/package.json+++ b/test/e2e/session-rotation/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Session Test - Rotation","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",@@ -18,9 +17,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/stagehand-claude/actor/Dockerfile b/test/e2e/stagehand-claude/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/stagehand-claude/actor/Dockerfile+++ b/test/e2e/stagehand-claude/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/stagehand-claude/actor/package.json b/test/e2e/stagehand-claude/actor/package.jsonindex 5c85a3c3e..6108fb3f0 100644--- a/test/e2e/stagehand-claude/actor/package.json+++ b/test/e2e/stagehand-claude/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Stagehand Test - Claude/Anthropic","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@browserbasehq/stagehand": "^3.0.7","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/stagehand-concurrent/actor/Dockerfile b/test/e2e/stagehand-concurrent/actor/Dockerfileindex 491acc4dd..77da7b583 100644--- a/test/e2e/stagehand-concurrent/actor/Dockerfile+++ b/test/e2e/stagehand-concurrent/actor/Dockerfile@@ -2,13 +2,16 @@ FROM node:24 AS builderCOPY /packages ./packagesCOPY /package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm updateFROM apify/actor-node-playwright-chrome:24-1.58.2-beta-RUN rm -r node_modules+# The browser base images ship a preinstalled node_modules - wipe it so the+# builder tree (with file: symlinks) can be copied over cleanly.+RUN rm -rf node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./@@ -16,7 +19,7 @@ COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/stagehand-concurrent/actor/main.js b/test/e2e/stagehand-concurrent/actor/main.jsindex 95ee351c5..b5cc03550 100644--- a/test/e2e/stagehand-concurrent/actor/main.js+++ b/test/e2e/stagehand-concurrent/actor/main.js@@ -11,7 +11,12 @@ const mainOptions = {};await Actor.main(async () => {- const browserIds = new Set();+ // Playwright's Browser has no `process()`, so number the instances by object identity.+ const browserIds = new Map();+ const getBrowserId = (browser) => {+ if (!browserIds.has(browser)) browserIds.set(browser, `browser-${browserIds.size + 1}`);+ return browserIds.get(browser);+ };const crawler = new StagehandCrawler({maxConcurrency: 3,@@ -29,9 +34,7 @@ await Actor.main(async () => {log.info(`Processing ${request.loadedUrl}`);// Track which browser instance handled this request via the underlying Playwright browser- const browser = page.context().browser();- const browserId = browser?.process()?.pid ?? 'unknown';- browserIds.add(browserId);+ const browserId = getBrowserId(page.context().browser());// Simple extraction - just get the page titleconst result = await page.extract(diff --git a/test/e2e/stagehand-concurrent/actor/package.json b/test/e2e/stagehand-concurrent/actor/package.jsonindex 9483c1f48..6d2a1b5dd 100644--- a/test/e2e/stagehand-concurrent/actor/package.json+++ b/test/e2e/stagehand-concurrent/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Stagehand Test - Concurrent browsers","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@browserbasehq/stagehand": "^3.0.7","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/storage-open-return-storage-object/actor/Dockerfile b/test/e2e/storage-open-return-storage-object/actor/Dockerfileindex c1788cb1f..e96d0b851 100644--- a/test/e2e/storage-open-return-storage-object/actor/Dockerfile+++ b/test/e2e/storage-open-return-storage-object/actor/Dockerfile@@ -3,11 +3,12 @@ FROM apify/actor-node:24-betaCOPY packages ./packagesCOPY package*.json ./+RUN rm -rf node_modules package-lock.jsonRUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \+ && npm install --only=prod --no-audit \&& npm update --no-audit \&& echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \+ && (npm list --only=prod --all || true) \&& echo "Node.js version:" \&& node --version \&& echo "NPM version:" \diff --git a/test/e2e/storage-open-return-storage-object/actor/main.js b/test/e2e/storage-open-return-storage-object/actor/main.jsindex cf8d7154d..70c090458 100644--- a/test/e2e/storage-open-return-storage-object/actor/main.js+++ b/test/e2e/storage-open-return-storage-object/actor/main.js@@ -12,7 +12,7 @@ await Actor.main(async () => {const kv = await KeyValueStore.open();const dataset = await Dataset.open();await kv.setValue('storageObject', {- keyValueStorageObject: kv.storageObject,- datasetStorageObject: dataset.storageObject,+ keyValueStorageObject: { id: kv.id, name: kv.name ?? null },+ datasetStorageObject: { id: dataset.id, name: dataset.name ?? null },});}, mainOptions);diff --git a/test/e2e/storage-open-return-storage-object/actor/package.json b/test/e2e/storage-open-return-storage-object/actor/package.jsonindex 3a6bda82f..1f9003689 100644--- a/test/e2e/storage-open-return-storage-object/actor/package.json+++ b/test/e2e/storage-open-return-storage-object/actor/package.json@@ -3,8 +3,7 @@"version": "0.0.1","description": "Key-Value Store - Return storage object on open","dependencies": {- "apify": "next-v4",- "@apify/storage-local": "^3.0.0",+ "apify": "^4.0.0-beta.24","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/core": "file:./packages/core","@crawlee/types": "file:./packages/types",@@ -14,9 +13,6 @@"apify": {"@crawlee/core": "file:./packages/core","@crawlee/utils": "file:./packages/utils"- },- "@apify/storage-local": {- "better-sqlite3": "^11.10.0"}},"scripts": {diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 416763ec1..31bb614ef 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -6,7 +6,7 @@ import { dirname, join } from 'node:path';import { setTimeout } from 'node:timers/promises';import { fileURLToPath } from 'node:url';-import { URL_NO_COMMAS_REGEX } from '@crawlee/utils';+import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal';import { Actor, ApifyClient } from 'apify';import fs from 'fs-extra';import { got } from 'got';@@ -24,12 +24,12 @@ function execSync(command, options) {* the base crawler created and installs its own, which lands on the next id.* @param {string} name*/-const isCrawlerStatisticsKey = (name) => name.startsWith('CRAWLEE_CRAWLER_STATISTICS_');+const isCrawlerStatisticsKey = (name) => name.startsWith('CRAWLEE_CRAWLER_STATISTICS');/*** @param {string} name*/-const isPrivateEntry = (name) => isCrawlerStatisticsKey(name) || name === 'CRAWLEE_SESSION_POOL_STATE';+const isPrivateEntry = (name) => isCrawlerStatisticsKey(name) || name.startsWith('CRAWLEE_SESSION_POOL_STATE');export const SKIPPED_TEST_CLOSE_CODE = 404;@@ -55,6 +55,7 @@ export function getStorage(dirName) {* @param {string} dirName*/export async function getStats(dirName) {+ // fs-storage writes key-value records extensionless (the filename is the record key).const dir = join(getStorage(dirName), 'key_value_stores/default');if (!existsSync(dir)) {@@ -191,8 +192,13 @@ export async function runActor(dirName, memory = 4096) {// id: runId,buildId,userId,+ status: runStatus,} = await client.run(runId).waitForFinish();+ if (runStatus !== 'SUCCEEDED') {+ console.log(`[run] actor run finished with status ${runStatus} - the assertions below will likely fail`);+ }+getKeyValueStoreItems = async (name) => {const kvResult = await client.keyValueStore(name ? `${userId}/${name}` : defaultKeyValueStoreId).get();@@ -467,7 +473,8 @@ export async function getLocalKeyValueStoreItems(dirName, kvName) {const filePath = join(storePath, fileName.name);const buffer = await readFile(filePath);- const name = fileName.name.split('.').slice(0, -1).join('.');+ // fs-storage writes records extensionless — the filename is the record key.+ const name = fileName.name;if (isPrivateEntry(name)) {continue;