Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: adapt E2E suite to v4 and fix regressions it caught (#4012)

Martin Adámekea9da4c1c2ec
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 796c87f2e..27fc79b55 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -18,7 +18,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- storage: [ LOCAL, MEMORY, PLATFORM ]
+ # TODO(v4): re-enable LOCAL once @apify/storage-local supports v4, see https://github.com/apify/crawlee/issues/3406
+ storage: [ MEMORY, PLATFORM ]
steps:
- name: Checkout repository
diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
index 1a22baf1d..b5a122e69 100644
--- a/docs/public-api/crawlee-jsdom.api.md
+++ b/docs/public-api/crawlee-jsdom.api.md
@@ -7,6 +7,7 @@
import { AnyPredicate } from 'ow';
import { ArrayPredicate } from 'ow';
import { BasePredicate } from 'ow';
+import type { BatchAddRequestsResult } from '@crawlee/types';
import { BooleanPredicate } from 'ow';
import { CheerioAPI } from 'cheerio';
import { CheerioRoot } from '@crawlee/utils/internal';
@@ -51,7 +52,7 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext
readonly body: string;
readonly document: Document;
} & {
- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<CheerioAPI>;
}>;
@@ -123,6 +124,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
body: string;
// (undocumented)
document: Document;
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
// (undocumented)
diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.md
index cf1b4affa..a88c86bc1 100644
--- a/docs/public-api/crawlee-linkedom.api.md
+++ b/docs/public-api/crawlee-linkedom.api.md
@@ -4,6 +4,7 @@
```ts
+import type { BatchAddRequestsResult } from '@crawlee/types';
import * as cheerio from 'cheerio';
import { CheerioRoot } from '@crawlee/utils/internal';
import { ContextPipeline } from '@crawlee/http';
@@ -33,14 +34,14 @@ export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = L
// @public
export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
+ constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
readonly window: Window;
readonly body: string;
readonly document: Document;
} & {
- enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<unknown>;
+ enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
}>;
@@ -61,6 +62,7 @@ export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // w
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
// (undocumented)
document: Document;
+ enqueueLinks(options?: LinkeDOMCrawlerEnqueueLinksOptions): Promise<BatchAddRequestsResult>;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
// (undocumented)
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 41053bb7a..c4825a5f5 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -464,6 +464,27 @@ const crawler = new CheerioCrawler({
Previously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.
+### The default HTTP client is now `impit`
+
+The HTTP crawlers (and `sendRequest`) no longer use `got-scraping` by default. The default client is now `ImpitHttpClient` from the optional `@crawlee/impit-client` package (a Rust-based client with TLS fingerprint impersonation); when it is not installed, the crawlers fall back to a plain `fetch`-based client with a logged warning (no proxy support or impersonation). To keep using `got-scraping`, install `@crawlee/got-scraping-client` and pass `httpClient: new GotScrapingHttpClient()` explicitly.
+
+Note that the session's fingerprint drives the impersonation: each new session gets a randomized realistic fingerprint by default, and its `browser` hint overrides the `browser` option passed to the `ImpitHttpClient` constructor. To force a specific browser family, pin the fingerprint on the sessions instead:
+
+```ts
+const crawler = new CheerioCrawler({
+ httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),
+ sessionPool: new SessionPool({
+ createSessionFunction: async (opts) =>
+ new Session({
+ ...opts?.sessionOptions,
+ fingerprint: { browser: 'firefox', platform: 'linux', device: 'desktop' },
+ }),
+ }),
+});
+```
+
+See the [Avoid getting blocked](https://crawlee.dev/js/docs/guides/avoid-blocking) guide for how the session fingerprint interacts with both HTTP and browser crawlers.
+
### `HttpClient` instances return `Response` objects
The interface of `HttpClient` instances was changed to return the [native `Response` objects](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of custom `HttpResponse` objects.
@@ -584,6 +605,19 @@ The mechanism can be disabled entirely with `transactionalStorage: false` on any
- `checkStorageAccess` and `withCheckedStorageAccess` are superseded by the transaction mechanism; the per-call-site helper is now `withDirectStorageAccess()`.
- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone — `requestsFromUrl` sources are now expanded when added, so the fetched URLs appear in `enqueuedUrls` (and are what `fullResultComparator` compares).
+### `storageObject` is removed from storage classes
+
+The `storageObject` property (the raw backend record exposed on `Dataset`, `KeyValueStore` and `RequestQueue` instances) is removed. The commonly used fields are available directly on the instance as `id` and `name`, and `Dataset.getInfo()` returns the full metadata:
+
+```ts
+// v3
+const { id, name } = dataset.storageObject;
+
+// v4
+const { id, name } = dataset;
+const info = await dataset.getInfo();
+```
+
### `KeyValueStore.getPublicUrl` is now async
The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public URL directly from the storage backend.
@@ -632,6 +666,26 @@ await enqueueLinks({
});
```
+#### `include` patterns no longer replace the enqueue strategy
+
+In v3, providing any URL patterns (`globs`, `regexps`, `pseudoUrls`) disabled the default enqueue strategy - the patterns were the only filter. In v4, the `strategy` **always applies** and is combined with `include` using AND logic: a URL must match an `include` pattern *and* satisfy the strategy (default `same-hostname`) to be enqueued. This mirrors the behavior of Crawlee for Python.
+
+The practical consequence: patterns that used to match URLs on other hostnames (e.g. subdomains) now silently enqueue nothing unless you relax the strategy explicitly:
+
+```ts
+// v3 - the glob alone allowed subdomains
+await enqueueLinks({
+ globs: ['https://*.example.com/'],
+});
+
+// v4 - the default same-hostname strategy would filter the subdomains out again,
+// so relax it explicitly
+await enqueueLinks({
+ include: ['https://*.example.com/'],
+ strategy: 'same-domain',
+});
+```
+
### `transformRequestFunction` precedence in `enqueueLinks`
The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`include`, `exclude`) instead of before. This means it has the highest priority and can overwrite any request options set by the global `label` / `userData` options.
@@ -734,6 +788,37 @@ The exported handler types were reshaped accordingly. `ErrorHandler` and `Reques
`serialize` and `deserialize` now take and return values rather than strings (so v3 records will not load) and each also accept a [Standard Schema](https://standardschema.dev) — a zod codec works as `deserialize` directly — `reset()` is synchronous, no longer clears the persisted record (the new `resetStore()` does) and doubles as a way to establish the state without awaiting `initialize()`, `persistStateKvsName` and `persistStateKvsId` collapsed into a single `keyValueStore` option taking a store (or a pending `KeyValueStore.open()`), `defaultState` also accepts a factory (which you need for a state `structuredClone` cannot rebuild, as the deep copy no longer goes through `serialize`/`deserialize`), and there is a new `persistenceTimeoutMillis` option. `teardown()` is no longer terminal — `initialize()` can be called again to open another persistence window — and a write that fails during a periodic `PERSIST_STATE` or during `teardown()` is warned about rather than thrown. A direct `persistState()` still throws.
+### The `log` crawler option is replaced by `logger`
+
+The crawler constructors no longer accept a `log` option with an `@apify/log` `Log` instance. Pass a `logger` implementing the `CrawleeLogger` interface instead. To keep using an `@apify/log` instance (e.g. a `child()` with a custom prefix), wrap it in the `ApifyLogAdapter` from `@crawlee/core`:
+
+```ts
+// v3
+import { CheerioCrawler, log } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+ log: log.child({ prefix: 'MyCrawler' }),
+});
+
+// v4
+import { ApifyLogAdapter, CheerioCrawler, log } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+ logger: new ApifyLogAdapter(log.child({ prefix: 'MyCrawler' })),
+});
+```
+
+Related: `crawler.log` and the crawling context `log` are typed as `CrawleeLogger`, which has no `setLevel()` method - level filtering belongs to the underlying logging library. With the default logger, set the level on the global `@apify/log` instance instead:
+
+```ts
+// v3
+crawler.log.setLevel(LogLevel.DEBUG);
+
+// v4
+import log, { LogLevel } from '@apify/log';
+log.setLevel(LogLevel.DEBUG);
+```
+
### The `log` property is typed as `CrawleeLogger`
The `log` property exposed throughout the public API (on the crawling context, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.
@@ -1005,6 +1090,40 @@ If you don't pass a predictor, nothing changes: the crawler builds one from `ren
This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.
+### `handleCloudflareChallenge` hooks must return the response
+
+In v3, calling `handleCloudflareChallenge()` in a `postNavigationHooks` entry was enough on its own - the helper received the session and removed `403` from the session pool's blocked status codes, so the challenge page (which is served with a 403 status) did not trip the blocked-request detection.
+
+In v4, blocked status code handling is internal to the crawler and runs *after* the post-navigation hooks, and the helper no longer touches it. Instead, `handleCloudflareChallenge()` returns the reloaded `Response` after solving the challenge, and the hook must return it as the new context response - otherwise the crawler still sees the original 403 challenge response and throws a `SessionError` before your `requestHandler` runs, even when the challenge was solved successfully.
+
+Use the pre-wrapped `handleCloudflareChallengeHook()` (it also handles the no-challenge case), or return the response yourself:
+
+```ts
+// v3
+postNavigationHooks: [
+ async ({ handleCloudflareChallenge }) => {
+ await handleCloudflareChallenge();
+ },
+],
+
+// v4
+import { handleCloudflareChallengeHook } from 'crawlee';
+
+postNavigationHooks: [handleCloudflareChallengeHook()],
+
+// v4 (manual equivalent)
+postNavigationHooks: [
+ async ({ handleCloudflareChallenge }) => {
+ // Returning `{ response: undefined }` would clobber the navigation response
+ // when there was no challenge, so only return it when one was solved.
+ const response = await handleCloudflareChallenge();
+ return response && { response };
+ },
+],
+```
+
+If you called the standalone `playwrightUtils.handleCloudflareChallenge(page, url, session, options)` directly, note that the `session` parameter is gone - the v4 signature is `handleCloudflareChallenge(page, url, options)`, so an options object passed in the old fourth position would be silently ignored.
+
## Only if you customize crawler statistics
Applies when you passed `statisticsOptions` to a crawler, or subclassed `Statistics`.
@@ -1725,6 +1844,7 @@ Besides the resource-detection helpers above, several other `@crawlee/utils` exp
- **Removed URL helpers:** `filterUrl(target, origin, strategy)`, `matchesEnqueueStrategy(strategy, target, origin)`, and the `UNSUPPORTED_SCHEME_MESSAGE` constant. URL filtering by enqueue strategy is now internal to `enqueueLinks`. The related `filterRequestsByPatterns(requests, patterns?, onSkippedUrl?)` function (from `@crawlee/core`) was removed for the same reason — pattern-based request filtering now happens inside `enqueueLinks`.
- **Relocated enums/types:** `EnqueueStrategy` now lives in `@crawlee/core` and `SearchParams` in `@crawlee/types`. They are no longer re-exported from `@crawlee/utils`, so `import { EnqueueStrategy } from '@crawlee/utils'` breaks — import them from `crawlee` (the meta-package) or from `@crawlee/core` / `@crawlee/types` instead.
- **Removed `RobotsFile` alias:** `RobotsFile` was an alias for the `RobotsTxtFile` class and is removed. Rename any usage to `RobotsTxtFile`; the class itself is unchanged apart from the signature change described below.
+- **Split into public and `/internal` entry points:** the main `@crawlee/utils` entry now exposes only the user-facing helpers (`sleep`, `htmlToText`, `extractUrls`, `downloadListOfUrls`, `expandShadowRoots`, the `social` namespace, the Open Graph parser, and the robots/sitemap utilities). Helpers that primarily serve the crawler packages - e.g. `URL_NO_COMMAS_REGEX`, `URL_WITH_COMMAS_REGEX`, `extractUrlsFromCheerio`, `tryAbsoluteURL`, the `CheerioRoot` type, and the blocked-detection and iterable helpers - moved to the `@crawlee/utils/internal` entry point. They keep working, but imports need updating: `import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal'`. As the name suggests, the internal entry follows no semver guarantees.
#### `RobotsTxtFile.find` signature changed; sitemap options removed
diff --git a/package.json b/package.json
index 9990f5f5a..b39a2f98c 100644
--- a/package.json
+++ b/package.json
@@ -65,12 +65,14 @@
"@crawlee/basic": "workspace:*",
"@crawlee/cheerio": "workspace:*",
"@crawlee/core": "workspace:*",
+ "@crawlee/http-client": "workspace:*",
"@crawlee/impit-client": "workspace:*",
"@crawlee/jsdom": "workspace:*",
"@crawlee/linkedom": "workspace:*",
"@crawlee/playwright": "workspace:*",
"@crawlee/puppeteer": "workspace:*",
"@crawlee/stagehand": "workspace:*",
+ "@crawlee/types": "workspace:*",
"@crawlee/utils": "workspace:*",
"@microsoft/api-extractor": "^7.58.9",
"@playwright/browser-chromium": "1.61.1",
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 90d881754..3afc05371 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -972,7 +972,16 @@ export class BasicCrawler<
logger ||
(configuration !== undefined && configuration !== serviceLocator.getConfiguration())
) {
- const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageBackend, logger);
+ // Inherit the ambient locator's already-set services for anything not explicitly
+ // provided - e.g. only passing a `logger` must not detach the crawler from a globally
+ // configured storage backend.
+ const ambientServices = serviceLocator.getServicesIfSet();
+ const scopedServiceLocator = new ServiceLocator(
+ configuration ?? ambientServices.configuration,
+ eventManager ?? ambientServices.eventManager,
+ storageBackend ?? ambientServices.storageBackend,
+ logger ?? ambientServices.logger,
+ );
serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 12d0898f4..4ec3dc7d5 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -265,7 +265,7 @@ export class CheerioCrawler<
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
enqueueLinks: originalEnqueueLinks,
- })) as BatchAddRequestsResult; // TODO make this type safe
+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
},
waitForSelector: async (selector: string, _timeoutMs?: number) => {
if (crawlingContext.$(selector).get().length === 0) {
diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.ts
index 09832c7ec..db33580fc 100644
--- a/packages/core/src/crawlers/error_snapshotter.ts
+++ b/packages/core/src/crawlers/error_snapshotter.ts
@@ -116,7 +116,7 @@ export class ErrorSnapshotter {
}
/**
- * Save the HTML snapshot of the page, and return the fileName with the extension.
+ * Save the HTML snapshot of the page, and return the key it was stored under.
*/
async saveHTMLSnapshot(
html: string,
@@ -125,7 +125,9 @@ export class ErrorSnapshotter {
): Promise<string | undefined> {
try {
await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
- return `${fileName}.html`;
+ // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
+ // where local storage put the extension in the key) would break `getPublicUrl`.
+ return fileName;
} catch {
return undefined;
}
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index ac1a2ac51..2b1f642ce 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -225,7 +225,10 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
userData.label = label;
}
- this.#userData = { __crawlee: {}, ...userData };
+ // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
+ // bag is non-enumerable, so the spread alone would silently drop the internal state
+ // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
+ this.#userData = { __crawlee: (userData as Dictionary).__crawlee ?? {}, ...userData };
// `userData` must stay an enumerable own accessor — serialization in the storages relies on it
Object.defineProperties(this, {
diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.ts
index cad4b8a3f..4b2437366 100644
--- a/packages/core/src/service_locator.ts
+++ b/packages/core/src/service_locator.ts
@@ -83,6 +83,18 @@ interface ServiceLocatorInterface {
*/
getStorageInstanceManager(): StorageInstanceManager;
+ /**
+ * Returns the currently set services without triggering the implicit creation of defaults.
+ * Used to inherit already-materialized services into crawler-scoped service locators.
+ * @internal
+ */
+ getServicesIfSet(): {
+ configuration?: Configuration;
+ eventManager?: EventManager;
+ storageBackend?: StorageBackend;
+ logger?: CrawleeLogger;
+ };
+
/**
* Resets the service locator to its initial state.
* Used mainly for testing purposes.
@@ -155,6 +167,21 @@ export class ServiceLocator implements ServiceLocatorInterface {
this.#logger = logger;
}
+ /** @internal */
+ getServicesIfSet(): {
+ configuration?: Configuration;
+ eventManager?: EventManager;
+ storageBackend?: StorageBackend;
+ logger?: CrawleeLogger;
+ } {
+ return {
+ configuration: this.#configuration,
+ eventManager: this.#eventManager,
+ storageBackend: this.#storageBackend,
+ logger: this.#logger,
+ };
+ }
+
getConfiguration(): Configuration {
if (!this.#configuration) {
this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts
index 656961a10..d0af8e27a 100644
--- a/packages/fs-storage/src/file-system-storage.ts
+++ b/packages/fs-storage/src/file-system-storage.ts
@@ -5,15 +5,20 @@ import type * as storage from '@crawlee/types';
import type { CrawleeLogger } from '@crawlee/types';
import { s } from '@sapphire/shapeshift';
-import {
- FileSystemDatasetClient as NativeDatasetBackend,
- FileSystemKeyValueStoreClient as NativeKeyValueStoreBackend,
- FileSystemRequestQueueClient as NativeRequestQueueBackend,
-} from '@crawlee/fs-storage-native';
import { DatasetBackend } from './resource-clients/dataset.js';
import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
import { RequestQueueBackend } from './resource-clients/request-queue.js';
+// The native package throws at load time on platforms without a published binary (e.g.
+// linux musl), and `@crawlee/core` imports this module eagerly via its service locator.
+// Load it lazily so merely importing `@crawlee/fs-storage` stays safe everywhere and the
+// native binding is only loaded when a file-system storage is actually used.
+let nativeModule: Promise<typeof import('@crawlee/fs-storage-native')> | undefined;
+async function importNativeModule() {
+ nativeModule ??= import('@crawlee/fs-storage-native');
+ return nativeModule;
+}
+
/** The alias `@crawlee/core` opens the default (unnamed) storage under. */
const DEFAULT_STORAGE_ALIAS = '__default__';
@@ -125,7 +130,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return found;
}
- const nativeBackend = await NativeDatasetBackend.open(id, name, alias, this.localDataDirectory);
+ const nativeBackend = await (
+ await importNativeModule()
+ ).FileSystemDatasetClient.open(id, name, alias, this.localDataDirectory);
const newStore = await DatasetBackend.create({
name: alias ? undefined : (name ?? cacheKey),
cacheKey,
@@ -150,7 +157,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return found;
}
- const nativeBackend = await NativeKeyValueStoreBackend.open(id, name, alias, this.localDataDirectory);
+ const nativeBackend = await (
+ await importNativeModule()
+ ).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);
const newStore = await KeyValueStoreBackend.create({
name: alias ? undefined : (name ?? cacheKey),
cacheKey,
@@ -175,7 +184,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
return found;
}
- const nativeBackend = await NativeRequestQueueBackend.open(
+ const nativeBackend = await (
+ await importNativeModule()
+ ).FileSystemRequestQueueClient.open(
id,
name,
alias,
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 23778694a..52700f8d7 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -23,7 +23,7 @@ import {
Router,
tryAbsoluteURL,
} from '@crawlee/http';
-import type { Dictionary } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import { type CheerioRoot } from '@crawlee/utils/internal';
import { type RobotsTxtFile, sleep } from '@crawlee/utils';
import type { DOMWindow } from 'jsdom';
@@ -97,6 +97,11 @@ export interface JSDOMCrawlingContext<
* ```
*/
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+
+ /**
+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
+ */
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
}
export type JSDOMRequestHandler<
@@ -360,7 +365,7 @@ export class JSDOMCrawler<
return {
enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {
- return domCrawlerEnqueueLinks({
+ return (await domCrawlerEnqueueLinks({
// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;
// pre-clamping it here would make the crawler log the internal limit as a user-provided one
options: enqueueOptions,
@@ -371,7 +376,7 @@ export class JSDOMCrawler<
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
enqueueLinks: originalEnqueueLinks,
- });
+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
},
async waitForSelector(selector: string, timeoutMs = 5_000) {
const cheerio = await import('cheerio');
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index a558e1023..cda4dc2d2 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -23,7 +23,7 @@ import {
Router,
tryAbsoluteURL,
} from '@crawlee/http';
-import type { Dictionary } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import { type CheerioRoot } from '@crawlee/utils/internal';
import { type RobotsTxtFile, sleep } from '@crawlee/utils';
import * as cheerio from 'cheerio';
@@ -90,6 +90,11 @@ export interface LinkeDOMCrawlingContext<
* ```
*/
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+
+ /**
+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
+ */
+ enqueueLinks(options?: LinkeDOMCrawlerEnqueueLinksOptions): Promise<BatchAddRequestsResult>;
}
export type LinkeDOMRequestHandler<
@@ -180,7 +185,7 @@ export class LinkeDOMCrawler<
> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
static #parser = new DOMParser();
- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {
+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes> = {}) {
const { contextPipelineBuilder, ...rest } = options;
super({
@@ -244,23 +249,23 @@ export class LinkeDOMCrawler<
}
}
- private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string }) {
+ private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: Window }) {
const originalEnqueueLinks = crawlingContext.enqueueLinks;
return {
enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {
- return linkedomCrawlerEnqueueLinks({
+ return (await linkedomCrawlerEnqueueLinks({
// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;
// pre-clamping it here would make the crawler log the internal limit as a user-provided one
options: enqueueOptions,
- window: document.defaultView,
+ window: crawlingContext.window,
requestManager: await this.getRequestManager(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
enqueueLinks: originalEnqueueLinks,
- });
+ })) as BatchAddRequestsResult; // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
},
async waitForSelector(selector: string, timeoutMs = 5_000) {
const $ = cheerio.load(crawlingContext.body);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e206172d3..c07c9e452 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -14,7 +14,7 @@ overrides:
tar: ^7.5.16
lerna>js-yaml: ^4.2.0
lerna>minimatch: ^3.1.4
- apify: 4.0.0-beta.19
+ apify: ^4.0.0-beta.24
apify>@crawlee/core: workspace:*
apify>@crawlee/types: workspace:*
apify>@crawlee/utils: workspace:*
@@ -49,6 +49,9 @@ importers:
'@crawlee/core':
specifier: workspace:*
version: link:packages/core
+ '@crawlee/http-client':
+ specifier: workspace:*
+ version: link:packages/http-client
'@crawlee/impit-client':
specifier: workspace:*
version: link:packages/impit-client
@@ -67,6 +70,9 @@ importers:
'@crawlee/stagehand':
specifier: workspace:*
version: link:packages/stagehand-crawler
+ '@crawlee/types':
+ specifier: workspace:*
+ version: link:packages/types
'@crawlee/utils':
specifier: workspace:*
version: link:packages/utils
@@ -143,8 +149,8 @@ importers:
specifier: ^4.0.16
version: 4.1.4([email protected])
apify:
- specifier: 4.0.0-beta.19
- version: 4.0.0-beta.19([email protected])
+ specifier: ^4.0.0-beta.24
+ version: 4.0.0-beta.24([email protected])
apify-node-curl-impersonate:
specifier: ^1.0.23
version: 1.0.29
@@ -266,8 +272,8 @@ importers:
specifier: workspace:*
version: link:../packages/stagehand-crawler
apify:
- specifier: 4.0.0-beta.19
- version: 4.0.0-beta.19([email protected])
+ specifier: ^4.0.0-beta.24
+ version: 4.0.0-beta.24([email protected])
crawlee:
specifier: workspace:*
version: link:../packages/crawlee
@@ -1307,9 +1313,6 @@ packages:
engines: {node: '>= 0.10'}
hasBin: true
- '@apify/[email protected]':
- resolution: {integrity: sha512-lyvwMXee8SJNjNyxhr+nSTNyvjyoxbxol51xikq9VytFOPNSEMz8N02mUAuLVJNqrnqCBFRybjeqZdg4Y5AZlA==}
-
resolution: {integrity: sha512-WOjdkuNRSo09ikse47t6jIW4R55FEmZdOaoUTbSmpwfmNNHVht+iTgoqeHIVr7vvI9s+mtTmvzPoesoZa0cjVA==}
@@ -5279,8 +5282,8 @@ packages:
resolution: {integrity: sha512-5Oa9VE5rpDBdRgg71Vl09bqraw5RV0nyL6+rnOzIrVe+YCFCOyjj1KwsCz92gPNjGXGQG53w005la3dCTTrgdw==}
- resolution: {integrity: sha512-p6fgxlqOqwtYX77u6MZiWjvNlcJwjkGGDJ9HVt56UEBC88q+n5igZgWgPpgj/WHp9yajTf7WtAdZUO16qKa75A==}
+ resolution: {integrity: sha512-CPoMf/o8SJ0rq/yPtHXIabT0ldcKD59q0r9Tgs+AN47txC4JlskyG0S1ck1HNGbuhPuGyX4blWR6yib7TugYOw==}
engines: {node: '>=22.0.0'}
@@ -12947,8 +12950,6 @@ snapshots:
dependencies:
event-stream: 3.3.4
- '@apify/[email protected]': {}
-
'@apify/[email protected]': {}
'@apify/[email protected]': {}
@@ -17793,12 +17794,13 @@ snapshots:
dependencies:
'@apify/consts': 2.52.1
+ '@apify/datastructures': 2.0.4
'@apify/input_secrets': 1.2.30
'@apify/log': 2.5.35
- '@apify/timeout': 0.3.3
+ '@apify/timeout': 0.4.4
'@apify/utilities': 2.27.0
'@crawlee/core': link:packages/core
'@crawlee/types': link:packages/types
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index ae18d1f10..44166d6fa 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -29,9 +29,9 @@ overrides:
# The e2e tests run the `apify` SDK against the local 4.x workspace packages.
# The default `latest`/`next` tags are the 3.x SDK, which is not runtime
# compatible with `@crawlee/core@4` (e.g. `Configuration.INTEGER_VARS` was
- # removed), so pin the v4 SDK and force it to use the workspace `@crawlee/*`
+ # removed), so keep the SDK on the latest v4 beta and force it to use the workspace `@crawlee/*`
# packages (mirroring the per-actor `overrides.apify` used for PLATFORM tests).
- apify: "4.0.0-beta.19"
+ apify: "^4.0.0-beta.24"
"apify>@crawlee/core": "workspace:*"
"apify>@crawlee/types": "workspace:*"
"apify>@crawlee/utils": "workspace:*"
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 1ef9b1a25..77086ab38 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -103,6 +103,24 @@ describe('BasicCrawler', () => {
serviceLocator.setStorageBackend(new MemoryStorageBackend());
});
+ test('crawler-scoped service locator inherits ambient services that are not explicitly overridden', async () => {
+ const ambientBackend = serviceLocator.getStorageBackend();
+
+ let seenBackend: unknown;
+ // Passing only a `logger` creates a crawler-scoped service locator - it must inherit the
+ // ambient storage backend rather than falling back to the default file-system one.
+ const crawler = new BasicCrawler({
+ logger: serviceLocator.getLogger(),
+ requestHandler: async () => {
+ seenBackend = serviceLocator.getStorageBackend();
+ },
+ });
+
+ await crawler.run(['https://example.com']);
+
+ expect(seenBackend).toBe(ambientBackend);
+ });
+
test('does not leak sigint events', async () => {
let count = 0;
diff --git a/test/core/error_snapshotter.test.ts b/test/core/error_snapshotter.test.ts
new file mode 100644
index 000000000..9263901f3
--- /dev/null
+++ b/test/core/error_snapshotter.test.ts
@@ -0,0 +1,29 @@
+import type { KeyValueStore } from '@crawlee/core';
+import { ErrorSnapshotter } from '@crawlee/core';
+import { describe, expect, test, vitest } from 'vitest';
+
+describe('ErrorSnapshotter', () => {
+ test('saveHTMLSnapshot returns the record key it stored the snapshot under', async () => {
+ const snapshotter = new ErrorSnapshotter();
+ const setValue = vitest.fn(async () => {});
+ const keyValueStore = { setValue } as unknown as KeyValueStore;
+
+ const key = await snapshotter.saveHTMLSnapshot('<html></html>', keyValueStore, 'ERROR_SNAPSHOT_foo');
+
+ expect(setValue).toHaveBeenCalledWith('ERROR_SNAPSHOT_foo', '<html></html>', { contentType: 'text/html' });
+ // The returned value must be the actual record key (no appended extension),
+ // as it is fed to `keyValueStore.getPublicUrl()`.
+ expect(key).toBe('ERROR_SNAPSHOT_foo');
+ });
+
+ test('saveHTMLSnapshot returns undefined when storing fails', async () => {
+ const snapshotter = new ErrorSnapshotter();
+ const keyValueStore = {
+ setValue: async () => {
+ throw new Error('nope');
+ },
+ } as unknown as KeyValueStore;
+
+ await expect(snapshotter.saveHTMLSnapshot('<html></html>', keyValueStore, 'KEY')).resolves.toBeUndefined();
+ });
+});
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index 36977f30d..fd298f569 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -342,6 +342,13 @@ describe('RequestQueue remote', () => {
bar: true,
crawlDepth: 10,
});
+ // Re-wrapping userData that comes from another Request instance (where `__crawlee` is
+ // non-enumerable) must preserve the internal state instead of dropping it via the spread.
+ const r4 = new Request({ url, method, userData: r1.userData });
+ expect(r4.skipNavigation).toBe(true);
+ expect(r4.maxRetries).toBe(5);
+ expect(r4.crawlDepth).toBe(10);
+ expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, maxRetries: 5 });
const desc2 = Object.getOwnPropertyDescriptor(r2.userData, '__crawlee');
expect(desc2!.enumerable).toBe(false);
expect(r2.maxRetries).toBeUndefined();
diff --git a/test/e2e/adaptive-playwright-default/actor/Dockerfile b/test/e2e/adaptive-playwright-default/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/adaptive-playwright-default/actor/Dockerfile
+++ b/test/e2e/adaptive-playwright-default/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/adaptive-playwright-default/actor/main.js b/test/e2e/adaptive-playwright-default/actor/main.js
index 308718583..f58262268 100644
--- a/test/e2e/adaptive-playwright-default/actor/main.js
+++ b/test/e2e/adaptive-playwright-default/actor/main.js
@@ -1,6 +1,6 @@
import { Actor } from 'apify';
import { AdaptivePlaywrightCrawler } from '@crawlee/playwright';
-import { LogLevel } from '@apify/log';
+import log, { LogLevel } from '@apify/log';
await Actor.init({
storage:
@@ -45,7 +45,8 @@ const crawler = new AdaptivePlaywrightCrawler({
},
});
-crawler.log.setLevel(LogLevel.DEBUG);
+// CrawleeLogger has no setLevel; raise the level on the global @apify/log instead.
+log.setLevel(LogLevel.DEBUG);
await crawler.run(['https://crawlee.dev/js/docs/next/examples/accept-user-input']);
diff --git a/test/e2e/adaptive-playwright-default/actor/package.json b/test/e2e/adaptive-playwright-default/actor/package.json
index 6b84deaed..2fcfe425c 100644
--- a/test/e2e/adaptive-playwright-default/actor/package.json
+++ b/test/e2e/adaptive-playwright-default/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Adaptive Playwright Test - Default",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile
index 6433c507b..3fd414cf0 100644
--- a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile
+++ b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/adaptive-playwright-robots-file/actor/package.json b/test/e2e/adaptive-playwright-robots-file/actor/package.json
index 65775d421..baf52ddc2 100644
--- a/test/e2e/adaptive-playwright-robots-file/actor/package.json
+++ b/test/e2e/adaptive-playwright-robots-file/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Adaptive Playwright Test - Robots file",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfile
index 9dee3c33a..eb099b2ac 100644
--- a/test/e2e/automatic-persist-value/actor/Dockerfile
+++ b/test/e2e/automatic-persist-value/actor/Dockerfile
@@ -3,12 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
-RUN rm -r node_modules
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.json
index f4abd9b47..cab800bfc 100644
--- a/test/e2e/automatic-persist-value/actor/package.json
+++ b/test/e2e/automatic-persist-value/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Key-Value Store - Automatic Persist Value Test",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
@@ -14,9 +13,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js b/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js
index 9f25e8edf..e7362c05c 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/main.js
@@ -1,10 +1,12 @@
import { Actor } from 'apify';
-import { BasicCrawler, log as defaultLog, LogLevel } from '@crawlee/basic';
+import { ApifyLogAdapter, BasicCrawler, log as defaultLog, LogLevel } from '@crawlee/basic';
-const crawlerLogger = defaultLog.child({
- prefix: 'AutoscalingTest',
- level: LogLevel.INFO,
-});
+const crawlerLogger = new ApifyLogAdapter(
+ defaultLog.child({
+ prefix: 'AutoscalingTest',
+ level: LogLevel.INFO,
+ }),
+);
const mainOptions = {
exit: Actor.isAtHome(),
@@ -18,7 +20,7 @@ let crawlCalledAt = Date.now();
await Actor.main(async () => {
const crawler = new BasicCrawler({
- log: crawlerLogger,
+ logger: crawlerLogger,
maxRequestsPerMinute: 1,
requestHandler({ log }) {
log.info(`Crawler requestHandler called after ${Date.now() - crawlCalledAt}ms`);
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
index 9a703cf0d..7bab52695 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Autoscaling Pool Test - Max Tasks per Minute",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
@@ -14,9 +13,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/camoufox-cloudflare/actor/Dockerfile b/test/e2e/camoufox-cloudflare/actor/Dockerfile
index 2c6befacb..810df0f5d 100644
--- a/test/e2e/camoufox-cloudflare/actor/Dockerfile
+++ b/test/e2e/camoufox-cloudflare/actor/Dockerfile
@@ -10,11 +10,12 @@ COPY --chown=myuser package*.json ./
# `--ignore-scripts` skips the `camoufox-js fetch` postinstall; the browser is already
# baked into the base image.
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit --ignore-scripts \
+ && npm install --only=prod --no-audit --ignore-scripts \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/camoufox-cloudflare/actor/main.js b/test/e2e/camoufox-cloudflare/actor/main.js
index 3667ebd6a..ac6c77954 100644
--- a/test/e2e/camoufox-cloudflare/actor/main.js
+++ b/test/e2e/camoufox-cloudflare/actor/main.js
@@ -1,4 +1,4 @@
-import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';
+import { Dataset, handleCloudflareChallengeHook, PlaywrightCrawler } from '@crawlee/playwright';
import { Actor } from 'apify';
import { launchOptions } from 'camoufox-js';
import { firefox } from 'playwright';
@@ -13,7 +13,9 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
- proxyConfiguration: await Actor.createProxyConfiguration(),
+ // The target hard-blocks datacenter IP ranges outright (block page, not a solvable
+ // challenge), so this test needs residential exit nodes.
+ proxyConfiguration: await Actor.createProxyConfiguration({ groups: ['RESIDENTIAL'] }),
// Camoufox ships its own anti-detection; Crawlee's fingerprint injection conflicts with it
// and keeps Cloudflare from ever clearing the challenge.
browserPoolOptions: { useFingerprints: false },
@@ -38,11 +40,8 @@ await Actor.main(async () => {
});
},
],
- postNavigationHooks: [
- async ({ handleCloudflareChallenge }) => {
- await handleCloudflareChallenge();
- },
- ],
+ // verbose keeps the challenge detection visible in the nightly run logs
+ postNavigationHooks: [handleCloudflareChallengeHook({ verbose: true })],
launchContext: {
launcher: firefox,
launchOptions: await launchOptions({
diff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.json
index 98886a293..be2e2bb8a 100644
--- a/test/e2e/camoufox-cloudflare/actor/package.json
+++ b/test/e2e/camoufox-cloudflare/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Camoufox - Solving Cloudflare Challenge",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -12,16 +11,13 @@
"@crawlee/playwright": "file:./packages/playwright-crawler",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils",
- "camoufox-js": "^0.11.0",
+ "camoufox-js": "^0.12.0",
"playwright": "1.60.0"
},
"overrides": {
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/camoufox-cloudflare/test.mjs b/test/e2e/camoufox-cloudflare/test.mjs
index a063f66c5..a44664b71 100644
--- a/test/e2e/camoufox-cloudflare/test.mjs
+++ b/test/e2e/camoufox-cloudflare/test.mjs
@@ -2,10 +2,12 @@ import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tool
// TODO: re-enable on LOCAL/MEMORY once Camoufox supports the repo's Playwright version.
// Those storage types import the actor in-process against the repo-root node_modules (currently
-// Playwright 1.61), but camoufox-js 0.11 ships Firefox 150 and only launches under Playwright 1.60.
-// Only the PLATFORM build isolates the actor's pinned 1.60, so the test runs there exclusively.
+// Playwright 1.61), but camoufox-js 0.12 declares a playwright-core <1.61 peer range.
+// Only the PLATFORM build isolates the actor's pinned playwright version, so the test runs there exclusively.
if (process.env.STORAGE_IMPLEMENTATION !== 'PLATFORM') {
- await skipTest('Camoufox needs Playwright 1.60; only the PLATFORM build can isolate it from the repo-root 1.61');
+ await skipTest(
+ 'Camoufox needs its paired Playwright version; only the PLATFORM build can isolate it from the repo-root one',
+ );
}
const testActorDirname = getActorTestDir(import.meta.url);
diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile
index eb6de5b07..792fbb7a8 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile
+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile
@@ -4,8 +4,9 @@ COPY /packages ./packages
COPY /package*.json ./
COPY /tsconfig.json ./
COPY /main.ts ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update \
&& npm run build
@@ -13,6 +14,9 @@ FROM lwthiker/curl-impersonate
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=builder /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm
+# Wipe any preinstalled node_modules so the builder tree (with file: symlinks)
+# can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -20,7 +24,7 @@ COPY --from=builder /main.js ./
COPY /.actor ./.actor
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
index f51d11483..fe36db3df 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
@@ -1,16 +1,6 @@
-import { Readable } from 'node:stream';
-
-import type { Dictionary } from '@crawlee/cheerio';
import { CheerioCrawler } from '@crawlee/cheerio';
-import type {
- BaseHttpClient,
- BaseHttpResponseData,
- HttpRequest,
- HttpResponse,
- RedirectHandler,
- ResponseTypes,
- StreamingHttpResponse,
-} from '@crawlee/core';
+import { BaseHttpClient, type CustomFetchOptions, ResponseWithUrl } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
import { Actor } from 'apify';
import { CurlImpersonate } from 'apify-node-curl-impersonate';
@@ -25,157 +15,73 @@ interface CurlImpersonateHttpClientOptions {
impersonate?: ConstructorParameters<typeof CurlImpersonate>[1]['impersonate'];
}
-type CurlResponse = Awaited<ReturnType<CurlImpersonate['makeRequest']>>;
-
-class CurlImpersonateHttpClient implements BaseHttpClient {
- constructor(private options: CurlImpersonateHttpClientOptions = {}) {}
-
- protected async curlOptionsForRequest(request: HttpRequest<any>) {
- const result = {
- method: request.method ?? 'get',
- headers: request.headers ?? {},
- flags: ['--compressed'],
- impersonate: this.options.impersonate,
- debugLogger: () => {},
- };
-
- if (request.proxyUrl !== undefined) {
- result.flags.push('--proxy', request.proxyUrl);
- }
-
- if (request.cookieJar) {
- result.headers.cookie = (await (request.cookieJar as any)?.getCookieString?.(request.url.toString())) ?? '';
- }
-
- return result;
+/**
+ * A v4 `BaseHttpClient` implementation backed by `apify-node-curl-impersonate`.
+ * The base class provides redirect following, proxy and cookie handling - this
+ * only performs the raw request and wraps the result in a fetch `Response`.
+ */
+class CurlImpersonateHttpClient extends BaseHttpClient {
+ constructor(private options: CurlImpersonateHttpClientOptions = {}) {
+ super();
}
- protected shouldRedirect(request: HttpRequest, response: CurlResponse, redirectUrls: URL[]): boolean {
- if (request.maxRedirects !== undefined && request.maxRedirects <= redirectUrls.length) {
- return false;
- }
+ protected async fetch(request: Request, init?: RequestInit & CustomFetchOptions): Promise<Response> {
+ const headers: Record<string, string> = {};
+ request.headers.forEach((value, key) => {
+ headers[key] = value;
+ });
- if (request.followRedirect instanceof Function) {
- return request.followRedirect(response);
+ const flags = ['--compressed'];
+ if (init?.proxyUrl) {
+ flags.push('--proxy', init.proxyUrl);
}
- return request.followRedirect === undefined || request.followRedirect;
- }
-
- protected async performRequest(request: HttpRequest<any>, onRedirect?: RedirectHandler) {
- let response: CurlResponse | undefined;
- const redirectUrls: URL[] = [];
-
- const updatedRequest: Required<Pick<HttpRequest, 'url' | 'headers'>> = {
- url: request.url.toString(),
- headers: {},
- };
-
- while (true) {
- const impersonate = new CurlImpersonate(updatedRequest.url.toString(), {
- ...(await this.curlOptionsForRequest(request)),
- followRedirects: false,
- });
-
- response = await impersonate.makeRequest();
+ const curl = new CurlImpersonate(request.url, {
+ method: request.method,
+ headers,
+ flags,
+ impersonate: this.options.impersonate,
+ followRedirects: false,
+ debugLogger: () => {},
+ });
- if (response.statusCode >= 300 && response.statusCode < 400) {
- if (!this.shouldRedirect(request, response, redirectUrls)) {
- break;
- }
+ const response = await curl.makeRequest();
- updatedRequest.url = response.responseHeaders.location;
- updatedRequest.headers = response.responseHeaders;
- redirectUrls.push(new URL(updatedRequest.url));
- onRedirect?.(this.transformResponse(response, redirectUrls), updatedRequest);
- } else if (request.throwHttpErrors !== false && response.statusCode >= 400) {
- throw Object.assign(new Error(`Error status code encountered - ${response.statusCode}`), {
- request,
- response,
- });
- } else {
- break;
+ // The parsed curl headers include the status line as a key (e.g. "HTTP/2 200 "),
+ // which the fetch Headers constructor rejects - keep only valid header names.
+ const responseHeaders: Record<string, string> = {};
+ for (const [name, value] of Object.entries(response.responseHeaders ?? {})) {
+ if (/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name)) {
+ responseHeaders[name] = String(value);
}
}
- return { ...response, redirectUrls };
- }
-
- protected transformResponse(response: CurlResponse, redirectUrls: URL[]): BaseHttpResponseData {
- return {
- redirectUrls,
- url: response.url,
- ip: response.ipAddress,
- statusCode: response.statusCode,
- headers: response.responseHeaders,
- trailers: {}, // TODO apify-node-curl-impersonate doesn't seem to expose this
- complete: true,
- };
- }
-
- async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(
- request: HttpRequest<TResponseType>,
- ): Promise<HttpResponse<TResponseType>> {
- const response = await this.performRequest(request);
-
- const body = ((): ResponseTypes[TResponseType] => {
- const buffer = Buffer.from(response.response, request.encoding);
-
- switch (request.responseType) {
- case 'buffer':
- return buffer as ResponseTypes[TResponseType];
- case 'json':
- return JSON.parse(buffer.toString());
- default:
- return buffer.toString() as ResponseTypes[TResponseType];
- }
- })();
-
- return {
- request,
- body,
- ...this.transformResponse(response, response.redirectUrls),
- };
- }
-
- async stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {
- const response = await this.performRequest(request, onRedirect);
-
- const stream = new Readable();
- stream.push(response.response);
- stream.push(null);
-
- return {
- request,
- url: response.url,
- ip: response.ipAddress,
- statusCode: response.statusCode,
- stream,
- complete: true,
- downloadProgress: { percent: 100, transferred: response.response.length },
- uploadProgress: { percent: 100, transferred: 0 },
- redirectUrls: response.redirectUrls,
- headers: response.responseHeaders,
- trailers: {},
- };
+ return new ResponseWithUrl(response.response, {
+ status: response.statusCode,
+ headers: responseHeaders,
+ url: response.url ?? request.url,
+ });
}
}
const crawler = new CheerioCrawler({
async requestHandler(context) {
- const { body: text } = await context.sendRequest({
- url: 'https://api.apify.com/v2/browser-info',
- });
-
- const { body: json } = await context.sendRequest<Dictionary>({
- url: 'https://api.apify.com/v2/browser-info',
- responseType: 'json',
- });
+ const text = await (
+ await context.sendRequest({
+ url: 'https://api.apify.com/v2/browser-info',
+ })
+ ).text();
+
+ const json = (await (
+ await context.sendRequest({
+ url: 'https://api.apify.com/v2/browser-info',
+ })
+ ).json()) as Dictionary;
await context.pushData({
body: context.body,
title: context.$('title').text(),
- userAgent: json.headers['user-agent'],
+ userAgent: (json.headers as Dictionary)['user-agent'],
clientIpTextResponse: text,
clientIpJsonResponse: json,
});
diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json
index 09a29a3e2..05255fdba 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json
+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json
@@ -3,16 +3,15 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - curl-impersonate HTTP client",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/cheerio": "file:./packages/cheerio-crawler",
"@crawlee/core": "file:./packages/core",
"@crawlee/http": "file:./packages/http-crawler",
+ "@crawlee/http-client": "file:./packages/http-client",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils",
- "apify": "next",
"apify-node-curl-impersonate": "^1.0.29"
},
"overrides": {
@@ -20,9 +19,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json
index 61f6d2984..5c269847a 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json
+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json
@@ -1,9 +1,10 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
- "module": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"],
+ "lib": ["ES2022", "DOM"],
"skipLibCheck": true,
"incremental": false
},
diff --git a/test/e2e/cheerio-curl-impersonate-ts/test.mjs b/test/e2e/cheerio-curl-impersonate-ts/test.mjs
index 48aea4fe7..ecc0b9d90 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/test.mjs
+++ b/test/e2e/cheerio-curl-impersonate-ts/test.mjs
@@ -19,5 +19,3 @@ await expect(
);
await expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains client IP');
await expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains client IP');
-await expect(result.uuidJsonResponse.uuid !== undefined, 'JSON response contains UUID');
-await expect(JSON.parse(result.uuidTextResponse).uuid !== undefined, 'Text response contains UUID');
diff --git a/test/e2e/cheerio-default-ts/actor/Dockerfile b/test/e2e/cheerio-default-ts/actor/Dockerfile
index 8f856bbb0..ba62560de 100644
--- a/test/e2e/cheerio-default-ts/actor/Dockerfile
+++ b/test/e2e/cheerio-default-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.json
index 9fbed6f67..18ff10310 100644
--- a/test/e2e/cheerio-default-ts/actor/package.json
+++ b/test/e2e/cheerio-default-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - TypeScript",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/cheerio-default-ts/actor/tsconfig.json b/test/e2e/cheerio-default-ts/actor/tsconfig.json
index 370e50c0a..dc194464d 100644
--- a/test/e2e/cheerio-default-ts/actor/tsconfig.json
+++ b/test/e2e/cheerio-default-ts/actor/tsconfig.json
@@ -4,7 +4,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"]
+ "lib": ["ES2022", "DOM"],
+ "skipLibCheck": true
},
"include": ["./**/*.ts"]
}
diff --git a/test/e2e/cheerio-default/actor/Dockerfile b/test/e2e/cheerio-default/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-default/actor/Dockerfile
+++ b/test/e2e/cheerio-default/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-default/actor/package.json b/test/e2e/cheerio-default/actor/package.json
index 2c64ec07d..e27b6bd77 100644
--- a/test/e2e/cheerio-default/actor/package.json
+++ b/test/e2e/cheerio-default/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Default",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile
+++ b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/package.json b/test/e2e/cheerio-enqueue-links-base/actor/package.json
index 0aa9f8c70..ae50efae7 100644
--- a/test/e2e/cheerio-enqueue-links-base/actor/package.json
+++ b/test/e2e/cheerio-enqueue-links-base/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Enqueue Links with <base href>",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-enqueue-links/actor/Dockerfile b/test/e2e/cheerio-enqueue-links/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-enqueue-links/actor/Dockerfile
+++ b/test/e2e/cheerio-enqueue-links/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-enqueue-links/actor/package.json b/test/e2e/cheerio-enqueue-links/actor/package.json
index ddea7a6d7..bdcd83b49 100644
--- a/test/e2e/cheerio-enqueue-links/actor/package.json
+++ b/test/e2e/cheerio-enqueue-links/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Enqueue Links",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-error-snapshot/actor/Dockerfile b/test/e2e/cheerio-error-snapshot/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-error-snapshot/actor/Dockerfile
+++ b/test/e2e/cheerio-error-snapshot/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-error-snapshot/actor/package.json b/test/e2e/cheerio-error-snapshot/actor/package.json
index 1c989a12b..42b4a09fb 100644
--- a/test/e2e/cheerio-error-snapshot/actor/package.json
+++ b/test/e2e/cheerio-error-snapshot/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Should save errors snapshots",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
index c96f6347d..2160f0afc 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
@@ -22,6 +22,7 @@ await Actor.main(async () => {
log.info('Bad ssl page opened!');
await enqueueLinks({
include: ['https://*.badssl.com/'],
+ strategy: 'same-domain',
label: 'DETAIL',
selector: '.group a.bad',
});
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
index cf6d1dae4..ea7198ee4 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Ignore SSL Errors",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-impit-ts/actor/Dockerfile b/test/e2e/cheerio-impit-ts/actor/Dockerfile
index 9e846faee..7cd3bcbf9 100644
--- a/test/e2e/cheerio-impit-ts/actor/Dockerfile
+++ b/test/e2e/cheerio-impit-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,6 +15,7 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-audit \
&& npm update --no-audit \
diff --git a/test/e2e/cheerio-impit-ts/actor/main.ts b/test/e2e/cheerio-impit-ts/actor/main.ts
index a4cccb647..810fd6ecc 100644
--- a/test/e2e/cheerio-impit-ts/actor/main.ts
+++ b/test/e2e/cheerio-impit-ts/actor/main.ts
@@ -1,5 +1,5 @@
-import type { Dictionary } from '@crawlee/cheerio';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Session, SessionPool } from '@crawlee/cheerio';
+import type { Dictionary } from '@crawlee/types';
import { Browser, ImpitHttpClient } from '@crawlee/impit-client';
import { Actor } from 'apify';
@@ -12,24 +12,36 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {
const crawler = new CheerioCrawler({
async requestHandler(context) {
- const { body: text } = await context.sendRequest({
- url: 'https://api.apify.com/v2/browser-info',
- });
+ const text = await (
+ await context.sendRequest({
+ url: 'https://api.apify.com/v2/browser-info',
+ })
+ ).text();
- const { body: json } = await context.sendRequest<Dictionary>({
- url: 'https://api.apify.com/v2/browser-info',
- responseType: 'json',
- });
+ const json = (await (
+ await context.sendRequest({
+ url: 'https://api.apify.com/v2/browser-info',
+ })
+ ).json()) as Dictionary;
await context.pushData({
body: context.body,
title: context.$('title').text(),
- userAgent: json.headers['user-agent'],
+ userAgent: (json.headers as Dictionary)['user-agent'],
clientIpTextResponse: text,
clientIpJsonResponse: json,
});
},
httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),
+ // The random default session fingerprint would override the client's browser
+ // impersonation, so pin the sessions to Firefox as well.
+ sessionPool: new SessionPool({
+ createSessionFunction: async (opts) =>
+ new Session({
+ ...opts?.sessionOptions,
+ fingerprint: { browser: 'firefox', platform: 'linux', device: 'desktop' },
+ }),
+ }),
});
await crawler.run(['https://crawlee.dev']);
diff --git a/test/e2e/cheerio-impit-ts/actor/package.json b/test/e2e/cheerio-impit-ts/actor/package.json
index f862e3e2e..e9058fe1f 100644
--- a/test/e2e/cheerio-impit-ts/actor/package.json
+++ b/test/e2e/cheerio-impit-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Impit HTTP client",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -19,9 +18,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/cheerio-impit-ts/actor/tsconfig.json b/test/e2e/cheerio-impit-ts/actor/tsconfig.json
index 3d4d42c61..5c269847a 100644
--- a/test/e2e/cheerio-impit-ts/actor/tsconfig.json
+++ b/test/e2e/cheerio-impit-ts/actor/tsconfig.json
@@ -4,7 +4,7 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"],
+ "lib": ["ES2022", "DOM"],
"skipLibCheck": true,
"incremental": false
},
diff --git a/test/e2e/cheerio-initial-cookies/actor/Dockerfile b/test/e2e/cheerio-initial-cookies/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-initial-cookies/actor/Dockerfile
+++ b/test/e2e/cheerio-initial-cookies/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.js
index a1dcf6874..d309d8484 100644
--- a/test/e2e/cheerio-initial-cookies/actor/main.js
+++ b/test/e2e/cheerio-initial-cookies/actor/main.js
@@ -14,14 +14,6 @@ const expectedCookies = [
name: 'hook_request',
value: 'true',
},
- {
- name: 'got_options_upper_case',
- value: 'true',
- },
- {
- name: 'got_options_lower_case',
- value: 'true',
- },
];
const mainOptions = {
@@ -36,21 +28,9 @@ await Actor.main(async () => {
const crawler = new CheerioCrawler({
additionalMimeTypes: ['application/json'],
preNavigationHooks: [
- ({ session, request }, gotOptions) => {
- session.setCookies(
- [
- {
- name: 'session',
- value: 'true',
- },
- ],
- request.url,
- );
+ async ({ session, request }) => {
+ await session.setCookie('session=true', request.url);
request.headers.cookie = 'hook_request=true';
-
- gotOptions.headers ??= {};
- gotOptions.headers.Cookie = 'got_options_upper_case=true';
- gotOptions.headers.cookie = 'got_options_lower_case=true';
},
],
async requestHandler({ json }) {
diff --git a/test/e2e/cheerio-initial-cookies/actor/package.json b/test/e2e/cheerio-initial-cookies/actor/package.json
index 747f78126..557deac9a 100644
--- a/test/e2e/cheerio-initial-cookies/actor/package.json
+++ b/test/e2e/cheerio-initial-cookies/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Initial Cookies",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjs
index 136a7d032..760a89858 100644
--- a/test/e2e/cheerio-initial-cookies/test.mjs
+++ b/test/e2e/cheerio-initial-cookies/test.mjs
@@ -6,7 +6,7 @@ await initialize(testActorDirname);
const { stats, datasetItems } = await runActor(testActorDirname);
await expect(stats.requestsFinished === 1, 'All requests finished');
-await expect(datasetItems[0].numberOfMatchingCookies === 5, 'Number of page cookies');
+await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');
await expect(
datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,
`Page cookies match the initial defined cookies. Number of non-matching cookies is ` +
diff --git a/test/e2e/cheerio-max-requests/actor/Dockerfile b/test/e2e/cheerio-max-requests/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-max-requests/actor/Dockerfile
+++ b/test/e2e/cheerio-max-requests/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-max-requests/actor/package.json b/test/e2e/cheerio-max-requests/actor/package.json
index 292a77daa..482e47c4e 100644
--- a/test/e2e/cheerio-max-requests/actor/package.json
+++ b/test/e2e/cheerio-max-requests/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Max Requests Per Crawl",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-page-info/actor/Dockerfile b/test/e2e/cheerio-page-info/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-page-info/actor/Dockerfile
+++ b/test/e2e/cheerio-page-info/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-page-info/actor/package.json b/test/e2e/cheerio-page-info/actor/package.json
index 2dfac264f..919fa6b2c 100644
--- a/test/e2e/cheerio-page-info/actor/package.json
+++ b/test/e2e/cheerio-page-info/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Page Info",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile b/test/e2e/cheerio-request-queue-v2/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile
+++ b/test/e2e/cheerio-request-queue-v2/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-request-queue-v2/actor/main.js b/test/e2e/cheerio-request-queue-v2/actor/main.js
index cc2db1d09..68e6ba564 100644
--- a/test/e2e/cheerio-request-queue-v2/actor/main.js
+++ b/test/e2e/cheerio-request-queue-v2/actor/main.js
@@ -1,5 +1,5 @@
-import { Actor, LogLevel, log as Logger } from 'apify';
-import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
+import { Actor, log as Logger } from 'apify';
+import { ApifyLogAdapter, CheerioCrawler, Dataset } from '@crawlee/cheerio';
const mainOptions = {
exit: Actor.isAtHome(),
@@ -22,10 +22,11 @@ await Actor.main(async () => {
await Dataset.pushData({ url, pageTitle });
},
- log: Logger.child({
- prefix: 'CheerioCrawler',
- // level: LogLevel.DEBUG,
- }),
+ logger: new ApifyLogAdapter(
+ Logger.child({
+ prefix: 'CheerioCrawler',
+ }),
+ ),
});
try {
diff --git a/test/e2e/cheerio-request-queue-v2/actor/package.json b/test/e2e/cheerio-request-queue-v2/actor/package.json
index 8fc0412f0..8d3395656 100644
--- a/test/e2e/cheerio-request-queue-v2/actor/package.json
+++ b/test/e2e/cheerio-request-queue-v2/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Request Queue V2",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-robots-file/actor/Dockerfile b/test/e2e/cheerio-robots-file/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-robots-file/actor/Dockerfile
+++ b/test/e2e/cheerio-robots-file/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-robots-file/actor/package.json b/test/e2e/cheerio-robots-file/actor/package.json
index 44d38a2cc..c79747c55 100644
--- a/test/e2e/cheerio-robots-file/actor/package.json
+++ b/test/e2e/cheerio-robots-file/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Test - Robots file",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile
index 8f856bbb0..ba62560de 100644
--- a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile
+++ b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-stop-resume-ts/actor/package.json b/test/e2e/cheerio-stop-resume-ts/actor/package.json
index 89fb8ae53..a0e00b3d5 100644
--- a/test/e2e/cheerio-stop-resume-ts/actor/package.json
+++ b/test/e2e/cheerio-stop-resume-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Crawler Stop-Resume Test - TypeScript",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json b/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json
index 370e50c0a..dc194464d 100644
--- a/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json
+++ b/test/e2e/cheerio-stop-resume-ts/actor/tsconfig.json
@@ -4,7 +4,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"]
+ "lib": ["ES2022", "DOM"],
+ "skipLibCheck": true
},
"include": ["./**/*.ts"]
}
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
index b926ccbcb..44f55c5a3 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
@@ -21,6 +21,7 @@ await Actor.main(async () => {
log.info('Bad ssl page opened!');
await enqueueLinks({
include: ['https://*.badssl.com/'],
+ strategy: 'same-domain',
label: 'DETAIL',
selector: '.group a.bad',
});
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
index e1b75655f..e5ecdf60c 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Should throw on SSL Errors",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/input-json5/actor/Dockerfile
+++ b/test/e2e/input-json5/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.json
index 785618c16..0d6e03127 100644
--- a/test/e2e/input-json5/actor/package.json
+++ b/test/e2e/input-json5/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "JSON5 input test",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
},
@@ -12,9 +11,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfile
index 8f856bbb0..ba62560de 100644
--- a/test/e2e/jsdom-default-ts/actor/Dockerfile
+++ b/test/e2e/jsdom-default-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.json
index 7c7142b7b..895e4b9be 100644
--- a/test/e2e/jsdom-default-ts/actor/package.json
+++ b/test/e2e/jsdom-default-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "JSDOM Crawler Test - TypeScript",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/jsdom-default-ts/actor/tsconfig.json b/test/e2e/jsdom-default-ts/actor/tsconfig.json
index 370e50c0a..dc194464d 100644
--- a/test/e2e/jsdom-default-ts/actor/tsconfig.json
+++ b/test/e2e/jsdom-default-ts/actor/tsconfig.json
@@ -4,7 +4,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"]
+ "lib": ["ES2022", "DOM"],
+ "skipLibCheck": true
},
"include": ["./**/*.ts"]
}
diff --git a/test/e2e/jsdom-react-ts/actor/Dockerfile b/test/e2e/jsdom-react-ts/actor/Dockerfile
index 8f856bbb0..ba62560de 100644
--- a/test/e2e/jsdom-react-ts/actor/Dockerfile
+++ b/test/e2e/jsdom-react-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/jsdom-react-ts/actor/package.json b/test/e2e/jsdom-react-ts/actor/package.json
index f09a8f586..4d5286746 100644
--- a/test/e2e/jsdom-react-ts/actor/package.json
+++ b/test/e2e/jsdom-react-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "JSDOM Crawler Test - React - TypeScript",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/jsdom-react-ts/actor/tsconfig.json b/test/e2e/jsdom-react-ts/actor/tsconfig.json
index 370e50c0a..dc194464d 100644
--- a/test/e2e/jsdom-react-ts/actor/tsconfig.json
+++ b/test/e2e/jsdom-react-ts/actor/tsconfig.json
@@ -4,7 +4,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"]
+ "lib": ["ES2022", "DOM"],
+ "skipLibCheck": true
},
"include": ["./**/*.ts"]
}
diff --git a/test/e2e/linkedom-default-ts/actor/Dockerfile b/test/e2e/linkedom-default-ts/actor/Dockerfile
index 8f856bbb0..ba62560de 100644
--- a/test/e2e/linkedom-default-ts/actor/Dockerfile
+++ b/test/e2e/linkedom-default-ts/actor/Dockerfile
@@ -3,6 +3,7 @@ FROM apify/actor-node:24-beta AS builder
# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
+RUN rm -rf node_modules package-lock.json
RUN npm install --include=dev \
&& npm run build
@@ -14,11 +15,12 @@ COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/linkedom-default-ts/actor/package.json b/test/e2e/linkedom-default-ts/actor/package.json
index eafbc38ac..5311cea3b 100644
--- a/test/e2e/linkedom-default-ts/actor/package.json
+++ b/test/e2e/linkedom-default-ts/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "LinkeDOM Crawler Test - TypeScript",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -18,9 +17,6 @@
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"devDependencies": {
diff --git a/test/e2e/linkedom-default-ts/actor/tsconfig.json b/test/e2e/linkedom-default-ts/actor/tsconfig.json
index 370e50c0a..dc194464d 100644
--- a/test/e2e/linkedom-default-ts/actor/tsconfig.json
+++ b/test/e2e/linkedom-default-ts/actor/tsconfig.json
@@ -4,7 +4,8 @@
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
- "lib": ["DOM"]
+ "lib": ["ES2022", "DOM"],
+ "skipLibCheck": true
},
"include": ["./**/*.ts"]
}
diff --git a/test/e2e/migration/actor/Dockerfile b/test/e2e/migration/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/migration/actor/Dockerfile
+++ b/test/e2e/migration/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/migration/actor/package.json b/test/e2e/migration/actor/package.json
index 007d21799..98c7a5c3f 100644
--- a/test/e2e/migration/actor/package.json
+++ b/test/e2e/migration/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Migration Test",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/playwright-default/actor/Dockerfile
+++ b/test/e2e/playwright-default/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.js
index 369ffa6a4..db1272731 100644
--- a/test/e2e/playwright-default/actor/main.js
+++ b/test/e2e/playwright-default/actor/main.js
@@ -12,8 +12,8 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
preNavigationHooks: [
- (_ctx, goToOptions) => {
- goToOptions.waitUntil = 'networkidle';
+ ({ gotoOptions }) => {
+ gotoOptions.waitUntil = 'networkidle';
},
],
async requestHandler({ page, enqueueLinks, request }) {
diff --git a/test/e2e/playwright-default/actor/package.json b/test/e2e/playwright-default/actor/package.json
index 431d8c34c..89d24593b 100644
--- a/test/e2e/playwright-default/actor/package.json
+++ b/test/e2e/playwright-default/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Default",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-enqueue-links-base/actor/package.json b/test/e2e/playwright-enqueue-links-base/actor/package.json
index b7f979514..c7effd3a8 100644
--- a/test/e2e/playwright-enqueue-links-base/actor/package.json
+++ b/test/e2e/playwright-enqueue-links-base/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Enqueue Links with <base href>",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -19,9 +18,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-enqueue-links/actor/package.json b/test/e2e/playwright-enqueue-links/actor/package.json
index 6894cb4f8..84675345d 100644
--- a/test/e2e/playwright-enqueue-links/actor/package.json
+++ b/test/e2e/playwright-enqueue-links/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Enqueue Links",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -20,9 +19,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-initial-cookies/actor/main.js b/test/e2e/playwright-initial-cookies/actor/main.js
index 0c1f2b05b..2510e6470 100644
--- a/test/e2e/playwright-initial-cookies/actor/main.js
+++ b/test/e2e/playwright-initial-cookies/actor/main.js
@@ -27,19 +27,11 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
preNavigationHooks: [
- ({ session, request }, goToOptions) => {
- session.setCookies(
- [
- {
- name: 'session',
- value: 'true',
- },
- ],
- request.url,
- );
+ async ({ session, request, gotoOptions }) => {
+ await session.setCookie('session=true', request.url);
request.headers.cookie = 'hook_request=true';
- goToOptions.waitUntil = 'networkidle';
+ gotoOptions.waitUntil = 'networkidle';
},
],
async requestHandler({ page }) {
diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.json
index 5c6fbab28..67fa86db6 100644
--- a/test/e2e/playwright-initial-cookies/actor/package.json
+++ b/test/e2e/playwright-initial-cookies/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Initial Cookies",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-introduction-guide/actor/Dockerfile b/test/e2e/playwright-introduction-guide/actor/Dockerfile
index 9849e3409..7d570fa0c 100644
--- a/test/e2e/playwright-introduction-guide/actor/Dockerfile
+++ b/test/e2e/playwright-introduction-guide/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional \
+ && npm install --only=prod \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-introduction-guide/actor/package.json b/test/e2e/playwright-introduction-guide/actor/package.json
index f7730a7ef..f5d96aa2d 100644
--- a/test/e2e/playwright-introduction-guide/actor/package.json
+++ b/test/e2e/playwright-introduction-guide/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Crawlee Introduction Guide (playwright + chrome)",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/playwright-multi-run/actor/Dockerfile
+++ b/test/e2e/playwright-multi-run/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-multi-run/actor/package.json b/test/e2e/playwright-multi-run/actor/package.json
index 9394b7d0c..376373749 100644
--- a/test/e2e/playwright-multi-run/actor/package.json
+++ b/test/e2e/playwright-multi-run/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Multiple run calls to the same crawler",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/playwright-robots-file/actor/Dockerfile b/test/e2e/playwright-robots-file/actor/Dockerfile
index 6433c507b..3fd414cf0 100644
--- a/test/e2e/playwright-robots-file/actor/Dockerfile
+++ b/test/e2e/playwright-robots-file/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/playwright-robots-file/actor/package.json b/test/e2e/playwright-robots-file/actor/package.json
index d444b0ab3..1cb1576fa 100644
--- a/test/e2e/playwright-robots-file/actor/package.json
+++ b/test/e2e/playwright-robots-file/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Robots file",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile
index 2956f2058..6122e22a2 100644
--- a/test/e2e/proxy-rotation/actor/Dockerfile
+++ b/test/e2e/proxy-rotation/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -18,7 +21,7 @@ COPY /main.js ./
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/proxy-rotation/actor/main.js b/test/e2e/proxy-rotation/actor/main.js
index db66b0ad1..29e30202c 100644
--- a/test/e2e/proxy-rotation/actor/main.js
+++ b/test/e2e/proxy-rotation/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { Dataset, KeyValueStore, PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, KeyValueStore, PuppeteerCrawler, Session, SessionPool } from '@crawlee/puppeteer';
const mainOptions = {
exit: Actor.isAtHome(),
@@ -10,10 +10,19 @@ const mainOptions = {
};
await Actor.main(async () => {
+ const proxyConfiguration = await Actor.createProxyConfiguration();
+ const sessionPool = new SessionPool({
+ createSessionFunction: async (opts) =>
+ new Session({
+ ...opts?.sessionOptions,
+ maxUsageCount: 1,
+ proxyInfo: await proxyConfiguration.newProxyInfo(),
+ }),
+ });
+
const crawler = new PuppeteerCrawler({
- proxyConfiguration: await Actor.createProxyConfiguration(),
+ sessionPool,
maxConcurrency: 1,
- sessionPoolOptions: { sessionOptions: { maxUsageCount: 1 } },
async requestHandler({ response }) {
const { clientIp } = await response.json();
diff --git a/test/e2e/proxy-rotation/actor/package.json b/test/e2e/proxy-rotation/actor/package.json
index 88257553c..40bfb00ad 100644
--- a/test/e2e/proxy-rotation/actor/package.json
+++ b/test/e2e/proxy-rotation/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Proxy Test - Rotation",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile
index 2956f2058..6122e22a2 100644
--- a/test/e2e/puppeteer-default/actor/Dockerfile
+++ b/test/e2e/puppeteer-default/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -18,7 +21,7 @@ COPY /main.js ./
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.js
index 30c121812..83f6f588a 100644
--- a/test/e2e/puppeteer-default/actor/main.js
+++ b/test/e2e/puppeteer-default/actor/main.js
@@ -12,8 +12,8 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
preNavigationHooks: [
- (_ctx, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ ({ gotoOptions }) => {
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page, enqueueLinks, request, infiniteScroll }) {
diff --git a/test/e2e/puppeteer-default/actor/package.json b/test/e2e/puppeteer-default/actor/package.json
index 7818af154..5a252a6d9 100644
--- a/test/e2e/puppeteer-default/actor/package.json
+++ b/test/e2e/puppeteer-default/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Default",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-enqueue-links/actor/package.json b/test/e2e/puppeteer-enqueue-links/actor/package.json
index 5acb45535..1703d799b 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/package.json
+++ b/test/e2e/puppeteer-enqueue-links/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Enqueue Links",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -20,9 +19,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile b/test/e2e/puppeteer-error-snapshot/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile
+++ b/test/e2e/puppeteer-error-snapshot/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-error-snapshot/actor/package.json b/test/e2e/puppeteer-error-snapshot/actor/package.json
index b5bfac8f3..ccfddff11 100644
--- a/test/e2e/puppeteer-error-snapshot/actor/package.json
+++ b/test/e2e/puppeteer-error-snapshot/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Should save errors snapshots",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
index 205900a7d..4c6f3b484 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
@@ -16,8 +16,8 @@ await Actor.main(async () => {
maxConcurrency: 2,
launchContext: { launchOptions: { acceptInsecureCerts: true } },
preNavigationHooks: [
- (_ctx, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ ({ gotoOptions }) => {
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page, enqueueLinks, request, log }) {
@@ -30,6 +30,7 @@ await Actor.main(async () => {
log.info('Bad ssl page opened!');
await enqueueLinks({
include: ['https://*.badssl.com/'],
+ strategy: 'same-domain',
label: 'DETAIL',
selector: '.group a.bad',
});
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
index 21a8be8ee..a33a6bf32 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Ignore SSL Errors",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.js
index 89ddc2319..259a945c4 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/main.js
+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js
@@ -27,19 +27,11 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
preNavigationHooks: [
- ({ session, request }, goToOptions) => {
- session.setCookies(
- [
- {
- name: 'session',
- value: 'true',
- },
- ],
- request.url,
- );
+ async ({ session, request, gotoOptions }) => {
+ await session.setCookie('session=true', request.url);
request.headers.cookie = 'hook_request=true';
- goToOptions.waitUntil = ['networkidle2'];
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page }) {
diff --git a/test/e2e/puppeteer-initial-cookies/actor/package.json b/test/e2e/puppeteer-initial-cookies/actor/package.json
index 89258fd2a..5e296be06 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/package.json
+++ b/test/e2e/puppeteer-initial-cookies/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Initial Cookies",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-page-info/actor/Dockerfile
+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.js
index a7837ee55..dd7140440 100644
--- a/test/e2e/puppeteer-page-info/actor/main.js
+++ b/test/e2e/puppeteer-page-info/actor/main.js
@@ -12,8 +12,8 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
preNavigationHooks: [
- (_ctx, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ ({ gotoOptions }) => {
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page, enqueueLinks, request }) {
diff --git a/test/e2e/puppeteer-page-info/actor/package.json b/test/e2e/puppeteer-page-info/actor/package.json
index 1dad46ec1..684633a9e 100644
--- a/test/e2e/puppeteer-page-info/actor/package.json
+++ b/test/e2e/puppeteer-page-info/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Page Info",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
index ae5efa02e..59a2ca48f 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
@@ -76,8 +76,11 @@ await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
maxRequestsPerCrawl: 10,
preNavigationHooks: [
- (_crawlingContext, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ async ({ page, gotoOptions }) => {
+ await page.evaluateOnNewDocument(() => {
+ localStorage.setItem('themeExitPopup', 'true');
+ });
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page, request, log, enqueueLinks, injectJQuery }) {
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
index aa4a680f6..74a23773f 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Store Pagination with jQuery",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.js
index 8c02ce9ee..d7a76b8d0 100644
--- a/test/e2e/puppeteer-store-pagination/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination/actor/main.js
@@ -74,8 +74,11 @@ await Actor.init({
const crawler = new PuppeteerCrawler({
maxRequestsPerCrawl: 10,
preNavigationHooks: [
- (_crawlingContext, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ async ({ page, gotoOptions }) => {
+ await page.evaluateOnNewDocument(() => {
+ localStorage.setItem('themeExitPopup', 'true');
+ });
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
});
diff --git a/test/e2e/puppeteer-store-pagination/actor/package.json b/test/e2e/puppeteer-store-pagination/actor/package.json
index f0200f35a..3be0ebe52 100644
--- a/test/e2e/puppeteer-store-pagination/actor/package.json
+++ b/test/e2e/puppeteer-store-pagination/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Store Pagination",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
index 4d84738ff..695dedcae 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-puppeteer-chrome:24-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
index 33ed50ead..8e8382e51 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
@@ -13,8 +13,8 @@ await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
launchContext: { launchOptions: { acceptInsecureCerts: false } }, // This is the default
preNavigationHooks: [
- (_ctx, goToOptions) => {
- goToOptions.waitUntil = ['networkidle2'];
+ ({ gotoOptions }) => {
+ gotoOptions.waitUntil = ['networkidle2'];
},
],
async requestHandler({ page, enqueueLinks, request, log }) {
@@ -27,6 +27,7 @@ await Actor.main(async () => {
log.info('Bad ssl page opened!');
await enqueueLinks({
include: ['https://*.badssl.com/'],
+ strategy: 'same-domain',
label: 'DETAIL',
selector: '.group a.bad',
});
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
index 31e5a2552..00ebde431 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Should throw on SSL Errors",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/request-queue-with-concurrency/actor/Dockerfile b/test/e2e/request-queue-with-concurrency/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/request-queue-with-concurrency/actor/Dockerfile
+++ b/test/e2e/request-queue-with-concurrency/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/request-queue-with-concurrency/actor/package.json b/test/e2e/request-queue-with-concurrency/actor/package.json
index b41e9d076..79caeb1ba 100644
--- a/test/e2e/request-queue-with-concurrency/actor/package.json
+++ b/test/e2e/request-queue-with-concurrency/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Request Queue Test - Zero Concurrency",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
+++ b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/request-queue-zero-concurrency/actor/package.json b/test/e2e/request-queue-zero-concurrency/actor/package.json
index 0d47e0acc..98bc9651b 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/package.json
+++ b/test/e2e/request-queue-zero-concurrency/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Request Queue Test - Zero Concurrency",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/request-queue-zero-concurrency/test.mjs b/test/e2e/request-queue-zero-concurrency/test.mjs
index 4c3e1eee0..0ad1ce2ef 100644
--- a/test/e2e/request-queue-zero-concurrency/test.mjs
+++ b/test/e2e/request-queue-zero-concurrency/test.mjs
@@ -1,4 +1,8 @@
-import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs';
+import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs';
+
+// TODO(v4): redesign for the v4 RequestQueue. The test stages a stuck queue via the v3
+// client-side `requestQueue.inProgress` set, which no longer exists in the rewritten queue.
+await skipTest('The v4 RequestQueue has no client-side `inProgress` set to stage a stuck queue with');
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
diff --git a/test/e2e/request-skip-navigation/actor/Dockerfile b/test/e2e/request-skip-navigation/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/request-skip-navigation/actor/Dockerfile
+++ b/test/e2e/request-skip-navigation/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/request-skip-navigation/actor/package.json b/test/e2e/request-skip-navigation/actor/package.json
index 136736e8d..f08f6d449 100644
--- a/test/e2e/request-skip-navigation/actor/package.json
+++ b/test/e2e/request-skip-navigation/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Request Test - skipNavigation",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/http": "file:./packages/http-crawler",
@@ -17,9 +16,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs
index 169e76c89..dcd0cc173 100644
--- a/test/e2e/run.mjs
+++ b/test/e2e/run.mjs
@@ -158,12 +158,12 @@ if (isMainThread) {
for (let attempt = 0; attempt < 5; attempt++) {
try {
execSync(`pnpm exec camoufox-js fetch`, { stdio: 'inherit' });
+ break;
} catch (e) {
console.error('Failed to fetch Camoufox', e);
if (attempt === 4) throw e;
console.log(`Retrying to fetch Camoufox (attempt ${attempt + 2}/5)...`);
await new Promise((resolve) => setTimeout(resolve, 10e3));
- continue;
}
}
}
diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/session-rotation/actor/Dockerfile
+++ b/test/e2e/session-rotation/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.js
index c9b3ba5e0..5d9d582f9 100644
--- a/test/e2e/session-rotation/actor/main.js
+++ b/test/e2e/session-rotation/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PlaywrightCrawler } from '@crawlee/playwright';
+import { PlaywrightCrawler, Session, SessionPool } from '@crawlee/playwright';
const mainOptions = {
exit: Actor.isAtHome(),
@@ -12,14 +12,20 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
maxRequestRetries: 10,
- sessionPoolOptions: {
- sessionOptions: {
- maxErrorScore: 2,
- },
- },
- requestHandler: async ({ session }) => {
+ sessionPool: new SessionPool({
+ createSessionFunction: async (opts) =>
+ new Session({
+ ...opts?.sessionOptions,
+ maxErrorScore: 2,
+ }),
+ }),
+ requestHandler: async ({ session, registerDeferredCleanup }) => {
+ // The handler runs in a storage transaction, so a plain pushData would be rolled
+ // back when the handler throws. Deferred cleanups run outside of it.
const { id, usageCount, errorScore } = session;
- await Actor.pushData({ id, usageCount, errorScore });
+ registerDeferredCleanup(async () => {
+ await Actor.pushData({ id, usageCount, errorScore });
+ });
throw new Error('retry');
},
});
diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.json
index 9bcc4a1d7..81e5a4d7c 100644
--- a/test/e2e/session-rotation/actor/package.json
+++ b/test/e2e/session-rotation/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Session Test - Rotation",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
@@ -18,9 +17,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/stagehand-claude/actor/Dockerfile b/test/e2e/stagehand-claude/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/stagehand-claude/actor/Dockerfile
+++ b/test/e2e/stagehand-claude/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/stagehand-claude/actor/package.json b/test/e2e/stagehand-claude/actor/package.json
index 5c85a3c3e..6108fb3f0 100644
--- a/test/e2e/stagehand-claude/actor/package.json
+++ b/test/e2e/stagehand-claude/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Stagehand Test - Claude/Anthropic",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@browserbasehq/stagehand": "^3.0.7",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/stagehand-concurrent/actor/Dockerfile b/test/e2e/stagehand-concurrent/actor/Dockerfile
index 491acc4dd..77da7b583 100644
--- a/test/e2e/stagehand-concurrent/actor/Dockerfile
+++ b/test/e2e/stagehand-concurrent/actor/Dockerfile
@@ -2,13 +2,16 @@ FROM node:24 AS builder
COPY /packages ./packages
COPY /package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update
FROM apify/actor-node-playwright-chrome:24-1.58.2-beta
-RUN rm -r node_modules
+# The browser base images ship a preinstalled node_modules - wipe it so the
+# builder tree (with file: symlinks) can be copied over cleanly.
+RUN rm -rf node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
@@ -16,7 +19,7 @@ COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/stagehand-concurrent/actor/main.js b/test/e2e/stagehand-concurrent/actor/main.js
index 95ee351c5..b5cc03550 100644
--- a/test/e2e/stagehand-concurrent/actor/main.js
+++ b/test/e2e/stagehand-concurrent/actor/main.js
@@ -11,7 +11,12 @@ const mainOptions = {
};
await Actor.main(async () => {
- const browserIds = new Set();
+ // Playwright's Browser has no `process()`, so number the instances by object identity.
+ const browserIds = new Map();
+ const getBrowserId = (browser) => {
+ if (!browserIds.has(browser)) browserIds.set(browser, `browser-${browserIds.size + 1}`);
+ return browserIds.get(browser);
+ };
const crawler = new StagehandCrawler({
maxConcurrency: 3,
@@ -29,9 +34,7 @@ await Actor.main(async () => {
log.info(`Processing ${request.loadedUrl}`);
// Track which browser instance handled this request via the underlying Playwright browser
- const browser = page.context().browser();
- const browserId = browser?.process()?.pid ?? 'unknown';
- browserIds.add(browserId);
+ const browserId = getBrowserId(page.context().browser());
// Simple extraction - just get the page title
const result = await page.extract(
diff --git a/test/e2e/stagehand-concurrent/actor/package.json b/test/e2e/stagehand-concurrent/actor/package.json
index 9483c1f48..6d2a1b5dd 100644
--- a/test/e2e/stagehand-concurrent/actor/package.json
+++ b/test/e2e/stagehand-concurrent/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Stagehand Test - Concurrent browsers",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@browserbasehq/stagehand": "^3.0.7",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/storage-open-return-storage-object/actor/Dockerfile b/test/e2e/storage-open-return-storage-object/actor/Dockerfile
index c1788cb1f..e96d0b851 100644
--- a/test/e2e/storage-open-return-storage-object/actor/Dockerfile
+++ b/test/e2e/storage-open-return-storage-object/actor/Dockerfile
@@ -3,11 +3,12 @@ FROM apify/actor-node:24-beta
COPY packages ./packages
COPY package*.json ./
+RUN rm -rf node_modules package-lock.json
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional --no-audit \
+ && npm install --only=prod --no-audit \
&& npm update --no-audit \
&& echo "Installed NPM packages:" \
- && (npm list --only=prod --no-optional --all || true) \
+ && (npm list --only=prod --all || true) \
&& echo "Node.js version:" \
&& node --version \
&& echo "NPM version:" \
diff --git a/test/e2e/storage-open-return-storage-object/actor/main.js b/test/e2e/storage-open-return-storage-object/actor/main.js
index cf8d7154d..70c090458 100644
--- a/test/e2e/storage-open-return-storage-object/actor/main.js
+++ b/test/e2e/storage-open-return-storage-object/actor/main.js
@@ -12,7 +12,7 @@ await Actor.main(async () => {
const kv = await KeyValueStore.open();
const dataset = await Dataset.open();
await kv.setValue('storageObject', {
- keyValueStorageObject: kv.storageObject,
- datasetStorageObject: dataset.storageObject,
+ keyValueStorageObject: { id: kv.id, name: kv.name ?? null },
+ datasetStorageObject: { id: dataset.id, name: dataset.name ?? null },
});
}, mainOptions);
diff --git a/test/e2e/storage-open-return-storage-object/actor/package.json b/test/e2e/storage-open-return-storage-object/actor/package.json
index 3a6bda82f..1f9003689 100644
--- a/test/e2e/storage-open-return-storage-object/actor/package.json
+++ b/test/e2e/storage-open-return-storage-object/actor/package.json
@@ -3,8 +3,7 @@
"version": "0.0.1",
"description": "Key-Value Store - Return storage object on open",
"dependencies": {
- "apify": "next-v4",
- "@apify/storage-local": "^3.0.0",
+ "apify": "^4.0.0-beta.24",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
"@crawlee/types": "file:./packages/types",
@@ -14,9 +13,6 @@
"apify": {
"@crawlee/core": "file:./packages/core",
"@crawlee/utils": "file:./packages/utils"
- },
- "@apify/storage-local": {
- "better-sqlite3": "^11.10.0"
}
},
"scripts": {
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index 416763ec1..31bb614ef 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -6,7 +6,7 @@ import { dirname, join } from 'node:path';
import { setTimeout } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
-import { URL_NO_COMMAS_REGEX } from '@crawlee/utils';
+import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal';
import { Actor, ApifyClient } from 'apify';
import fs from 'fs-extra';
import { got } from 'got';
@@ -24,12 +24,12 @@ function execSync(command, options) {
* the base crawler created and installs its own, which lands on the next id.
* @param {string} name
*/
-const isCrawlerStatisticsKey = (name) => name.startsWith('CRAWLEE_CRAWLER_STATISTICS_');
+const isCrawlerStatisticsKey = (name) => name.startsWith('CRAWLEE_CRAWLER_STATISTICS');
/**
* @param {string} name
*/
-const isPrivateEntry = (name) => isCrawlerStatisticsKey(name) || name === 'CRAWLEE_SESSION_POOL_STATE';
+const isPrivateEntry = (name) => isCrawlerStatisticsKey(name) || name.startsWith('CRAWLEE_SESSION_POOL_STATE');
export const SKIPPED_TEST_CLOSE_CODE = 404;
@@ -55,6 +55,7 @@ export function getStorage(dirName) {
* @param {string} dirName
*/
export async function getStats(dirName) {
+ // fs-storage writes key-value records extensionless (the filename is the record key).
const dir = join(getStorage(dirName), 'key_value_stores/default');
if (!existsSync(dir)) {
@@ -191,8 +192,13 @@ export async function runActor(dirName, memory = 4096) {
// id: runId,
buildId,
userId,
+ status: runStatus,
} = await client.run(runId).waitForFinish();
+ if (runStatus !== 'SUCCEEDED') {
+ console.log(`[run] actor run finished with status ${runStatus} - the assertions below will likely fail`);
+ }
+
getKeyValueStoreItems = async (name) => {
const kvResult = await client.keyValueStore(name ? `${userId}/${name}` : defaultKeyValueStoreId).get();
@@ -467,7 +473,8 @@ export async function getLocalKeyValueStoreItems(dirName, kvName) {
const filePath = join(storePath, fileName.name);
const buffer = await readFile(filePath);
- const name = fileName.name.split('.').slice(0, -1).join('.');
+ // fs-storage writes records extensionless — the filename is the record key.
+ const name = fileName.name;
if (isPrivateEntry(name)) {
continue;