Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: resolve last direct @apify/log calls

Richard Solar0eda96aec9f5
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json
index 515a05715..fb7bed6e8 100644
--- a/packages/basic-crawler/package.json
+++ b/packages/basic-crawler/package.json
@@ -40,7 +40,6 @@
},
"dependencies": {
"@apify/datastructures": "^2.0.0",
- "@apify/log": "^2.5.18",
"@apify/timeout": "^0.3.2",
"@apify/utilities": "^2.15.5",
"@crawlee/core": "4.0.0",
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 0377cd3a0..c53a3defa 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -832,7 +832,7 @@ export class BasicCrawler<
this.requestQueue = requestQueue;
}
- this.httpClient = httpClient ?? new GotScrapingHttpClient();
+ this.httpClient = httpClient ?? new GotScrapingHttpClient({ logger: this.log });
this.proxyConfiguration = proxyConfiguration;
this.statusMessageLoggingInterval = statusMessageLoggingInterval;
this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;
@@ -1775,7 +1775,7 @@ export class BasicCrawler<
return cachedRobotsTxtFile;
}
- const robotsTxtFile = await RobotsTxtFile.find(url);
+ const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
this.robotsTxtFileCache.add(origin, robotsTxtFile);
return robotsTxtFile;
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index c92ae0e14..00efaf577 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -30,7 +30,6 @@
"copy": "tsx ../../scripts/copy.ts"
},
"dependencies": {
- "@apify/log": "^2.5.18",
"@apify/timeout": "^0.3.2",
"@crawlee/core": "4.0.0",
"@crawlee/types": "4.0.0",
diff --git a/packages/core/src/autoscaling/memory_load_signal.ts b/packages/core/src/autoscaling/memory_load_signal.ts
index 205e96e1e..ab9376cd5 100644
--- a/packages/core/src/autoscaling/memory_load_signal.ts
+++ b/packages/core/src/autoscaling/memory_load_signal.ts
@@ -1,14 +1,13 @@
import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
-import type { Log } from '@apify/log';
-
import type { Configuration } from '../configuration.js';
import type { EventManager } from '../events/event_manager.js';
import { EventType } from '../events/event_manager.js';
-import { log as defaultLog } from '../log.js';
+import type { CrawleeLogger } from '../log.js';
import type { LoadSignal, LoadSnapshot } from './load_signal.js';
import { SnapshotStore } from './load_signal.js';
import type { SystemInfo } from './system_status.js';
+import { serviceLocator } from '../service_locator.js';
const RESERVE_MEMORY_RATIO = 0.5;
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
@@ -22,7 +21,7 @@ export interface MemoryLoadSignalOptions {
overloadedRatio?: number;
snapshotHistoryMillis?: number;
config: Configuration;
- log?: Log;
+ log?: CrawleeLogger;
}
/**
@@ -36,7 +35,7 @@ export class MemoryLoadSignal implements LoadSignal {
private readonly store: SnapshotStore<MemorySnapshot>;
private readonly config: Configuration;
private readonly events: EventManager;
- private readonly log: Log;
+ private readonly log: CrawleeLogger;
private readonly maxUsedMemoryRatio: number;
private maxMemoryRatio: number | undefined;
private maxMemoryBytes!: number;
@@ -46,7 +45,7 @@ export class MemoryLoadSignal implements LoadSignal {
this.store = new SnapshotStore(options.snapshotHistoryMillis);
this.config = options.config;
this.events = this.config.getEventManager();
- this.log = options.log ?? defaultLog.child({ prefix: 'MemoryLoadSignal' });
+ this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
this.overloadedRatio = options.overloadedRatio ?? 0.2;
this._onSystemInfo = this._onSystemInfo.bind(this);
diff --git a/packages/core/src/crawlers/context_pipeline.ts b/packages/core/src/crawlers/context_pipeline.ts
index 8f7fe8483..f22f0bb0e 100644
--- a/packages/core/src/crawlers/context_pipeline.ts
+++ b/packages/core/src/crawlers/context_pipeline.ts
@@ -1,7 +1,5 @@
import type { Awaitable } from '@crawlee/types';
-import log from '@apify/log';
-
import {
ContextPipelineCleanupError,
ContextPipelineInitializationError,
@@ -9,6 +7,7 @@ import {
RequestHandlerError,
SessionError,
} from '../errors.js';
+import { serviceLocator } from '../service_locator.js';
/**
* Represents a middleware step in the context pipeline.
@@ -175,7 +174,9 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e
);
}
} catch (error: any) {
- log.debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+ serviceLocator
+ .getLogger()
+ .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
}
}
diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.ts
index 5aa079a2c..badc23278 100644
--- a/packages/core/src/events/local_event_manager.ts
+++ b/packages/core/src/events/local_event_manager.ts
@@ -93,7 +93,10 @@ export class LocalEventManager extends EventManager {
private async createCpuInfo(options: { maxUsedCpuRatio: number }) {
const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
- const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
+ const usedCpuRatio = await getCurrentCpuTicksV2({
+ containerized: await this.isContainerizedWrapper(),
+ logger: serviceLocator.getLogger(),
+ });
return {
cpuCurrentUsage: usedCpuRatio * 100,
isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -114,9 +117,10 @@ export class LocalEventManager extends EventManager {
}
private async getMemoryInfo() {
- if (this.config.get('systemInfoV2')) {
- return getMemoryInfoV2(await this.isContainerizedWrapper());
- }
- return getMemoryInfo();
+ const { getMemoryInfo } = await import('@crawlee/utils');
+ return getMemoryInfo({
+ containerized: await this.isContainerizedWrapper(),
+ logger: serviceLocator.getLogger(),
+ });
}
}
diff --git a/packages/core/src/log.ts b/packages/core/src/log.ts
index 069070949..01231bc33 100644
--- a/packages/core/src/log.ts
+++ b/packages/core/src/log.ts
@@ -1,85 +1,9 @@
+import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+
import type { LoggerOptions } from '@apify/log';
import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
-/**
- * Configuration options for Crawlee logger implementations.
- */
-export interface CrawleeLoggerOptions {
- /** Prefix to be prepended to each logged line. */
- prefix?: string | null;
-}
-
-/**
- * Interface for Crawlee logger implementations.
- * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
- * compatibility with the default `@apify/log` implementation.
- */
-export interface CrawleeLogger {
- /**
- * Returns the logger configuration.
- */
- getOptions(): CrawleeLoggerOptions;
-
- /**
- * Configures logger options.
- */
- setOptions(options: Partial<CrawleeLoggerOptions>): void;
-
- /**
- * Creates a new instance of logger that inherits settings from a parent logger.
- */
- child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
-
- /**
- * Logs an `ERROR` message.
- */
- error(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs an `ERROR` level message with a nicely formatted exception.
- */
- exception(exception: Error, message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs a `SOFT_FAIL` level message.
- */
- softFail(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs a `WARNING` level message.
- */
- warning(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs a `WARNING` level message only once.
- */
- warningOnce(message: string): void;
-
- /**
- * Logs an `INFO` message.
- */
- info(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs a `DEBUG` message.
- */
- debug(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs a `PERF` level message for performance tracking.
- */
- perf(message: string, data?: Record<string, unknown>): void;
-
- /**
- * Logs given message only once as WARNING for deprecated features.
- */
- deprecated(message: string): void;
-
- /**
- * Logs a message at the given level. Useful when the log level is determined dynamically.
- */
- logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
-}
+export type { CrawleeLogger, CrawleeLoggerOptions };
/**
* Abstract base class for custom Crawlee logger implementations.
diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.ts
index cbd1db611..4aac07403 100644
--- a/packages/core/src/service_locator.ts
+++ b/packages/core/src/service_locator.ts
@@ -127,10 +127,6 @@ interface ServiceLocatorInterface {
* // Crawler has its own isolated ServiceLocator instance
* ```
*/
-// Used as fallback in ServiceLocator methods that need to log before a logger is explicitly set,
-// without implicitly locking the logger slot (which getLogger() would do).
-const fallbackLog = new ApifyLogAdapter(log);
-
export class ServiceLocator implements ServiceLocatorInterface {
private configuration?: Configuration;
private eventManager?: EventManager;
@@ -165,9 +161,7 @@ export class ServiceLocator implements ServiceLocatorInterface {
getConfiguration(): Configuration {
if (!this.configuration) {
- (this.logger ?? fallbackLog).debug(
- 'No configuration set, implicitly creating and using default Configuration.',
- );
+ this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
this.configuration = new Configuration();
}
return this.configuration;
@@ -189,11 +183,9 @@ export class ServiceLocator implements ServiceLocatorInterface {
getEventManager(): EventManager {
if (!this.eventManager) {
- (this.logger ?? fallbackLog).debug(
- 'No event manager set, implicitly creating and using default LocalEventManager.',
- );
+ this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
if (!this.configuration) {
- (this.logger ?? fallbackLog).warning(
+ this.getLogger().warning(
'Implicit creation of event manager will implicitly set configuration as side effect. ' +
'It is advised to explicitly first set the configuration instead.',
);
@@ -219,11 +211,9 @@ export class ServiceLocator implements ServiceLocatorInterface {
getStorageClient(): StorageClient {
if (!this.storageClient) {
- (this.logger ?? fallbackLog).debug(
- 'No storage client set, implicitly creating and using default MemoryStorage.',
- );
+ this.getLogger().debug('No storage client set, implicitly creating and using default MemoryStorage.');
if (!this.configuration) {
- (this.logger ?? fallbackLog).warning(
+ this.getLogger().warning(
'Implicit creation of storage client will implicitly set configuration as side effect. ' +
'It is advised to explicitly first set the configuration instead.',
);
@@ -231,6 +221,7 @@ export class ServiceLocator implements ServiceLocatorInterface {
const config = this.getConfiguration();
this.storageClient = new MemoryStorage({
persistStorage: config.get('persistStorage'),
+ logger: this.getLogger().child({ prefix: 'MemoryStorage' }),
});
}
return this.storageClient;
diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.ts
index 39c53298d..0105b5074 100644
--- a/packages/core/src/storages/sitemap_request_list.ts
+++ b/packages/core/src/storages/sitemap_request_list.ts
@@ -438,7 +438,9 @@ export class SitemapRequestList implements IRequestList {
persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
});
await requestList.restoreState();
- void requestList.load({ parseSitemapOptions: { ...options.parseSitemapOptions, httpClient } });
+ void requestList.load({
+ parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
+ });
if (requestList.persistenceOptions.enable) {
requestList.events.on(EventType.PERSIST_STATE, requestList.persistState);
diff --git a/packages/core/test/core/service_locator.test.ts b/packages/core/test/core/service_locator.test.ts
index b589af743..4429a1ac6 100644
--- a/packages/core/test/core/service_locator.test.ts
+++ b/packages/core/test/core/service_locator.test.ts
@@ -23,7 +23,7 @@ function makeMockLogger(overrides: Partial<CrawleeLogger> = {}): CrawleeLogger {
debug: () => {},
perf: () => {},
deprecated: () => {},
- internal: () => {},
+ logWithLevel: () => {},
...overrides,
};
return logger;
@@ -205,6 +205,17 @@ describe('ServiceLocator', () => {
}).toThrow(/Logger is already in use/);
});
+ test('setting logger after getStorageClient throws ServiceConflictError (logger already locked)', () => {
+ // getStorageClient() implicitly calls getLogger(), locking the logger
+ serviceLocator.getStorageClient();
+
+ const customLogger = makeMockLogger();
+
+ expect(() => {
+ serviceLocator.setLogger(customLogger);
+ }).toThrow(ServiceConflictError);
+ });
+
test('reset clears the logger', () => {
const customLogger = makeMockLogger();
serviceLocator.setLogger(customLogger);
diff --git a/packages/http-client/package.json b/packages/http-client/package.json
index f94278cc9..e7d4c5b4f 100644
--- a/packages/http-client/package.json
+++ b/packages/http-client/package.json
@@ -47,7 +47,6 @@
"access": "public"
},
"dependencies": {
- "@apify/log": "^2.5.32",
"@crawlee/types": "4.0.0",
"tough-cookie": "^6.0.0"
}
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts
index 2491c432a..c012a98da 100644
--- a/packages/http-client/src/base-http-client.ts
+++ b/packages/http-client/src/base-http-client.ts
@@ -1,9 +1,6 @@
-import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types';
+import type { BaseHttpClient as BaseHttpClientInterface, CrawleeLogger, SendRequestOptions } from '@crawlee/types';
import { CookieJar } from 'tough-cookie';
-import type { Log } from '@apify/log';
-import defaultLog from '@apify/log';
-
export interface CustomFetchOptions {
proxyUrl?: string;
}
@@ -14,10 +11,10 @@ export interface CustomFetchOptions {
* implement only the low-level network call in `fetch`.
*/
export abstract class BaseHttpClient implements BaseHttpClientInterface {
- protected log: Log;
+ protected log?: CrawleeLogger;
- constructor(log?: Log) {
- this.log = log ?? defaultLog;
+ constructor(options?: { logger?: CrawleeLogger }) {
+ this.log = options?.logger;
}
/**
@@ -36,7 +33,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
request.headers.set('cookie', cookies.join('; '));
}
} catch (e) {
- this.log.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);
+ this.log?.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);
}
return request;
}
@@ -48,7 +45,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
try {
await cookieJar.setCookie(header, response.url);
} catch (e) {
- this.log.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);
+ this.log?.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);
}
}
}
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.ts
index 0be7642b9..0c52b9736 100644
--- a/packages/impit-client/src/index.ts
+++ b/packages/impit-client/src/index.ts
@@ -1,5 +1,6 @@
import type { CustomFetchOptions } from '@crawlee/http-client';
import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
import { Impit, type ImpitOptions } from 'impit';
import type { CookieJar as ToughCookieJar } from 'tough-cookie';
@@ -48,9 +49,11 @@ export class ImpitHttpClient extends BaseHttpClient {
/**
* @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
*/
- constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & { cacheClients?: boolean }) {
- super();
- const { cacheClients = true, ...impitOptions } = options ?? {};
+ constructor(
+ options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & { cacheClients?: boolean; logger?: CrawleeLogger },
+ ) {
+ super({ logger: options?.logger });
+ const { cacheClients = true, logger: _logger, ...impitOptions } = options ?? {};
this.impitOptions = impitOptions;
this.cacheClients = cacheClients;
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index f66258214..f57d7b262 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -42,7 +42,6 @@
"access": "public"
},
"dependencies": {
- "@apify/log": "^2.5.18",
"@crawlee/types": "4.0.0",
"@sapphire/async-queue": "^1.5.5",
"@sapphire/shapeshift": "^4.0.0",
diff --git a/packages/memory-storage/src/background-handler/fs-utils.ts b/packages/memory-storage/src/background-handler/fs-utils.ts
index 387784cec..2dc1e1ea8 100644
--- a/packages/memory-storage/src/background-handler/fs-utils.ts
+++ b/packages/memory-storage/src/background-handler/fs-utils.ts
@@ -3,16 +3,13 @@ import { writeFile as writeFileP } from 'node:fs/promises';
import { resolve } from 'node:path';
import { setTimeout } from 'node:timers/promises';
+import type { CrawleeLogger } from '@crawlee/types';
import { ensureDir } from 'fs-extra/esm';
import { lock } from 'proper-lockfile';
-import log from '@apify/log';
-
import type { BackgroundHandlerReceivedMessage, BackgroundHandlerUpdateMetadataMessage } from '../utils.js';
-const backgroundHandlerLog = log.child({ prefix: 'MemoryStorageBackgroundHandler' });
-
-export async function handleMessage(message: BackgroundHandlerReceivedMessage) {
+export async function handleMessage(message: BackgroundHandlerReceivedMessage, logger?: CrawleeLogger) {
switch (message.action) {
case 'update-metadata':
await updateMetadata(message);
@@ -20,7 +17,7 @@ export async function handleMessage(message: BackgroundHandlerReceivedMessage) {
default:
// We're keeping this to make eslint happy + in the event we add a new action without adding checks for it
// we should be aware of them
- backgroundHandlerLog.warning(
+ logger?.warning(
`Unknown background handler message action ${(message as BackgroundHandlerReceivedMessage).action}`,
);
}
diff --git a/packages/memory-storage/src/background-handler/index.ts b/packages/memory-storage/src/background-handler/index.ts
index 5912cc7ad..50417f56c 100644
--- a/packages/memory-storage/src/background-handler/index.ts
+++ b/packages/memory-storage/src/background-handler/index.ts
@@ -1,5 +1,7 @@
import { randomUUID } from 'node:crypto';
+import type { CrawleeLogger } from '@crawlee/types';
+
import type { BackgroundHandlerReceivedMessage } from '../utils.js';
import { handleMessage } from './fs-utils.js';
@@ -16,7 +18,7 @@ export const promiseMap: Map<
}
> = new Map();
-export function scheduleBackgroundTask(message: BackgroundHandlerReceivedMessage) {
+export function scheduleBackgroundTask(message: BackgroundHandlerReceivedMessage, logger?: CrawleeLogger) {
const id = randomUUID();
let promiseResolve: () => void;
@@ -29,14 +31,20 @@ export function scheduleBackgroundTask(message: BackgroundHandlerReceivedMessage
resolve: promiseResolve!,
});
- void handleBackgroundMessage({
- ...message,
- messageId: id,
- });
+ void handleBackgroundMessage(
+ {
+ ...message,
+ messageId: id,
+ },
+ logger,
+ );
}
-async function handleBackgroundMessage(message: BackgroundHandlerReceivedMessage & { messageId: string }) {
- await handleMessage(message);
+async function handleBackgroundMessage(
+ message: BackgroundHandlerReceivedMessage & { messageId: string },
+ logger?: CrawleeLogger,
+) {
+ await handleMessage(message, logger);
promiseMap.get(message.messageId)?.resolve();
promiseMap.delete(message.messageId);
diff --git a/packages/memory-storage/src/cache-helpers.ts b/packages/memory-storage/src/cache-helpers.ts
index c7db0a9ea..69ab0001f 100644
--- a/packages/memory-storage/src/cache-helpers.ts
+++ b/packages/memory-storage/src/cache-helpers.ts
@@ -196,7 +196,7 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
try {
json5.parse(stringifiedJson);
} catch {
- memoryStorageLog.warning(
+ client.logger?.warning(
`Key-value entry "${entry.name}" for store ${entryNameOrId} has invalid JSON content and will be ignored from the store.`,
);
continue;
@@ -250,6 +250,7 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
persistStorage: true,
storeDirectory: keyValueStoreDir,
writeMetadata: hasSeenMetadataForEntry,
+ logger: client.logger,
});
// eslint-disable-next-line dot-notation
@@ -336,7 +337,7 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryN
entries.add(entryName);
} catch {
- memoryStorageLog.warning(
+ client.logger?.warning(
`Request queue entry "${entry.name}" for store ${entryNameOrId} has invalid JSON content and will be ignored from the store.`,
);
}
@@ -392,4 +393,4 @@ import { DatasetClient } from './resource-clients/dataset.js';
import type { InternalKeyRecord } from './resource-clients/key-value-store.js';
import { KeyValueStoreClient } from './resource-clients/key-value-store.js';
import { RequestQueueClient } from './resource-clients/request-queue.js';
-import { memoryStorageLog, resolveWithinDirectory } from './utils.js';
+import { resolveWithinDirectory } from './utils.js';
diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.ts
index 8bf024add..6af3e567c 100644
--- a/packages/memory-storage/src/fs/key-value-store/fs.ts
+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts
@@ -1,19 +1,21 @@
import { readFile, rm } from 'node:fs/promises';
import { basename, dirname } from 'node:path';
+import type { CrawleeLogger } from '@crawlee/types';
import { AsyncQueue } from '@sapphire/async-queue';
import { ensureDir } from 'fs-extra/esm';
import mime from 'mime-types';
import { lockAndWrite } from '../../background-handler/fs-utils.js';
import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js';
-import { memoryStorageLog, resolveWithinDirectory } from '../../utils.js';
+import { resolveWithinDirectory } from '../../utils.js';
import type { StorageImplementation } from '../common.js';
import type { CreateStorageImplementationOptions } from './index.js';
export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeyRecord> {
private storeDirectory: string;
private writeMetadata: boolean;
+ private logger?: CrawleeLogger;
private filePath!: string;
private fileMetadataPath!: string;
@@ -23,6 +25,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
constructor(options: CreateStorageImplementationOptions) {
this.storeDirectory = options.storeDirectory;
this.writeMetadata = options.writeMetadata;
+ this.logger = options.logger;
}
async get(): Promise<InternalKeyRecord> {
@@ -36,7 +39,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
const noExtFilePath = resolve(this.storeDirectory, this.rawRecord.key);
// Try without extension
file = await readFile(noExtFilePath);
- memoryStorageLog.warning(
+ this.logger?.warning(
[
`Key-value entry "${this.rawRecord.key}" for store ${basename(
this.storeDirectory,
diff --git a/packages/memory-storage/src/fs/key-value-store/index.ts b/packages/memory-storage/src/fs/key-value-store/index.ts
index 20e664ae0..6947b2b10 100644
--- a/packages/memory-storage/src/fs/key-value-store/index.ts
+++ b/packages/memory-storage/src/fs/key-value-store/index.ts
@@ -1,3 +1,5 @@
+import type { CrawleeLogger } from '@crawlee/types';
+
import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js';
import type { StorageImplementation } from '../common.js';
import { KeyValueFileSystemEntry } from './fs.js';
@@ -17,4 +19,5 @@ export interface CreateStorageImplementationOptions {
persistStorage: boolean;
storeDirectory: string;
writeMetadata: boolean;
+ logger?: CrawleeLogger;
}
diff --git a/packages/memory-storage/src/memory-storage.ts b/packages/memory-storage/src/memory-storage.ts
index 5ba10b597..ec50f24bf 100644
--- a/packages/memory-storage/src/memory-storage.ts
+++ b/packages/memory-storage/src/memory-storage.ts
@@ -4,6 +4,7 @@ import { resolve } from 'node:path';
import type * as storage from '@crawlee/types';
import type { Dictionary } from '@crawlee/types';
+import type { CrawleeLogger } from '@crawlee/types';
import { s } from '@sapphire/shapeshift';
import { ensureDirSync, move, moveSync, pathExistsSync } from 'fs-extra/esm';
@@ -35,6 +36,11 @@ export interface MemoryStorageOptions {
* @default true
*/
persistStorage?: boolean;
+
+ /**
+ * Optional logger for MemoryStorage warnings.
+ */
+ logger?: CrawleeLogger;
}
export class MemoryStorage implements storage.StorageClient {
@@ -44,6 +50,7 @@ export class MemoryStorage implements storage.StorageClient {
readonly requestQueuesDirectory: string;
readonly writeMetadata: boolean;
readonly persistStorage: boolean;
+ readonly logger?: CrawleeLogger;
readonly keyValueStoresHandled: KeyValueStoreClient[] = [];
readonly datasetClientsHandled: DatasetClient[] = [];
@@ -56,6 +63,8 @@ export class MemoryStorage implements storage.StorageClient {
persistStorage: s.boolean().optional(),
}).parse(options);
+ this.logger = options.logger;
+
// v3.0.0 used `crawlee_storage` as the default, we changed this in v3.0.1 to just `storage`,
// this function handles it without making BC breaks - it respects existing `crawlee_storage`
// directories, and uses the `storage` only if it's not there.
diff --git a/packages/memory-storage/src/resource-clients/dataset-collection.ts b/packages/memory-storage/src/resource-clients/dataset-collection.ts
index 4aa2462aa..104e9e55c 100644
--- a/packages/memory-storage/src/resource-clients/dataset-collection.ts
+++ b/packages/memory-storage/src/resource-clients/dataset-collection.ts
@@ -52,15 +52,18 @@ export class DatasetCollectionClient implements storage.DatasetCollectionClient
// Schedule the worker to write to the disk
const datasetInfo = newStore.toDatasetInfo();
- scheduleBackgroundTask({
- action: 'update-metadata',
- entityType: 'datasets',
- entityDirectory: newStore.datasetDirectory,
- id: datasetInfo.name ?? datasetInfo.id,
- data: datasetInfo,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ entityType: 'datasets',
+ entityDirectory: newStore.datasetDirectory,
+ id: datasetInfo.name ?? datasetInfo.id,
+ data: datasetInfo,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
return datasetInfo;
}
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.ts
index d069c101e..716cea3d8 100644
--- a/packages/memory-storage/src/resource-clients/dataset.ts
+++ b/packages/memory-storage/src/resource-clients/dataset.ts
@@ -305,14 +305,17 @@ export class DatasetClient<Data extends Dictionary = Dictionary>
}
const data = this.toDatasetInfo();
- scheduleBackgroundTask({
- action: 'update-metadata',
- data,
- entityType: 'datasets',
- entityDirectory: this.datasetDirectory,
- id: this.name ?? this.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ data,
+ entityType: 'datasets',
+ entityDirectory: this.datasetDirectory,
+ id: this.name ?? this.id,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
}
}
diff --git a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts b/packages/memory-storage/src/resource-clients/key-value-store-collection.ts
index f656b9019..4db3b1e05 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store-collection.ts
@@ -56,15 +56,18 @@ export class KeyValueStoreCollectionClient implements storage.KeyValueStoreColle
// Schedule the worker to write to the disk
const kvStoreInfo = newStore.toKeyValueStoreInfo();
- scheduleBackgroundTask({
- action: 'update-metadata',
- entityType: 'keyValueStores',
- entityDirectory: newStore.keyValueStoreDirectory,
- id: kvStoreInfo.name ?? kvStoreInfo.id,
- data: kvStoreInfo,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ entityType: 'keyValueStores',
+ entityDirectory: newStore.keyValueStoreDirectory,
+ id: kvStoreInfo.name ?? kvStoreInfo.id,
+ data: kvStoreInfo,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
return kvStoreInfo;
}
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts
index 53bc17404..28cf34c17 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts
@@ -485,6 +485,7 @@ export class KeyValueStoreClient extends BaseClient {
persistStorage: this.client.persistStorage,
storeDirectory: existingStoreById.keyValueStoreDirectory,
writeMetadata: existingStoreById.client.writeMetadata,
+ logger: this.client.logger,
});
await entry.update(_record);
@@ -532,14 +533,17 @@ export class KeyValueStoreClient extends BaseClient {
}
const data = this.toKeyValueStoreInfo();
- scheduleBackgroundTask({
- action: 'update-metadata',
- data,
- entityType: 'keyValueStores',
- entityDirectory: this.keyValueStoreDirectory,
- id: this.name ?? this.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ data,
+ entityType: 'keyValueStores',
+ entityDirectory: this.keyValueStoreDirectory,
+ id: this.name ?? this.id,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
}
}
diff --git a/packages/memory-storage/src/resource-clients/request-queue-collection.ts b/packages/memory-storage/src/resource-clients/request-queue-collection.ts
index 81c4f634d..90ed3c81d 100644
--- a/packages/memory-storage/src/resource-clients/request-queue-collection.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue-collection.ts
@@ -56,15 +56,18 @@ export class RequestQueueCollectionClient implements storage.RequestQueueCollect
// Schedule the worker to write to the disk
const queueInfo = newStore.toRequestQueueInfo();
- scheduleBackgroundTask({
- action: 'update-metadata',
- entityType: 'requestQueues',
- entityDirectory: newStore.requestQueueDirectory,
- id: queueInfo.name ?? queueInfo.id,
- data: queueInfo,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ entityType: 'requestQueues',
+ entityDirectory: newStore.requestQueueDirectory,
+ id: queueInfo.name ?? queueInfo.id,
+ data: queueInfo,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
return queueInfo;
}
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts
index 518ffcb16..aaa1f6c78 100644
--- a/packages/memory-storage/src/resource-clients/request-queue.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue.ts
@@ -624,15 +624,18 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
forefrontRequestIds: this.forefrontRequestIds,
};
- scheduleBackgroundTask({
- action: 'update-metadata',
- data,
- entityType: 'requestQueues',
- entityDirectory: this.requestQueueDirectory,
- id: this.name ?? this.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ scheduleBackgroundTask(
+ {
+ action: 'update-metadata',
+ data,
+ entityType: 'requestQueues',
+ entityDirectory: this.requestQueueDirectory,
+ id: this.name ?? this.id,
+ writeMetadata: this.client.writeMetadata,
+ persistStorage: this.client.persistStorage,
+ },
+ this.client.logger,
+ );
}
private _jsonToRequest<T>(requestJson?: string): T | undefined {
diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.ts
index 3d0068e55..70a596195 100644
--- a/packages/memory-storage/src/utils.ts
+++ b/packages/memory-storage/src/utils.ts
@@ -4,8 +4,6 @@ import { resolve, sep } from 'node:path';
import type * as storage from '@crawlee/types';
import { s } from '@sapphire/shapeshift';
-import defaultLog from '@apify/log';
-
import { REQUEST_ID_LENGTH } from './consts.js';
/**
@@ -71,8 +69,6 @@ export function isStream(value: any): boolean {
);
}
-export const memoryStorageLog = defaultLog.child({ prefix: 'MemoryStorage' });
-
export type BackgroundHandlerReceivedMessage = BackgroundHandlerUpdateMetadataMessage;
export type BackgroundHandlerUpdateMetadataMessage =
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 791a0cd17..d5a6206b4 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -48,7 +48,6 @@
},
"dependencies": {
"@apify/datastructures": "^2.0.3",
- "@apify/log": "^2.5.18",
"@apify/timeout": "^0.3.2",
"@crawlee/browser": "4.0.0",
"@crawlee/browser-pool": "4.0.0",
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 49db82d95..62d712329 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -48,7 +48,6 @@
},
"dependencies": {
"@apify/datastructures": "^2.0.3",
- "@apify/log": "^2.5.18",
"@crawlee/browser": "4.0.0",
"@crawlee/browser-pool": "4.0.0",
"@crawlee/types": "4.0.0",
diff --git a/packages/stagehand-crawler/package.json b/packages/stagehand-crawler/package.json
index 92bdff384..b4a160253 100644
--- a/packages/stagehand-crawler/package.json
+++ b/packages/stagehand-crawler/package.json
@@ -56,7 +56,6 @@
"access": "public"
},
"dependencies": {
- "@apify/log": "^2.5.18",
"@apify/timeout": "^0.3.2",
"@crawlee/browser": "4.0.0",
"@crawlee/browser-pool": "4.0.0",
diff --git a/packages/stagehand-crawler/src/internals/stagehand-controller.ts b/packages/stagehand-crawler/src/internals/stagehand-controller.ts
index 0815a2ce2..8e75206ae 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-controller.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-controller.ts
@@ -1,11 +1,10 @@
import type { Stagehand } from '@browserbasehq/stagehand';
import { BrowserController } from '@crawlee/browser-pool';
+import { serviceLocator } from '@crawlee/core';
import type { Cookie } from '@crawlee/types';
import { sleep } from '@crawlee/utils';
import type { Browser as PlaywrightBrowser, BrowserType, LaunchOptions, Page } from 'playwright';
-import log from '@apify/log';
-
import type { StagehandPlugin } from './stagehand-plugin';
/**
@@ -177,7 +176,7 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOp
try {
await stagehand.close();
} catch (error) {
- log.error('Error closing Stagehand', { error });
+ serviceLocator.getLogger().error('Error closing Stagehand', { error });
}
}
diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
index 5ec07ead7..f409fd949 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
@@ -1,13 +1,12 @@
import type { Stagehand, V3Options } from '@browserbasehq/stagehand';
import type { BrowserController, BrowserPluginOptions, LaunchContext } from '@crawlee/browser-pool';
import { anonymizeProxySugar, BrowserPlugin } from '@crawlee/browser-pool';
+import { serviceLocator } from '@crawlee/core';
import type { Browser as PlaywrightBrowser, BrowserType, LaunchOptions } from 'playwright';
// Stagehand is built on CDP (Chrome DevTools Protocol), which only works with Chromium-based browsers.
// Firefox and WebKit are not supported by Stagehand.
import { chromium } from 'playwright';
-import log from '@apify/log';
-
import { StagehandController } from './stagehand-controller';
import type { StagehandOptions } from './stagehand-crawler';
@@ -122,7 +121,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P
await closeAnonymizedProxy();
const augmentedError = this._augmentLaunchError(error, launchContext);
- log.error('Stagehand browser launch failed', { message: augmentedError.message });
+ serviceLocator.getLogger().error('Stagehand browser launch failed', { message: augmentedError.message });
throw augmentedError;
}
}
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index ef5273f56..d7058d57c 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -3,3 +3,4 @@ export * from './utility-types.js';
export * from './browser.js';
export * from './http-client.js';
export * from './session.js';
+export * from './logger.js';
diff --git a/packages/types/src/logger.ts b/packages/types/src/logger.ts
new file mode 100644
index 000000000..a3520c20f
--- /dev/null
+++ b/packages/types/src/logger.ts
@@ -0,0 +1,79 @@
+/**
+ * Configuration options for Crawlee logger implementations.
+ */
+export interface CrawleeLoggerOptions {
+ /** Prefix to be prepended to each logged line. */
+ prefix?: string | null;
+}
+
+/**
+ * Interface for Crawlee logger implementations.
+ * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
+ * compatibility with the default `@apify/log` implementation.
+ */
+export interface CrawleeLogger {
+ /**
+ * Returns the logger configuration.
+ */
+ getOptions(): CrawleeLoggerOptions;
+
+ /**
+ * Configures logger options.
+ */
+ setOptions(options: Partial<CrawleeLoggerOptions>): void;
+
+ /**
+ * Creates a new instance of logger that inherits settings from a parent logger.
+ */
+ child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+
+ /**
+ * Logs an `ERROR` message.
+ */
+ error(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs an `ERROR` level message with a nicely formatted exception.
+ */
+ exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs a `SOFT_FAIL` level message.
+ */
+ softFail(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs a `WARNING` level message.
+ */
+ warning(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs a `WARNING` level message only once.
+ */
+ warningOnce(message: string): void;
+
+ /**
+ * Logs an `INFO` message.
+ */
+ info(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs a `DEBUG` message.
+ */
+ debug(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs a `PERF` level message for performance tracking.
+ */
+ perf(message: string, data?: Record<string, unknown>): void;
+
+ /**
+ * Logs given message only once as WARNING for deprecated features.
+ */
+ deprecated(message: string): void;
+
+ /**
+ * Logs a message at the given level. Useful when the log level is determined dynamically.
+ */
+ logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+}
diff --git a/packages/utils/package.json b/packages/utils/package.json
index d0cddd15d..d52756614 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -41,7 +41,6 @@
"copy": "tsx ../../scripts/copy.ts"
},
"dependencies": {
- "@apify/log": "^2.5.18",
"@apify/ps-tree": "^1.2.0",
"@crawlee/http-client": "4.0.0",
"@crawlee/types": "4.0.0",
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index a91e46662..4cfc55b87 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -1,10 +1,8 @@
import { FetchHttpClient } from '@crawlee/http-client';
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
import type { Robot } from 'robots-parser';
import robotsParser from 'robots-parser';
-import log from '@apify/log';
-
import { Sitemap } from './sitemap.js';
import { type EnqueueStrategy, filterUrl } from './url.js';
@@ -40,6 +38,7 @@ export class RobotsTxtFile {
private url: string,
private robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>,
private proxyUrl?: string,
+ private logger?: CrawleeLogger,
) {}
/**
@@ -52,7 +51,13 @@ export class RobotsTxtFile {
*/
static async find(
url: string,
- options?: { signal?: AbortSignal; timeoutMillis?: number; proxyUrl?: string; httpClient?: BaseHttpClient },
+ options?: {
+ signal?: AbortSignal;
+ timeoutMillis?: number;
+ proxyUrl?: string;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+ },
): Promise<RobotsTxtFile> {
const robotsTxtFileUrl = new URL(url);
robotsTxtFileUrl.pathname = '/robots.txt';
@@ -73,9 +78,15 @@ export class RobotsTxtFile {
protected static async load(
url: string,
- options?: { signal?: AbortSignal; timeoutMillis?: number; proxyUrl?: string; httpClient?: BaseHttpClient },
+ options?: {
+ signal?: AbortSignal;
+ timeoutMillis?: number;
+ proxyUrl?: string;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+ },
): Promise<RobotsTxtFile> {
- const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
+ const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
proxyUrl,
@@ -99,10 +110,11 @@ export class RobotsTxtFile {
},
},
proxyUrl,
+ logger,
);
}
- return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl);
+ return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
}
/**
@@ -127,7 +139,7 @@ export class RobotsTxtFile {
// `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
const { allowed, reason } = filterUrl(sitemapUrl, this.url, enqueueStrategy);
if (!allowed) {
- log.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);
+ this.logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);
continue;
}
sitemaps.push(sitemapUrl);
@@ -141,7 +153,7 @@ export class RobotsTxtFile {
* and the sitemap parser.
*/
async parseSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<Sitemap> {
- return Sitemap.load(this.getSitemaps(options), this.proxyUrl, options);
+ return Sitemap.load(this.getSitemaps(options), this.proxyUrl, { ...options, logger: this.logger });
}
/**
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index b2e5f04a1..82cdb3c42 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -5,13 +5,11 @@ import { StringDecoder } from 'node:string_decoder';
import { createGunzip } from 'node:zlib';
import { FetchHttpClient } from '@crawlee/http-client';
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
import { fileTypeStream } from 'file-type';
import sax from 'sax';
import MIMEType from 'whatwg-mimetype';
-import log from '@apify/log';
-
import { mergeAsyncIterables } from './iterables.js';
import { RobotsFile } from './robots.js';
import { type EnqueueStrategy, filterUrl } from './url.js';
@@ -219,6 +217,10 @@ export interface ParseSitemapOptions {
* @default 'same-hostname'
*/
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+ /**
+ * Optional logger for reporting warnings during sitemap parsing.
+ */
+ logger?: CrawleeLogger;
}
export async function* parseSitemap<T extends ParseSitemapOptions>(
@@ -235,6 +237,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
reportNetworkErrors = true,
nestedSitemapFilter,
enqueueStrategy = 'same-hostname',
+ logger,
} = options ?? {};
const sources = [...initialSources];
@@ -264,9 +267,6 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
const source = sources.shift()!;
if ((source?.depth ?? 0) > maxDepth) {
- log.debug(
- `Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`,
- );
continue;
}
@@ -356,7 +356,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
break;
}
} catch (e) {
- log.warning(
+ logger?.warning(
`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`,
);
}
@@ -364,7 +364,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
} else if (source.type === 'raw') {
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
if (error !== undefined) {
- log.warning(`Malformed sitemap content: ${error}`);
+ logger?.warning(`Malformed sitemap content: ${error}`);
}
});
}
@@ -380,7 +380,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
for await (const item of items) {
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
- log.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
continue;
}
@@ -389,7 +389,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
if (source.type === 'url') {
const { allowed, reason } = filterUrl(item.url, sitemapUrl!, enqueueStrategy);
if (!allowed) {
- log.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
continue;
}
}
@@ -406,7 +406,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
const { allowed, reason } = filterUrl(item.loc, sitemapUrl!, enqueueStrategy);
if (!allowed) {
droppedUrlEntries++;
- log.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
continue;
}
}
@@ -422,7 +422,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
}
if (droppedUrlEntries > 0 && source.type === 'url') {
- log.warning(
+ logger?.warning(
`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`,
);
}
@@ -511,7 +511,9 @@ export class Sitemap {
urls.push(item.loc);
}
} catch (e) {
- log.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
+ parseSitemapOptions?.logger?.warning(
+ `Sitemap.load: Failed to load sitemap, returning empty result. (${e})`,
+ );
return new Sitemap([]);
}
@@ -555,6 +557,10 @@ export async function* discoverValidSitemaps(
* HTTP client to be used for network requests.
*/
httpClient?: BaseHttpClient;
+ /**
+ * Optional logger for reporting warnings during sitemap discovery.
+ */
+ logger?: CrawleeLogger;
} = {},
): AsyncIterable<string> {
const {
@@ -563,6 +569,7 @@ export async function* discoverValidSitemaps(
signal: externalSignal,
requestTimeoutMillis = 20_000,
httpClient = new FetchHttpClient(),
+ logger,
} = options;
const controller = new AbortController();
@@ -618,6 +625,7 @@ export async function* discoverValidSitemaps(
timeoutMillis: requestTimeoutMillis,
signal,
httpClient,
+ logger,
});
// Surface all referenced sitemaps, including cross-host; scoping happens at load time.
for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
@@ -626,7 +634,7 @@ export async function* discoverValidSitemaps(
}
}
} catch (err) {
- log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
}
const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
@@ -652,7 +660,7 @@ export async function* discoverValidSitemaps(
yield candidateSitemapUrl;
}
} else {
- log.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
error: result.reason,
});
}
diff --git a/packages/utils/src/internals/system-info/cpu-info.ts b/packages/utils/src/internals/system-info/cpu-info.ts
index aa5b4826d..a2f3f5289 100644
--- a/packages/utils/src/internals/system-info/cpu-info.ts
+++ b/packages/utils/src/internals/system-info/cpu-info.ts
@@ -2,7 +2,7 @@ import { execSync } from 'node:child_process';
import { readFile } from 'node:fs/promises';
import os from 'node:os';
-import log from '@apify/log';
+import type { CrawleeLogger } from '@crawlee/types';
import { getCgroupsVersion } from '../general.js';
@@ -55,16 +55,14 @@ export function getCurrentCpuTicks() {
}
/**
- * Reads the linux tick rate
- * @returns the number of ticks per second
+ * Reads the linux tick rate.
+ * @returns the number of ticks per second, or `null` if detection failed
*/
-function getClockTicks(): number {
+function getClockTicks(): number | null {
try {
- const result = execSync('getconf CLK_TCK').toString().trim();
- return parseInt(result, 10);
- } catch (err) {
- log.warningOnce('Failed to get clock ticks; defaulting to 100');
- return 100;
+ return parseInt(execSync('getconf CLK_TCK').toString().trim(), 10);
+ } catch {
+ return null;
}
}
@@ -192,7 +190,11 @@ let previousSample: CpuSample = { containerUsage: 0, systemUsage: 0 };
* @returns a number between 0 and 1 for the cpu load
* @internal
*/
-export async function getCurrentCpuTicksV2(containerized = false): Promise<number> {
+export async function getCurrentCpuTicksV2(
+ options: { containerized?: boolean; logger?: CrawleeLogger } = {},
+): Promise<number> {
+ const { containerized = false, logger } = options;
+
try {
// if not containerized
if (!containerized) {
@@ -200,13 +202,19 @@ export async function getCurrentCpuTicksV2(containerized = false): Promise<numbe
return getCurrentCpuTicks();
}
if (!CLOCK_TICKS_CHECKED) {
- CLOCK_TICKS_PER_SECOND = getClockTicks();
+ const ticks = getClockTicks();
+ if (ticks === null) {
+ CLOCK_TICKS_PER_SECOND = 100;
+ logger?.warning('Failed to get clock ticks; defaulting to 100. CPU metrics may be inaccurate.');
+ } else {
+ CLOCK_TICKS_PER_SECOND = ticks;
+ }
CLOCK_TICKS_CHECKED = true;
}
const cgroupsVersion = await getCgroupsVersion();
// if cgroup is not detected, return bare metal cpu limit
if (cgroupsVersion === null) {
- log.deprecated(
+ logger?.warningOnce(
'Your environment is containerized, but your system does not support cgroups.\n' +
"If you're running containers with limited cpu, cpu auto-scaling will not work properly.",
);
@@ -235,7 +243,7 @@ export async function getCurrentCpuTicksV2(containerized = false): Promise<numbe
return ((containerDelta / systemDelta) * numCpus) / cpuAllowance;
} catch (err) {
// if anything fails, default to bare metal metrics
- log.exception(err as Error, 'Cpu snapshot failed.');
+ logger?.warning('Cpu snapshot failed, falling back to bare-metal metrics.', { error: err as Error });
return getCurrentCpuTicks();
}
}
diff --git a/packages/utils/src/internals/system-info/memory-info.ts b/packages/utils/src/internals/system-info/memory-info.ts
index 855fde459..bbbf24f23 100644
--- a/packages/utils/src/internals/system-info/memory-info.ts
+++ b/packages/utils/src/internals/system-info/memory-info.ts
@@ -2,7 +2,7 @@ import { execSync } from 'node:child_process';
import { readFile } from 'node:fs/promises';
import { freemem, totalmem } from 'node:os';
-import log from '@apify/log';
+import type { CrawleeLogger } from '@crawlee/types';
import { getCgroupsVersion, isLambda } from '../general.js';
import { psTree } from './ps-tree.js';
@@ -49,7 +49,10 @@ export interface MemoryInfo {
* @returns An object containing the free and used memory metrics.
* @internal
*/
-export async function getMemoryInfo(containerized = false): Promise<MemoryInfo> {
+export async function getMemoryInfo(
+ options: { containerized?: boolean; logger?: CrawleeLogger } = {},
+): Promise<MemoryInfo> {
+ const { containerized = false, logger } = options;
let mainProcessBytes = -1;
let childProcessesBytes = 0;
@@ -88,8 +91,6 @@ export async function getMemoryInfo(containerized = false): Promise<MemoryInfo>
totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE!, 10) * 1000000;
usedBytes = mainProcessBytes + childProcessesBytes;
freeBytes = totalBytes - usedBytes;
-
- log.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
} else if (containerized) {
// When running inside a container, use container memory limits
@@ -122,8 +123,7 @@ export async function getMemoryInfo(containerized = false): Promise<MemoryInfo>
usedBytes = parseInt(usedBytesStr, 10);
freeBytes = totalBytes - usedBytes;
} catch (err) {
- // log.deprecated logs a warning only once
- log.deprecated(
+ logger?.warningOnce(
'Your environment is containerized, but your system does not support memory cgroups. ' +
"If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" +
`Cause: ${(err as Error).message}`,
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index 260c351bc..33273ea06 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -1,8 +1,6 @@
import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
-
-import log from '@apify/log';
+import { beforeEach, describe, expect, it } from 'vitest';
import type { SitemapUrl } from '../src/internals/sitemap.js';
import { discoverValidSitemaps, parseSitemap, Sitemap } from '../src/internals/sitemap.js';
@@ -417,14 +415,16 @@ describe('Sitemap', () => {
});
it('keeps quiet if autodetection does not find anything', async () => {
- const spy = vi.spyOn(log, 'warning');
+ const logger = { warning: vi.fn(), warningOnce: vi.fn() } as any;
const sitemap = await Sitemap.tryCommonNames('http://not-exists-2.com/arbitrary_url?search=xyz', undefined, {
httpClient: new FetchHttpClient(),
+ logger,
});
expect(sitemap.urls).toHaveLength(0);
- expect(spy).not.toHaveBeenCalled();
+ expect(logger.warning).not.toHaveBeenCalled();
+ expect(logger.warningOnce).not.toHaveBeenCalled();
});
it('handles sitemap.txt correctly', async () => {
diff --git a/test/stagehand-crawler/stagehand-controller.test.ts b/test/stagehand-crawler/stagehand-controller.test.ts
index 96e3d86dc..43e34f31d 100644
--- a/test/stagehand-crawler/stagehand-controller.test.ts
+++ b/test/stagehand-crawler/stagehand-controller.test.ts
@@ -1,4 +1,4 @@
-import log from '@apify/log';
+import { serviceLocator } from '@crawlee/core';
import { StagehandController } from '../../packages/stagehand-crawler/src/internals/stagehand-controller.js';
import type { StagehandPlugin } from '../../packages/stagehand-crawler/src/internals/stagehand-plugin.js';
@@ -140,8 +140,9 @@ describe('StagehandController', () => {
const controller = new StagehandController(mockPlugin, stagehandInstances);
(controller as any).browser = mockBrowser;
- // Mock log.error to prevent output and verify it's called
- const logErrorSpy = vi.spyOn(log, 'error').mockImplementation(() => {});
+ // Mock logger.error to prevent output and verify it's called
+ const logger = serviceLocator.getLogger();
+ const logErrorSpy = vi.spyOn(logger, 'error').mockImplementation(() => {});
// Should not throw
await expect((controller as any)._close()).resolves.toBeUndefined();
diff --git a/test/utils/cpu-infoV2.test.ts b/test/utils/cpu-infoV2.test.ts
index 5f76af90c..8786e3c3e 100644
--- a/test/utils/cpu-infoV2.test.ts
+++ b/test/utils/cpu-infoV2.test.ts
@@ -299,7 +299,7 @@ describe('getCpuInfo()', () => {
{ times: { user: 200, nice: 0, sys: 100, idle: 100, irq: 0 } },
] as os.CpuInfo[]);
// Initially, previousSample is { containerUsage: 0, systemUsage: 0 }.
- const result = await getCurrentCpuTicksV2(true);
+ const result = await getCurrentCpuTicksV2({ containerized: true });
// Calculation:
// containerDelta = 1000000, systemDelta = 3000000000, numCpus = 2, cpuAllowance = 2.
// So: ((1000000000 / 3000000000) * 2) / 2 ≈ 0.3333
@@ -307,6 +307,34 @@ describe('getCpuInfo()', () => {
cpusMock.mockRestore();
});
+ test('logs warningOnce when containerized but cgroups not available', async () => {
+ getCgroupsVersionSpy.mockResolvedValueOnce(null);
+ const cpusMock = vitest
+ .spyOn(os, 'cpus')
+ .mockReturnValue([{ times: { user: 100, nice: 0, sys: 50, idle: 50, irq: 0 } }] as os.CpuInfo[]);
+ const logger = { warningOnce: vitest.fn(), warning: vitest.fn() } as any;
+ await getCurrentCpuTicksV2({ containerized: true, logger });
+ expect(logger.warningOnce).toHaveBeenCalledOnce();
+ expect(logger.warningOnce.mock.calls[0][0]).toContain('does not support cgroups');
+ cpusMock.mockRestore();
+ });
+
+ test('logs warning when cgroup cpu snapshot fails', async () => {
+ getCgroupsVersionSpy.mockResolvedValueOnce('V1');
+ readFileSpy.mockRejectedValue(new Error('permission denied'));
+ const cpusMock = vitest
+ .spyOn(os, 'cpus')
+ .mockReturnValue([{ times: { user: 100, nice: 0, sys: 50, idle: 50, irq: 0 } }] as os.CpuInfo[]);
+ const logger = { warningOnce: vitest.fn(), warning: vitest.fn() } as any;
+ const result = await getCurrentCpuTicksV2({ containerized: true, logger });
+ expect(logger.warning).toHaveBeenCalledOnce();
+ expect(logger.warning.mock.calls[0][0]).toContain('Cpu snapshot failed');
+ expect(logger.warning.mock.calls[0][1]).toHaveProperty('error');
+ // Should fall back to bare metal
+ expect(result).toBeCloseTo(0.75);
+ cpusMock.mockRestore();
+ });
+
test('returns bare metal cpu ticks when containerized but no cgroup quota', async () => {
getCgroupsVersionSpy.mockResolvedValueOnce('V1');
// For V1, a quota of "-1" signals no limit → quota becomes null.
@@ -315,7 +343,7 @@ describe('getCpuInfo()', () => {
const cpusMock = vitest
.spyOn(os, 'cpus')
.mockReturnValue([{ times: { user: 300, nice: 0, sys: 150, idle: 150, irq: 0 } }] as os.CpuInfo[]);
- const result = await getCurrentCpuTicksV2(true);
+ const result = await getCurrentCpuTicksV2({ containerized: true });
// For one CPU: total = 300+0+150+150 = 600, idle = 150 → load = 0.75.
expect(result).toBeCloseTo(0.75);
cpusMock.mockRestore();
diff --git a/test/utils/memory-infoV2.test.ts b/test/utils/memory-infoV2.test.ts
index 1623336a9..9094cd4ee 100644
--- a/test/utils/memory-infoV2.test.ts
+++ b/test/utils/memory-infoV2.test.ts
@@ -73,7 +73,7 @@ describe('getMemoryInfoV2()', () => {
throw new Error(`Unexpected path ${path}`);
});
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
@@ -130,7 +130,7 @@ describe('getMemoryInfoV2()', () => {
let browser!: Awaited<ReturnType<typeof launchPuppeteer>>;
try {
browser = await launchPuppeteer();
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
@@ -145,6 +145,44 @@ describe('getMemoryInfoV2()', () => {
}
});
+ test('logs warningOnce when containerized but cgroups not available', async () => {
+ getCgroupsVersionSpy.mockResolvedValueOnce(null);
+ freememSpy.mockReturnValueOnce(222);
+ totalmemSpy.mockReturnValueOnce(333);
+
+ const logger = { warningOnce: vitest.fn(), warning: vitest.fn() } as any;
+ const data = await getMemoryInfo({ containerized: true, logger });
+
+ expect(logger.warningOnce).toHaveBeenCalledOnce();
+ expect(logger.warningOnce.mock.calls[0][0]).toContain('does not support memory cgroups');
+ // Should fall back to host memory
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ });
+
+ test('logs warningOnce when cgroup memory files are unreadable', async () => {
+ getCgroupsVersionSpy.mockResolvedValueOnce('V1');
+ accessSpy.mockResolvedValueOnce();
+ readFileSpy.mockRejectedValue(new Error('permission denied'));
+ freememSpy.mockReturnValueOnce(222);
+ totalmemSpy.mockReturnValueOnce(333);
+
+ const logger = { warningOnce: vitest.fn(), warning: vitest.fn() } as any;
+ const data = await getMemoryInfo({ containerized: true, logger });
+
+ expect(logger.warningOnce).toHaveBeenCalledOnce();
+ expect(logger.warningOnce.mock.calls[0][0]).toContain('permission denied');
+ // Should fall back to host memory
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ });
+
test('works with cgroup V1 with LIMITED memory', async () => {
getCgroupsVersionSpy.mockResolvedValueOnce('V1');
accessSpy.mockResolvedValueOnce();
@@ -161,7 +199,7 @@ describe('getMemoryInfoV2()', () => {
throw new Error(`Unexpected path ${path}`);
});
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
freeBytes: 222,
@@ -187,7 +225,7 @@ describe('getMemoryInfoV2()', () => {
totalmemSpy.mockReturnValueOnce(333);
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
freeBytes: 222,
@@ -211,7 +249,7 @@ describe('getMemoryInfoV2()', () => {
throw new Error(`Unexpected path ${path}`);
});
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
freeBytes: 222,
@@ -237,7 +275,7 @@ describe('getMemoryInfoV2()', () => {
totalmemSpy.mockReturnValueOnce(333);
- const data = await getMemoryInfo(true);
+ const data = await getMemoryInfo({ containerized: true });
expect(data).toMatchObject({
totalBytes: 333,
freeBytes: 222,