Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: Exclude internal symbols from public API docs (#3882)

Jan Buchar83003fbd3b3f
diff --git a/docs/public-api/README.md b/docs/public-api/README.md
index da55871b7..2d0a2f61d 100644
--- a/docs/public-api/README.md
+++ b/docs/public-api/README.md
@@ -23,7 +23,12 @@ They are produced by [API Extractor](https://api-extractor.com/) from the built
## Notes
-- `docs/public-api/temp/` holds intermediate reports and is git-ignored.
+- The reports are generated as API Extractor's **`public`** variant, so symbols tagged
+ `@internal` (`@alpha`/`@beta` too) are excluded — only `@public` surface is tracked.
+ The generator stages the variant as `<name>.public.api.md` under `temp/` and promotes it
+ onto the committed `<name>.api.md`, so the tracked filenames stay stable.
+- `docs/public-api/temp/` holds intermediate reports (including the staged `.public.api.md`
+ files) and is git-ignored.
- `@crawlee/cli` and `@crawlee/templates` are deliberately excluded — they are tooling
(a CLI binary and project scaffolding), not an importable API where we promise BC. The
exclude list lives in `scripts/api-extractor/run.ts`.
@@ -32,6 +37,6 @@ They are produced by [API Extractor](https://api-extractor.com/) from the built
afterwards) because API Extractor's AST walker trips over some of them; a small number
of packages additionally need a sanitized-mirror fallback. See the comments in
`scripts/api-extractor/run.ts` for details.
-- Symbols tagged `@internal` still show up here if they are exported. Shrinking these
- reports (hiding internals, e.g. via `@internal` + a trimmed rollup, or by not exporting
- them at all) is the goal tracked in issue #3109.
+- These reports now cover only the `@public` surface. Further shrinking them — genuinely
+ hiding class internals (untagged `protected`/`_`-prefixed members) rather than merely
+ tagging them — is the goal tracked in issue #3109.
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index d63a56d53..16e8c55cc 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/basic"
+## Public API Report File for "@crawlee/basic"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -61,8 +61,6 @@ import { TimeoutError } from '@apify/timeout';
// @public
export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
- // @internal
- protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
// (undocumented)
protected additionalHttpErrorStatusCodes: Set<number>;
addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
@@ -90,8 +88,6 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
protected delayRequest(request: Request_2, source: IRequestManager): boolean;
// (undocumented)
protected domainAccessedTime: Map<string, number>;
- // @internal
- protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request_2<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
// (undocumented)
protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;
diff --git a/docs/public-api/crawlee-browser-pool.api.md b/docs/public-api/crawlee-browser-pool.api.md
index 804a628d4..9ba695c44 100644
--- a/docs/public-api/crawlee-browser-pool.api.md
+++ b/docs/public-api/crawlee-browser-pool.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/browser-pool"
+## Public API Report File for "@crawlee/browser-pool"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -25,9 +25,6 @@ import type * as PuppeteerTypes from 'puppeteer';
import QuickLRU from 'quick-lru';
import { TypedEmitter } from 'tiny-typed-emitter';
-// @internal (undocumented)
-export const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
-
// @public (undocumented)
export interface AnonymizeProxySugarOptions {
// (undocumented)
@@ -148,18 +145,12 @@ export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrar
name: string;
// (undocumented)
proxyUrl?: string;
- // @internal
- remoteConnection?: RemoteConnection;
- // @internal
- remoteConnectionParameters?: RemoteConnectionParameters;
// (undocumented)
protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
// (undocumented)
useIncognitoPages: boolean;
// (undocumented)
userDataDir?: string;
- // @internal
- useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
}
// @public (undocumented)
@@ -312,16 +303,6 @@ export interface CommonLibrary {
product?: string;
}
-// @internal (undocumented)
-export interface CommonPage {
- // (undocumented)
- close(...args: unknown[]): Promise<unknown>;
- // (undocumented)
- evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
- // (undocumented)
- url(): string | Promise<string>;
-}
-
// @public
export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
@@ -408,8 +389,6 @@ export class LaunchContext<Library extends CommonLibrary = CommonLibrary, Librar
launchOptions: LibraryOptions;
set proxyUrl(url: string | undefined);
get proxyUrl(): string | undefined;
- // @internal
- _remoteToken?: number;
// (undocumented)
useIncognitoPages: boolean;
// (undocumented)
@@ -463,8 +442,6 @@ export class PlaywrightBrowser extends EventEmitter {
newContext(): Promise<never>;
// (undocumented)
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
- // @internal (undocumented)
- _setBrowserType(browserType: BrowserType): void;
// (undocumented)
startTracing(): Promise<never>;
// (undocumented)
@@ -603,17 +580,6 @@ export abstract class RemoteBrowserProvider<TContext extends Record<string, unkn
release(_context: TContext): Promise<void>;
}
-// @internal
-export interface RemoteConnection {
- release(token: number): Promise<void>;
- resolve(options?: {
- proxyUrl?: string;
- }): Promise<{
- url: string;
- token: number;
- }>;
-}
-
// @public
export interface RemoteConnectionParameters {
connectOptions?: Record<string, unknown>;
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 1b0fa60ae..7c253e07e 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/browser"
+## Public API Report File for "@crawlee/browser"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.md
index 38bc6f886..4624e9c48 100644
--- a/docs/public-api/crawlee-cheerio.api.md
+++ b/docs/public-api/crawlee-cheerio.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/cheerio"
+## Public API Report File for "@crawlee/cheerio"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -42,9 +42,6 @@ export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContex
}>;
}
-// @internal (undocumented)
-export function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
-
// @public (undocumented)
export interface CheerioCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 5f81762a6..852662871 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/core"
+## Public API Report File for "@crawlee/core"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -70,9 +70,6 @@ export interface AddRequestsBatchedResult {
waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
}
-// @internal
-export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
-
// @public
export class ApifyLogAdapter extends BaseCrawleeLogger {
constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
@@ -82,9 +79,6 @@ export class ApifyLogAdapter extends BaseCrawleeLogger {
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
}
-// @internal
-export function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
-
// @public
export function assertJsonSerializable<T>(item: T, index?: number): void;
@@ -170,19 +164,9 @@ export abstract class BaseCrawleeLogger implements CrawleeLogger {
warningOnce(message: string): void;
}
-// @internal
-export function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
- run: <T>(fn: () => T) => T;
- enterScope: () => void;
- exitScope: () => void;
-};
-
// @public (undocumented)
export const BLOCKED_STATUS_CODES: number[];
-// @internal
-export function browserPoolCookieToToughCookie(cookieObject: Cookie, maxAgeSecs?: number): Cookie_2;
-
// @public
export const checkStorageAccess: () => void | undefined;
@@ -196,9 +180,6 @@ export interface ClientInfo {
limitRatio: number;
}
-// @internal
-export type ClientLoadSignal = ReturnType<typeof createClientLoadSignal>;
-
// @public (undocumented)
export interface ClientLoadSignalOptions {
// (undocumented)
@@ -299,12 +280,6 @@ export class CookieParseError extends Error {
readonly cookieHeaderString: unknown;
}
-// @internal (undocumented)
-export function cookieStringToToughCookie(cookieString: string): Cookie | null;
-
-// @internal
-export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;
-
// @public (undocumented)
export interface CpuLoadSignalOptions {
// (undocumented)
@@ -374,12 +349,6 @@ export function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<LoadSig
getSample(sampleDurationMillis?: number): CpuSnapshot[];
};
-// @internal
-export function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
-
-// @internal
-export function createDualIterable<TItem, TRawPage>(options: DualIterableOptions<TItem, TRawPage>): AsyncIterable<TItem> & Promise<TItem[]>;
-
// @public
export function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<LoadSignal, "getSample"> & {
store: SnapshotStore<EventLoopSnapshot>;
@@ -405,8 +374,6 @@ export class CriticalError extends NonRetryableError {
// @public
export class Dataset<Data extends Dictionary = Dictionary> {
[Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
- // @internal
- constructor(options: DatasetOptions, config?: Configuration);
// (undocumented)
backend: DatasetBackend<Data>;
// (undocumented)
@@ -440,9 +407,6 @@ export class Dataset<Data extends Dictionary = Dictionary> {
values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
}
-// @internal (undocumented)
-export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
-
// @public
export interface DatasetConsumer<Data> {
// (undocumented)
@@ -486,18 +450,6 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
// @public (undocumented)
export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
- // @internal (undocumented)
- clean?: boolean;
- // @internal (undocumented)
- format?: string;
- // @internal
- limit?: number;
- // @internal (undocumented)
- offset?: number;
- // @internal (undocumented)
- skipEmpty?: boolean;
- // @internal (undocumented)
- skipHidden?: boolean;
}
// @public
@@ -537,17 +489,8 @@ export interface DefaultStorageIdentifier {
name?: never;
}
-// @internal
-export function deserializeArray<T extends string | Buffer>(compressedData: Buffer | Uint8Array): Promise<T[]>;
-
export { Dictionary }
-// @internal (undocumented)
-export interface DualIterableOptions<TItem, TRawPage> {
- createPages: () => AsyncGenerator<TRawPage>;
- extractItems: (page: TRawPage) => TItem[];
-}
-
// @public
export function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
requestManager: {
@@ -664,9 +607,6 @@ export interface ErrorTrackerOptions {
// @public
export function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
-// @internal
-export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;
-
// @public (undocumented)
export interface EventLoopLoadSignalOptions {
// (undocumented)
@@ -700,18 +640,12 @@ export abstract class EventManager {
protected intervals: Intervals;
// (undocumented)
isInitialized(): boolean;
- // @internal (undocumented)
- listenerCount(event: EventTypeName): number;
- // @internal (undocumented)
- listeners(event: EventTypeName): (() => Promise<unknown>)[];
// (undocumented)
protected log: CrawleeLogger;
// (undocumented)
off(event: EventTypeName, listener?: (...args: any[]) => any): void;
// (undocumented)
on(event: EventTypeName, listener: (...args: any[]) => any): void;
- // @internal (undocumented)
- waitForAllListenersToComplete(): Promise<boolean>;
}
// @public (undocumented)
@@ -801,15 +735,6 @@ export interface FinalStatistics {
retryHistogram: number[];
}
-// @internal (undocumented)
-export function getCookiesFromResponse(response: Response): Cookie_2[];
-
-// @internal
-export function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
-
-// @internal
-export function getRequestId(uniqueKey: string): string;
-
// @public (undocumented)
export type GetUserDataFromRequest<T> = T extends Request_2<infer Y> ? Y : never;
@@ -821,20 +746,6 @@ export type GlobObject = {
glob: string;
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
-// @internal
-export function handleRequestTimeout(input: {
- session?: ISession;
- errorMessage: string;
-}): void;
-
-// @internal (undocumented)
-export interface InternalSource {
- // (undocumented)
- regex?: RegExp;
- // (undocumented)
- requestsFromUrl: string;
-}
-
// @public
export interface IRequestLoader {
[Symbol.asyncIterator](): AsyncGenerator<Request_2>;
@@ -860,9 +771,6 @@ export interface IRequestManager extends IRequestLoader {
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
-// @internal (undocumented)
-export type IsAny<T> = 0 extends 1 & T ? true : false;
-
// @public
export interface IStorage {
// (undocumented)
@@ -882,10 +790,6 @@ export interface KeyConsumer {
// @public
export class KeyValueStore {
[Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
- // @internal
- constructor(options: KeyValueStoreOptions, config?: Configuration);
- // @internal (undocumented)
- clearCache(): void;
// (undocumented)
readonly config: Configuration;
drop(): Promise<void>;
@@ -945,11 +849,6 @@ export interface KeyValueStoreStats {
writeCount: number;
}
-// @internal (undocumented)
-export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
- request: LoadedRequest<Context['request']>;
-} & Omit<Context, 'request'>;
-
// @public (undocumented)
export type LoadedRequest<R extends Request_2> = WithRequired<R, 'id' | 'loadedUrl'>;
@@ -975,12 +874,8 @@ export class LocalEventManager extends EventManager {
constructor(options: LocalEventManagerOptions);
// (undocumented)
close(): Promise<void>;
- // @internal (undocumented)
- emitSystemInfoEvent(intervalCallback: () => unknown): Promise<void>;
static fromConfig(config?: Configuration): LocalEventManager;
init(): Promise<void>;
- // @internal (undocumented)
- isContainerizedWrapper(): Promise<boolean>;
}
// @public (undocumented)
@@ -1005,21 +900,14 @@ export { LogLevel }
// @public (undocumented)
export const MAX_POOL_SIZE = 1000;
-// @internal
-export const MAX_QUERIES_FOR_CONSISTENCY = 6;
-
// @public
export class MemoryLoadSignal implements LoadSignal {
constructor(options: MemoryLoadSignalOptions);
getMemorySnapshots(): MemorySnapshot[];
// (undocumented)
getSample(sampleDurationMillis?: number): MemorySnapshot[];
- // @internal (undocumented)
- _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
// (undocumented)
readonly name = "memInfo";
- // @internal (undocumented)
- _onSystemInfo(systemInfo: SystemInfo): void;
// (undocumented)
readonly overloadedRatio: number;
// (undocumented)
@@ -1077,9 +965,6 @@ export interface MemoryStorageOptions {
logger?: CrawleeLogger;
}
-// @internal
-export function mergeCookies(url: string, sourceCookies: string[]): string;
-
// @public (undocumented)
export class MissingRouteError extends CriticalError {
}
@@ -1171,12 +1056,6 @@ export interface PushErrorMessageOptions {
omitStack?: boolean;
}
-// @internal (undocumented)
-export const QUERY_HEAD_BUFFER = 3;
-
-// @internal
-export const QUERY_HEAD_MIN_LENGTH = 100;
-
export { QueueOperationInfo }
// @public (undocumented)
@@ -1224,14 +1103,10 @@ export type RegExpObject = {
// @public
class Request_2<UserData extends Dictionary = Dictionary> {
constructor(options: RequestOptions<UserData>);
- // @internal (undocumented)
- static computeUniqueKey(input: ComputeUniqueKeyOptions): string;
get crawlDepth(): number;
set crawlDepth(value: number);
errorMessages: string[];
handledAt?: string;
- // @internal (undocumented)
- static hashPayload(payload: BinaryLike): string;
headers?: Record<string, string>;
id?: string;
intoFetchAPIRequest(): Request;
@@ -1305,8 +1180,6 @@ export class RequestList implements IRequestLoader {
protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;
protected _addRequest(source: RequestListSource): void;
protected _addRequestsFromSources(): Promise<void>;
- // @internal
- areRequestsPersisted: boolean;
protected _ensureInProgress(uniqueKey: string): void;
protected _ensureIsInitialized(): void;
protected _ensureUniqueKeyValid(uniqueKey: string): void;
@@ -1320,14 +1193,10 @@ export class RequestList implements IRequestLoader {
protected _getPersistedState<T>(key: string): Promise<T>;
getState(): RequestListState;
getTotalCount(): Promise<number>;
- // @internal
- inProgress: Set<string>;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
isFinished(): Promise<boolean>;
- // @internal
- isStatePersisted: boolean;
protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;
// (undocumented)
markRequestAsHandled(request: Request_2): Promise<void>;
@@ -1335,8 +1204,6 @@ export class RequestList implements IRequestLoader {
protected _persistRequests(): Promise<void>;
// (undocumented)
persistState(): Promise<void>;
- // @internal
- requests: (Request_2 | RequestOptions)[];
protected _restoreState(state?: RequestListState): void;
teardown(): Promise<void>;
toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
@@ -1344,8 +1211,6 @@ export class RequestList implements IRequestLoader {
// @public (undocumented)
export interface RequestListOptions {
- // @internal (undocumented)
- config?: Configuration;
httpClient?: BaseHttpClient;
keepDuplicateUrls?: boolean;
persistRequestsKey?: string;
@@ -1399,25 +1264,15 @@ export class RequestManagerTandem implements IRequestManager {
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
alwaysEnqueue?: boolean;
crawlDepth?: number;
- // @internal (undocumented)
- enqueueStrategy?: EnqueueLinksOptions['strategy'];
- // @internal (undocumented)
- handledAt?: string;
headers?: Record<string, string>;
- // @internal (undocumented)
- id?: string;
keepUrlFragment?: boolean;
label?: string;
- // @internal (undocumented)
- lockExpiresAt?: Date;
maxRetries?: number;
method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
noRetry?: boolean;
payload?: string;
sessionId?: string;
skipNavigation?: boolean;
- // @internal
- skippedReason?: SkippedRequestReason;
uniqueKey?: string;
url: string;
useExtendedUniqueKey?: boolean;
@@ -1428,8 +1283,6 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
export class RequestQueue implements IStorage, IRequestManager {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
- // @internal
- constructor(options: RequestQueueOptions, config?: Configuration);
protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
@@ -1477,18 +1330,8 @@ export class RequestQueue implements IStorage, IRequestManager {
get stats(): RequestQueueStats;
}
-// @internal (undocumented)
-export interface RequestQueueOperationInfo extends QueueOperationInfo {
- // (undocumented)
- forefront: boolean;
- // (undocumented)
- uniqueKey: string;
-}
-
// @public (undocumented)
export interface RequestQueueOperationOptions {
- // @internal
- cache?: boolean;
forefront?: boolean;
}
@@ -1506,9 +1349,6 @@ export interface RequestQueueStats {
writeCount: number;
}
-// @internal (undocumented)
-export const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
-
// @public (undocumented)
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
@@ -1557,21 +1397,6 @@ export class RequestValidationError extends NonRetryableError {
readonly label: string | symbol;
}
-// @internal (undocumented)
-export interface ResolveBaseUrl {
- // (undocumented)
- enqueueStrategy?: EnqueueLinksOptions['strategy'];
- // (undocumented)
- finalRequestUrl?: string;
- // (undocumented)
- originalRequestUrl: string;
- // (undocumented)
- userProvidedBaseUrl?: string;
-}
-
-// @internal
-export function resolveBaseUrlForEnqueueLinksFiltering(input: ResolveBaseUrl): string | undefined;
-
// @public (undocumented)
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
@@ -1619,8 +1444,6 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
// (undocumented)
static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
getHandler(label?: string | symbol): (ctx: Context) => Awaitable_2<void>;
- // @internal
- getSchema(label?: string | symbol): StandardSchemaV1 | undefined;
use(middleware: (ctx: Context) => Awaitable_2<void>): void;
}
@@ -1655,9 +1478,6 @@ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
[Label in Extract<keyof Schemas, string>]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary ? StandardSchemaV1.InferOutput<Schemas[Label]> : Dictionary;
};
-// @internal
-export function serializeArray<T>(data: T[]): Promise<Buffer>;
-
// @public
export function serializeValue(value: unknown, contentType?: string): {
value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
@@ -1831,8 +1651,6 @@ export class SessionPool implements ISessionPool {
export interface SessionPoolOptions {
createSessionFunction?: CreateSession;
id?: string | number;
- // @internal (undocumented)
- log?: CrawleeLogger;
maxPoolSize?: number;
persistenceOptions?: PersistenceOptions;
persistStateKey?: string;
@@ -1856,8 +1674,6 @@ export class SitemapRequestLoader implements IRequestLoader {
getPendingCount(): Promise<number>;
// (undocumented)
getTotalCount(): Promise<number>;
- // @internal
- inProgress: Set<string>;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
@@ -1975,14 +1791,8 @@ export class Snapshotter {
// @public (undocumented)
export interface SnapshotterOptions {
- // @internal (undocumented)
- client?: StorageBackend;
clientSnapshotIntervalSecs?: number;
- // @internal (undocumented)
- config?: Configuration;
eventLoopSnapshotIntervalSecs?: number;
- // @internal (undocumented)
- log?: CrawleeLogger;
maxBlockedMillis?: number;
maxClientErrors?: number;
maxUsedMemoryRatio?: number;
@@ -1995,9 +1805,6 @@ export type Source = (Partial<RequestOptions> & {
regex?: RegExp;
}) | Request_2;
-// @internal (undocumented)
-export const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
-
// @public
export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
// (undocumented)
@@ -2020,8 +1827,6 @@ export interface StatisticPersistedState extends Omit<StatisticState, 'statsPers
// @public
export class Statistics {
- // @internal
- constructor(options?: StatisticsOptions);
calculate(): {
requestAvgFailedDurationMillis: number;
requestAvgFinishedDurationMillis: number;
@@ -2106,9 +1911,6 @@ export interface StatisticState {
statsPersistedAt: Date | string | null;
}
-// @internal
-export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
-
export { StorageBackend }
export { StorageIdentifier }
@@ -2142,16 +1944,10 @@ export class StorageStatsTracker<T extends Record<keyof T, number>> {
export interface SystemInfo {
// (undocumented)
clientInfo: ClientInfo;
- // @internal
- cpuCurrentUsage?: number;
// (undocumented)
cpuInfo: ClientInfo;
- // @internal
- createdAt?: Date;
// (undocumented)
eventLoopInfo: ClientInfo;
- // @internal
- isCpuOverloaded?: boolean;
isSystemIdle: boolean;
loadSignalInfo?: Record<string, ClientInfo>;
// (undocumented)
@@ -2181,9 +1977,6 @@ export interface SystemStatusOptions {
snapshotter?: Snapshotter;
}
-// @internal
-export function toughCookieToBrowserPoolCookie(toughCookie: Cookie_2): Cookie;
-
export { tryAbsoluteURL }
// @public (undocumented)
@@ -2205,48 +1998,9 @@ export interface UseStateOptions {
keyValueStoreName?: string | null;
}
-// @internal (undocumented)
-export function validateGlobPattern(glob: string): string;
-
-// @internal
-export function validateUserData(label: string | symbol, schema: StandardSchemaV1, userData: unknown): Promise<Dictionary>;
-
-// @internal (undocumented)
-export const validators: {
- browserPage: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
- proxyConfiguration: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
- requestList: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
- requestQueue: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
- browserPool: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
- sessionPool: (value: Dictionary) => {
- validator: boolean;
- message: (label: string) => string;
- };
-};
-
// @public
export const withCheckedStorageAccess: <T>(checkFunction: () => void, callback: () => Awaitable_2<T>) => Promise<T>;
-// @internal (undocumented)
-export type WithRequired<T, K extends keyof T> = T & {
- [P in K]-?: T[P];
-};
-
// (No @packageDocumentation comment for this package)
```
diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.md
index bba73cf4c..e86f39013 100644
--- a/docs/public-api/crawlee-fs-storage.api.md
+++ b/docs/public-api/crawlee-fs-storage.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/fs-storage"
+## Public API Report File for "@crawlee/fs-storage"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-got-scraping-client.api.md b/docs/public-api/crawlee-got-scraping-client.api.md
index 5987bfffe..fb497938b 100644
--- a/docs/public-api/crawlee-got-scraping-client.api.md
+++ b/docs/public-api/crawlee-got-scraping-client.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/got-scraping-client"
+## Public API Report File for "@crawlee/got-scraping-client"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-http-client.api.md b/docs/public-api/crawlee-http-client.api.md
index b4d3f8218..98c0a2e64 100644
--- a/docs/public-api/crawlee-http-client.api.md
+++ b/docs/public-api/crawlee-http-client.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/http-client"
+## Public API Report File for "@crawlee/http-client"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 57126d66d..da7d20e74 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/http"
+## Public API Report File for "@crawlee/http"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -227,22 +227,6 @@ JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserDat
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
-// @internal (undocumented)
-export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
- body: string | Buffer;
- contentType: {
- type: string;
- encoding: BufferEncoding;
- };
- json: JSONData;
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-}
-
-// @internal (undocumented)
-export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
-
// @public
export function MinimumSpeedStream(input: {
minSpeedKbps: number;
diff --git a/docs/public-api/crawlee-impit-client.api.md b/docs/public-api/crawlee-impit-client.api.md
index 4f5db677e..5bdbb61f8 100644
--- a/docs/public-api/crawlee-impit-client.api.md
+++ b/docs/public-api/crawlee-impit-client.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/impit-client"
+## Public API Report File for "@crawlee/impit-client"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
index c75092752..c5befb5cc 100644
--- a/docs/public-api/crawlee-jsdom.api.md
+++ b/docs/public-api/crawlee-jsdom.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/jsdom"
+## Public API Report File for "@crawlee/jsdom"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -45,9 +45,6 @@ export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCr
// @public (undocumented)
export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
-// @internal (undocumented)
-export function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
-
// @public (undocumented)
export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.md
index b138303de..c102e6795 100644
--- a/docs/public-api/crawlee-linkedom.api.md
+++ b/docs/public-api/crawlee-linkedom.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/linkedom"
+## Public API Report File for "@crawlee/linkedom"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -50,9 +50,6 @@ export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedConte
}>;
}
-// @internal (undocumented)
-export function linkedomCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
-
// @public (undocumented)
export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {
}
diff --git a/docs/public-api/crawlee-memory-storage.api.md b/docs/public-api/crawlee-memory-storage.api.md
deleted file mode 100644
index 1da99e83d..000000000
--- a/docs/public-api/crawlee-memory-storage.api.md
+++ /dev/null
@@ -1,44 +0,0 @@
-## API Report File for "@crawlee/memory-storage"
-
-> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
-
-```ts
-
-import type { CrawleeLogger } from '@crawlee/types';
-import type { Dictionary } from '@crawlee/types';
-import type * as storage from '@crawlee/types';
-
-// @public (undocumented)
-export class MemoryStorageClient implements storage.StorageClient {
- constructor(options?: MemoryStorageOptions);
- // (undocumented)
- createDatasetClient(options?: storage.CreateDatasetClientOptions): Promise<storage.DatasetClient>;
- // (undocumented)
- createKeyValueStoreClient(options?: storage.CreateKeyValueStoreClientOptions): Promise<storage.KeyValueStoreClient>;
- // (undocumented)
- createRequestQueueClient(options?: storage.CreateRequestQueueClientOptions): Promise<storage.RequestQueueClient>;
- // (undocumented)
- readonly datasetClientCache: DatasetClient[];
- getStorageClientCacheKey(): string;
- // (undocumented)
- readonly keyValueStoreCache: KeyValueStoreClient[];
- // (undocumented)
- readonly logger?: CrawleeLogger;
- purge(): Promise<void>;
- // (undocumented)
- readonly requestQueueCache: RequestQueueClient[];
- // (undocumented)
- setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
- // (undocumented)
- storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
- teardown(): Promise<void>;
-}
-
-// @public (undocumented)
-export interface MemoryStorageOptions {
- logger?: CrawleeLogger;
-}
-
-// (No @packageDocumentation comment for this package)
-
-```
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index 3d2af3dc3..1cd0c8163 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/playwright"
+## Public API Report File for "@crawlee/playwright"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -258,22 +258,6 @@ declare namespace playwrightClickElements {
}
}
-// @internal (undocumented)
-interface PlaywrightContextUtils {
- blockRequests(options?: BlockRequestsOptions): Promise<void>;
- closeCookieModals(): Promise<void>;
- compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
- enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
- handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response_2 | undefined>;
- infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
- injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
- injectJQuery(): Promise<unknown>;
- listDownloads(): Promise<Download[]>;
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
- saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-}
-
// @public
export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response_2, {
browserPlugins: [PlaywrightPlugin];
@@ -430,22 +414,6 @@ declare namespace playwrightUtils {
}
}
-// @internal (undocumented)
-const playwrightUtils_2: {
- injectFile: typeof injectFile;
- injectJQuery: typeof injectJQuery;
- gotoExtended: typeof gotoExtended;
- blockRequests: typeof blockRequests;
- enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
- parseWithCheerio: typeof parseWithCheerio;
- infiniteScroll: typeof infiniteScroll;
- saveSnapshot: typeof saveSnapshot;
- compileScript: typeof compileScript;
- closeCookieModals: typeof closeCookieModals;
- RenderingTypePredictor: typeof RenderingTypePredictor;
- handleCloudflareChallenge: typeof handleCloudflareChallenge;
-};
-
// @public (undocumented)
export type RenderingType = 'clientOnly' | 'static';
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
index 18215bad6..bc6e01783 100644
--- a/docs/public-api/crawlee-puppeteer.api.md
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/puppeteer"
+## Public API Report File for "@crawlee/puppeteer"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -175,22 +175,6 @@ declare namespace puppeteerClickElements {
}
}
-// @internal (undocumented)
-interface PuppeteerContextUtils {
- addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
- blockRequests(options?: BlockRequestsOptions): Promise<void>;
- closeCookieModals(): Promise<void>;
- compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
- enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
- infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
- injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
- injectJQuery(): Promise<unknown>;
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
- removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
- saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-}
-
// @public
export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, {
browserPlugins: [PuppeteerPlugin];
@@ -355,22 +339,6 @@ declare namespace puppeteerUtils {
}
}
-// @internal (undocumented)
-const puppeteerUtils_2: {
- injectFile: typeof injectFile;
- injectJQuery: typeof injectJQuery;
- enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
- blockRequests: typeof blockRequests;
- compileScript: typeof compileScript;
- gotoExtended: typeof gotoExtended;
- addInterceptRequestHandler: typeof addInterceptRequestHandler;
- removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
- infiniteScroll: typeof infiniteScroll;
- saveSnapshot: typeof saveSnapshot;
- parseWithCheerio: typeof parseWithCheerio;
- closeCookieModals: typeof closeCookieModals;
-};
-
// @public
function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
@@ -387,9 +355,6 @@ export interface SaveSnapshotOptions {
screenshotQuality?: number;
}
-// @internal (undocumented)
-function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
-
export * from "@crawlee/browser";
diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
index de99a0ff0..5e5c4fcae 100644
--- a/docs/public-api/crawlee-stagehand.api.md
+++ b/docs/public-api/crawlee-stagehand.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/stagehand"
+## Public API Report File for "@crawlee/stagehand"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index b0435846c..557467f01 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/types"
+## Public API Report File for "@crawlee/types"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
index d7d10e194..9e7e450f9 100644
--- a/docs/public-api/crawlee-utils.api.md
+++ b/docs/public-api/crawlee-utils.api.md
@@ -1,4 +1,4 @@
-## API Report File for "@crawlee/utils"
+## Public API Report File for "@crawlee/utils"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
@@ -16,12 +16,6 @@ import { Element as Element_2 } from 'domhandler';
import type { IncomingMessage } from 'node:http';
import type { SearchParams } from '@crawlee/types';
-// @internal
-export function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
-
-// @internal
-export function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
-
export { Awaitable }
export { Cheerio }
@@ -34,9 +28,6 @@ export type CheerioRoot = CheerioAPI;
// @public (undocumented)
export function chunk<T>(array: readonly T[], chunkSize: number): T[][];
-// @internal
-export function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
-
// @public (undocumented)
export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
@@ -119,24 +110,15 @@ export function getObjectType(value: unknown): string;
// @public
export function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;
-// @internal
-export function inspectValue(value: unknown): string;
-
// @public
const INSTAGRAM_REGEX: RegExp;
// @public
const INSTAGRAM_REGEX_GLOBAL: RegExp;
-// @internal
-export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
-
// @public
export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
-// @internal
-export function isIterable<T>(value: unknown): value is Iterable<T>;
-
// @public
export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
@@ -186,20 +168,6 @@ export interface ParseSitemapOptions {
timeoutMillis?: number;
}
-// @internal
-export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
- // (undocumented)
- [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
-}
-
-// @internal
-export function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
-
-// @internal
-export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
- peek(): Promise<T | undefined>;
-}
-
// @public
function phonesFromText(text: string): string[];
diff --git a/docs/public-api/crawlee.api.md b/docs/public-api/crawlee.api.md
index ed64562e3..87b1b818c 100644
--- a/docs/public-api/crawlee.api.md
+++ b/docs/public-api/crawlee.api.md
@@ -1,4 +1,4 @@
-## API Report File for "crawlee"
+## Public API Report File for "crawlee"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
diff --git a/scripts/api-extractor/run.ts b/scripts/api-extractor/run.ts
index 0599b8c75..1f02be59e 100644
--- a/scripts/api-extractor/run.ts
+++ b/scripts/api-extractor/run.ts
@@ -1,6 +1,6 @@
/* eslint-disable no-console */
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
-import { dirname, relative, resolve } from 'node:path';
+import { basename, dirname, relative, resolve } from 'node:path';
import { Extractor, ExtractorConfig, type IConfigFile } from '@microsoft/api-extractor';
import { globbySync } from 'globby';
@@ -24,6 +24,10 @@ const root = resolve(import.meta.dirname, '..', '..');
const baseConfigPath = resolve(import.meta.dirname, 'api-extractor.base.json');
const baseConfig = JSON.parse(readFileSync(baseConfigPath, 'utf8')) as IConfigFile;
const reportFolder = resolve(root, 'docs', 'public-api');
+// API Extractor writes the "public" variant to a `.public.api.md` staging file here; we then
+// promote it onto the committed `<name>.api.md` ourselves (see `extract`), so the committed
+// filenames stay stable while the report content is @public-only (no @internal symbols).
+const stagingFolder = resolve(reportFolder, 'temp');
const mirrorRoot = resolve(root, 'node_modules', '.cache', 'api-extractor-dts');
const verify = process.argv.includes('--verify');
@@ -70,7 +74,11 @@ const stripTsIgnore = (content: string) =>
.filter((line) => !TS_IGNORE_LINE.test(line))
.join('\n');
-const reportFileName = (name: string) => `${name.replace('@', '').replace('/', '-')}.api.md`;
+const reportBaseName = (name: string) => name.replace('@', '').replace('/', '-');
+const reportFileName = (name: string) => `${reportBaseName(name)}.api.md`;
+// With `reportVariants: ['public']`, API Extractor appends the variant kind to the file name,
+// producing `<base>.public.api.md`. We stage that, then promote it to `<base>.api.md`.
+const stagedFileName = (name: string) => `${reportBaseName(name)}.public.api.md`;
/** Lazily built sanitized mirror of the dist tree, with a `@crawlee/*` -> mirror paths map. */
let mirror: { packages: string; paths: Record<string, string[]> } | undefined;
@@ -93,6 +101,7 @@ function getMirror() {
}
function extract(pkgDir: string, pkgJsonPath: string, entry: string, paths?: Record<string, string[]>) {
+ const name = manifest(pkgJsonPath).name;
const config = ExtractorConfig.prepare({
configObjectFullPath: baseConfigPath,
packageJsonFullPath: pkgJsonPath,
@@ -105,18 +114,42 @@ function extract(pkgDir: string, pkgJsonPath: string, entry: string, paths?: Rec
: baseConfig.compiler,
apiReport: {
enabled: true,
- reportFileName: reportFileName(manifest(pkgJsonPath).name),
- reportFolder,
- reportTempFolder: resolve(reportFolder, 'temp'),
+ // @public-only: drops @internal/@alpha/@beta symbols from the surface map.
+ reportVariants: ['public'],
+ reportFileName: reportFileName(name),
+ // Stage into temp; we promote the `.public.api.md` output onto the committed
+ // `<base>.api.md` ourselves so the tracked filenames don't change.
+ reportFolder: stagingFolder,
+ reportTempFolder: stagingFolder,
},
},
});
- return Extractor.invoke(config, { localBuild: !verify, showVerboseMessages: false });
+ // Let API Extractor always write the staged report (localBuild), then diff it against the
+ // committed report ourselves so `--verify` keys off the stable `<base>.api.md` name.
+ Extractor.invoke(config, { localBuild: true, showVerboseMessages: false });
+
+ const staged = readFileSync(resolve(stagingFolder, stagedFileName(name)), 'utf8');
+ const committedPath = resolve(reportFolder, reportFileName(name));
+ const committed = existsSync(committedPath) ? readFileSync(committedPath, 'utf8') : undefined;
+ const apiReportChanged = staged !== committed;
+ if (apiReportChanged && !verify) writeFileSync(committedPath, staged);
+ return { apiReportChanged };
}
function main() {
let failed = 0;
+ // Report filenames owned by an in-scope package. Any committed `*.api.md` not in this set
+ // is orphaned (e.g. its package was removed or renamed) and gets pruned in extract mode.
+ // Keyed off package existence, not per-run success, so a transient build/extract failure
+ // never deletes an otherwise-valid report.
+ const expectedReports = new Set(
+ packageJsonPaths
+ .map(manifest)
+ .filter((pkg) => !pkg.private && !EXCLUDED.has(pkg.name))
+ .map((pkg) => reportFileName(pkg.name)),
+ );
+
// The build injects `// @ts-ignore` lines into the `.d.ts` files that crash API
// Extractor's AST walker, so strip them for the duration of the run and restore after.
const originals = new Map<string, string>();
@@ -179,6 +212,21 @@ function main() {
rmSync(mirrorRoot, { recursive: true, force: true });
}
+ // Prune orphaned reports: committed `*.api.md` files with no owning in-scope package
+ // (e.g. a removed/renamed package). Delete them in extract mode; flag them in verify mode.
+ for (const file of globbySync('*.api.md', { cwd: reportFolder, absolute: true })) {
+ if (expectedReports.has(basename(file))) continue;
+ if (verify) {
+ const message = `${basename(file)}: orphaned report (no matching package) — run "pnpm api:extract" to remove it`;
+ console.error(`✗ ${message}`);
+ ghCommand('error', message);
+ failed++;
+ } else {
+ rmSync(file);
+ console.log(`✓ removed orphaned report ${basename(file)}`);
+ }
+ }
+
if (failed > 0) {
if (verify) console.error('\nRun "pnpm api:extract" and commit the changes in docs/public-api/.');
process.exit(1);