Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Public API Report File for "@crawlee/utils"
Do not edit this file. It is a report generated by API Extractor.
import { ArgumentValidationError } from '@apify/validations';
import type { BaseHttpClient } from '@crawlee/http-client';
import type { CheerioAPI } from 'cheerio';
import type { CrawleeLogger } from '@crawlee/types';
import type { Dictionary } from '@crawlee/types';
import type { ISession } from '@crawlee/types';
export { ArgumentValidationError }
// @public
const DISCORD_REGEX: RegExp;
// @public
const DISCORD_REGEX_GLOBAL: RegExp;
// @public
export function discoverValidSitemaps(urls: string[], options?: {
proxyUrl?: string;
timeoutMillis?: number;
signal?: AbortSignal;
requestTimeoutMillis?: number;
httpClient?: BaseHttpClient;
logger?: CrawleeLogger;
}): AsyncIterable<string>;
// @public
export function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise<string[]>;
// @public (undocumented)
export interface DownloadListOfUrlsOptions {
encoding?: BufferEncoding;
httpClient?: BaseHttpClient;
proxyUrl?: string;
url: string;
urlRegExp?: RegExp;
}
// @public
const EMAIL_REGEX: RegExp;
// @public
const EMAIL_REGEX_GLOBAL: RegExp;
// @public
function emailsFromText(text: string): string[];
// @public
function emailsFromUrls(urls: string[]): string[];
// @public
export enum EnqueueStrategy {
All = "all",
SameDomain = "same-domain",
SameHostname = "same-hostname",
SameOrigin = "same-origin"
}
// @public
export function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
// @public (undocumented)
export function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;
// @public
export function extractUrls(options: ExtractUrlsOptions): string[];
// @public (undocumented)
export interface ExtractUrlsOptions {
string: string;
urlRegExp?: RegExp;
}
// @public
const FACEBOOK_REGEX: RegExp;
// @public
const FACEBOOK_REGEX_GLOBAL: RegExp;
// @public
export function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;
// @public
const INSTAGRAM_REGEX: RegExp;
// @public
const INSTAGRAM_REGEX_GLOBAL: RegExp;
// @public
const LINKEDIN_REGEX: RegExp;
// @public
const LINKEDIN_REGEX_GLOBAL: RegExp;
// @public
export interface MicrodataItem {
id?: string;
properties: Record<string, MicrodataValue | MicrodataValue[]>;
type?: string[];
}
// @public
export type MicrodataValue = string | MicrodataItem;
// @public (undocumented)
export interface OpenGraphProperty {
// (undocumented)
children: OpenGraphProperty[];
// (undocumented)
name: string;
// (undocumented)
outputName: string;
}
// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;
// @public
function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
// @public
export function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
// @public (undocumented)
export function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
// @public (undocumented)
export interface ParseSitemapOptions {
emitNestedSitemaps?: true | false;
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
httpClient?: BaseHttpClient;
logger?: CrawleeLogger;
maxDepth?: number;
nestedSitemapFilter?: (sitemapUrl: string) => boolean;
reportNetworkErrors?: boolean;
sitemapRetries?: number;
timeoutMillis?: number;
}
// @public
function phonesFromText(text: string): string[];
// @public
function phonesFromUrls(urls: string[]): string[];
// @public
const PINTEREST_REGEX: RegExp;
// @public
const PINTEREST_REGEX_GLOBAL: RegExp;
// @public
export class RobotsTxtFile {
static find(url: string, options?: {
signal?: AbortSignal;
timeoutMillis?: number;
proxyUrl?: string;
httpClient?: BaseHttpClient;
session?: ISession;
logger?: CrawleeLogger;
}): Promise<RobotsTxtFile>;
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
getCrawlDelay(userAgent?: string): number | undefined;
getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
isAllowed(url: string, userAgent?: string): boolean;
parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
}
// @public (undocumented)
export interface RobotsTxtFileSitemapsOptions {
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
}
// @public
export class Sitemap {
constructor(urls: string[]);
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
// (undocumented)
readonly urls: string[];
}
// @public
export function sleep(millis?: number): Promise<void>;
declare namespace social {
export {
emailsFromText,
emailsFromUrls,
phonesFromText,
phonesFromUrls,
parseHandlesFromHtml,
EMAIL_REGEX,
EMAIL_REGEX_GLOBAL,
SocialHandles,
LINKEDIN_REGEX,
LINKEDIN_REGEX_GLOBAL,
INSTAGRAM_REGEX,
INSTAGRAM_REGEX_GLOBAL,
TWITTER_REGEX,
TWITTER_REGEX_GLOBAL,
FACEBOOK_REGEX,
FACEBOOK_REGEX_GLOBAL,
YOUTUBE_REGEX,
YOUTUBE_REGEX_GLOBAL,
TIKTOK_REGEX,
TIKTOK_REGEX_GLOBAL,
PINTEREST_REGEX,
PINTEREST_REGEX_GLOBAL,
DISCORD_REGEX,
DISCORD_REGEX_GLOBAL
}
}
// @public
interface SocialHandles {
// (undocumented)
discords: string[];
// (undocumented)
emails: string[];
// (undocumented)
facebooks: string[];
// (undocumented)
instagrams: string[];
// (undocumented)
linkedIns: string[];
// (undocumented)
phones: string[];
// (undocumented)
phonesUncertain: string[];
// (undocumented)
pinterests: string[];
// (undocumented)
tiktoks: string[];
// (undocumented)
twitters: string[];
// (undocumented)
youtubes: string[];
}
// @public
const TIKTOK_REGEX: RegExp;
// @public
const TIKTOK_REGEX_GLOBAL: RegExp;
// @public
const TWITTER_REGEX: RegExp;
// @public
const TWITTER_REGEX_GLOBAL: RegExp;
// @public
const YOUTUBE_REGEX: RegExp;
// @public
const YOUTUBE_REGEX_GLOBAL: RegExp;
// (No @packageDocumentation comment for this package)