Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix formatting and regenerate public API reports after the rebase
Martin Adámekc324386fde91
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 8d8bdb9d2..24aba9fc6 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -18,6 +18,7 @@ import type { CrawleeLoggerOptions } from '@crawlee/types';import type { DatasetBackend } from '@crawlee/types';import type { DatasetInfo } from '@crawlee/types';import { Dictionary } from '@crawlee/types';+import { EnqueueStrategy } from '@crawlee/utils';import type { HttpRequestOptions } from '@crawlee/types';import type { ISession } from '@crawlee/types';import type { ISessionPool } from '@crawlee/types';@@ -496,13 +497,7 @@ export interface DefaultStorageIdentifier {// @publicexport type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;-// @public-export enum EnqueueStrategy {- All = "all",- SameDomain = "same-domain",- SameHostname = "same-hostname",- SameOrigin = "same-origin"-}+export { EnqueueStrategy }// @publicexport type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';@@ -1798,7 +1793,9 @@ export class SessionPool implements ISessionPool {// (undocumented)resetStore(options?: PersistenceOptions): Promise<void>;retiredSessionsCount(): Promise<number>;- teardown(): Promise<void>;+ teardown(input?: {+ persistState?: boolean;+ }): Promise<void>;usableSessionsCount(): Promise<number>;}@@ -1845,6 +1842,7 @@ export class SitemapRequestLoader implements IRequestLoader {// @public (undocumented)export interface SitemapRequestLoaderOptions extends UrlConstraints {+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;httpClient?: BaseHttpClient;maxBufferSize?: number;parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex 278ffb96c..1f4935737 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -61,6 +61,14 @@ function emailsFromText(text: string): string[];// @publicfunction emailsFromUrls(urls: string[]): string[];+// @public+export enum EnqueueStrategy {+ All = "all",+ SameDomain = "same-domain",+ SameHostname = "same-hostname",+ SameOrigin = "same-origin"+}+// @publicexport function extractUrls(options: ExtractUrlsOptions): string[];@@ -129,6 +137,7 @@ export function parseSitemap<T extends ParseSitemapOptions>(initialSources: Site// @public (undocumented)export interface ParseSitemapOptions {emitNestedSitemaps?: true | false;+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;httpClient?: BaseHttpClient;logger?: CrawleeLogger;maxDepth?: number;@@ -161,10 +170,15 @@ export class RobotsTxtFile {}): Promise<RobotsTxtFile>;static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;getCrawlDelay(userAgent?: string): number | undefined;- getSitemaps(): string[];+ getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];isAllowed(url: string, userAgent?: string): boolean;- parseSitemaps(): Promise<Sitemap>;- parseUrlsFromSitemaps(): Promise<string[]>;+ parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;+ parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;+}++// @public (undocumented)+export interface RobotsTxtFileSitemapsOptions {+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;}// @publicdiff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex fe62d5af3..397f8e9d2 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -163,7 +163,9 @@ export class RobotsTxtFile {// `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);if (!allowed) {- this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);+ this.#logger?.warning(+ `Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`,+ );continue;}sitemaps.push(sitemapUrl);diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex 2f493003e..679e91dbc 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -484,16 +484,16 @@ describe('playwrightUtils', () => {test('detects the current challenge markup', async () => {await page.setContent(`<html><body>${currentChallengeBody}</body></html>`);- await expect(- handleCloudflareChallenge(page, 'https://example.com', fastOptions()),- ).rejects.toThrow(/Blocked by Cloudflare/);+ await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(+ /Blocked by Cloudflare/,+ );});test('detects the old challenge markup', async () => {await page.setContent(`<html><body>${oldChallengeBody}</body></html>`);- await expect(- handleCloudflareChallenge(page, 'https://example.com', fastOptions()),- ).rejects.toThrow(/Blocked by Cloudflare/);+ await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(+ /Blocked by Cloudflare/,+ );});test('resolves without detection on a regular page', async () => {diff --git a/test/core/sitemap_request_loader.test.ts b/test/core/sitemap_request_loader.test.tsindex 759933f85..019af1ba4 100644--- a/test/core/sitemap_request_loader.test.ts+++ b/test/core/sitemap_request_loader.test.ts@@ -267,7 +267,10 @@ beforeEach(async () => {describe('SitemapRequestLoader', () => {test('requests are available before the sitemap is fully loaded', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-stream.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-stream.xml`],+ enqueueStrategy: 'all',+ });while (await list.isEmpty()) {await sleep(20);@@ -287,7 +290,10 @@ describe('SitemapRequestLoader', () => {});test('retry sitemap load on error', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-unreliable.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-unreliable.xml`],+ enqueueStrategy: 'all',+ });for await (const request of list) {await list.markRequestAsHandled(request);@@ -297,7 +303,10 @@ describe('SitemapRequestLoader', () => {});test('broken off sitemap load resurrects correctly and does not duplicate / lose requests', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`],+ enqueueStrategy: 'all',+ });const urls = new Set<string>();@@ -319,7 +328,10 @@ describe('SitemapRequestLoader', () => {});test('teardown works', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-index.xml`],+ enqueueStrategy: 'all',+ });for await (const request of list) {await list.markRequestAsHandled(request);@@ -377,7 +389,10 @@ describe('SitemapRequestLoader', () => {});test('draining the request list between sitemaps', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-index.xml`],+ enqueueStrategy: 'all',+ });while (await list.isEmpty()) {await sleep(20);@@ -412,7 +427,10 @@ describe('SitemapRequestLoader', () => {});test('for..await syntax works with SitemapRequestLoader', async () => {- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });+ const list = await SitemapRequestLoader.open({+ sitemapUrls: [`${url}/sitemap-index.xml`],+ enqueueStrategy: 'all',+ });for await (const request of list) {await list.markRequestAsHandled(request);@@ -510,7 +528,11 @@ describe('SitemapRequestLoader', () => {});test('persists state', async () => {- const options = { sitemapUrls: [`${url}/sitemap-stream.xml`], persistStateKey: 'some-key', enqueueStrategy: 'all' as const };+ const options = {+ sitemapUrls: [`${url}/sitemap-stream.xml`],+ persistStateKey: 'some-key',+ enqueueStrategy: 'all' as const,+ };const list = await SitemapRequestLoader.open(options);const firstRequest = await list.fetchNextRequest();diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.tsindex f453e3012..860c12260 100644--- a/test/core/storages/utils.test.ts+++ b/test/core/storages/utils.test.ts@@ -1,5 +1,12 @@import type { Dictionary, StorageBackend } from '@crawlee/types';-import { Configuration, KeyValueStore, MemoryStorageBackend, purgeDefaultStorages, serviceLocator, useState } from '@crawlee/core';+import {+ Configuration,+ KeyValueStore,+ MemoryStorageBackend,+ purgeDefaultStorages,+ serviceLocator,+ useState,+} from '@crawlee/core';describe('useState', () => {beforeEach(async () => {@@ -80,6 +87,4 @@ describe('purgeDefaultStorages', () => {expect(purgedWhileAlreadyPurging).toBe(false);});--});