Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: fix formatting and regenerate public API reports after the rebase

Martin Adámekc324386fde91
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 8d8bdb9d2..24aba9fc6 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -18,6 +18,7 @@ import type { CrawleeLoggerOptions } from '@crawlee/types';
import type { DatasetBackend } from '@crawlee/types';
import type { DatasetInfo } from '@crawlee/types';
import { Dictionary } from '@crawlee/types';
+import { EnqueueStrategy } from '@crawlee/utils';
import type { HttpRequestOptions } from '@crawlee/types';
import type { ISession } from '@crawlee/types';
import type { ISessionPool } from '@crawlee/types';
@@ -496,13 +497,7 @@ export interface DefaultStorageIdentifier {
// @public
export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
-// @public
-export enum EnqueueStrategy {
- All = "all",
- SameDomain = "same-domain",
- SameHostname = "same-hostname",
- SameOrigin = "same-origin"
-}
+export { EnqueueStrategy }
// @public
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
@@ -1798,7 +1793,9 @@ export class SessionPool implements ISessionPool {
// (undocumented)
resetStore(options?: PersistenceOptions): Promise<void>;
retiredSessionsCount(): Promise<number>;
- teardown(): Promise<void>;
+ teardown(input?: {
+ persistState?: boolean;
+ }): Promise<void>;
usableSessionsCount(): Promise<number>;
}
@@ -1845,6 +1842,7 @@ export class SitemapRequestLoader implements IRequestLoader {
// @public (undocumented)
export interface SitemapRequestLoaderOptions extends UrlConstraints {
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
httpClient?: BaseHttpClient;
maxBufferSize?: number;
parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
index 278ffb96c..1f4935737 100644
--- a/docs/public-api/crawlee-utils.api.md
+++ b/docs/public-api/crawlee-utils.api.md
@@ -61,6 +61,14 @@ function emailsFromText(text: string): string[];
// @public
function emailsFromUrls(urls: string[]): string[];
+// @public
+export enum EnqueueStrategy {
+ All = "all",
+ SameDomain = "same-domain",
+ SameHostname = "same-hostname",
+ SameOrigin = "same-origin"
+}
+
// @public
export function extractUrls(options: ExtractUrlsOptions): string[];
@@ -129,6 +137,7 @@ export function parseSitemap<T extends ParseSitemapOptions>(initialSources: Site
// @public (undocumented)
export interface ParseSitemapOptions {
emitNestedSitemaps?: true | false;
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
httpClient?: BaseHttpClient;
logger?: CrawleeLogger;
maxDepth?: number;
@@ -161,10 +170,15 @@ export class RobotsTxtFile {
}): Promise<RobotsTxtFile>;
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
getCrawlDelay(userAgent?: string): number | undefined;
- getSitemaps(): string[];
+ getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
isAllowed(url: string, userAgent?: string): boolean;
- parseSitemaps(): Promise<Sitemap>;
- parseUrlsFromSitemaps(): Promise<string[]>;
+ parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
+ parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
+}
+
+// @public (undocumented)
+export interface RobotsTxtFileSitemapsOptions {
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
}
// @public
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index fe62d5af3..397f8e9d2 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -163,7 +163,9 @@ export class RobotsTxtFile {
// `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
if (!allowed) {
- this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
+ this.#logger?.warning(
+ `Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`,
+ );
continue;
}
sitemaps.push(sitemapUrl);
diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
index 2f493003e..679e91dbc 100644
--- a/test/core/playwright_utils.test.ts
+++ b/test/core/playwright_utils.test.ts
@@ -484,16 +484,16 @@ describe('playwrightUtils', () => {
test('detects the current challenge markup', async () => {
await page.setContent(`<html><body>${currentChallengeBody}</body></html>`);
- await expect(
- handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
- ).rejects.toThrow(/Blocked by Cloudflare/);
+ await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(
+ /Blocked by Cloudflare/,
+ );
});
test('detects the old challenge markup', async () => {
await page.setContent(`<html><body>${oldChallengeBody}</body></html>`);
- await expect(
- handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
- ).rejects.toThrow(/Blocked by Cloudflare/);
+ await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(
+ /Blocked by Cloudflare/,
+ );
});
test('resolves without detection on a regular page', async () => {
diff --git a/test/core/sitemap_request_loader.test.ts b/test/core/sitemap_request_loader.test.ts
index 759933f85..019af1ba4 100644
--- a/test/core/sitemap_request_loader.test.ts
+++ b/test/core/sitemap_request_loader.test.ts
@@ -267,7 +267,10 @@ beforeEach(async () => {
describe('SitemapRequestLoader', () => {
test('requests are available before the sitemap is fully loaded', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-stream.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-stream.xml`],
+ enqueueStrategy: 'all',
+ });
while (await list.isEmpty()) {
await sleep(20);
@@ -287,7 +290,10 @@ describe('SitemapRequestLoader', () => {
});
test('retry sitemap load on error', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-unreliable.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-unreliable.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestAsHandled(request);
@@ -297,7 +303,10 @@ describe('SitemapRequestLoader', () => {
});
test('broken off sitemap load resurrects correctly and does not duplicate / lose requests', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`],
+ enqueueStrategy: 'all',
+ });
const urls = new Set<string>();
@@ -319,7 +328,10 @@ describe('SitemapRequestLoader', () => {
});
test('teardown works', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestAsHandled(request);
@@ -377,7 +389,10 @@ describe('SitemapRequestLoader', () => {
});
test('draining the request list between sitemaps', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
while (await list.isEmpty()) {
await sleep(20);
@@ -412,7 +427,10 @@ describe('SitemapRequestLoader', () => {
});
test('for..await syntax works with SitemapRequestLoader', async () => {
- const list = await SitemapRequestLoader.open({ sitemapUrls: [`${url}/sitemap-index.xml`], enqueueStrategy: 'all' });
+ const list = await SitemapRequestLoader.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestAsHandled(request);
@@ -510,7 +528,11 @@ describe('SitemapRequestLoader', () => {
});
test('persists state', async () => {
- const options = { sitemapUrls: [`${url}/sitemap-stream.xml`], persistStateKey: 'some-key', enqueueStrategy: 'all' as const };
+ const options = {
+ sitemapUrls: [`${url}/sitemap-stream.xml`],
+ persistStateKey: 'some-key',
+ enqueueStrategy: 'all' as const,
+ };
const list = await SitemapRequestLoader.open(options);
const firstRequest = await list.fetchNextRequest();
diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.ts
index f453e3012..860c12260 100644
--- a/test/core/storages/utils.test.ts
+++ b/test/core/storages/utils.test.ts
@@ -1,5 +1,12 @@
import type { Dictionary, StorageBackend } from '@crawlee/types';
-import { Configuration, KeyValueStore, MemoryStorageBackend, purgeDefaultStorages, serviceLocator, useState } from '@crawlee/core';
+import {
+ Configuration,
+ KeyValueStore,
+ MemoryStorageBackend,
+ purgeDefaultStorages,
+ serviceLocator,
+ useState,
+} from '@crawlee/core';
describe('useState', () => {
beforeEach(async () => {
@@ -80,6 +87,4 @@ describe('purgeDefaultStorages', () => {
expect(purgedWhileAlreadyPurging).toBe(false);
});
-
-
});