Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor!: remove unused exports from `@crawlee/utils` (#3929)

Jindřich Bär6be354a66c8f
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
index 8536945df..6e7d591a0 100644
--- a/docs/public-api/crawlee-utils.api.md
+++ b/docs/public-api/crawlee-utils.api.md
@@ -23,9 +23,6 @@ export { CheerioAPI }
// @public (undocumented)
export type CheerioRoot = CheerioAPI;
-// @public (undocumented)
-export function chunk<T>(array: readonly T[], chunkSize: number): T[][];
-
// @public (undocumented)
export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
@@ -161,7 +158,7 @@ const PINTEREST_REGEX_GLOBAL: RegExp;
export const RETRY_CSS_SELECTORS: string[];
// @public
-class RobotsTxtFile {
+export class RobotsTxtFile {
static find(url: string, options?: {
signal?: AbortSignal;
timeoutMillis?: number;
@@ -175,8 +172,6 @@ class RobotsTxtFile {
parseSitemaps(): Promise<Sitemap>;
parseUrlsFromSitemaps(): Promise<string[]>;
}
-export { RobotsTxtFile as RobotsFile }
-export { RobotsTxtFile }
// @public
export const ROTATE_PROXY_ERRORS: string[];
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 7bd5df24e..d034e0031 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -435,6 +435,7 @@ Besides the resource-detection helpers above, several other `@crawlee/utils` exp
- **Removed URL helpers:** `filterUrl(target, origin, strategy)`, `matchesEnqueueStrategy(strategy, target, origin)`, and the `UNSUPPORTED_SCHEME_MESSAGE` constant. URL filtering by enqueue strategy is now internal to `enqueueLinks`.
- **Relocated enums/types:** `EnqueueStrategy` now lives in `@crawlee/core` and `SearchParams` in `@crawlee/types`. They are no longer re-exported from `@crawlee/utils`, so `import { EnqueueStrategy } from '@crawlee/utils'` breaks — import them from `crawlee` (the meta-package) or from `@crawlee/core` / `@crawlee/types` instead.
+- **Removed `RobotsFile` alias:** `RobotsFile` was an alias for the `RobotsTxtFile` class and is removed. Rename any usage to `RobotsTxtFile`; the class itself is unchanged apart from the signature change described below.
### `RobotsTxtFile.find` signature changed; sitemap options removed
diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts
index b58e56dcf..164f8b95f 100644
--- a/packages/utils/src/index.ts
+++ b/packages/utils/src/index.ts
@@ -1,6 +1,5 @@
export * from './internals/blocked.js';
export * from './internals/cheerio.js';
-export * from './internals/chunk.js';
export * from './internals/extract-urls.js';
export * from './internals/general.js';
export * as social from './internals/social.js';
diff --git a/packages/utils/src/internals/chunk.ts b/packages/utils/src/internals/chunk.ts
deleted file mode 100644
index b842b760d..000000000
--- a/packages/utils/src/internals/chunk.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
-The MIT License (MIT)
-
-Copyright © `2020` `The Sapphire Community and its contributors`
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the “Software”), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-export function chunk<T>(array: readonly T[], chunkSize: number): T[][] {
- if (!Array.isArray(array)) throw new TypeError('entries must be an array.');
- if (!Number.isInteger(chunkSize)) throw new TypeError('chunkSize must be an integer.');
- if (chunkSize < 1) throw new RangeError('chunkSize must be 1 or greater.');
- const clone: T[] = array.slice();
- const chunks: T[][] = [];
- while (clone.length) chunks.push(clone.splice(0, chunkSize));
- return chunks;
-}
diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.ts
index 7d59c1b3b..975f326f3 100644
--- a/packages/utils/src/internals/general.ts
+++ b/packages/utils/src/internals/general.ts
@@ -51,20 +51,6 @@ export async function sleep(millis?: number): Promise<void> {
return setTimeout(millis ?? undefined);
}
-/**
- * Converts SNAKE_CASE to camelCase.
- * @ignore
- */
-export function snakeCaseToCamelCase(snakeCaseStr: string): string {
- return snakeCaseStr
- .toLowerCase()
- .split('_')
- .map((part, index) => {
- return index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part;
- })
- .join('');
-}
-
/**
* Traverses DOM and expands shadow-root elements (created by custom components).
* @ignore
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index ff2c7907f..e20ff678f 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -164,6 +164,3 @@ export class RobotsTxtFile {
return (await this.parseSitemaps(options)).urls;
}
}
-
-// to stay backwards compatible
-export { RobotsTxtFile as RobotsFile };
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 5969b259d..514f1b180 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -11,7 +11,7 @@ import sax from 'sax';
import MIMEType from 'whatwg-mimetype';
import { mergeAsyncIterables } from './iterables.js';
-import { RobotsFile } from './robots.js';
+import { RobotsTxtFile } from './robots.js';
import { type EnqueueStrategy, filterUrl } from './url.js';
interface SitemapUrlData {
@@ -620,7 +620,7 @@ export async function* discoverValidSitemaps(
}
try {
- const robotsFile = await RobotsFile.find(domainUrls[0], {
+ const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
proxyUrl,
timeoutMillis: requestTimeoutMillis,
signal,
diff --git a/test/utils/general.test.ts b/test/utils/general.test.ts
index 1267f76f9..8db466708 100644
--- a/test/utils/general.test.ts
+++ b/test/utils/general.test.ts
@@ -1,4 +1,4 @@
-import { sleep, snakeCaseToCamelCase, weightedAvg } from '@crawlee/utils';
+import { sleep, weightedAvg } from '@crawlee/utils';
describe('weightedAvg()', () => {
test('works', () => {
@@ -28,18 +28,3 @@ describe('sleep()', () => {
expect(timeAfter - timeBefore).toBeGreaterThanOrEqual(95);
});
});
-
-describe('snakeCaseToCamelCase()', () => {
- test('should camel case all sneaky cases of snake case', () => {
- const tests = {
- 'aaa_bbb_': 'aaaBbb',
- '': '',
- 'AaA_bBb_cCc': 'aaaBbbCcc',
- 'a_1_b_1a': 'a1B1a',
- };
-
- Object.entries(tests).forEach(([snakeCase, camelCase]) => {
- expect(snakeCaseToCamelCase(snakeCase)).toEqual(camelCase);
- });
- });
-});