Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: filter out empty globs (#2205)

Jindřich Bär41322ab32d7d
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index cdca85df9..7f6adf42f 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -75,22 +75,24 @@ export function constructRegExpObjectsFromPseudoUrls(pseudoUrls: PseudoUrlInput[
* @ignore
*/
export function constructGlobObjectsFromGlobs(globs: GlobInput[]): GlobObject[] {
- return globs.map((item) => {
+ return globs
+ .filter((glob) => ((glob as GlobObject).glob || (glob as string)).trim().length > 0)
+ .map((item) => {
// Get glob object from cache.
- let globObject = enqueueLinksPatternCache.get(item);
- if (globObject) return globObject;
-
- if (typeof item === 'string') {
- globObject = { glob: validateGlobPattern(item) };
- } else {
- const { glob, ...requestOptions } = item;
- globObject = { glob: validateGlobPattern(glob), ...requestOptions };
- }
+ let globObject = enqueueLinksPatternCache.get(item);
+ if (globObject) return globObject;
+
+ if (typeof item === 'string') {
+ globObject = { glob: validateGlobPattern(item) };
+ } else {
+ const { glob, ...requestOptions } = item;
+ globObject = { glob: validateGlobPattern(glob), ...requestOptions };
+ }
- updateEnqueueLinksPatternCache(item, globObject);
+ updateEnqueueLinksPatternCache(item, globObject);
- return globObject;
- });
+ return globObject;
+ });
}
/**
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index aa91f4711..559f38286 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -151,6 +151,25 @@ describe('enqueueLinks()', () => {
expect(enqueued[2].userData).toEqual({ label: 'COOL' });
});
+ test('does not throw with empty globs', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+ const globs = [
+ 'https://example.com/**/*',
+ '',
+ { glob: ' ' },
+ { glob: '?(http|https)://cool.com/', method: 'POST' as const },
+ ];
+
+ await expect(browserCrawlerEnqueueLinks({
+ options: { selector: '.click', globs },
+ page,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ })).resolves.not.toThrow();
+
+ expect(enqueued).toHaveLength(3);
+ });
+
test('works with regexps', async () => {
const { enqueued, requestQueue } = createRequestQueueMock();
const regexps = [
@@ -609,6 +628,25 @@ describe('enqueueLinks()', () => {
expect(enqueued[2].userData).toEqual({ label: 'COOL' });
});
+ test('does not throw with empty globs', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+ const globs = [
+ 'https://example.com/**/*',
+ { glob: '?(http|https)://cool.com/', method: 'POST' as const, userData: { label: 'COOL' } },
+ '',
+ { glob: ' ' },
+ ];
+
+ await expect(cheerioCrawlerEnqueueLinks({
+ options: { selector: '.click', globs },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ })).resolves.not.toThrow();
+
+ expect(enqueued).toHaveLength(3);
+ });
+
test('works with RegExps', async () => {
const { enqueued, requestQueue } = createRequestQueueMock();
const regexps = [