Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(enqueueLinks): filter out empty/nullish globs (#2286)

Vlad Frangu84319b39efb5
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index a9197e8c0..ee14d925c 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -78,7 +78,22 @@ export function constructRegExpObjectsFromPseudoUrls(pseudoUrls: PseudoUrlInput[
*/
export function constructGlobObjectsFromGlobs(globs: GlobInput[]): GlobObject[] {
return globs
- .filter((glob) => ((glob as GlobObject).glob || (glob as string)).trim().length > 0)
+ .filter((glob) => {
+ // Skip possibly nullish, empty strings
+ if (!glob) {
+ return false;
+ }
+
+ if (typeof glob === 'string') {
+ return glob.trim().length > 0;
+ }
+
+ if (glob.glob) {
+ return glob.glob.trim().length > 0;
+ }
+
+ return false;
+ })
.map((item) => {
// Get glob object from cache.
let globObject = enqueueLinksPatternCache.get(item);
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 559f38286..bbbecc95e 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -157,6 +157,8 @@ describe('enqueueLinks()', () => {
'https://example.com/**/*',
'',
{ glob: ' ' },
+ // Empty string used to throw an error (https://console.apify.com/actors/aYG0l9s7dbB7j3gbS/issues/Wd0Ahfk9Vd2OPk4Uf)
+ { glob: '' },
{ glob: '?(http|https)://cool.com/', method: 'POST' as const },
];