Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Filter non-HTTP(S) protocols (telnet, ftp, ssh, file, mailto) with separate error message (#2357)
devin-ai-integration[bot]5f3104435581
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 81aef2f2d..fa8244cf1 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -120,6 +120,7 @@ const FILE_TYPE: &str = "FILE_TYPE";const SOCIAL_MEDIA: &str = "SOCIAL_MEDIA";const EXTERNAL_LINK: &str = "EXTERNAL_LINK";const SECTION_LINK: &str = "SECTION_LINK";+const NON_WEB_PROTOCOL: &str = "NON_WEB_PROTOCOL";#[inline]fn is_file(path: &str) -> bool {@@ -169,6 +170,23 @@ fn no_sections(url_str: &str) -> bool {}}+#[inline]+fn is_non_web_protocol(url_str: &str) -> bool {+ const NON_WEB_PROTOCOLS: &[&str] = &[+ "mailto:",+ "tel:",+ "telnet:",+ "ftp:",+ "ftps:",+ "ssh:",+ "file:",+ ];++ NON_WEB_PROTOCOLS+ .iter()+ .any(|protocol| url_str.starts_with(protocol))+}+#[inline]fn is_social_media_or_email(url_str: &str) -> bool {const SOCIAL_MEDIA_OR_EMAIL: &[&str] = &[@@ -177,7 +195,6 @@ fn is_social_media_or_email(url_str: &str) -> bool {"linkedin.com","instagram.com","pinterest.com",- "mailto:","github.com","calendly.com","discord.gg",@@ -267,6 +284,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultlet path = url.path();let url_str = url.as_str();+ if is_non_web_protocol(url_str) {+ denial_reasons.insert(link, NON_WEB_PROTOCOL.to_string());+ continue;+ }+if get_url_depth(path) > data.max_depth {denial_reasons.insert(link, DEPTH_LIMIT.to_string());continue;@@ -417,6 +439,14 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Strilet path = url.path();let url_str = url.as_str();+ if is_non_web_protocol(url_str) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(NON_WEB_PROTOCOL.to_string()),+ });+ }+let excludes_regex: Vec<Regex> = data.excludes.iter()diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.tsindex 4be811e02..6b2ef7c38 100644--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts@@ -397,4 +397,28 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Crawl tests", () => {expect(typeof response.body.id).toBe("string");},);++ it.concurrent(+ "filters out non-web protocol links (telnet, ftp, ssh, file, mailto)",+ async () => {+ const res = await crawl(+ {+ url: base,+ limit: 10,+ },+ identity,+ );++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ for (const page of res.data) {+ const url = page.metadata.url ?? page.metadata.sourceURL!;+ expect(url).not.toMatch(/^(mailto|tel|telnet|ftp|ftps|ssh|file):/);+ expect(url).toMatch(/^https?:/);+ }+ }+ },+ 10 * scrapeTimeout,+ );});diff --git a/apps/api/src/lib/format-utils.ts b/apps/api/src/lib/format-utils.tsindex 119fbd2b9..598b0c561 100644--- a/apps/api/src/lib/format-utils.ts+++ b/apps/api/src/lib/format-utils.ts@@ -22,22 +22,3 @@ export function hasFormatOfType<T extends FormatObject["type"]>(const found = formats.find(f => f.type === type);return found as Extract<FormatObject, { type: T }> | undefined;}--/**- * Checks if any of the specified format types exist in the formats array.- * Returns true if at least one of the types is found.- *- * @param formats - Array of format objects- * @param types - Array of format types to search for- * @returns true if any of the types are found, false otherwise- */-export function hasAnyFormatOfTypes(- formats: FormatObject[] | undefined,- types: FormatObject["type"][],-): boolean {- if (!formats) {- return false;- }-- return formats.some(f => types.includes(f.type));-}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 6e5c67065..228ef32b8 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -35,6 +35,7 @@ enum DenialReason {SOCIAL_MEDIA = "URL is a social media or email link",EXTERNAL_LINK = "External URL not allowed",SECTION_LINK = "URL contains section anchor (#)",+ NON_WEB_PROTOCOL = "URL uses a non-web protocol (telnet, ftp, ssh, file, mailto, etc.)",}interface FilterLinksResult {@@ -219,6 +220,24 @@ export class WebCrawler {}const path = url.pathname;+ const urlStr = url.toString();+ const nonWebProtocols = [+ "mailto:",+ "tel:",+ "telnet:",+ "ftp:",+ "ftps:",+ "ssh:",+ "file:",+ ];+ if (nonWebProtocols.some(protocol => urlStr.startsWith(protocol))) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} NON-WEB PROTOCOL FAIL`);+ }+ denialReasons.set(link, DenialReason.NON_WEB_PROTOCOL);+ return false;+ }+const depth = getURLDepth(url.toString());// Check if the link exceeds the maximum depth allowed