Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Filter non-HTTP(S) protocols (telnet, ftp, ssh, file, mailto) with separate error message (#2357)

devin-ai-integration[bot]5f3104435581
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 81aef2f2d..fa8244cf1 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -120,6 +120,7 @@ const FILE_TYPE: &str = "FILE_TYPE";
const SOCIAL_MEDIA: &str = "SOCIAL_MEDIA";
const EXTERNAL_LINK: &str = "EXTERNAL_LINK";
const SECTION_LINK: &str = "SECTION_LINK";
+const NON_WEB_PROTOCOL: &str = "NON_WEB_PROTOCOL";
#[inline]
fn is_file(path: &str) -> bool {
@@ -169,6 +170,23 @@ fn no_sections(url_str: &str) -> bool {
}
}
+#[inline]
+fn is_non_web_protocol(url_str: &str) -> bool {
+ const NON_WEB_PROTOCOLS: &[&str] = &[
+ "mailto:",
+ "tel:",
+ "telnet:",
+ "ftp:",
+ "ftps:",
+ "ssh:",
+ "file:",
+ ];
+
+ NON_WEB_PROTOCOLS
+ .iter()
+ .any(|protocol| url_str.starts_with(protocol))
+}
+
#[inline]
fn is_social_media_or_email(url_str: &str) -> bool {
const SOCIAL_MEDIA_OR_EMAIL: &[&str] = &[
@@ -177,7 +195,6 @@ fn is_social_media_or_email(url_str: &str) -> bool {
"linkedin.com",
"instagram.com",
"pinterest.com",
- "mailto:",
"github.com",
"calendly.com",
"discord.gg",
@@ -267,6 +284,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
let path = url.path();
let url_str = url.as_str();
+ if is_non_web_protocol(url_str) {
+ denial_reasons.insert(link, NON_WEB_PROTOCOL.to_string());
+ continue;
+ }
+
if get_url_depth(path) > data.max_depth {
denial_reasons.insert(link, DEPTH_LIMIT.to_string());
continue;
@@ -417,6 +439,14 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
let path = url.path();
let url_str = url.as_str();
+ if is_non_web_protocol(url_str) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(NON_WEB_PROTOCOL.to_string()),
+ });
+ }
+
let excludes_regex: Vec<Regex> = data
.excludes
.iter()
diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts
index 4be811e02..6b2ef7c38 100644
--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts
@@ -397,4 +397,28 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Crawl tests", () => {
expect(typeof response.body.id).toBe("string");
},
);
+
+ it.concurrent(
+ "filters out non-web protocol links (telnet, ftp, ssh, file, mailto)",
+ async () => {
+ const res = await crawl(
+ {
+ url: base,
+ limit: 10,
+ },
+ identity,
+ );
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ expect(res.completed).toBeGreaterThan(0);
+ for (const page of res.data) {
+ const url = page.metadata.url ?? page.metadata.sourceURL!;
+ expect(url).not.toMatch(/^(mailto|tel|telnet|ftp|ftps|ssh|file):/);
+ expect(url).toMatch(/^https?:/);
+ }
+ }
+ },
+ 10 * scrapeTimeout,
+ );
});
diff --git a/apps/api/src/lib/format-utils.ts b/apps/api/src/lib/format-utils.ts
index 119fbd2b9..598b0c561 100644
--- a/apps/api/src/lib/format-utils.ts
+++ b/apps/api/src/lib/format-utils.ts
@@ -22,22 +22,3 @@ export function hasFormatOfType<T extends FormatObject["type"]>(
const found = formats.find(f => f.type === type);
return found as Extract<FormatObject, { type: T }> | undefined;
}
-
-/**
- * Checks if any of the specified format types exist in the formats array.
- * Returns true if at least one of the types is found.
- *
- * @param formats - Array of format objects
- * @param types - Array of format types to search for
- * @returns true if any of the types are found, false otherwise
- */
-export function hasAnyFormatOfTypes(
- formats: FormatObject[] | undefined,
- types: FormatObject["type"][],
-): boolean {
- if (!formats) {
- return false;
- }
-
- return formats.some(f => types.includes(f.type));
-}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 6e5c67065..228ef32b8 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -35,6 +35,7 @@ enum DenialReason {
SOCIAL_MEDIA = "URL is a social media or email link",
EXTERNAL_LINK = "External URL not allowed",
SECTION_LINK = "URL contains section anchor (#)",
+ NON_WEB_PROTOCOL = "URL uses a non-web protocol (telnet, ftp, ssh, file, mailto, etc.)",
}
interface FilterLinksResult {
@@ -219,6 +220,24 @@ export class WebCrawler {
}
const path = url.pathname;
+ const urlStr = url.toString();
+ const nonWebProtocols = [
+ "mailto:",
+ "tel:",
+ "telnet:",
+ "ftp:",
+ "ftps:",
+ "ssh:",
+ "file:",
+ ];
+ if (nonWebProtocols.some(protocol => urlStr.startsWith(protocol))) {
+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ this.logger.debug(`${link} NON-WEB PROTOCOL FAIL`);
+ }
+ denialReasons.set(link, DenialReason.NON_WEB_PROTOCOL);
+ return false;
+ }
+
const depth = getURLDepth(url.toString());
// Check if the link exceeds the maximum depth allowed