Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): scope file-seed crawls to their directory and read markdown links (#4250)

tomsideguide6272c64a2eec
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 3dbb1b2b4..66023c2a3 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -135,6 +135,45 @@ fn is_file(path: &str) -> bool {
}
}
+static DOCUMENT_SEGMENT: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"\.[A-Za-z0-9]{2,}$").unwrap());
+
+struct CrawlScope {
+ prefix: String,
+ exact: Option<String>,
+}
+
+/// Resolves the path a crawl is scoped to when backward crawling is disabled.
+/// Keep in sync with getCrawlScope() in apps/api/src/lib/crawl-scope.ts.
+fn crawl_scope(initial_url: &Url) -> CrawlScope {
+ let path = initial_url.path();
+
+ if path.ends_with('/') {
+ return CrawlScope {
+ prefix: path.to_string(),
+ exact: None,
+ };
+ }
+
+ let last_slash = path.rfind('/').map_or(0, |i| i + 1);
+ if DOCUMENT_SEGMENT.is_match(&path[last_slash..]) {
+ return CrawlScope {
+ prefix: path[..last_slash].to_string(),
+ exact: None,
+ };
+ }
+
+ CrawlScope {
+ prefix: format!("{path}/"),
+ exact: Some(path.to_string()),
+ }
+}
+
+#[inline]
+fn is_within_crawl_scope(path: &str, scope: &CrawlScope) -> bool {
+ scope.exact.as_deref() == Some(path) || path.starts_with(&scope.prefix)
+}
+
#[inline]
fn get_url_depth(path: &str) -> u32 {
path
@@ -257,7 +296,7 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {e}"))?;
let initial_url =
Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;
- let initial_path = initial_url.path();
+ let scope = crawl_scope(&initial_url);
let excludes_regex: Vec<Regex> = data
.excludes
@@ -317,7 +356,7 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
continue;
}
- if !data.allow_backward_crawling && !path.starts_with(initial_path) {
+ if !data.allow_backward_crawling && !is_within_crawl_scope(path, &scope) {
denial_reasons.insert(link, BACKWARD_CRAWLING.to_string());
continue;
}
@@ -753,6 +792,54 @@ pub async fn process_sitemap(xml_content: String) -> Result<SitemapProcessingRes
mod tests {
use super::*;
+ fn scope_of(url: &str) -> CrawlScope {
+ crawl_scope(&Url::parse(url).unwrap())
+ }
+
+ #[test]
+ fn test_crawl_scope_document_seed_scopes_to_directory() {
+ let scope = scope_of("https://example.com/docs/guide.md");
+ assert_eq!(scope.prefix, "/docs/");
+ assert_eq!(scope.exact, None);
+
+ assert!(is_within_crawl_scope("/docs/other.md", &scope));
+ assert!(is_within_crawl_scope("/docs/nested/deep.md", &scope));
+ assert!(!is_within_crawl_scope("/blog/post.md", &scope));
+ assert!(!is_within_crawl_scope("/", &scope));
+ }
+
+ #[test]
+ fn test_crawl_scope_directory_seed() {
+ let scope = scope_of("https://example.com/docs/");
+ assert_eq!(scope.prefix, "/docs/");
+ assert_eq!(scope.exact, None);
+
+ assert!(is_within_crawl_scope("/docs/guide.md", &scope));
+ assert!(!is_within_crawl_scope("/docs", &scope));
+ assert!(!is_within_crawl_scope("/docsearch/x", &scope));
+ }
+
+ #[test]
+ fn test_crawl_scope_extensionless_seed_allows_seed_and_children() {
+ let scope = scope_of("https://example.com/docs");
+ assert_eq!(scope.prefix, "/docs/");
+ assert_eq!(scope.exact.as_deref(), Some("/docs"));
+
+ assert!(is_within_crawl_scope("/docs", &scope));
+ assert!(is_within_crawl_scope("/docs/guide", &scope));
+ // Sibling paths sharing the seed's prefix must not sneak in.
+ assert!(!is_within_crawl_scope("/docsearch", &scope));
+ }
+
+ #[test]
+ fn test_crawl_scope_root_seed() {
+ let scope = scope_of("https://example.com");
+ assert_eq!(scope.prefix, "/");
+ assert_eq!(scope.exact, None);
+
+ assert!(is_within_crawl_scope("/anything/at/all", &scope));
+ }
+
#[test]
fn test_parse_sitemap_xml_urlset() {
let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
diff --git a/apps/api/src/lib/crawl-scope.ts b/apps/api/src/lib/crawl-scope.ts
new file mode 100644
index 000000000..21e7f1268
--- /dev/null
+++ b/apps/api/src/lib/crawl-scope.ts
@@ -0,0 +1,46 @@
+// Keep in sync with crawl_scope() in apps/api/native/src/crawler.rs.
+
+interface CrawlScope {
+ /** Directory every crawled path must sit under. Always ends with "/". */
+ prefix: string;
+ /** Single path allowed alongside the prefix, or null. */
+ exact: string | null;
+}
+
+const DOCUMENT_SEGMENT = /\.[A-Za-z0-9]{2,}$/;
+
+/** Resolves the path a crawl is scoped to when backward crawling is disabled. */
+export function getCrawlScope(initialUrl: string): CrawlScope {
+ let pathname: string;
+ try {
+ pathname = new URL(initialUrl).pathname;
+ } catch {
+ return { prefix: "/", exact: null };
+ }
+
+ if (!pathname.startsWith("/")) {
+ pathname = "/" + pathname;
+ }
+
+ if (pathname.endsWith("/")) {
+ return { prefix: pathname, exact: null };
+ }
+
+ const lastSlash = pathname.lastIndexOf("/");
+ if (DOCUMENT_SEGMENT.test(pathname.slice(lastSlash + 1))) {
+ return { prefix: pathname.slice(0, lastSlash + 1), exact: null };
+ }
+
+ return { prefix: pathname + "/", exact: pathname };
+}
+
+export function isWithinCrawlScope(path: string, scope: CrawlScope): boolean {
+ return path === scope.exact || path.startsWith(scope.prefix);
+}
+
+/** Human-readable scope for denial messages. */
+export function describeCrawlScope(scope: CrawlScope): string {
+ return scope.exact
+ ? `${scope.exact} or ${scope.prefix}*`
+ : `${scope.prefix}*`;
+}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 9b75fba3b..7e0b1f597 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -17,6 +17,15 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { ScrapeOptions } from "../../controllers/v2/types";
import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";
import { extractBaseDomain } from "../../lib/url-utils";
+import {
+ describeCrawlScope,
+ getCrawlScope,
+ isWithinCrawlScope,
+} from "../../lib/crawl-scope";
+import {
+ extractLinksFromMarkdown,
+ isMarkdownContentType,
+} from "../scrapeURL/lib/extractLinksFromMarkdown";
export const SITEMAP_LIMIT = 25;
const SITEMAP_FETCH_CONCURRENCY = 3;
@@ -153,7 +162,7 @@ export class WebCrawler {
public setBaseUrl(newBase: string): void {
this.baseUrl = newBase;
- this.robotsTxtUrl = `${this.baseUrl}${this.baseUrl.endsWith("/") ? "" : "/"}robots.txt`;
+ this.robotsTxtUrl = new URL("/robots.txt", newBase).href;
}
public async filterLinks(
@@ -245,7 +254,9 @@ export class WebCrawler {
);
break;
case "BACKWARD_CRAWLING":
- const initialPath = new URL(this.initialUrl).pathname;
+ const initialPath = describeCrawlScope(
+ getCrawlScope(this.initialUrl),
+ );
fancyDenialReasons.set(
key,
`This URL's path ("${urlPath}") is outside the initial URL's path hierarchy ("${initialPath}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,
@@ -288,6 +299,8 @@ export class WebCrawler {
});
}
+ const scope = getCrawlScope(this.initialUrl);
+
const filteredLinks = sitemapLinks
.filter(link => {
let url: URL;
@@ -396,17 +409,15 @@ export class WebCrawler {
// }
if (!this.allowBackwardCrawling) {
- if (
- !normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)
- ) {
+ if (!isWithinCrawlScope(normalizedLink.pathname, scope)) {
if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(
- `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,
+ `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${describeCrawlScope(scope)}`,
);
}
denialReasons.set(
link,
- `This URL's path ("${normalizedLink.pathname}") is outside the initial URL's path hierarchy ("${normalizedInitialUrl.pathname}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,
+ `This URL's path ("${normalizedLink.pathname}") is outside the initial URL's path hierarchy ("${describeCrawlScope(scope)}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,
);
return false;
}
@@ -762,7 +773,26 @@ export class WebCrawler {
return links;
}
- public async extractLinksFromHTML(html: string, url: string) {
+ private async extractLinksFromMarkdownContent(text: string, url: string) {
+ const filteredLinks: string[] = [];
+ for (const link of extractLinksFromMarkdown(text, url)) {
+ const filterResult = await this.filterURL(link, url);
+ if (filterResult.allowed && filterResult.url) {
+ filteredLinks.push(filterResult.url);
+ }
+ }
+ return filteredLinks;
+ }
+
+ public async extractLinksFromContent(
+ html: string,
+ url: string,
+ contentType?: string,
+ ) {
+ if (isMarkdownContentType(contentType)) {
+ return await this.extractLinksFromMarkdownContent(html, url);
+ }
+
try {
return [
...new Set(
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
index 7aecd12de..e4f520f61 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
@@ -5,6 +5,10 @@ import {
extractLinks as _extractLinks,
extractBaseHref as _extractBaseHref,
} from "@mendable/firecrawl-rs";
+import {
+ extractLinksFromMarkdown,
+ isMarkdownContentType,
+} from "./extractLinksFromMarkdown";
function resolveUrlWithBaseHref(
href: string,
@@ -67,7 +71,12 @@ async function extractLinksRust(
export async function extractLinks(
html: string,
baseUrl: string,
+ contentType?: string,
): Promise<string[]> {
+ if (isMarkdownContentType(contentType)) {
+ return extractLinksFromMarkdown(html, baseUrl);
+ }
+
try {
return await extractLinksRust(html, baseUrl);
} catch (error) {
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.ts
new file mode 100644
index 000000000..a43a0d1cd
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.ts
@@ -0,0 +1,118 @@
+const MARKDOWN_CONTENT_TYPES = [
+ "text/plain",
+ "text/markdown",
+ "text/x-markdown",
+];
+
+/** Bodies served as text but authored as markdown, e.g. llms.txt. */
+export function isMarkdownContentType(contentType?: string): boolean {
+ if (!contentType) {
+ return false;
+ }
+ // Media types are case-insensitive and may carry parameters ("; charset=").
+ const mediaType = contentType.split(";")[0].trim().toLowerCase();
+ return MARKDOWN_CONTENT_TYPES.includes(mediaType);
+}
+
+const IMAGE = /!\[[^\]]*\]\([^)]*\)/g;
+const INLINE_LINK =
+ /\[[^\]]*\]\(\s*(<[^>\n]*>|[^)\s]+)(?:\s+["'][^"']*["'])?\s*\)/g;
+const REFERENCE_DEFINITION = /^[ \t]{0,3}\[[^\]]+\]:[ \t]*(<[^>]+>|\S+)/gm;
+const AUTOLINK = /<((?:https?:\/\/|mailto:)[^>\s]+)>/g;
+const BARE_URL = /(?:^|[\s(<])(https?:\/\/[^\s<>"'`)\]]+)/g;
+const TRAILING_PUNCTUATION = /[.,;:!?]+$/;
+
+/** Blanks out fenced blocks and inline code spans. */
+function stripCode(text: string): string {
+ const out: string[] = [];
+ let fence: string | null = null;
+
+ for (const line of text.split("\n")) {
+ const match = line.match(/^[ \t]{0,3}(`{3,}|~{3,})/);
+
+ if (fence !== null) {
+ // A closing fence carries no info string, so anything trailing it means
+ // the line is code rather than the end of the block.
+ if (
+ match &&
+ match[1][0] === fence[0] &&
+ match[1].length >= fence.length &&
+ line.slice(match[0].length).trim() === ""
+ ) {
+ fence = null;
+ }
+ out.push("");
+ continue;
+ }
+
+ if (match) {
+ fence = match[1];
+ out.push("");
+ continue;
+ }
+
+ // Code spans are delimited by a run of backticks of matching length, so a
+ // shorter run inside a longer span must not terminate it.
+ out.push(line.replace(/(`+)(?:(?!\1)[^\n])*\1/g, " "));
+ }
+
+ return out.join("\n");
+}
+
+function resolve(
+ href: string,
+ baseUrl: string,
+ trimPunctuation: boolean,
+): string | null {
+ let candidate = href.trim();
+
+ if (candidate.startsWith("<") && candidate.endsWith(">")) {
+ candidate = candidate.slice(1, -1).trim();
+ }
+ // Only bare URLs pick up sentence punctuation; delimited destinations mean
+ // every character the author wrote.
+ if (trimPunctuation) {
+ candidate = candidate.replace(TRAILING_PUNCTUATION, "");
+ }
+
+ if (candidate === "" || candidate.startsWith("#")) {
+ return null;
+ }
+
+ try {
+ return new URL(candidate, baseUrl).href;
+ } catch {
+ return null;
+ }
+}
+
+/** Extracts links from a markdown or plaintext body, excluding images. */
+export function extractLinksFromMarkdown(
+ text: string,
+ baseUrl: string,
+): string[] {
+ const body = stripCode(text).replace(IMAGE, " ");
+ const links: string[] = [];
+
+ const push = (href: string, trimPunctuation = false) => {
+ const resolved = resolve(href, baseUrl, trimPunctuation);
+ if (resolved) {
+ links.push(resolved);
+ }
+ };
+
+ for (const match of body.matchAll(INLINE_LINK)) {
+ push(match[1]);
+ }
+ for (const match of body.matchAll(REFERENCE_DEFINITION)) {
+ push(match[1]);
+ }
+ for (const match of body.matchAll(AUTOLINK)) {
+ push(match[1]);
+ }
+ for (const match of body.matchAll(BARE_URL)) {
+ push(match[1], true);
+ }
+
+ return [...new Set(links)];
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 5ac616209..2df9b5da8 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -3,6 +3,7 @@ import { Meta } from "..";
import { Document } from "../../../controllers/v2/types";
import { htmlTransform } from "../lib/removeUnwantedElements";
import { extractLinks } from "../lib/extractLinks";
+import { isMarkdownContentType } from "../lib/extractLinksFromMarkdown";
import { extractImages } from "../lib/extractImages";
import { extractMetadata } from "../lib/extractMetadata";
import {
@@ -196,7 +197,9 @@ async function deriveLinksFromHTML(
meta: Meta,
document: Document,
): Promise<Document> {
- if (document.html === undefined) {
+ const isMarkdown = isMarkdownContentType(document.metadata.contentType);
+
+ if (document.html === undefined && !isMarkdown) {
throw new Error(
"html is undefined -- this transformer is being called out of order",
);
@@ -222,12 +225,15 @@ async function deriveLinksFromHTML(
return document;
}
+ // Engines that carry markdown natively (exchange) put synthetic metadata in
+ // rawHtml, so the markdown field is the only place their links live.
document.links = await extractLinks(
- document.html,
+ isMarkdown ? (document.markdown ?? document.rawHtml ?? "") : document.html!,
document.metadata.url ??
document.metadata.sourceURL ??
meta.rewrittenUrl ??
meta.url,
+ document.metadata.contentType,
);
if (forwardToIndexer) {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 8657077ad..c57744744 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -66,6 +66,7 @@ import {
calculateCreditsToBeBilled,
calculateThreatScanCredits,
} from "../../lib/scrape-billing";
+import { getCrawlScope } from "../../lib/crawl-scope";
import { billTeam } from "../billing/credit_billing";
import { getBillingQueue } from "../queue-service";
import type { Logger } from "winston";
@@ -74,6 +75,7 @@ import {
JobCancelledError,
RacedRedirectError,
ScrapeJobTimeoutError,
+ SitemapError,
TransportableError,
UnknownError,
} from "../../lib/error";
@@ -509,9 +511,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
if (!sc.crawlerOptions?.sitemapOnly) {
const links = await crawler.filterLinks(
- await crawler.extractLinksFromHTML(
+ await crawler.extractLinksFromContent(
rawHtml ?? "",
doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
+ doc.metadata?.contentType,
),
Infinity,
sc.crawlerOptions?.maxDepth ?? 10,
@@ -1229,12 +1232,10 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
const attempts: string[] = crawler.robots.getSitemaps();
- // Append sitemap.xml
+ // sitemap.xml in the directory the crawl is scoped to
const urlWithSitemap = new URL(urlObj.href);
urlWithSitemap.pathname =
- urlWithSitemap.pathname +
- (urlObj.pathname.endsWith("/") ? "" : "/") +
- "sitemap.xml";
+ getCrawlScope(urlObj.href).prefix + "sitemap.xml";
urlWithSitemap.search = "";
urlWithSitemap.hash = "";
attempts.push(urlWithSitemap.href);
@@ -1252,7 +1253,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
attempts.push(urlRootSitemap.href);
}
- for (const attempt of attempts) {
+ for (const attempt of new Set(attempts)) {
await addKickoffSitemapJob(attempt, job, sc, logger);
}
}
@@ -1539,7 +1540,11 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
}
return { success: true };
} catch (error) {
- logger.error("An error occurred!", { error });
+ if (error instanceof SitemapError && error.cause === 404) {
+ logger.debug("Sitemap not found", { sitemapUrl: job.data.sitemapUrl });
+ } else {
+ logger.error("An error occurred!", { error });
+ }
return { success: false, error };
} finally {
await redisEvictConnection.sadd(