Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): scope file-seed crawls to their directory and read markdown links (#4250)
tomsideguide6272c64a2eec
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 3dbb1b2b4..66023c2a3 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -135,6 +135,45 @@ fn is_file(path: &str) -> bool {}}+static DOCUMENT_SEGMENT: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"\.[A-Za-z0-9]{2,}$").unwrap());++struct CrawlScope {+ prefix: String,+ exact: Option<String>,+}++/// Resolves the path a crawl is scoped to when backward crawling is disabled.+/// Keep in sync with getCrawlScope() in apps/api/src/lib/crawl-scope.ts.+fn crawl_scope(initial_url: &Url) -> CrawlScope {+ let path = initial_url.path();++ if path.ends_with('/') {+ return CrawlScope {+ prefix: path.to_string(),+ exact: None,+ };+ }++ let last_slash = path.rfind('/').map_or(0, |i| i + 1);+ if DOCUMENT_SEGMENT.is_match(&path[last_slash..]) {+ return CrawlScope {+ prefix: path[..last_slash].to_string(),+ exact: None,+ };+ }++ CrawlScope {+ prefix: format!("{path}/"),+ exact: Some(path.to_string()),+ }+}++#[inline]+fn is_within_crawl_scope(path: &str, scope: &CrawlScope) -> bool {+ scope.exact.as_deref() == Some(path) || path.starts_with(&scope.prefix)+}+#[inline]fn get_url_depth(path: &str) -> u32 {path@@ -257,7 +296,7 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultlet base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {e}"))?;let initial_url =Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;- let initial_path = initial_url.path();+ let scope = crawl_scope(&initial_url);let excludes_regex: Vec<Regex> = data.excludes@@ -317,7 +356,7 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultcontinue;}- if !data.allow_backward_crawling && !path.starts_with(initial_path) {+ if !data.allow_backward_crawling && !is_within_crawl_scope(path, &scope) {denial_reasons.insert(link, BACKWARD_CRAWLING.to_string());continue;}@@ -753,6 +792,54 @@ pub async fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResmod tests {use super::*;+ fn scope_of(url: &str) -> CrawlScope {+ crawl_scope(&Url::parse(url).unwrap())+ }++ #[test]+ fn test_crawl_scope_document_seed_scopes_to_directory() {+ let scope = scope_of("https://example.com/docs/guide.md");+ assert_eq!(scope.prefix, "/docs/");+ assert_eq!(scope.exact, None);++ assert!(is_within_crawl_scope("/docs/other.md", &scope));+ assert!(is_within_crawl_scope("/docs/nested/deep.md", &scope));+ assert!(!is_within_crawl_scope("/blog/post.md", &scope));+ assert!(!is_within_crawl_scope("/", &scope));+ }++ #[test]+ fn test_crawl_scope_directory_seed() {+ let scope = scope_of("https://example.com/docs/");+ assert_eq!(scope.prefix, "/docs/");+ assert_eq!(scope.exact, None);++ assert!(is_within_crawl_scope("/docs/guide.md", &scope));+ assert!(!is_within_crawl_scope("/docs", &scope));+ assert!(!is_within_crawl_scope("/docsearch/x", &scope));+ }++ #[test]+ fn test_crawl_scope_extensionless_seed_allows_seed_and_children() {+ let scope = scope_of("https://example.com/docs");+ assert_eq!(scope.prefix, "/docs/");+ assert_eq!(scope.exact.as_deref(), Some("/docs"));++ assert!(is_within_crawl_scope("/docs", &scope));+ assert!(is_within_crawl_scope("/docs/guide", &scope));+ // Sibling paths sharing the seed's prefix must not sneak in.+ assert!(!is_within_crawl_scope("/docsearch", &scope));+ }++ #[test]+ fn test_crawl_scope_root_seed() {+ let scope = scope_of("https://example.com");+ assert_eq!(scope.prefix, "/");+ assert_eq!(scope.exact, None);++ assert!(is_within_crawl_scope("/anything/at/all", &scope));+ }+#[test]fn test_parse_sitemap_xml_urlset() {let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>diff --git a/apps/api/src/lib/crawl-scope.ts b/apps/api/src/lib/crawl-scope.tsnew file mode 100644index 000000000..21e7f1268--- /dev/null+++ b/apps/api/src/lib/crawl-scope.ts@@ -0,0 +1,46 @@+// Keep in sync with crawl_scope() in apps/api/native/src/crawler.rs.++interface CrawlScope {+ /** Directory every crawled path must sit under. Always ends with "/". */+ prefix: string;+ /** Single path allowed alongside the prefix, or null. */+ exact: string | null;+}++const DOCUMENT_SEGMENT = /\.[A-Za-z0-9]{2,}$/;++/** Resolves the path a crawl is scoped to when backward crawling is disabled. */+export function getCrawlScope(initialUrl: string): CrawlScope {+ let pathname: string;+ try {+ pathname = new URL(initialUrl).pathname;+ } catch {+ return { prefix: "/", exact: null };+ }++ if (!pathname.startsWith("/")) {+ pathname = "/" + pathname;+ }++ if (pathname.endsWith("/")) {+ return { prefix: pathname, exact: null };+ }++ const lastSlash = pathname.lastIndexOf("/");+ if (DOCUMENT_SEGMENT.test(pathname.slice(lastSlash + 1))) {+ return { prefix: pathname.slice(0, lastSlash + 1), exact: null };+ }++ return { prefix: pathname + "/", exact: pathname };+}++export function isWithinCrawlScope(path: string, scope: CrawlScope): boolean {+ return path === scope.exact || path.startsWith(scope.prefix);+}++/** Human-readable scope for denial messages. */+export function describeCrawlScope(scope: CrawlScope): string {+ return scope.exact+ ? `${scope.exact} or ${scope.prefix}*`+ : `${scope.prefix}*`;+}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 9b75fba3b..7e0b1f597 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -17,6 +17,15 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { ScrapeOptions } from "../../controllers/v2/types";import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";import { extractBaseDomain } from "../../lib/url-utils";+import {+ describeCrawlScope,+ getCrawlScope,+ isWithinCrawlScope,+} from "../../lib/crawl-scope";+import {+ extractLinksFromMarkdown,+ isMarkdownContentType,+} from "../scrapeURL/lib/extractLinksFromMarkdown";export const SITEMAP_LIMIT = 25;const SITEMAP_FETCH_CONCURRENCY = 3;@@ -153,7 +162,7 @@ export class WebCrawler {public setBaseUrl(newBase: string): void {this.baseUrl = newBase;- this.robotsTxtUrl = `${this.baseUrl}${this.baseUrl.endsWith("/") ? "" : "/"}robots.txt`;+ this.robotsTxtUrl = new URL("/robots.txt", newBase).href;}public async filterLinks(@@ -245,7 +254,9 @@ export class WebCrawler {);break;case "BACKWARD_CRAWLING":- const initialPath = new URL(this.initialUrl).pathname;+ const initialPath = describeCrawlScope(+ getCrawlScope(this.initialUrl),+ );fancyDenialReasons.set(key,`This URL's path ("${urlPath}") is outside the initial URL's path hierarchy ("${initialPath}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,@@ -288,6 +299,8 @@ export class WebCrawler {});}+ const scope = getCrawlScope(this.initialUrl);+const filteredLinks = sitemapLinks.filter(link => {let url: URL;@@ -396,17 +409,15 @@ export class WebCrawler {// }if (!this.allowBackwardCrawling) {- if (- !normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)- ) {+ if (!isWithinCrawlScope(normalizedLink.pathname, scope)) {if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(- `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,+ `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${describeCrawlScope(scope)}`,);}denialReasons.set(link,- `This URL's path ("${normalizedLink.pathname}") is outside the initial URL's path hierarchy ("${normalizedInitialUrl.pathname}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,+ `This URL's path ("${normalizedLink.pathname}") is outside the initial URL's path hierarchy ("${describeCrawlScope(scope)}"), and backward crawling is disabled. By default, Firecrawl only crawls URLs that are 'below' or 'within' the starting URL path. To crawl this URL, either set allowBackwardCrawling: true or set crawlEntireDomain: true to crawl the entire domain.`,);return false;}@@ -762,7 +773,26 @@ export class WebCrawler {return links;}- public async extractLinksFromHTML(html: string, url: string) {+ private async extractLinksFromMarkdownContent(text: string, url: string) {+ const filteredLinks: string[] = [];+ for (const link of extractLinksFromMarkdown(text, url)) {+ const filterResult = await this.filterURL(link, url);+ if (filterResult.allowed && filterResult.url) {+ filteredLinks.push(filterResult.url);+ }+ }+ return filteredLinks;+ }++ public async extractLinksFromContent(+ html: string,+ url: string,+ contentType?: string,+ ) {+ if (isMarkdownContentType(contentType)) {+ return await this.extractLinksFromMarkdownContent(html, url);+ }+try {return [...new Set(diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinks.tsindex 7aecd12de..e4f520f61 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts@@ -5,6 +5,10 @@ import {extractLinks as _extractLinks,extractBaseHref as _extractBaseHref,} from "@mendable/firecrawl-rs";+import {+ extractLinksFromMarkdown,+ isMarkdownContentType,+} from "./extractLinksFromMarkdown";function resolveUrlWithBaseHref(href: string,@@ -67,7 +71,12 @@ async function extractLinksRust(export async function extractLinks(html: string,baseUrl: string,+ contentType?: string,): Promise<string[]> {+ if (isMarkdownContentType(contentType)) {+ return extractLinksFromMarkdown(html, baseUrl);+ }+try {return await extractLinksRust(html, baseUrl);} catch (error) {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.tsnew file mode 100644index 000000000..a43a0d1cd--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinksFromMarkdown.ts@@ -0,0 +1,118 @@+const MARKDOWN_CONTENT_TYPES = [+ "text/plain",+ "text/markdown",+ "text/x-markdown",+];++/** Bodies served as text but authored as markdown, e.g. llms.txt. */+export function isMarkdownContentType(contentType?: string): boolean {+ if (!contentType) {+ return false;+ }+ // Media types are case-insensitive and may carry parameters ("; charset=").+ const mediaType = contentType.split(";")[0].trim().toLowerCase();+ return MARKDOWN_CONTENT_TYPES.includes(mediaType);+}++const IMAGE = /!\[[^\]]*\]\([^)]*\)/g;+const INLINE_LINK =+ /\[[^\]]*\]\(\s*(<[^>\n]*>|[^)\s]+)(?:\s+["'][^"']*["'])?\s*\)/g;+const REFERENCE_DEFINITION = /^[ \t]{0,3}\[[^\]]+\]:[ \t]*(<[^>]+>|\S+)/gm;+const AUTOLINK = /<((?:https?:\/\/|mailto:)[^>\s]+)>/g;+const BARE_URL = /(?:^|[\s(<])(https?:\/\/[^\s<>"'`)\]]+)/g;+const TRAILING_PUNCTUATION = /[.,;:!?]+$/;++/** Blanks out fenced blocks and inline code spans. */+function stripCode(text: string): string {+ const out: string[] = [];+ let fence: string | null = null;++ for (const line of text.split("\n")) {+ const match = line.match(/^[ \t]{0,3}(`{3,}|~{3,})/);++ if (fence !== null) {+ // A closing fence carries no info string, so anything trailing it means+ // the line is code rather than the end of the block.+ if (+ match &&+ match[1][0] === fence[0] &&+ match[1].length >= fence.length &&+ line.slice(match[0].length).trim() === ""+ ) {+ fence = null;+ }+ out.push("");+ continue;+ }++ if (match) {+ fence = match[1];+ out.push("");+ continue;+ }++ // Code spans are delimited by a run of backticks of matching length, so a+ // shorter run inside a longer span must not terminate it.+ out.push(line.replace(/(`+)(?:(?!\1)[^\n])*\1/g, " "));+ }++ return out.join("\n");+}++function resolve(+ href: string,+ baseUrl: string,+ trimPunctuation: boolean,+): string | null {+ let candidate = href.trim();++ if (candidate.startsWith("<") && candidate.endsWith(">")) {+ candidate = candidate.slice(1, -1).trim();+ }+ // Only bare URLs pick up sentence punctuation; delimited destinations mean+ // every character the author wrote.+ if (trimPunctuation) {+ candidate = candidate.replace(TRAILING_PUNCTUATION, "");+ }++ if (candidate === "" || candidate.startsWith("#")) {+ return null;+ }++ try {+ return new URL(candidate, baseUrl).href;+ } catch {+ return null;+ }+}++/** Extracts links from a markdown or plaintext body, excluding images. */+export function extractLinksFromMarkdown(+ text: string,+ baseUrl: string,+): string[] {+ const body = stripCode(text).replace(IMAGE, " ");+ const links: string[] = [];++ const push = (href: string, trimPunctuation = false) => {+ const resolved = resolve(href, baseUrl, trimPunctuation);+ if (resolved) {+ links.push(resolved);+ }+ };++ for (const match of body.matchAll(INLINE_LINK)) {+ push(match[1]);+ }+ for (const match of body.matchAll(REFERENCE_DEFINITION)) {+ push(match[1]);+ }+ for (const match of body.matchAll(AUTOLINK)) {+ push(match[1]);+ }+ for (const match of body.matchAll(BARE_URL)) {+ push(match[1], true);+ }++ return [...new Set(links)];+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 5ac616209..2df9b5da8 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -3,6 +3,7 @@ import { Meta } from "..";import { Document } from "../../../controllers/v2/types";import { htmlTransform } from "../lib/removeUnwantedElements";import { extractLinks } from "../lib/extractLinks";+import { isMarkdownContentType } from "../lib/extractLinksFromMarkdown";import { extractImages } from "../lib/extractImages";import { extractMetadata } from "../lib/extractMetadata";import {@@ -196,7 +197,9 @@ async function deriveLinksFromHTML(meta: Meta,document: Document,): Promise<Document> {- if (document.html === undefined) {+ const isMarkdown = isMarkdownContentType(document.metadata.contentType);++ if (document.html === undefined && !isMarkdown) {throw new Error("html is undefined -- this transformer is being called out of order",);@@ -222,12 +225,15 @@ async function deriveLinksFromHTML(return document;}+ // Engines that carry markdown natively (exchange) put synthetic metadata in+ // rawHtml, so the markdown field is the only place their links live.document.links = await extractLinks(- document.html,+ isMarkdown ? (document.markdown ?? document.rawHtml ?? "") : document.html!,document.metadata.url ??document.metadata.sourceURL ??meta.rewrittenUrl ??meta.url,+ document.metadata.contentType,);if (forwardToIndexer) {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 8657077ad..c57744744 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -66,6 +66,7 @@ import {calculateCreditsToBeBilled,calculateThreatScanCredits,} from "../../lib/scrape-billing";+import { getCrawlScope } from "../../lib/crawl-scope";import { billTeam } from "../billing/credit_billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";@@ -74,6 +75,7 @@ import {JobCancelledError,RacedRedirectError,ScrapeJobTimeoutError,+ SitemapError,TransportableError,UnknownError,} from "../../lib/error";@@ -509,9 +511,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {if (!sc.crawlerOptions?.sitemapOnly) {const links = await crawler.filterLinks(- await crawler.extractLinksFromHTML(+ await crawler.extractLinksFromContent(rawHtml ?? "",doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,+ doc.metadata?.contentType,),Infinity,sc.crawlerOptions?.maxDepth ?? 10,@@ -1229,12 +1232,10 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {const attempts: string[] = crawler.robots.getSitemaps();- // Append sitemap.xml+ // sitemap.xml in the directory the crawl is scoped toconst urlWithSitemap = new URL(urlObj.href);urlWithSitemap.pathname =- urlWithSitemap.pathname +- (urlObj.pathname.endsWith("/") ? "" : "/") +- "sitemap.xml";+ getCrawlScope(urlObj.href).prefix + "sitemap.xml";urlWithSitemap.search = "";urlWithSitemap.hash = "";attempts.push(urlWithSitemap.href);@@ -1252,7 +1253,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {attempts.push(urlRootSitemap.href);}- for (const attempt of attempts) {+ for (const attempt of new Set(attempts)) {await addKickoffSitemapJob(attempt, job, sc, logger);}}@@ -1539,7 +1540,11 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {}return { success: true };} catch (error) {- logger.error("An error occurred!", { error });+ if (error instanceof SitemapError && error.cause === 404) {+ logger.debug("Sitemap not found", { sitemapUrl: job.data.sitemapUrl });+ } else {+ logger.error("An error occurred!", { error });+ }return { success: false, error };} finally {await redisEvictConnection.sadd(