Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add firecrawl-local, a single binary for scrape, crawl and map on your own machine (#5685)
Eric Ciarla9f66dd4c677c
diff --git a/.github/workflows/publish-local.yml b/.github/workflows/publish-local.ymlnew file mode 100644index 000000000..95b1c3009--- /dev/null+++ b/.github/workflows/publish-local.yml@@ -0,0 +1,205 @@+name: Publish Firecrawl Local++# Publishes the firecrawl-local crate to crates.io with trusted publishing+# when the version in apps/local/Cargo.toml is newer than the published one.+#+# crates.io setup (once, by a crate owner): the first version is published by+# hand, because trusted publishing only works for a crate that already exists.+# Then add a trusted publisher under the crate's settings with repository+# firecrawl/firecrawl, workflow publish-local.yml and environment release.++on:+ workflow_dispatch:+ push:+ branches:+ - main+ paths:+ - 'apps/local/**'++permissions: {}++jobs:+ build:+ name: Build and package+ runs-on: blacksmith-4vcpu-ubuntu-2404+ permissions:+ contents: read+ outputs:+ publish: ${{ steps.gate.outputs.publish }}+ crate_sha256: ${{ steps.package.outputs.sha256 }}+ steps:+ - name: Checkout repository+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ with:+ persist-credentials: false++ - name: Set up Python+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5+ with:+ python-version: '3.x'++ - name: Install version check dependencies+ run: |+ python -m pip install --no-cache-dir -r .github/pip-bootstrap.txt+ pip install requests==2.34.2 packaging==26.3 toml==0.10.2++ - name: Run version check script+ id: version_check+ run: |+ VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py rust ./apps/local firecrawl-local)+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"++ # Trusted publishing cannot create a crate. Until the first version is+ # published by hand, there is nothing for this workflow to do.+ - name: Check the crate exists on crates.io+ id: gate+ env:+ VERSION_INCREMENTED: ${{ steps.version_check.outputs.publish }}+ run: |+ status=$(curl -s -o /dev/null -w '%{http_code}' -A 'firecrawl-version-check' https://crates.io/api/v1/crates/firecrawl-local)+ if [ "$status" = "404" ]; then+ echo "::notice::firecrawl-local is not on crates.io yet. Publish the first version by hand (see apps/local/README.md), then this workflow takes over."+ echo "publish=false" >> "$GITHUB_OUTPUT"+ elif [ "$status" != "200" ]; then+ echo "::error::crates.io answered $status when asked about firecrawl-local"+ exit 1+ else+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"+ fi++ - name: Install Rust toolchain+ if: steps.gate.outputs.publish == 'true'+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable++ # A published crate cannot reach apps/api/native/src, so it carries a copy.+ - name: Copy the shared cleaning code into the crate+ if: steps.gate.outputs.publish == 'true'+ working-directory: ./apps/local+ run: ./scripts/vendor-native.sh++ # cargo package builds the crate from the packaged files alone, which is+ # what `cargo install firecrawl-local` will do.+ - name: Package+ id: package+ if: steps.gate.outputs.publish == 'true'+ working-directory: ./apps/local+ run: |+ cargo package --allow-dirty+ crate=$(ls target/package/*.crate)+ echo "sha256=$(sha256sum "$crate" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"+ mkdir -p "$RUNNER_TEMP/dist" && cp "$crate" "$RUNNER_TEMP/dist/"++ - name: Run tests+ if: steps.gate.outputs.publish == 'true'+ working-directory: ./apps/local+ run: cargo test --locked++ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ if: steps.gate.outputs.publish == 'true'+ with:+ name: firecrawl-local-crate+ path: ${{ runner.temp }}/dist+ if-no-files-found: error+ retention-days: 7++ supersede:+ name: Supersede unapproved runs+ needs: build+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ permissions:+ actions: write+ contents: read+ steps:+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ continue-on-error: true+ with:+ sparse-checkout: .github/actions+ persist-credentials: false+ - uses: ./.github/actions/supersede-waiting-runs+ continue-on-error: true+ timeout-minutes: 2++ release-summary:+ name: Release summary+ needs: build+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ permissions:+ contents: read+ actions: read+ pull-requests: read+ steps:+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ continue-on-error: true+ with:+ fetch-depth: 0+ filter: blob:none+ persist-credentials: false+ - name: Read the version+ id: version+ continue-on-error: true+ run: |+ echo "version=$(sed -nE 's/^version *= *"([^"]+)".*/\1/p' apps/local/Cargo.toml | head -1)" >> "$GITHUB_OUTPUT"+ - uses: ./.github/actions/release-summary+ continue-on-error: true+ timeout-minutes: 3+ with:+ gated-job: Publish to crates.io+ version: firecrawl-local ${{ steps.version.outputs.version }}+ paths: apps/local++ publish:+ name: Publish to crates.io+ needs: [build, release-summary]+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ environment: release+ concurrency:+ group: ${{ github.workflow }}-release-${{ github.ref }}+ cancel-in-progress: false+ permissions:+ contents: read+ id-token: write+ steps:+ - name: Checkout repository+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ with:+ persist-credentials: false++ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: firecrawl-local-crate+ path: ${{ runner.temp }}/dist++ - name: Install Rust toolchain+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable++ - name: Copy the shared cleaning code into the crate+ working-directory: ./apps/local+ run: ./scripts/vendor-native.sh++ # cargo publish uploads what cargo package produces from this commit; refuse+ # unless that is byte-identical to the crate the build job tested.+ - name: Check the crate matches the build+ working-directory: ./apps/local+ env:+ EXPECTED: ${{ needs.build.outputs.crate_sha256 }}+ run: |+ cargo package --no-verify --allow-dirty+ actual=$(sha256sum target/package/*.crate | cut -d' ' -f1)+ artifact=$(sha256sum "$RUNNER_TEMP"/dist/*.crate | cut -d' ' -f1)+ if [ "$actual" != "$EXPECTED" ] || [ "$artifact" != "$EXPECTED" ]; then+ echo "::error::Crate mismatch: build $EXPECTED, artifact $artifact, publish $actual"+ exit 1+ fi++ - name: Authenticate to crates.io+ id: auth+ uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6++ - name: Publish to crates.io+ working-directory: ./apps/local+ env:+ CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}+ run: cargo publish --no-verify --allow-dirtydiff --git a/.github/workflows/test-local.yml b/.github/workflows/test-local.ymlnew file mode 100644index 000000000..075d86b0b--- /dev/null+++ b/.github/workflows/test-local.yml@@ -0,0 +1,66 @@+name: Firecrawl Local Test Suite++on:+ pull_request:+ branches:+ - main+ paths:+ - apps/local/**+ - apps/api/native/src/**+ - .github/workflows/test-local.yml++permissions:+ contents: read++jobs:+ test:+ name: Build and Test+ runs-on: blacksmith-4vcpu-ubuntu-2404+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false++ - name: Install Rust toolchain+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable+ with:+ components: rustfmt++ - name: Cache cargo registry+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ apps/local/target+ key: ${{ runner.os }}-cargo-local-${{ hashFiles('apps/local/Cargo.lock') }}+ restore-keys: |+ ${{ runner.os }}-cargo-local-++ # Firecrawl Local compiles the API's native cleaning code directly, so this+ # also checks those shared files.+ - name: Check formatting+ working-directory: ./apps/local+ run: cargo fmt --check++ - name: Build+ working-directory: ./apps/local+ run: cargo build --locked++ - name: Run unit tests+ working-directory: ./apps/local+ run: cargo test --locked++ # Test the actual archive and normalized manifest, including registry+ # dependencies, rather than a copy of the repository manifest.+ - name: Build and test the packaged crate+ working-directory: ./apps/local+ run: |+ ./scripts/vendor-native.sh+ cargo package --no-verify --allow-dirty+ out="$RUNNER_TEMP/firecrawl-local-package"+ mkdir -p "$out"+ tar -xzf target/package/firecrawl-local-*.crate -C "$out" --strip-components=1+ test -f "$out/src/native/html.rs"+ # Keep repository build-script output out of the packaged build.+ CARGO_TARGET_DIR="$out/target" cargo test --locked --manifest-path "$out/Cargo.toml"diff --git a/apps/api/native/src/document.rs b/apps/api/native/src/document.rsindex b0976b269..832fdb315 100644--- a/apps/api/native/src/document.rs+++ b/apps/api/native/src/document.rs@@ -65,7 +65,7 @@ pub async fn convert_document_to_markdown(})?}-fn convert_document_bytes_to_markdown(+pub fn convert_document_bytes_to_markdown(data: &[u8],extension_hint: Option<String>,) -> Result<String> {diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 72ff8d0a1..7b9deecb0 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -35,6 +35,12 @@ fn _extract_base_href_from_document(Ok(url.to_string())}+/// Base URL of an already-parsed document: its `<base href>` if it has one,+/// otherwise the page URL.+pub fn base_href_of_document(document: &NodeRef, url: &Url) -> String {+ _extract_base_href_from_document(document, url).unwrap_or_else(|_| url.to_string())+}+fn _extract_base_href(html: &str,url: &str,@@ -59,6 +65,33 @@ pub async fn extract_base_href(html: String, url: String) -> napi::Result<Stringres.map_err(to_napi_err)}+/// Raw `href` of every link in an already-parsed document.+pub fn extract_links_from_document(document: &NodeRef) -> Result<Vec<String>, String> {+ let anchors: Vec<_> = document+ .select("a[href]")+ .map_err(|_| "Failed to select links".to_string())?+ .collect();++ let mut out: Vec<String> = Vec::new();++ for anchor in anchors {+ let mut href = match anchor.attributes.borrow().get("href") {+ Some(x) => x.to_string(),+ None => continue,+ };++ if href.starts_with("http:/") && !href.starts_with("http://") {+ href = format!("http://{}", &href[6..]);+ } else if href.starts_with("https:/") && !href.starts_with("https://") {+ href = format!("https://{}", &href[7..]);+ }++ out.push(href);+ }++ Ok(out)+}+/// Extract all links from HTML document.#[napi]pub async fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {@@ -69,30 +102,7 @@ pub async fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {};let document = parse_html().one(html.as_str());-- let anchors: Vec<_> = document- .select("a[href]")- .map_err(|_| to_napi_err("Failed to select links"))?- .collect();-- let mut out: Vec<String> = Vec::new();-- for anchor in anchors {- let mut href = match anchor.attributes.borrow().get("href") {- Some(x) => x.to_string(),- None => continue,- };-- if href.starts_with("http:/") && !href.starts_with("http://") {- href = format!("http://{}", &href[6..]);- } else if href.starts_with("https:/") && !href.starts_with("https://") {- href = format!("https://{}", &href[7..]);- }-- out.push(href);- }-- Ok(out)+ extract_links_from_document(&document).map_err(to_napi_err)}).await.map_err(|e| {@@ -145,6 +155,13 @@ fn _extract_metadata(html: &str,) -> Result<HashMap<String, Value>, Box<dyn std::error::Error + Send + Sync>> {let document = parse_html().one(html);+ extract_metadata_from_document(&document)+}++/// Metadata of an already-parsed document.+pub fn extract_metadata_from_document(+ document: &NodeRef,+) -> Result<HashMap<String, Value>, Box<dyn std::error::Error + Send + Sync>> {let mut out = HashMap::<String, Value>::new();let head_node = document@@ -428,7 +445,16 @@ struct ImageSource {fn _transform_html_inner(opts: TransformHtmlOptions,) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {- let mut document = parse_html().one(opts.html.as_ref());+ let document = parse_html().one(opts.html.as_ref());+ Ok(transform_document(document, &opts)?.to_string())+}++/// Clean an already-parsed document with the same rules as `transform_html`,+/// without parsing it again. `opts.html` is ignored.+pub fn transform_document(+ mut document: NodeRef,+ opts: &TransformHtmlOptions,+) -> Result<NodeRef, Box<dyn std::error::Error + Send + Sync>> {let url = Url::parse(&_extract_base_href_from_document(&document,&Url::parse(&opts.url)?,@@ -678,7 +704,7 @@ fn _transform_html_inner(}}- Ok(document.to_string())+ Ok(document)}/// Transform and clean HTML content based on provided options.@@ -982,42 +1008,45 @@ pub async fn extract_images(html: String, base_url: String) -> napi::Result<Vec<res.map_err(to_napi_err)}-/// Process multi-line links in markdown.-#[napi]-pub async fn post_process_markdown(markdown: String) -> napi::Result<String> {- let res = task::spawn_blocking(move || {- let mut link_open_count = 0usize;- let mut out = String::with_capacity(markdown.len());-- for ch in markdown.chars() {- match ch {- '[' => {- link_open_count += 1;- }- ']' => {- link_open_count = link_open_count.saturating_sub(1);- }- _ => {}- }+/// The synchronous body of `post_process_markdown`.+pub fn post_process_markdown_text(markdown: &str) -> String {+ let mut link_open_count = 0usize;+ let mut out = String::with_capacity(markdown.len());- let inside_link_content = link_open_count > 0;- if inside_link_content && ch == '\n' {- out.push('\\');- out.push('\n');- } else {- out.push(ch);+ for ch in markdown.chars() {+ match ch {+ '[' => {+ link_open_count += 1;+ }+ ']' => {+ link_open_count = link_open_count.saturating_sub(1);}+ _ => {}}- remove_skip_to_content_links(&out)- })- .await- .map_err(|e| {- napi::Error::new(- napi::Status::GenericFailure,- format!("post_process_markdown join error: {e}"),- )- })?;+ let inside_link_content = link_open_count > 0;+ if inside_link_content && ch == '\n' {+ out.push('\\');+ out.push('\n');+ } else {+ out.push(ch);+ }+ }++ remove_skip_to_content_links(&out)+}++/// Process multi-line links in markdown.+#[napi]+pub async fn post_process_markdown(markdown: String) -> napi::Result<String> {+ let res = task::spawn_blocking(move || post_process_markdown_text(&markdown))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("post_process_markdown join error: {e}"),+ )+ })?;Ok(res)}diff --git a/apps/local/.gitignore b/apps/local/.gitignorenew file mode 100644index 000000000..d20dce13d--- /dev/null+++ b/apps/local/.gitignore@@ -0,0 +1,5 @@+target/+target-linux/++# The shared cleaning code, copied in only while packaging a release.+/src/native/diff --git a/apps/local/Cargo.lock b/apps/local/Cargo.locknew file mode 100644index 000000000..fd3a489eb--- /dev/null+++ b/apps/local/Cargo.lock@@ -0,0 +1,4054 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 4++[[package]]+name = "adler2"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"++[[package]]+name = "aes"+version = "0.9.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1"+dependencies = [+ "cipher",+ "cpubits",+ "cpufeatures 0.3.1",+]++[[package]]+name = "aho-corasick"+version = "1.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"+dependencies = [+ "memchr",+]++[[package]]+name = "alloc-no-stdlib"+version = "2.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"++[[package]]+name = "alloc-no-stdlib"+version = "3.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"++[[package]]+name = "alloc-stdlib"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"+dependencies = [+ "alloc-no-stdlib 2.0.4",+]++[[package]]+name = "alloc-stdlib"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"+dependencies = [+ "alloc-no-stdlib 3.0.0",+]++[[package]]+name = "allocator-api2"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"++[[package]]+name = "android_system_properties"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"+dependencies = [+ "libc",+]++[[package]]+name = "anstream"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"+dependencies = [+ "anstyle",+ "anstyle-parse",+ "anstyle-query",+ "anstyle-wincon",+ "colorchoice",+ "is_terminal_polyfill",+ "utf8parse",+]++[[package]]+name = "anstyle"+version = "1.0.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"++[[package]]+name = "anstyle-parse"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"+dependencies = [+ "utf8parse",+]++[[package]]+name = "anstyle-query"+version = "1.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "anstyle-wincon"+version = "3.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"+dependencies = [+ "anstyle",+ "once_cell_polyfill",+ "windows-sys 0.61.2",+]++[[package]]+name = "anydoc"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf0d78e4cfe3654eb3ea04422ec5a94352000ee56e97902125e68e51f7527062"+dependencies = [+ "cfb",+ "csv",+ "encoding_rs",+ "flate2",+ "log",+ "pdf-inspector",+ "quick-xml",+ "zip",+]++[[package]]+name = "anyhow"+version = "1.0.104"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"++[[package]]+name = "async-compression"+version = "0.4.52"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"+dependencies = [+ "compression-codecs",+ "compression-core",+ "pin-project-lite",+ "tokio",+]++[[package]]+name = "async-tungstenite"+version = "0.32.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8acc405d38be14342132609f06f02acaf825ddccfe76c4824a69281e0458ebd4"+dependencies = [+ "atomic-waker",+ "futures-core",+ "futures-io",+ "futures-task",+ "futures-util",+ "log",+ "pin-project-lite",+ "tokio",+ "tungstenite",+]++[[package]]+name = "atomic-waker"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"++[[package]]+name = "autocfg"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"++[[package]]+name = "aws-lc-rs"+version = "1.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53"+dependencies = [+ "aws-lc-sys",+ "zeroize",+]++[[package]]+name = "aws-lc-sys"+version = "0.47.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52"+dependencies = [+ "cc",+ "cmake",+ "dunce",+ "fs_extra",+ "pkg-config",+]++[[package]]+name = "axum"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"+dependencies = [+ "axum-core",+ "bytes",+ "form_urlencoded",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-util",+ "itoa 1.0.19",+ "matchit",+ "memchr",+ "mime",+ "percent-encoding",+ "pin-project-lite",+ "serde_core",+ "serde_json",+ "serde_path_to_error",+ "serde_urlencoded",+ "sync_wrapper",+ "tokio",+ "tower",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "axum-core"+version = "0.5.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "http-body-util",+ "mime",+ "pin-project-lite",+ "sync_wrapper",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "base64"+version = "0.22.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"++[[package]]+name = "base64"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"++[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"++[[package]]+name = "bitflags"+version = "2.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"++[[package]]+name = "block-buffer"+version = "0.10.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"+dependencies = [+ "generic-array",+]++[[package]]+name = "block-buffer"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"+dependencies = [+ "hybrid-array",+]++[[package]]+name = "block-padding"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"+dependencies = [+ "hybrid-array",+]++[[package]]+name = "brotli"+version = "9.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81"+dependencies = [+ "alloc-no-stdlib 3.0.0",+ "alloc-stdlib 0.3.0",+ "brotli-decompressor 6.3.1",+]++[[package]]+name = "brotli-decompressor"+version = "5.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"+dependencies = [+ "alloc-no-stdlib 2.0.4",+ "alloc-stdlib 0.2.4",+]++[[package]]+name = "brotli-decompressor"+version = "6.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f"+dependencies = [+ "alloc-no-stdlib 3.0.0",+ "alloc-stdlib 0.3.0",+]++[[package]]+name = "bstr"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"+dependencies = [+ "lazy_static",+ "memchr",+ "regex-automata 0.1.10",+]++[[package]]+name = "bumpalo"+version = "3.20.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"++[[package]]+name = "byteorder"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"++[[package]]+name = "bytes"+version = "1.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"+dependencies = [+ "serde",+]++[[package]]+name = "cbc"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"+dependencies = [+ "cipher",+]++[[package]]+name = "cc"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155"+dependencies = [+ "find-msvc-tools",+ "jobserver",+ "libc",+ "shlex",+]++[[package]]+name = "cfb"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3"+dependencies = [+ "fnv",+ "uuid",+ "web-time",+]++[[package]]+name = "cfg-if"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"++[[package]]+name = "cfg_aliases"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"++[[package]]+name = "chacha20"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"+dependencies = [+ "cfg-if",+ "cpufeatures 0.3.1",+ "rand_core 0.10.1",+]++[[package]]+name = "chromiumoxide"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26ed067eb6c1f660bdb87c05efb964421d2ca262bae0296cdfe38cf0cd949a3e"+dependencies = [+ "async-tungstenite",+ "base64 0.22.1",+ "bytes",+ "chromiumoxide_cdp",+ "chromiumoxide_types",+ "dunce",+ "fnv",+ "futures",+ "futures-timer",+ "pin-project-lite",+ "reqwest",+ "serde",+ "serde_json",+ "thiserror 2.0.21",+ "tokio",+ "tracing",+ "url",+ "which",+ "windows-registry",+]++[[package]]+name = "chromiumoxide_cdp"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68a6a03a7ebac4ea85308f285d6959a3e6b2ce32a0c9465dc7a7b1db0144eec7"+dependencies = [+ "chromiumoxide_pdl",+ "chromiumoxide_types",+ "serde",+ "serde_json",+]++[[package]]+name = "chromiumoxide_pdl"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c602dea92337bc4d824668d78c5b79c3b4ddb29b40dd7218282bbe8fd3fc2091"+dependencies = [+ "chromiumoxide_types",+ "either",+ "heck",+ "once_cell",+ "proc-macro2",+ "quote",+ "regex",+ "serde_json",+]++[[package]]+name = "chromiumoxide_types"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "678d5146e74f16fc4a41978b275af572cd913de1f10270d2b93b6c276bc57d80"+dependencies = [+ "serde",+ "serde_json",+]++[[package]]+name = "chrono"+version = "0.4.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"+dependencies = [+ "iana-time-zone",+ "js-sys",+ "num-traits",+ "wasm-bindgen",+ "windows-link",+]++[[package]]+name = "cipher"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"+dependencies = [+ "crypto-common 0.2.2",+ "inout",+]++[[package]]+name = "clap"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"+dependencies = [+ "clap_builder",+ "clap_derive",+]++[[package]]+name = "clap_builder"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"+dependencies = [+ "anstream",+ "anstyle",+ "clap_lex",+ "strsim",+]++[[package]]+name = "clap_derive"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"+dependencies = [+ "heck",+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "clap_lex"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"++[[package]]+name = "cmake"+version = "0.1.58"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"+dependencies = [+ "cc",+]++[[package]]+name = "colorchoice"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"++[[package]]+name = "combine"+version = "4.6.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92"+dependencies = [+ "bytes",+ "memchr",+]++[[package]]+name = "compression-codecs"+version = "0.4.47"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"+dependencies = [+ "brotli",+ "compression-core",+ "flate2",+ "memchr",+ "zstd",+ "zstd-safe",+]++[[package]]+name = "compression-core"+version = "0.4.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a"++[[package]]+name = "const-oid"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a"++[[package]]+name = "convert_case"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"++[[package]]+name = "convert_case"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b"+dependencies = [+ "unicode-segmentation",+]++[[package]]+name = "core-foundation"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"++[[package]]+name = "core_detect"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"++[[package]]+name = "cpubits"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"++[[package]]+name = "cpufeatures"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"+dependencies = [+ "libc",+]++[[package]]+name = "cpufeatures"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"+dependencies = [+ "libc",+]++[[package]]+name = "crc32fast"+version = "1.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64"+dependencies = [+ "cfg-if",+]++[[package]]+name = "crossbeam-deque"+version = "0.8.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"+dependencies = [+ "crossbeam-epoch",+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-epoch"+version = "0.9.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"+dependencies = [+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-utils"+version = "0.8.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"++[[package]]+name = "crypto-common"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"+dependencies = [+ "generic-array",+ "typenum",+]++[[package]]+name = "crypto-common"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"+dependencies = [+ "hybrid-array",+]++[[package]]+name = "cssparser"+version = "0.27.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"+dependencies = [+ "cssparser-macros",+ "dtoa-short",+ "itoa 0.4.8",+ "matches",+ "phf 0.8.0",+ "proc-macro2",+ "quote",+ "smallvec",+ "syn 1.0.109",+]++[[package]]+name = "cssparser"+version = "0.36.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2"+dependencies = [+ "cssparser-macros",+ "dtoa-short",+ "itoa 1.0.19",+ "phf 0.13.1",+ "smallvec",+]++[[package]]+name = "cssparser-macros"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"+dependencies = [+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "csv"+version = "1.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"+dependencies = [+ "csv-core",+ "itoa 1.0.19",+ "ryu",+ "serde_core",+]++[[package]]+name = "csv-core"+version = "0.1.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"+dependencies = [+ "memchr",+]++[[package]]+name = "ctor"+version = "1.0.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"++[[package]]+name = "cty"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"++[[package]]+name = "data-encoding"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"++[[package]]+name = "derive_more"+version = "0.99.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f"+dependencies = [+ "convert_case 0.4.0",+ "proc-macro2",+ "quote",+ "rustc_version",+ "syn 2.0.119",+]++[[package]]+name = "derive_more"+version = "2.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"+dependencies = [+ "derive_more-impl",+]++[[package]]+name = "derive_more-impl"+version = "2.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"+dependencies = [+ "proc-macro2",+ "quote",+ "rustc_version",+ "syn 2.0.119",+]++[[package]]+name = "digest"+version = "0.10.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"+dependencies = [+ "block-buffer 0.10.4",+ "crypto-common 0.1.7",+]++[[package]]+name = "digest"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"+dependencies = [+ "block-buffer 0.12.1",+ "const-oid",+ "crypto-common 0.2.2",+]++[[package]]+name = "displaydoc"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "dtoa"+version = "1.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"++[[package]]+name = "dtoa-short"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"+dependencies = [+ "dtoa",+]++[[package]]+name = "dunce"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"++[[package]]+name = "ecb"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26f2a8b3e564eba0877223dc343703ad0385794e882e6d13f3a4dd5c6b1f41ac"+dependencies = [+ "cipher",+]++[[package]]+name = "either"+version = "1.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"++[[package]]+name = "encoding_rs"+version = "0.8.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"+dependencies = [+ "cfg-if",+ "core_detect",+ "multiversion_no_op",+ "rustversion",+ "scopeguard",+ "simdutf8",+]++[[package]]+name = "env_filter"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"+dependencies = [+ "log",+ "regex",+]++[[package]]+name = "env_logger"+version = "0.11.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"+dependencies = [+ "anstream",+ "anstyle",+ "env_filter",+ "jiff",+ "log",+]++[[package]]+name = "equivalent"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"++[[package]]+name = "errno"+version = "0.3.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"+dependencies = [+ "libc",+ "windows-sys 0.61.2",+]++[[package]]+name = "fastrand"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"++[[package]]+name = "find-msvc-tools"+version = "0.1.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"++[[package]]+name = "firecrawl-local"+version = "0.1.0"+dependencies = [+ "anydoc",+ "axum",+ "base64 0.22.1",+ "bytes",+ "chromiumoxide",+ "chrono",+ "clap",+ "encoding_rs",+ "futures",+ "getrandom 0.3.4",+ "kuchikiki",+ "libmimalloc-sys",+ "lol_html",+ "mimalloc",+ "napi",+ "napi-derive",+ "nodesig",+ "once_cell",+ "pdf-inspector",+ "psl",+ "regex",+ "reqwest",+ "roxmltree",+ "serde",+ "serde_json",+ "sha2 0.10.9",+ "strsim",+ "texting_robots",+ "tinyvec",+ "tokio",+ "url",+ "uuid",+ "zip",+]++[[package]]+name = "flate2"+version = "1.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"+dependencies = [+ "crc32fast",+ "miniz_oxide",+ "zlib-rs",+]++[[package]]+name = "fnv"+version = "1.0.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"++[[package]]+name = "foldhash"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"++[[package]]+name = "form_urlencoded"+version = "1.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"+dependencies = [+ "percent-encoding",+]++[[package]]+name = "fs_extra"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"++[[package]]+name = "futf"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"+dependencies = [+ "mac",+ "new_debug_unreachable",+]++[[package]]+name = "futures"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"+dependencies = [+ "futures-channel",+ "futures-core",+ "futures-executor",+ "futures-io",+ "futures-sink",+ "futures-task",+ "futures-util",+]++[[package]]+name = "futures-channel"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"+dependencies = [+ "futures-core",+ "futures-sink",+]++[[package]]+name = "futures-core"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"++[[package]]+name = "futures-executor"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+]++[[package]]+name = "futures-io"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"++[[package]]+name = "futures-macro"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "futures-sink"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"++[[package]]+name = "futures-task"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"++[[package]]+name = "futures-timer"+version = "3.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"++[[package]]+name = "futures-util"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"+dependencies = [+ "futures-channel",+ "futures-core",+ "futures-io",+ "futures-macro",+ "futures-sink",+ "futures-task",+ "memchr",+ "pin-project-lite",+ "slab",+]++[[package]]+name = "fxhash"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"+dependencies = [+ "byteorder",+]++[[package]]+name = "generic-array"+version = "0.14.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"+dependencies = [+ "typenum",+ "version_check",+]++[[package]]+name = "getrandom"+version = "0.1.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"+dependencies = [+ "cfg-if",+ "libc",+ "wasi 0.9.0+wasi-snapshot-preview1",+]++[[package]]+name = "getrandom"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"+dependencies = [+ "cfg-if",+ "js-sys",+ "libc",+ "wasi 0.11.1+wasi-snapshot-preview1",+ "wasm-bindgen",+]++[[package]]+name = "getrandom"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi 5.3.0",+ "wasip2",+]++[[package]]+name = "getrandom"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"+dependencies = [+ "cfg-if",+ "js-sys",+ "libc",+ "r-efi 6.0.0",+ "rand_core 0.10.1",+ "wasm-bindgen",+]++[[package]]+name = "h2"+version = "0.4.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118"+dependencies = [+ "atomic-waker",+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "http",+ "indexmap 2.14.2",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]++[[package]]+name = "hashbrown"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"++[[package]]+name = "hashbrown"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"+dependencies = [+ "allocator-api2",+ "equivalent",+ "foldhash",+]++[[package]]+name = "heck"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"++[[package]]+name = "hex"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"++[[package]]+name = "html5ever"+version = "0.26.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"+dependencies = [+ "log",+ "mac",+ "markup5ever",+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]++[[package]]+name = "http"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"+dependencies = [+ "bytes",+ "itoa 1.0.19",+]++[[package]]+name = "http-body"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"+dependencies = [+ "bytes",+ "http",+]++[[package]]+name = "http-body-util"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "pin-project-lite",+]++[[package]]+name = "httparse"+version = "1.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"++[[package]]+name = "httpdate"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"++[[package]]+name = "hybrid-array"+version = "0.4.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17"+dependencies = [+ "typenum",+]++[[package]]+name = "hyper"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c"+dependencies = [+ "atomic-waker",+ "bytes",+ "futures-channel",+ "futures-core",+ "h2",+ "http",+ "http-body",+ "httparse",+ "httpdate",+ "itoa 1.0.19",+ "pin-project-lite",+ "smallvec",+ "tokio",+ "want",+]++[[package]]+name = "hyper-rustls"+version = "0.27.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53"+dependencies = [+ "http",+ "hyper",+ "hyper-util",+ "rustls",+ "tokio",+ "tokio-rustls",+ "tower-service",+]++[[package]]+name = "hyper-util"+version = "0.1.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"+dependencies = [+ "base64 0.23.1",+ "bytes",+ "futures-channel",+ "futures-util",+ "http",+ "http-body",+ "httparse",+ "hyper",+ "ipnet",+ "libc",+ "percent-encoding",+ "pin-project-lite",+ "socket2",+ "tokio",+ "tower-service",+ "tracing",+]++[[package]]+name = "iana-time-zone"+version = "0.1.65"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"+dependencies = [+ "android_system_properties",+ "core-foundation-sys",+ "iana-time-zone-haiku",+ "js-sys",+ "log",+ "wasm-bindgen",+ "windows-core",+]++[[package]]+name = "iana-time-zone-haiku"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"+dependencies = [+ "cc",+]++[[package]]+name = "icu_collections"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd"+dependencies = [+ "displaydoc",+ "potential_utf",+ "utf8_iter",+ "yoke",+ "zerofrom",+ "zerovec",+]++[[package]]+name = "icu_locale_core"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0"+dependencies = [+ "displaydoc",+ "litemap",+ "tinystr",+ "writeable",+ "zerovec",+]++[[package]]+name = "icu_normalizer"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599"+dependencies = [+ "icu_collections",+ "icu_normalizer_data",+ "icu_properties",+ "icu_provider",+ "smallvec",+ "zerovec",+]++[[package]]+name = "icu_normalizer_data"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc"++[[package]]+name = "icu_properties"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f"+dependencies = [+ "displaydoc",+ "icu_collections",+ "icu_locale_core",+ "icu_properties_data",+ "icu_provider",+ "zerotrie",+ "zerovec",+]++[[package]]+name = "icu_properties_data"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf"++[[package]]+name = "icu_provider"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514"+dependencies = [+ "displaydoc",+ "icu_locale_core",+ "writeable",+ "yoke",+ "zerofrom",+ "zerotrie",+ "zerovec",+]++[[package]]+name = "idna"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"+dependencies = [+ "idna_adapter",+ "smallvec",+ "utf8_iter",+]++[[package]]+name = "idna_adapter"+version = "1.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"+dependencies = [+ "icu_normalizer",+ "icu_properties",+]++[[package]]+name = "include_dir"+version = "0.7.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"+dependencies = [+ "include_dir_macros",+]++[[package]]+name = "include_dir_macros"+version = "0.7.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"+dependencies = [+ "proc-macro2",+ "quote",+]++[[package]]+name = "indexmap"+version = "1.9.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"+dependencies = [+ "autocfg",+ "hashbrown 0.12.3",+]++[[package]]+name = "indexmap"+version = "2.14.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"+dependencies = [+ "equivalent",+ "hashbrown 0.17.1",+]++[[package]]+name = "inout"+version = "0.2.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a580c13f0ffc335ef42dd3f6b1a41b726051c5e1656d9a3aa6fddb3c5dfbbf9"+dependencies = [+ "block-padding",+ "hybrid-array",+]++[[package]]+name = "ipnet"+version = "2.12.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"++[[package]]+name = "is_terminal_polyfill"+version = "1.70.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"++[[package]]+name = "itoa"+version = "0.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"++[[package]]+name = "itoa"+version = "1.0.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"++[[package]]+name = "jiff"+version = "0.2.42"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fefca06093eeaed707a905795e68206da0edca4edef23d2a3997a3ddbee462aa"+dependencies = [+ "jiff-core",+ "jiff-static",+ "log",+ "portable-atomic",+ "portable-atomic-util",+ "serde_core",+]++[[package]]+name = "jiff-core"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77a24020db0ee88d1c681d677681bd31f9f4ea189a455425223d52ce4f1508aa"+dependencies = [+ "log",+]++[[package]]+name = "jiff-static"+version = "0.2.42"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41d3213549742662eea1a18e6c5af9be5dfbabeb7611ab71356a54928ec398f0"+dependencies = [+ "jiff-core",+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "jni"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"+dependencies = [+ "cfg-if",+ "combine",+ "jni-macros",+ "jni-sys",+ "log",+ "simd_cesu8",+ "thiserror 2.0.21",+ "walkdir",+ "windows-link",+]++[[package]]+name = "jni-macros"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"+dependencies = [+ "proc-macro2",+ "quote",+ "rustc_version",+ "simd_cesu8",+ "syn 2.0.119",+]++[[package]]+name = "jni-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"+dependencies = [+ "jni-sys-macros",+]++[[package]]+name = "jni-sys-macros"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"+dependencies = [+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "jobserver"+version = "0.1.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"+dependencies = [+ "getrandom 0.4.3",+ "libc",+]++[[package]]+name = "js-sys"+version = "0.3.107"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"+dependencies = [+ "cfg-if",+ "futures-util",+ "wasm-bindgen",+]++[[package]]+name = "kuchikiki"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f29e4755b7b995046f510a7520c42b2fed58b77bd94d5a87a8eb43d2fd126da8"+dependencies = [+ "cssparser 0.27.2",+ "html5ever",+ "indexmap 1.9.3",+ "matches",+ "selectors 0.22.0",+]++[[package]]+name = "lazy_static"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"++[[package]]+name = "libc"+version = "0.2.191"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"++[[package]]+name = "libloading"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"+dependencies = [+ "cfg-if",+ "windows-link",+]++[[package]]+name = "libmimalloc-sys"+version = "0.1.49"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"+dependencies = [+ "cc",+ "cty",+]++[[package]]+name = "litemap"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"++[[package]]+name = "lock_api"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"+dependencies = [+ "scopeguard",+]++[[package]]+name = "log"+version = "0.4.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"++[[package]]+name = "lol_html"+version = "2.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652"+dependencies = [+ "bitflags 2.13.2",+ "cfg-if",+ "cssparser 0.36.0",+ "encoding_rs",+ "foldhash",+ "hashbrown 0.17.1",+ "memchr",+ "mime",+ "precomputed-hash",+ "selectors 0.37.0",+ "thiserror 2.0.21",+]++[[package]]+name = "lopdf"+version = "0.45.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bfffda0fe1ab0157e1a13c14bebd3f28671f2fccb7922f0722ec53926e6922d3"+dependencies = [+ "aes",+ "bitflags 2.13.2",+ "brotli-decompressor 5.0.3",+ "cbc",+ "chrono",+ "ecb",+ "encoding_rs",+ "flate2",+ "getrandom 0.4.3",+ "indexmap 2.14.2",+ "itoa 1.0.19",+ "log",+ "md-5",+ "nom 8.0.0",+ "rand 0.10.3",+ "rangemap",+ "rayon",+ "sha2 0.11.1",+ "stringprep",+ "thiserror 2.0.21",+ "weezl",+]++[[package]]+name = "lru-slab"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f"++[[package]]+name = "mac"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"++[[package]]+name = "markup5ever"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"+dependencies = [+ "log",+ "phf 0.10.1",+ "phf_codegen 0.10.0",+ "string_cache",+ "string_cache_codegen",+ "tendril",+]++[[package]]+name = "matches"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"++[[package]]+name = "matchit"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"++[[package]]+name = "md-5"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"+dependencies = [+ "cfg-if",+ "digest 0.11.3",+]++[[package]]+name = "memchr"+version = "2.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"++[[package]]+name = "mimalloc"+version = "0.1.52"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"+dependencies = [+ "libmimalloc-sys",+]++[[package]]+name = "mime"+version = "0.3.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d5848f5a9e8117cf94f2f90faac24a87ff81600ff8628c1e3deaab572e76bfa1"++[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"++[[package]]+name = "miniz_oxide"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"+dependencies = [+ "adler2",+ "simd-adler32",+]++[[package]]+name = "mio"+version = "1.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"+dependencies = [+ "libc",+ "wasi 0.11.1+wasi-snapshot-preview1",+ "windows-sys 0.61.2",+]++[[package]]+name = "multiversion_no_op"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"++[[package]]+name = "napi"+version = "3.16.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "907ff7366fb7182e28edd0f08ef8ac5d6b13b8d65a3c0b3a10bf3ac092437d6b"+dependencies = [+ "bitflags 2.13.2",+ "ctor",+ "futures",+ "libc",+ "napi-build",+ "napi-sys",+ "nohash-hasher",+ "rustc-hash",+ "serde",+ "serde_json",+ "tokio",+]++[[package]]+name = "napi-build"+version = "2.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b899b545d3aa6dca985939059f258c5488d34e4ecf39c274e20009748f4b846d"++[[package]]+name = "napi-derive"+version = "3.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d32eb00f92f84cf462fc328f49082b29c2a7d2eebeb1d447d19c1f9fdd9f89ee"+dependencies = [+ "convert_case 0.12.0",+ "napi-derive-backend",+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "napi-derive-backend"+version = "7.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1fe28c64d8fc12e69f9b817d0febdbe7914555760f1e353106281b8ff99cc90"+dependencies = [+ "convert_case 0.12.0",+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "napi-sys"+version = "3.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d153beb7d5c973f4b9395be8a83054a28968b61188f1dd1a2cc2321af22daea2"+dependencies = [+ "libloading",+]++[[package]]+name = "new_debug_unreachable"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"++[[package]]+name = "nodesig"+version = "1.0.0"+source = "git+https://github.com/firecrawl/nodesig?rev=b8ebc4a710810dbc36e189d9a0df87b877702b3c#b8ebc4a710810dbc36e189d9a0df87b877702b3c"+dependencies = [+ "base64 0.22.1",+ "hex",+ "kuchikiki",+ "rand 0.9.5",+ "sha2 0.10.9",+]++[[package]]+name = "nodrop"+version = "0.1.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"++[[package]]+name = "nohash-hasher"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"++[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]++[[package]]+name = "nom"+version = "8.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"+dependencies = [+ "memchr",+]++[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]++[[package]]+name = "once_cell"+version = "1.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"++[[package]]+name = "once_cell_polyfill"+version = "1.70.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"++[[package]]+name = "openssl-probe"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1"++[[package]]+name = "parking_lot"+version = "0.12.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"+dependencies = [+ "lock_api",+ "parking_lot_core",+]++[[package]]+name = "parking_lot_core"+version = "0.9.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"+dependencies = [+ "cfg-if",+ "libc",+ "redox_syscall",+ "smallvec",+ "windows-link",+]++[[package]]+name = "pdf-inspector"+version = "1.25.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ef5970f05a55de82c8d6d6fd5f8ff162b50fc7828cc1fd0abd965abad6fcbc5b"+dependencies = [+ "env_logger",+ "include_dir",+ "log",+ "lopdf",+ "once_cell",+ "rayon",+ "regex",+ "thiserror 2.0.21",+ "ttf-parser",+ "unicode-bidi",+ "unicode-normalization",+]++[[package]]+name = "percent-encoding"+version = "2.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"++[[package]]+name = "phf"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"+dependencies = [+ "phf_macros 0.8.0",+ "phf_shared 0.8.0",+ "proc-macro-hack",+]++[[package]]+name = "phf"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"+dependencies = [+ "phf_shared 0.10.0",+]++[[package]]+name = "phf"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"+dependencies = [+ "phf_macros 0.13.1",+ "phf_shared 0.13.1",+ "serde",+]++[[package]]+name = "phf_codegen"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"+dependencies = [+ "phf_generator 0.8.0",+ "phf_shared 0.8.0",+]++[[package]]+name = "phf_codegen"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"+dependencies = [+ "phf_generator 0.10.0",+ "phf_shared 0.10.0",+]++[[package]]+name = "phf_codegen"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"+dependencies = [+ "phf_generator 0.13.1",+ "phf_shared 0.13.1",+]++[[package]]+name = "phf_generator"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"+dependencies = [+ "phf_shared 0.8.0",+ "rand 0.7.3",+]++[[package]]+name = "phf_generator"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"+dependencies = [+ "phf_shared 0.10.0",+ "rand 0.8.8",+]++[[package]]+name = "phf_generator"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"+dependencies = [+ "phf_shared 0.11.3",+ "rand 0.8.8",+]++[[package]]+name = "phf_generator"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"+dependencies = [+ "fastrand",+ "phf_shared 0.13.1",+]++[[package]]+name = "phf_macros"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"+dependencies = [+ "phf_generator 0.8.0",+ "phf_shared 0.8.0",+ "proc-macro-hack",+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]++[[package]]+name = "phf_macros"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"+dependencies = [+ "phf_generator 0.13.1",+ "phf_shared 0.13.1",+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "phf_shared"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"+dependencies = [+ "siphasher 0.3.11",+]++[[package]]+name = "phf_shared"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"+dependencies = [+ "siphasher 0.3.11",+]++[[package]]+name = "phf_shared"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"+dependencies = [+ "siphasher 1.0.4",+]++[[package]]+name = "phf_shared"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"+dependencies = [+ "siphasher 1.0.4",+]++[[package]]+name = "pin-project-lite"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"++[[package]]+name = "pkg-config"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"++[[package]]+name = "portable-atomic"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"++[[package]]+name = "portable-atomic-util"+version = "0.2.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"+dependencies = [+ "portable-atomic",+]++[[package]]+name = "potential_utf"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"+dependencies = [+ "zerovec",+]++[[package]]+name = "ppv-lite86"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"+dependencies = [+ "zerocopy",+]++[[package]]+name = "precomputed-hash"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"++[[package]]+name = "proc-macro-hack"+version = "0.5.20+deprecated"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"++[[package]]+name = "proc-macro2"+version = "1.0.107"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "psl"+version = "2.1.250"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9c036f34929bf0b11edcba0799b63f360beb20aef1957b3432b1c3954bd3a631"+dependencies = [+ "psl-types",+]++[[package]]+name = "psl-types"+version = "2.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"++[[package]]+name = "quick-xml"+version = "0.41.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"+dependencies = [+ "memchr",+]++[[package]]+name = "quinn"+version = "0.11.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11"+dependencies = [+ "bytes",+ "cfg_aliases",+ "pin-project-lite",+ "quinn-proto",+ "quinn-udp",+ "rustc-hash",+ "rustls",+ "socket2",+ "thiserror 2.0.21",+ "tokio",+ "tracing",+ "web-time",+]++[[package]]+name = "quinn-proto"+version = "0.11.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe"+dependencies = [+ "aws-lc-rs",+ "bytes",+ "getrandom 0.4.3",+ "lru-slab",+ "rand 0.10.3",+ "rand_pcg 0.10.2",+ "ring",+ "rustc-hash",+ "rustls",+ "rustls-pki-types",+ "slab",+ "thiserror 2.0.21",+ "tinyvec",+ "tracing",+ "web-time",+]++[[package]]+name = "quinn-udp"+version = "0.5.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016"+dependencies = [+ "cfg_aliases",+ "libc",+ "once_cell",+ "socket2",+ "tracing",+ "windows-sys 0.61.2",+]++[[package]]+name = "quote"+version = "1.0.47"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "r-efi"+version = "5.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"++[[package]]+name = "r-efi"+version = "6.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"++[[package]]+name = "rand"+version = "0.7.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"+dependencies = [+ "getrandom 0.1.16",+ "libc",+ "rand_chacha 0.2.2",+ "rand_core 0.5.1",+ "rand_hc",+ "rand_pcg 0.2.1",+]++[[package]]+name = "rand"+version = "0.8.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"+dependencies = [+ "libc",+ "rand_chacha 0.3.1",+ "rand_core 0.6.4",+]++[[package]]+name = "rand"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"+dependencies = [+ "rand_chacha 0.9.0",+ "rand_core 0.9.5",+]++[[package]]+name = "rand"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"+dependencies = [+ "chacha20",+ "getrandom 0.4.3",+ "rand_core 0.10.1",+]++[[package]]+name = "rand_chacha"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"+dependencies = [+ "ppv-lite86",+ "rand_core 0.5.1",+]++[[package]]+name = "rand_chacha"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"+dependencies = [+ "ppv-lite86",+ "rand_core 0.6.4",+]++[[package]]+name = "rand_chacha"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"+dependencies = [+ "ppv-lite86",+ "rand_core 0.9.5",+]++[[package]]+name = "rand_core"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"+dependencies = [+ "getrandom 0.1.16",+]++[[package]]+name = "rand_core"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"+dependencies = [+ "getrandom 0.2.17",+]++[[package]]+name = "rand_core"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"+dependencies = [+ "getrandom 0.3.4",+]++[[package]]+name = "rand_core"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"++[[package]]+name = "rand_hc"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"+dependencies = [+ "rand_core 0.5.1",+]++[[package]]+name = "rand_pcg"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"+dependencies = [+ "rand_core 0.5.1",+]++[[package]]+name = "rand_pcg"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"+dependencies = [+ "rand_core 0.10.1",+]++[[package]]+name = "rangemap"+version = "1.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d"++[[package]]+name = "rayon"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"+dependencies = [+ "either",+ "rayon-core",+]++[[package]]+name = "rayon-core"+version = "1.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"+dependencies = [+ "crossbeam-deque",+ "crossbeam-utils",+]++[[package]]+name = "redox_syscall"+version = "0.5.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"+dependencies = [+ "bitflags 2.13.2",+]++[[package]]+name = "regex"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata 0.4.18",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"++[[package]]+name = "regex-automata"+version = "0.4.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"++[[package]]+name = "reqwest"+version = "0.13.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"+dependencies = [+ "base64 0.23.1",+ "bytes",+ "encoding_rs",+ "futures-core",+ "futures-util",+ "h2",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-rustls",+ "hyper-util",+ "js-sys",+ "log",+ "mime",+ "percent-encoding",+ "pin-project-lite",+ "quinn",+ "rustls",+ "rustls-pki-types",+ "rustls-platform-verifier",+ "serde",+ "serde_json",+ "sync_wrapper",+ "tokio",+ "tokio-rustls",+ "tokio-util",+ "tower",+ "tower-http",+ "tower-service",+ "url",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "wasm-streams",+ "web-sys",+]++[[package]]+name = "ring"+version = "0.17.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"+dependencies = [+ "cc",+ "cfg-if",+ "getrandom 0.2.17",+ "libc",+ "untrusted",+ "windows-sys 0.52.0",+]++[[package]]+name = "roxmltree"+version = "0.20.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"++[[package]]+name = "rustc-hash"+version = "2.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"++[[package]]+name = "rustc_version"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"+dependencies = [+ "semver",+]++[[package]]+name = "rustls"+version = "0.23.46"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"+dependencies = [+ "aws-lc-rs",+ "once_cell",+ "rustls-pki-types",+ "rustls-webpki",+ "subtle",+ "zeroize",+]++[[package]]+name = "rustls-native-certs"+version = "0.8.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4"+dependencies = [+ "openssl-probe",+ "rustls-pki-types",+ "schannel",+ "security-framework",+]++[[package]]+name = "rustls-pki-types"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb"+dependencies = [+ "web-time",+ "zeroize",+]++[[package]]+name = "rustls-platform-verifier"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98"+dependencies = [+ "core-foundation",+ "core-foundation-sys",+ "jni",+ "log",+ "once_cell",+ "rustls",+ "rustls-native-certs",+ "rustls-platform-verifier-android",+ "rustls-webpki",+ "security-framework",+ "security-framework-sys",+ "webpki-root-certs",+ "windows-sys 0.61.2",+]++[[package]]+name = "rustls-platform-verifier-android"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"++[[package]]+name = "rustls-webpki"+version = "0.103.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"+dependencies = [+ "aws-lc-rs",+ "ring",+ "rustls-pki-types",+ "untrusted",+]++[[package]]+name = "rustversion"+version = "1.0.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"++[[package]]+name = "ryu"+version = "1.0.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"++[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]++[[package]]+name = "schannel"+version = "0.1.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "scopeguard"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"++[[package]]+name = "security-framework"+version = "3.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"+dependencies = [+ "bitflags 2.13.2",+ "core-foundation",+ "core-foundation-sys",+ "libc",+ "security-framework-sys",+]++[[package]]+name = "security-framework-sys"+version = "2.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "selectors"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"+dependencies = [+ "bitflags 1.3.2",+ "cssparser 0.27.2",+ "derive_more 0.99.20",+ "fxhash",+ "log",+ "matches",+ "phf 0.8.0",+ "phf_codegen 0.8.0",+ "precomputed-hash",+ "servo_arc 0.1.1",+ "smallvec",+ "thin-slice",+]++[[package]]+name = "selectors"+version = "0.37.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9"+dependencies = [+ "bitflags 2.13.2",+ "cssparser 0.36.0",+ "derive_more 2.1.1",+ "log",+ "new_debug_unreachable",+ "phf 0.13.1",+ "phf_codegen 0.13.1",+ "precomputed-hash",+ "rustc-hash",+ "servo_arc 0.4.3",+ "smallvec",+]++[[package]]+name = "semver"+version = "1.0.28"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"++[[package]]+name = "serde"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"+dependencies = [+ "serde_core",+ "serde_derive",+]++[[package]]+name = "serde_core"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"+dependencies = [+ "serde_derive",+]++[[package]]+name = "serde_derive"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "serde_json"+version = "1.0.154"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"+dependencies = [+ "foldhash",+ "indexmap 2.14.2",+ "itoa 1.0.19",+ "memchr",+ "serde",+ "serde_core",+ "zmij",+]++[[package]]+name = "serde_path_to_error"+version = "0.1.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"+dependencies = [+ "itoa 1.0.19",+ "serde",+ "serde_core",+]++[[package]]+name = "serde_urlencoded"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"+dependencies = [+ "form_urlencoded",+ "itoa 1.0.19",+ "ryu",+ "serde",+]++[[package]]+name = "servo_arc"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"+dependencies = [+ "nodrop",+ "stable_deref_trait",+]++[[package]]+name = "servo_arc"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"+dependencies = [+ "stable_deref_trait",+]++[[package]]+name = "sha1"+version = "0.10.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"+dependencies = [+ "cfg-if",+ "cpufeatures 0.2.17",+ "digest 0.10.7",+]++[[package]]+name = "sha2"+version = "0.10.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"+dependencies = [+ "cfg-if",+ "cpufeatures 0.2.17",+ "digest 0.10.7",+]++[[package]]+name = "sha2"+version = "0.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"+dependencies = [+ "cfg-if",+ "cpufeatures 0.3.1",+ "digest 0.11.3",+]++[[package]]+name = "shlex"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"++[[package]]+name = "signal-hook-registry"+version = "1.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"+dependencies = [+ "errno",+ "libc",+]++[[package]]+name = "simd-adler32"+version = "0.3.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"++[[package]]+name = "simd_cesu8"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"+dependencies = [+ "rustc_version",+ "simdutf8",+]++[[package]]+name = "simdutf8"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"++[[package]]+name = "siphasher"+version = "0.3.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"++[[package]]+name = "siphasher"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"++[[package]]+name = "slab"+version = "0.4.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"++[[package]]+name = "smallvec"+version = "1.16.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46"++[[package]]+name = "socket2"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"+dependencies = [+ "libc",+ "windows-sys 0.61.2",+]++[[package]]+name = "stable_deref_trait"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"++[[package]]+name = "string_cache"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"+dependencies = [+ "new_debug_unreachable",+ "parking_lot",+ "phf_shared 0.11.3",+ "precomputed-hash",+ "serde",+]++[[package]]+name = "string_cache_codegen"+version = "0.5.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"+dependencies = [+ "phf_generator 0.11.3",+ "phf_shared 0.11.3",+ "proc-macro2",+ "quote",+]++[[package]]+name = "stringprep"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"+dependencies = [+ "unicode-bidi",+ "unicode-normalization",+ "unicode-properties",+]++[[package]]+name = "strsim"+version = "0.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"++[[package]]+name = "subtle"+version = "2.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"++[[package]]+name = "syn"+version = "1.0.109"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "syn"+version = "2.0.119"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "syn"+version = "3.0.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "sync_wrapper"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"+dependencies = [+ "futures-core",+]++[[package]]+name = "synstructure"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "tendril"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"+dependencies = [+ "futf",+ "mac",+ "utf-8",+]++[[package]]+name = "texting_robots"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"+dependencies = [+ "anyhow",+ "bstr",+ "lazy_static",+ "nom 7.1.3",+ "percent-encoding",+ "regex",+ "thiserror 1.0.69",+ "url",+]++[[package]]+name = "thin-slice"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl 1.0.69",+]++[[package]]+name = "thiserror"+version = "2.0.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"+dependencies = [+ "thiserror-impl 2.0.21",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "thiserror-impl"+version = "2.0.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "tinystr"+version = "0.8.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a"+dependencies = [+ "displaydoc",+ "zerovec",+]++[[package]]+name = "tinyvec"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"+dependencies = [+ "tinyvec_macros",+]++[[package]]+name = "tinyvec_macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"++[[package]]+name = "tokio"+version = "1.54.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3"+dependencies = [+ "bytes",+ "libc",+ "mio",+ "pin-project-lite",+ "signal-hook-registry",+ "socket2",+ "tokio-macros",+ "windows-sys 0.61.2",+]++[[package]]+name = "tokio-macros"+version = "2.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "tokio-rustls"+version = "0.26.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db"+dependencies = [+ "rustls",+ "tokio",+]++[[package]]+name = "tokio-util"+version = "0.7.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b"+dependencies = [+ "bytes",+ "futures-core",+ "futures-sink",+ "libc",+ "pin-project-lite",+ "tokio",+]++[[package]]+name = "tower"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"+dependencies = [+ "futures-core",+ "futures-util",+ "pin-project-lite",+ "sync_wrapper",+ "tokio",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "tower-http"+version = "0.6.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"+dependencies = [+ "async-compression",+ "bitflags 2.13.2",+ "bytes",+ "futures-core",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "pin-project-lite",+ "tokio",+ "tokio-util",+ "tower",+ "tower-layer",+ "tower-service",+ "url",+]++[[package]]+name = "tower-layer"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"++[[package]]+name = "tower-service"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"++[[package]]+name = "tracing"+version = "0.1.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"+dependencies = [+ "pin-project-lite",+ "tracing-attributes",+ "tracing-core",+]++[[package]]+name = "tracing-attributes"+version = "0.1.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "tracing-core"+version = "0.1.36"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"+dependencies = [+ "once_cell",+]++[[package]]+name = "try-lock"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"++[[package]]+name = "ttf-parser"+version = "0.25.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"++[[package]]+name = "tungstenite"+version = "0.28.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442"+dependencies = [+ "bytes",+ "data-encoding",+ "http",+ "httparse",+ "log",+ "rand 0.9.5",+ "sha1",+ "thiserror 2.0.21",+ "utf-8",+]++[[package]]+name = "typed-path"+version = "0.12.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c"++[[package]]+name = "typenum"+version = "1.20.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"++[[package]]+name = "unicode-bidi"+version = "0.3.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"++[[package]]+name = "unicode-ident"+version = "1.0.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"++[[package]]+name = "unicode-normalization"+version = "0.1.26"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d"+dependencies = [+ "tinyvec",+]++[[package]]+name = "unicode-properties"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833"++[[package]]+name = "unicode-segmentation"+version = "1.13.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"++[[package]]+name = "untrusted"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"++[[package]]+name = "url"+version = "2.5.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"+dependencies = [+ "form_urlencoded",+ "idna",+ "percent-encoding",+ "serde",+]++[[package]]+name = "utf-8"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"++[[package]]+name = "utf8_iter"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0"+dependencies = [+ "utf_types",+]++[[package]]+name = "utf8parse"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"++[[package]]+name = "utf_types"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6"++[[package]]+name = "uuid"+version = "1.29.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c"+dependencies = [+ "getrandom 0.4.3",+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "version_check"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"++[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]++[[package]]+name = "want"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b"+dependencies = [+ "try-lock",+]++[[package]]+name = "wasi"+version = "0.9.0+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"++[[package]]+name = "wasi"+version = "0.11.1+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"++[[package]]+name = "wasip2"+version = "1.0.4+wasi-0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"+dependencies = [+ "wit-bindgen",+]++[[package]]+name = "wasm-bindgen"+version = "0.2.130"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"+dependencies = [+ "cfg-if",+ "rustversion",+ "wasm-bindgen-macro",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-futures"+version = "0.4.80"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501"+dependencies = [+ "js-sys",+ "tokio",+ "wasm-bindgen",+]++[[package]]+name = "wasm-bindgen-macro"+version = "0.2.130"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"+dependencies = [+ "quote",+ "wasm-bindgen-macro-support",+]++[[package]]+name = "wasm-bindgen-macro-support"+version = "0.2.130"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"+dependencies = [+ "bumpalo",+ "proc-macro2",+ "quote",+ "syn 3.0.10",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-shared"+version = "0.2.130"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "wasm-streams"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"+dependencies = [+ "futures-util",+ "js-sys",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+]++[[package]]+name = "web-sys"+version = "0.3.107"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "web-time"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "webpki-root-certs"+version = "1.0.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585"+dependencies = [+ "rustls-pki-types",+]++[[package]]+name = "weezl"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2"++[[package]]+name = "which"+version = "8.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f"+dependencies = [+ "libc",+]++[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "windows-core"+version = "0.62.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"+dependencies = [+ "windows-implement",+ "windows-interface",+ "windows-link",+ "windows-result",+ "windows-strings",+]++[[package]]+name = "windows-implement"+version = "0.60.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "windows-interface"+version = "0.59.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "windows-link"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"++[[package]]+name = "windows-registry"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"+dependencies = [+ "windows-link",+ "windows-result",+ "windows-strings",+]++[[package]]+name = "windows-result"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-strings"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-sys"+version = "0.52.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"+dependencies = [+ "windows-targets",+]++[[package]]+name = "windows-sys"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-targets"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc",+ "windows_i686_gnu",+ "windows_i686_gnullvm",+ "windows_i686_msvc",+ "windows_x86_64_gnu",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"++[[package]]+name = "windows_aarch64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"++[[package]]+name = "windows_i686_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"++[[package]]+name = "windows_i686_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"++[[package]]+name = "windows_i686_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"++[[package]]+name = "windows_x86_64_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"++[[package]]+name = "windows_x86_64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"++[[package]]+name = "wit-bindgen"+version = "0.57.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"++[[package]]+name = "writeable"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"++[[package]]+name = "yoke"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3"+dependencies = [+ "stable_deref_trait",+ "yoke-derive",+ "zerofrom",+]++[[package]]+name = "yoke-derive"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+ "synstructure",+]++[[package]]+name = "zerocopy"+version = "0.8.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"+dependencies = [+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.8.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "zerofrom"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"+dependencies = [+ "zerofrom-derive",+]++[[package]]+name = "zerofrom-derive"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+ "synstructure",+]++[[package]]+name = "zeroize"+version = "1.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"++[[package]]+name = "zerotrie"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"+dependencies = [+ "displaydoc",+ "yoke",+ "zerofrom",+]++[[package]]+name = "zerovec"+version = "0.11.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40"+dependencies = [+ "yoke",+ "zerofrom",+ "zerovec-derive",+]++[[package]]+name = "zerovec-derive"+version = "0.11.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.10",+]++[[package]]+name = "zip"+version = "8.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"+dependencies = [+ "crc32fast",+ "flate2",+ "indexmap 2.14.2",+ "memchr",+ "typed-path",+ "zopfli",+]++[[package]]+name = "zlib-rs"+version = "0.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"++[[package]]+name = "zmij"+version = "1.0.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"++[[package]]+name = "zopfli"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11"+dependencies = [+ "bumpalo",+ "crc32fast",+ "log",+ "simd-adler32",+]++[[package]]+name = "zstd"+version = "0.14.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"+dependencies = [+ "zstd-safe",+]++[[package]]+name = "zstd-safe"+version = "8.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b"+dependencies = [+ "zstd-sys",+]++[[package]]+name = "zstd-sys"+version = "2.1.2+zstd.1.5.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"+dependencies = [+ "cc",+ "pkg-config",+]diff --git a/apps/local/Cargo.toml b/apps/local/Cargo.tomlnew file mode 100644index 000000000..0b72f0a33--- /dev/null+++ b/apps/local/Cargo.toml@@ -0,0 +1,72 @@+[package]+name = "firecrawl-local"+version = "0.1.0"+edition = "2021"+description = "Firecrawl Local: scrape, crawl and map on your own machine from one binary, with a hand-off to Firecrawl cloud."+license = "AGPL-3.0"+repository = "https://github.com/firecrawl/firecrawl"+homepage = "https://www.firecrawl.dev"+readme = "README.md"+keywords = ["scraping", "crawler", "markdown", "mcp", "firecrawl"]+categories = ["command-line-utilities", "web-programming"]+# Listed by hand so the vendored copy in src/native, which git ignores, is+# part of a published crate.+include = ["src/**/*", "build.rs", "README.md"]++[lib]+name = "firecrawl_local"+path = "src/lib.rs"++[[bin]]+name = "firecrawl-local"+path = "src/main.rs"++[dependencies]+# --- shared with apps/api/native -------------------------------------------+# The HTML cleaning, link, metadata and crawl-filter code is compiled straight+# from apps/api/native/src (see src/lib.rs), so local and cloud output stay the+# same. `noop` turns the N-API macros into plain Rust so no Node is needed.+napi = { version = "3", default-features = false, features = ["noop", "napi4", "serde-json", "tokio_rt"] }+napi-derive = { version = "3", default-features = false, features = ["noop"] }+# Pinned to the commit in Cargo.lock so `cargo update` cannot change the+# cleaning rules. Move it together with apps/api/native. The version is what+# a published crate depends on, since crates.io does not take git sources.+nodesig = { version = "1.0.0", git = "https://github.com/firecrawl/nodesig", rev = "b8ebc4a710810dbc36e189d9a0df87b877702b3c" }+kuchikiki = "0.8.2"+lol_html = "2.6.0"+psl = "2.1.140"+regex = "1.11.2"+roxmltree = "0.20.0"+serde = { version = "1.0.219", features = ["derive"] }+serde_json = { version = "1.0.143", features = ["preserve_order"] }+strsim = "0.11"+texting_robots = "0.2.2"+url = "2.5.7"+tokio = { version = "1.48.0", features = ["rt-multi-thread", "macros", "net", "time", "sync", "fs", "io-util", "io-std", "process", "signal"] }++# --- engine -----------------------------------------------------------------+reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "gzip", "brotli", "zstd", "deflate", "charset", "json", "stream"] }+axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio", "query"] }+clap = { version = "4", features = ["derive", "env"] }+futures = "0.3"+bytes = "1"+encoding_rs = "0.8"+once_cell = "1"+uuid = { version = "1", features = ["v4"] }+chromiumoxide = { version = "0.9", default-features = false, features = ["bytes"] }+base64 = "0.22"+anydoc = "0.2.2"+pdf-inspector = "1.21.0"+chrono = "0.4"+zip = { version = "8.6.0", default-features = false, features = ["deflate"] }+tinyvec = ">=1.12, <1.13"+mimalloc = "0.1.52"+libmimalloc-sys = { version = "0.1.49", features = ["extended"] }+sha2 = "0.10"+getrandom = "0.3"++[profile.release]+lto = "fat"+codegen-units = 1+strip = true+opt-level = 3diff --git a/apps/local/README.md b/apps/local/README.mdnew file mode 100644index 000000000..146a6c612--- /dev/null+++ b/apps/local/README.md@@ -0,0 +1,114 @@+# Firecrawl Local++A single Rust binary that scrapes, crawls and maps the web from your own machine. It speaks the Firecrawl v2 API, plugs into AI agents through the Firecrawl MCP server, and needs no database, queue or container.++It shares its HTML cleaning code with the Firecrawl API. Firecrawl Local compiles `apps/api/native/src` directly, so a page cleaned locally and a page cleaned in the cloud go through the same rules.++## Build++```sh+cd apps/local+cargo build --release+./target/release/firecrawl-local --help+```++Once a release is on crates.io, `cargo install firecrawl-local` installs the same binary without cloning the repository.++Pages that need JavaScript are rendered in the Chrome or Chromium already installed on the machine. Without one it still handles static pages. Set `FIRECRAWL_CHROME_PATH` to point it at a specific browser.++## Use it from the command line++```sh+firecrawl-local scrape https://example.com --markdown+firecrawl-local crawl https://docs.example.com --limit 200 --out ./pages+firecrawl-local map https://example.com+```++`scrape` returns the same JSON shape as the Firecrawl API. `--format` takes `markdown`, `html`, `rawHtml` and `links`, and `--full-page` keeps navigation and footers. PDFs and office documents are parsed locally.++## Use it from an AI agent++```sh+firecrawl-local init+```++`init` adds Firecrawl Local as an MCP server to the AI coding tools it finds on the machine (Claude Code, Claude desktop, Cursor, Windsurf, Gemini CLI and Codex). It backs up each config file before changing it, and `--dry-run` shows the change without writing it. To wire it up by hand, run `firecrawl-local mcp` as a stdio MCP server.++Firecrawl Local does not carry an MCP server of its own. `firecrawl-local mcp` starts its API on a loopback port and runs the [Firecrawl MCP server](https://github.com/firecrawl/firecrawl-mcp-server) against it, so the tools, their descriptions and their updates are the same ones every other Firecrawl MCP user gets. Scrape, map, crawl and batch scrape are answered on your machine, and the other tools are passed on to Firecrawl cloud. It needs Node.js 22 or newer, because the MCP server is the `firecrawl-mcp` package run through `npx`.++## Use it as a local API++```sh+firecrawl-local serve --port 3002+curl -s localhost:3002/v2/scrape -H 'content-type: application/json' \+ -d '{"url":"https://example.com","formats":["markdown"]}'+```++`serve` answers `/v2/scrape`, `/v2/map`, `/v2/crawl` and `/v2/batch/scrape`, so existing Firecrawl SDKs work when pointed at it. The older v0 and v1 routes are not served. Crawl and batch jobs are kept in memory and expire an hour after they are created.++Every other `/v2` route, such as `/search`, `/agent` and data providers, is passed on to Firecrawl cloud when the cloud setting allows that route, and answers `CLOUD_ONLY` otherwise. With no key and no `FIRECRAWL_CLOUD`, `serve` passes nothing on. `FIRECRAWL_CLOUD=search` passes on search only, `FIRECRAWL_CLOUD=off` passes on nothing, and a value that is not recognised is read as `off`. Forwarded requests must be `application/json` or come from a tool, not a web page, and bodies may be up to 64 MB. Structured JSON output and browser actions on a local scrape are not implemented. The full self-hosted server in `apps/api` is unchanged.++## What runs locally and what runs on Firecrawl cloud++Scrape, crawl, map and document parsing run on your machine and send nothing to Firecrawl.++Everything else uses Firecrawl cloud, because it needs an index, data or a service the machine does not have. The three you will meet most are these.++- **Search.** `firecrawl-local search` and the MCP server's `firecrawl_search` tool query Firecrawl's search.+- **Pages that will not load locally.** When a site refuses a plain fetch and the local browser, the CLI and MCP server ask Firecrawl cloud for the page and mark the result with `source: firecrawl-cloud`.+- **Alexandria data providers.** `find-tools` and `data` reach structured data providers and need a Firecrawl account.++`firecrawl-local login` signs in through the browser and saves the key where the Firecrawl CLI keeps it. Set `FIRECRAWL_CLOUD=off` to turn off search, the cloud fallback and data providers, or `FIRECRAWL_CLOUD=search` to allow search only. `login` always talks to Firecrawl, since signing in is its whole job. `serve` stays local unless a key or `FIRECRAWL_CLOUD` is set.++## Safety defaults++- `serve` listens on `127.0.0.1`. Binding any other address requires `--token`, and requests are checked against an allowed `Host` list.+- `serve` callers cannot reach localhost or private networks unless `--allow-private` is set. The command line and the MCP server run for the person at the keyboard, so they allow local and private targets by default; pass `--public-only` to refuse them there too.+- Because the command line and the MCP server can reach local and private addresses, an AI agent that follows instructions hidden in a page could be steered into reading an internal page. If the agent browses sites you do not trust, register the MCP server as `firecrawl-local mcp --public-only`.+- Link-local and cloud metadata addresses are refused in every mode, including after redirects and DNS resolution.+- The same rules cover the browser. Chrome runs behind a loopback proxy inside Firecrawl Local that checks every connection it opens, including redirects, subresources, scripts' own requests and WebSockets. With private targets off, a public page cannot make the browser call a private address. With them on, as in the command line's default, a page the browser loads can reach the local network the way it could in your own browser.+- The local browser runs headless and sandboxed with a throwaway profile.+- It identifies itself in its User-Agent and reads `robots.txt` when crawling.++## Environment variables++| Variable | Effect |+| --- | --- |+| `FIRECRAWL_LOCAL_BROWSER` | `auto` (default), `never` or `always` |+| `FIRECRAWL_CHROME_PATH` | Path to the Chrome or Chromium to use |+| `FIRECRAWL_CLOUD` | `auto` (also `on`, `yes`, `fallback`), `search` or `off` (also `no`, `none`, `disabled`). Unrecognized values are read as `off` |+| `FIRECRAWL_API_KEY`, `FIRECRAWL_API_URL` | Firecrawl cloud key and address |+| `FIRECRAWL_LOCAL_TOKEN` | Bearer token for `serve` |+| `FIRECRAWL_LOCAL_ALLOW_PRIVATE` | Let `serve` callers reach private addresses |+| `FIRECRAWL_LOCAL_USER_AGENT` | Override the User-Agent |+| `FIRECRAWL_LOCAL_SCROLL` | Scroll rendered pages before reading them |+| `FIRECRAWL_LOCAL_TRACE` | Print browser settle timing to stderr |+| `FIRECRAWL_LOCAL_HOME` | Folder for Firecrawl Local's own files, such as its install ID |+| `FIRECRAWL_CLI_CREDENTIALS` | Path of the saved login file, which defaults to the Firecrawl CLI's |+| `FIRECRAWL_WEB_URL` | Address of the site `login` opens, for testing against another deployment |+| `FIRECRAWL_MCP_PACKAGE` | The npm package `mcp` runs, which defaults to `firecrawl-mcp` |++## Releases++`.github/workflows/publish-local.yml` publishes the crate to crates.io when the version in `Cargo.toml` on main is newer than the published one. It uses trusted publishing, so there is no token stored in the repository, and the publish step waits for approval in the `release` environment.++A crate on crates.io cannot reach `apps/api/native/src`, so `scripts/vendor-native.sh` copies the shared cleaning code into `src/native` before packaging. `build.rs` uses that copy only when the repository's files are not there, so a build inside the repository always uses the shared files.++Two things have to happen once, by hand, before the workflow can publish.++1. Publish `nodesig` 1.0.0 to crates.io from [firecrawl/nodesig](https://github.com/firecrawl/nodesig). crates.io does not accept git dependencies, and the cleaning code depends on it.+2. Publish the first version of `firecrawl-local` with an API token, since trusted publishing only works for a crate that already exists. Then add a trusted publisher in the crate's settings on crates.io with repository `firecrawl/firecrawl`, workflow `publish-local.yml` and environment `release`.++```sh+cd apps/local+./scripts/vendor-native.sh+cargo publish --allow-dirty+```++After that, a release is a version bump in `Cargo.toml`.++## Tests++```sh+cargo test+```diff --git a/apps/local/build.rs b/apps/local/build.rsnew file mode 100644index 000000000..e4290fa20--- /dev/null+++ b/apps/local/build.rs@@ -0,0 +1,24 @@+//! Decides where the shared cleaning code is compiled from.+//!+//! In the repository it comes straight from `apps/api/native/src`, the same+//! files the cloud API loads. A crate published to crates.io cannot reach+//! outside its own folder, so `scripts/vendor-native.sh` copies those files+//! to `src/native` before packaging, and the copy is used only when the+//! repository's files are not there.++use std::path::Path;++fn main() {+ println!("cargo:rustc-check-cfg=cfg(vendored_native)");+ println!("cargo:rerun-if-changed=build.rs");+ let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());+ let root = Path::new(&root);+ if root.join("src/native").exists() {+ println!("cargo:rerun-if-changed=src/native");+ }+ let in_repository = root.join("../api/native/src/html.rs").exists();+ let vendored = root.join("src/native/html.rs").exists();+ if !in_repository && vendored {+ println!("cargo:rustc-cfg=vendored_native");+ }+}diff --git a/apps/local/rustfmt.toml b/apps/local/rustfmt.tomlnew file mode 100644index 000000000..b196eaa2d--- /dev/null+++ b/apps/local/rustfmt.toml@@ -0,0 +1 @@+tab_spaces = 2diff --git a/apps/local/scripts/vendor-native.sh b/apps/local/scripts/vendor-native.shnew file mode 100755index 000000000..130d18881--- /dev/null+++ b/apps/local/scripts/vendor-native.sh@@ -0,0 +1,20 @@+#!/usr/bin/env bash+# Copy the shared cleaning code from apps/api/native/src into src/native, so a+# packaged crate carries it. Run before `cargo package` or `cargo publish`.+# The list of files is read from src/lib.rs, so it cannot drift.+set -euo pipefail++cd "$(dirname "$0")/.."+shared="../api/native/src"+files=$(sed -nE 's/.*path = "\.\.\/\.\.\/api\/native\/src\/([a-z_]+\.rs)".*/\1/p' src/lib.rs | sort -u)+if [ -z "$files" ]; then+ echo "vendor-native: found no shared files listed in src/lib.rs" >&2+ exit 1+fi++rm -rf src/native+mkdir -p src/native+for file in $files; do+ cp "$shared/$file" "src/native/$file"+ echo "vendored $file"+donediff --git a/apps/local/src/browser.rs b/apps/local/src/browser.rsnew file mode 100644index 000000000..0408bda41--- /dev/null+++ b/apps/local/src/browser.rs@@ -0,0 +1,835 @@+//! JavaScript rendering through the Chrome already installed on this machine.+//!+//! Nothing is downloaded. Chrome runs headless, sandboxed, in a throwaway+//! profile: never the person's own profile, cookies or logins. It starts on+//! the first page that needs it and is reused after that.++use std::{+ path::PathBuf,+ sync::{Arc, Mutex as StdMutex},+ time::{Duration, Instant},+};++use chromiumoxide::{+ browser::{Browser, BrowserConfig},+ cdp::browser_protocol::network::{+ EventLoadingFailed, EventLoadingFinished, EventRequestWillBeSent, ResourceType,+ },+ page::ScreenshotParams,+};+use futures::StreamExt;+use tokio::sync::{Mutex, Semaphore};+use url::Url;++use crate::guard::{self, NetworkPolicy};++/// Counts fetch and XHR calls still in flight, so "the page has settled" can+/// mean "nothing is still loading", not just "the text stopped changing".+const NETWORK_TRACKER: &str = r#"(() => {+ if (window.__fcTracker) return;+ window.__fcTracker = true;+ // Requests still open. One that has been open for more than 1.5 s is a+ // long poll or a stream, not the page loading, so it stops counting.+ const open = new Set();+ Object.defineProperty(window, '__fcPending', {+ configurable: true,+ get() {+ const now = performance.now();+ let n = 0;+ for (const r of open) if (now - r.at < 1500) n++;+ return n;+ },+ });+ window.__fcLastMut = performance.now();+ try {+ new MutationObserver(() => { window.__fcLastMut = performance.now(); })+ .observe(document, { subtree: true, childList: true, characterData: true });+ } catch (e) {}+ const begin = () => {+ const r = { at: performance.now() };+ open.add(r);+ return () => { open.delete(r); };+ };+ const origFetch = window.fetch;+ if (origFetch) {+ window.fetch = function () {+ const done = begin();+ const p = origFetch.apply(this, arguments);+ p.then(done, done);+ return p;+ };+ }+ const origSend = XMLHttpRequest.prototype.send;+ XMLHttpRequest.prototype.send = function () {+ this.addEventListener('loadend', begin(), { once: true });+ return origSend.apply(this, arguments);+ };+})()"#;++/// Serialises the page with open shadow roots flattened into their hosts, so+/// content inside web components is not lost. Returns null when the page has+/// no shadow roots and the normal serialisation is enough.+const FLATTEN_SHADOW_DOM: &str = r#"(() => {+ let has = false;+ for (const el of document.querySelectorAll('*')) { if (el.shadowRoot) { has = true; break; } }+ if (!has) return null;+ const VOID = new Set(['area','base','br','col','embed','hr','img','input','link','meta','source','track','wbr']);+ const SKIP = new Set(['script','style','noscript','template','svg']);+ const esc = s => s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');+ const escA = s => s.replace(/&/g,'&').replace(/"/g,'"');+ const ser = n => {+ if (n.nodeType === 3) return esc(n.nodeValue);+ if (n.nodeType !== 1) return '';+ const t = n.localName;+ if (SKIP.has(t)) return '';+ let a = '';+ for (const at of n.attributes) a += ' ' + at.name + '="' + escA(at.value) + '"';+ if (VOID.has(t)) return '<' + t + a + '>';+ let inner = '';+ if (n.shadowRoot) for (const c of n.shadowRoot.childNodes) inner += ser(c);+ for (const c of n.childNodes) inner += ser(c);+ return '<' + t + a + '>' + inner + '</' + t + '>';+ };+ return '<!doctype html>' + ser(document.documentElement);+})()"#;++/// Watches the page's network requests through the DevTools protocol, so the+/// engine can tell "still loading scripts and data" from "done".+struct NetWatch {+ in_flight: Arc<StdMutex<std::collections::HashMap<String, (Instant, String)>>>,+ last_event: Arc<StdMutex<Instant>>,+ /// The address of the latest document request, which after a redirect is+ /// not the address the page was asked for.+ last_document: Arc<StdMutex<Option<String>>>,+ tasks: Vec<tokio::task::JoinHandle<()>>,+}++impl NetWatch {+ async fn start(page: &chromiumoxide::Page) -> Self {+ let in_flight = Arc::new(StdMutex::new(std::collections::HashMap::new()));+ let last_event = Arc::new(StdMutex::new(Instant::now()));+ let last_document = Arc::new(StdMutex::new(None));+ let mut tasks = Vec::new();+ if let Ok(mut events) = page.event_listener::<EventRequestWillBeSent>().await {+ let (set, last) = (in_flight.clone(), last_event.clone());+ let document = last_document.clone();+ tasks.push(tokio::spawn(async move {+ while let Some(e) = events.next().await {+ if e.r#type == Some(ResourceType::Document) {+ if let Ok(mut d) = document.lock() {+ *d = Some(e.request.url.clone());+ }+ }+ if let Ok(mut s) = set.lock() {+ s.insert(+ e.request_id.inner().clone(),+ (Instant::now(), e.request.url.clone()),+ );+ }+ if let Ok(mut l) = last.lock() {+ *l = Instant::now();+ }+ }+ }));+ }+ if let Ok(mut events) = page.event_listener::<EventLoadingFinished>().await {+ let (set, last) = (in_flight.clone(), last_event.clone());+ tasks.push(tokio::spawn(async move {+ while let Some(e) = events.next().await {+ if let Ok(mut s) = set.lock() {+ s.remove(e.request_id.inner());+ }+ if let Ok(mut l) = last.lock() {+ *l = Instant::now();+ }+ }+ }));+ }+ if let Ok(mut events) = page.event_listener::<EventLoadingFailed>().await {+ let (set, last) = (in_flight.clone(), last_event.clone());+ tasks.push(tokio::spawn(async move {+ while let Some(e) = events.next().await {+ if let Ok(mut s) = set.lock() {+ s.remove(e.request_id.inner());+ }+ if let Ok(mut l) = last.lock() {+ *l = Instant::now();+ }+ }+ }));+ }+ Self {+ in_flight,+ last_event,+ last_document,+ tasks,+ }+ }++ fn last_document(&self) -> Option<String> {+ self.last_document.lock().ok().and_then(|d| d.clone())+ }++ /// True when the network has gone quiet: nothing started or finished for+ /// a third of a second and no recent request is still in flight. A request open+ /// for more than 1.5 seconds is a long-lived connection or one that will+ /// never report back (a beacon, a blocked script), not "still loading";+ /// slow data calls are still caught by the in-page fetch and XHR counter.+ fn idle(&self) -> bool {+ let quiet = self+ .last_event+ .lock()+ .map(|l| l.elapsed())+ .unwrap_or_default();+ let pending = self+ .in_flight+ .lock()+ .map(|s| {+ s.values()+ .filter(|(started, _)| started.elapsed() < Duration::from_millis(1500))+ .count()+ })+ .unwrap_or(0);+ pending == 0 && quiet >= Duration::from_millis(350)+ }++ fn stop(self) {+ for t in self.tasks {+ t.abort();+ }+ }+}++pub struct Rendered {+ pub html: String,+ pub final_url: String,+ pub status: Option<u16>,+ pub screenshot_png: Option<Vec<u8>>,+}++struct Launched {+ browser: Browser,+ handler: tokio::task::JoinHandle<()>,+ profile: PathBuf,+ /// Every connection Chrome makes goes through this; dropped with Chrome.+ _proxy: crate::netguard::GuardProxy,+}++pub struct BrowserPool {+ chrome: PathBuf,+ policy: NetworkPolicy,+ user_agent: String,+ launched: Mutex<Option<Launched>>,+ pages: Semaphore,+}++/// Find a Chromium-family browser. `FIRECRAWL_CHROME_PATH` wins.+pub fn find_chrome() -> Option<PathBuf> {+ if let Some(p) = std::env::var_os("FIRECRAWL_CHROME_PATH").map(PathBuf::from) {+ if p.exists() {+ return Some(p);+ }+ }+ let fixed = [+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",+ "/Applications/Chromium.app/Contents/MacOS/Chromium",+ "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",+ "/usr/bin/google-chrome",+ "/usr/bin/google-chrome-stable",+ "/usr/bin/chromium",+ "/usr/bin/chromium-browser",+ "/snap/bin/chromium",+ "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",+ "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",+ "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",+ ];+ if let Some(p) = fixed.iter().map(PathBuf::from).find(|p| p.exists()) {+ return Some(p);+ }+ let path = std::env::var_os("PATH")?;+ for dir in std::env::split_paths(&path) {+ for name in [+ "google-chrome",+ "google-chrome-stable",+ "chromium",+ "chromium-browser",+ "chrome",+ ] {+ let candidate = dir.join(name);+ if candidate.exists() {+ return Some(candidate);+ }+ }+ }+ None+}++impl BrowserPool {+ pub fn new(policy: NetworkPolicy, user_agent: &str, max_pages: usize) -> Option<Arc<Self>> {+ let chrome = find_chrome()?;+ Some(Arc::new(Self {+ chrome,+ policy,+ user_agent: user_agent.to_string(),+ launched: Mutex::new(None),+ pages: Semaphore::new(max_pages.max(1)),+ }))+ }++ async fn ensure_launched(&self) -> Result<(), String> {+ let mut guard = self.launched.lock().await;+ if guard.is_some() {+ return Ok(());+ }+ let profile = std::env::temp_dir().join(format!(+ "firecrawl-local-chrome-{}-{}",+ std::process::id(),+ uuid::Uuid::new_v4().simple()+ ));+ std::fs::create_dir_all(&profile)+ .map_err(|e| format!("could not create a browser profile: {e}"))?;+ // Chrome does its own networking, so it is sent through a loopback proxy+ // that applies the network policy to every connection it opens: the+ // page, its redirects, subresources, scripts' requests and WebSockets.+ let proxy = crate::netguard::GuardProxy::start(self.policy)+ .await+ .map_err(|e| format!("could not start the browser's network guard: {e}"))?;+ let config = BrowserConfig::builder()+ .chrome_executable(&self.chrome)+ .user_data_dir(&profile)+ .new_headless_mode()+ .respect_https_errors()+ .window_size(1366, 900)+ .launch_timeout(Duration::from_secs(20))+ .request_timeout(Duration::from_secs(45))+ .args([+ "disable-gpu",+ "mute-audio",+ "hide-scrollbars",+ "no-default-browser-check",+ "disable-notifications",+ "disable-save-password-bubble",+ ])+ .args(proxy.chrome_args())+ .build()+ .map_err(|e| format!("browser configuration failed: {e}"))?;+ let (browser, mut handler) = Browser::launch(config)+ .await+ .map_err(|e| format!("could not start Chrome at {}: {e}", self.chrome.display()))?;+ let handler = tokio::spawn(async move {+ while let Some(event) = handler.next().await {+ if event.is_err() {+ break;+ }+ }+ });+ *guard = Some(Launched {+ browser,+ handler,+ profile,+ _proxy: proxy,+ });+ Ok(())+ }++ /// Load a page, let it settle, and return its rendered HTML.+ pub async fn render(+ &self,+ url: &Url,+ wait_for: Duration,+ timeout: Duration,+ screenshot: bool,+ ) -> Result<Rendered, String> {+ guard::check_url(url, self.policy).map_err(|e| e.0)?;+ let _permit = self.pages.acquire().await.map_err(|e| e.to_string())?;+ self.ensure_launched().await?;++ let page = {+ let guard = self.launched.lock().await;+ let launched = guard.as_ref().ok_or("browser is not running")?;+ launched+ .browser+ .new_page("about:blank")+ .await+ .map_err(|e| format!("could not open a tab: {e}"))?+ };++ let result = tokio::time::timeout(timeout, self.drive(&page, url, wait_for, screenshot)).await;+ let _ = page.close().await;+ match result {+ Ok(r) => r,+ Err(_) => Err("the page did not finish loading in time".to_string()),+ }+ }++ async fn drive(+ &self,+ page: &chromiumoxide::Page,+ url: &Url,+ wait_for: Duration,+ screenshot: bool,+ ) -> Result<Rendered, String> {+ page+ .set_user_agent(self.user_agent.as_str())+ .await+ .map_err(|e| format!("could not set the user agent: {e}"))?;+ let _ = page.evaluate_on_new_document(NETWORK_TRACKER).await;+ let watch = NetWatch::start(page).await;+ let result = self+ .drive_watched(page, url, wait_for, screenshot, &watch)+ .await;+ watch.stop();+ result+ }++ async fn drive_watched(+ &self,+ page: &chromiumoxide::Page,+ url: &Url,+ wait_for: Duration,+ screenshot: bool,+ watch: &NetWatch,+ ) -> Result<Rendered, String> {+ if let Err(e) = page.goto(url.as_str()).await {+ // The guard may have refused this address or one it redirected to.+ let mut reason = crate::netguard::refusal_reason(url, self.policy).await;+ if reason.is_none() {+ if let Some(redirected) = watch.last_document().and_then(|u| Url::parse(&u).ok()) {+ reason = crate::netguard::refusal_reason(&redirected, self.policy).await;+ }+ }+ return Err(reason.unwrap_or_else(|| format!("navigation failed: {e}")));+ }+ let response = page.wait_for_navigation_response().await.ok().flatten();++ // Which address each connection reaches is enforced by the guard proxy+ // Chrome runs behind, for this response and everything else the page+ // loads, so there is nothing further to check here.+ let status = response+ .as_ref()+ .and_then(|r| r.response.as_ref())+ .map(|resp| resp.status as u16);++ let trace = std::env::var_os("FIRECRAWL_LOCAL_TRACE").is_some();+ let t0 = Instant::now();+ settle(page, watch, Duration::from_secs(10), true).await;+ if trace {+ eprintln!("[render] first settle {:?}", t0.elapsed());+ if let Ok(open) = watch.in_flight.lock() {+ for (started, url) in open.values() {+ eprintln!(+ "[render] still open after {:?}: {}",+ started.elapsed(),+ &url[..url.len().min(120)]+ );+ }+ }+ }+ // Lazy-loaded content only appears once it scrolls into view. Wait again+ // only if scrolling actually made the page load or change something.+ let before_scroll = Instant::now();+ // Scrolling costs most of a second per rendered page and helped on few+ // pages in testing, so it is opt-in: FIRECRAWL_LOCAL_SCROLL=1.+ let scrolled =+ std::env::var_os("FIRECRAWL_LOCAL_SCROLL").is_some() && scroll_through(page).await;+ let changed = scrolled && page_changed_since(page, watch, before_scroll).await;+ if changed {+ settle(page, watch, Duration::from_millis(2500), false).await;+ }+ if trace {+ eprintln!(+ "[render] scrolled={scrolled} changed={changed} total {:?}",+ t0.elapsed()+ );+ }+ if !wait_for.is_zero() {+ tokio::time::sleep(wait_for).await;+ }++ let final_url = page+ .url()+ .await+ .ok()+ .flatten()+ .unwrap_or_else(|| url.to_string());+ if let Ok(parsed) = Url::parse(&final_url) {+ // A page that navigated itself somewhere forbidden is discarded.+ guard::check_url(&parsed, self.policy).map_err(|e| e.0)?;+ }++ let flattened = match page.evaluate(FLATTEN_SHADOW_DOM).await {+ Ok(v) => v.into_value::<Option<String>>().ok().flatten(),+ Err(_) => None,+ };+ let html = match flattened {+ Some(html) => html,+ None => page+ .content()+ .await+ .map_err(|e| format!("could not read the page: {e}"))?,+ };+ let screenshot_png = if screenshot {+ page+ .screenshot(ScreenshotParams::builder().full_page(true).build())+ .await+ .ok()+ } else {+ None+ };++ Ok(Rendered {+ html,+ final_url,+ status,+ screenshot_png,+ })+ }++ /// Stop Chrome and delete its throwaway profile.+ pub async fn shutdown(&self) {+ if let Some(mut launched) = self.launched.lock().await.take() {+ let _ = launched.browser.close().await;+ let _ = tokio::time::timeout(Duration::from_secs(3), launched.browser.wait()).await;+ launched.handler.abort();+ let _ = std::fs::remove_dir_all(&launched.profile);+ }+ }+}++/// Wait until the page has really finished: the document is complete, the+/// network is idle, no fetch or XHR is in flight, and neither the DOM nor the+/// text has changed for about half a second. A page with no text yet gets up+/// to three seconds to produce some.+async fn settle(+ page: &chromiumoxide::Page,+ watch: &NetWatch,+ budget: Duration,+ wait_for_text: bool,+) {+ let started = Instant::now();+ let empty_budget = if wait_for_text {+ Duration::from_secs(3).min(budget)+ } else {+ Duration::ZERO+ };+ let probe = "(() => document.readyState + ':' + (window.__fcPending || 0) + ':' + (document.body ? document.body.innerText.length : 0) + ':' + Math.round(performance.now() - (window.__fcLastMut || 0)))()";+ let mut last_text = usize::MAX;+ let mut stable = 0u8;+ while started.elapsed() < budget {+ let now = match page.evaluate(probe).await {+ Ok(v) => v.into_value::<String>().unwrap_or_default(),+ Err(_) => String::new(),+ };+ let mut parts = now.split(':');+ let complete = parts.next() == Some("complete");+ let pending: usize = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0);+ let text_len: usize = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0);+ let dom_quiet_ms: u64 = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0);+ if std::env::var_os("FIRECRAWL_LOCAL_TRACE").is_some() {+ eprintln!(+ "[settle] {:?} state={now} net_idle={}",+ started.elapsed(),+ watch.idle()+ );+ }+ if complete && pending == 0 && watch.idle() && dom_quiet_ms >= 300 && text_len == last_text {+ stable += 1;+ let has_text = text_len >= 50;+ if stable >= 2 && (has_text || started.elapsed() >= empty_budget) {+ return;+ }+ } else {+ stable = 0;+ }+ last_text = text_len;+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+}++/// True when a request started or finished, or the DOM changed, after `since`.+async fn page_changed_since(page: &chromiumoxide::Page, watch: &NetWatch, since: Instant) -> bool {+ let network = watch.last_event.lock().map(|l| *l > since).unwrap_or(false);+ if network {+ return true;+ }+ let dom_quiet_ms = match page+ .evaluate("(() => Math.round(performance.now() - (window.__fcLastMut || 0)))()")+ .await+ {+ Ok(v) => v.into_value::<u64>().unwrap_or(u64::MAX),+ Err(_) => u64::MAX,+ };+ Duration::from_millis(dom_quiet_ms) < since.elapsed()+}++/// Scroll a tall page to the bottom in steps and back to the top. Returns+/// false, without scrolling, when the page fits in a couple of screens.+async fn scroll_through(page: &chromiumoxide::Page) -> bool {+ let tall = match page+ .evaluate("(() => document.documentElement.scrollHeight > window.innerHeight * 2)()")+ .await+ {+ Ok(v) => v.into_value::<bool>().unwrap_or(false),+ Err(_) => false,+ };+ if !tall {+ return false;+ }+ for _ in 0..8 {+ let at_bottom = match page+ .evaluate("(() => { window.scrollBy(0, window.innerHeight * 2); return window.scrollY + window.innerHeight >= document.documentElement.scrollHeight - 4; })()")+ .await+ {+ Ok(v) => v.into_value::<bool>().unwrap_or(true),+ Err(_) => true,+ };+ tokio::time::sleep(Duration::from_millis(80)).await;+ if at_bottom {+ break;+ }+ }+ let _ = page.evaluate("window.scrollTo(0, 0)").await;+ true+}++/// True when a fetched page is a JavaScript shell that needs a browser.+///+/// Signals, strongest first: the page says it needs JavaScript or carries a+/// framework mount point and has little text; or it has scripts and almost+/// no text; or its text is a tiny share of its HTML (data waiting in scripts).+/// A short static page with an analytics tag is none of these.+pub fn needs_browser(raw_html: &str, markdown: &str) -> bool {+ let visible = markdown.chars().filter(|c| !c.is_whitespace()).count();+ if visible >= 1500 {+ return false;+ }+ let mut end = raw_html.len().min(200_000);+ while !raw_html.is_char_boundary(end) {+ end -= 1;+ }+ let head = raw_html[..end].to_ascii_lowercase();+ if !head.contains("<script") {+ return false;+ }+ let asks_for_js = head.contains("enable javascript")+ || head.contains("requires javascript")+ || head.contains("javascript is disabled")+ || head.contains("javascript is required")+ || head.contains("need to enable javascript");+ let mount_point = [+ "id=\"root\"",+ "id=\"app\"",+ "id=\"__next\"",+ "id=\"__nuxt\"",+ "id=\"svelte\"",+ "id=\"___gatsby\"",+ "data-reactroot",+ "ng-app",+ "ng-view",+ "ng-version",+ "<app-root",+ "id=\"q-app\"",+ "id=\"elm\"",+ ]+ .iter()+ .any(|m| head.contains(m));+ if asks_for_js || mount_point {+ return true;+ }+ let ratio = visible as f64 / raw_html.len().max(1) as f64;+ // Text is under 1% of a sizeable page: the content is waiting in scripts.+ if ratio < 0.01 && raw_html.len() > 20_000 {+ return true;+ }+ if visible >= 300 {+ return false;+ }+ visible < 60 || ratio < 0.03+}++/// Share of a markdown document's visible text that is link text. Pages that+/// are nearly all links are navigation, not content.+pub fn link_text_share(markdown: &str) -> f64 {+ let mut total = 0usize;+ let mut in_links = 0usize;+ let mut depth = 0usize;+ let mut pending = 0usize;+ let mut chars = markdown.chars().peekable();+ while let Some(c) = chars.next() {+ match c {+ '[' => {+ depth += 1;+ pending = 0;+ }+ ']' if depth > 0 => {+ depth -= 1;+ if chars.peek() == Some(&'(') {+ in_links += pending;+ // Skip the URL: it is not visible text.+ for u in chars.by_ref() {+ if u == ')' {+ break;+ }+ }+ }+ pending = 0;+ }+ c if !c.is_whitespace() => {+ total += 1;+ if depth > 0 {+ pending += 1;+ }+ }+ _ => {}+ }+ }+ if total == 0 {+ 0.0+ } else {+ in_links as f64 / total as f64+ }+}++/// True when a server-rendered framework page came back as navigation with+/// no body: it has an app mount point, a modest amount of text, and most of+/// that text is links.+pub fn looks_like_nav_only_shell(raw_html: &str, markdown: &str) -> bool {+ let visible = markdown.chars().filter(|c| !c.is_whitespace()).count();+ if !(1500..12_000).contains(&visible) {+ return false;+ }+ let mut end = raw_html.len().min(200_000);+ while !raw_html.is_char_boundary(end) {+ end -= 1;+ }+ let head = raw_html[..end].to_ascii_lowercase();+ let framework = [+ "id=\"__nuxt\"",+ "id=\"__next\"",+ "id=\"root\"",+ "id=\"app\"",+ "ng-version",+ "<app-root",+ "id=\"___gatsby\"",+ "id=\"svelte\"",+ ]+ .iter()+ .any(|m| head.contains(m));+ framework && link_text_share(markdown) > 0.6+}++/// Target of an instant `<meta http-equiv="refresh">` redirect, if the page+/// is one.+pub fn meta_refresh_target(raw_html: &str, base: &Url) -> Option<Url> {+ if raw_html.len() > 100_000 {+ return None;+ }+ let lower = raw_html.to_ascii_lowercase();+ let at = lower+ .find("http-equiv=\"refresh\"")+ .or_else(|| lower.find("http-equiv='refresh'"))+ .or_else(|| lower.find("http-equiv=refresh"))?;+ let tag_start = lower[..at].rfind("<meta")?;+ let tag_end = at + lower[at..].find('>')?;+ let tag = &raw_html[tag_start..tag_end];+ let tag_lower = &lower[tag_start..tag_end];+ let content_at = tag_lower.find("content=")? + "content=".len();+ let value = tag[content_at..].trim_start_matches(['"', '\'']);+ let value_lower = value.to_ascii_lowercase();+ let delay: f64 = value+ .split(|c| c == ';' || c == ',')+ .next()+ .and_then(|d| d.trim().parse().ok())+ .unwrap_or(99.0);+ if delay > 5.0 {+ return None;+ }+ let url_at = value_lower.find("url=")? + "url=".len();+ let target = value[url_at..]+ .trim_start_matches(['"', '\''])+ .split(|c| c == '"' || c == '\'' || c == '>')+ .next()?+ .trim();+ if target.is_empty() {+ return None;+ }+ let resolved = base.join(target).ok()?;+ if resolved == *base {+ None+ } else {+ Some(resolved)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn link_share_and_nav_only_shells() {+ assert!((link_text_share("[abcd](https://x.dev/long/url) ef") - 4.0 / 6.0).abs() < 1e-9);+ assert_eq!(link_text_share("plain text only"), 0.0);+ let nav: String = (0..200)+ .map(|i| format!("- [Section number {i}](https://x.dev/s/{i})\n"))+ .collect();+ let html = "<html><body><div id=\"__nuxt\"><nav></nav></div><script src=\"/a.js\"></script></body></html>";+ assert!(looks_like_nav_only_shell(html, &nav));+ let article = "word ".repeat(800);+ assert!(!looks_like_nav_only_shell(html, &article));+ assert!(!looks_like_nav_only_shell(+ "<html><body><nav></nav></body></html>",+ &nav+ ));+ }++ #[test]+ fn meta_refresh_is_followed_only_when_instant() {+ let base = Url::parse("https://blog.example.com/2024/old.html").unwrap();+ let html = "<html><head><meta http-equiv=\"refresh\" content=\"0; url=/2024/new/\"></head><body>Redirecting</body></html>";+ assert_eq!(+ meta_refresh_target(html, &base).unwrap().as_str(),+ "https://blog.example.com/2024/new/"+ );+ let quoted = "<meta http-equiv='refresh' content='0;URL=https://other.dev/x'>";+ assert_eq!(+ meta_refresh_target(quoted, &base).unwrap().as_str(),+ "https://other.dev/x"+ );+ let slow = "<meta http-equiv=\"refresh\" content=\"300; url=/reload\">";+ assert!(meta_refresh_target(slow, &base).is_none());+ assert!(meta_refresh_target("<html><body>plain</body></html>", &base).is_none());+ }++ #[test]+ fn shell_detection() {+ let shell =+ "<html><body><div id=\"root\"></div><script src=\"/app.js\"></script></body></html>";+ assert!(needs_browser(shell, ""));+ let article = format!(+ "<html><body><p>{}</p><script></script></body></html>",+ "word ".repeat(200)+ );+ assert!(!needs_browser(&article, &"word ".repeat(200)));+ // A short static page with no scripts is just short.+ assert!(!needs_browser("<html><body><p>Hi</p></body></html>", "Hi"));+ // Little text, lots of markup and data in scripts: render it.+ let data_page = format!(+ "<html><body><div class=\"quotes\"></div><script>var data = [{}];</script></body></html>",+ "{\"text\":\"q\"},".repeat(400)+ );+ assert!(needs_browser(&data_page, &"word ".repeat(20)));+ // A framework mount point with some text around it: render it.+ let app = format!(+ "<html><body><div ng-view></div><p>{}</p><script src=\"/app.js\"></script></body></html>",+ "word ".repeat(80)+ );+ assert!(needs_browser(&app, &"word ".repeat(80)));+ // A short static page with an analytics script is still just short.+ let small = format!(+ "<html><body><p>{}</p><script src=\"/a.js\"></script></body></html>",+ "word ".repeat(30)+ );+ assert!(!needs_browser(&small, &"word ".repeat(30)));+ }+}diff --git a/apps/local/src/cloud.rs b/apps/local/src/cloud.rsnew file mode 100644index 000000000..0a89c1ee2--- /dev/null+++ b/apps/local/src/cloud.rs@@ -0,0 +1,905 @@+//! Hand-off to Firecrawl cloud for what a laptop cannot do: web search, pages+//! that block a local fetch, and Alexandria data providers.+//!+//! Nothing here runs unless the caller asks for search or the cloud setting+//! allows a fallback. Every result says it came from the cloud and what it+//! cost, as data fields.++use std::time::Duration;++use serde_json::{json, Value};++pub const DEFAULT_API_URL: &str = "https://api.firecrawl.dev";++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum CloudMode {+ /// Never contact Firecrawl cloud.+ Off,+ /// Search goes to the cloud; scrapes stay local.+ SearchOnly,+ /// Search goes to the cloud, and a scrape the local engine could not+ /// complete is retried there.+ Fallback,+}++impl CloudMode {+ pub fn parse(s: &str) -> Option<CloudMode> {+ match s.trim().to_ascii_lowercase().as_str() {+ "off" | "false" | "0" | "none" | "no" | "disabled" | "disable" => Some(CloudMode::Off),+ "search" | "search-only" => Some(CloudMode::SearchOnly),+ "on" | "auto" | "fallback" | "true" | "1" | "yes" | "enabled" | "enable" => {+ Some(CloudMode::Fallback)+ }+ _ => None,+ }+ }++ /// The mode a `FIRECRAWL_CLOUD` value asks for. Unset or empty asks for+ /// nothing. A value that is not understood asks for off: someone who set+ /// it was restricting the cloud, and a typo must not open it up.+ pub fn requested(raw: Option<&str>) -> Option<CloudMode> {+ let raw = raw.map(str::trim).filter(|r| !r.is_empty())?;+ Some(CloudMode::parse(raw).unwrap_or(CloudMode::Off))+ }+}++/// Whether a path has a `.` or `..` segment, plain or percent-encoded. The+/// HTTP client resolves those before sending, so `/v2/search/../agent` would+/// arrive as `/v2/agent`. Such paths are never passed on.+pub fn has_dot_segment(path: &str) -> bool {+ path.split(['/', '\\']).any(|segment| {+ let segment = segment.to_ascii_lowercase().replace("%2e", ".");+ segment == "." || segment == ".."+ })+}++/// Say once that `FIRECRAWL_CLOUD` holds a value this build does not know.+fn warn_unknown_mode(raw: &str) {+ static ONCE: std::sync::Once = std::sync::Once::new();+ ONCE.call_once(|| {+ eprintln!(+ "FIRECRAWL_CLOUD={raw} is not a setting Firecrawl Local knows (off, search or auto). The cloud stays off."+ );+ });+}++#[derive(Clone)]+pub struct Cloud {+ client: reqwest::Client,+ base: String,+ api_key: Option<String>,+ pub mode: CloudMode,+ /// Random ID for this install, so Firecrawl can count installs, not IP+ /// addresses. Created on the first cloud call; never tied to a person.+ install_id: Option<String>,+ /// The agent harness that launched the engine (from the MCP handshake).+ harness: Option<String>,+ origin: String,+}++pub const SIGN_IN_HINT: &str = "Run `firecrawl login`, or set FIRECRAWL_API_KEY.";++fn config_dir() -> Option<std::path::PathBuf> {+ if let Some(dir) = std::env::var_os("FIRECRAWL_LOCAL_HOME") {+ return Some(std::path::PathBuf::from(dir));+ }+ std::env::var_os("HOME").map(|h| {+ std::path::PathBuf::from(h)+ .join(".config")+ .join("firecrawl-local")+ })+}++/// Where the Firecrawl CLI keeps the login from `firecrawl login`. Same+/// paths as the CLI's own `credentials.ts`.+pub fn cli_credentials_path() -> Option<std::path::PathBuf> {+ if let Some(p) = std::env::var_os("FIRECRAWL_CLI_CREDENTIALS") {+ return Some(std::path::PathBuf::from(p));+ }+ let home =+ std::path::PathBuf::from(std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE"))?);+ let dir = if cfg!(target_os = "macos") {+ home+ .join("Library")+ .join("Application Support")+ .join("firecrawl-cli")+ } else if cfg!(target_os = "windows") {+ home.join("AppData").join("Roaming").join("firecrawl-cli")+ } else {+ home.join(".config").join("firecrawl-cli")+ };+ Some(dir.join("credentials.json"))+}++/// The API key and API URL saved by `firecrawl login`, if the user has+/// logged in with the CLI. This is what makes sign-in one step: log in once+/// in the browser and the engine picks it up.+fn cli_credentials() -> (Option<String>, Option<String>) {+ let Some(path) = cli_credentials_path() else {+ return (None, None);+ };+ let Ok(text) = std::fs::read_to_string(path) else {+ return (None, None);+ };+ let Ok(v) = serde_json::from_str::<Value>(&text) else {+ return (None, None);+ };+ let field = |name: &str| {+ v.get(name)+ .and_then(Value::as_str)+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(str::to_string)+ };+ (field("apiKey"), field("apiUrl"))+}++/// Read this install's ID, creating it on first use.+fn load_install_id() -> Option<String> {+ let dir = config_dir()?;+ let path = dir.join("install-id");+ if let Ok(existing) = std::fs::read_to_string(&path) {+ let id = existing.trim().to_string();+ if !id.is_empty() {+ return Some(id);+ }+ }+ let id = uuid::Uuid::new_v4().to_string();+ std::fs::create_dir_all(&dir).ok()?;+ std::fs::write(&path, &id).ok()?;+ Some(id)+}++/// How the engine was started, which decides the origin its cloud calls carry.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Surface {+ /// A command typed in a terminal, or run by an AI agent through one.+ Cli,+ /// The stdio MCP server.+ Mcp,+ /// The local HTTP API.+ Serve,+}++/// The origin for a surface, in the forms Firecrawl already uses:+/// `cli@<version>` for the CLI and `mcp-<client>@<version>` for an MCP+/// server, where the client is the name the MCP client gave at initialize.+/// The version is marked `local-` so Firecrawl Local can be told apart from the+/// Node CLI and the hosted MCP server.+fn origin_for(surface: Surface, client: Option<&str>) -> String {+ let version = concat!("local-", env!("CARGO_PKG_VERSION"));+ match surface {+ Surface::Cli => format!("cli@{version}"),+ Surface::Serve => format!("firecrawl-local@{version}"),+ Surface::Mcp => {+ let token: String = client+ .unwrap_or("")+ .to_ascii_lowercase()+ .chars()+ .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })+ .collect();+ let token: String = token+ .split('-')+ .filter(|part| !part.is_empty())+ .collect::<Vec<_>>()+ .join("-")+ .chars()+ .take(48)+ .collect();+ if token.is_empty() {+ format!("mcp-local@{version}")+ } else {+ format!("mcp-{token}@{version}")+ }+ }+ }+}++/// Firecrawl cloud's answer to a forwarded request, passed back untouched.+pub struct Forwarded {+ pub status: u16,+ pub content_type: Option<String>,+ pub body: bytes::Bytes,+}++#[derive(Debug)]+pub struct CloudError {+ pub status: u16,+ pub message: String,+}++impl std::fmt::Display for CloudError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}", self.message)+ }+}++impl Cloud {+ /// Reads `FIRECRAWL_API_KEY`, `FIRECRAWL_API_URL` and `FIRECRAWL_CLOUD`,+ /// falling back to the login saved by `firecrawl login`.+ /// With no explicit setting: a key turns the fallback on, no key means+ /// search only.+ pub fn from_env() -> Self {+ let (saved_key, saved_url) = cli_credentials();+ let api_key = std::env::var("FIRECRAWL_API_KEY")+ .ok()+ .filter(|k| !k.trim().is_empty())+ .or(saved_key);+ let base = std::env::var("FIRECRAWL_API_URL")+ .ok()+ .filter(|u| !u.trim().is_empty())+ .or(saved_url)+ .unwrap_or_else(|| DEFAULT_API_URL.to_string());+ let setting = std::env::var("FIRECRAWL_CLOUD").ok();+ if let Some(raw) = setting.as_deref().map(str::trim) {+ if !raw.is_empty() && CloudMode::parse(raw).is_none() {+ warn_unknown_mode(raw);+ }+ }+ let mode = CloudMode::requested(setting.as_deref()).unwrap_or(CloudMode::Fallback);+ Self::new(base, api_key, mode)+ }++ /// Same as `from_env`, but with the cloud off unless a key or an explicit+ /// `FIRECRAWL_CLOUD` setting turns it on. For `serve`, where the people+ /// calling the API did not install this engine themselves.+ pub fn from_env_opt_in() -> Self {+ let mut cloud = Self::from_env().with_surface(Surface::Serve, None);+ let setting = std::env::var("FIRECRAWL_CLOUD").ok();+ if CloudMode::requested(setting.as_deref()).is_none() && !cloud.has_key() {+ cloud.mode = CloudMode::Off;+ }+ cloud+ }++ /// Give the cloud a fixed install ID, so a test never writes one to the+ /// developer's config folder.+ #[cfg(test)]+ pub(crate) fn with_install_id(mut self, id: &str) -> Self {+ self.install_id = Some(id.to_string());+ self+ }++ /// Use a key obtained mid-session (after a one-step sign-in).+ pub fn with_key(mut self, api_key: String) -> Self {+ self.api_key = Some(api_key);+ self+ }++ /// Tag cloud calls with the harness that launched the engine.+ pub fn with_harness(mut self, harness: Option<String>) -> Self {+ self.harness = harness.map(|h| {+ h.chars()+ .filter(|c| c.is_ascii_graphic() || *c == ' ')+ .take(64)+ .collect()+ });+ self+ }++ pub fn new(base: String, api_key: Option<String>, mode: CloudMode) -> Self {+ let client = reqwest::Client::builder()+ .user_agent(concat!("firecrawl-local/", env!("CARGO_PKG_VERSION")))+ .connect_timeout(Duration::from_secs(10))+ .build()+ .unwrap_or_default();+ Self {+ client,+ base: base.trim_end_matches('/').to_string(),+ api_key,+ mode,+ install_id: None,+ harness: None,+ origin: origin_for(Surface::Cli, None),+ }+ }++ /// Stamp cloud calls with the surface the engine is running as, so the+ /// request is attributed the way the Firecrawl CLI and MCP server are.+ pub fn with_surface(mut self, surface: Surface, client: Option<&str>) -> Self {+ self.origin = origin_for(surface, client);+ self+ }++ pub fn origin(&self) -> &str {+ &self.origin+ }++ pub fn has_key(&self) -> bool {+ self.api_key.is_some()+ }++ /// Use the origin a client stamped on its own request, so a call the+ /// engine passes on is attributed to that client and not to the engine.+ pub fn with_origin(mut self, origin: &str) -> Self {+ let clean: String = origin+ .chars()+ .filter(|c| c.is_ascii_graphic())+ .take(100)+ .collect();+ if !clean.is_empty() {+ self.origin = clean;+ }+ self+ }++ /// Whether a request for `path` may be passed on to Firecrawl cloud.+ pub fn allows(&self, path: &str) -> bool {+ if has_dot_segment(path) {+ return false;+ }+ match self.mode {+ CloudMode::Off => false,+ CloudMode::SearchOnly => path == "/v2/search" || path.starts_with("/v2/search/"),+ CloudMode::Fallback => true,+ }+ }++ /// Pass a request this engine does not serve itself on to Firecrawl cloud+ /// and return the answer as it came. The engine's own key is used when it+ /// has one; otherwise the caller's, and with neither the call is keyless.+ pub async fn forward(+ &self,+ method: reqwest::Method,+ path_and_query: &str,+ content_type: Option<&str>,+ body: bytes::Bytes,+ client_auth: Option<&str>,+ client_origin: Option<&str>,+ ) -> Result<Forwarded, CloudError> {+ let path = path_and_query.split('?').next().unwrap_or(path_and_query);+ if !self.allows(path) {+ return Err(CloudError {+ status: 400,+ message: format!("{path} is not a path Firecrawl Local passes on to Firecrawl cloud."),+ });+ }+ let origin = match client_origin {+ Some(o) => self.clone().with_origin(o).origin,+ None => self.origin.clone(),+ };+ let mut req = self+ .client+ .request(method, format!("{}{}", self.base, path_and_query))+ .timeout(Duration::from_secs(190))+ .header("x-origin", origin)+ .body(body);+ if let Some(ct) = content_type {+ req = req.header(reqwest::header::CONTENT_TYPE, ct);+ }+ if let Some(id) = self.install_id.clone().or_else(load_install_id) {+ req = req.header("x-firecrawl-install", id);+ }+ if let Some(key) = self.api_key.as_deref().or(client_auth) {+ req = req.bearer_auth(key);+ }+ let resp = req.send().await.map_err(|e| CloudError {+ status: 502,+ message: format!("Could not reach Firecrawl cloud: {e}"),+ })?;+ let status = resp.status().as_u16();+ let content_type = resp+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .and_then(|v| v.to_str().ok())+ .map(str::to_string);+ let body = resp.bytes().await.map_err(|e| CloudError {+ status: 502,+ message: format!("Firecrawl cloud's answer was cut short: {e}"),+ })?;+ Ok(Forwarded {+ status,+ content_type,+ body,+ })+ }++ async fn post(+ &self,+ path: &str,+ mut body: Value,+ timeout: Duration,+ ) -> Result<Value, CloudError> {+ // The scrape route reads the origin from the body only, so it travels+ // there as well as in the header.+ if let Some(obj) = body.as_object_mut() {+ obj+ .entry("origin")+ .or_insert_with(|| Value::String(self.origin.clone()));+ }+ let mut req = self+ .client+ .post(format!("{}{}", self.base, path))+ .timeout(timeout)+ .header("x-origin", &self.origin)+ .json(&body);+ if let Some(id) = self.install_id.clone().or_else(load_install_id) {+ req = req.header("x-firecrawl-install", id);+ }+ if let Some(h) = &self.harness {+ req = req.header("x-firecrawl-harness", h);+ }+ if let Some(key) = &self.api_key {+ req = req.bearer_auth(key);+ }+ let resp = req.send().await.map_err(|e| CloudError {+ status: 502,+ message: format!("Could not reach Firecrawl cloud: {e}"),+ })?;+ let status = resp.status().as_u16();+ let payload: Value = resp.json().await.unwrap_or(Value::Null);+ if !(200..300).contains(&status) {+ let detail = payload+ .get("error")+ .and_then(Value::as_str)+ .unwrap_or("request failed")+ .to_string();+ let message = match status {+ 401 | 402 | 403 | 429 if !self.has_key() => {+ // The cloud sends a signup link made for this caller; pass it on.+ let link = payload+ .get("signupUrl")+ .and_then(Value::as_str)+ .map(|u| format!(" Or sign up at {u}"))+ .unwrap_or_default();+ let reason = detail.lines().next().unwrap_or("").trim();+ format!("Firecrawl cloud needs a sign-in for this ({reason}). {SIGN_IN_HINT}{link}")+ }+ 401 if self.has_key() => format!(+ "Firecrawl rejected the saved login ({detail}). It may have been rotated or revoked. Run `firecrawl login` again, or update FIRECRAWL_API_KEY."+ ),+ _ => format!("Firecrawl cloud returned {status}: {detail}"),+ };+ return Err(CloudError { status, message });+ }+ Ok(payload)+ }++ /// Web search. Always a cloud call: there is no index on a laptop.+ pub async fn search(&self, query: &str, limit: u32) -> Result<Value, CloudError> {+ if self.mode == CloudMode::Off {+ return Err(CloudError {+ status: 403,+ message:+ "Web search runs on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off)."+ .into(),+ });+ }+ let mut payload = self+ .post(+ "/v2/search",+ json!({ "query": query, "limit": limit.clamp(1, 50) }),+ Duration::from_secs(60),+ )+ .await?;+ annotate(&mut payload);+ Ok(payload)+ }++ /// Retry a scrape in the cloud. Only called when the mode is `Fallback`.+ pub async fn scrape(&self, url: &str, options: &Value) -> Result<Value, CloudError> {+ let mut body = options.clone();+ if !body.is_object() {+ body = json!({});+ }+ body["url"] = Value::String(url.to_string());+ let mut payload = self+ .post("/v2/scrape", body, Duration::from_secs(120))+ .await?;+ annotate(&mut payload);+ Ok(payload)+ }+}++impl Cloud {+ /// Run one or more Alexandria provider calls. `calls` is one call object+ /// or an array of up to ten: `{provider, capability, options}`. Needs an+ /// account: providers cost Firecrawl money per call.+ pub async fn alexandria(&self, calls: &Value) -> Result<Value, CloudError> {+ if self.mode == CloudMode::Off {+ return Err(CloudError {+ status: 403,+ message: "Alexandria data providers run on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off).".into(),+ });+ }+ if !self.has_key() {+ return Err(CloudError {+ status: 401,+ message: format!("Alexandria data providers need a free Firecrawl account. {SIGN_IN_HINT} The same call works after that."),+ });+ }+ let calls = match calls {+ Value::Array(_) => calls.clone(),+ Value::Object(_) => Value::Array(vec![calls.clone()]),+ _ => return Err(CloudError {+ status: 400,+ message:+ "'alexandria' must be a call object or an array of them: {provider, capability, options}."+ .into(),+ }),+ };+ let mut payload = self+ .post(+ "/v2/scrape",+ json!({ "alexandria": calls, "origin": self.origin }),+ Duration::from_secs(190),+ )+ .await?;+ annotate(&mut payload);+ Ok(payload)+ }++ /// Browse or search the Alexandria catalogue. Tried without a key first,+ /// since browsing costs nothing; the cloud decides whether to allow it.+ pub async fn find_tools(&self, options: &Value) -> Result<Value, CloudError> {+ if self.mode == CloudMode::Off {+ return Err(CloudError {+ status: 403,+ message: "The Alexandria catalogue lives on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off).".into(),+ });+ }+ let call = json!([{ "provider": "firecrawl", "capability": "find-tools", "options": options }]);+ let mut payload = self+ .post(+ "/v2/scrape",+ json!({ "alexandria": call, "origin": self.origin }),+ Duration::from_secs(60),+ )+ .await?;+ annotate(&mut payload);+ Ok(payload)+ }+}++/// Scrape locally; if the page was refused and the cloud fallback is on,+/// retry it through Firecrawl. The result says where it ran.+pub async fn scrape_with_fallback(+ engine: &crate::scrape::Engine,+ cloud: &Cloud,+ url: &str,+ opts: &crate::scrape::ScrapeOptions,+) -> Result<Value, crate::scrape::ScrapeError> {+ let local = engine.scrape(url, opts).await;+ let blocked = match &local {+ Ok(doc) => {+ let status = doc+ .pointer("/metadata/statusCode")+ .and_then(Value::as_u64)+ .unwrap_or(200) as u16;+ let md = doc.get("markdown").and_then(Value::as_str).unwrap_or("");+ looks_blocked(status, md)+ }+ Err(e) => matches!(e.code, "FETCH_FAILED" | "SCRAPE_TIMEOUT"),+ };+ if blocked && cloud.mode == CloudMode::Fallback {+ let cloud_opts = json!({ "formats": ["markdown"], "onlyMainContent": opts.only_main_content });+ if let Ok(payload) = cloud.scrape(url, &cloud_opts).await {+ if let Some(mut data) = payload.get("data").cloned() {+ if let Some(obj) = data.as_object_mut() {+ obj.insert("source".into(), Value::String("firecrawl-cloud".into()));+ }+ return Ok(data);+ }+ }+ }+ match local {+ Ok(mut doc) => {+ if let Some(obj) = doc.as_object_mut() {+ obj.insert("source".into(), Value::String("local".into()));+ if blocked {+ obj.insert("blocked".into(), Value::Bool(true));+ }+ }+ Ok(doc)+ }+ Err(e) => Err(e),+ }+}++/// Mark a cloud response so callers can tell where it ran and what it cost.+fn annotate(payload: &mut Value) {+ if let Some(obj) = payload.as_object_mut() {+ obj.insert("source".into(), Value::String("firecrawl-cloud".into()));+ }+}++/// Signs that a local fetch was refused rather than served.+pub fn looks_blocked(status: u16, markdown: &str) -> bool {+ matches!(status, 401 | 403 | 407 | 429 | 503) || looks_like_bot_check(markdown)+}++/// True when a short page is a bot check or a refusal, not content.+pub fn looks_like_bot_check(markdown: &str) -> bool {+ if markdown.len() > 2000 {+ return false;+ }+ let lower = markdown.to_ascii_lowercase();+ [+ "verify you are human",+ "verifying you are human",+ "checking your browser",+ "verifying your browser",+ "quick browser check",+ "security checkpoint",+ "performing security verification",+ "enable javascript and cookies to continue",+ "attention required! | cloudflare",+ "access denied",+ "are you a robot",+ "captcha",+ "unusual traffic",+ "pardon our interruption",+ ]+ .iter()+ .any(|needle| lower.contains(needle))+}++/// A stand-in for Firecrawl cloud that returns what it was sent.+#[cfg(test)]+pub(crate) async fn echo_cloud() -> String {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))+ .await+ .unwrap();+ let base = format!("http://{}", listener.local_addr().unwrap());+ tokio::spawn(async move {+ while let Ok((mut socket, _)) = listener.accept().await {+ tokio::spawn(async move {+ // Read the whole request, keeping only its first 8 KB to echo.+ let mut seen: Vec<u8> = Vec::new();+ let mut buf = vec![0u8; 64 * 1024];+ let mut wanted: Option<usize> = None;+ let mut total = 0usize;+ loop {+ let n = socket.read(&mut buf).await.unwrap_or(0);+ if n == 0 {+ break;+ }+ total += n;+ if seen.len() < 8192 {+ let room = 8192 - seen.len();+ seen.extend_from_slice(&buf[..n.min(room)]);+ }+ if wanted.is_none() {+ let text = String::from_utf8_lossy(&seen).to_lowercase();+ if let Some(head_end) = text.find("\r\n\r\n") {+ let length = text[..head_end]+ .lines()+ .find_map(|l| l.strip_prefix("content-length:"))+ .and_then(|v| v.trim().parse::<usize>().ok())+ .unwrap_or(0);+ wanted = Some(head_end + 4 + length);+ }+ }+ if wanted.is_some_and(|w| total >= w) {+ break;+ }+ }+ let request = String::from_utf8_lossy(&seen).to_string();+ let body = serde_json::json!({ "request": request }).to_string();+ let answer = format!(+ "HTTP/1.1 207 Multi-Status\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",+ body.len(),+ body+ );+ let _ = socket.write_all(answer.as_bytes()).await;+ });+ }+ });+ base+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn cli_login_is_picked_up() {+ let dir = std::env::temp_dir().join(format!("fc-cred-test-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ let path = dir.join("credentials.json");+ std::fs::write(+ &path,+ r#"{"apiKey":"fc-test-key","apiUrl":"https://api.example.test"}"#,+ )+ .unwrap();+ std::env::set_var("FIRECRAWL_CLI_CREDENTIALS", &path);+ let (key, url) = cli_credentials();+ assert_eq!(key.as_deref(), Some("fc-test-key"));+ assert_eq!(url.as_deref(), Some("https://api.example.test"));+ std::fs::write(&path, "not json").unwrap();+ assert_eq!(cli_credentials(), (None, None));+ std::env::remove_var("FIRECRAWL_CLI_CREDENTIALS");+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn origins_follow_the_cli_and_mcp_conventions() {+ let v = concat!("local-", env!("CARGO_PKG_VERSION"));+ assert_eq!(origin_for(Surface::Cli, None), format!("cli@{v}"));+ assert_eq!(+ origin_for(Surface::Serve, None),+ format!("firecrawl-local@{v}")+ );+ assert_eq!(+ origin_for(Surface::Mcp, Some("Claude Code")),+ format!("mcp-claude-code@{v}")+ );+ assert_eq!(+ origin_for(Surface::Mcp, Some(" ")),+ format!("mcp-local@{v}")+ );+ assert_eq!(origin_for(Surface::Mcp, None), format!("mcp-local@{v}"));+ assert!(origin_for(Surface::Mcp, Some("cursor/1.0 (x)")).starts_with("mcp-cursor-1-0-x@"));+ }++ #[tokio::test]+ async fn forward_passes_the_request_and_the_answer_through() {+ let base = echo_cloud().await;+ let keyless =+ Cloud::new(base.clone(), None, CloudMode::Fallback).with_install_id("test-install");+ let answer = keyless+ .forward(+ reqwest::Method::POST,+ "/v2/search?x=1",+ Some("application/json"),+ bytes::Bytes::from_static(b"{\"query\":\"q\"}"),+ None,+ Some("[email protected]"),+ )+ .await+ .unwrap();+ assert_eq!(answer.status, 207);+ assert_eq!(answer.content_type.as_deref(), Some("application/json"));+ let seen = String::from_utf8_lossy(&answer.body).to_lowercase();+ assert!(seen.contains("post /v2/search?x=1"), "{seen}");+ assert!(seen.contains("x-origin: [email protected]"), "{seen}");+ assert!(seen.contains("{\\\"query\\\":\\\"q\\\"}"), "{seen}");+ assert!(+ !seen.contains("authorization:"),+ "a keyless call carries no key: {seen}"+ );++ // The caller's key is used when the engine has none, and the engine's wins when it has one.+ let with_client_key = keyless+ .forward(+ reqwest::Method::GET,+ "/v2/team/credit-usage",+ None,+ bytes::Bytes::new(),+ Some("fc-client"),+ None,+ )+ .await+ .unwrap();+ assert!(String::from_utf8_lossy(&with_client_key.body).contains("Bearer fc-client"));+ let own = Cloud::new(base, Some("fc-engine".into()), CloudMode::Fallback)+ .with_install_id("test-install");+ let with_own_key = own+ .forward(+ reqwest::Method::GET,+ "/v2/team/credit-usage",+ None,+ bytes::Bytes::new(),+ Some("fc-client"),+ None,+ )+ .await+ .unwrap();+ let seen = String::from_utf8_lossy(&with_own_key.body).to_string();+ assert!(+ seen.contains("Bearer fc-engine") && !seen.contains("fc-client"),+ "{seen}"+ );+ }++ #[test]+ fn modes_decide_what_is_passed_on() {+ let cloud = |mode| Cloud::new("https://api.example.test".into(), None, mode);+ assert!(!cloud(CloudMode::Off).allows("/v2/search"));+ assert!(cloud(CloudMode::SearchOnly).allows("/v2/search"));+ assert!(cloud(CloudMode::SearchOnly).allows("/v2/search/research/papers"));+ assert!(!cloud(CloudMode::SearchOnly).allows("/v2/agent"));+ assert!(!cloud(CloudMode::SearchOnly).allows("/v2/searching"));+ assert!(cloud(CloudMode::Fallback).allows("/v2/agent"));+ }++ #[test]+ fn dot_segments_are_never_passed_on() {+ let cloud = |mode| Cloud::new("https://api.example.test".into(), None, mode);+ for path in [+ "/v2/search/../agent",+ "/v2/search/%2e%2e/agent",+ "/v2/search/%2E./agent",+ "/v2/search/./x",+ "/v2/search\\..\\agent",+ "/v2/search/..",+ ] {+ assert!(has_dot_segment(path), "{path}");+ assert!(!cloud(CloudMode::SearchOnly).allows(path), "{path}");+ assert!(!cloud(CloudMode::Fallback).allows(path), "{path}");+ }+ assert!(!has_dot_segment("/v2/search/v1.2/..hidden"));+ assert!(!has_dot_segment("/v2/search/a.b"));+ }++ #[tokio::test]+ async fn forward_refuses_a_path_it_would_not_send_as_written() {+ let cloud =+ Cloud::new(echo_cloud().await, None, CloudMode::SearchOnly).with_install_id("test-install");+ let refused = cloud+ .forward(+ reqwest::Method::POST,+ "/v2/search/../agent?x=1",+ Some("application/json"),+ bytes::Bytes::from_static(b"{}"),+ None,+ None,+ )+ .await;+ assert_eq!(refused.err().map(|e| e.status), Some(400));+ }++ #[test]+ fn a_client_origin_replaces_the_engine_origin() {+ let cloud = Cloud::new("https://api.example.test".into(), None, CloudMode::Fallback);+ assert_eq!(+ cloud.clone().with_origin("[email protected]").origin(),+ );+ assert_eq!(cloud.clone().with_origin(" ").origin(), cloud.origin());+ assert_eq!(cloud.with_origin("a b\nc").origin(), "abc");+ }++ #[test]+ fn modes_parse() {+ assert_eq!(CloudMode::parse("off"), Some(CloudMode::Off));+ assert_eq!(CloudMode::parse("search"), Some(CloudMode::SearchOnly));+ assert_eq!(CloudMode::parse("auto"), Some(CloudMode::Fallback));+ assert_eq!(CloudMode::parse("nonsense"), None);+ assert_eq!(CloudMode::parse(" Disabled "), Some(CloudMode::Off));+ assert_eq!(CloudMode::parse("no"), Some(CloudMode::Off));+ assert_eq!(CloudMode::parse("yes"), Some(CloudMode::Fallback));+ }++ #[test]+ fn a_setting_that_is_not_understood_turns_the_cloud_off() {+ assert_eq!(CloudMode::requested(None), None);+ assert_eq!(CloudMode::requested(Some(" ")), None);+ assert_eq!(+ CloudMode::requested(Some("search")),+ Some(CloudMode::SearchOnly)+ );+ assert_eq!(+ CloudMode::requested(Some("auto")),+ Some(CloudMode::Fallback)+ );+ assert_eq!(CloudMode::requested(Some("offf")), Some(CloudMode::Off));+ assert_eq!(CloudMode::requested(Some("nope")), Some(CloudMode::Off));+ }++ #[test]+ fn blocked_detection() {+ assert!(looks_blocked(403, "anything"));+ assert!(looks_blocked(+ 200,+ "Please verify you are human to continue"+ ));+ assert!(!looks_blocked(+ 200,+ "A normal article about captcha history. "+ .repeat(100)+ .as_str()+ ));+ assert!(!looks_blocked(200, "Hello world"));+ assert!(looks_blocked(200, "We're verifying your browser\n\n[Website owner? Click here to fix](https://vercel.link/security-checkpoint)"));+ assert!(looks_blocked(+ 200,+ "We're performing a quick browser check to help us fight bots."+ ));+ assert!(looks_like_bot_check("## Performing security verification"));+ assert!(!looks_like_bot_check(+ "A long guide to browser checks. ".repeat(100).as_str()+ ));+ }+}diff --git a/apps/local/src/crawl.rs b/apps/local/src/crawl.rsnew file mode 100644index 000000000..32c09900e--- /dev/null+++ b/apps/local/src/crawl.rs@@ -0,0 +1,671 @@+//! Crawl and map, in one process: a frontier, a visited set and a concurrency+//! limit. No queue, no database.++use std::{+ collections::{HashSet, VecDeque},+ sync::Arc,+ time::Duration,+};++use futures::stream::{FuturesUnordered, StreamExt};+use regex::Regex;+use serde::Deserialize;+use serde_json::{json, Value};+use texting_robots::Robot;+use url::Url;++use crate::{+ crawler,+ fetch::FetchOptions,+ scrape::{parse_url, Engine, ScrapeError, ScrapeOptions},+};++/// The token site owners can target in robots.txt.+pub const ROBOTS_AGENT: &str = "FirecrawlLocal";++const SKIP_EXTENSIONS: &[&str] = &[+ "png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "bmp", "tiff", "avif", "css", "js", "mjs",+ "map", "woff", "woff2", "ttf", "otf", "eot", "mp3", "mp4", "m4a", "mov", "avi", "webm", "wav",+ "ogg", "zip", "gz", "tar", "tgz", "bz2", "7z", "rar", "dmg", "exe", "msi", "pkg", "deb", "rpm",+ "apk", "iso", "bin", "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "csv", "epub",+];++#[derive(Debug, Clone, Deserialize)]+#[serde(rename_all = "camelCase", default)]+pub struct CrawlOptions {+ /// Maximum pages to return.+ pub limit: usize,+ /// How many links deep to follow from the start URL.+ pub max_discovery_depth: Option<u32>,+ /// Regexes on the URL path; a URL must match one when any are given.+ pub include_paths: Vec<String>,+ /// Regexes on the URL path; a URL matching any is skipped.+ pub exclude_paths: Vec<String>,+ /// Follow links anywhere on the site, not only under the start path.+ pub crawl_entire_domain: bool,+ pub allow_subdomains: bool,+ pub allow_external_links: bool,+ /// "include" (default), "skip" or "only".+ pub sitemap: String,+ pub ignore_robots_txt: bool,+ /// Pages fetched at once.+ pub max_concurrency: Option<usize>,+ /// Milliseconds to wait between requests.+ pub delay: Option<u64>,+ pub scrape_options: ScrapeOptions,+}++impl Default for CrawlOptions {+ fn default() -> Self {+ Self {+ limit: 100,+ max_discovery_depth: None,+ include_paths: Vec::new(),+ exclude_paths: Vec::new(),+ crawl_entire_domain: false,+ allow_subdomains: false,+ allow_external_links: false,+ sitemap: "include".into(),+ ignore_robots_txt: false,+ max_concurrency: None,+ delay: None,+ scrape_options: ScrapeOptions::default(),+ }+ }+}++pub struct CrawlOutcome {+ pub pages: Vec<Value>,+ pub errors: Vec<Value>,+ pub robots_blocked: Vec<String>,+}++fn host_key(host: &str) -> &str {+ host.strip_prefix("www.").unwrap_or(host)+}++fn registrable(host: &str) -> String {+ psl::domain_str(host).unwrap_or(host).to_string()+}++/// Canonical form used for de-duplication: no fragment, no default port,+/// no trailing slash except on the root.+pub fn canonical(url: &Url) -> String {+ let mut u = url.clone();+ u.set_fragment(None);+ let mut s = u.to_string();+ if s.ends_with('/') && u.path() != "/" {+ s.pop();+ }+ s+}++fn has_skipped_extension(url: &Url) -> bool {+ let path = url.path().to_ascii_lowercase();+ match path.rsplit_once('.') {+ Some((_, ext)) if !ext.contains('/') => SKIP_EXTENSIONS.contains(&ext),+ _ => false,+ }+}++/// Directory prefix a crawl stays under when `crawlEntireDomain` is off.+fn scope_prefix(seed: &Url) -> String {+ let path = seed.path();+ if path.ends_with('/') {+ return path.to_string();+ }+ let last = path.rsplit('/').next().unwrap_or("");+ if last.contains('.') {+ // A document: scope to its directory.+ match path.rfind('/') {+ Some(i) => path[..=i].to_string(),+ None => "/".to_string(),+ }+ } else {+ // An extensionless page: the page and its children.+ format!("{path}/")+ }+}++struct Scope {+ seed: Url,+ prefix: String,+ seed_host: String,+ seed_domain: String,+ includes: Vec<Regex>,+ excludes: Vec<Regex>,+ entire_domain: bool,+ allow_subdomains: bool,+ allow_external: bool,+}++impl Scope {+ fn new(seed: &Url, opts: &CrawlOptions) -> Result<Self, ScrapeError> {+ let compile = |patterns: &[String]| -> Result<Vec<Regex>, ScrapeError> {+ patterns+ .iter()+ .map(|p| {+ Regex::new(p).map_err(|e| ScrapeError {+ status: 400,+ code: "BAD_REQUEST",+ message: format!("Invalid path pattern '{p}': {e}"),+ })+ })+ .collect()+ };+ let host = seed.host_str().unwrap_or("").to_ascii_lowercase();+ Ok(Self {+ seed: seed.clone(),+ prefix: scope_prefix(seed),+ seed_host: host_key(&host).to_string(),+ seed_domain: registrable(&host),+ includes: compile(&opts.include_paths)?,+ excludes: compile(&opts.exclude_paths)?,+ entire_domain: opts.crawl_entire_domain,+ allow_subdomains: opts.allow_subdomains,+ allow_external: opts.allow_external_links,+ })+ }++ fn same_site(&self, url: &Url) -> bool {+ let host = url.host_str().unwrap_or("").to_ascii_lowercase();+ if host_key(&host) == self.seed_host {+ return true;+ }+ self.allow_subdomains && registrable(&host) == self.seed_domain+ }++ fn allows(&self, url: &Url) -> bool {+ if !matches!(url.scheme(), "http" | "https") || has_skipped_extension(url) {+ return false;+ }+ let same_site = self.same_site(url);+ if !same_site && !self.allow_external {+ return false;+ }+ let path = url.path();+ if same_site && !self.entire_domain {+ let seed_path = self.seed.path();+ let under =+ path == seed_path || path.starts_with(&self.prefix) || format!("{path}/") == self.prefix;+ if !under {+ return false;+ }+ }+ if self.excludes.iter().any(|r| r.is_match(path)) {+ return false;+ }+ if !self.includes.is_empty() && !self.includes.iter().any(|r| r.is_match(path)) {+ return false;+ }+ true+ }+}++struct Robots {+ robot: Option<Robot>,+ sitemaps: Vec<String>,+ delay: Option<f32>,+}++async fn load_robots(engine: &Engine, seed: &Url) -> Robots {+ let empty = Robots {+ robot: None,+ sitemaps: Vec::new(),+ delay: None,+ };+ let Ok(robots_url) = seed.join("/robots.txt") else {+ return empty;+ };+ let opts = FetchOptions {+ timeout: Some(Duration::from_secs(10)),+ ..Default::default()+ };+ match engine.fetcher().fetch(&robots_url, &opts).await {+ Ok(res) if (200..300).contains(&res.status) => match Robot::new(ROBOTS_AGENT, &res.body) {+ Ok(robot) => Robots {+ sitemaps: robot.sitemaps.clone(),+ delay: robot.delay,+ robot: Some(robot),+ },+ Err(_) => empty,+ },+ _ => empty,+ }+}++/// URLs listed in the site's sitemaps, following sitemap indexes.+pub async fn sitemap_urls(+ engine: &Engine,+ seed: &Url,+ extra: &[String],+ cap: usize,+) -> Vec<String> {+ let mut queue: VecDeque<String> = VecDeque::new();+ for s in extra {+ queue.push_back(s.clone());+ }+ if let Ok(u) = seed.join("/sitemap.xml") {+ queue.push_back(u.to_string());+ }+ let mut seen_files = HashSet::new();+ let mut out = Vec::new();+ let mut seen_urls = HashSet::new();+ let opts = FetchOptions {+ timeout: Some(Duration::from_secs(15)),+ ..Default::default()+ };+ while let Some(sitemap) = queue.pop_front() {+ if seen_files.len() >= 50 || out.len() >= cap {+ break;+ }+ if !seen_files.insert(sitemap.clone()) {+ continue;+ }+ let Ok(url) = Url::parse(&sitemap) else {+ continue;+ };+ let Ok(res) = engine.fetcher().fetch(&url, &opts).await else {+ continue;+ };+ if !(200..300).contains(&res.status) {+ continue;+ }+ let Ok(processed) = crawler::process_sitemap(res.text()).await else {+ continue;+ };+ for instruction in processed.instructions {+ if instruction.action == "recurse" {+ for u in instruction.urls {+ queue.push_back(u);+ }+ } else {+ for u in instruction.urls {+ if out.len() >= cap {+ break;+ }+ if seen_urls.insert(u.clone()) {+ out.push(u);+ }+ }+ }+ }+ }+ out+}++/// Crawl from `start`. `on_page` runs as each page finishes, so callers can+/// stream results or write files while the crawl is still going.+pub async fn crawl<F>(+ engine: Arc<Engine>,+ start: &str,+ opts: &CrawlOptions,+ mut on_page: F,+) -> Result<CrawlOutcome, ScrapeError>+where+ F: FnMut(&Value),+{+ let seed = parse_url(start)?;+ let scope = Scope::new(&seed, opts)?;+ let limit = opts.limit.max(1);+ let max_depth = opts.max_discovery_depth.unwrap_or(u32::MAX);+ let robots = if opts.ignore_robots_txt {+ Robots {+ robot: None,+ sitemaps: Vec::new(),+ delay: None,+ }+ } else {+ load_robots(&engine, &seed).await+ };++ // robots.txt Crawl-delay means one request at a time, spaced out.+ let robots_delay = robots+ .delay+ .map(|d| Duration::from_secs_f32(d.clamp(0.0, 30.0)));+ let delay = opts.delay.map(Duration::from_millis).or(robots_delay);+ let concurrency = if delay.is_some() {+ 1+ } else {+ opts.max_concurrency.unwrap_or(8).clamp(1, 64)+ };++ let mut frontier: VecDeque<(Url, u32)> = VecDeque::new();+ let mut visited: HashSet<String> = HashSet::new();+ let mut robots_blocked = Vec::new();++ let enqueue = |url: Url,+ depth: u32,+ frontier: &mut VecDeque<(Url, u32)>,+ visited: &mut HashSet<String>,+ blocked: &mut Vec<String>| {+ let key = canonical(&url);+ if visited.contains(&key) {+ return;+ }+ if let Some(robot) = &robots.robot {+ if !robot.allowed(url.as_str()) {+ visited.insert(key);+ blocked.push(url.to_string());+ return;+ }+ }+ visited.insert(key);+ frontier.push_back((url, depth));+ };++ if opts.sitemap != "only" {+ enqueue(+ seed.clone(),+ 0,+ &mut frontier,+ &mut visited,+ &mut robots_blocked,+ );+ }+ if opts.sitemap != "skip" {+ let urls = sitemap_urls(&engine, &seed, &robots.sitemaps, limit * 4).await;+ for u in urls {+ if let Ok(parsed) = Url::parse(&u) {+ if scope.allows(&parsed) {+ enqueue(parsed, 1, &mut frontier, &mut visited, &mut robots_blocked);+ }+ }+ }+ }++ let mut pages: Vec<Value> = Vec::new();+ let mut errors: Vec<Value> = Vec::new();+ let mut in_flight = FuturesUnordered::new();+ let mut started = 0usize;++ loop {+ while in_flight.len() < concurrency && started < limit {+ let Some((url, depth)) = frontier.pop_front() else {+ break;+ };+ started += 1;+ let engine = engine.clone();+ let scrape_opts = opts.scrape_options.clone();+ in_flight.push(async move {+ if let Some(d) = delay {+ tokio::time::sleep(d).await;+ }+ let res = engine.scrape_with_links(url.as_str(), &scrape_opts).await;+ (url, depth, res)+ });+ }+ let Some((url, depth, res)) = in_flight.next().await else {+ break;+ };+ match res {+ Ok((doc, links)) => {+ on_page(&doc);+ pages.push(doc);+ if depth < max_depth && opts.sitemap != "only" {+ for link in links {+ if let Ok(parsed) = Url::parse(&link) {+ if scope.allows(&parsed) {+ enqueue(+ parsed,+ depth + 1,+ &mut frontier,+ &mut visited,+ &mut robots_blocked,+ );+ }+ }+ }+ }+ }+ Err(e) => {+ // A failed page does not count against the limit.+ started -= 1;+ errors.push(json!({ "url": url.to_string(), "code": e.code, "error": e.message }));+ }+ }+ }++ Ok(CrawlOutcome {+ pages,+ errors,+ robots_blocked,+ })+}++#[derive(Debug, Clone, Deserialize)]+#[serde(rename_all = "camelCase", default)]+pub struct MapOptions {+ pub search: Option<String>,+ pub limit: usize,+ pub include_subdomains: bool,+ /// "include" (default), "skip" or "only".+ pub sitemap: String,+}++impl Default for MapOptions {+ fn default() -> Self {+ Self {+ search: None,+ limit: 5000,+ include_subdomains: true,+ sitemap: "include".into(),+ }+ }+}++/// Every URL the engine can find for a site without crawling it: the+/// sitemaps plus the links on the start page.+pub async fn map(+ engine: Arc<Engine>,+ start: &str,+ opts: &MapOptions,+) -> Result<Vec<String>, ScrapeError> {+ let seed = parse_url(start)?;+ let host = seed.host_str().unwrap_or("").to_ascii_lowercase();+ let seed_host = host_key(&host).to_string();+ let seed_domain = registrable(&host);+ let limit = opts.limit.clamp(1, 100_000);++ let robots = load_robots(&engine, &seed).await;+ let mut candidates: Vec<String> = Vec::new();++ if opts.sitemap != "skip" {+ candidates.extend(sitemap_urls(&engine, &seed, &robots.sitemaps, limit * 2).await);+ }+ if opts.sitemap != "only" {+ let page_opts = ScrapeOptions {+ formats: vec![Value::String("links".into())],+ ..Default::default()+ };+ if let Ok((_, links)) = engine.scrape_with_links(seed.as_str(), &page_opts).await {+ candidates.push(seed.to_string());+ candidates.extend(links);+ }+ }++ let needle = opts.search.as_ref().map(|s| s.to_ascii_lowercase());+ let mut seen = HashSet::new();+ let mut out = Vec::new();+ for c in candidates {+ let Ok(url) = Url::parse(&c) else { continue };+ if !matches!(url.scheme(), "http" | "https") {+ continue;+ }+ let h = url.host_str().unwrap_or("").to_ascii_lowercase();+ let on_site =+ host_key(&h) == seed_host || (opts.include_subdomains && registrable(&h) == seed_domain);+ if !on_site {+ continue;+ }+ let key = canonical(&url);+ if let Some(n) = &needle {+ if !key.to_ascii_lowercase().contains(n) {+ continue;+ }+ }+ if seen.insert(key.clone()) {+ out.push(key);+ if out.len() >= limit {+ break;+ }+ }+ }+ Ok(out)+}++fn file_name_for(url: &str, index: usize) -> String {+ let slug: String = url+ .trim_start_matches("https://")+ .trim_start_matches("http://")+ .chars()+ .map(|c| {+ if c.is_ascii_alphanumeric() || c == '-' || c == '.' {+ c+ } else {+ '_'+ }+ })+ .collect();+ let slug = slug.trim_matches('_');+ let slug = if slug.len() > 120 { &slug[..120] } else { slug };+ format!("{:04}-{slug}.md", index + 1)+}++/// A value on one line: control characters, tabs and line breaks included,+/// become spaces, so it cannot break a line-based or tab-separated record.+fn one_line(value: &str) -> String {+ value+ .chars()+ .map(|c| if c.is_control() { ' ' } else { c })+ .collect::<String>()+ .trim()+ .to_string()+}++/// A value as a double-quoted YAML scalar. A JSON string is one, and quoting+/// keeps a title such as "Docs: Setup" from being read as a mapping.+fn yaml_string(value: &str) -> String {+ serde_json::to_string(&one_line(value)).unwrap_or_else(|_| "\"\"".to_string())+}++/// Write one markdown file per page and return an index of "file url title".+pub async fn write_pages(dir: &std::path::Path, pages: &[Value]) -> Result<Vec<String>, String> {+ tokio::fs::create_dir_all(dir)+ .await+ .map_err(|e| format!("{}: {e}", dir.display()))?;+ let mut index = Vec::new();+ for (i, page) in pages.iter().enumerate() {+ let url = page+ .pointer("/metadata/url")+ .and_then(Value::as_str)+ .unwrap_or("");+ let title = page+ .pointer("/metadata/title")+ .and_then(Value::as_str)+ .unwrap_or("");+ let md = page.get("markdown").and_then(Value::as_str).unwrap_or("");+ let name = file_name_for(url, i);+ let body = format!(+ "---\nurl: {}\ntitle: {}\n---\n\n{md}\n",+ yaml_string(url),+ yaml_string(title)+ );+ tokio::fs::write(dir.join(&name), body)+ .await+ .map_err(|e| format!("{name}: {e}"))?;+ index.push(format!("{name}\t{}\t{}", one_line(url), one_line(title)));+ }+ Ok(index)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn scope(seed: &str, opts: CrawlOptions) -> Scope {+ Scope::new(&Url::parse(seed).unwrap(), &opts).unwrap()+ }++ fn u(s: &str) -> Url {+ Url::parse(s).unwrap()+ }++ #[tokio::test]+ async fn a_page_title_cannot_break_the_front_matter_or_the_index() {+ let dir = std::env::temp_dir().join(format!("fc-local-pages-{}", uuid::Uuid::new_v4()));+ let pages = vec![serde_json::json!({+ "markdown": "body",+ "metadata": {+ "url": "https://example.com/a",+ "title": "Docs: Setup\ninjected: yes\tand \"quotes\" # not a comment"+ }+ })];+ let index = write_pages(&dir, &pages).await.unwrap();+ assert_eq!(index.len(), 1);+ assert_eq!(index[0].split('\t').count(), 3, "{}", index[0]);+ assert!(!index[0].contains('\n'));+ let name = index[0].split('\t').next().unwrap();+ let written = tokio::fs::read_to_string(dir.join(name)).await.unwrap();+ let _ = tokio::fs::remove_dir_all(&dir).await;+ let front: Vec<&str> = written.split("---\n").nth(1).unwrap().lines().collect();+ assert_eq!(front.len(), 2, "{written}");+ assert_eq!(front[0], "url: \"https://example.com/a\"");+ assert_eq!(+ front[1],+ "title: \"Docs: Setup injected: yes and \\\"quotes\\\" # not a comment\""+ );+ }++ #[test]+ fn stays_under_start_path_by_default() {+ let s = scope("https://example.com/docs/", CrawlOptions::default());+ assert!(s.allows(&u("https://example.com/docs/intro")));+ assert!(s.allows(&u("https://www.example.com/docs/a/b")));+ assert!(!s.allows(&u("https://example.com/blog/post")));+ assert!(!s.allows(&u("https://other.com/docs/intro")));+ assert!(!s.allows(&u("https://sub.example.com/docs/intro")));+ }++ #[test]+ fn document_seed_scopes_to_directory() {+ let s = scope(+ "https://example.com/docs/index.html",+ CrawlOptions::default(),+ );+ assert!(s.allows(&u("https://example.com/docs/other.html")));+ assert!(!s.allows(&u("https://example.com/about.html")));+ }++ #[test]+ fn extensionless_seed_allows_itself_and_children() {+ let s = scope("https://example.com/docs", CrawlOptions::default());+ assert!(s.allows(&u("https://example.com/docs")));+ assert!(s.allows(&u("https://example.com/docs/a")));+ assert!(!s.allows(&u("https://example.com/docs-old/a")));+ }++ #[test]+ fn entire_domain_subdomains_and_patterns() {+ let s = scope(+ "https://example.com/docs/",+ CrawlOptions {+ crawl_entire_domain: true,+ allow_subdomains: true,+ exclude_paths: vec!["^/private".into()],+ ..Default::default()+ },+ );+ assert!(s.allows(&u("https://example.com/blog/post")));+ assert!(s.allows(&u("https://sub.example.com/x")));+ assert!(!s.allows(&u("https://example.com/private/x")));+ assert!(!s.allows(&u("https://example.com/logo.png")));+ assert!(!s.allows(&u("mailto:[email protected]")));+ }++ #[test]+ fn canonical_drops_fragment_and_trailing_slash() {+ assert_eq!(canonical(&u("https://a.dev/x/#top")), "https://a.dev/x");+ assert_eq!(canonical(&u("https://a.dev/")), "https://a.dev/");+ }+}diff --git a/apps/local/src/documents.rs b/apps/local/src/documents.rsnew file mode 100644index 000000000..556d94615--- /dev/null+++ b/apps/local/src/documents.rs@@ -0,0 +1,175 @@+//! PDFs and office documents, parsed with the same libraries the cloud API+//! uses: `pdf-inspector` for PDFs and `anydoc` (through the shared+//! `document` module) for DOCX, XLSX, PPTX, ODT, RTF and friends.++use url::Url;++pub enum DocKind {+ Pdf,+ /// An office document, with the URL's file extension as a format hint.+ Office(Option<String>),+}++const OFFICE_EXTENSIONS: &[&str] = &[+ "docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "epub",+];++fn extension_of(url: &Url) -> Option<String> {+ let last = url.path().rsplit('/').next()?;+ let (_, ext) = last.rsplit_once('.')?;+ let ext = ext.to_ascii_lowercase();+ if ext.len() <= 5 && ext.chars().all(|c| c.is_ascii_alphanumeric()) {+ Some(ext)+ } else {+ None+ }+}++/// Decide from the first bytes, then the content type and extension, whether+/// a response is a document rather than a web page. The bytes win: servers+/// often send PDFs as `text/html` or `application/octet-stream`.+pub fn sniff(body: &[u8], mime: Option<&str>, url: &Url) -> Option<DocKind> {+ // "%PDF-" at the very start, allowing a byte-order mark or a few stray+ // bytes of whitespace before it.+ let head = &body[..body.len().min(32)];+ let skip = head+ .iter()+ .take_while(|b| b.is_ascii_whitespace() || matches!(**b, 0xEF | 0xBB | 0xBF))+ .count();+ if head[skip..].starts_with(b"%PDF-") {+ return Some(DocKind::Pdf);+ }+ let ext = extension_of(url);+ let office_ext = ext+ .as_deref()+ .map(|e| OFFICE_EXTENSIONS.contains(&e))+ .unwrap_or(false);+ let office_mime = mime+ .map(|m| {+ m.contains("officedocument")+ || m.contains("msword")+ || m.contains("ms-excel")+ || m.contains("ms-powerpoint")+ || m.contains("opendocument")+ || m == "application/rtf"+ || m == "text/rtf"+ || m == "application/epub+zip"+ })+ .unwrap_or(false);+ let zip_magic = body.starts_with(b"PK\x03\x04");+ let ole_magic = body.starts_with(&[0xD0, 0xCF, 0x11, 0xE0]);+ let rtf_magic = body.starts_with(b"{\\rtf");+ if office_mime+ || rtf_magic+ || ((zip_magic || ole_magic) && (office_ext || office_mime))+ || (office_ext && (zip_magic || ole_magic))+ {+ return Some(DocKind::Office(ext));+ }+ if mime == Some("application/pdf") {+ return Some(DocKind::Pdf);+ }+ None+}++pub struct ParsedDocument {+ pub markdown: String,+ pub title: Option<String>,+ pub pages: Option<u32>,+ /// Pages a text extraction could not read (scanned images).+ pub pages_needing_ocr: usize,+}++pub fn parse(kind: &DocKind, body: &[u8]) -> Result<ParsedDocument, String> {+ match kind {+ DocKind::Pdf => {+ let result =+ pdf_inspector::process_pdf_mem(body).map_err(|e| format!("could not read the PDF: {e}"))?;+ Ok(ParsedDocument {+ markdown: result.markdown.unwrap_or_default(),+ title: result.title.filter(|t| !t.trim().is_empty()),+ pages: Some(result.page_count),+ pages_needing_ocr: result.pages_needing_ocr.len(),+ })+ }+ DocKind::Office(ext) => {+ let markdown = crate::document::convert_document_bytes_to_markdown(body, ext.clone())+ .map_err(|e| format!("could not read the document: {e}"))?;+ Ok(ParsedDocument {+ markdown,+ title: None,+ pages: None,+ pages_needing_ocr: 0,+ })+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn u(s: &str) -> Url {+ Url::parse(s).unwrap()+ }++ #[test]+ fn pdf_bytes_win_over_a_wrong_content_type() {+ let body = b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj";+ assert!(matches!(+ sniff(body, Some("text/html"), &u("https://x.dev/download/12")),+ Some(DocKind::Pdf)+ ));+ assert!(matches!(+ sniff(b"", Some("application/pdf"), &u("https://x.dev/a")),+ Some(DocKind::Pdf)+ ));+ }++ #[test]+ fn office_documents_by_magic_and_extension() {+ let zip = b"PK\x03\x04rest";+ assert!(+ matches!(sniff(zip, Some("application/octet-stream"), &u("https://x.dev/f/report.xlsx")), Some(DocKind::Office(Some(ref e))) if e == "xlsx")+ );+ assert!(matches!(+ sniff(+ zip,+ Some("application/vnd.openxmlformats-officedocument.wordprocessingml.document"),+ &u("https://x.dev/dl?id=1")+ ),+ Some(DocKind::Office(None))+ ));+ // A plain zip download is not a document.+ assert!(sniff(+ zip,+ Some("application/zip"),+ &u("https://x.dev/archive.zip")+ )+ .is_none());+ }++ #[test]+ fn web_pages_are_not_documents() {+ assert!(sniff(+ b"<!doctype html><html>",+ Some("text/html"),+ &u("https://x.dev/page.html")+ )+ .is_none());+ assert!(sniff(+ b"plain text mentioning %PDF- later on",+ Some("text/plain"),+ &u("https://x.dev/a.txt")+ )+ .is_none());+ assert!(matches!(+ sniff(+ b"\n\n%PDF-1.4",+ Some("application/octet-stream"),+ &u("https://x.dev/d")+ ),+ Some(DocKind::Pdf)+ ));+ }+}diff --git a/apps/local/src/fetch.rs b/apps/local/src/fetch.rsnew file mode 100644index 000000000..7876b5dcc--- /dev/null+++ b/apps/local/src/fetch.rs@@ -0,0 +1,327 @@+//! Plain HTTP fetch: one shared connection pool, guarded DNS, guarded+//! redirects, a body size cap and charset-aware decoding.++use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration};++use bytes::BytesMut;+use futures::StreamExt;+use reqwest::{+ dns::{Addrs, Name, Resolve, Resolving},+ header::{HeaderMap, HeaderName, HeaderValue, ACCEPT, ACCEPT_LANGUAGE, CONTENT_TYPE},+ redirect, Client,+};+use url::Url;++use crate::guard::{self, Blocked, NetworkPolicy};++/// Looks like a browser to naive checks, and names itself so a site owner can+/// find and block it.+pub const USER_AGENT: &str = concat!(+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ",+ "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 ",+ "FirecrawlLocal/",+ env!("CARGO_PKG_VERSION"),+ " (+https://firecrawl.dev/local)"+);++pub const MAX_BODY_BYTES: usize = 32 * 1024 * 1024;+const MAX_REDIRECTS: usize = 10;++#[derive(Debug)]+pub enum FetchError {+ Blocked(String),+ Timeout,+ TooLarge,+ Network(String),+}++impl std::fmt::Display for FetchError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ FetchError::Blocked(m) => write!(f, "blocked: {m}"),+ FetchError::Timeout => write!(f, "the request timed out"),+ FetchError::TooLarge => write!(f, "the response is larger than {MAX_BODY_BYTES} bytes"),+ FetchError::Network(m) => write!(f, "network error: {m}"),+ }+ }+}++impl std::error::Error for FetchError {}++pub struct FetchResult {+ pub status: u16,+ pub final_url: Url,+ pub content_type: Option<String>,+ pub body: bytes::Bytes,+ pub headers: HeaderMap,+}++impl FetchResult {+ /// Media type without parameters, lowercased.+ pub fn mime(&self) -> Option<String> {+ self.content_type.as_deref().map(|ct| {+ ct.split(';')+ .next()+ .unwrap_or("")+ .trim()+ .to_ascii_lowercase()+ })+ }++ /// Decode the body to text: charset from the header, then a `<meta>` sniff,+ /// then UTF-8.+ pub fn text(&self) -> String {+ decode_body(&self.body, self.content_type.as_deref())+ }+}++fn charset_from_content_type(ct: &str) -> Option<&str> {+ ct.split(';').skip(1).find_map(|part| {+ let (k, v) = part.split_once('=')?;+ if k.trim().eq_ignore_ascii_case("charset") {+ Some(v.trim().trim_matches('"').trim_matches('\''))+ } else {+ None+ }+ })+}++fn sniff_meta_charset(body: &[u8]) -> Option<String> {+ let head = &body[..body.len().min(4096)];+ let lower = String::from_utf8_lossy(head).to_ascii_lowercase();+ let idx = lower.find("charset=")?;+ let rest = &lower[idx + 8..];+ let rest = rest.trim_start_matches(['"', '\'', ' ']);+ let end = rest+ .find(|c: char| !(c.is_ascii_alphanumeric() || c == '-' || c == '_'))+ .unwrap_or(rest.len());+ if end == 0 {+ None+ } else {+ Some(rest[..end].to_string())+ }+}++pub fn decode_body(body: &[u8], content_type: Option<&str>) -> String {+ let label = content_type+ .and_then(charset_from_content_type)+ .map(str::to_string)+ .or_else(|| sniff_meta_charset(body));+ if let Some(label) = label {+ if let Some(enc) = encoding_rs::Encoding::for_label(label.as_bytes()) {+ if enc != encoding_rs::UTF_8 {+ let (text, _, _) = enc.decode(body);+ return text.into_owned();+ }+ }+ }+ match std::str::from_utf8(body) {+ Ok(s) => s.to_owned(),+ Err(_) => String::from_utf8_lossy(body).into_owned(),+ }+}++/// DNS resolver that drops every address the policy forbids. A hostname that+/// only resolves to forbidden addresses fails to connect.+struct GuardedResolver {+ policy: NetworkPolicy,+}++impl Resolve for GuardedResolver {+ fn resolve(&self, name: Name) -> Resolving {+ let policy = self.policy;+ Box::pin(async move {+ let host = name.as_str().to_string();+ let resolved: Vec<SocketAddr> = tokio::net::lookup_host((host.as_str(), 0)).await?.collect();+ let allowed: Vec<SocketAddr> = resolved+ .into_iter()+ .filter(|a| guard::check_ip(a.ip(), policy).is_ok())+ .collect();+ if allowed.is_empty() {+ let err: Box<dyn std::error::Error + Send + Sync> = Box::new(Blocked(format!(+ "{host} does not resolve to an address Firecrawl Local may reach"+ )));+ return Err(err);+ }+ let addrs: Addrs = Box::new(allowed.into_iter());+ Ok(addrs)+ })+ }+}++#[derive(Clone)]+pub struct Fetcher {+ client: Client,+ policy: NetworkPolicy,+}++#[derive(Default, Clone)]+pub struct FetchOptions {+ pub headers: HashMap<String, String>,+ pub timeout: Option<Duration>,+}++impl Fetcher {+ pub fn new(policy: NetworkPolicy, user_agent: Option<&str>) -> Result<Self, String> {+ let mut default_headers = HeaderMap::new();+ default_headers.insert(+ ACCEPT,+ HeaderValue::from_static(+ "text/html,application/xhtml+xml,application/xml;q=0.9,text/markdown;q=0.9,*/*;q=0.8",+ ),+ );+ default_headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));++ let redirect_policy = redirect::Policy::custom(move |attempt| {+ if attempt.previous().len() >= MAX_REDIRECTS {+ return attempt.error(Blocked(format!("more than {MAX_REDIRECTS} redirects")));+ }+ match guard::check_url(attempt.url(), policy) {+ Ok(()) => attempt.follow(),+ Err(e) => attempt.error(e),+ }+ });++ let client = Client::builder()+ .user_agent(user_agent.unwrap_or(USER_AGENT))+ .default_headers(default_headers)+ .redirect(redirect_policy)+ .dns_resolver(Arc::new(GuardedResolver { policy }))+ .connect_timeout(Duration::from_secs(10))+ .pool_idle_timeout(Duration::from_secs(60))+ .pool_max_idle_per_host(64)+ .tcp_nodelay(true)+ .build()+ .map_err(|e| format!("could not build HTTP client: {e}"))?;++ Ok(Self { client, policy })+ }++ pub fn policy(&self) -> NetworkPolicy {+ self.policy+ }++ /// Fetch a URL, retrying once when the connection itself failed (a reset,+ /// a DNS hiccup, a dropped handshake). Those are usually gone a moment+ /// later; a refused certificate, a timeout or a guard refusal is not.+ pub async fn fetch(&self, url: &Url, opts: &FetchOptions) -> Result<FetchResult, FetchError> {+ match self.fetch_once(url, opts).await {+ Err(FetchError::Network(msg)) if is_transient(&msg) => {+ tokio::time::sleep(Duration::from_millis(250)).await;+ self.fetch_once(url, opts).await+ }+ other => other,+ }+ }++ async fn fetch_once(&self, url: &Url, opts: &FetchOptions) -> Result<FetchResult, FetchError> {+ guard::check_url(url, self.policy).map_err(|e| FetchError::Blocked(e.0))?;++ let timeout = opts.timeout.unwrap_or(Duration::from_secs(30));+ let mut req = self.client.get(url.clone()).timeout(timeout);+ for (k, v) in &opts.headers {+ if let (Ok(name), Ok(value)) = (+ HeaderName::from_bytes(k.as_bytes()),+ HeaderValue::from_str(v),+ ) {+ req = req.header(name, value);+ }+ }++ let resp = req.send().await.map_err(classify)?;+ let status = resp.status().as_u16();+ let final_url = resp.url().clone();+ let headers = resp.headers().clone();+ let content_type = headers+ .get(CONTENT_TYPE)+ .and_then(|v| v.to_str().ok())+ .map(str::to_string);++ if let Some(len) = resp.content_length() {+ if len as usize > MAX_BODY_BYTES {+ return Err(FetchError::TooLarge);+ }+ }++ let mut buf = BytesMut::with_capacity(resp.content_length().unwrap_or(64 * 1024) as usize);+ let mut stream = resp.bytes_stream();+ while let Some(chunk) = stream.next().await {+ let chunk = chunk.map_err(classify)?;+ if buf.len() + chunk.len() > MAX_BODY_BYTES {+ return Err(FetchError::TooLarge);+ }+ buf.extend_from_slice(&chunk);+ }++ Ok(FetchResult {+ status,+ final_url,+ content_type,+ body: buf.freeze(),+ headers,+ })+ }+}++/// A network failure worth one more try: anything but a certificate the+/// client refuses, which fails the same way every time.+fn is_transient(message: &str) -> bool {+ let lower = message.to_ascii_lowercase();+ !(lower.contains("certificate") || lower.contains("unsupported protocol"))+}++fn classify(e: reqwest::Error) -> FetchError {+ if e.is_timeout() {+ return FetchError::Timeout;+ }+ // The guard's refusals travel inside reqwest errors (redirect policy and+ // DNS resolver); surface them as Blocked instead of a generic failure.+ let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&e);+ while let Some(err) = source {+ if let Some(blocked) = err.downcast_ref::<Blocked>() {+ return FetchError::Blocked(blocked.0.clone());+ }+ source = err.source();+ }+ let mut msg = e.to_string();+ let mut source = std::error::Error::source(&e);+ while let Some(err) = source {+ msg.push_str(": ");+ msg.push_str(&err.to_string());+ source = err.source();+ }+ FetchError::Network(msg)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn charset_header_wins() {+ assert_eq!(+ charset_from_content_type("text/html; charset=ISO-8859-1"),+ Some("ISO-8859-1")+ );+ let body = [b'c', b'a', b'f', 0xe9];+ assert_eq!(+ decode_body(&body, Some("text/html; charset=iso-8859-1")),+ "café"+ );+ }++ #[test]+ fn meta_charset_sniffed() {+ let mut body = b"<html><head><meta charset=\"windows-1252\"></head><body>".to_vec();+ body.push(0x93);+ body.extend_from_slice(b"hi");+ body.push(0x94);+ let text = decode_body(&body, Some("text/html"));+ assert!(text.contains('\u{201c}') && text.contains('\u{201d}'));+ }++ #[test]+ fn utf8_default() {+ assert_eq!(decode_body("héllo".as_bytes(), None), "héllo");+ }+}diff --git a/apps/local/src/guard.rs b/apps/local/src/guard.rsnew file mode 100644index 000000000..32722deda--- /dev/null+++ b/apps/local/src/guard.rs@@ -0,0 +1,252 @@+//! Private-network guard.+//!+//! Every outbound request passes through here twice: once on the URL (scheme,+//! IP literals, localhost names) and once on the addresses DNS actually+//! returned, so a hostname that resolves to a private address is refused at+//! connect time. That second check is what defeats DNS rebinding.++use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};++use url::{Host, Url};++/// What the caller is allowed to reach.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum NetworkPolicy {+ /// Public internet only. The default for `serve`.+ PublicOnly,+ /// Public internet plus loopback and private ranges. Link-local and cloud+ /// metadata addresses stay blocked. Used when a person runs the CLI or MCP+ /// against their own localhost or intranet.+ AllowPrivate,+}++#[derive(Debug)]+pub struct Blocked(pub String);++impl std::fmt::Display for Blocked {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}", self.0)+ }+}++impl std::error::Error for Blocked {}++fn v4_is_metadata_or_link_local(ip: Ipv4Addr) -> bool {+ // 169.254.0.0/16 covers link-local and most cloud metadata endpoints.+ // Alibaba Cloud serves its metadata from 100.100.100.200 instead.+ ip.is_link_local() || ip == Ipv4Addr::new(100, 100, 100, 200)+}++fn v4_is_private(ip: Ipv4Addr) -> bool {+ let o = ip.octets();+ ip.is_loopback()+ || ip.is_private()+ || ip.is_unspecified()+ || ip.is_broadcast()+ || ip.is_multicast()+ || ip.is_documentation()+ // 100.64.0.0/10 carrier-grade NAT+ || (o[0] == 100 && (o[1] & 0b1100_0000) == 64)+ // 192.0.0.0/24 IETF protocol assignments+ || (o[0] == 192 && o[1] == 0 && o[2] == 0)+ // 198.18.0.0/15 benchmarking+ || (o[0] == 198 && (o[1] & 0xfe) == 18)+ // 240.0.0.0/4 reserved+ || o[0] >= 240+ || o[0] == 0+}++fn v6_embedded_v4(ip: Ipv6Addr) -> Option<Ipv4Addr> {+ if let Some(v4) = ip.to_ipv4_mapped() {+ return Some(v4);+ }+ let s = ip.segments();+ // 64:ff9b::/96 NAT64, and the deprecated ::a.b.c.d compatible form.+ if (s[0] == 0x64 && s[1] == 0xff9b && s[2..6].iter().all(|x| *x == 0))+ || (s[..6].iter().all(|x| *x == 0) && (s[6] != 0 || s[7] > 1))+ {+ return Some(Ipv4Addr::new(+ (s[6] >> 8) as u8,+ s[6] as u8,+ (s[7] >> 8) as u8,+ s[7] as u8,+ ));+ }+ None+}++fn v6_is_link_local(ip: Ipv6Addr) -> bool {+ (ip.segments()[0] & 0xffc0) == 0xfe80+}++fn v6_is_private(ip: Ipv6Addr) -> bool {+ let s = ip.segments();+ ip.is_loopback()+ || ip.is_unspecified()+ || ip.is_multicast()+ // fc00::/7 unique local+ || (s[0] & 0xfe00) == 0xfc00+ // fec0::/10 site-local (deprecated)+ || (s[0] & 0xffc0) == 0xfec0+ // 2001:db8::/32 documentation+ || (s[0] == 0x2001 && s[1] == 0x0db8)+}++/// Addresses nobody may reach through the engine, whatever the policy.+pub fn ip_always_blocked(ip: IpAddr) -> bool {+ match ip {+ IpAddr::V4(v4) => v4_is_metadata_or_link_local(v4),+ IpAddr::V6(v6) => match v6_embedded_v4(v6) {+ Some(v4) => v4_is_metadata_or_link_local(v4),+ None => v6_is_link_local(v6),+ },+ }+}++/// Addresses that are not on the public internet.+pub fn ip_is_private(ip: IpAddr) -> bool {+ match ip {+ IpAddr::V4(v4) => v4_is_private(v4) || v4_is_metadata_or_link_local(v4),+ IpAddr::V6(v6) => match v6_embedded_v4(v6) {+ Some(v4) => v4_is_private(v4) || v4_is_metadata_or_link_local(v4),+ None => v6_is_private(v6) || v6_is_link_local(v6),+ },+ }+}++pub fn check_ip(ip: IpAddr, policy: NetworkPolicy) -> Result<(), Blocked> {+ if ip_always_blocked(ip) {+ return Err(Blocked(format!(+ "{ip} is a link-local or metadata address and is never reachable"+ )));+ }+ if policy == NetworkPolicy::PublicOnly && ip_is_private(ip) {+ return Err(Blocked(format!(+ "{ip} is a private address; private targets are off"+ )));+ }+ Ok(())+}++fn hostname_is_local(host: &str) -> bool {+ let h = host.trim_end_matches('.').to_ascii_lowercase();+ h == "localhost"+ || h.ends_with(".localhost")+ || h.ends_with(".local")+ || h.ends_with(".internal")+ || h == "metadata"+ || h == "metadata.google.internal"+}++/// Check a URL before any network activity. Called on the first URL and on+/// every redirect hop.+pub fn check_url(url: &Url, policy: NetworkPolicy) -> Result<(), Blocked> {+ match url.scheme() {+ "http" | "https" => {}+ other => {+ return Err(Blocked(format!(+ "unsupported URL scheme '{other}': only http and https are fetched"+ )))+ }+ }+ if !url.username().is_empty() || url.password().is_some() {+ return Err(Blocked(+ "URLs with embedded credentials are not fetched".to_string(),+ ));+ }+ match url.host() {+ None => Err(Blocked("URL has no host".to_string())),+ Some(Host::Ipv4(ip)) => check_ip(IpAddr::V4(ip), policy),+ Some(Host::Ipv6(ip)) => check_ip(IpAddr::V6(ip), policy),+ Some(Host::Domain(name)) => {+ if policy == NetworkPolicy::PublicOnly && hostname_is_local(name) {+ return Err(Blocked(format!(+ "'{name}' is a local hostname; private targets are off"+ )));+ }+ if name.eq_ignore_ascii_case("metadata.google.internal") {+ return Err(Blocked(+ "cloud metadata endpoints are never reachable".to_string(),+ ));+ }+ Ok(())+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn alibaba_metadata_is_blocked_in_every_mode() {+ let ip: IpAddr = "100.100.100.200".parse().unwrap();+ assert!(check_ip(ip, NetworkPolicy::AllowPrivate).is_err());+ assert!(check_ip(ip, NetworkPolicy::PublicOnly).is_err());+ // The rest of the carrier-grade NAT range is only private.+ let neighbour: IpAddr = "100.100.100.201".parse().unwrap();+ assert!(check_ip(neighbour, NetworkPolicy::AllowPrivate).is_ok());+ }++ fn u(s: &str) -> Url {+ Url::parse(s).unwrap()+ }++ #[test]+ fn public_urls_pass() {+ assert!(check_url(&u("https://example.com/a"), NetworkPolicy::PublicOnly).is_ok());+ assert!(check_url(&u("http://93.184.216.34/"), NetworkPolicy::PublicOnly).is_ok());+ }++ #[test]+ fn private_literals_blocked_when_public_only() {+ for s in [+ "http://127.0.0.1/",+ "http://10.0.0.5/",+ "http://192.168.1.1/",+ "http://172.16.0.1/",+ "http://[::1]/",+ "http://[fd00::1]/",+ "http://[::ffff:127.0.0.1]/",+ "http://0.0.0.0/",+ "http://100.64.0.1/",+ "http://localhost:3000/",+ "http://foo.localhost/",+ "http://printer.local/",+ ] {+ assert!(+ check_url(&u(s), NetworkPolicy::PublicOnly).is_err(),+ "{s} should be blocked"+ );+ }+ }++ #[test]+ fn private_allowed_when_opted_in_but_metadata_never() {+ assert!(check_url(&u("http://127.0.0.1:3000/"), NetworkPolicy::AllowPrivate).is_ok());+ assert!(check_url(&u("http://localhost:3000/"), NetworkPolicy::AllowPrivate).is_ok());+ assert!(check_url(&u("http://192.168.1.10/"), NetworkPolicy::AllowPrivate).is_ok());+ for s in [+ "http://169.254.169.254/latest/meta-data/",+ "http://[fe80::1]/",+ "http://[::ffff:169.254.169.254]/",+ "http://metadata.google.internal/",+ ] {+ assert!(+ check_url(&u(s), NetworkPolicy::AllowPrivate).is_err(),+ "{s} should always be blocked"+ );+ }+ }++ #[test]+ fn schemes_and_credentials() {+ assert!(check_url(&u("file:///etc/passwd"), NetworkPolicy::AllowPrivate).is_err());+ assert!(check_url(&u("ftp://example.com/"), NetworkPolicy::PublicOnly).is_err());+ assert!(check_url(+ &u("https://user:[email protected]/"),+ NetworkPolicy::PublicOnly+ )+ .is_err());+ }+}diff --git a/apps/local/src/init.rs b/apps/local/src/init.rsnew file mode 100644index 000000000..2ab06e472--- /dev/null+++ b/apps/local/src/init.rs@@ -0,0 +1,352 @@+//! `firecrawl-local init`: add Firecrawl to every AI coding tool on this+//! machine in one command.+//!+//! Each tool keeps its MCP servers in its own config file. This finds the+//! ones that exist, backs each up, and adds a `firecrawl` entry that runs+//! `firecrawl-local mcp`. Nothing else in the file is touched.++use std::{+ fs,+ path::{Path, PathBuf},+};++use serde_json::{json, Map, Value};++pub struct InitOptions {+ /// Limit to these tools (names as in `targets()`); empty means every tool found.+ pub only: Vec<String>,+ /// Print what would change and write nothing.+ pub dry_run: bool,+ /// Home directory to look in. Tests point this at a temp directory.+ pub home: PathBuf,+ /// Path to this binary, written into each config.+ pub binary: PathBuf,+}++enum Kind {+ /// JSON file with a top-level object of servers under `key`.+ Json { key: &'static str },+ /// Codex's TOML file with `[mcp_servers.<name>]` tables.+ CodexToml,+}++struct Target {+ name: &'static str,+ label: &'static str,+ /// Path relative to the home directory.+ config: &'static str,+ /// Directory whose existence means the tool is installed.+ marker: &'static str,+ kind: Kind,+}++fn targets() -> Vec<Target> {+ vec![+ Target {+ name: "claude-code",+ label: "Claude Code",+ config: ".claude.json",+ marker: ".claude",+ kind: Kind::Json { key: "mcpServers" },+ },+ Target {+ name: "claude-desktop",+ label: "Claude desktop",+ config: "Library/Application Support/Claude/claude_desktop_config.json",+ marker: "Library/Application Support/Claude",+ kind: Kind::Json { key: "mcpServers" },+ },+ Target {+ name: "cursor",+ label: "Cursor",+ config: ".cursor/mcp.json",+ marker: ".cursor",+ kind: Kind::Json { key: "mcpServers" },+ },+ Target {+ name: "windsurf",+ label: "Windsurf",+ config: ".codeium/windsurf/mcp_config.json",+ marker: ".codeium/windsurf",+ kind: Kind::Json { key: "mcpServers" },+ },+ Target {+ name: "gemini",+ label: "Gemini CLI",+ config: ".gemini/settings.json",+ marker: ".gemini",+ kind: Kind::Json { key: "mcpServers" },+ },+ Target {+ name: "codex",+ label: "Codex",+ config: ".codex/config.toml",+ marker: ".codex",+ kind: Kind::CodexToml,+ },+ ]+}++pub fn target_names() -> Vec<&'static str> {+ targets().iter().map(|t| t.name).collect()+}++#[derive(Debug, PartialEq, Eq)]+pub enum Outcome {+ Added,+ Updated,+ AlreadySet,+}++fn backup(path: &Path) -> Result<(), String> {+ if path.exists() {+ let mut bak = path.as_os_str().to_owned();+ bak.push(".firecrawl-bak");+ fs::copy(path, PathBuf::from(bak))+ .map_err(|e| format!("could not back up {}: {e}", path.display()))?;+ }+ Ok(())+}++fn server_entry(binary: &Path) -> Value {+ json!({ "command": binary.to_string_lossy(), "args": ["mcp"] })+}++/// Merge the server entry into a JSON config, keeping every other key and+/// the file's key order.+fn merge_json(+ existing: Option<&str>,+ key: &str,+ binary: &Path,+) -> Result<(String, Outcome), String> {+ let mut root: Value = match existing {+ Some(text) if !text.trim().is_empty() => serde_json::from_str(text)+ .map_err(|e| format!("the file is not valid JSON ({e}); left untouched"))?,+ _ => Value::Object(Map::new()),+ };+ let obj = root+ .as_object_mut()+ .ok_or_else(|| "the file's top level is not a JSON object; left untouched".to_string())?;+ let servers = obj+ .entry(key.to_string())+ .or_insert_with(|| Value::Object(Map::new()));+ let servers = servers+ .as_object_mut()+ .ok_or_else(|| format!("'{key}' is not an object; left untouched"))?;+ let entry = server_entry(binary);+ let outcome = match servers.get("firecrawl") {+ Some(current) if *current == entry => Outcome::AlreadySet,+ Some(_) => Outcome::Updated,+ None => Outcome::Added,+ };+ servers.insert("firecrawl".to_string(), entry);+ let mut text = serde_json::to_string_pretty(&root).map_err(|e| e.to_string())?;+ text.push('\n');+ Ok((text, outcome))+}++fn toml_string(s: &str) -> String {+ format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))+}++/// Add or replace the `[mcp_servers.firecrawl]` table in Codex's config,+/// leaving the rest of the file byte-for-byte as it was.+fn merge_codex_toml(existing: Option<&str>, binary: &Path) -> (String, Outcome) {+ let block = format!(+ "[mcp_servers.firecrawl]\ncommand = {}\nargs = [\"mcp\"]\n",+ toml_string(&binary.to_string_lossy())+ );+ let text = existing.unwrap_or("");+ let header = "[mcp_servers.firecrawl]";+ match text.find(header) {+ Some(start) => {+ // The table runs until the next table header or the end of the file.+ let after = &text[start + header.len()..];+ let end = after+ .match_indices("\n[")+ .next()+ .map(|(i, _)| start + header.len() + i + 1)+ .unwrap_or(text.len());+ let current = &text[start..end];+ if current.trim_end() == block.trim_end() {+ return (text.to_string(), Outcome::AlreadySet);+ }+ let mut out = String::with_capacity(text.len() + block.len());+ out.push_str(&text[..start]);+ out.push_str(&block);+ if end < text.len() {+ out.push('\n');+ }+ out.push_str(&text[end..]);+ (out, Outcome::Updated)+ }+ None => {+ let mut out = text.to_string();+ if !out.is_empty() && !out.ends_with('\n') {+ out.push('\n');+ }+ if !out.is_empty() {+ out.push('\n');+ }+ out.push_str(&block);+ (out, Outcome::Added)+ }+ }+}++pub struct Report {+ pub lines: Vec<String>,+ pub configured: usize,+}++pub fn run(opts: &InitOptions) -> Report {+ let mut lines = Vec::new();+ let mut configured = 0usize;+ for target in targets() {+ if !opts.only.is_empty() && !opts.only.iter().any(|o| o == target.name) {+ continue;+ }+ let explicitly_asked = opts.only.iter().any(|o| o == target.name);+ if !explicitly_asked && !opts.home.join(target.marker).exists() {+ continue;+ }+ let path = opts.home.join(target.config);+ let existing = fs::read_to_string(&path).ok();+ let merged = match &target.kind {+ Kind::Json { key } => merge_json(existing.as_deref(), key, &opts.binary),+ Kind::CodexToml => Ok(merge_codex_toml(existing.as_deref(), &opts.binary)),+ };+ match merged {+ Err(e) => lines.push(format!(+ " {}: skipped, {e} ({})",+ target.label,+ path.display()+ )),+ Ok((_, Outcome::AlreadySet)) => {+ configured += 1;+ lines.push(format!(" {}: already set up", target.label));+ }+ Ok((text, outcome)) => {+ let verb = if outcome == Outcome::Added {+ "added"+ } else {+ "updated"+ };+ if opts.dry_run {+ lines.push(format!(+ " {}: would be {verb} in {}",+ target.label,+ path.display()+ ));+ continue;+ }+ let write = || -> Result<(), String> {+ if let Some(parent) = path.parent() {+ fs::create_dir_all(parent).map_err(|e| e.to_string())?;+ }+ backup(&path)?;+ fs::write(&path, &text).map_err(|e| e.to_string())+ };+ match write() {+ Ok(()) => {+ configured += 1;+ lines.push(format!(" {}: {verb} ({})", target.label, path.display()));+ }+ Err(e) => lines.push(format!(+ " {}: could not write {} ({e})",+ target.label,+ path.display()+ )),+ }+ }+ }+ }+ Report { lines, configured }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn bin() -> PathBuf {+ PathBuf::from("/usr/local/bin/firecrawl-local")+ }++ #[test]+ fn json_merge_keeps_other_servers_and_keys() {+ let existing = r#"{"theme":"dark","mcpServers":{"other":{"command":"x"}}}"#;+ let (text, outcome) = merge_json(Some(existing), "mcpServers", &bin()).unwrap();+ assert_eq!(outcome, Outcome::Added);+ let v: Value = serde_json::from_str(&text).unwrap();+ assert_eq!(v["theme"], "dark");+ assert_eq!(v["mcpServers"]["other"]["command"], "x");+ assert_eq!(v["mcpServers"]["firecrawl"]["args"][0], "mcp");+ // Key order is preserved: theme still comes first.+ assert!(text.find("theme").unwrap() < text.find("mcpServers").unwrap());++ let (_, again) = merge_json(Some(&text), "mcpServers", &bin()).unwrap();+ assert_eq!(again, Outcome::AlreadySet);+ }++ #[test]+ fn json_merge_refuses_broken_files() {+ assert!(merge_json(Some("{not json"), "mcpServers", &bin()).is_err());+ assert!(merge_json(Some("[]"), "mcpServers", &bin()).is_err());+ }++ #[test]+ fn codex_toml_append_and_replace() {+ let existing = "model = \"gpt\"\n\n[mcp_servers.other]\ncommand = \"x\"\n";+ let (text, outcome) = merge_codex_toml(Some(existing), &bin());+ assert_eq!(outcome, Outcome::Added);+ assert!(text.starts_with(existing));+ assert!(text.contains(+ "[mcp_servers.firecrawl]\ncommand = \"/usr/local/bin/firecrawl-local\"\nargs = [\"mcp\"]"+ ));++ let (again, outcome) = merge_codex_toml(Some(&text), &bin());+ assert_eq!(outcome, Outcome::AlreadySet);+ assert_eq!(again, text);++ let old =+ "[mcp_servers.firecrawl]\ncommand = \"/old/path\"\n\n[mcp_servers.other]\ncommand = \"x\"\n";+ let (replaced, outcome) = merge_codex_toml(Some(old), &bin());+ assert_eq!(outcome, Outcome::Updated);+ assert!(replaced.contains("/usr/local/bin/firecrawl-local"));+ assert!(!replaced.contains("/old/path"));+ assert!(replaced.contains("[mcp_servers.other]\ncommand = \"x\""));+ }++ #[test]+ fn run_only_touches_installed_tools_and_backs_up() {+ let home = std::env::temp_dir().join(format!("fc-init-test-{}", std::process::id()));+ let _ = fs::remove_dir_all(&home);+ fs::create_dir_all(home.join(".cursor")).unwrap();+ fs::write(+ home.join(".cursor/mcp.json"),+ r#"{"mcpServers":{"a":{"command":"b"}}}"#,+ )+ .unwrap();++ let opts = InitOptions {+ only: Vec::new(),+ dry_run: false,+ home: home.clone(),+ binary: bin(),+ };+ let report = run(&opts);+ assert_eq!(report.configured, 1, "{:?}", report.lines);+ let written: Value =+ serde_json::from_str(&fs::read_to_string(home.join(".cursor/mcp.json")).unwrap()).unwrap();+ assert_eq!(written["mcpServers"]["a"]["command"], "b");+ assert_eq!(+ written["mcpServers"]["firecrawl"]["command"],+ "/usr/local/bin/firecrawl-local"+ );+ assert!(home.join(".cursor/mcp.json.firecrawl-bak").exists());+ // Tools that are not installed get no config file.+ assert!(!home.join(".codex/config.toml").exists());+ assert!(!home.join(".claude.json").exists());+ let _ = fs::remove_dir_all(&home);+ }+}diff --git a/apps/local/src/lib.rs b/apps/local/src/lib.rsnew file mode 100644index 000000000..3aea477ae--- /dev/null+++ b/apps/local/src/lib.rs@@ -0,0 +1,44 @@+//! Firecrawl Local: the local primitive behind `firecrawl scrape|crawl|map`.+//!+//! The HTML cleaning, link extraction, metadata and crawl-filter modules are+//! compiled directly from `apps/api/native/src`, the same Rust the cloud API+//! loads through N-API. One source of truth means a page cleaned locally and a+//! page cleaned in the cloud go through identical code.+//!+//! A published crate carries a copy of those files in `src/native` (see+//! `build.rs` and `scripts/vendor-native.sh`).++#![allow(dead_code)]++#[cfg_attr(not(vendored_native), path = "../../api/native/src/utils.rs")]+#[cfg_attr(vendored_native, path = "native/utils.rs")]+mod utils;+#[cfg_attr(not(vendored_native), path = "../../api/native/src/xml.rs")]+#[cfg_attr(vendored_native, path = "native/xml.rs")]+mod xml;++#[cfg_attr(not(vendored_native), path = "../../api/native/src/html.rs")]+#[cfg_attr(vendored_native, path = "native/html.rs")]+pub mod html;++#[cfg_attr(not(vendored_native), path = "../../api/native/src/crawler.rs")]+#[cfg_attr(vendored_native, path = "native/crawler.rs")]+pub mod crawler;++#[cfg_attr(not(vendored_native), path = "../../api/native/src/document.rs")]+#[cfg_attr(vendored_native, path = "native/document.rs")]+pub mod document;++pub mod browser;+pub mod cloud;+pub mod crawl;+pub mod documents;+pub mod fetch;+pub mod guard;+pub mod init;+pub mod login;+pub mod markdown;+pub mod mem;+pub mod netguard;+pub mod scrape;+pub mod serve;diff --git a/apps/local/src/login.rs b/apps/local/src/login.rsnew file mode 100644index 000000000..a667f812b--- /dev/null+++ b/apps/local/src/login.rs@@ -0,0 +1,190 @@+//! One-step sign-in: the same browser handshake as `firecrawl login`.+//!+//! The engine makes a link, the person opens it and signs in, and the key+//! comes back by itself and is saved where the Firecrawl CLI keeps its login.+//! Nobody copies a key. The secret half of the handshake (the verifier) never+//! leaves this process except in the POST that redeems it.++use std::time::{Duration, Instant};++use base64::Engine as _;+use serde_json::{json, Value};+use sha2::{Digest, Sha256};++pub const WEB_URL: &str = "https://www.firecrawl.dev";+const POLL_EVERY: Duration = Duration::from_secs(2);++#[derive(Clone)]+pub struct LoginSession {+ pub url: String,+ session_id: String,+ code_verifier: String,+ web_url: String,+}++pub struct SignedIn {+ pub api_key: String,+ pub api_url: Option<String>,+ pub team_name: Option<String>,+}++pub enum Poll {+ Pending,+ Complete(SignedIn),+ /// Rate limited or a server fault: keep polling.+ Busy(String),+ /// A refusal that more polling will not change.+ Failed(String),+}++fn random_bytes<const N: usize>() -> Result<[u8; N], String> {+ let mut buf = [0u8; N];+ getrandom::fill(&mut buf).map_err(|e| format!("no secure random source: {e}"))?;+ Ok(buf)+}++fn challenge_for(verifier: &str) -> String {+ base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()))+}++impl LoginSession {+ pub fn start() -> Result<Self, String> {+ let web_url = std::env::var("FIRECRAWL_WEB_URL")+ .ok()+ .filter(|u| !u.trim().is_empty())+ .unwrap_or_else(|| WEB_URL.to_string());+ let session_id: String = random_bytes::<32>()?+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect();+ let code_verifier =+ base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(random_bytes::<32>()?);+ let url = format!(+ "{}/cli-auth?code_challenge={}#session_id={}",+ web_url.trim_end_matches('/'),+ challenge_for(&code_verifier),+ session_id+ );+ Ok(Self {+ url,+ session_id,+ code_verifier,+ web_url,+ })+ }++ pub async fn poll(&self, client: &reqwest::Client) -> Poll {+ let resp = client+ .post(format!(+ "{}/api/auth/cli/status",+ self.web_url.trim_end_matches('/')+ ))+ .timeout(Duration::from_secs(10))+ .json(&json!({ "session_id": self.session_id, "code_verifier": self.code_verifier }))+ .send()+ .await;+ let resp = match resp {+ Ok(r) => r,+ Err(e) => return Poll::Busy(format!("could not reach {}: {e}", self.web_url)),+ };+ let status = resp.status().as_u16();+ if status == 429 || status >= 500 {+ return Poll::Busy(format!("server answered {status}"));+ }+ if !(200..300).contains(&status) {+ return Poll::Failed(format!("server answered {status}"));+ }+ let data: Value = match resp.json().await {+ Ok(v) => v,+ Err(e) => return Poll::Busy(format!("unreadable reply: {e}")),+ };+ let field = |name: &str| data.get(name).and_then(Value::as_str).map(str::to_string);+ match (data.get("status").and_then(Value::as_str), field("apiKey")) {+ (Some("complete"), Some(api_key)) if !api_key.is_empty() => Poll::Complete(SignedIn {+ api_key,+ api_url: field("apiUrl"),+ team_name: field("teamName"),+ }),+ _ => Poll::Pending,+ }+ }++ /// Poll until the person finishes signing in, or `limit` passes.+ pub async fn wait(&self, limit: Duration) -> Result<SignedIn, String> {+ let client = reqwest::Client::new();+ let started = Instant::now();+ loop {+ match self.poll(&client).await {+ Poll::Complete(s) => return Ok(s),+ Poll::Failed(detail) => return Err(format!("sign-in was refused: {detail}")),+ Poll::Pending | Poll::Busy(_) => {}+ }+ if started.elapsed() >= limit {+ return Err("sign-in was not finished in time".to_string());+ }+ tokio::time::sleep(POLL_EVERY).await;+ }+ }+}++/// Save the login where the Firecrawl CLI keeps it, readable only by the+/// user, so the CLI and the engine share one sign-in.+pub fn save(signed_in: &SignedIn) -> Result<std::path::PathBuf, String> {+ let path =+ crate::cloud::cli_credentials_path().ok_or("no home directory to save the login in")?;+ if let Some(dir) = path.parent() {+ std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;+ }+ let mut body = json!({ "apiKey": signed_in.api_key });+ if let Some(url) = &signed_in.api_url {+ body["apiUrl"] = Value::String(url.clone());+ }+ let text = serde_json::to_string_pretty(&body).map_err(|e| e.to_string())?;+ std::fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))?;+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600));+ }+ Ok(path)+}++/// Open a URL in the default browser. Best effort.+pub fn open_browser(url: &str) -> bool {+ let result = if cfg!(target_os = "macos") {+ std::process::Command::new("open").arg(url).status()+ } else if cfg!(target_os = "windows") {+ std::process::Command::new("cmd")+ .args(["/C", "start", "", url])+ .status()+ } else {+ std::process::Command::new("xdg-open").arg(url).status()+ };+ matches!(result, Ok(s) if s.success())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn challenge_matches_the_pkce_test_vector() {+ // RFC 7636 appendix B.+ assert_eq!(+ challenge_for("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk"),+ "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM"+ );+ }++ #[test]+ fn session_link_has_the_challenge_but_not_the_verifier() {+ let s = LoginSession::start().unwrap();+ assert!(s+ .url+ .starts_with("https://www.firecrawl.dev/cli-auth?code_challenge="));+ assert!(s.url.contains(&challenge_for(&s.code_verifier)));+ assert!(!s.url.contains(&s.code_verifier));+ assert_eq!(s.session_id.len(), 64);+ assert!(s.url.ends_with(&format!("#session_id={}", s.session_id)));+ }+}diff --git a/apps/local/src/main.rs b/apps/local/src/main.rsnew file mode 100644index 000000000..296679adf--- /dev/null+++ b/apps/local/src/main.rs@@ -0,0 +1,673 @@+use std::{net::SocketAddr, process::ExitCode, sync::Arc};++use clap::{Parser, Subcommand};+use futures::stream::{self, StreamExt};+use serde_json::{json, Value};++use firecrawl_local::{+ cloud::Cloud,+ crawl::{self, CrawlOptions, MapOptions},+ guard::NetworkPolicy,+ init::{self, InitOptions},+ login::{self, LoginSession},+ scrape::{BrowserMode, Engine, EngineConfig, ScrapeOptions},+ serve::{self, ServeConfig},+};++// DOM-heavy work allocates and frees a lot; mimalloc is markedly faster at it+// than the system allocator.+#[global_allocator]+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;++#[derive(Parser)]+#[command(+ name = "firecrawl-local",+ version,+ about = "Scrape, crawl and map the web from your own machine."+)]+struct Cli {+ /// When to use the local Chrome for JavaScript pages: auto, never or always.+ #[arg(+ long,+ global = true,+ env = "FIRECRAWL_LOCAL_BROWSER",+ default_value = "auto"+ )]+ browser: String,+ #[command(subcommand)]+ command: Command,+}++#[derive(Subcommand)]+enum Command {+ /// Scrape one or more URLs and print Firecrawl-shaped JSON.+ Scrape {+ /// One or more URLs.+ #[arg(required = true)]+ urls: Vec<String>,+ /// Comma-separated formats: markdown, html, rawHtml, links.+ #[arg(long, short = 'f', default_value = "markdown", value_delimiter = ',')]+ format: Vec<String>,+ /// Keep navigation, headers and footers instead of main content only.+ #[arg(long)]+ full_page: bool,+ /// Print only the markdown, not the JSON envelope.+ #[arg(long)]+ markdown: bool,+ /// Per-page timeout in milliseconds.+ #[arg(long, default_value_t = 30_000)]+ timeout: u64,+ /// How many URLs to fetch at once.+ #[arg(long, short = 'c', default_value_t = 16)]+ concurrency: usize,+ /// Refuse localhost and private-network targets.+ #[arg(long)]+ public_only: bool,+ /// Override the User-Agent header.+ #[arg(long, env = "FIRECRAWL_LOCAL_USER_AGENT")]+ user_agent: Option<String>,+ },+ /// Crawl a site from a start URL.+ Crawl {+ url: String,+ /// Maximum pages.+ #[arg(long, short = 'l', default_value_t = 100)]+ limit: usize,+ /// How many links deep to follow.+ #[arg(long)]+ max_depth: Option<u32>,+ /// Follow links anywhere on the site, not only under the start path.+ #[arg(long)]+ entire_domain: bool,+ /// Regex a URL path must match. Repeatable.+ #[arg(long = "include")]+ include_paths: Vec<String>,+ /// Regex that skips a URL path. Repeatable.+ #[arg(long = "exclude")]+ exclude_paths: Vec<String>,+ /// Pages fetched at once.+ #[arg(long, short = 'c')]+ concurrency: Option<usize>,+ /// Write one markdown file per page into this directory.+ #[arg(long, short = 'o')]+ out: Option<std::path::PathBuf>,+ /// Do not read robots.txt.+ #[arg(long)]+ ignore_robots: bool,+ /// Refuse localhost and private-network targets.+ #[arg(long)]+ public_only: bool,+ },+ /// List a site's URLs from its sitemaps and start page.+ Map {+ url: String,+ /// Only URLs containing this text.+ #[arg(long)]+ search: Option<String>,+ #[arg(long, short = 'l', default_value_t = 5000)]+ limit: usize,+ },+ /// Search the web through Firecrawl cloud.+ Search {+ query: String,+ #[arg(long, short = 'l', default_value_t = 5)]+ limit: u32,+ },+ /// Find Alexandria data providers on Firecrawl cloud.+ FindTools {+ /// What data you need, in plain words.+ query: String,+ #[arg(long, short = 'l', default_value_t = 10)]+ limit: u32,+ },+ /// Run an Alexandria data provider on Firecrawl cloud.+ Data {+ provider: String,+ capability: String,+ /// The capability's options as JSON.+ #[arg(long, default_value = "{}")]+ options: String,+ },+ /// Sign in to Firecrawl in the browser. One step: the key is saved for you.+ Login {+ /// Print the link instead of opening the browser.+ #[arg(long)]+ no_browser: bool,+ },+ /// Run as an MCP server over stdio, for AI agents.+ Mcp {+ /// Refuse localhost and private-network targets.+ #[arg(long)]+ public_only: bool,+ },+ /// Add Firecrawl to the AI coding tools on this machine.+ Init {+ /// Only set up this tool. Repeatable. Default: every tool found.+ #[arg(long = "agent")]+ agents: Vec<String>,+ /// Show what would change without writing anything.+ #[arg(long)]+ dry_run: bool,+ },+ /// Serve the Firecrawl HTTP API (scrape, crawl, map) on one port.+ Serve {+ #[arg(long, default_value = "127.0.0.1")]+ host: String,+ #[arg(long, short = 'p', default_value_t = 3002)]+ port: u16,+ /// Bearer token clients must send. Required when --host is not loopback.+ #[arg(long, env = "FIRECRAWL_LOCAL_TOKEN")]+ token: Option<String>,+ /// Extra hostname accepted in the Host header. Repeatable.+ #[arg(long = "allowed-host")]+ allowed_hosts: Vec<String>,+ /// Let API callers scrape localhost and private-network targets.+ #[arg(long, env = "FIRECRAWL_LOCAL_ALLOW_PRIVATE")]+ allow_private: bool,+ /// Override the User-Agent header.+ #[arg(long, env = "FIRECRAWL_LOCAL_USER_AGENT")]+ user_agent: Option<String>,+ },+}++/// Run the Firecrawl MCP server against this engine.+///+/// Firecrawl already maintains one MCP server, the `firecrawl-mcp` package,+/// and it can talk to any Firecrawl API address. So this starts the engine's+/// API on a loopback port the system picks, then runs that server pointed at+/// it, with this process's stdin and stdout handed straight through. Scrape,+/// crawl and map are answered here; everything else the server offers is+/// passed on to Firecrawl cloud.+async fn launch_mcp(engine: Arc<Engine>) -> Result<(), String> {+ let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))+ .await+ .map_err(|e| format!("could not open a local port for Firecrawl Local: {e}"))?;+ let addr = listener+ .local_addr()+ .map_err(|e| format!("could not read the local port: {e}"))?;+ let cfg = ServeConfig {+ addr,+ token: None,+ allowed_hosts: Vec::new(),+ cloud: Cloud::from_env().with_surface(firecrawl_local::cloud::Surface::Mcp, None),+ };+ let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();+ let server = tokio::spawn(serve::serve_on(listener, engine, cfg, async move {+ let _ = stop_rx.await;+ }));++ let package = std::env::var("FIRECRAWL_MCP_PACKAGE")+ .ok()+ .filter(|p| !p.trim().is_empty())+ .unwrap_or_else(|| "firecrawl-mcp".to_string());+ let npx = if cfg!(windows) { "npx.cmd" } else { "npx" };+ let child = tokio::process::Command::new(npx)+ .args(["-y", &package])+ .env("FIRECRAWL_API_URL", format!("http://{addr}"))+ .stdin(std::process::Stdio::inherit())+ .stdout(std::process::Stdio::inherit())+ .stderr(std::process::Stdio::inherit())+ .spawn();+ let outcome = match child {+ Ok(mut child) => match child.wait().await {+ Ok(status) if status.success() => Ok(()),+ Ok(status) => Err(format!("the Firecrawl MCP server stopped ({status})")),+ Err(e) => Err(format!(+ "the Firecrawl MCP server could not be watched: {e}"+ )),+ },+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(+ "The MCP server needs Node.js 22 or newer, and `npx` was not found on this machine. \+ Install Node.js from https://nodejs.org and try again."+ .to_string(),+ ),+ Err(e) => Err(format!("could not start the Firecrawl MCP server: {e}")),+ };+ let _ = stop_tx.send(());+ let _ = server.await;+ outcome+}++fn main() -> ExitCode {+ let cli = Cli::parse();+ // Parsing is CPU-bound, so more blocking threads than cores only adds+ // memory: each thread keeps its own allocator heap.+ let cores = std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(4);+ let runtime = match tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .max_blocking_threads(cores.max(4))+ .build()+ {+ Ok(rt) => rt,+ Err(e) => {+ eprintln!("could not start the async runtime: {e}");+ return ExitCode::FAILURE;+ }+ };+ runtime.block_on(run(cli))+}++/// Engine for a person or agent on their own machine: private targets are+/// reachable unless they ask otherwise.+fn local_engine(public_only: bool, browser: BrowserMode) -> Option<Arc<Engine>> {+ let policy = if public_only {+ NetworkPolicy::PublicOnly+ } else {+ NetworkPolicy::AllowPrivate+ };+ match Engine::new(EngineConfig {+ policy,+ user_agent: std::env::var("FIRECRAWL_LOCAL_USER_AGENT").ok(),+ browser,+ }) {+ Ok(e) => Some(Arc::new(e)),+ Err(e) => {+ eprintln!("{e}");+ None+ }+ }+}++async fn run(cli: Cli) -> ExitCode {+ let browser = match cli.browser.to_ascii_lowercase().as_str() {+ "auto" => BrowserMode::Auto,+ "never" | "off" => BrowserMode::Never,+ "always" => BrowserMode::Always,+ other => {+ eprintln!("--browser must be auto, never or always, got '{other}'");+ return ExitCode::FAILURE;+ }+ };+ match cli.command {+ Command::Scrape {+ urls,+ format,+ full_page,+ markdown,+ timeout,+ concurrency,+ public_only,+ user_agent,+ } => {+ // A person at their own terminal may scrape their own localhost.+ let policy = if public_only {+ NetworkPolicy::PublicOnly+ } else {+ NetworkPolicy::AllowPrivate+ };+ let engine = match Engine::new(EngineConfig {+ policy,+ user_agent,+ browser,+ }) {+ Ok(e) => Arc::new(e),+ Err(e) => {+ eprintln!("{e}");+ return ExitCode::FAILURE;+ }+ };+ let opts = ScrapeOptions {+ formats: format.into_iter().map(Value::String).collect(),+ only_main_content: !full_page,+ timeout: Some(timeout),+ ..Default::default()+ };++ let single = urls.len() == 1;+ let cloud = Cloud::from_env();+ let results: Vec<Value> = stream::iter(urls.into_iter().map(|url| {+ let engine = engine.clone();+ let opts = opts.clone();+ let cloud = cloud.clone();+ async move {+ match firecrawl_local::cloud::scrape_with_fallback(&engine, &cloud, &url, &opts).await {+ Ok(data) => json!({ "success": true, "data": data }),+ Err(e) => json!({ "success": false, "url": url, "code": e.code, "error": e.message }),+ }+ }+ }))+ .buffered(concurrency.max(1))+ .collect()+ .await;+ engine.shutdown().await;++ let failed = results+ .iter()+ .any(|r| r.get("success").and_then(Value::as_bool) != Some(true));++ if markdown {+ for r in &results {+ match r.pointer("/data/markdown").and_then(Value::as_str) {+ Some(md) => println!("{md}\n"),+ None => eprintln!(+ "{}",+ r.get("error")+ .and_then(Value::as_str)+ .unwrap_or("no markdown produced")+ ),+ }+ }+ } else if single {+ println!("{}", results.into_iter().next().unwrap_or(Value::Null));+ } else {+ println!("{}", Value::Array(results));+ }++ if failed {+ ExitCode::FAILURE+ } else {+ ExitCode::SUCCESS+ }+ }+ Command::Crawl {+ url,+ limit,+ max_depth,+ entire_domain,+ include_paths,+ exclude_paths,+ concurrency,+ out,+ ignore_robots,+ public_only,+ } => {+ let Some(engine) = local_engine(public_only, browser) else {+ return ExitCode::FAILURE;+ };+ let opts = CrawlOptions {+ limit,+ max_discovery_depth: max_depth,+ include_paths,+ exclude_paths,+ crawl_entire_domain: entire_domain,+ ignore_robots_txt: ignore_robots,+ max_concurrency: concurrency,+ ..Default::default()+ };+ let started = std::time::Instant::now();+ let crawled = crawl::crawl(engine.clone(), &url, &opts, |_| {}).await;+ engine.shutdown().await;+ match crawled {+ Ok(outcome) => {+ let secs = started.elapsed().as_secs_f64();+ match out {+ Some(dir) => match crawl::write_pages(&dir, &outcome.pages).await {+ Ok(index) => {+ eprintln!(+ "{} pages in {secs:.2}s ({:.1} pages/s), {} errors, {} skipped by robots.txt, saved to {}",+ outcome.pages.len(),+ outcome.pages.len() as f64 / secs.max(0.001),+ outcome.errors.len(),+ outcome.robots_blocked.len(),+ dir.display()+ );+ for line in index {+ println!("{line}");+ }+ }+ Err(e) => {+ eprintln!("{e}");+ return ExitCode::FAILURE;+ }+ },+ None => {+ println!(+ "{}",+ json!({+ "success": true,+ "status": "completed",+ "total": outcome.pages.len(),+ "completed": outcome.pages.len(),+ "creditsUsed": 0,+ "elapsedSeconds": secs,+ "errors": outcome.errors,+ "robotsBlocked": outcome.robots_blocked,+ "data": outcome.pages,+ })+ );+ }+ }+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("{}", e.message);+ ExitCode::FAILURE+ }+ }+ }+ Command::Map { url, search, limit } => {+ let Some(engine) = local_engine(false, browser) else {+ return ExitCode::FAILURE;+ };+ let opts = MapOptions {+ search,+ limit,+ ..Default::default()+ };+ let mapped = crawl::map(engine.clone(), &url, &opts).await;+ engine.shutdown().await;+ match mapped {+ Ok(links) => {+ println!("{}", json!({ "success": true, "links": links }));+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("{}", e.message);+ ExitCode::FAILURE+ }+ }+ }+ Command::Search { query, limit } => match Cloud::from_env().search(&query, limit).await {+ Ok(payload) => {+ println!("{payload}");+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("{}", e.message);+ ExitCode::FAILURE+ }+ },+ Command::FindTools { query, limit } => {+ match Cloud::from_env()+ .find_tools(&json!({ "query": query, "limit": limit }))+ .await+ {+ Ok(payload) => {+ println!("{payload}");+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("{}", e.message);+ ExitCode::FAILURE+ }+ }+ }+ Command::Data {+ provider,+ capability,+ options,+ } => {+ let options: Value = match serde_json::from_str(&options) {+ Ok(v) => v,+ Err(e) => {+ eprintln!("--options is not valid JSON: {e}");+ return ExitCode::FAILURE;+ }+ };+ let call = json!({ "provider": provider, "capability": capability, "options": options });+ match Cloud::from_env().alexandria(&call).await {+ Ok(payload) => {+ println!("{payload}");+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("{}", e.message);+ ExitCode::FAILURE+ }+ }+ }+ Command::Login { no_browser } => {+ let session = match LoginSession::start() {+ Ok(s) => s,+ Err(e) => {+ eprintln!("{e}");+ return ExitCode::FAILURE;+ }+ };+ println!("Sign in to Firecrawl:");+ println!(" {}", session.url);+ if !no_browser && login::open_browser(&session.url) {+ println!("Your browser should open now. Waiting for you to finish...");+ } else {+ println!("Open that link in your browser. Waiting for you to finish...");+ }+ match session.wait(std::time::Duration::from_secs(600)).await {+ Ok(signed_in) => match login::save(&signed_in) {+ Ok(path) => {+ match &signed_in.team_name {+ Some(team) => println!("Signed in to {team}. Login saved to {}.", path.display()),+ None => println!("Signed in. Login saved to {}.", path.display()),+ }+ ExitCode::SUCCESS+ }+ Err(e) => {+ eprintln!("Signed in, but could not save the login: {e}");+ ExitCode::FAILURE+ }+ },+ Err(e) => {+ eprintln!("{e}");+ ExitCode::FAILURE+ }+ }+ }+ Command::Mcp { public_only } => {+ let Some(engine) = local_engine(public_only, browser) else {+ return ExitCode::FAILURE;+ };+ tokio::spawn(engine.clone().trim_memory_when_idle());+ let served = launch_mcp(engine.clone()).await;+ engine.shutdown().await;+ match served {+ Ok(()) => ExitCode::SUCCESS,+ Err(e) => {+ eprintln!("{e}");+ ExitCode::FAILURE+ }+ }+ }+ Command::Init { agents, dry_run } => {+ let known = init::target_names();+ if let Some(bad) = agents.iter().find(|a| !known.contains(&a.as_str())) {+ eprintln!("Unknown tool '{bad}'. Known: {}.", known.join(", "));+ return ExitCode::FAILURE;+ }+ let Some(home) = std::env::var_os("HOME").map(std::path::PathBuf::from) else {+ eprintln!("HOME is not set.");+ return ExitCode::FAILURE;+ };+ let binary = match std::env::current_exe() {+ Ok(p) => p,+ Err(e) => {+ eprintln!("could not find this binary's path: {e}");+ return ExitCode::FAILURE;+ }+ };+ let report = init::run(&InitOptions {+ only: agents,+ dry_run,+ home,+ binary: binary.clone(),+ });+ if report.lines.is_empty() {+ println!("No supported AI tool found. Known: {}.", known.join(", "));+ } else {+ println!("Firecrawl MCP server:");+ for line in &report.lines {+ println!("{line}");+ }+ // The MCP server is the firecrawl-mcp package, run through npx.+ let has_npx = std::env::var_os("PATH")+ .map(|path| {+ std::env::split_paths(&path)+ .any(|dir| dir.join("npx").exists() || dir.join("npx.cmd").exists())+ })+ .unwrap_or(false);+ if !has_npx {+ println!(" Needs Node.js 22 or newer, and `npx` was not found. Install it from https://nodejs.org.");+ }+ }+ println!();+ println!("Use it from any app:");+ println!(+ " {} serve # Firecrawl API on http://127.0.0.1:3002",+ binary.display()+ );+ println!(" FIRECRAWL_API_URL=http://127.0.0.1:3002 # then your existing Firecrawl SDK code runs locally");+ println!();+ println!("What goes to Firecrawl cloud:");+ println!(" - your search queries (there is no search index on a laptop)");+ println!(+ " - the URL of a page that blocked this machine, so Firecrawl can fetch it for you"+ );+ println!(" - Alexandria data-provider calls, once you sign in");+ println!(" - a random install ID and the name of your AI tool, to count installs");+ println!(+ "Pages that load fine here are never sent. FIRECRAWL_CLOUD=search keeps only search;"+ );+ println!("FIRECRAWL_CLOUD=off turns the cloud off. FIRECRAWL_API_KEY uses your account.");+ ExitCode::SUCCESS+ }+ Command::Serve {+ host,+ port,+ token,+ allowed_hosts,+ allow_private,+ user_agent,+ } => {+ let ip: std::net::IpAddr = match host.parse() {+ Ok(ip) => ip,+ Err(_) if host == "localhost" => std::net::IpAddr::from([127, 0, 0, 1]),+ Err(_) => {+ eprintln!("--host must be an IP address, got '{host}'");+ return ExitCode::FAILURE;+ }+ };+ let policy = if allow_private {+ NetworkPolicy::AllowPrivate+ } else {+ NetworkPolicy::PublicOnly+ };+ let engine = match Engine::new(EngineConfig {+ policy,+ user_agent,+ browser,+ }) {+ Ok(e) => Arc::new(e),+ Err(e) => {+ eprintln!("{e}");+ return ExitCode::FAILURE;+ }+ };+ let cfg = ServeConfig {+ addr: SocketAddr::new(ip, port),+ token: token.filter(|t| !t.is_empty()),+ allowed_hosts,+ cloud: Cloud::from_env_opt_in(),+ };+ tokio::spawn(engine.clone().trim_memory_when_idle());+ let served = serve::serve(engine.clone(), cfg).await;+ engine.shutdown().await;+ match served {+ Ok(()) => ExitCode::SUCCESS,+ Err(e) => {+ eprintln!("{e}");+ ExitCode::FAILURE+ }+ }+ }+ }+}diff --git a/apps/local/src/markdown.rs b/apps/local/src/markdown.rsnew file mode 100644index 000000000..ad7af88bc--- /dev/null+++ b/apps/local/src/markdown.rs@@ -0,0 +1,947 @@+//! Cleaned HTML to markdown, written for what an LLM reads: every code block+//! fenced (with its language when the page says it), every data table kept as+//! a table, layout tables unwrapped, permalink anchors and inline data images+//! dropped, and no padding that only costs tokens.++use kuchikiki::{parse_html, traits::TendrilSink, NodeRef};++const SKIPPED: &[&str] = &[+ "script", "style", "noscript", "template", "svg", "head", "iframe", "canvas", "object", "embed",+ "select", "textarea", "input", "link", "meta", "title", "audio", "video", "source", "track",+ "map", "area", "math",+];++const BLOCKS: &[&str] = &[+ "address",+ "article",+ "aside",+ "blockquote",+ "body",+ "center",+ "dd",+ "details",+ "dialog",+ "div",+ "dl",+ "dt",+ "fieldset",+ "figcaption",+ "figure",+ "footer",+ "form",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "header",+ "hgroup",+ "hr",+ "html",+ "li",+ "main",+ "nav",+ "ol",+ "p",+ "pre",+ "section",+ "summary",+ "table",+ "tbody",+ "td",+ "tfoot",+ "th",+ "thead",+ "tr",+ "ul",+ "caption",+];++fn tag_of(node: &NodeRef) -> Option<&str> {+ node.as_element().map(|e| e.name.local.as_ref())+}++fn attr(node: &NodeRef, name: &str) -> Option<String> {+ node+ .as_element()+ .and_then(|e| e.attributes.borrow().get(name).map(str::to_string))+}++fn is_block(name: &str) -> bool {+ BLOCKS.contains(&name)+}++fn is_skipped(name: &str) -> bool {+ SKIPPED.contains(&name)+}++/// Collapse every run of whitespace to one space.+fn collapse_ws(text: &str, out: &mut String) {+ let mut last_space = out.ends_with(' ') || out.ends_with('\n');+ for ch in text.chars() {+ if ch.is_whitespace() {+ if !last_space {+ out.push(' ');+ last_space = true;+ }+ } else {+ out.push(ch);+ last_space = false;+ }+ }+}++/// Tidy one paragraph of inline output: no spaces around line breaks, no+/// doubled spaces, trimmed.+fn normalize_inline(text: &str) -> String {+ let mut out = String::with_capacity(text.len());+ for (i, line) in text.split('\n').enumerate() {+ if i > 0 {+ out.push('\n');+ }+ let mut last_space = true;+ for ch in line.trim().chars() {+ if ch == ' ' {+ if !last_space {+ out.push(' ');+ }+ last_space = true;+ } else {+ out.push(ch);+ last_space = false;+ }+ }+ }+ let trimmed = out.trim_matches(|c: char| c == '\n' || c == ' ');+ // Collapse blank lines left by stray <br> runs.+ let mut result = String::with_capacity(trimmed.len());+ let mut newline_run = 0;+ for ch in trimmed.chars() {+ if ch == '\n' {+ newline_run += 1;+ if newline_run > 1 {+ continue;+ }+ } else {+ newline_run = 0;+ }+ result.push(ch);+ }+ result+}++/// Stop plain text from being read as a heading, quote or list item.+fn escape_line_starts(paragraph: &str) -> String {+ let mut out = String::with_capacity(paragraph.len() + 4);+ for (i, line) in paragraph.split('\n').enumerate() {+ if i > 0 {+ out.push('\n');+ }+ let bytes = line.as_bytes();+ let needs = match bytes.first() {+ Some(b'#') => line.trim_start_matches('#').starts_with(' '),+ Some(b'>') => true,+ Some(b'-') | Some(b'+') => bytes.get(1) == Some(&b' '),+ Some(c) if c.is_ascii_digit() => {+ let digits = line.chars().take_while(|c| c.is_ascii_digit()).count();+ matches!(bytes.get(digits), Some(b'.') | Some(b')')) && bytes.get(digits + 1) == Some(&b' ')+ }+ _ => false,+ };+ if needs {+ out.push('\\');+ }+ out.push_str(line);+ }+ out+}++fn wrap(marker: &str, inner: &str) -> String {+ let trimmed = inner.trim();+ if trimmed.is_empty() {+ return inner.to_string();+ }+ let lead = if inner.starts_with(char::is_whitespace) {+ " "+ } else {+ ""+ };+ let trail = if inner.ends_with(char::is_whitespace) {+ " "+ } else {+ ""+ };+ format!("{lead}{marker}{trimmed}{marker}{trail}")+}++fn code_span(text: &str) -> String {+ let text = text.replace('\n', " ");+ let text = text.trim();+ if text.is_empty() {+ return String::new();+ }+ if text.contains('`') {+ format!("`` {text} ``")+ } else {+ format!("`{text}`")+ }+}++fn inline_children(node: &NodeRef) -> String {+ let mut out = String::new();+ for child in node.children() {+ inline(&child, &mut out);+ }+ out+}++fn is_permalink_text(text: &str) -> bool {+ matches!(text.trim(), "¶" | "#" | "§" | "🔗" | "" | "🔖" | "¤")+}++fn inline(node: &NodeRef, out: &mut String) {+ if let Some(text) = node.as_text() {+ collapse_ws(&text.borrow(), out);+ return;+ }+ let Some(name) = tag_of(node) else {+ // Documents, comments, doctypes.+ if node.as_document().is_some() {+ for child in node.children() {+ inline(&child, out);+ }+ }+ return;+ };+ if is_skipped(name) {+ return;+ }+ match name {+ "br" => out.push('\n'),+ "a" => {+ let text = normalize_inline(&inline_children(node)).replace('\n', " ");+ let href = attr(node, "href").unwrap_or_default();+ let href = href.trim();+ let has_image = text.contains(""));+ }+ }+ "img" => {+ let src = attr(node, "src").unwrap_or_default();+ let src = src.trim();+ if src.is_empty() || src.starts_with("data:") {+ return;+ }+ let alt = attr(node, "alt").unwrap_or_default();+ let alt = alt.replace(['\n', '[', ']'], " ");+ let src = src+ .replace(' ', "%20")+ .replace('(', "%28")+ .replace(')', "%29");+ out.push_str(&format!("", alt.trim()));+ }+ "strong" | "b" => out.push_str(&wrap("**", &inline_children(node))),+ "em" | "i" | "cite" | "dfn" | "var" => out.push_str(&wrap("*", &inline_children(node))),+ "del" | "s" | "strike" => out.push_str(&wrap("~~", &inline_children(node))),+ "code" | "kbd" | "samp" | "tt" => {+ let lead = out.ends_with(|c: char| c.is_alphanumeric());+ if lead {+ out.push(' ');+ }+ out.push_str(&code_span(&node.text_contents()));+ }+ "pre" => {+ out.push('\n');+ out.push_str(&code_block(node));+ out.push('\n');+ }+ _ if is_block(name) => {+ // A block inside inline content (a div inside a link, say).+ if !out.ends_with(' ') && !out.is_empty() {+ out.push(' ');+ }+ for child in node.children() {+ inline(&child, out);+ }+ if !out.ends_with(' ') {+ out.push(' ');+ }+ }+ _ => {+ for child in node.children() {+ inline(&child, out);+ }+ }+ }+}++fn clean_language(token: &str) -> Option<String> {+ let token = token.trim_matches(|c: char| !(c.is_ascii_alphanumeric() || c == '+' || c == '#'));+ if token.is_empty() || token.len() > 24 {+ return None;+ }+ let lower = token.to_ascii_lowercase();+ if matches!(+ lower.as_str(),+ "none"+ | "text"+ | "plain"+ | "plaintext"+ | "default"+ | "notranslate"+ | "highlight"+ | "code"+ | "wrap"+ | "nohighlight"+ ) {+ return None;+ }+ if lower+ .chars()+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '#' | '-' | '_'))+ {+ Some(lower)+ } else {+ None+ }+}++fn language_from_classes(classes: &str) -> Option<String> {+ for token in classes.split_whitespace() {+ for prefix in [+ "language-",+ "lang-",+ "highlight-source-",+ "highlight-",+ "brush:",+ "sourceCode-",+ "hljs-language-",+ ] {+ if let Some(rest) = token.strip_prefix(prefix) {+ if let Some(lang) = clean_language(rest) {+ return Some(lang);+ }+ }+ }+ }+ None+}++fn code_language(pre: &NodeRef) -> Option<String> {+ let mut candidates: Vec<NodeRef> = vec![pre.clone()];+ if let Some(code) = pre.children().find(|c| tag_of(c) == Some("code")) {+ candidates.push(code);+ }+ let mut up = pre.parent();+ for _ in 0..3 {+ match up {+ Some(p) => {+ candidates.push(p.clone());+ up = p.parent();+ }+ None => break,+ }+ }+ for node in candidates {+ for name in ["class", "data-lang", "data-language", "lang"] {+ if let Some(value) = attr(&node, name) {+ let found = if name == "class" {+ language_from_classes(&value)+ } else if name == "lang" {+ // `lang` on an ancestor is usually the page's human language.+ None+ } else {+ clean_language(&value)+ };+ if found.is_some() {+ return found;+ }+ }+ }+ }+ None+}++/// Text of a `<pre>`, keeping line breaks written as `<br>`.+fn pre_text(node: &NodeRef, out: &mut String) {+ for child in node.children() {+ if let Some(text) = child.as_text() {+ out.push_str(&text.borrow());+ } else if let Some(name) = tag_of(&child) {+ if name == "br" {+ out.push('\n');+ } else if !is_skipped(name) {+ pre_text(&child, out);+ }+ }+ }+}++fn code_block(pre: &NodeRef) -> String {+ let mut text = String::new();+ pre_text(pre, &mut text);+ let text = text.replace("\r\n", "\n");+ let text = text.trim_matches('\n').trim_end();+ if text.trim().is_empty() {+ return String::new();+ }+ let lang = code_language(pre).unwrap_or_default();+ let fence = if text.contains("```") { "````" } else { "```" };+ format!("{fence}{lang}\n{text}\n{fence}")+}++fn heading(node: &NodeRef, level: usize) -> String {+ let text = normalize_inline(&inline_children(node)).replace('\n', " ");+ if text.is_empty() {+ return String::new();+ }+ format!("{} {}", "#".repeat(level), text)+}++fn indent_continuation(text: &str, width: usize) -> String {+ let pad = " ".repeat(width);+ let mut out = String::with_capacity(text.len());+ for (i, line) in text.split('\n').enumerate() {+ if i > 0 {+ out.push('\n');+ if !line.is_empty() {+ out.push_str(&pad);+ }+ }+ out.push_str(line);+ }+ out+}++fn list(node: &NodeRef, ordered: bool) -> String {+ let mut number: usize = attr(node, "start")+ .and_then(|s| s.trim().parse().ok())+ .unwrap_or(1);+ let mut items: Vec<String> = Vec::new();+ for child in node.children() {+ let Some(name) = tag_of(&child) else { continue };+ match name {+ "li" => {+ let parts = blocks(&child);+ if parts.is_empty() {+ continue;+ }+ // Keep an item and its nested list tight; separate real paragraphs.+ let mut body = String::new();+ for (i, part) in parts.iter().enumerate() {+ if i > 0 {+ let nested_list =+ part.starts_with("- ") || part.starts_with(|c: char| c.is_ascii_digit());+ body.push_str(if nested_list { "\n" } else { "\n\n" });+ }+ body.push_str(part);+ }+ let marker = if ordered {+ let m = format!("{number}. ");+ number += 1;+ m+ } else {+ "- ".to_string()+ };+ let width = marker.len();+ items.push(format!("{marker}{}", indent_continuation(&body, width)));+ }+ "ul" | "ol" => {+ // A list nested directly in a list, without its own <li>.+ let nested = list(&child, name == "ol");+ if !nested.is_empty() {+ match items.last_mut() {+ Some(last) => {+ last.push('\n');+ last.push_str(&indent_continuation(&format!(" {nested}"), 2));+ }+ None => items.push(nested),+ }+ }+ }+ _ => {}+ }+ }+ items.join("\n")+}++fn quote(node: &NodeRef) -> String {+ let inner = blocks(node).join("\n\n");+ if inner.is_empty() {+ return String::new();+ }+ inner+ .split('\n')+ .map(|line| {+ if line.is_empty() {+ ">".to_string()+ } else {+ format!("> {line}")+ }+ })+ .collect::<Vec<_>>()+ .join("\n")+}++struct Row {+ cells: Vec<NodeRef>,+ header: bool,+}++fn collect_rows(node: &NodeRef, in_head: bool, rows: &mut Vec<Row>, caption: &mut Option<String>) {+ for child in node.children() {+ match tag_of(&child) {+ Some("thead") => collect_rows(&child, true, rows, caption),+ Some("tbody") | Some("tfoot") => collect_rows(&child, false, rows, caption),+ Some("caption") => {+ let text = normalize_inline(&inline_children(&child)).replace('\n', " ");+ if !text.is_empty() {+ *caption = Some(text);+ }+ }+ Some("tr") => {+ let cells: Vec<NodeRef> = child+ .children()+ .filter(|c| matches!(tag_of(c), Some("td") | Some("th")))+ .collect();+ if cells.is_empty() {+ continue;+ }+ let all_th = cells.iter().all(|c| tag_of(c) == Some("th"));+ rows.push(Row {+ cells,+ header: in_head || all_th,+ });+ }+ _ => {}+ }+ }+}++fn cell_span(cell: &NodeRef) -> usize {+ attr(cell, "colspan")+ .and_then(|s| s.trim().parse::<usize>().ok())+ .unwrap_or(1)+ .clamp(1, 50)+}++fn cell_text(cell: &NodeRef) -> String {+ let text = blocks(cell).join(" ");+ let text = text.replace('\n', " ").replace('|', "\\|");+ normalize_inline(&text)+}++fn has_nested_table(node: &NodeRef) -> bool {+ node+ .descendants()+ .skip(1)+ .any(|d| tag_of(&d) == Some("table"))+}++fn table(node: &NodeRef) -> String {+ let mut rows = Vec::new();+ let mut caption = None;+ collect_rows(node, false, &mut rows, &mut caption);+ if rows.is_empty() {+ return blocks(node).join("\n\n");+ }+ let columns = rows+ .iter()+ .map(|r| r.cells.iter().map(cell_span).sum::<usize>())+ .max()+ .unwrap_or(0);++ // One column, tables inside tables, or header-less tables holding whole+ // blocks of prose are page layout, not data.+ let has_header = rows.iter().any(|r| r.header)+ || rows+ .iter()+ .any(|r| r.cells.iter().any(|c| tag_of(c) == Some("th")));+ let presentational = matches!(+ attr(node, "role").as_deref(),+ Some("presentation") | Some("none")+ );+ let prose_cells = !has_header+ && rows.iter().any(|r| {+ r.cells.iter().any(|c| {+ c.descendants().skip(1).any(|d| {+ matches!(+ tag_of(&d),+ Some("p")+ | Some("ul")+ | Some("ol")+ | Some("pre")+ | Some("blockquote")+ | Some("h1")+ | Some("h2")+ | Some("h3")+ | Some("h4")+ )+ }) || c.text_contents().len() > 400+ })+ });+ if columns <= 1 || has_nested_table(node) || presentational || prose_cells {+ let mut parts = Vec::new();+ for row in &rows {+ for cell in &row.cells {+ let text = blocks(cell).join("\n\n");+ if !text.is_empty() {+ parts.push(text);+ }+ }+ }+ return parts.join("\n\n");+ }++ let render = |row: &Row| -> String {+ let mut line = String::from("|");+ let mut used = 0;+ for cell in &row.cells {+ let span = cell_span(cell);+ line.push(' ');+ line.push_str(&cell_text(cell));+ line.push_str(" |");+ for _ in 1..span {+ line.push_str(" |");+ }+ used += span;+ }+ for _ in used..columns {+ line.push_str(" |");+ }+ line+ };++ let mut out = String::new();+ if let Some(c) = caption {+ out.push_str(&format!("**{c}**\n\n"));+ }+ let mut body = rows.iter();+ match rows.first() {+ Some(first) if first.header => {+ out.push_str(&render(first));+ body.next();+ }+ _ => {+ out.push('|');+ for _ in 0..columns {+ out.push_str(" |");+ }+ }+ }+ out.push_str("\n|");+ for _ in 0..columns {+ out.push_str(" --- |");+ }+ for row in body {+ out.push('\n');+ out.push_str(&render(row));+ }+ out+}++fn render_block(node: &NodeRef, name: &str) -> String {+ match name {+ "h1" => heading(node, 1),+ "h2" => heading(node, 2),+ "h3" => heading(node, 3),+ "h4" => heading(node, 4),+ "h5" => heading(node, 5),+ "h6" => heading(node, 6),+ "pre" => code_block(node),+ "blockquote" => quote(node),+ "ul" => list(node, false),+ "ol" => list(node, true),+ "table" => table(node),+ "hr" => "---".to_string(),+ "summary" => {+ let text = normalize_inline(&inline_children(node)).replace('\n', " ");+ if text.is_empty() {+ String::new()+ } else {+ format!("**{text}**")+ }+ }+ _ => blocks(node).join("\n\n"),+ }+}++/// Render a container's children as a list of blocks.+fn blocks(parent: &NodeRef) -> Vec<String> {+ let mut out: Vec<String> = Vec::new();+ let mut run = String::new();+ let flush = |run: &mut String, out: &mut Vec<String>| {+ if !run.is_empty() {+ let paragraph = normalize_inline(run);+ if !paragraph.is_empty() {+ out.push(escape_line_starts(¶graph));+ }+ run.clear();+ }+ };+ for child in parent.children() {+ if child.as_text().is_some() {+ inline(&child, &mut run);+ continue;+ }+ let Some(name) = tag_of(&child) else {+ if child.as_document().is_some() {+ flush(&mut run, &mut out);+ out.extend(blocks(&child));+ }+ continue;+ };+ if is_skipped(name) {+ continue;+ }+ if is_block(name) {+ flush(&mut run, &mut out);+ let rendered = render_block(&child, name);+ if !rendered.is_empty() {+ out.push(rendered);+ }+ } else if contains_block(&child) {+ // An inline or custom element (a link card, <turbo-frame>, <my-app>)+ // wrapping real blocks: keep the blocks instead of flattening them.+ flush(&mut run, &mut out);+ let inner = blocks(&child);+ if name == "a" && inner.len() == 1 && !inner[0].contains('\n') {+ let href = attr(&child, "href").unwrap_or_default();+ let href = href.trim();+ if !href.is_empty() && !href.starts_with('#') && !inner[0].contains("](") {+ let href = href+ .replace(' ', "%20")+ .replace('(', "%28")+ .replace(')', "%29");+ let (marks, text) = match inner[0].find(|c: char| c != '#' && c != ' ') {+ Some(i) if inner[0].starts_with('#') => inner[0].split_at(i),+ _ => ("", inner[0].as_str()),+ };+ out.push(format!("{marks}[{text}]({href})"));+ continue;+ }+ }+ out.extend(inner);+ } else {+ inline(&child, &mut run);+ }+ }+ flush(&mut run, &mut out);+ out+}++/// True when an element that is not itself a block has a block somewhere+/// inside it.+fn contains_block(node: &NodeRef) -> bool {+ node.children().any(|child| match tag_of(&child) {+ Some(name) if is_skipped(name) => false,+ Some(name) if is_block(name) => true,+ Some(_) => contains_block(&child),+ None => false,+ })+}++/// Collapse runs of blank lines and trailing spaces, leaving fenced code+/// untouched.+fn tidy(md: &str) -> String {+ let mut out = String::with_capacity(md.len());+ let mut blank_run = 0usize;+ let mut fence: Option<&str> = None;+ for line in md.lines() {+ let trimmed_start = line.trim_start();+ match fence {+ Some(open) => {+ if trimmed_start.starts_with(open) && trimmed_start.trim_end().len() == open.len() {+ fence = None;+ }+ out.push_str(line);+ out.push('\n');+ blank_run = 0;+ continue;+ }+ None => {+ if trimmed_start.starts_with("````") {+ fence = Some("````");+ } else if trimmed_start.starts_with("```") {+ fence = Some("```");+ }+ if fence.is_some() {+ out.push_str(line);+ out.push('\n');+ blank_run = 0;+ continue;+ }+ }+ }+ let line = line.trim_end();+ if line.is_empty() {+ blank_run += 1;+ if blank_run > 1 {+ continue;+ }+ } else {+ blank_run = 0;+ }+ out.push_str(line);+ out.push('\n');+ }+ out.trim().to_string()+}++pub fn html_to_markdown(html: &str) -> String {+ document_to_markdown(&parse_html().one(html))+}++/// Markdown for an already-parsed document.+pub fn document_to_markdown(document: &NodeRef) -> String {+ tidy(&blocks(document).join("\n\n"))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn md(html: &str) -> String {+ html_to_markdown(html)+ }++ #[test]+ fn headings_paragraphs_inline() {+ let out = md("<h1>Title</h1><p>Hello <strong>bold</strong>, <em>em</em>, <code>x = 1</code> and <a href=\"https://x.dev/a\">a link</a>.</p><h2>Sub <a class=\"headerlink\" href=\"#sub\">¶</a></h2>");+ assert_eq!(+ out,+ "# Title\n\nHello **bold**, *em*, `x = 1` and [a link](https://x.dev/a).\n\n## Sub"+ );+ }++ #[test]+ fn bare_pre_is_fenced_with_language_from_wrapper() {+ let out = md("<div class=\"highlight-python3 notranslate\"><div class=\"highlight\"><pre><span></span><span class=\"c1\"># Expected output:</span>\n<span class=\"k\">def</span> main():\n return 1\n</pre></div></div>");+ assert_eq!(+ out,+ "```python3\n# Expected output:\ndef main():\n return 1\n```"+ );+ }++ #[test]+ fn pre_code_language_and_blank_lines_kept() {+ let out = md(+ "<pre><code class=\"language-rust\">fn main() {\n\n\n println!(\"hi\");\n}\n</code></pre>",+ );+ assert_eq!(+ out,+ "```rust\nfn main() {\n\n\n println!(\"hi\");\n}\n```"+ );+ }++ #[test]+ fn data_table_with_and_without_header() {+ let with = md("<table><thead><tr><th>Plan</th><th>Price</th></tr></thead><tbody><tr><td>Pro</td><td>$20 | month</td></tr></tbody></table>");+ assert_eq!(+ with,+ "| Plan | Price |\n| --- | --- |\n| Pro | $20 \\| month |"+ );+ let without = md("<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>");+ assert_eq!(without, "| | |\n| --- | --- |\n| a | b |\n| c | d |");+ }++ #[test]+ fn colspan_and_ragged_rows_stay_aligned() {+ let out = md("<table><tr><th>A</th><th>B</th><th>C</th></tr><tr><td colspan=\"2\">wide</td><td>x</td></tr><tr><td>only</td></tr></table>");+ assert_eq!(+ out,+ "| A | B | C |\n| --- | --- | --- |\n| wide | | x |\n| only | | |"+ );+ }++ #[test]+ fn layout_tables_are_unwrapped() {+ let out = md("<table><tr><td><p>First block.</p><p>Second block.</p></td></tr></table>");+ assert_eq!(out, "First block.\n\nSecond block.");+ }++ #[test]+ fn header_less_tables_with_prose_are_layout() {+ let out = md("<table><tr><td><img src=\"/s.gif\"></td><td><p>A long comment that is really a paragraph.</p><p>And another.</p></td></tr></table>");+ assert_eq!(+ out,+ "\n\nA long comment that is really a paragraph.\n\nAnd another."+ );+ let out = md("<table role=\"presentation\"><tr><td>a</td><td>b</td></tr></table>");+ assert_eq!(out, "a\n\nb");+ }++ #[test]+ fn custom_and_inline_wrappers_keep_their_blocks() {+ let out = md("<turbo-frame><span><h2>Why Rust?</h2><ul><li>Fast</li><li>Safe</li></ul><p>Text.</p></span></turbo-frame>");+ assert_eq!(out, "## Why Rust?\n\n- Fast\n- Safe\n\nText.");+ let out = md("<a href=\"/post\"><div><h3>Card title</h3></div></a><a href=\"/two\"><div><h3>Title</h3><p>Summary</p></div></a>");+ assert_eq!(out, "### [Card title](/post)\n\n### Title\n\nSummary");+ }++ #[test]+ fn nested_and_ordered_lists() {+ let out = md("<ul><li>one<ul><li>nested a</li><li>nested b</li></ul></li><li>two</li></ul><ol start=\"3\"><li>third</li><li>fourth</li></ol>");+ assert_eq!(+ out,+ "- one\n - nested a\n - nested b\n- two\n\n3. third\n4. fourth"+ );+ }++ #[test]+ fn list_item_with_paragraphs_and_code() {+ let out = md("<ol><li><p>Run it:</p><pre><code>npm start</code></pre></li><li>Done</li></ol>");+ assert_eq!(out, "1. Run it:\n\n ```\n npm start\n ```\n2. Done");+ }++ #[test]+ fn blockquote_images_and_skipped_elements() {+ let out = md("<blockquote><p>Quoted</p><p>Twice</p></blockquote><p><img src=\"/a.png\" alt=\"An image\"><img src=\"data:image/png;base64,AAAA\" alt=\"inline\"></p><script>var x;</script><style>p{}</style>");+ assert_eq!(out, "> Quoted\n>\n> Twice\n\n");+ }++ #[test]+ fn plain_text_that_looks_like_markdown_is_escaped() {+ let out =+ md("<p># not a heading</p><p>- not a list</p><p>1. not ordered</p><p>#hashtag stays</p>");+ assert_eq!(+ out,+ "\\# not a heading\n\n\\- not a list\n\n\\1. not ordered\n\n#hashtag stays"+ );+ }++ #[test]+ fn links_without_targets_and_with_spaces() {+ let out = md(+ "<p><a href=\"javascript:void(0)\">click</a> <a>bare</a> <a href=\"/a b(1)\">spaced</a></p>",+ );+ assert_eq!(out, "click bare [spaced](/a%20b%281%29)");+ }++ #[test]+ fn whitespace_and_breaks() {+ let out = md("<p> many spaces\n and\tnewlines <br> next line </p><div>inline <span>text</span> in a div</div>");+ assert_eq!(+ out,+ "many spaces and newlines\nnext line\n\ninline text in a div"+ );+ }++ #[test]+ fn tidy_keeps_code_blank_lines() {+ let out = tidy("a\n\n\n\nb\n```\nx\n\n\n\ny\n```\n");+ assert_eq!(out, "a\n\nb\n```\nx\n\n\n\ny\n```");+ }+}diff --git a/apps/local/src/mem.rs b/apps/local/src/mem.rsnew file mode 100644index 000000000..e26454e97--- /dev/null+++ b/apps/local/src/mem.rs@@ -0,0 +1,13 @@+//! Giving memory back.+//!+//! Parsing big pages allocates a lot for a moment. The allocator keeps that+//! memory for reuse, which makes an idle engine look far larger than it is.+//! This asks it to return what it is not using.++/// Return free heap memory to the operating system. Safe to call at any+/// time.+pub fn release_free_memory() {+ unsafe {+ libmimalloc_sys::mi_collect(true);+ }+}diff --git a/apps/local/src/netguard.rs b/apps/local/src/netguard.rsnew file mode 100644index 000000000..f7499a3c4--- /dev/null+++ b/apps/local/src/netguard.rs@@ -0,0 +1,385 @@+//! A loopback SOCKS5 proxy that the local Chrome is forced through.+//!+//! The plain fetcher resolves and checks every address itself. A browser does+//! its own networking, so checking only the URL we navigate to would miss+//! redirects, subresources, scripts' own requests and WebSockets. Routing+//! Chrome through this proxy puts every connection it makes through the same+//! network policy, applied to the address that is actually dialled.++use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};+use std::time::Duration;++use tokio::io::{AsyncReadExt, AsyncWriteExt};+use tokio::net::{TcpListener, TcpStream};+use url::Url;++use crate::guard::{self, NetworkPolicy};++const REPLY_OK: u8 = 0;+const REPLY_FAILURE: u8 = 1;+const REPLY_NOT_ALLOWED: u8 = 2;+const REPLY_HOST_UNREACHABLE: u8 = 4;+const REPLY_COMMAND_NOT_SUPPORTED: u8 = 7;+const REPLY_ADDRESS_NOT_SUPPORTED: u8 = 8;++pub struct GuardProxy {+ addr: SocketAddr,+ task: tokio::task::JoinHandle<()>,+}++impl GuardProxy {+ /// Listen on a free loopback port and start serving.+ pub async fn start(policy: NetworkPolicy) -> std::io::Result<Self> {+ let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;+ let addr = listener.local_addr()?;+ let task = tokio::spawn(async move {+ loop {+ // A failed accept (too many open files, say) is not a reason to+ // stop: Chrome would lose its only way out for the rest of the run.+ let Ok((client, _)) = listener.accept().await else {+ tokio::time::sleep(Duration::from_millis(50)).await;+ continue;+ };+ tokio::spawn(async move {+ let _ = serve(client, policy).await;+ });+ }+ });+ Ok(Self { addr, task })+ }++ pub fn addr(&self) -> SocketAddr {+ self.addr+ }++ /// The Chrome switches that send every connection through this proxy:+ /// loopback included, no QUIC or other UDP path around it, and no DNS+ /// lookups of its own.+ pub fn chrome_args(&self) -> Vec<String> {+ vec![+ format!("proxy-server=socks5://{}", self.addr),+ "proxy-bypass-list=<-loopback>".to_string(),+ "disable-quic".to_string(),+ "force-webrtc-ip-handling-policy=disable_non_proxied_udp".to_string(),+ format!(+ "host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE {}",+ self.addr.ip()+ ),+ ]+ }+}++impl Drop for GuardProxy {+ fn drop(&mut self) {+ self.task.abort();+ }+}++async fn reply(client: &mut TcpStream, code: u8) -> std::io::Result<()> {+ client.write_all(&[5, code, 0, 1, 0, 0, 0, 0, 0, 0]).await+}++/// Why a target is refused: the SOCKS reply code and, when the network+/// policy is the cause, the guard's own explanation.+struct Refused {+ code: u8,+ reason: Option<String>,+}++impl Refused {+ fn policy(reason: String) -> Self {+ Self {+ code: REPLY_NOT_ALLOWED,+ reason: Some(reason),+ }+ }+}++/// The addresses a SOCKS request may connect to, or why it is refused.+async fn allowed_targets(+ host: Target,+ port: u16,+ policy: NetworkPolicy,+) -> Result<Vec<SocketAddr>, Refused> {+ match host {+ Target::Ip(ip) => {+ guard::check_ip(ip, policy).map_err(|e| Refused::policy(e.0))?;+ Ok(vec![SocketAddr::new(ip, port)])+ }+ Target::Name(name) => {+ // Names the guard treats as local ("localhost", "*.internal") are+ // refused before any lookup, the same as for a URL.+ let as_url = Url::parse(&format!("http://{name}:{port}/")).map_err(|_| {+ Refused::policy(format!("'{name}' is not a host Firecrawl Local may reach"))+ })?;+ guard::check_url(&as_url, policy).map_err(|e| Refused::policy(e.0))?;+ let resolved: Vec<SocketAddr> = tokio::net::lookup_host((name.as_str(), port))+ .await+ .map_err(|_| Refused {+ code: REPLY_HOST_UNREACHABLE,+ reason: None,+ })?+ .collect();+ // The reason names the kind of address, never the address itself:+ // what an internal name resolves to is not the caller's to learn.+ let mut never_reachable = false;+ let mut allowed = Vec::new();+ let any = !resolved.is_empty();+ for addr in resolved {+ match guard::check_ip(addr.ip(), policy) {+ Ok(()) => allowed.push(addr),+ Err(_) => never_reachable |= guard::ip_always_blocked(addr.ip()),+ }+ }+ if allowed.is_empty() {+ return Err(Refused::policy(name_refusal(&name, any, never_reachable)));+ }+ Ok(allowed)+ }+ }+}++/// The reason a hostname is refused. It names the kind of address the name+/// resolved to and never the address.+fn name_refusal(name: &str, resolved_any: bool, never_reachable: bool) -> String {+ if !resolved_any {+ format!("'{name}' does not resolve to any address")+ } else if never_reachable {+ format!("'{name}' resolves to a link-local or metadata address, which is never reachable")+ } else {+ format!("'{name}' resolves to a private address; private targets are off")+ }+}++enum Target {+ Ip(IpAddr),+ Name(String),+}++/// Why the proxy would refuse a URL's host, if the network policy is the+/// cause. Chrome reports a refusal only as a failed proxy connection, so the+/// browser asks here to tell "this target is off limits" from an ordinary+/// connection failure.+pub async fn refusal_reason(url: &Url, policy: NetworkPolicy) -> Option<String> {+ let host = url.host_str()?;+ let bare = host.trim_start_matches('[').trim_end_matches(']');+ let target = match bare.parse::<IpAddr>() {+ Ok(ip) => Target::Ip(ip),+ Err(_) => Target::Name(bare.to_string()),+ };+ allowed_targets(target, url.port_or_known_default().unwrap_or(80), policy)+ .await+ .err()+ .and_then(|refused| refused.reason)+}++async fn serve(mut client: TcpStream, policy: NetworkPolicy) -> std::io::Result<()> {+ // Greeting: version, then the auth methods the client offers. No auth is+ // needed on a loopback listener only this process's Chrome is told about.+ let mut head = [0u8; 2];+ client.read_exact(&mut head).await?;+ if head[0] != 5 {+ return Ok(());+ }+ let mut methods = vec![0u8; head[1] as usize];+ client.read_exact(&mut methods).await?;+ client.write_all(&[5, 0]).await?;++ // Request: version, command, reserved, address type.+ let mut request = [0u8; 4];+ client.read_exact(&mut request).await?;+ if request[0] != 5 || request[1] != 1 {+ return reply(&mut client, REPLY_COMMAND_NOT_SUPPORTED).await;+ }+ let host = match request[3] {+ 1 => {+ let mut raw = [0u8; 4];+ client.read_exact(&mut raw).await?;+ Target::Ip(IpAddr::V4(Ipv4Addr::from(raw)))+ }+ 4 => {+ let mut raw = [0u8; 16];+ client.read_exact(&mut raw).await?;+ Target::Ip(IpAddr::V6(Ipv6Addr::from(raw)))+ }+ 3 => {+ let mut len = [0u8; 1];+ client.read_exact(&mut len).await?;+ let mut raw = vec![0u8; len[0] as usize];+ client.read_exact(&mut raw).await?;+ let name = String::from_utf8_lossy(&raw).to_string();+ // Chrome sends IP literals as names when a page uses one.+ match name+ .trim_start_matches('[')+ .trim_end_matches(']')+ .parse::<IpAddr>()+ {+ Ok(ip) => Target::Ip(ip),+ Err(_) => Target::Name(name),+ }+ }+ _ => return reply(&mut client, REPLY_ADDRESS_NOT_SUPPORTED).await,+ };+ let mut port = [0u8; 2];+ client.read_exact(&mut port).await?;+ let port = u16::from_be_bytes(port);++ let targets = match allowed_targets(host, port, policy).await {+ Ok(t) => t,+ Err(refused) => return reply(&mut client, refused.code).await,+ };+ let mut upstream = None;+ for target in targets {+ if let Ok(Ok(stream)) =+ tokio::time::timeout(Duration::from_secs(15), TcpStream::connect(target)).await+ {+ upstream = Some(stream);+ break;+ }+ }+ let Some(mut upstream) = upstream else {+ return reply(&mut client, REPLY_FAILURE).await;+ };+ reply(&mut client, REPLY_OK).await?;+ let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Speak just enough SOCKS5 to ask the proxy for one connection, and+ /// return its reply code and the stream.+ async fn ask(proxy: SocketAddr, address: &[u8], port: u16) -> (u8, TcpStream) {+ let mut stream = TcpStream::connect(proxy).await.unwrap();+ stream.write_all(&[5, 1, 0]).await.unwrap();+ let mut chosen = [0u8; 2];+ stream.read_exact(&mut chosen).await.unwrap();+ assert_eq!(chosen, [5, 0]);+ let mut request = vec![5, 1, 0];+ request.extend_from_slice(address);+ request.extend_from_slice(&port.to_be_bytes());+ stream.write_all(&request).await.unwrap();+ let mut answer = [0u8; 10];+ stream.read_exact(&mut answer).await.unwrap();+ (answer[1], stream)+ }++ fn name(host: &str) -> Vec<u8> {+ let mut v = vec![3, host.len() as u8];+ v.extend_from_slice(host.as_bytes());+ v+ }++ async fn echo_server() -> u16 {+ let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();+ let port = listener.local_addr().unwrap().port();+ tokio::spawn(async move {+ while let Ok((mut s, _)) = listener.accept().await {+ tokio::spawn(async move {+ let mut buf = [0u8; 5];+ if s.read_exact(&mut buf).await.is_ok() {+ let _ = s.write_all(&buf).await;+ }+ });+ }+ });+ port+ }++ #[tokio::test]+ async fn public_only_refuses_loopback_private_and_local_names() {+ let port = echo_server().await;+ let proxy = GuardProxy::start(NetworkPolicy::PublicOnly).await.unwrap();+ for address in [+ vec![1, 127, 0, 0, 1],+ vec![1, 10, 0, 0, 5],+ vec![1, 192, 168, 1, 1],+ vec![1, 169, 254, 169, 254],+ name("localhost"),+ name("127.0.0.1"),+ name("[::1]"),+ name("metadata.google.internal"),+ ] {+ let (code, _) = ask(proxy.addr(), &address, port).await;+ assert_eq!(code, REPLY_NOT_ALLOWED, "{address:?} should be refused");+ }+ }++ #[tokio::test]+ async fn allow_private_reaches_loopback_but_never_metadata() {+ let port = echo_server().await;+ let proxy = GuardProxy::start(NetworkPolicy::AllowPrivate)+ .await+ .unwrap();++ let (code, mut stream) = ask(proxy.addr(), &[1, 127, 0, 0, 1], port).await;+ assert_eq!(code, REPLY_OK);+ stream.write_all(b"hello").await.unwrap();+ let mut back = [0u8; 5];+ stream.read_exact(&mut back).await.unwrap();+ assert_eq!(&back, b"hello");++ let (code, _) = ask(proxy.addr(), &[1, 169, 254, 169, 254], 80).await;+ assert_eq!(code, REPLY_NOT_ALLOWED);+ }++ #[tokio::test]+ async fn refusals_are_explained() {+ let local = Url::parse("http://127.0.0.1:8080/").unwrap();+ assert!(refusal_reason(&local, NetworkPolicy::PublicOnly)+ .await+ .is_some_and(|m| m.contains("private")));+ assert!(refusal_reason(&local, NetworkPolicy::AllowPrivate)+ .await+ .is_none());+ // With private targets on, the reason is the metadata rule, not+ // "private targets are off".+ let metadata = Url::parse("http://169.254.169.254/latest/").unwrap();+ let why = refusal_reason(&metadata, NetworkPolicy::AllowPrivate)+ .await+ .unwrap();+ assert!(+ why.contains("never reachable") && !why.contains("are off"),+ "{why}"+ );+ }++ #[test]+ fn a_named_refusal_does_not_reveal_the_address() {+ let private = name_refusal("intranet.example", true, false);+ assert!(+ private.contains("private") && private.contains("intranet.example"),+ "{private}"+ );+ let metadata = name_refusal("meta.example", true, true);+ assert!(metadata.contains("never reachable"), "{metadata}");+ for message in [+ private,+ metadata,+ name_refusal("gone.example", false, false),+ ] {+ assert!(!message.chars().any(|c| c.is_ascii_digit()), "{message}");+ }+ }++ #[tokio::test]+ async fn only_connect_is_supported() {+ let proxy = GuardProxy::start(NetworkPolicy::AllowPrivate)+ .await+ .unwrap();+ let mut stream = TcpStream::connect(proxy.addr()).await.unwrap();+ stream.write_all(&[5, 1, 0]).await.unwrap();+ let mut chosen = [0u8; 2];+ stream.read_exact(&mut chosen).await.unwrap();+ // Command 3 is UDP associate.+ stream+ .write_all(&[5, 3, 0, 1, 127, 0, 0, 1, 0, 53])+ .await+ .unwrap();+ let mut answer = [0u8; 10];+ stream.read_exact(&mut answer).await.unwrap();+ assert_eq!(answer[1], REPLY_COMMAND_NOT_SUPPORTED);+ }+}diff --git a/apps/local/src/scrape.rs b/apps/local/src/scrape.rsnew file mode 100644index 000000000..2c8bdbf2b--- /dev/null+++ b/apps/local/src/scrape.rs@@ -0,0 +1,1085 @@+//! One URL in, one Firecrawl-shaped document out.++use std::{collections::HashMap, sync::Arc, time::Duration};++use serde::Deserialize;+use serde_json::{json, Map, Value};+use url::Url;++use crate::{+ browser::{looks_like_nav_only_shell, meta_refresh_target, needs_browser, BrowserPool},+ fetch::{FetchError, FetchOptions, Fetcher},+ guard::NetworkPolicy,+ html::{self, TransformHtmlOptions},+ markdown,+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum Format {+ Markdown,+ Html,+ RawHtml,+ Links,+ Screenshot,+}++impl Format {+ pub fn parse(name: &str) -> Option<Format> {+ match name {+ "markdown" => Some(Format::Markdown),+ "html" => Some(Format::Html),+ "rawHtml" | "raw_html" | "rawhtml" => Some(Format::RawHtml),+ "links" => Some(Format::Links),+ "screenshot" | "screenshot@fullPage" => Some(Format::Screenshot),+ _ => None,+ }+ }+}++/// The subset of Firecrawl's scrape options the local engine honours. Unknown+/// fields are accepted and ignored so existing clients keep working.+#[derive(Debug, Clone, Deserialize)]+#[serde(rename_all = "camelCase", default)]+pub struct ScrapeOptions {+ pub formats: Vec<Value>,+ pub only_main_content: bool,+ pub include_tags: Vec<String>,+ pub exclude_tags: Vec<String>,+ /// Milliseconds, as in the Firecrawl API.+ pub timeout: Option<u64>,+ pub headers: HashMap<String, String>,+ /// Milliseconds to wait after the page loads. Uses the local browser.+ pub wait_for: Option<u64>,+}++impl Default for ScrapeOptions {+ fn default() -> Self {+ Self {+ formats: vec![Value::String("markdown".into())],+ only_main_content: true,+ include_tags: Vec::new(),+ exclude_tags: Vec::new(),+ timeout: None,+ headers: HashMap::new(),+ wait_for: None,+ }+ }+}++impl ScrapeOptions {+ /// Formats the engine can produce locally, and the names of any it cannot.+ pub fn resolved_formats(&self) -> (Vec<Format>, Vec<String>) {+ let mut known = Vec::new();+ let mut unknown = Vec::new();+ for f in &self.formats {+ let name = match f {+ Value::String(s) => s.clone(),+ Value::Object(o) => o+ .get("type")+ .and_then(Value::as_str)+ .unwrap_or("")+ .to_string(),+ _ => String::new(),+ };+ match Format::parse(&name) {+ Some(fmt) if !known.contains(&fmt) => known.push(fmt),+ Some(_) => {}+ None if !name.is_empty() => unknown.push(name),+ None => {}+ }+ }+ if known.is_empty() && unknown.is_empty() {+ known.push(Format::Markdown);+ }+ (known, unknown)+ }+}++#[derive(Debug)]+pub struct ScrapeError {+ pub status: u16,+ pub code: &'static str,+ pub message: String,+}++impl ScrapeError {+ fn new(status: u16, code: &'static str, message: impl Into<String>) -> Self {+ Self {+ status,+ code,+ message: message.into(),+ }+ }+}++impl std::fmt::Display for ScrapeError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}", self.message)+ }+}++impl From<FetchError> for ScrapeError {+ fn from(e: FetchError) -> Self {+ match e {+ FetchError::Blocked(m) => ScrapeError::new(403, "BLOCKED_TARGET", m),+ FetchError::Timeout => ScrapeError::new(408, "SCRAPE_TIMEOUT", "The scrape timed out."),+ FetchError::TooLarge => ScrapeError::new(413, "RESPONSE_TOO_LARGE", e.to_string()),+ FetchError::Network(m) => ScrapeError::new(502, "FETCH_FAILED", m),+ }+ }+}++/// When to use the local browser.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum BrowserMode {+ /// Plain fetch first; the browser only for pages that come back as an+ /// empty JavaScript shell, or when a screenshot or wait is asked for.+ Auto,+ Never,+ Always,+}++pub struct EngineConfig {+ pub policy: NetworkPolicy,+ pub user_agent: Option<String>,+ pub browser: BrowserMode,+}++pub struct Engine {+ fetcher: Fetcher,+ browser: Option<Arc<BrowserPool>>,+ browser_mode: BrowserMode,+ in_flight: std::sync::atomic::AtomicUsize,+ dirty: std::sync::atomic::AtomicBool,+ parse_slots: tokio::sync::Semaphore,+}++struct InFlight<'a>(&'a Engine);++impl Drop for InFlight<'_> {+ fn drop(&mut self) {+ use std::sync::atomic::Ordering;+ self.0.in_flight.fetch_sub(1, Ordering::SeqCst);+ self.0.dirty.store(true, Ordering::SeqCst);+ }+}++fn looks_like_html(mime: Option<&str>, body: &[u8]) -> bool {+ match mime {+ Some("text/html") | Some("application/xhtml+xml") => true,+ Some(m) if m.starts_with("text/") || m.contains("json") || m.contains("xml") => {+ // Mislabelled HTML is common; trust the first bytes over the header.+ sniff_html(body) && !m.contains("json")+ }+ Some(_) => false,+ None => sniff_html(body),+ }+}++fn sniff_html(body: &[u8]) -> bool {+ let head = String::from_utf8_lossy(&body[..body.len().min(1024)]).to_ascii_lowercase();+ let head = head.trim_start();+ head.starts_with("<!doctype html") || head.starts_with("<html") || head.contains("<head")+}++fn resolve_links(hrefs: Vec<String>, base: &Url) -> Vec<String> {+ let mut seen = std::collections::HashSet::new();+ let mut out = Vec::new();+ for href in hrefs {+ let href = href.trim();+ if href.is_empty() || href.starts_with('#') {+ continue;+ }+ let lower = href.to_ascii_lowercase();+ if lower.starts_with("javascript:") || lower.starts_with("data:") {+ continue;+ }+ let resolved = match base.join(href) {+ Ok(u) => u,+ Err(_) => continue,+ };+ if !matches!(resolved.scheme(), "http" | "https" | "mailto" | "tel") {+ continue;+ }+ let s = resolved.to_string();+ if seen.insert(s.clone()) {+ out.push(s);+ }+ }+ out+}++impl Engine {+ pub fn new(cfg: EngineConfig) -> Result<Self, String> {+ let user_agent = cfg+ .user_agent+ .clone()+ .unwrap_or_else(|| crate::fetch::USER_AGENT.to_string());+ let browser = match cfg.browser {+ BrowserMode::Never => None,+ _ => BrowserPool::new(cfg.policy, &user_agent, 6),+ };+ Ok(Self {+ fetcher: Fetcher::new(cfg.policy, cfg.user_agent.as_deref())?,+ browser,+ browser_mode: cfg.browser,+ in_flight: std::sync::atomic::AtomicUsize::new(0),+ dirty: std::sync::atomic::AtomicBool::new(false),+ parse_slots: tokio::sync::Semaphore::new(+ std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(4),+ ),+ })+ }++ pub fn policy(&self) -> NetworkPolicy {+ self.fetcher.policy()+ }++ pub fn fetcher(&self) -> &Fetcher {+ &self.fetcher+ }++ /// True when a Chromium-family browser was found for JavaScript pages.+ pub fn has_browser(&self) -> bool {+ self.browser.is_some()+ }++ /// Run for the life of a long-lived process (`serve`, `mcp`): whenever the+ /// engine goes idle after doing work, hand freed memory back to the system.+ pub async fn trim_memory_when_idle(self: Arc<Self>) {+ use std::sync::atomic::Ordering;+ loop {+ tokio::time::sleep(Duration::from_secs(2)).await;+ if self.in_flight.load(Ordering::SeqCst) == 0 && self.dirty.swap(false, Ordering::SeqCst) {+ let _ = tokio::task::spawn_blocking(crate::mem::release_free_memory).await;+ }+ }+ }++ /// Stop the browser, if one was started.+ pub async fn shutdown(&self) {+ if let Some(b) = &self.browser {+ b.shutdown().await;+ }+ }++ /// Scrape one URL and also return every link on the page, resolved against+ /// the page's base URL. The crawler uses this to avoid parsing twice.+ pub async fn scrape_with_links(+ &self,+ url: &str,+ opts: &ScrapeOptions,+ ) -> Result<(Value, Vec<String>), ScrapeError> {+ let mut with_links = opts.clone();+ let asked_for_links = opts.resolved_formats().0.contains(&Format::Links);+ if !asked_for_links {+ with_links.formats.push(Value::String("links".into()));+ }+ let mut data = self.scrape(url, &with_links).await?;+ let links: Vec<String> = data+ .get("links")+ .and_then(Value::as_array)+ .map(|a| {+ a.iter()+ .filter_map(|v| v.as_str().map(str::to_string))+ .collect()+ })+ .unwrap_or_default();+ if !asked_for_links {+ if let Some(obj) = data.as_object_mut() {+ obj.remove("links");+ }+ }+ Ok((data, links))+ }++ /// Scrape one URL and return the `data` object of a Firecrawl scrape+ /// response.+ pub async fn scrape(&self, url: &str, opts: &ScrapeOptions) -> Result<Value, ScrapeError> {+ self+ .in_flight+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ let _in_flight = InFlight(self);+ let parsed = parse_url(url)?;+ let (formats, mut unsupported) = opts.resolved_formats();+ let timeout = Duration::from_millis(opts.timeout.unwrap_or(30_000));+ let wait_for = Duration::from_millis(opts.wait_for.unwrap_or(0).min(30_000));+ let wants_screenshot = formats.contains(&Format::Screenshot);+ if wants_screenshot && self.browser.is_none() {+ unsupported.push("screenshot (no Chrome found on this machine)".into());+ }++ let must_render = self.browser.is_some()+ && (self.browser_mode == BrowserMode::Always || wants_screenshot || !wait_for.is_zero());++ let mut data = if must_render {+ self+ .render_document(&parsed, opts, &formats, wait_for, timeout, wants_screenshot)+ .await?+ } else {+ let fetch_opts = FetchOptions {+ headers: opts.headers.clone(),+ timeout: Some(timeout),+ };+ let can_render = self.browser_mode == BrowserMode::Auto && self.browser.is_some();+ let mut target = parsed.clone();+ let mut hops = 0;+ let fetched = loop {+ let fetched = match self.fetcher.fetch(&target, &fetch_opts).await {+ Ok(f) => f,+ // A plain fetch that fails at the network level (reset, protocol+ // error, timeout) often works in a real browser.+ Err(e @ (FetchError::Network(_) | FetchError::Timeout)) if can_render => {+ match self+ .render_document(&parsed, opts, &formats, wait_for, timeout, false)+ .await+ {+ Ok(mut doc) => {+ if !unsupported.is_empty() {+ doc.insert(+ "warning".into(),+ Value::String(format!(+ "These formats are not produced by the local engine and were skipped: {}.",+ unsupported.join(", ")+ )),+ );+ }+ return Ok(Value::Object(doc));+ }+ Err(_) => return Err(e.into()),+ }+ }+ Err(e) => return Err(e.into()),+ };+ // Follow an instant <meta refresh> the way a browser would.+ let next = if hops < 3 && looks_like_html(fetched.mime().as_deref(), &fetched.body) {+ meta_refresh_target(&fetched.text(), &fetched.final_url)+ } else {+ None+ };+ match next {+ Some(url) => {+ hops += 1;+ target = url;+ }+ None => break fetched,+ }+ };+ let mime = fetched.mime();+ if let Some(kind) =+ crate::documents::sniff(&fetched.body, mime.as_deref(), &fetched.final_url)+ {+ let body = fetched.body.clone();+ let parsed_doc = tokio::task::spawn_blocking(move || {+ let label = match kind {+ crate::documents::DocKind::Pdf => "local-pdf",+ crate::documents::DocKind::Office(_) => "local-document",+ };+ crate::documents::parse(&kind, &body).map(|d| (d, label))+ })+ .await+ .map_err(|e| ScrapeError::new(500, "PARSE_FAILED", e.to_string()))?+ .map_err(|e| ScrapeError::new(422, "DOCUMENT_PARSE_FAILED", e))?;+ let (doc, label) = parsed_doc;+ let mut data = Map::new();+ let mut metadata = Map::new();+ if let Some(title) = &doc.title {+ metadata.insert("title".into(), Value::String(title.clone()));+ }+ if let Some(pages) = doc.pages {+ metadata.insert("numPages".into(), json!(pages));+ }+ if doc.pages_needing_ocr > 0 {+ data.insert(+ "warning".into(),+ Value::String(format!(+ "{} page(s) of this PDF are scanned images; the local engine has no OCR, so their text is missing.",+ doc.pages_needing_ocr+ )),+ );+ }+ if formats.contains(&Format::Markdown) {+ data.insert("markdown".into(), Value::String(doc.markdown));+ }+ finish_metadata(+ &mut metadata,+ &parsed,+ fetched.final_url.as_str(),+ fetched.status,+ fetched.content_type.as_deref(),+ label,+ );+ data.insert("metadata".into(), Value::Object(metadata));+ data+ } else if looks_like_html(mime.as_deref(), &fetched.body) {+ let raw_html = fetched.text();+ let (mut data, mut metadata, markdown) = self+ .html_parts(raw_html.clone(), &fetched.final_url, opts, &formats)+ .await?;+ // A bot check (a 403 or 503, a "verifying your browser" page) often+ // clears in a real browser. Keep what the browser got only when it is+ // real content, so a page that stays blocked still reads as blocked.+ let challenged = can_render+ && (matches!(fetched.status, 403 | 503) || crate::cloud::looks_like_bot_check(&markdown));+ let nav_only = looks_like_nav_only_shell(&raw_html, &markdown);+ let shell = can_render+ && !challenged+ && (200..400).contains(&fetched.status)+ && (needs_browser(&raw_html, &markdown) || nav_only);+ let rendered = if challenged {+ match self+ .render_with_markdown(&parsed, opts, &formats, wait_for, timeout, false)+ .await+ {+ Ok((doc, rendered_md))+ if !crate::cloud::looks_like_bot_check(&rendered_md)+ && visible_chars(&rendered_md) > visible_chars(&markdown).max(200) =>+ {+ Some(doc)+ }+ _ => None,+ }+ } else if shell {+ // Keep the render unless it came back with clearly less text than+ // the plain fetch had (a consent wall, a half-loaded page).+ match self+ .render_with_markdown(&parsed, opts, &formats, wait_for, timeout, false)+ .await+ {+ Ok((doc, rendered_md))+ if nav_only || visible_chars(&rendered_md) * 10 >= visible_chars(&markdown) * 8 =>+ {+ Some(doc)+ }+ _ => None,+ }+ } else {+ None+ };+ match rendered {+ Some(doc) => doc,+ None => {+ finish_metadata(+ &mut metadata,+ &parsed,+ fetched.final_url.as_str(),+ fetched.status,+ fetched.content_type.as_deref(),+ "local-fetch",+ );+ data.insert("metadata".into(), Value::Object(metadata));+ data+ }+ }+ } else if mime+ .as_deref()+ .map(|m| m.starts_with("text/") || m.contains("json") || m.contains("xml"))+ .unwrap_or(false)+ {+ let text = fetched.text();+ let mut data = Map::new();+ if formats.contains(&Format::Markdown) {+ data.insert("markdown".into(), Value::String(text.clone()));+ }+ if formats.contains(&Format::RawHtml) {+ data.insert("rawHtml".into(), Value::String(text));+ }+ let mut metadata = Map::new();+ finish_metadata(+ &mut metadata,+ &parsed,+ fetched.final_url.as_str(),+ fetched.status,+ fetched.content_type.as_deref(),+ "local-fetch",+ );+ data.insert("metadata".into(), Value::Object(metadata));+ data+ } else {+ return Err(ScrapeError::new(+ 415,+ "UNSUPPORTED_CONTENT_TYPE",+ format!(+ "The local engine does not parse '{}' yet.",+ mime.as_deref().unwrap_or("unknown content type")+ ),+ ));+ }+ };++ if !unsupported.is_empty() {+ data.insert(+ "warning".into(),+ Value::String(format!(+ "These formats are not produced by the local engine and were skipped: {}.",+ unsupported.join(", ")+ )),+ );+ }+ Ok(Value::Object(data))+ }++ /// Load the page in the local browser and build the document from what it+ /// rendered.+ async fn render_document(+ &self,+ requested: &Url,+ opts: &ScrapeOptions,+ formats: &[Format],+ wait_for: Duration,+ timeout: Duration,+ screenshot: bool,+ ) -> Result<Map<String, Value>, ScrapeError> {+ self+ .render_with_markdown(requested, opts, formats, wait_for, timeout, screenshot)+ .await+ .map(|(data, _)| data)+ }++ /// `render_document`, plus the page's markdown even when markdown was not+ /// a requested format, so the caller can judge what the browser got.+ async fn render_with_markdown(+ &self,+ requested: &Url,+ opts: &ScrapeOptions,+ formats: &[Format],+ wait_for: Duration,+ timeout: Duration,+ screenshot: bool,+ ) -> Result<(Map<String, Value>, String), ScrapeError> {+ let browser = self+ .browser+ .as_ref()+ .ok_or_else(|| ScrapeError::new(501, "NO_BROWSER", "No Chrome was found on this machine."))?;+ let rendered = browser+ .render(+ requested,+ wait_for,+ timeout.max(Duration::from_secs(10)),+ screenshot,+ )+ .await+ .map_err(|m| {+ if m.contains("private") || m.contains("never reachable") || m.contains("local hostname") {+ ScrapeError::new(403, "BLOCKED_TARGET", m)+ } else {+ ScrapeError::new(502, "RENDER_FAILED", m)+ }+ })?;+ let final_url = Url::parse(&rendered.final_url).unwrap_or_else(|_| requested.clone());+ let (mut data, mut metadata, markdown) = self+ .html_parts(rendered.html, &final_url, opts, formats)+ .await?;+ if let Some(png) = rendered.screenshot_png {+ use base64::Engine as _;+ let encoded = base64::engine::general_purpose::STANDARD.encode(png);+ data.insert(+ "screenshot".into(),+ Value::String(format!("data:image/png;base64,{encoded}")),+ );+ }+ finish_metadata(+ &mut metadata,+ requested,+ final_url.as_str(),+ rendered.status.unwrap_or(200),+ Some("text/html"),+ "local-browser",+ );+ data.insert("metadata".into(), Value::Object(metadata));+ Ok((data, markdown))+ }++ /// Turn one HTML document into the requested formats. Also returns the+ /// markdown (even when not requested as a format) so the caller can tell an+ /// empty JavaScript shell from a real page.+ ///+ /// The page is parsed once. Metadata, links, cleaning and markdown all run+ /// on that one tree, on a single blocking thread.+ async fn html_parts(+ &self,+ raw_html: String,+ final_url: &Url,+ opts: &ScrapeOptions,+ formats: &[Format],+ ) -> Result<(Map<String, Value>, Map<String, Value>, String), ScrapeError> {+ let final_url = final_url.clone();+ let opts = opts.clone();+ let formats = formats.to_vec();+ // Parsing is CPU-bound: more pages in memory than cores only raises the+ // peak, not the throughput.+ let _permit = self+ .parse_slots+ .acquire()+ .await+ .map_err(|e| ScrapeError::new(500, "PARSE_FAILED", e.to_string()))?;+ tokio::task::spawn_blocking(move || process_html(raw_html, &final_url, &opts, &formats))+ .await+ .map_err(|e| ScrapeError::new(500, "PARSE_FAILED", e.to_string()))?+ }+}++fn process_html(+ raw_html: String,+ final_url: &Url,+ opts: &ScrapeOptions,+ formats: &[Format],+) -> Result<(Map<String, Value>, Map<String, Value>, String), ScrapeError> {+ use kuchikiki::traits::TendrilSink;+ let parse_err = |e: Box<dyn std::error::Error + Send + Sync>| {+ ScrapeError::new(500, "PARSE_FAILED", e.to_string())+ };+ let mut data = Map::new();+ let mut metadata: Map<String, Value> = Map::new();++ let document = kuchikiki::parse_html().one(raw_html.as_str());++ // Read-only passes first, on the page as it was served.+ for (k, v) in html::extract_metadata_from_document(&document).map_err(parse_err)? {+ metadata.insert(k, v);+ }+ if formats.contains(&Format::Links) {+ let base = Url::parse(&html::base_href_of_document(&document, final_url))+ .unwrap_or_else(|_| final_url.clone());+ let hrefs = html::extract_links_from_document(&document)+ .map_err(|e| ScrapeError::new(500, "PARSE_FAILED", e))?;+ data.insert("links".into(), json!(resolve_links(hrefs, &base)));+ }++ // Then the passes that change the tree.+ let prose_before = if opts.only_main_content {+ paragraph_chars(&document)+ } else {+ 0+ };+ if opts.only_main_content {+ keep_content_headers(&document);+ strip_page_furniture(&document);+ }+ let transform = |document: kuchikiki::NodeRef, only_main: bool, extra_excludes: &[&str]| {+ let mut exclude_tags = opts.exclude_tags.clone();+ exclude_tags.extend(extra_excludes.iter().map(|s| s.to_string()));+ html::transform_document(+ document,+ &TransformHtmlOptions {+ html: String::new(),+ url: final_url.to_string(),+ include_tags: opts.include_tags.clone(),+ exclude_tags,+ only_main_content: only_main,+ omce_signatures: None,+ },+ )+ };+ let mut cleaned = transform(document, opts.only_main_content, &[]).map_err(parse_err)?;++ // Safety net: when the main-content rules strip most of a long page's+ // paragraphs, they have misfired (an article body inside a `.widget`, say).+ // Redo the page keeping everything except the semantic page furniture.+ if opts.only_main_content && opts.include_tags.is_empty() && prose_before >= 2_000 {+ let prose_after = paragraph_chars(&cleaned);+ if prose_after * 100 < prose_before * 35 {+ let again = kuchikiki::parse_html().one(raw_html.as_str());+ keep_content_headers(&again);+ strip_page_furniture(&again);+ cleaned = transform(+ again,+ false,+ &[+ "header",+ "footer",+ "nav",+ "aside",+ "[role=navigation]",+ "[role=banner]",+ "[role=contentinfo]",+ ],+ )+ .map_err(parse_err)?;+ }+ }++ if opts.only_main_content && opts.include_tags.is_empty() {+ strip_edge_link_blocks(&cleaned);+ }++ let md = html::post_process_markdown_text(&markdown::document_to_markdown(&cleaned));+ if formats.contains(&Format::Markdown) {+ data.insert("markdown".into(), Value::String(md.clone()));+ }+ if formats.contains(&Format::Html) {+ data.insert("html".into(), Value::String(cleaned.to_string()));+ }+ if formats.contains(&Format::RawHtml) {+ data.insert("rawHtml".into(), Value::String(raw_html));+ }+ Ok((data, metadata, md))+}++const CONSENT_SELECTORS: &str = "#onetrust-consent-sdk, #onetrust-banner-sdk, #CybotCookiebotDialog, #cookie-law-info-bar, \+ #cookieConsent, .cc-window, .cc-banner, [id*=\"cookie-banner\" i], [class*=\"cookie-banner\" i], [id*=\"cookie-consent\" i], \+ [class*=\"cookie-consent\" i], [id*=\"cookie-notice\" i], [class*=\"cookie-notice\" i], [id*=\"cookieconsent\" i], \+ [class*=\"cookieconsent\" i], [id*=\"consent-banner\" i], [class*=\"consent-banner\" i], [id*=\"gdpr\" i], [class*=\"gdpr\" i], \+ [id*=\"cookie-bar\" i], [class*=\"cookie-bar\" i], [id*=\"cookiebar\" i], [class*=\"cookiebar\" i]";++fn has_descendant(node: &kuchikiki::NodeRef, selector: &str) -> bool {+ node+ .select(selector)+ .map(|mut i| i.next().is_some())+ .unwrap_or(false)+}++/// Remove, in place, two kinds of clutter the shared main-content rules miss:+/// cookie and consent banners, and small sign-up or search forms sitting in+/// the middle of content. Anything holding real content (a heading, an+/// article, a table, a lot of text) is left alone.+fn strip_page_furniture(document: &kuchikiki::NodeRef) {+ let mut doomed: Vec<kuchikiki::NodeRef> = Vec::new();+ if let Ok(banners) = document.select(CONSENT_SELECTORS) {+ for b in banners {+ let node = b.as_node();+ if node.text_contents().len() < 2_500+ && !has_descendant(node, "h1, article, main, table, pre")+ {+ doomed.push(node.clone());+ }+ }+ }+ if let Ok(forms) = document.select("form") {+ for f in forms {+ let node = f.as_node();+ let small = node.text_contents().trim().len() < 500;+ let has_fields = has_descendant(node, "input, select, textarea, button");+ if small+ && has_fields+ && !has_descendant(node, "h1, h2, article, main, table, pre, p + p + p")+ {+ doomed.push(node.clone());+ }+ }+ }+ for node in doomed {+ node.detach();+ }+}++/// Text measurements of one subtree, in non-whitespace characters.+#[derive(Default, Clone, Copy)]+struct TextStats {+ text: usize,+ link_text: usize,+ links: usize,+}++/// One element seen while measuring the page, with its place in document+/// order: `start` on the way in, `end` once its subtree is done.+struct Measured {+ node: kuchikiki::NodeRef,+ start: usize,+ end: usize,+ stats: TextStats,+ tag: String,+ after_heading: bool,+}++fn measure(+ node: &kuchikiki::NodeRef,+ in_link: bool,+ counter: &mut usize,+ out: &mut Vec<Measured>,+) -> TextStats {+ let mut stats = TextStats::default();+ let mut previous_was_heading = false;+ for child in node.children() {+ if let Some(text) = child.as_text() {+ let n = text.borrow().chars().filter(|c| !c.is_whitespace()).count();+ stats.text += n;+ if in_link {+ stats.link_text += n;+ }+ continue;+ }+ let Some(element) = child.as_element() else {+ continue;+ };+ let tag = element.name.local.to_string();+ let is_link = tag == "a";+ *counter += 1;+ let start = *counter;+ let inner = measure(&child, in_link || is_link, counter, out);+ *counter += 1;+ stats.text += inner.text;+ stats.link_text += inner.link_text;+ stats.links += inner.links + usize::from(is_link);+ let is_heading = matches!(tag.as_str(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6");+ out.push(Measured {+ node: child.clone(),+ start,+ end: *counter,+ stats: TextStats {+ links: inner.links + usize::from(is_link),+ ..inner+ },+ tag,+ after_heading: previous_was_heading,+ });+ previous_was_heading = is_heading;+ }+ stats+}++/// Remove, in place, menus and footer link rows that survive the shared+/// main-content rules because the site builds them from plain `<div>`s.+///+/// Deliberately narrow. A block goes only when all of this holds: the page+/// has real prose; the block sits wholly before the first or after the last+/// paragraph of it; it is short, nearly all link text, with three or more+/// short links; and no heading introduces it (a "See also" list stays). A+/// short copyright line after the last paragraph goes too. Pages that are+/// themselves lists of links have no prose to anchor on and are left alone.+fn strip_edge_link_blocks(document: &kuchikiki::NodeRef) {+ let mut measured = Vec::new();+ let mut counter = 0;+ measure(document, false, &mut counter, &mut measured);++ let prose = |m: &Measured| {+ matches!(m.tag.as_str(), "p" | "blockquote" | "pre" | "dd")+ && m.stats.text - m.stats.link_text >= 60+ };+ let anchors: Vec<&Measured> = measured.iter().filter(|m| prose(m)).collect();+ let prose_chars: usize = anchors+ .iter()+ .map(|m| m.stats.text - m.stats.link_text)+ .sum();+ if anchors.len() < 2 || prose_chars < 500 {+ return;+ }+ let content_start = anchors.iter().map(|m| m.start).min().unwrap_or(0);+ let content_end = anchors.iter().map(|m| m.end).max().unwrap_or(usize::MAX);++ let mut doomed: Vec<kuchikiki::NodeRef> = Vec::new();+ for m in &measured {+ let outside = m.end < content_start || m.start > content_end;+ if !outside || m.after_heading {+ continue;+ }+ let block = matches!(+ m.tag.as_str(),+ "ul" | "ol" | "menu" | "nav" | "div" | "p" | "section" | "aside" | "footer" | "table" | "dl"+ );+ if !block || m.stats.text == 0 {+ continue;+ }+ let link_row = m.stats.links >= 3+ && m.stats.text <= 500+ && m.stats.link_text * 100 >= m.stats.text * 80+ && m.stats.link_text / m.stats.links <= 30+ && !has_descendant(&m.node, "h1, h2, h3, img, picture, video, pre, form");+ let copyright = m.start > content_end && m.stats.text <= 200 && m.stats.links <= 6 && {+ let text = m.node.text_contents().to_lowercase();+ text.contains('©') || text.contains("all rights reserved") || text.contains("(c) 20")+ };+ if link_row || copyright {+ doomed.push(m.node.clone());+ }+ }+ for node in doomed {+ node.detach();+ }+}++/// Characters of text inside `<p>` elements: a cheap measure of how much+/// prose a page holds.+fn paragraph_chars(document: &kuchikiki::NodeRef) -> usize {+ match document.select("p") {+ Ok(ps) => ps.map(|p| p.text_contents().trim().len()).sum(),+ Err(_) => 0,+ }+}++/// The main-content rules drop every `<header>`, which also drops the title+/// of an article written as `<article><header><h1>`. Unwrap, in place,+/// headers that sit inside the page's content and hold a heading, so the+/// title survives.+fn keep_content_headers(document: &kuchikiki::NodeRef) {+ let Ok(selection) = document.select("article header, main header, [role=main] header") else {+ return;+ };+ let headers: Vec<_> = selection+ .filter(|h| {+ let node = h.as_node();+ let has_heading = node+ .select("h1, h2, h3")+ .map(|mut i| i.next().is_some())+ .unwrap_or(false);+ let has_nav = node+ .select("nav, [role=navigation]")+ .map(|mut i| i.next().is_some())+ .unwrap_or(false);+ has_heading && !has_nav+ })+ .collect();+ for header in headers {+ let node = header.as_node();+ let children: Vec<_> = node.children().collect();+ for child in children {+ node.insert_before(child);+ }+ node.detach();+ }+}++fn finish_metadata(+ metadata: &mut Map<String, Value>,+ requested: &Url,+ final_url: &str,+ status: u16,+ content_type: Option<&str>,+ engine: &str,+) {+ metadata.insert("sourceURL".into(), Value::String(requested.to_string()));+ metadata.insert("url".into(), Value::String(final_url.to_string()));+ metadata.insert("statusCode".into(), json!(status));+ if let Some(ct) = content_type {+ metadata.insert("contentType".into(), Value::String(ct.to_string()));+ }+ metadata.insert("engine".into(), Value::String(engine.to_string()));+ metadata.insert("creditsUsed".into(), json!(0));+}++/// Non-whitespace characters, the measure of how much a page actually says.+fn visible_chars(markdown: &str) -> usize {+ markdown.chars().filter(|c| !c.is_whitespace()).count()+}++pub fn parse_url(input: &str) -> Result<Url, ScrapeError> {+ let trimmed = input.trim();+ let candidate = if trimmed.contains("://") {+ trimmed.to_string()+ } else {+ format!("https://{trimmed}")+ };+ Url::parse(&candidate)+ .map_err(|e| ScrapeError::new(400, "BAD_REQUEST", format!("Invalid URL '{input}': {e}")))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn cleaned_text(html: &str) -> String {+ use kuchikiki::traits::TendrilSink;+ let document = kuchikiki::parse_html().one(html);+ strip_edge_link_blocks(&document);+ document+ .text_contents()+ .split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ }++ const PROSE: &str = "<p>The first paragraph of the article says something long enough to count as real prose on a page, well past sixty characters.</p>\+ <p>The second paragraph keeps going so the page clearly has an article in it, with more than five hundred characters of text overall, \+ which is what tells the cleaner that menus around it are furniture and not the content itself. It needs to be reasonably long, so here \+ is another sentence, and one more after it to be safely over the bar that the rule sets for itself before it removes anything at all. \+ A last sentence pads the paragraph out a little further, because the bar counts letters and not the spaces between the words.</p>";++ #[test]+ fn edge_menus_and_footer_rows_are_dropped() {+ let html = format!(+ "<body><div><a href=/>HOME</a> <a href=/f>FOTO'S</a> <a href=/c>CONTACT</a> <a href=/d>DIENSTEN</a></div>\+ <h1>Title</h1>{PROSE}\+ <div><a href=/t>Terms of use</a> <a href=/p>Privacy Policy</a> <a href=/x>Disclaimer</a> <a href=/q>FAQ</a></div>\+ <div>© Example 2026 All rights reserved.</div></body>"+ );+ let text = cleaned_text(&html);+ assert!(+ text.contains("Title")+ && text.contains("first paragraph")+ && text.contains("sets for itself")+ );+ assert!(!text.contains("DIENSTEN"), "{text}");+ assert!(!text.contains("Privacy Policy"), "{text}");+ assert!(!text.contains("All rights reserved"), "{text}");+ }++ #[test]+ fn link_lists_that_are_content_stay() {+ // A list a heading introduces, links inside the article, and a page that is only links.+ let html = format!(+ "<body><h1>Title</h1>{PROSE}<ul><li><a href=/in>inside one</a></li><li><a href=/in2>inside two</a></li><li><a href=/in3>inside three</a></li></ul>\+ <p>A closing paragraph that is also long enough to count as prose, so the list above sits inside the content.</p>\+ <h2>See also</h2><ul><li><a href=/a>Alpha</a></li><li><a href=/b>Beta</a></li><li><a href=/c>Gamma</a></li></ul></body>"+ );+ let text = cleaned_text(&html);+ assert!(+ text.contains("inside two") && text.contains("Alpha") && text.contains("Gamma"),+ "{text}"+ );++ let hub = "<body><ul><li><a href=/a>Alpha</a></li><li><a href=/b>Beta</a></li><li><a href=/c>Gamma</a></li><li><a href=/d>Delta</a></li></ul></body>";+ assert!(cleaned_text(hub).contains("Delta"));++ // Long link titles after the article (related reading) are not a menu.+ let related = format!(+ "<body>{PROSE}<div><a href=/1>A much longer related article headline that reads as a title</a> \+ <a href=/2>Another long related article headline that reads as a title</a> \+ <a href=/3>A third long related article headline that reads as a title</a></div></body>"+ );+ assert!(cleaned_text(&related).contains("third long related"));+ }++ #[test]+ fn formats_default_and_objects() {+ let o = ScrapeOptions::default();+ assert_eq!(o.resolved_formats().0, vec![Format::Markdown]);++ let o: ScrapeOptions = serde_json::from_value(json!({+ "formats": ["markdown", {"type": "json", "schema": {}}, "links", "summary"],+ "onlyMainContent": false,+ "somethingNew": 1+ }))+ .unwrap();+ let (known, unknown) = o.resolved_formats();+ assert_eq!(known, vec![Format::Markdown, Format::Links]);+ assert_eq!(unknown, vec!["json".to_string(), "summary".to_string()]);+ assert!(!o.only_main_content);+ }++ #[test]+ fn links_resolved_and_deduped() {+ let base = Url::parse("https://example.com/docs/page").unwrap();+ let links = resolve_links(+ vec![+ "/a".into(),+ "b".into(),+ "#frag".into(),+ "https://other.dev/x".into(),+ "/a".into(),+ "javascript:void(0)".into(),+ "mailto:[email protected]".into(),+ ],+ &base,+ );+ assert_eq!(+ links,+ vec![+ "https://example.com/a",+ "https://example.com/docs/b",+ "https://other.dev/x",+ "mailto:[email protected]"+ ]+ );+ }++ #[test]+ fn bare_host_gets_https() {+ assert_eq!(+ parse_url("example.com/a").unwrap().as_str(),+ "https://example.com/a"+ );+ assert!(parse_url("http://").is_err());+ }+}diff --git a/apps/local/src/serve.rs b/apps/local/src/serve.rsnew file mode 100644index 000000000..04450975b--- /dev/null+++ b/apps/local/src/serve.rs@@ -0,0 +1,926 @@+//! `firecrawl-local serve`: the Firecrawl HTTP API on one port.+//!+//! Safety defaults:+//! - binds to loopback unless told otherwise;+//! - a non-loopback bind refuses to start without a token;+//! - every request must carry a Host header naming this machine, which stops+//! a web page from reaching the API through DNS rebinding;+//! - bodies must be `application/json`, so a browser cannot post to it+//! cross-origin without a preflight that this server never approves.++use std::{+ collections::HashMap,+ net::SocketAddr,+ sync::{Arc, Mutex},+ time::{Duration, Instant},+};++use axum::{+ body::Bytes,+ extract::{Path, Request, State},+ http::{header, HeaderMap, Method, StatusCode},+ middleware::{self, Next},+ response::{IntoResponse, Response},+ routing::{delete, get, post},+ Json, Router,+};+use serde_json::{json, Value};++use futures::stream::{self, StreamExt};++use crate::{+ cloud::{self, Cloud},+ crawl::{self, CrawlOptions, MapOptions},+ scrape::{Engine, ScrapeError, ScrapeOptions},+};++const JOB_TTL: Duration = Duration::from_secs(60 * 60);++/// The largest body passed on to Firecrawl cloud. Local routes keep axum's+/// 2 MB default; uploads to cloud routes such as /v2/parse need more.+const FORWARD_BODY_LIMIT: usize = 64 * 1024 * 1024;++/// A crawl or batch scrape. Jobs live in memory and expire after an hour.+struct Job {+ status: &'static str,+ total: usize,+ data: Vec<Value>,+ errors: Vec<Value>,+ created: Instant,+ task: Option<tokio::task::AbortHandle>,+}++type Jobs = Mutex<HashMap<String, Arc<Mutex<Job>>>>;++pub struct ServeConfig {+ pub addr: SocketAddr,+ /// Required when `addr` is not loopback.+ pub token: Option<String>,+ /// Extra hostnames accepted in the Host header (for example a compose+ /// service name).+ pub allowed_hosts: Vec<String>,+ /// What may be passed on to Firecrawl cloud. `serve` keeps it off unless a+ /// key or setting turns it on; the MCP launcher runs with it on.+ pub cloud: Cloud,+}++struct AppState {+ engine: Arc<Engine>,+ cloud: Cloud,+ token: Option<String>,+ allowed_hosts: Vec<String>,+ jobs: Jobs,+}++fn error_response(status: StatusCode, code: &str, message: impl Into<String>) -> Response {+ (+ status,+ Json(json!({ "success": false, "code": code, "error": message.into() })),+ )+ .into_response()+}++fn host_without_port(host: &str) -> &str {+ if let Some(rest) = host.strip_prefix('[') {+ return rest.split(']').next().unwrap_or(rest);+ }+ host.rsplit_once(':').map(|(h, _)| h).unwrap_or(host)+}++fn host_allowed(host: &str, extra: &[String]) -> bool {+ let h = host_without_port(host).to_ascii_lowercase();+ h == "localhost"+ || h == "127.0.0.1"+ || h == "::1"+ || extra.iter().any(|e| e.eq_ignore_ascii_case(&h))+}++fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {+ if a.len() != b.len() {+ return false;+ }+ a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0+}++async fn gate(State(state): State<Arc<AppState>>, req: Request, next: Next) -> Response {+ let host = req+ .headers()+ .get(header::HOST)+ .and_then(|v| v.to_str().ok())+ .unwrap_or("");+ if !host_allowed(host, &state.allowed_hosts) {+ return error_response(+ StatusCode::FORBIDDEN,+ "HOST_NOT_ALLOWED",+ "Firecrawl Local only answers requests addressed to this machine. Start it with --allowed-host to accept another name.",+ );+ }+ if let Some(expected) = &state.token {+ let presented = req+ .headers()+ .get(header::AUTHORIZATION)+ .and_then(|v| v.to_str().ok())+ .and_then(|v| {+ v.strip_prefix("Bearer ")+ .or_else(|| v.strip_prefix("bearer "))+ })+ .unwrap_or("");+ if !constant_time_eq(presented.as_bytes(), expected.as_bytes()) {+ return error_response(+ StatusCode::UNAUTHORIZED,+ "UNAUTHORIZED",+ "Missing or wrong token. Send 'Authorization: Bearer <token>'.",+ );+ }+ }+ next.run(req).await+}++/// The origin a client put on its request, in the body or the header.+fn client_origin(headers: &HeaderMap, body_origin: Option<&str>) -> Option<String> {+ body_origin+ .filter(|o| !o.is_empty() && *o != "api")+ .map(str::to_string)+ .or_else(|| {+ headers+ .get("x-origin")+ .and_then(|v| v.to_str().ok())+ .filter(|o| !o.is_empty())+ .map(str::to_string)+ })+}++fn is_json(headers: &HeaderMap) -> bool {+ headers+ .get(header::CONTENT_TYPE)+ .and_then(|v| v.to_str().ok())+ .map(|v| v.to_ascii_lowercase().starts_with("application/json"))+ .unwrap_or(false)+}++/// Whether a browser sent this request on behalf of a web page. Browsers+/// put `Origin` on every cross-origin request that can change something and+/// `Sec-Fetch-Site` on all of them; command-line tools and SDKs send neither.+fn from_a_web_page(headers: &HeaderMap) -> bool {+ let value = |name: &str| {+ headers+ .get(name)+ .and_then(|v| v.to_str().ok())+ .map(|v| v.trim().to_ascii_lowercase())+ .filter(|v| !v.is_empty())+ };+ if let Some(site) = value("sec-fetch-site") {+ if site != "same-origin" && site != "none" {+ return true;+ }+ }+ value("origin").is_some()+}++/// Why a request may not be passed on to Firecrawl cloud, if it may not.+///+/// A page open in the user's browser can send "simple" requests (a form+/// post, an upload, a GET) to a loopback port without asking first, and a+/// forwarded request spends the user's Firecrawl key. So a forwarded request+/// must either be `application/json`, which a page cannot send across+/// origins without a preflight this server never grants, or carry none of+/// the headers a browser adds.+fn forward_refusal(+ state: &AppState,+ method: &Method,+ path: &str,+ headers: &HeaderMap,+) -> Option<Response> {+ if cloud::has_dot_segment(path) {+ return Some(error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ "The path may not contain '.' or '..' segments.",+ ));+ }+ if !state.cloud.allows(path) {+ return Some(+ (+ StatusCode::NOT_IMPLEMENTED,+ Json(json!({+ "success": false,+ "code": "CLOUD_ONLY",+ "error": format!(+ "{path} runs on Firecrawl cloud, and the cloud is off here. Set FIRECRAWL_API_KEY or FIRECRAWL_CLOUD=auto to pass these requests on."+ ),+ "cloud": { "url": "https://www.firecrawl.dev", "docs": "https://docs.firecrawl.dev" }+ })),+ )+ .into_response(),+ );+ }+ if *method == Method::OPTIONS || (!is_json(headers) && from_a_web_page(headers)) {+ return Some(error_response(+ StatusCode::FORBIDDEN,+ "CROSS_SITE_REQUEST",+ "Firecrawl Local does not pass requests from web pages on to Firecrawl cloud. Call it from a script, an SDK or a command-line tool.",+ ));+ }+ None+}++/// Pass a request on to Firecrawl cloud and hand its answer back, or say+/// plainly why not.+async fn pass_on(+ state: &AppState,+ method: Method,+ path_and_query: &str,+ headers: &HeaderMap,+ body: Bytes,+) -> Response {+ let path = path_and_query.split('?').next().unwrap_or(path_and_query);+ if let Some(refusal) = forward_refusal(state, &method, path, headers) {+ return refusal;+ }+ // With a token set, the caller's bearer is this engine's token and must+ // not travel on. Without one it is the caller's own Firecrawl key.+ let client_auth = if state.token.is_none() {+ headers+ .get(header::AUTHORIZATION)+ .and_then(|v| v.to_str().ok())+ .and_then(|v| {+ v.strip_prefix("Bearer ")+ .or_else(|| v.strip_prefix("bearer "))+ })+ .filter(|k| !k.is_empty())+ } else {+ None+ };+ let content_type = headers+ .get(header::CONTENT_TYPE)+ .and_then(|v| v.to_str().ok());+ let origin = client_origin(headers, None);+ match state+ .cloud+ .forward(+ method,+ path_and_query,+ content_type,+ body,+ client_auth,+ origin.as_deref(),+ )+ .await+ {+ Ok(answer) => {+ let status = StatusCode::from_u16(answer.status).unwrap_or(StatusCode::BAD_GATEWAY);+ let mut response = (status, answer.body).into_response();+ if let Some(ct) = answer.content_type.and_then(|ct| ct.parse().ok()) {+ response.headers_mut().insert(header::CONTENT_TYPE, ct);+ }+ response+ }+ Err(e) => error_response(+ StatusCode::from_u16(e.status).unwrap_or(StatusCode::BAD_GATEWAY),+ "CLOUD_ERROR",+ e.message,+ ),+ }+}++/// Everything this engine does not serve itself. Routes under /v2 go to+/// Firecrawl cloud: search, data providers, the agent and the rest of the+/// API. Anything else is not an API route.+async fn forward_handler(State(state): State<Arc<AppState>>, req: Request) -> Response {+ let (parts, body) = req.into_parts();+ let (method, uri, headers) = (parts.method, parts.uri, parts.headers);+ if !uri.path().starts_with("/v2/") {+ return error_response(+ StatusCode::NOT_FOUND,+ "NOT_FOUND",+ "Firecrawl Local serves the Firecrawl v2 API. Use /v2/scrape, /v2/map, /v2/crawl or /v2/batch/scrape.",+ );+ }+ // Decide before reading the body, so a refused upload is not buffered.+ if let Some(refusal) = forward_refusal(&state, &method, uri.path(), &headers) {+ return refusal;+ }+ let body = match axum::body::to_bytes(body, FORWARD_BODY_LIMIT).await {+ Ok(body) => body,+ Err(_) => {+ return error_response(+ StatusCode::PAYLOAD_TOO_LARGE,+ "PAYLOAD_TOO_LARGE",+ format!(+ "The request body could not be read, or is larger than the {} MB Firecrawl Local passes on to Firecrawl cloud.",+ FORWARD_BODY_LIMIT / (1024 * 1024)+ ),+ )+ }+ };+ let path_and_query = uri+ .path_and_query()+ .map(|pq| pq.as_str())+ .unwrap_or(uri.path());+ pass_on(&state, method, path_and_query, &headers, body).await+}++fn scrape_error(e: ScrapeError) -> Response {+ let status = StatusCode::from_u16(e.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);+ error_response(status, e.code, e.message)+}++/// Parse a JSON body. Anything that is not `application/json` is refused, so+/// a web page cannot post here without a preflight.+fn json_body(headers: &HeaderMap, body: &Bytes) -> Result<Value, Response> {+ if !is_json(headers) {+ return Err(error_response(+ StatusCode::UNSUPPORTED_MEDIA_TYPE,+ "BAD_REQUEST",+ "Send the request body as application/json.",+ ));+ }+ serde_json::from_slice(body).map_err(|e| {+ error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ format!("Invalid JSON: {e}"),+ )+ })+}++fn required_url(payload: &Value) -> Result<String, Response> {+ match payload.get("url").and_then(Value::as_str) {+ Some(u) if !u.trim().is_empty() => Ok(u.to_string()),+ _ => Err(error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ "The 'url' field is required.",+ )),+ }+}++fn new_job(state: &AppState, total: usize) -> (String, Arc<Mutex<Job>>) {+ let id = uuid::Uuid::new_v4().to_string();+ let job = Arc::new(Mutex::new(Job {+ status: "scraping",+ total,+ data: Vec::new(),+ errors: Vec::new(),+ created: Instant::now(),+ task: None,+ }));+ let mut jobs = locked(&state.jobs);+ drop_expired(&mut jobs);+ jobs.insert(id.clone(), job.clone());+ (id, job)+}++/// Lock a mutex, taking the data even if another thread panicked while+/// holding it. A job map with one half-updated job is still usable, and a+/// panic here would take every other request down with it.+fn locked<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {+ mutex+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+}++/// Remove jobs older than `JOB_TTL`, stopping any that are still running.+/// Runs whenever a job is created or looked up, so an expired job is never+/// served and never keeps working unobserved.+fn drop_expired(jobs: &mut HashMap<String, Arc<Mutex<Job>>>) {+ jobs.retain(|_, job| {+ let mut job = locked(job);+ let live = job.created.elapsed() < JOB_TTL;+ if !live {+ if let Some(task) = job.task.take() {+ task.abort();+ }+ }+ live+ });+}++fn find_job(state: &AppState, id: &str) -> Option<Arc<Mutex<Job>>> {+ let mut jobs = locked(&state.jobs);+ drop_expired(&mut jobs);+ jobs.get(id).cloned()+}++fn job_url(headers: &HeaderMap, path: &str, id: &str) -> String {+ let host = headers+ .get(header::HOST)+ .and_then(|v| v.to_str().ok())+ .unwrap_or("127.0.0.1");+ format!("http://{host}{path}/{id}")+}++async fn map_handler(+ State(state): State<Arc<AppState>>,+ headers: HeaderMap,+ body: Bytes,+) -> Response {+ let payload = match json_body(&headers, &body) {+ Ok(v) => v,+ Err(r) => return r,+ };+ let url = match required_url(&payload) {+ Ok(u) => u,+ Err(r) => return r,+ };+ let opts: MapOptions = match serde_json::from_value(payload) {+ Ok(o) => o,+ Err(e) => {+ return error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ format!("Invalid options: {e}"),+ )+ }+ };+ match crawl::map(state.engine.clone(), &url, &opts).await {+ Ok(links) => {+ let links = Value::Array(links.into_iter().map(|u| json!({ "url": u })).collect());+ Json(json!({ "success": true, "links": links })).into_response()+ }+ Err(e) => scrape_error(e),+ }+}++async fn crawl_handler(+ State(state): State<Arc<AppState>>,+ req_uri: axum::http::Uri,+ headers: HeaderMap,+ body: Bytes,+) -> Response {+ let payload = match json_body(&headers, &body) {+ Ok(v) => v,+ Err(r) => return r,+ };+ let url = match required_url(&payload) {+ Ok(u) => u,+ Err(r) => return r,+ };+ let mut payload = payload;+ // v1 named the depth option maxDepth.+ if let Some(depth) = payload.get("maxDepth").cloned() {+ if payload.get("maxDiscoveryDepth").is_none() {+ payload["maxDiscoveryDepth"] = depth;+ }+ }+ let opts: CrawlOptions = match serde_json::from_value(payload) {+ Ok(o) => o,+ Err(e) => {+ return error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ format!("Invalid options: {e}"),+ )+ }+ };+ let (id, job) = new_job(&state, 0);+ let engine = state.engine.clone();+ let task_job = job.clone();+ let handle = tokio::spawn(async move {+ let sink = task_job.clone();+ let result = crawl::crawl(engine, &url, &opts, move |page| {+ if let Ok(mut j) = sink.lock() {+ j.data.push(page.clone());+ j.total = j.data.len();+ }+ })+ .await;+ if let Ok(mut j) = task_job.lock() {+ match result {+ Ok(outcome) => {+ j.errors = outcome.errors;+ j.total = j.data.len();+ j.status = "completed";+ }+ Err(e) => {+ j.errors.push(json!({ "error": e.message, "code": e.code }));+ j.status = "failed";+ }+ }+ }+ });+ locked(&job).task = Some(handle.abort_handle());+ let base = req_uri.path().trim_end_matches('/').to_string();+ Json(json!({ "success": true, "id": id, "url": job_url(&headers, &base, &id) })).into_response()+}++async fn batch_handler(+ State(state): State<Arc<AppState>>,+ req_uri: axum::http::Uri,+ headers: HeaderMap,+ body: Bytes,+) -> Response {+ let payload = match json_body(&headers, &body) {+ Ok(v) => v,+ Err(r) => return r,+ };+ let urls: Vec<String> = payload+ .get("urls")+ .and_then(Value::as_array)+ .map(|a| {+ a.iter()+ .filter_map(|v| v.as_str().map(str::to_string))+ .collect()+ })+ .unwrap_or_default();+ if urls.is_empty() {+ return error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ "The 'urls' field needs at least one URL.",+ );+ }+ let concurrency = payload+ .get("maxConcurrency")+ .and_then(Value::as_u64)+ .map(|n| n as usize)+ .unwrap_or(16)+ .clamp(1, 128);+ let opts: ScrapeOptions = match serde_json::from_value(payload) {+ Ok(o) => o,+ Err(e) => {+ return error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ format!("Invalid options: {e}"),+ )+ }+ };+ let (id, job) = new_job(&state, urls.len());+ let engine = state.engine.clone();+ let task_job = job.clone();+ let handle = tokio::spawn(async move {+ let mut results = stream::iter(urls.into_iter().map(|url| {+ let engine = engine.clone();+ let opts = opts.clone();+ async move {+ let res = engine.scrape(&url, &opts).await;+ (url, res)+ }+ }))+ .buffer_unordered(concurrency);+ while let Some((url, res)) = results.next().await {+ if let Ok(mut j) = task_job.lock() {+ match res {+ Ok(doc) => j.data.push(doc),+ Err(e) => j+ .errors+ .push(json!({ "url": url, "error": e.message, "code": e.code })),+ }+ }+ }+ if let Ok(mut j) = task_job.lock() {+ j.status = "completed";+ }+ });+ locked(&job).task = Some(handle.abort_handle());+ let base = req_uri.path().trim_end_matches('/').to_string();+ Json(json!({ "success": true, "id": id, "url": job_url(&headers, &base, &id) })).into_response()+}++async fn job_status(State(state): State<Arc<AppState>>, Path(id): Path<String>) -> Response {+ let job = find_job(&state, &id);+ let Some(job) = job else {+ return error_response(StatusCode::NOT_FOUND, "JOB_NOT_FOUND", "Job not found.");+ };+ let j = locked(&job);+ let expires = (JOB_TTL.saturating_sub(j.created.elapsed())).as_secs();+ Json(json!({+ "success": true,+ "status": j.status,+ "total": j.total.max(j.data.len()),+ "completed": j.data.len(),+ "creditsUsed": 0,+ "expiresInSeconds": expires,+ "errors": j.errors,+ "data": j.data,+ }))+ .into_response()+}++async fn job_cancel(State(state): State<Arc<AppState>>, Path(id): Path<String>) -> Response {+ let job = find_job(&state, &id);+ let Some(job) = job else {+ return error_response(StatusCode::NOT_FOUND, "JOB_NOT_FOUND", "Job not found.");+ };+ let mut j = locked(&job);+ if j.status == "scraping" {+ if let Some(task) = j.task.take() {+ task.abort();+ }+ j.status = "cancelled";+ }+ Json(json!({ "success": true, "status": "cancelled" })).into_response()+}++async fn scrape_handler(+ State(state): State<Arc<AppState>>,+ headers: HeaderMap,+ body: Bytes,+) -> Response {+ let payload = match json_body(&headers, &body) {+ Ok(v) => v,+ Err(r) => return r,+ };+ // An Alexandria provider call. This engine has no data providers of its+ // own, so the request goes to Firecrawl cloud as it came.+ if payload.get("alexandria").is_some() {+ return pass_on(&state, Method::POST, "/v2/scrape", &headers, body).await;+ }+ let url = match required_url(&payload) {+ Ok(u) => u,+ Err(r) => return r,+ };+ let origin = payload+ .get("origin")+ .and_then(Value::as_str)+ .map(str::to_string);+ let opts: ScrapeOptions = match serde_json::from_value(payload) {+ Ok(o) => o,+ Err(e) => {+ return error_response(+ StatusCode::BAD_REQUEST,+ "BAD_REQUEST",+ format!("Invalid options: {e}"),+ )+ }+ };+ // A page this machine cannot load is fetched by Firecrawl cloud under the+ // origin the client gave, so the call is counted as that client's.+ let cloud = match client_origin(&headers, origin.as_deref()) {+ Some(origin) => state.cloud.clone().with_origin(&origin),+ None => state.cloud.clone(),+ };+ match cloud::scrape_with_fallback(&state.engine, &cloud, &url, &opts).await {+ Ok(data) => Json(json!({ "success": true, "data": data })).into_response(),+ Err(e) => scrape_error(e),+ }+}++async fn health() -> impl IntoResponse {+ Json(json!({ "status": "ok", "engine": "firecrawl-local", "version": env!("CARGO_PKG_VERSION") }))+}++/// Resolves on Ctrl-C or, on Unix, SIGTERM, so the caller can stop the+/// browser before the process exits.+async fn shutdown_signal() {+ #[cfg(unix)]+ {+ use tokio::signal::unix::{signal, SignalKind};+ match signal(SignalKind::terminate()) {+ Ok(mut term) => {+ tokio::select! {+ _ = tokio::signal::ctrl_c() => {}+ _ = term.recv() => {}+ }+ }+ Err(_) => {+ let _ = tokio::signal::ctrl_c().await;+ }+ }+ }+ #[cfg(not(unix))]+ {+ let _ = tokio::signal::ctrl_c().await;+ }+}++pub async fn serve(engine: Arc<Engine>, cfg: ServeConfig) -> Result<(), String> {+ if !cfg.addr.ip().is_loopback() && cfg.token.is_none() {+ return Err(format!(+ "Refusing to listen on {} without a token. Anyone who can reach that address could use it. \+ Pass --token <secret> (or FIRECRAWL_LOCAL_TOKEN), or bind to 127.0.0.1.",+ cfg.addr+ ));+ }+ let listener = tokio::net::TcpListener::bind(cfg.addr)+ .await+ .map_err(|e| format!("could not listen on {}: {e}", cfg.addr))?;+ eprintln!("firecrawl-local listening on http://{}", cfg.addr);+ serve_on(listener, engine, cfg, shutdown_signal()).await+}++/// Serve the API on a listener the caller already bound, until `stop`+/// resolves. The MCP launcher uses this with a port the system picked.+pub async fn serve_on(+ listener: tokio::net::TcpListener,+ engine: Arc<Engine>,+ cfg: ServeConfig,+ stop: impl std::future::Future<Output = ()> + Send + 'static,+) -> Result<(), String> {+ let mut allowed_hosts = cfg.allowed_hosts.clone();+ if !cfg.addr.ip().is_unspecified() {+ allowed_hosts.push(cfg.addr.ip().to_string());+ }++ let state = Arc::new(AppState {+ engine,+ cloud: cfg.cloud,+ token: cfg.token,+ allowed_hosts,+ jobs: Mutex::new(HashMap::new()),+ });++ let app = Router::new()+ .route("/v2/scrape", post(scrape_handler))+ .route("/v2/map", post(map_handler))+ .route("/v2/crawl", post(crawl_handler))+ .route("/v2/crawl/{id}", get(job_status))+ .route("/v2/crawl/{id}", delete(job_cancel))+ .route("/v2/batch/scrape", post(batch_handler))+ .route("/v2/batch/scrape/{id}", get(job_status))+ .route("/v2/batch/scrape/{id}", delete(job_cancel))+ .route("/health", get(health))+ .fallback(forward_handler)+ .layer(middleware::from_fn_with_state(state.clone(), gate))+ .with_state(state);++ axum::serve(listener, app)+ .with_graceful_shutdown(stop)+ .await+ .map_err(|e| format!("server error: {e}"))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn host_checks() {+ assert!(host_allowed("localhost:3002", &[]));+ assert!(host_allowed("127.0.0.1:3002", &[]));+ assert!(host_allowed("[::1]:3002", &[]));+ assert!(!host_allowed("evil.example:3002", &[]));+ assert!(!host_allowed("", &[]));+ assert!(host_allowed("engine:3002", &["engine".to_string()]));+ }++ fn headers(pairs: &[(&str, &str)]) -> HeaderMap {+ let mut map = HeaderMap::new();+ for (name, value) in pairs {+ map.insert(+ axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(),+ value.parse().unwrap(),+ );+ }+ map+ }++ #[test]+ fn browser_requests_are_told_apart_from_tools() {+ assert!(!from_a_web_page(&headers(&[])));+ assert!(!from_a_web_page(&headers(&[("user-agent", "curl/8")])));+ assert!(from_a_web_page(&headers(&[(+ "origin",+ "https://evil.example"+ )])));+ assert!(from_a_web_page(&headers(&[("origin", "null")])));+ assert!(from_a_web_page(&headers(&[(+ "sec-fetch-site",+ "cross-site"+ )])));+ assert!(from_a_web_page(&headers(&[(+ "sec-fetch-site",+ "same-site"+ )])));+ assert!(!from_a_web_page(&headers(&[("sec-fetch-site", "none")])));+ }++ /// Start the server against a stand-in cloud and return its address.+ async fn start(mode: cloud::CloudMode) -> String {+ let engine = Arc::new(+ Engine::new(crate::scrape::EngineConfig {+ policy: crate::guard::NetworkPolicy::PublicOnly,+ user_agent: None,+ browser: crate::scrape::BrowserMode::Never,+ })+ .unwrap(),+ );+ let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))+ .await+ .unwrap();+ let addr = listener.local_addr().unwrap();+ let cfg = ServeConfig {+ addr,+ token: None,+ allowed_hosts: Vec::new(),+ cloud: Cloud::new(cloud::echo_cloud().await, Some("fc-own".into()), mode)+ .with_install_id("test-install"),+ };+ tokio::spawn(serve_on(listener, engine, cfg, std::future::pending()));+ format!("http://{addr}")+ }++ #[tokio::test]+ async fn a_web_page_cannot_spend_the_key_through_a_forwarded_route() {+ let base = start(cloud::CloudMode::Fallback).await;+ let http = reqwest::Client::new();++ // A form post or an upload from a page: no preflight, so it must be refused here.+ for content_type in [+ "text/plain",+ "multipart/form-data; boundary=x",+ "application/x-www-form-urlencoded",+ ] {+ let answer = http+ .post(format!("{base}/v2/search"))+ .header("origin", "https://evil.example")+ .header("content-type", content_type)+ .body("{\"query\":\"q\"}")+ .send()+ .await+ .unwrap();+ assert_eq!(answer.status().as_u16(), 403, "{content_type}");+ }+ let image_tag = http+ .get(format!("{base}/v2/team/credit-usage"))+ .header("sec-fetch-site", "cross-site")+ .send()+ .await+ .unwrap();+ assert_eq!(image_tag.status().as_u16(), 403);+ let preflight = http+ .request(reqwest::Method::OPTIONS, format!("{base}/v2/search"))+ .header("origin", "https://evil.example")+ .send()+ .await+ .unwrap();+ assert_eq!(preflight.status().as_u16(), 403);++ // The same upload from a tool goes through, and so does JSON.+ let upload = http+ .post(format!("{base}/v2/parse"))+ .header("content-type", "multipart/form-data; boundary=x")+ .body("--x--")+ .send()+ .await+ .unwrap();+ assert_eq!(upload.status().as_u16(), 207);+ let json = http+ .post(format!("{base}/v2/search"))+ .header("content-type", "application/json")+ .body("{\"query\":\"q\"}")+ .send()+ .await+ .unwrap();+ assert_eq!(json.status().as_u16(), 207);+ assert!(json.text().await.unwrap().contains("Bearer fc-own"));+ }++ #[tokio::test]+ async fn search_only_mode_cannot_be_walked_out_of() {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ let base = start(cloud::CloudMode::SearchOnly).await;+ // Sent raw: an HTTP client would tidy the path before it left.+ for path in ["/v2/search/../agent", "/v2/search/%2e%2e/agent"] {+ let mut socket = tokio::net::TcpStream::connect(base.trim_start_matches("http://"))+ .await+ .unwrap();+ let request = format!(+ "POST {path} HTTP/1.1\r\nhost: {}\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{{}}",+ base.trim_start_matches("http://")+ );+ socket.write_all(request.as_bytes()).await.unwrap();+ let mut answer = String::new();+ socket.read_to_string(&mut answer).await.unwrap();+ assert!(answer.starts_with("HTTP/1.1 400"), "{path}: {answer}");+ }+ let http = reqwest::Client::new();+ let agent = http+ .post(format!("{base}/v2/agent"))+ .header("content-type", "application/json")+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(agent.status().as_u16(), 501);+ let search = http+ .post(format!("{base}/v2/search"))+ .header("content-type", "application/json")+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(search.status().as_u16(), 207);+ }++ #[tokio::test]+ async fn a_forwarded_upload_may_be_larger_than_two_megabytes() {+ let base = start(cloud::CloudMode::Fallback).await;+ let answer = reqwest::Client::new()+ .post(format!("{base}/v2/parse"))+ .header("content-type", "application/octet-stream")+ .body(vec![b'a'; 3 * 1024 * 1024])+ .send()+ .await+ .unwrap();+ assert_eq!(answer.status().as_u16(), 207);+ }++ #[test]+ fn token_compare() {+ assert!(constant_time_eq(b"abc", b"abc"));+ assert!(!constant_time_eq(b"abc", b"abd"));+ assert!(!constant_time_eq(b"abc", b"ab"));+ }+}