Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(rust-sdk): upgrade to official v2 SDK
firecrawl-spring[bot]1c40db842426
diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.pyindex 34c56d838..7c4f4a21e 100644--- a/.github/scripts/check_version_has_incremented.py+++ b/.github/scripts/check_version_has_incremented.py@@ -82,18 +82,25 @@ def get_maven_central_version(package_name: str) -> str:return version_match.group(1).strip()return "0.0.0"-# def get_rust_version(file_path: str) -> str:-# """Extract version string from Cargo.toml."""-# cargo_toml = toml.load(file_path)-# if 'package' in cargo_toml and 'version' in cargo_toml['package']:-# return cargo_toml['package']['version'].strip()-# raise RuntimeError("Unable to find version string in Cargo.toml.")--# def get_crates_version(package_name: str) -> str:-# """Get latest version of Rust package from crates.io."""-# response = requests.get(f"https://crates.io/api/v1/crates/{package_name}")-# version = response.json()['crate']['newest_version']-# return version.strip()+def get_rust_version(file_path: str) -> str:+ """Extract version string from Cargo.toml."""+ import toml+ cargo_toml = toml.load(file_path)+ if 'package' in cargo_toml and 'version' in cargo_toml['package']:+ return cargo_toml['package']['version'].strip()+ raise RuntimeError("Unable to find version string in Cargo.toml.")++def get_crates_version(package_name: str) -> str:+ """Get latest version of Rust package from crates.io."""+ response = requests.get(+ f"https://crates.io/api/v1/crates/{package_name}",+ headers={"User-Agent": "firecrawl-version-check"}+ )+ if response.status_code == 404:+ return "0.0.0"+ response.raise_for_status()+ version = response.json()['crate']['newest_version']+ return version.strip()def is_version_incremented(local_version: str, published_version: str) -> bool:"""Compare local and published versions."""@@ -121,14 +128,14 @@ if __name__ == "__main__":current_version = get_gradle_version(os.path.join(package_path, 'build.gradle.kts'))# Get published version from Maven Centralpublished_version = get_maven_central_version(package_name)- # if package_type == "rust":- # # Get current version from Cargo.toml- # current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))- # # Get published version from crates.io- # published_version = get_crates_version(package_name)+ elif package_type == "rust":+ # Get current version from Cargo.toml+ current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))+ # Get published version from crates.io+ published_version = get_crates_version(package_name)else:- raise ValueError("Invalid package type. Use 'python', 'js', or 'java'.")+ raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'rust'.")# Print versions for debugging# print(f"Local version: {current_version}")diff --git a/.github/workflows/publish-rust-sdk.yml b/.github/workflows/publish-rust-sdk.ymlnew file mode 100644index 000000000..a005553e0--- /dev/null+++ b/.github/workflows/publish-rust-sdk.yml@@ -0,0 +1,64 @@+name: Publish Rust SDK++on:+ workflow_dispatch:+ push:+ branches:+ - main+ paths:+ - 'apps/rust-sdk/**'++jobs:+ publish:+ name: Publish to crates.io+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: '3.x'++ - name: Install version check dependencies+ run: pip install requests packaging toml++ - name: Run version check script+ id: version_check+ run: |+ VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py rust ./apps/rust-sdk firecrawl)+ echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV++ - name: Install Rust toolchain+ if: ${{ env.VERSION_INCREMENTED == 'true' }}+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable++ - name: Cache cargo registry+ if: ${{ env.VERSION_INCREMENTED == 'true' }}+ uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ apps/rust-sdk/target+ key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}+ restore-keys: |+ ${{ runner.os }}-cargo-++ - name: Build+ if: ${{ env.VERSION_INCREMENTED == 'true' }}+ working-directory: ./apps/rust-sdk+ run: cargo build --release++ - name: Run tests+ if: ${{ env.VERSION_INCREMENTED == 'true' }}+ working-directory: ./apps/rust-sdk+ run: cargo test --lib++ - name: Publish to crates.io+ if: ${{ env.VERSION_INCREMENTED == 'true' }}+ working-directory: ./apps/rust-sdk+ env:+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}+ run: cargo publishdiff --git a/README.md b/README.mdindex 239d3577b..eb818202d 100644--- a/README.md+++ b/README.md@@ -655,10 +655,43 @@ end{:ok, response} = Firecrawl.map_urls(url: "https://example.com")```+### Rust++Add the dependency:+```toml+[dependencies]+firecrawl = "2"+```+```rust+use firecrawl::{Client, ScrapeOptions, Format, CrawlOptions};++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let client = Client::new("fc-YOUR_API_KEY")?;++ // Scrape a URL+ let document = client.scrape("https://firecrawl.dev", None).await?;+ println!("{:?}", document.markdown);++ // Crawl a website+ let options = CrawlOptions {+ limit: Some(50),+ ..Default::default()+ };+ let result = client.crawl("https://docs.firecrawl.dev", options).await?;+ println!("Crawled {} pages", result.data.len());++ // Search the web+ let response = client.search("best web scraping tools 2024", None).await?;+ println!("{:?}", response.data);++ Ok(())+}+```+### Community SDKs- [Go SDK](https://github.com/mendableai/firecrawl-go)-- [Rust SDK](https://docs.firecrawl.dev/sdks/rust)---diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 120fe7e5e..701d04b9b 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -41,83 +41,6 @@ dependencies = ["libc",]-[[package]]-name = "ansi_colours"-version = "1.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "14eec43e0298190790f41679fe69ef7a829d2a2ddd78c8c00339e84710e435fe"-dependencies = [- "rgb",-]--[[package]]-name = "anstream"-version = "0.6.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"-dependencies = [- "anstyle",- "anstyle-parse",- "anstyle-query",- "anstyle-wincon",- "colorchoice",- "is_terminal_polyfill",- "utf8parse",-]--[[package]]-name = "anstyle"-version = "1.0.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"--[[package]]-name = "anstyle-parse"-version = "0.2.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"-dependencies = [- "utf8parse",-]--[[package]]-name = "anstyle-query"-version = "1.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"-dependencies = [- "windows-sys 0.60.2",-]--[[package]]-name = "anstyle-wincon"-version = "3.0.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"-dependencies = [- "anstyle",- "once_cell_polyfill",- "windows-sys 0.60.2",-]--[[package]]-name = "anyhow"-version = "1.0.99"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"--[[package]]-name = "arrayref"-version = "0.3.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"--[[package]]-name = "arrayvec"-version = "0.5.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"-[[package]]name = "assert-json-diff"version = "2.0.2"@@ -128,12 +51,6 @@ dependencies = ["serde_json",]-[[package]]-name = "assert_matches"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9"-[[package]]name = "atomic-waker"version = "1.1.2"@@ -146,72 +63,6 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"-[[package]]-name = "axum"-version = "0.8.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"-dependencies = [- "axum-core",- "axum-macros",- "bytes",- "form_urlencoded",- "futures-util",- "http",- "http-body",- "http-body-util",- "hyper",- "hyper-util",- "itoa",- "matchit",- "memchr",- "mime",- "percent-encoding",- "pin-project-lite",- "rustversion",- "serde",- "serde_json",- "serde_path_to_error",- "serde_urlencoded",- "sync_wrapper",- "tokio",- "tower",- "tower-layer",- "tower-service",- "tracing",-]--[[package]]-name = "axum-core"-version = "0.5.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"-dependencies = [- "bytes",- "futures-core",- "http",- "http-body",- "http-body-util",- "mime",- "pin-project-lite",- "rustversion",- "sync_wrapper",- "tower-layer",- "tower-service",- "tracing",-]--[[package]]-name = "axum-macros"-version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "backtrace"version = "0.3.75"@@ -227,154 +78,36 @@ dependencies = ["windows-targets 0.52.6",]-[[package]]-name = "base64"-version = "0.13.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"-[[package]]name = "base64"version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"-[[package]]-name = "bat"-version = "0.25.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ab792c2ad113a666f08856c88cdec0a62d732559b1f3982eedf0142571e669a"-dependencies = [- "ansi_colours",- "anyhow",- "bincode",- "bugreport",- "bytesize",- "clap",- "clircle",- "console",- "content_inspector",- "encoding_rs",- "etcetera",- "flate2",- "git2",- "globset",- "grep-cli",- "home",- "indexmap 2.10.0",- "itertools",- "nu-ansi-term",- "once_cell",- "path_abs",- "plist",- "regex",- "semver",- "serde",- "serde_derive",- "serde_with",- "serde_yaml",- "shell-words",- "syntect",- "terminal-colorsaurus",- "thiserror",- "toml",- "unicode-width 0.1.14",- "walkdir",- "wild",-]--[[package]]-name = "bincode"-version = "1.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"-dependencies = [- "serde",-]--[[package]]-name = "bitflags"-version = "1.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"-[[package]]name = "bitflags"version = "2.9.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"-[[package]]-name = "blake2b_simd"-version = "0.5.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"-dependencies = [- "arrayref",- "arrayvec",- "constant_time_eq",-]--[[package]]-name = "bstr"-version = "1.12.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"-dependencies = [- "memchr",- "regex-automata",- "serde",-]--[[package]]-name = "bugreport"-version = "0.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f280f65ce85b880919349bbfcb204930291251eedcb2e5f84ce2f51df969c162"-dependencies = [- "git-version",- "shell-escape",- "sysinfo",-]-[[package]]name = "bumpalo"version = "3.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"-[[package]]-name = "bytemuck"-version = "1.23.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677"--[[package]]-name = "byteorder"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"-[[package]]name = "bytes"version = "1.10.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"-[[package]]-name = "bytesize"-version = "1.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"-[[package]]name = "cc"version = "1.2.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e"dependencies = [- "jobserver",- "libc","shlex",]@@ -397,72 +130,6 @@ dependencies = ["windows-link",]-[[package]]-name = "clap"-version = "4.5.44"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1c1f056bae57e3e54c3375c41ff79619ddd13460a17d7438712bd0d83fda4ff8"-dependencies = [- "clap_builder",- "clap_derive",-]--[[package]]-name = "clap_builder"-version = "4.5.44"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"-dependencies = [- "anstream",- "anstyle",- "clap_lex",- "strsim",- "terminal_size",-]--[[package]]-name = "clap_derive"-version = "4.5.41"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"-dependencies = [- "heck",- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "clap_lex"-version = "0.7.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"--[[package]]-name = "clippy"-version = "0.0.302"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d911ee15579a3f50880d8c1d59ef6e79f9533127a3bd342462f5d584f5e8c294"-dependencies = [- "term",-]--[[package]]-name = "clircle"-version = "0.6.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7d9334f725b46fb9bed8580b9b47a932587e044fadb344ed7fa98774b067ac1a"-dependencies = [- "cfg-if",- "windows 0.56.0",-]--[[package]]-name = "colorchoice"-version = "1.0.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"-[[package]]name = "colored"version = "3.0.0"@@ -472,34 +139,6 @@ dependencies = ["windows-sys 0.59.0",]-[[package]]-name = "console"-version = "0.15.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"-dependencies = [- "encode_unicode",- "libc",- "once_cell",- "unicode-width 0.2.1",- "windows-sys 0.59.0",-]--[[package]]-name = "constant_time_eq"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"--[[package]]-name = "content_inspector"-version = "0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38"-dependencies = [- "memchr",-]-[[package]]name = "core-foundation"version = "0.9.4"@@ -516,40 +155,6 @@ version = "0.8.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"-[[package]]-name = "crc32fast"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"-dependencies = [- "cfg-if",-]--[[package]]-name = "crossbeam-deque"-version = "0.8.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"-dependencies = [- "crossbeam-epoch",- "crossbeam-utils",-]--[[package]]-name = "crossbeam-epoch"-version = "0.9.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"-dependencies = [- "crossbeam-utils",-]--[[package]]-name = "crossbeam-utils"-version = "0.8.21"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"-[[package]]name = "darling"version = "0.20.11"@@ -595,17 +200,6 @@ dependencies = ["serde",]-[[package]]-name = "dirs"-version = "1.0.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"-dependencies = [- "libc",- "redox_users",- "winapi",-]-[[package]]name = "displaydoc"version = "0.2.5"@@ -617,30 +211,12 @@ dependencies = ["syn",]-[[package]]-name = "dotenvy"-version = "0.15.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"-[[package]]name = "dyn-clone"version = "1.0.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"-[[package]]-name = "either"-version = "1.15.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"--[[package]]-name = "encode_unicode"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"-[[package]]name = "encoding_rs"version = "0.8.35"@@ -666,17 +242,6 @@ dependencies = ["windows-sys 0.60.2",]-[[package]]-name = "etcetera"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"-dependencies = [- "cfg-if",- "home",- "windows-sys 0.48.0",-]-[[package]]name = "fastrand"version = "2.3.0"@@ -685,36 +250,15 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "1.2.3"-dependencies = [- "assert_matches",- "axum",- "bat",- "clap",- "clippy",- "dotenvy",- "futures",- "log",+version = "2.0.0"+dependencies = ["mockito","reqwest",- "schemars 0.8.22","serde","serde_json","serde_with","thiserror","tokio",- "tracing",- "uuid",-]--[[package]]-name = "flate2"-version = "1.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d"-dependencies = [- "crc32fast",- "miniz_oxide",][[package]]@@ -747,21 +291,6 @@ dependencies = ["percent-encoding",]-[[package]]-name = "futures"-version = "0.3.31"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"-dependencies = [- "futures-channel",- "futures-core",- "futures-executor",- "futures-io",- "futures-sink",- "futures-task",- "futures-util",-]-[[package]]name = "futures-channel"version = "0.3.31"@@ -769,7 +298,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"dependencies = ["futures-core",- "futures-sink",][[package]]@@ -778,34 +306,6 @@ version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"-[[package]]-name = "futures-executor"-version = "0.3.31"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"-dependencies = [- "futures-core",- "futures-task",- "futures-util",-]--[[package]]-name = "futures-io"-version = "0.3.31"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"--[[package]]-name = "futures-macro"-version = "0.3.31"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "futures-sink"version = "0.3.31"@@ -824,27 +324,10 @@ version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"dependencies = [- "futures-channel","futures-core",- "futures-io",- "futures-macro",- "futures-sink","futures-task",- "memchr","pin-project-lite","pin-utils",- "slab",-]--[[package]]-name = "getrandom"-version = "0.1.16"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"-dependencies = [- "cfg-if",- "libc",- "wasi 0.9.0+wasi-snapshot-preview1",][[package]]@@ -876,72 +359,6 @@ version = "0.31.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"-[[package]]-name = "git-version"-version = "0.3.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19"-dependencies = [- "git-version-macro",-]--[[package]]-name = "git-version-macro"-version = "0.3.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "git2"-version = "0.19.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"-dependencies = [- "bitflags 2.9.1",- "libc",- "libgit2-sys",- "log",- "url",-]--[[package]]-name = "glob"-version = "0.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"--[[package]]-name = "globset"-version = "0.4.16"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"-dependencies = [- "aho-corasick",- "bstr",- "log",- "regex-automata",- "regex-syntax",-]--[[package]]-name = "grep-cli"-version = "0.1.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "47f1288f0e06f279f84926fa4c17e3fcd2a22b357927a82f2777f7be26e4cec0"-dependencies = [- "bstr",- "globset",- "libc",- "log",- "termcolor",- "winapi-util",-]-[[package]]name = "h2"version = "0.4.12"@@ -973,27 +390,12 @@ version = "0.15.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"-[[package]]-name = "heck"-version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"-[[package]]name = "hex"version = "0.4.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"-[[package]]-name = "home"-version = "0.5.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"-dependencies = [- "windows-sys 0.59.0",-]-[[package]]name = "http"version = "1.3.1"@@ -1099,7 +501,7 @@ version = "0.1.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"dependencies = [- "base64 0.22.1",+ "base64","bytes","futures-channel","futures-core",@@ -1131,7 +533,7 @@ dependencies = ["js-sys","log","wasm-bindgen",- "windows-core 0.61.2",+ "windows-core",][[package]]@@ -1284,7 +686,7 @@ version = "0.7.9"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"dependencies = [- "bitflags 2.9.1",+ "bitflags","cfg-if","libc",]@@ -1305,37 +707,12 @@ dependencies = ["serde",]-[[package]]-name = "is_terminal_polyfill"-version = "1.70.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"--[[package]]-name = "itertools"-version = "0.13.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"-dependencies = [- "either",-]-[[package]]name = "itoa"version = "1.0.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"-[[package]]-name = "jobserver"-version = "0.1.33"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"-dependencies = [- "getrandom 0.3.3",- "libc",-]-[[package]]name = "js-sys"version = "0.3.77"@@ -1352,36 +729,6 @@ version = "0.2.175"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"-[[package]]-name = "libgit2-sys"-version = "0.17.0+1.8.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224"-dependencies = [- "cc",- "libc",- "libz-sys",- "pkg-config",-]--[[package]]-name = "libz-sys"-version = "1.1.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"-dependencies = [- "cc",- "libc",- "pkg-config",- "vcpkg",-]--[[package]]-name = "linked-hash-map"-version = "0.5.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"-[[package]]name = "linux-raw-sys"version = "0.9.4"@@ -1410,12 +757,6 @@ version = "0.4.27"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"-[[package]]-name = "matchit"-version = "0.8.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"-[[package]]name = "memchr"version = "2.7.5"@@ -1489,24 +830,6 @@ dependencies = ["tempfile",]-[[package]]-name = "ntapi"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"-dependencies = [- "winapi",-]--[[package]]-name = "nu-ansi-term"-version = "0.50.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"-dependencies = [- "windows-sys 0.52.0",-]-[[package]]name = "num-conv"version = "0.1.0"@@ -1537,41 +860,13 @@ version = "1.21.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"-[[package]]-name = "once_cell_polyfill"-version = "1.70.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"--[[package]]-name = "onig"-version = "6.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"-dependencies = [- "bitflags 2.9.1",- "libc",- "once_cell",- "onig_sys",-]--[[package]]-name = "onig_sys"-version = "69.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"-dependencies = [- "cc",- "pkg-config",-]-[[package]]name = "openssl"version = "0.10.73"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8"dependencies = [- "bitflags 2.9.1",+ "bitflags","cfg-if","foreign-types","libc",@@ -1627,20 +922,11 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"dependencies = ["cfg-if","libc",- "redox_syscall 0.5.17",+ "redox_syscall","smallvec","windows-targets 0.52.6",]-[[package]]-name = "path_abs"-version = "0.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3"-dependencies = [- "std_prelude",-]-[[package]]name = "percent-encoding"version = "2.3.1"@@ -1665,19 +951,6 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"-[[package]]-name = "plist"-version = "1.7.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1"-dependencies = [- "base64 0.22.1",- "indexmap 2.10.0",- "quick-xml",- "serde",- "time",-]-[[package]]name = "potential_utf"version = "0.1.2"@@ -1711,15 +984,6 @@ dependencies = ["unicode-ident",]-[[package]]-name = "quick-xml"-version = "0.38.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9845d9dccf565065824e69f9f235fafba1587031eda353c1f1561cd6a6be78f4"-dependencies = [- "memchr",-]-[[package]]name = "quote"version = "1.0.40"@@ -1764,50 +1028,13 @@ dependencies = ["getrandom 0.3.3",]-[[package]]-name = "rayon"-version = "1.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"-dependencies = [- "either",- "rayon-core",-]--[[package]]-name = "rayon-core"-version = "1.12.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"-dependencies = [- "crossbeam-deque",- "crossbeam-utils",-]--[[package]]-name = "redox_syscall"-version = "0.1.57"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"-[[package]]name = "redox_syscall"version = "0.5.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"dependencies = [- "bitflags 2.9.1",-]--[[package]]-name = "redox_users"-version = "0.3.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"-dependencies = [- "getrandom 0.1.16",- "redox_syscall 0.1.57",- "rust-argon2",+ "bitflags",][[package]]@@ -1865,12 +1092,10 @@ version = "0.12.23"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"dependencies = [- "base64 0.22.1",+ "base64","bytes","encoding_rs",- "futures-channel","futures-core",- "futures-util","h2","http","http-body",@@ -1901,15 +1126,6 @@ dependencies = ["web-sys",]-[[package]]-name = "rgb"-version = "0.8.52"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce"-dependencies = [- "bytemuck",-]-[[package]]name = "ring"version = "0.17.14"@@ -1924,18 +1140,6 @@ dependencies = ["windows-sys 0.52.0",]-[[package]]-name = "rust-argon2"-version = "0.8.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"-dependencies = [- "base64 0.13.1",- "blake2b_simd",- "constant_time_eq",- "crossbeam-utils",-]-[[package]]name = "rustc-demangle"version = "0.1.26"@@ -1948,7 +1152,7 @@ version = "1.0.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"dependencies = [- "bitflags 2.9.1",+ "bitflags","errno","libc","linux-raw-sys",@@ -2000,15 +1204,6 @@ version = "1.0.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"-[[package]]-name = "same-file"-version = "1.0.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"-dependencies = [- "winapi-util",-]-[[package]]name = "schannel"version = "0.1.27"@@ -2018,18 +1213,6 @@ dependencies = ["windows-sys 0.59.0",]-[[package]]-name = "schemars"-version = "0.8.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"-dependencies = [- "dyn-clone",- "schemars_derive",- "serde",- "serde_json",-]-[[package]]name = "schemars"version = "0.9.0"@@ -2054,18 +1237,6 @@ dependencies = ["serde_json",]-[[package]]-name = "schemars_derive"-version = "0.8.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"-dependencies = [- "proc-macro2",- "quote",- "serde_derive_internals",- "syn",-]-[[package]]name = "scopeguard"version = "1.2.0"@@ -2078,7 +1249,7 @@ version = "2.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"dependencies = [- "bitflags 2.9.1",+ "bitflags","core-foundation","core-foundation-sys","libc",@@ -2095,12 +1266,6 @@ dependencies = ["libc",]-[[package]]-name = "semver"-version = "1.0.26"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"-[[package]]name = "serde"version = "1.0.219"@@ -2121,17 +1286,6 @@ dependencies = ["syn",]-[[package]]-name = "serde_derive_internals"-version = "0.29.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "serde_json"version = "1.0.142"@@ -2144,25 +1298,6 @@ dependencies = ["serde",]-[[package]]-name = "serde_path_to_error"-version = "0.1.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"-dependencies = [- "itoa",- "serde",-]--[[package]]-name = "serde_spanned"-version = "0.6.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"-dependencies = [- "serde",-]-[[package]]name = "serde_urlencoded"version = "0.7.1"@@ -2181,7 +1316,7 @@ version = "3.14.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5"dependencies = [- "base64 0.22.1",+ "base64","chrono","hex","indexmap 1.9.3",@@ -2207,31 +1342,6 @@ dependencies = ["syn",]-[[package]]-name = "serde_yaml"-version = "0.9.34+deprecated"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"-dependencies = [- "indexmap 2.10.0",- "itoa",- "ryu",- "serde",- "unsafe-libyaml",-]--[[package]]-name = "shell-escape"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"--[[package]]-name = "shell-words"-version = "1.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"-[[package]]name = "shlex"version = "1.3.0"@@ -2281,12 +1391,6 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"-[[package]]-name = "std_prelude"-version = "0.2.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe"-[[package]]name = "strsim"version = "0.11.1"@@ -2323,47 +1427,11 @@ dependencies = [name = "synstructure"version = "0.13.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "syntect"-version = "5.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"-dependencies = [- "bincode",- "bitflags 1.3.2",- "flate2",- "fnv",- "once_cell",- "onig",- "plist",- "regex-syntax",- "serde",- "serde_derive",- "serde_json",- "thiserror",- "walkdir",- "yaml-rust",-]--[[package]]-name = "sysinfo"-version = "0.33.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"-dependencies = [- "core-foundation-sys",- "libc",- "memchr",- "ntapi",- "rayon",- "windows 0.57.0",+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",][[package]]@@ -2372,7 +1440,7 @@ version = "0.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"dependencies = [- "bitflags 2.9.1",+ "bitflags","core-foundation","system-configuration-sys",]@@ -2400,62 +1468,6 @@ dependencies = ["windows-sys 0.59.0",]-[[package]]-name = "term"-version = "0.5.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42"-dependencies = [- "byteorder",- "dirs",- "winapi",-]--[[package]]-name = "termcolor"-version = "1.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"-dependencies = [- "winapi-util",-]--[[package]]-name = "terminal-colorsaurus"-version = "0.4.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b7afe4c174a3cbfb52ebcb11b28965daf74fe9111d4e07e40689d05af06e26e8"-dependencies = [- "cfg-if",- "libc",- "memchr",- "mio",- "terminal-trx",- "windows-sys 0.59.0",- "xterm-color",-]--[[package]]-name = "terminal-trx"-version = "0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "975b4233aefa1b02456d5e53b22c61653c743e308c51cf4181191d8ce41753ab"-dependencies = [- "cfg-if",- "libc",- "windows-sys 0.59.0",-]--[[package]]-name = "terminal_size"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"-dependencies = [- "rustix",- "windows-sys 0.60.2",-]-[[package]]name = "thiserror"version = "1.0.69"@@ -2581,48 +1593,6 @@ dependencies = ["tokio",]-[[package]]-name = "toml"-version = "0.8.23"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"-dependencies = [- "indexmap 2.10.0",- "serde",- "serde_spanned",- "toml_datetime",- "toml_edit",-]--[[package]]-name = "toml_datetime"-version = "0.6.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"-dependencies = [- "serde",-]--[[package]]-name = "toml_edit"-version = "0.22.27"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"-dependencies = [- "indexmap 2.10.0",- "serde",- "serde_spanned",- "toml_datetime",- "toml_write",- "winnow",-]--[[package]]-name = "toml_write"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"-[[package]]name = "tower"version = "0.5.2"@@ -2636,7 +1606,6 @@ dependencies = ["tokio","tower-layer","tower-service",- "tracing",][[package]]@@ -2645,7 +1614,7 @@ version = "0.6.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"dependencies = [- "bitflags 2.9.1",+ "bitflags","bytes","futures-util","http",@@ -2675,23 +1644,10 @@ version = "0.1.41"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"dependencies = [- "log","pin-project-lite",- "tracing-attributes","tracing-core",]-[[package]]-name = "tracing-attributes"-version = "0.1.30"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "tracing-core"version = "0.1.34"@@ -2713,24 +1669,6 @@ version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"-[[package]]-name = "unicode-width"-version = "0.1.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"--[[package]]-name = "unicode-width"-version = "0.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"--[[package]]-name = "unsafe-libyaml"-version = "0.2.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"-[[package]]name = "untrusted"version = "0.9.0"@@ -2754,39 +1692,12 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"-[[package]]-name = "utf8parse"-version = "0.2.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"--[[package]]-name = "uuid"-version = "1.18.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be"-dependencies = [- "getrandom 0.3.3",- "js-sys",- "wasm-bindgen",-]-[[package]]name = "vcpkg"version = "0.2.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"-[[package]]-name = "walkdir"-version = "2.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"-dependencies = [- "same-file",- "winapi-util",-]-[[package]]name = "want"version = "0.3.1"@@ -2796,12 +1707,6 @@ dependencies = ["try-lock",]-[[package]]-name = "wasi"-version = "0.9.0+wasi-snapshot-preview1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"-[[package]]name = "wasi"version = "0.11.1+wasi-snapshot-preview1"@@ -2898,125 +1803,19 @@ dependencies = ["wasm-bindgen",]-[[package]]-name = "wild"-version = "2.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1"-dependencies = [- "glob",-]--[[package]]-name = "winapi"-version = "0.3.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"-dependencies = [- "winapi-i686-pc-windows-gnu",- "winapi-x86_64-pc-windows-gnu",-]--[[package]]-name = "winapi-i686-pc-windows-gnu"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"--[[package]]-name = "winapi-util"-version = "0.1.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"-dependencies = [- "windows-sys 0.59.0",-]--[[package]]-name = "winapi-x86_64-pc-windows-gnu"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"--[[package]]-name = "windows"-version = "0.56.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132"-dependencies = [- "windows-core 0.56.0",- "windows-targets 0.52.6",-]--[[package]]-name = "windows"-version = "0.57.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"-dependencies = [- "windows-core 0.57.0",- "windows-targets 0.52.6",-]--[[package]]-name = "windows-core"-version = "0.56.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6"-dependencies = [- "windows-implement 0.56.0",- "windows-interface 0.56.0",- "windows-result 0.1.2",- "windows-targets 0.52.6",-]--[[package]]-name = "windows-core"-version = "0.57.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"-dependencies = [- "windows-implement 0.57.0",- "windows-interface 0.57.0",- "windows-result 0.1.2",- "windows-targets 0.52.6",-]-[[package]]name = "windows-core"version = "0.61.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"dependencies = [- "windows-implement 0.60.0",- "windows-interface 0.59.1",+ "windows-implement",+ "windows-interface","windows-link",- "windows-result 0.3.4",+ "windows-result","windows-strings",]-[[package]]-name = "windows-implement"-version = "0.56.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "windows-implement"-version = "0.57.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "windows-implement"version = "0.60.0"@@ -3028,28 +1827,6 @@ dependencies = ["syn",]-[[package]]-name = "windows-interface"-version = "0.56.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "windows-interface"-version = "0.57.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"-dependencies = [- "proc-macro2",- "quote",- "syn",-]-[[package]]name = "windows-interface"version = "0.59.1"@@ -3074,19 +1851,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e"dependencies = ["windows-link",- "windows-result 0.3.4",+ "windows-result","windows-strings",]-[[package]]-name = "windows-result"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"-dependencies = [- "windows-targets 0.52.6",-]-[[package]]name = "windows-result"version = "0.3.4"@@ -3105,15 +1873,6 @@ dependencies = ["windows-link",]-[[package]]-name = "windows-sys"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"-dependencies = [- "windows-targets 0.48.5",-]-[[package]]name = "windows-sys"version = "0.52.0"@@ -3141,21 +1900,6 @@ dependencies = ["windows-targets 0.53.3",]-[[package]]-name = "windows-targets"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"-dependencies = [- "windows_aarch64_gnullvm 0.48.5",- "windows_aarch64_msvc 0.48.5",- "windows_i686_gnu 0.48.5",- "windows_i686_msvc 0.48.5",- "windows_x86_64_gnu 0.48.5",- "windows_x86_64_gnullvm 0.48.5",- "windows_x86_64_msvc 0.48.5",-]-[[package]]name = "windows-targets"version = "0.52.6"@@ -3189,12 +1933,6 @@ dependencies = ["windows_x86_64_msvc 0.53.0",]-[[package]]-name = "windows_aarch64_gnullvm"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"-[[package]]name = "windows_aarch64_gnullvm"version = "0.52.6"@@ -3207,12 +1945,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"-[[package]]-name = "windows_aarch64_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"-[[package]]name = "windows_aarch64_msvc"version = "0.52.6"@@ -3225,12 +1957,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"-[[package]]-name = "windows_i686_gnu"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"-[[package]]name = "windows_i686_gnu"version = "0.52.6"@@ -3255,12 +1981,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"-[[package]]-name = "windows_i686_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"-[[package]]name = "windows_i686_msvc"version = "0.52.6"@@ -3273,12 +1993,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"-[[package]]-name = "windows_x86_64_gnu"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"-[[package]]name = "windows_x86_64_gnu"version = "0.52.6"@@ -3291,12 +2005,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"-[[package]]-name = "windows_x86_64_gnullvm"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"-[[package]]name = "windows_x86_64_gnullvm"version = "0.52.6"@@ -3309,12 +2017,6 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"-[[package]]-name = "windows_x86_64_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"-[[package]]name = "windows_x86_64_msvc"version = "0.52.6"@@ -3327,22 +2029,13 @@ version = "0.53.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"-[[package]]-name = "winnow"-version = "0.7.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"-dependencies = [- "memchr",-]-[[package]]name = "wit-bindgen-rt"version = "0.39.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"dependencies = [- "bitflags 2.9.1",+ "bitflags",][[package]]@@ -3351,21 +2044,6 @@ version = "0.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"-[[package]]-name = "xterm-color"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4de5f056fb9dc8b7908754867544e26145767187aaac5a98495e88ad7cb8a80f"--[[package]]-name = "yaml-rust"-version = "0.4.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"-dependencies = [- "linked-hash-map",-]-[[package]]name = "yoke"version = "0.8.0"diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex c47284047..b1b74cee5 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,41 +1,28 @@[package]name = "firecrawl"-author= "Mendable.ai"-version = "1.4.0"+version = "2.0.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"-repository ="https://github.com/firecrawl/firecrawl"-description = "Rust SDK for Firecrawl API."+repository = "https://github.com/firecrawl/firecrawl"+description = "Official Rust SDK for Firecrawl API v2."authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>", "kkharji <[email protected]>"]+keywords = ["firecrawl", "web-scraping", "crawling", "api"]+categories = ["api-bindings", "web-programming"][lib]path = "src/lib.rs"name = "firecrawl"-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[dependencies]-reqwest = { version = "0.12.22", features = ["json", "blocking"] }-serde = { version = "^1.0", features = ["derive"] }-serde_json = "^1.0"-serde_with = "^3.9"-log = "^0.4"-thiserror = "^1.0"-uuid = { version = "^1.10", features = ["v4"] }-tokio = { version = "^1", features = ["full"] }-futures = "0.3.31"-schemars = "0.8.22"-tracing = ">=0.1.0,<0.2.0"+reqwest = { version = "0.12", features = ["json"] }+serde = { version = "1", features = ["derive"] }+serde_json = "1"+serde_with = "3"+thiserror = "1"+tokio = { version = "1", features = ["time"] }[dev-dependencies]-clippy = "^0.0.302"-assert_matches = "^1.5"-dotenvy = "^0.15"-tokio = { version = "1", features = ["full"] }-mockito = "1.7.0"-clap = { version ="4.5.35", features = ["derive"] }-axum = { version = "0.8.3", features = ["tokio", "macros"] }-bat = "0.25.0"--[build-dependencies]tokio = { version = "1", features = ["full"] }+mockito = "1.7"+serde_json = "1"diff --git a/apps/rust-sdk/examples/batch_scrape_example.rs b/apps/rust-sdk/examples/batch_scrape_example.rsdeleted file mode 100644index 7db7856ec..000000000--- a/apps/rust-sdk/examples/batch_scrape_example.rs+++ /dev/null@@ -1,175 +0,0 @@-use clap::{Parser, Subcommand};-use firecrawl::{- batch_scrape::{BatchScrapeParams, WebhookOptions},- map::MapOptions,- scrape::{ScrapeFormats, ScrapeOptions},- FirecrawlApp,-};-use serde_json::Value;-use std::error::Error;-use std::net::SocketAddr;-use std::sync::Arc;-use tokio::sync::Mutex;--// Store webhook responses-struct WebhookState {- responses: Vec<Value>,-}--#[derive(Parser)]-#[command(version, about, long_about = None)]-struct Cli {- #[command(subcommand)]- command: Commands,-}--#[derive(Subcommand)]-enum Commands {- /// Multiple URL scraping with webhook monitoring- Basic,-}--async fn create_firecrawl_app() -> Result<FirecrawlApp, Box<dyn Error>> {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- FirecrawlApp::new_selfhosted(api_url, None::<&str>).map_err(|e| e.into())-}--// Start webhook server and return its address-async fn start_webhook_server(- port: u16,- state: Arc<Mutex<WebhookState>>,-) -> Result<String, Box<dyn Error>> {- let state = state.clone();- use axum::routing::post;- use axum::Json;-- let app = axum::Router::new().route(- "/",- post(move |body: Json<Value>| {- let state = state.clone();- async move {- state.lock().await.responses.push(body.0.clone());- match serde_json::to_string_pretty(&body.0) {- Ok(data) => println!(- "Received webhook: {}",- serde_json::to_string_pretty(&data).unwrap()- ),- Err(_) => println!("Received webhook: {}", body.0),- }- "OK"- }- }),- );-- let addr = SocketAddr::from(([0, 0, 0, 0], port));- let webhook_url = format!("http://host.docker.internal:{}", port);-- tokio::spawn(async move {- let listener = tokio::net::TcpListener::bind(addr)- .await- .inspect_err(|err| println!("{err:?}"))- .unwrap();-- if let Err(e) = axum::serve(listener, app).await {- eprintln!("Webhook server error: {}", e);- }- });-- println!("Webhook server running at {}", webhook_url);-- Ok(webhook_url)-}--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- let cli = Cli::parse();- let firecrawl = create_firecrawl_app().await?;-- let state = Arc::new(Mutex::new(WebhookState { responses: vec![] }));- let webhook_url = start_webhook_server(39120, state.clone()).await?;-- match cli.command {- Commands::Basic => {- let mut urls = Vec::new();-- let url_one = "https://invalid-url.url/";- println!("Mapping: {}", url_one);- match firecrawl.map_url(url_one, None).await {- Ok(mapped_urls) => urls.extend(mapped_urls),- Err(e) => println!("Error mapping {}: {}", url_one, e),- }-- let url_two = "https://www.devjobsscanner.com";- println!("Mapping: {}", url_two);- match firecrawl- .map_url(- url_two,- Some(MapOptions {- search: Some("rust".into()),- limit: Some(20),- ..Default::default()- }),- )- .await- {- Ok(mapped_urls) => urls.extend(mapped_urls),- Err(e) => println!("Error mapping {}: {}", url_two, e),- }-- test_multiple_urls(&firecrawl, urls, &webhook_url).await?;-- // Give time for webhooks to arrive- tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;- println!(- "Received {} webhook responses",- state.lock().await.responses.len()- );- }- }-- Ok(())-}--async fn test_multiple_urls(- app: &FirecrawlApp,- urls: Vec<String>,- webhook_url: &str,-) -> Result<(), Box<dyn Error>> {- println!("Testing batch scraping of {} URLs", urls.len());-- let webhook = WebhookOptions {- url: webhook_url.to_string(),- headers: None,- auth_token: None,- };-- let params = BatchScrapeParams {- urls,- webhook: Some(webhook),- ignore_invalid_urls: true,- options: Some(ScrapeOptions {- formats: Some(vec![ScrapeFormats::Markdown, ScrapeFormats::Links]),- ..Default::default()- }),- ..Default::default()- };-- let batch = app.async_batch_scrape_urls(params).await?;- println!("Batch job started: {}", batch.id);-- // Poll status periodically- loop {- let status = app.check_batch_scrape_status(&batch.id).await?;- println!("Progress: {}/{} pages", status.completed, status.total);-- if status.completed >= status.total {- println!("Batch job completed!");- break;- }-- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;- }-- Ok(())-}diff --git a/apps/rust-sdk/examples/cancel_crawl_example.rs b/apps/rust-sdk/examples/cancel_crawl_example.rsdeleted file mode 100644index 3451cc104..000000000--- a/apps/rust-sdk/examples/cancel_crawl_example.rs+++ /dev/null@@ -1,33 +0,0 @@-use firecrawl::FirecrawlApp;-use std::error::Error;-use std::time::Duration;--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- // Get API URL from environment- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");-- // Create the FirecrawlApp instance- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;-- // Start a crawl job- println!("Starting a crawl job...");- let crawl_response = firecrawl- .crawl_url_async("https://example.com", None)- .await?;- println!("Crawl job started with ID: {}", crawl_response.id);-- // Wait for a moment to let the crawl job start- println!("Waiting for a moment...");- tokio::time::sleep(Duration::from_secs(2)).await;-- // Cancel the crawl job- println!("Cancelling the crawl job...");- let cancel_response = firecrawl.cancel_crawl(&crawl_response.id).await?;-- println!("Cancellation result:");- println!(" Status: {:?}", cancel_response.status);-- Ok(())-}diff --git a/apps/rust-sdk/examples/check_crawl_errors_example.rs b/apps/rust-sdk/examples/check_crawl_errors_example.rsdeleted file mode 100644index 7629d36cb..000000000--- a/apps/rust-sdk/examples/check_crawl_errors_example.rs+++ /dev/null@@ -1,59 +0,0 @@-use firecrawl::FirecrawlApp;-use std::error::Error;--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- // Get API URL from environment- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");-- // Create the FirecrawlApp instance- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;-- // Start a crawl job that will likely have some errors (invalid URL format)- println!("Starting a crawl job...");- let crawl_response = firecrawl- .crawl_url_async("https://no-wer-agg.invalid", None)- .await?;- println!("Crawl job started with ID: {}", crawl_response.id);-- println!("Let it do it's thing...");- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;-- // Check the crawl errors- println!("Checking for crawl errors...");- match firecrawl.check_crawl_errors(&crawl_response.id).await {- Ok(error_response) => {- println!("Crawl errors response:");- println!(" Number of errors: {}", error_response.errors.len());-- if !error_response.errors.is_empty() {- println!("\nDetailed errors:");- for (i, error) in error_response.errors.iter().enumerate() {- println!("Error #{}", i + 1);- println!(" ID: {}", error.id);- if let Some(timestamp) = &error.timestamp {- println!(" Timestamp: {}", timestamp);- }- println!(" URL: {}", error.url);- println!(" Error: {}", error.error);- }- }-- println!(- "\nRobots.txt blocked URLs: {}",- error_response.robots_blocked.len()- );- for (i, url) in error_response.robots_blocked.iter().enumerate() {- println!(" {}. {}", i + 1, url);- }- }- Err(e) => {- println!("Failed to check crawl errors: {}", e);- }- }- let cancel = firecrawl.cancel_crawl(&crawl_response.id).await?;- println!("Cancel: {}", cancel.status);-- Ok(())-}diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rsindex 37a3df7d2..57b418e93 100644--- a/apps/rust-sdk/examples/example.rs+++ b/apps/rust-sdk/examples/example.rs@@ -1,90 +1,216 @@+//! Example usage of Firecrawl API.+//!+//! Run with: cargo run --example example+use firecrawl::{- crawl::CrawlOptions,- scrape::{JsonOptions, ScrapeFormats, ScrapeOptions},- FirecrawlApp,+ AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,+ ScrapeOptions, SearchOptions, SitemapMode,};+use serde::Deserialize;use serde_json::json;#[tokio::main]async fn main() {- // Initialize the FirecrawlApp with the API key- let app = FirecrawlApp::new("fc-YOUR-API-KEY").expect("Failed to initialize FirecrawlApp");+ // Initialize the Client with the API key+ let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");// Or, connect to a self-hosted instance:- // let app = FirecrawlApp::new_selfhosted("http://localhost:3002", None).expect("Failed to initialize FirecrawlApp");-- // Scrape a website- let scrape_result = app.scrape_url("https://firecrawl.dev", None).await;+ // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)+ // .expect("Failed to initialize Client");- match scrape_result {- Ok(data) => println!("Scrape Result:\n{}", data.markdown.unwrap()),- Err(e) => eprintln!("Scrape failed: {:#?}", e),+ // Example 1: Simple Scrape+ println!("=== Example 1: Simple Scrape ===");+ let result = client.scrape("https://example.com", None).await;+ match result {+ Ok(doc) => println!("Markdown: {:?}", doc.markdown),+ Err(e) => eprintln!("Scrape failed: {}", e),}- // Crawl a website- let crawl_options = CrawlOptions {- exclude_paths: vec!["blog/*".into()].into(),+ // Example 2: Scrape with Options+ println!("\n=== Example 2: Scrape with Options ===");+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ only_main_content: Some(true),..Default::default()};+ let result = client.scrape("https://example.com", options).await;+ match result {+ Ok(doc) => {+ println!("Links: {:?}", doc.links);+ }+ Err(e) => eprintln!("Scrape with options failed: {}", e),+ }- let crawl_result = app.crawl_url("https://mendable.ai", crawl_options).await;-- match crawl_result {+ // Example 3: Scrape with JSON Schema Extraction+ println!("\n=== Example 3: JSON Schema Extraction ===");+ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "description": { "type": "string" }+ }+ });+ let result = client+ .scrape_with_schema(+ "https://example.com",+ schema,+ Some("Extract title and description"),+ )+ .await;+ match result {Ok(data) => println!(- "Crawl Result (used {} credits):\n{:#?}",- data.credits_used, data.data+ "Extracted: {}",+ serde_json::to_string_pretty(&data).unwrap()),- Err(e) => eprintln!("Crawl failed: {}", e),+ Err(e) => eprintln!("Schema extraction failed: {}", e),}- // Scrape with Extract- let json_schema = json!({- "type": "object",- "properties": {- "top": {- "type": "array",- "items": {- "type": "object",- "properties": {- "title": {"type": "string"},- "points": {"type": "number"},- "by": {"type": "string"},- "commentsURL": {"type": "string"}- },- "required": ["title", "points", "by", "commentsURL"]- },- "minItems": 5,- "maxItems": 5,- "description": "Top 5 stories on Hacker News"+ // Example 4: Search+ println!("\n=== Example 4: Search ===");+ let options = SearchOptions {+ limit: Some(5),+ ..Default::default()+ };+ let result = client.search("rust programming", options).await;+ match result {+ Ok(response) => {+ if let Some(web) = response.data.web {+ for item in web {+ match item {+ firecrawl::SearchResultOrDocument::WebResult(r) => {+ println!("Result: {} - {}", r.url, r.title.unwrap_or_default());+ }+ firecrawl::SearchResultOrDocument::Document(d) => {+ if let Some(meta) = d.metadata {+ println!("Document: {:?}", meta.title);+ }+ }+ }+ }}- },- "required": ["top"]- });+ }+ Err(e) => eprintln!("Search failed: {}", e),+ }++ // Example 5: Map a Website+ println!("\n=== Example 5: Map ===");+ let options = MapOptions {+ sitemap: Some(SitemapMode::Include),+ limit: Some(20),+ ..Default::default()+ };+ let result = client.map("https://example.com", options).await;+ match result {+ Ok(response) => {+ println!("Found {} links", response.links.len());+ for link in response.links.iter().take(5) {+ println!(" - {}", link.url);+ }+ }+ Err(e) => eprintln!("Map failed: {}", e),+ }- let llm_extraction_options = ScrapeOptions {- formats: vec![ScrapeFormats::Json].into(),- json_options: JsonOptions {- schema: json_schema.into(),+ // Example 6: Crawl a Website+ println!("\n=== Example 6: Crawl ===");+ let options = CrawlOptions {+ limit: Some(5),+ sitemap: Some(SitemapMode::Include),+ poll_interval: Some(3000),+ ..Default::default()+ };+ let result = client.crawl("https://example.com", options).await;+ match result {+ Ok(job) => {+ println!("Crawled {} pages", job.data.len());+ for doc in job.data.iter().take(3) {+ if let Some(meta) = &doc.metadata {+ println!(" - {:?}", meta.source_url);+ }+ }+ }+ Err(e) => eprintln!("Crawl failed: {}", e),+ }++ // Example 7: Batch Scrape+ println!("\n=== Example 7: Batch Scrape ===");+ let urls = vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ];+ let options = BatchScrapeOptions {+ options: Some(ScrapeOptions {+ formats: Some(vec![Format::Markdown]),..Default::default()+ }),+ poll_interval: Some(2000),+ ..Default::default()+ };+ let result = client.batch_scrape(urls, options).await;+ match result {+ Ok(job) => {+ println!("Batch scraped {} pages", job.data.len());}- .into(),+ Err(e) => eprintln!("Batch scrape failed: {}", e),+ }++ // Example 8: Agent (Autonomous Web Browsing)+ println!("\n=== Example 8: Agent ===");+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Describe what this website is about and list any key features mentioned"+ .to_string(),+ model: Some(AgentModel::Spark1Pro),+ timeout: Some(60),..Default::default()};+ let result = client.agent(options).await;+ match result {+ Ok(response) => {+ println!("Agent status: {:?}", response.status);+ if let Some(data) = response.data {+ println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());+ }+ }+ Err(e) => eprintln!("Agent failed: {}", e),+ }- let llm_extraction_result = app- .scrape_url("https://news.ycombinator.com", llm_extraction_options)- .await;+ // Example 9: Agent with Typed Schema+ println!("\n=== Example 9: Agent with Typed Schema ===");- match llm_extraction_result {- Ok(data) => println!("LLM Extraction Result:\n{:#?}", data.extract.unwrap()),- Err(e) => eprintln!("LLM Extraction failed: {}", e),+ #[derive(Debug, Deserialize)]+ struct CompanyInfo {+ name: String,+ description: Option<String>,+ industry: Option<String>,}- // Map a website (Alpha)- let map_result = app.map_url("https://firecrawl.dev", None).await;+ let schema = json!({+ "type": "object",+ "properties": {+ "name": { "type": "string" },+ "description": { "type": "string" },+ "industry": { "type": "string" }+ },+ "required": ["name"]+ });++ let result: Result<Option<CompanyInfo>, _> = client+ .agent_with_schema(+ vec!["https://firecrawl.dev".to_string()],+ "Extract company information from this website",+ schema,+ )+ .await;- match map_result {- Ok(data) => println!("Mapped URLs: {:#?}", data),- Err(e) => eprintln!("Map failed: {}", e),+ match result {+ Ok(Some(info)) => {+ println!("Company: {}", info.name);+ println!("Description: {:?}", info.description);+ println!("Industry: {:?}", info.industry);+ }+ Ok(None) => println!("No data extracted"),+ Err(e) => eprintln!("Agent with schema failed: {}", e),}++ println!("\n=== All examples completed ===");}diff --git a/apps/rust-sdk/examples/extract_example.rs b/apps/rust-sdk/examples/extract_example.rsdeleted file mode 100644index 47f446ae3..000000000--- a/apps/rust-sdk/examples/extract_example.rs+++ /dev/null@@ -1,237 +0,0 @@-use firecrawl::{extract::ExtractParams, FirecrawlApp};-use serde_json::json;-use std::error::Error;--use clap::{Parser, ValueEnum};--#[derive(Parser)]-#[command(author, version, about, long_about = None)]-struct Args {- #[arg(value_enum)]- command: Examples,-}--#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]-enum Examples {- Basic,- Schema,- JsonSchema,-}--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- let args = Args::parse();-- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;- let urls = vec![- "https://www.firecrawl.dev/".to_string(),- "https://betteruptime.com".to_string(),- ];-- match args.command {- Examples::Basic => {- println!("Example 1: Extracting with URLs and prompt");-- let extract_params = ExtractParams {- prompt: Some(- "Extract Product promise, consice descirption and category".to_string(),- ),- url_trace: Some(true),- ..Default::default()- };-- println!("Starting asynchronous extraction job...");- let response = firecrawl- .async_extract(ExtractParams {- urls: Some(urls.iter().map(|u| u.to_string()).collect()),- prompt: extract_params.prompt.clone(),- url_trace: extract_params.url_trace,- ..Default::default()- })- .await?;-- println!("Extract job initiated:");- println!(" Job ID: {}", response.id);-- println!("\nChecking extract status...");- for _ in 0..5 {- let response = firecrawl.get_extract_status(&response.id).await?;-- println!("Extract status: {}", response.status);- if let Some(url_trace) = &response.url_trace {- println!("URL traces:");- for trace in url_trace {- println!(" URL: {}", trace.url);- println!(" Status: {}", trace.status);- }- }- println!("Extract data: {:#?}", response.data);- if response.status == "completed" {- break;- }-- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;- }- }- Examples::Schema => {- println!("Example 2: Extracting with schema");-- let schema = json!({- "type": "object",- "properties": {- "category": { "type": "string" },- "promise": { "type": "string" },- "descirption": { "type": "string" }- },- "required": ["category", "promise", "description"]- });-- println!("Starting synchronous extraction job...");-- match firecrawl- .extract(ExtractParams {- urls: urls.into(),- schema: Some(schema),- ..Default::default()- })- .await- {- Ok(result) => {- println!("Extraction completed successfully!");- println!("Status: {}", result.status);-- if let Some(data) = result.data {- println!("\nExtracted data:");- println!(" Title: {}", data["title"]);- if let Some(desc) = data.get("description") {- println!(" Description: {}", desc);- }- println!(- " Content (preview): {:.100}...",- data["content"].as_str().unwrap_or("N/A")- );- }-- if let Some(sources) = result.sources {- println!("\nSources:");- for (field, urls) in sources {- println!(" {}: {}", field, urls.join(", "));- }- }- }- Err(e) => {- println!("Extraction failed: {}", e);- }- }- }- Examples::JsonSchema => {- println!("Example 3: Using JsonSchema derive");-- /// A comprehensive analysis of given product- #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]- struct ProductAnalysis {- /// The full name of the product- product_name: String,- /// The company/brand behind the product- brand: String,- /// The general price range (e.g. "Premium", "$10-50", "Enterprise")- price_range: String,- /// The main customer segments this product targets- target_audience: Vec<String>,- /// Primary benefits and value propositions of the product- key_benefits: Vec<String>,- /// Distinctive features that set this product apart from competitors- unique_selling_points: Vec<String>,- /// Direct comparisons with competing products/services- competitor_comparison: Vec<String>,- /// Technologies, frameworks, or platforms used (if applicable)- tech_stack: Option<Vec<String>>,- /// Aggregated review data and sentiment analysis- reviews_summary: ReviewsSummary,- // /// Score from 0-10 indicating product-market fit based on analysis- // market_fit_score: f32, // NOTE: Breaks- /// Assessment of future growth prospects (e.g. "High", "Moderate", "Limited")- growth_potential: String,- /// Relevant compliance standards and certifications- regulatory_compliance: Option<Vec<String>>,- }-- /// Aggregated analysis of product reviews from multiple sources- #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]- struct ReviewsSummary {- /// Overall sentiment from review analysis (e.g. "Highly Positive", "Mixed", "Negative")- sentiment_analysis: String,- /// Most frequently mentioned positive aspects- common_praises: Vec<String>,- /// Most frequently mentioned criticisms or issues- common_complaints: Vec<String>,- /// Platforms or websites where reviews were sourced from- review_sources: Vec<String>,- }- println!("Starting extraction with derived schema...");- match firecrawl- .extract_with_schemars::<ProductAnalysis>(ExtractParams {- urls: urls.into(),- ..Default::default()- })- .await- {- Ok(result) => {- println!("Extraction completed!");- println!("Status: {}", result.status);-- if let Some(data) = result.data {- if let Ok(analysis) = serde_json::from_value::<ProductAnalysis>(data) {- println!("\nExtracted Product Analysis:");- println!(" Product: {}", analysis.product_name);- println!(" Brand: {}", analysis.brand);- println!(" Price Range: {}", analysis.price_range);- println!(" Target Audience:");- for audience in analysis.target_audience {- println!(" - {}", audience);- }- println!(" Key Benefits:");- for benefit in analysis.key_benefits {- println!(" - {}", benefit);- }- println!(" USPs:");- for usp in analysis.unique_selling_points {- println!(" - {}", usp);- }-- println!("\n Reviews Summary:");- println!(- " Sentiment: {}",- analysis.reviews_summary.sentiment_analysis- );- println!(" Common Praises:");- for praise in analysis.reviews_summary.common_praises {- println!(" - {}", praise);- }- println!(" Common Complaints:");- for complaint in analysis.reviews_summary.common_complaints {- println!(" - {}", complaint);- }- } else {- println!("Failed to parse extracted data");- }- }-- if let Some(sources) = result.sources {- println!("\nSources:");- for (field, urls) in sources {- println!(" {}: {}", field, urls.join(", "));- }- }- }- Err(e) => {- println!("Extraction failed: {}", e);- }- }- }- }-- Ok(())-}diff --git a/apps/rust-sdk/examples/llmstxt_example.rs b/apps/rust-sdk/examples/llmstxt_example.rsdeleted file mode 100644index 64fb33179..000000000--- a/apps/rust-sdk/examples/llmstxt_example.rs+++ /dev/null@@ -1,173 +0,0 @@-#![allow(clippy::option_map_unit_fn)]-use bat::{Input, PrettyPrinter};-use firecrawl::{llmstxt::GenerateLLMsTextParams, FirecrawlApp};-use std::error::Error;--use clap::{Parser, ValueEnum};--#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]-enum Mode {- Basic,- Pool,- Fulltext,-}--#[derive(Parser)]-#[command(author, version, about, long_about = None)]-struct Args {- /// URL for which to generate LLMs.txt- #[arg(default_value = "https://www.firecrawl.dev/")]- url: String,-- #[arg(long, short = 'm', value_enum, default_value = "Mode::Basic")]- mode: Mode,-- /// Maximum number of URLs to process- #[arg(long, short = 'd', default_value = "1")]- max_urls: u32,-- /// Whether to show the full LLMs-full.txt in the response- #[arg(long, short = 'f', default_value = "false")]- full_text: bool,-- /// Experimental streaming option- #[arg(long, short = 's', default_value = "false")]- stream: bool,-}--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- let args = Args::parse();-- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;-- let params = GenerateLLMsTextParams {- url: args.url.clone(),- max_urls: args.max_urls,- show_full_text: args.full_text,- experimental_stream: args.stream,- };-- match args.mode {- Mode::Basic => {- println!("Example 1: Basic LLMs.txt generation (synchronous)");- println!("Generating LLMs.txt for {}...", args.url);- firecrawl- .generate_llms_text(params)- .await- .inspect(|result| {- println!("Expires at: {}", result.expires_at);- let text = (if args.full_text {- result.data.full.as_ref()- } else {- result.data.compact.as_ref()- })- .expect("LLM Text");-- pretty_print_content("Firecrawl Result", text).expect("Print");- })?;- }- Mode::Pool => {- println!("Example 2: Asynchronous LLMs.txt generation with manual polling");-- println!("Starting asynchronous LLMs.txt generation job...");- let response = firecrawl.async_generate_llms_text(params).await?;-- println!("LLMs.txt generation job initiated:");- println!(" Job ID: {}", response.id);- println!("\nManually polling for status...");- for _ in 0..10 {- let status = firecrawl- .check_generate_llms_text_status(&response.id)- .await?;-- match status.status.as_str() {- "completed" => {- println!("LLMs.txt generation completed!");- let text = (if args.full_text {- status.data.full.as_ref()- } else {- status.data.compact.as_ref()- })- .expect("LLM Text");-- pretty_print_content("Pool Result", text).expect("Print");-- break;- }- "failed" => {- println!(- "LLMs.txt generation failed: {}",- status.error.unwrap_or_default()- );- break;- }- status => println!("Generation status: {}", status),- }-- println!("Waiting 2 seconds before checking again...");- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;- }- }- Mode::Fulltext => {- println!("Example 3: LLMs.txt generation with full text");-- println!("Generating LLMs.txt with full text...");- match firecrawl.generate_llms_text(params).await {- Ok(result) => {- println!("LLMs.txt generation completed successfully!");- let llmstxt = result.data.compact.expect("LLMs Text Expected");- let fulltxt = result.data.full.expect("Full LLMs Text Expected");-- pretty_print_contents(&[- ("LLMs.txt (compact)", llmstxt),- ("LLMs.txt (full text)", fulltxt),- ])- .expect("Print")- }- Err(e) => {- println!("LLMs.txt generation failed: {}", e);- }- }- }- }-- Ok(())-}--/// Pretty prints the provided content with syntax highlighting-fn pretty_print_content(title: &str, content: &str) -> Result<(), Box<dyn Error>> {- PrettyPrinter::new()- .header(true)- .grid(true)- .input(- Input::from_bytes(content.as_bytes())- .title(title)- .name("file.md"),- )- .print()?;-- Ok(())-}--/// Pretty prints multiple contents with syntax highlighting-fn pretty_print_contents(title_contents: &[(&'static str, String)]) -> Result<(), Box<dyn Error>> {- let mut inputs = Vec::new();- for (title, content) in title_contents {- inputs.push(- Input::from_bytes(content.as_bytes())- .title(*title)- .name("file.md"),- );- }-- PrettyPrinter::new()- .header(true)- .grid(true)- .inputs(inputs)- .print()?;-- Ok(())-}diff --git a/apps/rust-sdk/examples/search_example.rs b/apps/rust-sdk/examples/search_example.rsdeleted file mode 100644index 28437193a..000000000--- a/apps/rust-sdk/examples/search_example.rs+++ /dev/null@@ -1,186 +0,0 @@-use clap::{Parser, ValueEnum};-use firecrawl::{- search::{SearchParams, SearchResponse},- FirecrawlApp,-};-use std::error::Error;--#[derive(Debug, Parser)]-#[command(author, version, about, long_about = None)]-struct Args {- /// Which example to run- #[arg(value_enum, default_value_t = Examples::All)]- example: Examples,-}--#[derive(Debug, Clone, ValueEnum)]-enum Examples {- All,- Basic,- Advanced,- Geo,- Temporal,- Social,- News,- Academic,- Commercial,-}--#[tokio::main]-async fn main() -> Result<(), Box<dyn Error>> {- let args = Args::parse();-- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;-- match args.example {- Examples::All => {- run_basic_example(&firecrawl).await?;- run_advanced_example(&firecrawl).await?;- run_geographic_example(&firecrawl).await?;- run_temporal_example(&firecrawl).await?;- run_social_example(&firecrawl).await?;- run_news_example(&firecrawl).await?;- run_academic_example(&firecrawl).await?;- run_commercial_example(&firecrawl).await?;- }- Examples::Basic => run_basic_example(&firecrawl).await?,- Examples::Advanced => run_advanced_example(&firecrawl).await?,- Examples::Geo => run_geographic_example(&firecrawl).await?,- Examples::Temporal => run_temporal_example(&firecrawl).await?,- Examples::Social => run_social_example(&firecrawl).await?,- Examples::News => run_news_example(&firecrawl).await?,- Examples::Academic => run_academic_example(&firecrawl).await?,- Examples::Commercial => run_commercial_example(&firecrawl).await?,- }-- Ok(())-}-async fn run_basic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "rust programming language";- let results = firecrawl.search(query, None).await?;- print_results("Basic Search", query, &results);- Ok(())-}--async fn run_advanced_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "rust web framework site:github.com OR site:gitlab.com";- let params = SearchParams {- query: query.to_string(),- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Advanced Repository Search", query, &results);- Ok(())-}--async fn run_geographic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "coworking space startup hub";- let params = SearchParams {- query: query.to_string(),- // WARN: Doesn't work with searxng- location: Some("Silicon Valley, CA".to_string()),- // WARN: Doesn't work with searxng- country: Some("us".to_string()),- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Geographic-Specific Search", query, &results);- Ok(())-}--async fn run_temporal_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "artificial intelligence breakthroughs";- let params = SearchParams {- query: query.to_string(),- // WARN: Doesn't work with searxng- tbs: Some("qdr:m1".to_string()),- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Recent AI News", query, &results);- Ok(())-}--async fn run_social_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "viral tech trends site:twitter.com";- let params = SearchParams {- query: query.to_string(),- // WARN: Doesn't work. Maybe searxng related- filter: Some("site:twitter.com OR site:linkedin.com".to_string()),- // WARN: Doesn't work with searxng- tbs: Some("qdr:w".to_string()), // Last week- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Social Media Tech Trends", query, &results);- Ok(())-}--async fn run_news_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query =- "cryptocurrency market analysis site:reuters.com OR site:bloomberg.com OR site:ft.com";- let params = SearchParams {- query: query.to_string(),- // WARN: Doesn't work with searxng- tbs: Some("qdr:d".to_string()), // Last 24 hours- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Financial News Search", query, &results);- Ok(())-}--async fn run_academic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "quantum computing research papers site:arxiv.org OR site:scholar.google.com";- let params = SearchParams {- query: query.to_string(),- // WARN: Doesn't work. Maybe searxng related- // filter: Some("site:arxiv.org OR site:scholar.google.com".to_string()),- // WARN: Doesn't work with searxng- tbs: Some("qdr:y".to_string()), // Last year- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Academic Research Search", query, &results);- Ok(())-}--async fn run_commercial_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {- let query = "enterprise cloud solutions reviews site:g2.com";- let params = SearchParams {- query: query.to_string(),- limit: Some(5),- ..Default::default()- };- let results = firecrawl.search_with_params(params).await?;- print_results("Commercial Product Search", query, &results);- Ok(())-}--fn print_results(name: &str, query: &str, results: &SearchResponse) {- let sec = "=".repeat(70);-- println!("\n{sec}");- println!("🔍 {name}");- println!("🔎 Query: \"{query}\"");- println!("{sec}");-- for (i, doc) in results.data.iter().enumerate() {- println!("{}. 📌 Title: {}", i + 1, doc.title);- println!(" - 🔗 URL: {}", doc.url);- println!(" - 📝 Description: \"{:.40}\"...", doc.description);- }-- if let Some(warning) = &results.warning {- println!("\n⚠️ Warning: {warning}");- }- println!("{sec}\n");-}diff --git a/apps/rust-sdk/examples/v2_example.rs b/apps/rust-sdk/examples/v2_example.rsdeleted file mode 100644index a5401c928..000000000--- a/apps/rust-sdk/examples/v2_example.rs+++ /dev/null@@ -1,216 +0,0 @@-//! Example usage of Firecrawl v2 API.-//!-//! Run with: cargo run --example v2_example--use firecrawl::v2::{- AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,- ScrapeOptions, SearchOptions, SitemapMode,-};-use serde::Deserialize;-use serde_json::json;--#[tokio::main]-async fn main() {- // Initialize the v2 Client with the API key- let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");-- // Or, connect to a self-hosted instance:- // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)- // .expect("Failed to initialize Client");-- // Example 1: Simple Scrape- println!("=== Example 1: Simple Scrape ===");- let result = client.scrape("https://example.com", None).await;- match result {- Ok(doc) => println!("Markdown: {:?}", doc.markdown),- Err(e) => eprintln!("Scrape failed: {}", e),- }-- // Example 2: Scrape with Options- println!("\n=== Example 2: Scrape with Options ===");- let options = ScrapeOptions {- formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),- only_main_content: Some(true),- ..Default::default()- };- let result = client.scrape("https://example.com", options).await;- match result {- Ok(doc) => {- println!("Links: {:?}", doc.links);- }- Err(e) => eprintln!("Scrape with options failed: {}", e),- }-- // Example 3: Scrape with JSON Schema Extraction- println!("\n=== Example 3: JSON Schema Extraction ===");- let schema = json!({- "type": "object",- "properties": {- "title": { "type": "string" },- "description": { "type": "string" }- }- });- let result = client- .scrape_with_schema(- "https://example.com",- schema,- Some("Extract title and description"),- )- .await;- match result {- Ok(data) => println!(- "Extracted: {}",- serde_json::to_string_pretty(&data).unwrap()- ),- Err(e) => eprintln!("Schema extraction failed: {}", e),- }-- // Example 4: Search- println!("\n=== Example 4: Search ===");- let options = SearchOptions {- limit: Some(5),- ..Default::default()- };- let result = client.search("rust programming", options).await;- match result {- Ok(response) => {- if let Some(web) = response.data.web {- for item in web {- match item {- firecrawl::v2::SearchResultOrDocument::WebResult(r) => {- println!("Result: {} - {}", r.url, r.title.unwrap_or_default());- }- firecrawl::v2::SearchResultOrDocument::Document(d) => {- if let Some(meta) = d.metadata {- println!("Document: {:?}", meta.title);- }- }- }- }- }- }- Err(e) => eprintln!("Search failed: {}", e),- }-- // Example 5: Map a Website- println!("\n=== Example 5: Map ===");- let options = MapOptions {- sitemap: Some(SitemapMode::Include),- limit: Some(20),- ..Default::default()- };- let result = client.map("https://example.com", options).await;- match result {- Ok(response) => {- println!("Found {} links", response.links.len());- for link in response.links.iter().take(5) {- println!(" - {}", link.url);- }- }- Err(e) => eprintln!("Map failed: {}", e),- }-- // Example 6: Crawl a Website- println!("\n=== Example 6: Crawl ===");- let options = CrawlOptions {- limit: Some(5),- sitemap: Some(SitemapMode::Include),- poll_interval: Some(3000),- ..Default::default()- };- let result = client.crawl("https://example.com", options).await;- match result {- Ok(job) => {- println!("Crawled {} pages", job.data.len());- for doc in job.data.iter().take(3) {- if let Some(meta) = &doc.metadata {- println!(" - {:?}", meta.source_url);- }- }- }- Err(e) => eprintln!("Crawl failed: {}", e),- }-- // Example 7: Batch Scrape- println!("\n=== Example 7: Batch Scrape ===");- let urls = vec![- "https://example.com".to_string(),- "https://example.org".to_string(),- ];- let options = BatchScrapeOptions {- options: Some(ScrapeOptions {- formats: Some(vec![Format::Markdown]),- ..Default::default()- }),- poll_interval: Some(2000),- ..Default::default()- };- let result = client.batch_scrape(urls, options).await;- match result {- Ok(job) => {- println!("Batch scraped {} pages", job.data.len());- }- Err(e) => eprintln!("Batch scrape failed: {}", e),- }-- // Example 8: Agent (Autonomous Web Browsing)- println!("\n=== Example 8: Agent ===");- let options = AgentOptions {- urls: Some(vec!["https://example.com".to_string()]),- prompt: "Describe what this website is about and list any key features mentioned"- .to_string(),- model: Some(AgentModel::Spark1Pro),- timeout: Some(60),- ..Default::default()- };- let result = client.agent(options).await;- match result {- Ok(response) => {- println!("Agent status: {:?}", response.status);- if let Some(data) = response.data {- println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());- }- }- Err(e) => eprintln!("Agent failed: {}", e),- }-- // Example 9: Agent with Typed Schema- println!("\n=== Example 9: Agent with Typed Schema ===");-- #[derive(Debug, Deserialize)]- struct CompanyInfo {- name: String,- description: Option<String>,- industry: Option<String>,- }-- let schema = json!({- "type": "object",- "properties": {- "name": { "type": "string" },- "description": { "type": "string" },- "industry": { "type": "string" }- },- "required": ["name"]- });-- let result: Result<Option<CompanyInfo>, _> = client- .agent_with_schema(- vec!["https://firecrawl.dev".to_string()],- "Extract company information from this website",- schema,- )- .await;-- match result {- Ok(Some(info)) => {- println!("Company: {}", info.name);- println!("Description: {:?}", info.description);- println!("Industry: {:?}", info.industry);- }- Ok(None) => println!("No data extracted"),- Err(e) => eprintln!("Agent with schema failed: {}", e),- }-- println!("\n=== All examples completed ===");-}diff --git a/apps/rust-sdk/src/v2/agent.rs b/apps/rust-sdk/src/agent.rssimilarity index 98%rename from apps/rust-sdk/src/v2/agent.rsrename to apps/rust-sdk/src/agent.rsindex 47b4f734f..bd4e77796 100644--- a/apps/rust-sdk/src/v2/agent.rs+++ b/apps/rust-sdk/src/agent.rs@@ -6,8 +6,8 @@use serde::{Deserialize, Serialize};use serde_json::Value;-use super::client::Client;-use super::types::{AgentModel, AgentWebhookConfig};+use crate::client::Client;+use crate::types::{AgentModel, AgentWebhookConfig};use crate::FirecrawlError;/// Options for running an agent task.@@ -111,7 +111,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::{Client, AgentOptions};+ /// use firecrawl::{Client, AgentOptions};////// #[tokio::main]/// async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -160,7 +160,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::Client;+ /// use firecrawl::Client;////// #[tokio::main]/// async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -209,7 +209,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::{Client, AgentOptions, AgentModel};+ /// use firecrawl::{Client, AgentOptions, AgentModel};/// use serde_json::json;////// #[tokio::main]@@ -305,7 +305,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::Client;+ /// use firecrawl::Client;////// #[tokio::main]/// async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -358,7 +358,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::Client;+ /// use firecrawl::Client;/// use serde::Deserialize;/// use serde_json::json;///diff --git a/apps/rust-sdk/src/batch_scrape.rs b/apps/rust-sdk/src/batch_scrape.rsindex 1429fbea6..2b2fef680 100644--- a/apps/rust-sdk/src/batch_scrape.rs+++ b/apps/rust-sdk/src/batch_scrape.rs@@ -1,243 +1,370 @@-use std::collections::HashMap;+//! Batch scrape endpoint for Firecrawl API v2.use serde::{Deserialize, Serialize};-use crate::{- crawl::{CrawlErrorsResponse, CrawlStatus, CrawlStatusTypes},- scrape::ScrapeOptions,- FirecrawlApp, FirecrawlError, API_VERSION,-};+use crate::client::Client;+use crate::scrape::ScrapeOptions;+use crate::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};+use crate::FirecrawlError;+/// Options for batch scraping.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]-pub struct BatchScrapeParams {- /// List of URLs to scrape- pub urls: Vec<String>,- /// Scrape options to apply to all URLs+pub struct BatchScrapeOptions {+ /// Scrape options to apply to all URLs.#[serde(flatten)]pub options: Option<ScrapeOptions>,- /// Whether to ignore invalid URLs- #[serde(rename = "ignoreInvalidURLs")]- pub ignore_invalid_urls: bool,- /// ID of an existing job to append these URLs to++ /// Webhook configuration for job notifications.+ pub webhook: Option<WebhookConfig>,++ /// ID of an existing batch job to append URLs to.pub append_to_id: Option<String>,- /// Webhook configuration- pub webhook: Option<WebhookOptions>,- /// Idempotency key to send to the crawl endpoint.+ /// Whether to ignore invalid URLs instead of failing.+ #[serde(rename = "ignoreInvalidURLs")]+ pub ignore_invalid_urls: Option<bool>,++ /// Maximum concurrent requests.+ pub max_concurrency: Option<u32>,++ /// Enable zero data retention mode.+ pub zero_data_retention: Option<bool>,++ /// Idempotency key for the request.#[serde(skip)]pub idempotency_key: Option<String>,-}-/// Options for webhook notifications-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct WebhookOptions {- /// URL to send webhook notifications to- pub url: String,- /// Custom headers to include in webhook requests- pub headers: Option<HashMap<String, String>>,- /// Authentication token for the webhook- pub auth_token: Option<String>,+ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Poll interval for synchronous batch scrape (milliseconds).+ #[serde(skip)]+ pub poll_interval: Option<u64>,}-impl From<&str> for WebhookOptions {- fn from(url: &str) -> Self {- Self {- url: url.to_string(),- headers: None,- auth_token: None,- }- }+/// Request body for batch scrape endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct BatchScrapeRequest {+ urls: Vec<String>,+ #[serde(flatten)]+ options: BatchScrapeOptions,}-/// Response from initiating a batch scrape job+/// Response from starting a batch scrape job.#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]pub struct BatchScrapeResponse {- /// Whether the request was successful+ /// Whether the request was successful.pub success: bool,- /// The ID of the batch scrape job+ /// The batch scrape job ID.pub id: String,- /// URL to get the status of the batch scrape job+ /// URL to check the batch scrape status.pub url: String,- /// List of URLs that were invalid and could not be processed+ /// URLs that were invalid and ignored.+ #[serde(rename = "invalidURLs")]pub invalid_urls: Option<Vec<String>>,}-impl From<String> for WebhookOptions {- fn from(url: String) -> Self {- Self {- url,- headers: None,- auth_token: None,- }- }+/// Status of a batch scrape job.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeJob {+ /// Current status of the batch scrape job.+ pub status: JobStatus,+ /// Number of URLs completed.+ pub completed: u32,+ /// Total number of URLs to scrape.+ pub total: u32,+ /// Credits used by the batch scrape.+ pub credits_used: Option<u32>,+ /// Expiry time of the batch data.+ pub expires_at: Option<String>,+ /// URL for the next page of results.+ pub next: Option<String>,+ /// Scraped documents.+ pub data: Vec<Document>,}-impl FirecrawlApp {- /// Initiates an asynchronous batch scrape job- pub async fn async_batch_scrape_urls(+impl Client {+ /// Starts a batch scrape job asynchronously.+ ///+ /// Returns immediately with a job ID that can be used to check status.+ ///+ /// # Arguments+ ///+ /// * `urls` - List of URLs to scrape.+ /// * `options` - Optional batch scrape configuration.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeResponse` containing the job ID.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, BatchScrapeOptions};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = vec![+ /// "https://example.com".to_string(),+ /// "https://example.org".to_string(),+ /// ];+ ///+ /// let response = client.start_batch_scrape(urls, None).await?;+ /// println!("Batch job started: {}", response.id);+ ///+ /// // Check status later+ /// let status = client.get_batch_scrape_status(&response.id).await?;+ /// println!("Completed: {}/{}", status.completed, status.total);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn start_batch_scrape(&self,- params: BatchScrapeParams,+ urls: Vec<String>,+ options: impl Into<Option<BatchScrapeOptions>>,) -> Result<BatchScrapeResponse, FirecrawlError> {- let headers = self.prepare_headers(params.idempotency_key.as_ref());+ let options = options.into().unwrap_or_default();+ let body = BatchScrapeRequest {+ urls,+ options: options.clone(),+ };++ let headers = self.prepare_headers(options.idempotency_key.as_ref());let response = self.client- .post(format!("{}{}/batch/scrape", self.api_url, API_VERSION))+ .post(self.url("/batch/scrape")).headers(headers)- .json(¶ms)+ .json(&body).send().await- .map_err(|e| FirecrawlError::HttpError("Initiating batch scrape job".to_string(), e))?;-- self.handle_response(response, "initiate batch scrape job")- .await- }+ .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;- /// Initiates a batch scrape job and waits for completion- pub async fn batch_scrape_urls(- &self,- params: BatchScrapeParams,- poll_interval: Option<u64>,- ) -> Result<CrawlStatus, FirecrawlError> {- let poll_interval_ms = poll_interval.unwrap_or(2000);-- let response = self.async_batch_scrape_urls(params).await?;-- self.monitor_batch_job_status(&response.id, poll_interval_ms)- .await+ self.handle_response(response, "start batch scrape").await}- /// Checks the status of a batch scrape job- pub async fn check_batch_scrape_status(+ /// Gets the status of a batch scrape job.+ ///+ /// If the job is completed, this will automatically fetch all pages of results.+ ///+ /// # Arguments+ ///+ /// * `id` - The batch scrape job ID.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeJob` containing the current status and any available documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let status = client.get_batch_scrape_status("job-id").await?;+ /// println!("Status: {:?}", status.status);+ /// println!("Completed: {}/{}", status.completed, status.total);+ /// println!("Documents: {}", status.data.len());+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_batch_scrape_status(&self,id: impl AsRef<str>,- ) -> Result<CrawlStatus, FirecrawlError> {+ ) -> Result<BatchScrapeJob, FirecrawlError> {let response = self.client- .get(format!(- "{}{}/batch/scrape/{}",- self.api_url,- API_VERSION,- id.as_ref()- ))+ .get(self.url(&format!("/batch/scrape/{}", id.as_ref()))).headers(self.prepare_headers(None)).send().await.map_err(|e| {FirecrawlError::HttpError(- format!("Checking status of batch scrape {}", id.as_ref()),+ format!("Checking batch scrape status {}", id.as_ref()),e,)})?;- let mut status: CrawlStatus = self- .handle_response(- response,- format!("Checking status of batch scrape {}", id.as_ref()),- )+ let mut status: BatchScrapeJob = self+ .handle_response(response, format!("batch scrape status {}", id.as_ref())).await?;- if status.status == CrawlStatusTypes::Completed {- while let Some(next) = status.next.clone() {- let new_status = self.check_batch_scrape_status_next(next).await?;- status.data.extend_from_slice(&new_status.data);- status.next = new_status.next;+ // Auto-paginate if completed+ if status.status == JobStatus::Completed {+ while let Some(next) = status.next.take() {+ let next_status = self.get_batch_scrape_status_next(&next).await?;+ status.data.extend(next_status.data);+ status.next = next_status.next;}}Ok(status)}- /// Helper function to paginate through batch scrape status results- async fn check_batch_scrape_status_next(+ /// Fetches the next page of batch scrape results.+ async fn get_batch_scrape_status_next(&self,- next: impl AsRef<str>,- ) -> Result<CrawlStatus, FirecrawlError> {+ next: &str,+ ) -> Result<BatchScrapeJob, FirecrawlError> {let response = self.client- .get(next.as_ref())+ .get(next).headers(self.prepare_headers(None)).send().await.map_err(|e| {- FirecrawlError::HttpError(- format!("Paginating batch scrape using URL {:?}", next.as_ref()),- e,- )+ FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)})?;- self.handle_response(- response,- format!("Paginating batch scrape using URL {:?}", next.as_ref()),- )- .await+ self.handle_response(response, "batch scrape pagination")+ .await}- /// Check for errors in a batch scrape job- pub async fn check_batch_scrape_errors(+ /// Scrapes multiple URLs and waits for completion.+ ///+ /// This method starts a batch scrape and polls until it completes or fails.+ ///+ /// # Arguments+ ///+ /// * `urls` - List of URLs to scrape.+ /// * `options` - Optional batch scrape configuration.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeJob` containing all scraped documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, BatchScrapeOptions, ScrapeOptions, Format};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = vec![+ /// "https://example.com/page1".to_string(),+ /// "https://example.com/page2".to_string(),+ /// "https://example.com/page3".to_string(),+ /// ];+ ///+ /// let options = BatchScrapeOptions {+ /// options: Some(ScrapeOptions {+ /// formats: Some(vec![Format::Markdown, Format::Links]),+ /// ..Default::default()+ /// }),+ /// ignore_invalid_urls: Some(true),+ /// poll_interval: Some(3000),+ /// ..Default::default()+ /// };+ ///+ /// let result = client.batch_scrape(urls, options).await?;+ /// println!("Scraped {} pages", result.data.len());+ ///+ /// for doc in result.data {+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn batch_scrape(&self,- id: impl AsRef<str>,- ) -> Result<CrawlErrorsResponse, FirecrawlError> {- let response = self- .client- .get(format!(- "{}{}/batch/scrape/{}/errors",- self.api_url,- API_VERSION,- id.as_ref()- ))- .headers(self.prepare_headers(None))- .send()+ urls: Vec<String>,+ options: impl Into<Option<BatchScrapeOptions>>,+ ) -> Result<BatchScrapeJob, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let poll_interval = options.poll_interval.unwrap_or(2000);++ let response = self.start_batch_scrape(urls, options).await?;+ self.wait_for_batch_scrape(&response.id, poll_interval).await- .map_err(|e| {- FirecrawlError::HttpError(- format!("Checking errors for batch scrape {}", id.as_ref()),- e,- )- })?;-- self.handle_response(- response,- format!("Checking errors for batch scrape {}", id.as_ref()),- )- .await}- /// Helper function to poll for batch job status until completion- async fn monitor_batch_job_status(+ /// Waits for a batch scrape job to complete.+ async fn wait_for_batch_scrape(&self,id: &str,poll_interval: u64,- ) -> Result<CrawlStatus, FirecrawlError> {+ ) -> Result<BatchScrapeJob, FirecrawlError> {loop {- let status_data = self.check_batch_scrape_status(id).await?;- match status_data.status {- CrawlStatusTypes::Completed => {- break Ok(status_data);- }- CrawlStatusTypes::Scraping => {+ let status = self.get_batch_scrape_status(id).await?;++ match status.status {+ JobStatus::Completed => return Ok(status),+ JobStatus::Scraping => {tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;}- CrawlStatusTypes::Failed => {- break Err(FirecrawlError::CrawlJobFailed(- "Batch scrape job failed".into(),- status_data,+ JobStatus::Failed => {+ return Err(FirecrawlError::JobFailed(+ "Batch scrape job failed".to_string(),));}- CrawlStatusTypes::Cancelled => {- break Err(FirecrawlError::CrawlJobFailed(- "Batch scrape job was cancelled".into(),- status_data,+ JobStatus::Cancelled => {+ return Err(FirecrawlError::JobFailed(+ "Batch scrape job was cancelled".to_string(),));}}}}++ /// Gets errors from a batch scrape job.+ ///+ /// # Arguments+ ///+ /// * `id` - The batch scrape job ID.+ ///+ /// # Returns+ ///+ /// A `CrawlErrorsResponse` containing error details.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let errors = client.get_batch_scrape_errors("job-id").await?;+ /// for error in errors.errors {+ /// println!("Error on {}: {}", error.url, error.error);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_batch_scrape_errors(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "batch scrape errors").await+ }}#[cfg(test)]@@ -246,98 +373,30 @@ mod tests {use serde_json::json;#[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_batch_scrape() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();-- // Start a batch scrape job- let params = BatchScrapeParams {- urls: vec![- "https://example.com".to_string(),- "https://example.org".to_string(),- ],- ignore_invalid_urls: true,- ..Default::default()- };-- let response = app.async_batch_scrape_urls(params).await.unwrap();-- assert!(response.success);- assert!(!response.id.is_empty());- assert!(!response.url.is_empty());- }-- #[tokio::test]- async fn test_async_batch_scrape_with_mock() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock- let mock = server- .mock("POST", "/v1/batch/scrape")- // Remove the match_body expectation which might be causing issues- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "batch-123",- "url": "https://api.example.com/v1/batch/batch-123",- "invalidUrls": []- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = BatchScrapeParams {- urls: vec![- "https://example.com".to_string(),- "https://example.org".to_string(),- ],- ignore_invalid_urls: true,- ..Default::default()- };-- let response = app.async_batch_scrape_urls(params).await.unwrap();-- assert!(response.success);- assert_eq!(response.id, "batch-123");- assert_eq!(response.url, "https://api.example.com/v1/batch/batch-123");- assert!(response.invalid_urls.unwrap_or_default().is_empty());- mock.assert();- }-- #[tokio::test]- async fn test_batch_scrape_with_webhook() {+ async fn test_start_batch_scrape_with_mock() {let mut server = mockito::Server::new_async().await;let mock = server- .mock("POST", "/v1/batch/scrape")- // Remove the match_body expectation to simplify+ .mock("POST", "/v2/batch/scrape").with_status(200).with_header("content-type", "application/json").with_body(json!({"success": true,"id": "batch-123",- "url": "https://api.example.com/v1/batch/batch-123"+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ];- let params = BatchScrapeParams {- urls: vec!["https://example.com".to_string()],- webhook: Some("https://webhook.example.com/notify".into()),- ..Default::default()- };-- let response = app.async_batch_scrape_urls(params).await.unwrap();+ let response = client.start_batch_scrape(urls, None).await.unwrap();assert!(response.success);assert_eq!(response.id, "batch-123");@@ -345,35 +404,27 @@ mod tests {}#[tokio::test]- async fn test_check_batch_scrape_status_with_mock() {+ async fn test_get_batch_scrape_status_with_mock() {let mut server = mockito::Server::new_async().await;let mock = server- .mock("GET", "/v1/batch/scrape/batch-123")+ .mock("GET", "/v2/batch/scrape/batch-123").with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": true,"status": "completed",- "total": 2,- "completed": 2,- "creditsUsed": 2,- "expiresAt": "2023-12-31T23:59:59Z",+ "total": 3,+ "completed": 3,+ "creditsUsed": 3,"data": [{- "metadata": {- "sourceURL": "https://example.com",- "statusCode": 200- },- "markdown": "Example Domain content"+ "markdown": "# Page 1",+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }},{- "metadata": {- "sourceURL": "https://example.org",- "statusCode": 200- },- "markdown": "Another example content"+ "markdown": "# Page 2",+ "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }}]})@@ -381,114 +432,129 @@ mod tests {).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let status = app.check_batch_scrape_status("batch-123").await.unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = client.get_batch_scrape_status("batch-123").await.unwrap();- assert_eq!(status.total, 2);- assert_eq!(status.completed, 2);+ assert_eq!(status.status, JobStatus::Completed);+ assert_eq!(status.total, 3);+ assert_eq!(status.completed, 3);assert_eq!(status.data.len(), 2);- assert_eq!(status.data[0].metadata.source_url, "https://example.com");- assert_eq!(status.data[1].metadata.source_url, "https://example.org");mock.assert();}#[tokio::test]- async fn test_check_batch_scrape_errors_with_mock() {+ async fn test_batch_scrape_with_invalid_urls() {let mut server = mockito::Server::new_async().await;let mock = server- .mock("GET", "/v1/batch/scrape/batch-123/errors")+ .mock("POST", "/v2/batch/scrape").with_status(200).with_header("content-type", "application/json").with_body(json!({"success": true,- "errors": [- {- "id": "error1",- "timestamp": "2023-01-01T00:00:00Z",- "url": "https://invalid.example.com",- "error": "Failed to load page"- }- ],- "robotsBlocked": [- "https://example.com/admin"- ]+ "id": "batch-456",+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",+ "invalidURLs": ["not-a-url"]}).to_string(),)- .create_async()- .await;+ .create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let errors = app.check_batch_scrape_errors("batch-123").await.unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];- assert_eq!(errors.errors.len(), 1);- assert_eq!(errors.errors[0].url, "https://invalid.example.com");- assert_eq!(errors.robots_blocked.len(), 1);- assert_eq!(errors.robots_blocked[0], "https://example.com/admin");+ let options = BatchScrapeOptions {+ ignore_invalid_urls: Some(true),+ ..Default::default()+ };++ let response = client.start_batch_scrape(urls, options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));mock.assert();}#[tokio::test]- async fn test_batch_scrape_with_invalid_urls() {+ async fn test_batch_scrape_sync() {let mut server = mockito::Server::new_async().await;- let mock = server- .mock("POST", "/v1/batch/scrape")- // Remove the match_body expectation+ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/batch/scrape").with_status(200).with_header("content-type", "application/json").with_body(json!({"success": true,- "id": "batch-123",- "url": "https://api.example.com/v1/batch/batch-123",- "invalidUrls": ["invalid-url"]+ "id": "batch-789",+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ // Mock the status endpoint (completed immediately)+ let status_mock = server+ .mock("GET", "/v2/batch/scrape/batch-789")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "completed",+ "total": 2,+ "completed": 2,+ "data": [+ {+ "markdown": "# Content",+ "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }+ }+ ]+ })+ .to_string(),+ )+ .create();- let params = BatchScrapeParams {- urls: vec!["https://example.com".to_string(), "invalid-url".to_string()],- ignore_invalid_urls: true,- ..Default::default()- };+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec!["https://example.com".to_string()];- let response = app.async_batch_scrape_urls(params).await.unwrap();+ let result = client.batch_scrape(urls, None).await.unwrap();- assert!(response.success);- assert_eq!(response.id, "batch-123");- assert_eq!(response.invalid_urls, Some(vec!["invalid-url".to_string()]));- mock.assert();+ assert_eq!(result.status, JobStatus::Completed);+ assert_eq!(result.data.len(), 1);+ start_mock.assert();+ status_mock.assert();}#[tokio::test]- async fn test_batch_scrape_error_response() {+ async fn test_get_batch_scrape_errors() {let mut server = mockito::Server::new_async().await;let mock = server- .mock("POST", "/v1/batch/scrape")- .with_status(400)+ .mock("GET", "/v2/batch/scrape/batch-123/errors")+ .with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": false,- "error": "No valid URLs provided"+ "errors": [+ {+ "id": "err-1",+ "url": "https://example.com/broken",+ "error": "Connection timeout"+ }+ ],+ "robotsBlocked": []}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();- let params = BatchScrapeParams::default();- let result = app.async_batch_scrape_urls(params).await;-- assert!(result.is_err());+ assert_eq!(errors.errors.len(), 1);+ assert_eq!(errors.errors[0].error, "Connection timeout");mock.assert();}}diff --git a/apps/rust-sdk/src/v2/client.rs b/apps/rust-sdk/src/client.rssimilarity index 97%rename from apps/rust-sdk/src/v2/client.rsrename to apps/rust-sdk/src/client.rsindex f3881d83d..67c0279ef 100644--- a/apps/rust-sdk/src/v2/client.rs+++ b/apps/rust-sdk/src/client.rs@@ -17,7 +17,7 @@ const CLOUD_API_URL: &str = "https://api.firecrawl.dev";/// # Example////// ```no_run-/// use firecrawl::v2::Client;+/// use firecrawl::Client;////// #[tokio::main]/// async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -51,7 +51,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::Client;+ /// use firecrawl::Client;////// let client = Client::new("your-api-key").unwrap();/// ```@@ -73,7 +73,7 @@ impl Client {/// # Example////// ```no_run- /// use firecrawl::v2::Client;+ /// use firecrawl::Client;////// // Self-hosted without authentication/// let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();@@ -164,9 +164,6 @@ impl Client {.and_then(|response_json| {serde_json::from_str::<Value>(&response_json).map_err(FirecrawlError::ResponseParseError)- .inspect(|data| {- tracing::debug!("Response JSON: {:#?}", data);- })}).and_then(|response_value| {// Check for success field, or allow responses without it for status checksdiff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rsindex bd3b4273a..c9639b6ac 100644--- a/apps/rust-sdk/src/crawl.rs+++ b/apps/rust-sdk/src/crawl.rs@@ -1,432 +1,364 @@-use std::collections::HashMap;+//! Crawl endpoint for Firecrawl API v2.use serde::{Deserialize, Serialize};-use crate::{- document::Document,- scrape::{ScrapeFormats, ScrapeOptions},- FirecrawlApp, FirecrawlError, API_VERSION,-};--#[derive(Deserialize, Serialize, Clone, Copy, Debug)]-pub enum CrawlScrapeFormats {- /// Will result in a copy of the Markdown content of the page.- #[serde(rename = "markdown")]- Markdown,-- /// Will result in a copy of the filtered, content-only HTML of the page.- #[serde(rename = "html")]- HTML,-- /// Will result in a copy of the raw HTML of the page.- #[serde(rename = "rawHtml")]- RawHTML,-- /// Will result in a Vec of URLs found on the page.- #[serde(rename = "links")]- Links,-- /// Will result in a URL to a screenshot of the page.- ///- /// Can not be used in conjunction with `CrawlScrapeFormats::ScreenshotFullPage`.- #[serde(rename = "screenshot")]- Screenshot,-- /// Will result in a URL to a full-page screenshot of the page.- ///- /// Can not be used in conjunction with `CrawlScrapeFormats::Screenshot`.- #[serde(rename = "screenshot@fullPage")]- ScreenshotFullPage,-}--impl From<CrawlScrapeFormats> for ScrapeFormats {- fn from(value: CrawlScrapeFormats) -> Self {- match value {- CrawlScrapeFormats::Markdown => Self::Markdown,- CrawlScrapeFormats::HTML => Self::HTML,- CrawlScrapeFormats::RawHTML => Self::RawHTML,- CrawlScrapeFormats::Links => Self::Links,- CrawlScrapeFormats::Screenshot => Self::Screenshot,- CrawlScrapeFormats::ScreenshotFullPage => Self::ScreenshotFullPage,- }- }-}+use crate::client::Client;+use crate::scrape::ScrapeOptions;+use crate::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};+use crate::FirecrawlError;+/// Options for crawling a website.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]-pub struct CrawlScrapeOptions {- /// Formats to extract from the page. (default: `[ Markdown ]`)- pub formats: Option<Vec<CrawlScrapeFormats>>,-- /// Only extract the main content of the page, excluding navigation and other miscellaneous content. (default: `true`)- pub only_main_content: Option<bool>,-- /// HTML tags to exclusively include.- ///- /// For example, if you pass `div`, you will only get content from `<div>`s and their children.- pub include_tags: Option<Vec<String>>,-- /// HTML tags to exclude.- ///- /// For example, if you pass `img`, you will never get image URLs in your results.- pub exclude_tags: Option<Vec<String>>,-- /// Additional HTTP headers to use when loading the page.- pub headers: Option<HashMap<String, String>>,-- // Amount of time to wait after loading the page, and before grabbing the content, in milliseconds. (default: `0`)- pub wait_for: Option<u32>,-- // Timeout before returning an error, in milliseconds. (default: `60000`)- pub timeout: Option<u32>,-}--impl From<CrawlScrapeOptions> for ScrapeOptions {- fn from(value: CrawlScrapeOptions) -> Self {- ScrapeOptions {- formats: value- .formats- .map(|formats| formats.into_iter().map(|x| x.into()).collect()),- only_main_content: value.only_main_content,- include_tags: value.include_tags,- exclude_tags: value.exclude_tags,- headers: value.headers,- wait_for: value.wait_for,- timeout: value.timeout,- ..Default::default()- }- }-}--/// Options for webhook notifications-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct WebhookOptions {- /// URL to send webhook notifications to- pub url: String,-- /// Custom headers to include in webhook requests- pub headers: Option<HashMap<String, String>>,-- /// Custom data included in all webhook payloads- pub metadata: Option<HashMap<String, String>>,+pub struct CrawlOptions {+ /// Natural language prompt to guide crawl behavior.+ pub prompt: Option<String>,- /// Event types to receive- pub events: Option<Vec<WebhookEvent>>,-}+ /// URL path patterns to exclude from crawling.+ pub exclude_paths: Option<Vec<String>>,-impl From<String> for WebhookOptions {- fn from(value: String) -> Self {- Self {- url: value,- ..Default::default()- }- }-}+ /// URL path patterns to include in crawling.+ pub include_paths: Option<Vec<String>>,-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]-#[serde(rename_all = "camelCase")]-pub enum WebhookEvent {- /// Crawl finished successfully- Completed,+ /// Maximum depth of links to follow from the initial URL.+ pub max_discovery_depth: Option<u32>,- /// Crawl encountered an error- Failed,+ /// How to handle the sitemap.+ pub sitemap: Option<SitemapMode>,- /// Individual page scraped- Page,+ /// Ignore query parameters when deduplicating URLs.+ pub ignore_query_parameters: Option<bool>,- /// Crawl job initiated- Started,-}+ /// Maximum number of pages to crawl.+ pub limit: Option<u32>,-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlOptions {- /// Options to pass through to the scraper.- pub scrape_options: Option<CrawlScrapeOptions>,+ /// Crawl the entire domain regardless of path structure.+ pub crawl_entire_domain: Option<bool>,- /// URL RegEx patterns to (exclusively) include.- ///- /// For example, if you specified `"blog"`, only pages that have `blog` somewhere in the URL would be crawled.- pub include_paths: Option<Vec<String>>,+ /// Allow following links to external domains.+ pub allow_external_links: Option<bool>,- /// URL RegEx patterns to exclude.- ///- /// For example, if you specified `"blog"`, pages that have `blog` somewhere in the URL would not be crawled.- pub exclude_paths: Option<Vec<String>>,+ /// Allow following links to subdomains.+ pub allow_subdomains: Option<bool>,- /// Maximum URL depth to crawl, relative to the base URL. (default: `2`)- pub max_depth: Option<u32>,+ /// Delay between requests in seconds.+ pub delay: Option<u32>,- /// Tells the crawler to ignore the sitemap when crawling. (default: `true`)- pub ignore_sitemap: Option<bool>,+ /// Maximum concurrent requests.+ pub max_concurrency: Option<u32>,- /// Maximum number of pages to crawl. (default: `10`)- pub limit: Option<u32>,+ /// Webhook configuration for job notifications.+ pub webhook: Option<WebhookConfig>,- /// Allows the crawler to navigate links that are backwards in the URL hierarchy. (default: `false`)- pub allow_backward_links: Option<bool>,+ /// Scrape options to apply to each page.+ pub scrape_options: Option<ScrapeOptions>,- /// Allows the crawler to follow links to external URLs. (default: `false`)- pub allow_external_links: Option<bool>,+ /// Enable zero data retention mode.+ pub zero_data_retention: Option<bool>,- /// URL to send Webhook crawl events to.- pub webhook: Option<WebhookOptions>,+ /// Integration identifier for tracking.+ pub integration: Option<String>,- /// Idempotency key to send to the crawl endpoint.+ /// Idempotency key for the request.#[serde(skip)]pub idempotency_key: Option<String>,- pub delay: Option<u32>,-- /// When using `FirecrawlApp::crawl_url`, this is how often the status of the job should be checked, in milliseconds. (default: `2000`)+ /// Poll interval for synchronous crawl (milliseconds).#[serde(skip)]pub poll_interval: Option<u64>,}+/// Request body for crawl endpoint.#[derive(Deserialize, Serialize, Debug, Default)]#[serde(rename_all = "camelCase")]-struct CrawlRequestBody {+struct CrawlRequest {url: String,-#[serde(flatten)]options: CrawlOptions,}-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]+/// Response from starting a crawl job.+#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]-pub enum CrawlStatusTypes {- /// The crawl job is in progress.- Scraping,-- /// The crawl job has been completed successfully.- Completed,-- /// The crawl job has failed.- Failed,-- /// The crawl job has been cancelled.- Cancelled,+pub struct CrawlResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// The crawl job ID.+ pub id: String,+ /// URL to check the crawl status.+ pub url: String,}+/// Status of a crawl job.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]-pub struct CrawlStatus {- /// The status of the crawl.- pub status: CrawlStatusTypes,-- /// Number of pages that will be scraped in total. This number may grow as the crawler discovers new pages.+pub struct CrawlJob {+ /// Current status of the crawl job.+ pub status: JobStatus,+ /// Total number of pages to crawl.pub total: u32,-- /// Number of pages that have been successfully scraped.+ /// Number of pages completed.pub completed: u32,-- /// Amount of credits used by the crawl job.- pub credits_used: u32,-- /// Expiry time of crawl data. After this date, the crawl data will be unavailable from the API.- pub expires_at: String, // TODO: parse into date-- /// URL to call to get the next batch of documents.- /// Unless you are sidestepping the SDK, you do not need to deal with this.+ /// Credits used by the crawl.+ pub credits_used: Option<u32>,+ /// Expiry time of the crawl data.+ pub expires_at: Option<String>,+ /// URL for the next page of results.pub next: Option<String>,-- /// List of documents returned by the crawl+ /// Crawled documents.pub data: Vec<Document>,}-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlError {- pub id: String,- pub timestamp: Option<String>,- pub url: String,- pub error: String,-}--#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlErrorsResponse {- pub errors: Vec<CrawlError>,- #[serde(rename = "robotsBlocked")]- pub robots_blocked: Vec<String>,-}-+/// Response from canceling a crawl.#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]pub struct CancelCrawlResponse {+ /// Status of the cancellation.pub status: String,}-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlAsyncResponse {- success: bool,-- /// Crawl ID- pub id: String,-- /// URL to get the status of the crawl job- pub url: String,-}--impl FirecrawlApp {- /// Initiates a crawl job for a URL using the Firecrawl API.- pub async fn crawl_url_async(+impl Client {+ /// Starts a crawl job asynchronously.+ ///+ /// Returns immediately with a job ID that can be used to check status.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to start crawling from.+ /// * `options` - Optional crawl configuration.+ ///+ /// # Returns+ ///+ /// A `CrawlResponse` containing the job ID.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, CrawlOptions};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let response = client.start_crawl("https://example.com", None).await?;+ /// println!("Crawl job started: {}", response.id);+ ///+ /// // Check status later+ /// let status = client.get_crawl_status(&response.id).await?;+ /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn start_crawl(&self,url: impl AsRef<str>,- options: Option<CrawlOptions>,- ) -> Result<CrawlAsyncResponse, FirecrawlError> {- let body = CrawlRequestBody {+ options: impl Into<Option<CrawlOptions>>,+ ) -> Result<CrawlResponse, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let body = CrawlRequest {url: url.as_ref().to_string(),- options: options.unwrap_or_default(),+ options: options.clone(),};- let headers = self.prepare_headers(body.options.idempotency_key.as_ref());+ let headers = self.prepare_headers(options.idempotency_key.as_ref());let response = self.client- .post(format!("{}{}/crawl", self.api_url, API_VERSION))- .headers(headers.clone())+ .post(self.url("/crawl"))+ .headers(headers).json(&body).send().await- .map_err(|e| FirecrawlError::HttpError(format!("Crawling {:?}", url.as_ref()), e))?;-- self.handle_response::<CrawlAsyncResponse>(response, "start crawl job")- .await- }+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)+ })?;- /// Performs a crawl job for a URL using the Firecrawl API, waiting for the end result. This may take a long time depending on the size of the target page and your options (namely `CrawlOptions.limit`).- pub async fn crawl_url(- &self,- url: impl AsRef<str>,- options: impl Into<Option<CrawlOptions>>,- ) -> Result<CrawlStatus, FirecrawlError> {- let options = options.into();- let poll_interval = options- .as_ref()- .and_then(|x| x.poll_interval)- .unwrap_or(2000);- let res = self.crawl_url_async(url, options).await?;-- self.monitor_job_status(&res.id, poll_interval).await+ self.handle_response(response, "start crawl").await}- async fn check_crawl_status_next(- &self,- next: impl AsRef<str>,- ) -> Result<CrawlStatus, FirecrawlError> {+ /// Gets the status of a crawl job.+ ///+ /// If the job is completed, this will automatically fetch all pages of results.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID.+ ///+ /// # Returns+ ///+ /// A `CrawlJob` containing the current status and any available documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let status = client.get_crawl_status("job-id").await?;+ /// println!("Status: {:?}", status.status);+ /// println!("Completed: {}/{}", status.completed, status.total);+ /// println!("Documents: {}", status.data.len());+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {let response = self.client- .get(next.as_ref())+ .get(self.url(&format!("/crawl/{}", id.as_ref()))).headers(self.prepare_headers(None)).send().await.map_err(|e| {- FirecrawlError::HttpError(- format!("Paginating crawl using URL {:?}", next.as_ref()),- e,- )+ FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)})?;- self.handle_response(- response,- format!("Paginating crawl using URL {:?}", next.as_ref()),- )- .await+ let mut status: CrawlJob = self+ .handle_response(response, format!("crawl status {}", id.as_ref()))+ .await?;++ // Auto-paginate if completed+ if status.status == JobStatus::Completed {+ while let Some(next) = status.next.take() {+ let next_status = self.get_crawl_status_next(&next).await?;+ status.data.extend(next_status.data);+ status.next = next_status.next;+ }+ }++ Ok(status)}- /// Checks for the status of a crawl, based on the crawl's ID. To be used in conjunction with `FirecrawlApp::crawl_url_async`.- pub async fn check_crawl_status(- &self,- id: impl AsRef<str>,- ) -> Result<CrawlStatus, FirecrawlError> {+ /// Fetches the next page of crawl results.+ async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {let response = self.client- .get(format!(- "{}{}/crawl/{}",- self.api_url,- API_VERSION,- id.as_ref()- ))+ .get(next).headers(self.prepare_headers(None)).send().await- .map_err(|e| {- FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e)- })?;+ .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;- let mut status: CrawlStatus = self- .handle_response(- response,- format!("Checking status of crawl {}", id.as_ref()),- )- .await?;+ self.handle_response(response, "crawl pagination").await+ }- if status.status == CrawlStatusTypes::Completed {- while let Some(next) = status.next {- let new_status = self.check_crawl_status_next(next).await?;- status.data.extend_from_slice(&new_status.data);- status.next = new_status.next;- }- }+ /// Crawls a website and waits for completion.+ ///+ /// This method starts a crawl and polls until it completes or fails.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to start crawling from.+ /// * `options` - Optional crawl configuration.+ ///+ /// # Returns+ ///+ /// A `CrawlJob` containing all crawled documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, CrawlOptions, SitemapMode};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let options = CrawlOptions {+ /// sitemap: Some(SitemapMode::Include),+ /// limit: Some(100),+ /// poll_interval: Some(5000), // Check every 5 seconds+ /// ..Default::default()+ /// };+ ///+ /// let result = client.crawl("https://example.com", options).await?;+ /// println!("Crawled {} pages", result.data.len());+ ///+ /// for doc in result.data {+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn crawl(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<CrawlOptions>>,+ ) -> Result<CrawlJob, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let poll_interval = options.poll_interval.unwrap_or(2000);- Ok(status)+ let response = self.start_crawl(url, options).await?;+ self.wait_for_crawl(&response.id, poll_interval).await}- async fn monitor_job_status(+ /// Waits for a crawl job to complete.+ async fn wait_for_crawl(&self,id: &str,poll_interval: u64,- ) -> Result<CrawlStatus, FirecrawlError> {+ ) -> Result<CrawlJob, FirecrawlError> {loop {- let status_data = self.check_crawl_status(id).await?;- match status_data.status {- CrawlStatusTypes::Completed => {- break Ok(status_data);- }- CrawlStatusTypes::Scraping => {+ let status = self.get_crawl_status(id).await?;++ match status.status {+ JobStatus::Completed => return Ok(status),+ JobStatus::Scraping => {tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;}- CrawlStatusTypes::Failed => {- break Err(FirecrawlError::CrawlJobFailed(- "Crawl job failed".into(),- status_data,+ JobStatus::Failed => {+ return Err(FirecrawlError::JobFailed(+ "Crawl job failed".to_string(),));}- CrawlStatusTypes::Cancelled => {- break Err(FirecrawlError::CrawlJobFailed(- "Crawl job was cancelled.".into(),- status_data,+ JobStatus::Cancelled => {+ return Err(FirecrawlError::JobFailed(+ "Crawl job was cancelled".to_string(),));}}}}- /// Cancel an asynchronous crawl job using the Firecrawl API.+ /// Cancels a running crawl job.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID to cancel.////// # Returns///- /// A response indicating whether the cancellation was successful, or a FirecrawlError if the request fails.+ /// A `CancelCrawlResponse` indicating the cancellation status.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let response = client.cancel_crawl("job-id").await?;+ /// println!("Cancellation status: {}", response.status);+ ///+ /// Ok(())+ /// }+ /// ```pub async fn cancel_crawl(&self,id: impl AsRef<str>,) -> Result<CancelCrawlResponse, FirecrawlError> {let response = self.client- .delete(format!(- "{}{}/crawl/{}",- self.api_url,- API_VERSION,- id.as_ref()- ))+ .delete(self.url(&format!("/crawl/{}", id.as_ref()))).headers(self.prepare_headers(None)).send().await@@ -434,34 +366,51 @@ impl FirecrawlApp {FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)})?;- self.handle_response(response, "crawl_cancel").await+ self.handle_response(response, "cancel crawl").await}- /// Returns information about crawl errors.+ /// Gets errors from a crawl job.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID.////// # Returns///- /// A response containing information about crawl errors, or a FirecrawlError if the request fails.- pub async fn check_crawl_errors(+ /// A `CrawlErrorsResponse` containing error details.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let errors = client.get_crawl_errors("job-id").await?;+ /// for error in errors.errors {+ /// println!("Error on {}: {}", error.url, error.error);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_crawl_errors(&self,id: impl AsRef<str>,) -> Result<CrawlErrorsResponse, FirecrawlError> {let response = self.client- .get(format!(- "{}{}/crawl/{}/errors",- self.api_url,- API_VERSION,- id.as_ref()- ))+ .get(self.url(&format!("/crawl/{}/errors", id.as_ref()))).headers(self.prepare_headers(None)).send().await.map_err(|e| {- FirecrawlError::HttpError(format!("Checking errors for crawl {}", id.as_ref()), e)+ FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)})?;- self.handle_response(response, "crawl_check").await+ self.handle_response(response, "crawl errors").await}}@@ -471,164 +420,190 @@ mod tests {use serde_json::json;#[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_cancel_crawl() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();-- // First start a crawl job- let crawl_response = app- .crawl_url_async("https://example.com", None)- .await- .unwrap();-- // Then cancel it- let cancel_response = app.cancel_crawl(crawl_response.id).await.unwrap();-- assert_eq!(cancel_response.status, "cancelled");- }-- #[tokio::test]- async fn test_cancel_crawl_with_mock() {+ async fn test_start_crawl_with_mock() {let mut server = mockito::Server::new_async().await;- // Set up the mock for the cancel requestlet mock = server- .mock("DELETE", "/v1/crawl/test-crawl-id")+ .mock("POST", "/v2/crawl").with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": null,- "status": "cancelled"+ "success": true,+ "id": "crawl-123",+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = app.cancel_crawl("test-crawl-id").await.unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client+ .start_crawl("https://example.com", None)+ .await+ .unwrap();- assert_eq!(response.status, "cancelled");+ assert!(response.success);+ assert_eq!(response.id, "crawl-123");mock.assert();}#[tokio::test]- async fn test_cancel_crawl_error_response() {+ async fn test_get_crawl_status_with_mock() {let mut server = mockito::Server::new_async().await;- // Set up the mock for an error responselet mock = server- .mock("DELETE", "/v1/crawl/invalid-id")- .with_status(404)+ .mock("GET", "/v2/crawl/crawl-123")+ .with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": false,- "error": "Crawl job not found"+ "status": "completed",+ "total": 5,+ "completed": 5,+ "creditsUsed": 5,+ "expiresAt": "2024-12-31T23:59:59Z",+ "data": [+ {+ "markdown": "# Page 1",+ "metadata": {+ "sourceURL": "https://example.com/page1",+ "statusCode": 200+ }+ },+ {+ "markdown": "# Page 2",+ "metadata": {+ "sourceURL": "https://example.com/page2",+ "statusCode": 200+ }+ }+ ]}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = app.cancel_crawl("invalid-id").await;+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = client.get_crawl_status("crawl-123").await.unwrap();- assert!(result.is_err());+ assert_eq!(status.status, JobStatus::Completed);+ assert_eq!(status.total, 5);+ assert_eq!(status.completed, 5);+ assert_eq!(status.data.len(), 2);mock.assert();}#[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_check_crawl_errors() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();-- // First start a crawl job- let crawl_response = app- .crawl_url_async("https://no-wer-agg.invalid", None)- .await- .unwrap();+ async fn test_cancel_crawl_with_mock() {+ let mut server = mockito::Server::new_async().await;- // Check for errors- let errors_response = app.check_crawl_errors(crawl_response.id).await.unwrap();- println!("{errors_response:?}");+ let mock = server+ .mock("DELETE", "/v2/crawl/crawl-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "cancelled"+ })+ .to_string(),+ )+ .create();- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.cancel_crawl("crawl-123").await.unwrap();- assert!(- !errors_response.errors.is_empty(),- "WARN: Error returned related to Supabase not in my environment. It may fail"- );+ assert_eq!(response.status, "cancelled");+ mock.assert();}#[tokio::test]- async fn test_check_crawl_errors_with_mock() {+ async fn test_get_crawl_errors_with_mock() {let mut server = mockito::Server::new_async().await;- // Set up the mock for the check errors requestlet mock = server- .mock("GET", "/v1/crawl/test-crawl-id/errors")+ .mock("GET", "/v2/crawl/crawl-123/errors").with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": true,"errors": [{- "id": "error1",- "timestamp": "2023-01-01T00:00:00Z",- "url": "https://example.com/error-page",- "error": "Failed to load page"+ "id": "error-1",+ "timestamp": "2024-01-01T00:00:00Z",+ "url": "https://example.com/broken",+ "error": "404 Not Found"}],"robotsBlocked": [- "https://example.com/blocked-by-robots"+ "https://example.com/admin"]}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = app.check_crawl_errors("test-crawl-id").await.unwrap();-- assert_eq!(response.errors.len(), 1);- assert_eq!(response.errors[0].id, "error1");- assert_eq!(response.errors[0].url, "https://example.com/error-page");- assert_eq!(response.errors[0].error, "Failed to load page");- assert_eq!(response.robots_blocked.len(), 1);- assert_eq!(- response.robots_blocked[0],- "https://example.com/blocked-by-robots"- );+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let errors = client.get_crawl_errors("crawl-123").await.unwrap();++ assert_eq!(errors.errors.len(), 1);+ assert_eq!(errors.errors[0].url, "https://example.com/broken");+ assert_eq!(errors.robots_blocked.len(), 1);mock.assert();}#[tokio::test]- async fn test_check_crawl_errors_error_response() {+ async fn test_crawl_with_options() {let mut server = mockito::Server::new_async().await;- // Set up the mock for an error response- let mock = server- .mock("GET", "/v1/crawl/invalid-id/errors")- .with_status(404)+ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/crawl")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "crawl-456",+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"+ })+ .to_string(),+ )+ .create();++ // Mock the status endpoint (completed immediately)+ let status_mock = server+ .mock("GET", "/v2/crawl/crawl-456")+ .with_status(200).with_header("content-type", "application/json").with_body(json!({- "success": false,- "error": "Crawl job not found"+ "status": "completed",+ "total": 2,+ "completed": 2,+ "data": [+ {+ "markdown": "# Page 1",+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }+ }+ ]}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = app.check_crawl_errors("invalid-id").await;+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = CrawlOptions {+ limit: Some(10),+ sitemap: Some(SitemapMode::Include),+ ..Default::default()+ };- assert!(result.is_err());- mock.assert();+ let result = client.crawl("https://example.com", options).await.unwrap();++ assert_eq!(result.status, JobStatus::Completed);+ assert_eq!(result.data.len(), 1);+ start_mock.assert();+ status_mock.assert();}}diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rsdeleted file mode 100644index 5dd2c1a47..000000000--- a/apps/rust-sdk/src/document.rs+++ /dev/null@@ -1,150 +0,0 @@-use serde::{Deserialize, Serialize};-use serde_json::Value;--use crate::serde_helpers::deserialize_string_or_array;--#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct DocumentMetadata {- // firecrawl specific- #[serde(rename = "sourceURL")]- pub source_url: String,- pub status_code: u16,- pub error: Option<String>,-- // basic meta tags- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub title: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub description: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub language: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub keywords: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub robots: Option<String>,-- // og: namespace- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_title: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_description: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_url: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_image: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_audio: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_determiner: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_locale: Option<String>,- pub og_locale_alternate: Option<Vec<String>>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_site_name: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub og_video: Option<String>,-- // article: namespace- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub article_section: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub article_tag: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub published_time: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub modified_time: Option<String>,-- // dc./dcterms. namespace- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dcterms_keywords: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dc_description: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dc_subject: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dcterms_subject: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dcterms_audience: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dc_type: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dcterms_type: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dc_date: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dc_date_created: Option<String>,- #[serde(default, deserialize_with = "deserialize_string_or_array")]- pub dcterms_created: Option<String>,-}--#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct Document {- /// A list of the links on the page, present if `ScrapeFormats::Markdown` is present in `ScrapeOptions.formats`. (default)- pub markdown: Option<String>,-- /// The HTML of the page, present if `ScrapeFormats::HTML` is present in `ScrapeOptions.formats`.- ///- /// This contains HTML that has non-content tags removed. If you need the original HTML, use `ScrapeFormats::RawHTML`.- pub html: Option<String>,-- /// The raw HTML of the page, present if `ScrapeFormats::RawHTML` is present in `ScrapeOptions.formats`.- ///- /// This contains the original, untouched HTML on the page. If you only need human-readable content, use `ScrapeFormats::HTML`.- pub raw_html: Option<String>,-- /// The URL to the screenshot of the page, present if `ScrapeFormats::Screenshot` or `ScrapeFormats::ScreenshotFullPage` is present in `ScrapeOptions.formats`.- pub screenshot: Option<String>,-- /// A list of the links on the page, present if `ScrapeFormats::Links` is present in `ScrapeOptions.formats`.- pub links: Option<Vec<String>>,-- /// The extracted data from the page, present if `ScrapeFormats::Extract` is present in `ScrapeOptions.formats`.- /// If `ScrapeOptions.extract.schema` is `Some`, this `Value` is guaranteed to match the provided schema.- pub extract: Option<Value>,-- /// The metadata from the page.- pub metadata: DocumentMetadata,-- /// Can be present if `ScrapeFormats::Extract` is present in `ScrapeOptions.formats`.- /// The warning message will contain any errors encountered during the extraction.- pub warning: Option<String>,-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[test]- fn test_full_document_with_array_metadata() {- let json = json!({- "markdown": "# Hello",- "metadata": {- "sourceURL": "https://example.com",- "statusCode": 200,- "title": "Example Page",- "description": ["A great page", "with multiple descriptions"],- "robots": ["index", "follow"],- "ogImage": ["https://img.jpg"],- "language": "en",- "keywords": ["rust", "sdk", "firecrawl"]- }- });- let doc: Document = serde_json::from_value(json).unwrap();- assert_eq!(doc.markdown, Some("# Hello".to_string()));- let meta = doc.metadata;- assert_eq!(meta.title, Some("Example Page".to_string()));- assert_eq!(- meta.description,- Some("A great page, with multiple descriptions".to_string())- );- assert_eq!(meta.robots, Some("index, follow".to_string()));- assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));- assert_eq!(meta.language, Some("en".to_string()));- assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));- }-}diff --git a/apps/rust-sdk/src/error.rs b/apps/rust-sdk/src/error.rsindex f452cd0e3..0f991503e 100644--- a/apps/rust-sdk/src/error.rs+++ b/apps/rust-sdk/src/error.rs@@ -4,8 +4,6 @@ use serde::{Deserialize, Serialize};use serde_json::Value;use thiserror::Error;-use crate::crawl::CrawlStatus;-#[derive(Debug, Deserialize, Serialize, Clone)]pub struct FirecrawlAPIError {/// Always false.@@ -40,8 +38,8 @@ pub enum FirecrawlError {ResponseParseError(serde_json::Error),#[error("{0} failed: {1}")]APIError(String, FirecrawlAPIError),- #[error("Crawl job failed: {0}")]- CrawlJobFailed(String, CrawlStatus),- #[error("Missuse: {0}")]- Missuse(String),+ #[error("Job failed: {0}")]+ JobFailed(String),+ #[error("Misuse: {0}")]+ Misuse(String),}diff --git a/apps/rust-sdk/src/extract.rs b/apps/rust-sdk/src/extract.rsdeleted file mode 100644index a8721ee08..000000000--- a/apps/rust-sdk/src/extract.rs+++ /dev/null@@ -1,607 +0,0 @@-use std::collections::HashMap;--use schemars::schema_for;-use serde::{Deserialize, Serialize};-use serde_json::Value;--use crate::{FirecrawlApp, FirecrawlError, API_VERSION};--/// Agent options for extract requests-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct AgentOptionsExtract {- /// Model to use for the agent- pub model: String,-}--/// Parameters for extract requests-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ExtractParams {- /// URLs to extract information from- pub urls: Option<Vec<String>>,-- /// Extraction prompt- pub prompt: Option<String>,-- /// Schema for structured output- pub schema: Option<Value>,-- /// System prompt for the LLM- pub system_prompt: Option<String>,-- /// Allow following external links- pub allow_external_links: Option<bool>,-- /// Enable web search for additional information- pub enable_web_search: Option<bool>,-- /// Show sources in the response- pub show_sources: Option<bool>,-- /// Origin information, defaults to "api-sdk"- pub origin: Option<String>,-- /// Timeout in milliseconds, defaults to 60000- pub timeout: Option<u32>,-- /// Whether to include URL trace information, defaults to false- pub url_trace: Option<bool>,-- /// Whether to ignore sitemap, defaults to false- pub ignore_sitemap: Option<bool>,-- /// Whether to include subdomains, defaults to true- pub include_subdomains: Option<bool>,-- /// Maximum number of URLs to process- pub limit: Option<u32>,-- /// Agent options- pub agent: Option<AgentOptionsExtract>,-- /// Experimental: Stream steps information- #[serde(rename = "__experimental_streamSteps")]- pub experimental_stream_steps: Option<bool>,-- /// Experimental: Include LLM usage information- #[serde(rename = "__experimental_llmUsage")]- pub experimental_llm_usage: Option<bool>,-- /// Experimental: Show sources information- #[serde(rename = "__experimental_showSources")]- pub experimental_show_sources: Option<bool>,-- /// Experimental: Cache key- #[serde(rename = "__experimental_cacheKey")]- pub experimental_cache_key: Option<String>,-- /// Experimental: Cache mode, defaults to "direct"- #[serde(rename = "__experimental_cacheMode")]- pub experimental_cache_mode: Option<String>,-}--/// Response from initiating an extract operation-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ExtractResponse {- /// Whether the request was successful- pub success: bool,-- /// The ID of the extract job- pub id: String,-- /// URL trace information if requested- pub url_trace: Option<Vec<URLTrace>>,-}--/// Information about URL processing during extraction-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct URLTrace {- /// The URL being processed- pub url: String,-- /// Status of processing this URL- pub status: String,-- /// Timing information for URL processing- pub timing: URLTraceTiming,-- /// Error message if processing failed- pub error: Option<String>,-- /// Warning message if there were issues- pub warning: Option<String>,-- /// Content statistics- pub content_stats: Option<ContentStats>,-- /// Relevance score for this URL (0-1)- pub relevance_score: Option<f64>,-- /// Whether this URL was used in the final completion- pub used_in_completion: Option<bool>,-- /// Fields extracted from this URL- pub extracted_fields: Option<Vec<String>>,-}--/// Timing information for URL processing-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct URLTraceTiming {- /// When the URL was discovered- pub discovered_at: String,-- /// When scraping began for this URL- pub scraped_at: Option<String>,-- /// When processing was completed for this URL- pub completed_at: Option<String>,-}--/// Statistics about processed content-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ContentStats {- /// Length of the raw content in characters- pub raw_content_length: u32,-- /// Length of the processed content in characters- pub processed_content_length: u32,-- /// Number of tokens used for this content- pub tokens_used: u32,-}--/// Response for extract status check-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ExtractStatusResponse {- /// Whether the request was successful- pub success: bool,-- /// Status of the extract job: "pending", "processing", "completed", "failed"- pub status: String,-- /// Extracted data, present when status is "completed"- pub data: Option<Value>,-- /// Error message if the job failed- pub error: Option<String>,-- /// URL trace information if requested- pub url_trace: Option<Vec<URLTrace>>,-- /// Sources information if requested- pub sources: Option<HashMap<String, Vec<String>>>,-}--impl FirecrawlApp {- /// Extracts information from URLs using the Firecrawl API.- ///- /// This is the synchronous version that polls until completion.- ///- /// Either `params.prompt` or `params.schema` must be provided.- pub async fn extract(- &self,- params: impl Into<ExtractParams>,- ) -> Result<ExtractStatusResponse, FirecrawlError> {- let mut params = params.into();- // Validation: Either prompt or schema must be provided- if params.prompt.is_none() && params.schema.is_none() {- return Err(FirecrawlError::APIError(- "Extract validation".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: "Either prompt or schema must be provided".to_string(),- details: None,- },- ));- }-- // Set default origin if not provided- if params.origin.is_none() {- params.origin = Some("api-sdk".to_string());- }-- // Initiate the extract job asynchronously- let response = self.async_extract(params).await?;-- // Poll for the result- let poll_interval = 2000; // Default to 2 seconds- self.monitor_extract_job_status(&response.id, poll_interval)- .await- }-- pub async fn extract_with_schemars<T>(- &self,- params: impl Into<ExtractParams>,- ) -> Result<ExtractStatusResponse, FirecrawlError>- where- T: schemars::JsonSchema,- {- let mut params = params.into();- let schema = schema_for!(T);- let schema_json = serde_json::to_value(schema).map_err(|e| {- FirecrawlError::APIError(- "Schema serialization".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: e.to_string(),- details: None,- },- )- })?;- params.schema = Some(schema_json);- self.extract(params).await- }-- /// Initiates an asynchronous extract operation.- ///- /// # Arguments- ///- /// * `params` - Parameters for the extract request- ///- /// # Returns- ///- /// A response containing the extract job ID, or a FirecrawlError if the request fails.- ///- /// # Notes- ///- /// Either `params.urls` or `params.prompt` must be provided.- /// Either `params.prompt` or `params.schema` must be provided.- pub async fn async_extract(- &self,- params: impl Into<ExtractParams>,- ) -> Result<ExtractResponse, FirecrawlError> {- let params = params.into();- // Validation: Either URLs or prompt must be provided- if params.urls.is_none() && params.prompt.is_none() {- return Err(FirecrawlError::APIError(- "Extract validation".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: "Either URLs or prompt must be provided".to_string(),- details: None,- },- ));- }-- // Validation: Either prompt or schema must be provided- if params.prompt.is_none() && params.schema.is_none() {- return Err(FirecrawlError::APIError(- "Extract validation".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: "Either prompt or schema must be provided".to_string(),- details: None,- },- ));- }-- let headers = self.prepare_headers(None);-- let response = self- .client- .post(format!("{}{}/extract", self.api_url, API_VERSION))- .headers(headers)- .json(¶ms)- .send()- .await- .map_err(|e| FirecrawlError::HttpError("Initiating extract job".to_string(), e))?;-- self.handle_response(response, "initiate extract job").await- }-- /// Checks the status of an extract job.- ///- /// # Arguments- ///- /// * `id` - The ID of the extract job- ///- /// # Returns- ///- /// A response containing the status of the extract job, or a FirecrawlError if the request fails.- pub async fn get_extract_status(- &self,- id: impl AsRef<str>,- ) -> Result<ExtractStatusResponse, FirecrawlError> {- let response = self- .client- .get(format!(- "{}{}/extract/{}",- self.api_url,- API_VERSION,- id.as_ref()- ))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Checking status of extract {}", id.as_ref()), e)- })?;-- self.handle_response(- response,- format!("Checking status of extract {}", id.as_ref()),- )- .await- }-- /// Helper function to poll for extract job status until completion- async fn monitor_extract_job_status(- &self,- id: &str,- poll_interval: u64,- ) -> Result<ExtractStatusResponse, FirecrawlError> {- loop {- let status_data = self.get_extract_status(id).await?;-- match status_data.status.as_str() {- "completed" => {- break Ok(status_data);- }- "pending" | "processing" => {- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;- }- "failed" => {- let error_msg = status_data- .error- .clone()- .unwrap_or_else(|| "Extract job failed".to_string());- break Err(FirecrawlError::APIError(- "Extract job failed".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: error_msg,- details: None,- },- ));- }- _ => {- break Err(FirecrawlError::APIError(- "Extract job status".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: format!("Unexpected status: {}", status_data.status),- details: None,- },- ));- }- }- }- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_extract() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();-- // Create extract params- let params = ExtractParams {- urls: Some(vec!["https://example.com".to_string()]),- prompt: Some("Extract the title and main content from this page".to_string()),- schema: None,- origin: Some("test".to_string()),- ..Default::default()- };-- // Start an extract job- let response = app.async_extract(params).await.unwrap();-- assert!(response.success);- assert!(!response.id.is_empty());- }-- #[tokio::test]- async fn test_async_extract_with_mock() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for the extract request- let mock = server- .mock("POST", "/v1/extract")- .match_body(mockito::Matcher::PartialJson(json!({- "urls": ["https://example.com"],- "prompt": "Extract the title and main content"- })))- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "extract-123",- "urlTrace": []- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = ExtractParams {- urls: Some(vec!["https://example.com".to_string()]),- prompt: Some("Extract the title and main content".to_string()),- schema: None,- ..Default::default()- };-- let response = app.async_extract(params).await.unwrap();-- assert!(response.success);- assert_eq!(response.id, "extract-123");- assert!(response.url_trace.unwrap_or_default().is_empty());- mock.assert();- }-- #[tokio::test]- async fn test_extract_with_schema() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for the extract request with schema- let mock = server- .mock("POST", "/v1/extract")- .match_body(mockito::Matcher::PartialJson(json!({- "urls": ["https://example.com"],- "schema": {- "type": "object",- "properties": {- "title": { "type": "string" },- "content": { "type": "string" }- }- }- })))- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "extract-123"- })- .to_string(),- )- .create();-- // Set up the mock for the status request- let status_mock = server- .mock("GET", "/v1/extract/extract-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "status": "completed",- "data": {- "title": "Example Domain",- "content": "This domain is for use in illustrative examples in documents."- }- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let urls = Some(vec!["https://example.com".to_string()]);- let params = ExtractParams {- urls,- schema: Some(json!({- "type": "object",- "properties": {- "title": { "type": "string" },- "content": { "type": "string" }- }- })),- ..Default::default()- };-- let response = app.extract(params).await.unwrap();-- assert!(response.success);- assert_eq!(response.status, "completed");-- let data = response.data.unwrap();- assert_eq!(data["title"], "Example Domain");- assert_eq!(- data["content"],- "This domain is for use in illustrative examples in documents."- );-- mock.assert();- status_mock.assert();- }-- #[tokio::test]- async fn test_extract_status_with_mock() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for the status check- let mock = server- .mock("GET", "/v1/extract/extract-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "status": "processing",- "urlTrace": [- {- "url": "https://example.com",- "status": "scraping",- "timing": {- "discoveredAt": "2023-01-01T00:00:00Z"- }- }- ]- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let status = app.get_extract_status("extract-123").await.unwrap();-- assert!(status.success);- assert_eq!(status.status, "processing");- assert_eq!(status.url_trace.unwrap()[0].url, "https://example.com");- mock.assert();- }-- #[tokio::test]- async fn test_extract_validation_errors() {- let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();-- // Test missing both URLs and prompt- let result = app.async_extract(ExtractParams::default()).await;- assert!(result.is_err());-- // Test having URLs but missing both prompt and schema- let params = ExtractParams {- urls: Some(vec!["https://example.com".to_string()]),- ..Default::default()- };- let result = app.async_extract(params).await;- assert!(result.is_err());- }-- #[tokio::test]- async fn test_extract_api_error() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for an error response- let mock = server- .mock("POST", "/v1/extract")- .with_status(400)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Invalid schema format"- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = ExtractParams {- urls: Some(vec!["https://example.com".to_string()]),- schema: Some(json!("invalid")), // Invalid schema format- ..Default::default()- };-- let result = app.async_extract(params).await;- assert!(result.is_err());- mock.assert();- }-}diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex 1b56c2e47..e439d2b8a 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -1,27 +1,12 @@//! Firecrawl Rust SDK//!-//! This SDK provides access to the Firecrawl API for web scraping and crawling.+//! This SDK provides access to the Firecrawl v2 API for web scraping, crawling,+//! searching, mapping, batch scraping, and agent operations.//!//! # Quick Start//!-//! ## v1 API (existing)-//!-//! ```no_run-//! use firecrawl::FirecrawlApp;-//!-//! #[tokio::main]-//! async fn main() -> Result<(), Box<dyn std::error::Error>> {-//! let app = FirecrawlApp::new("your-api-key")?;-//! let document = app.scrape_url("https://example.com", None).await?;-//! println!("{:?}", document.markdown);-//! Ok(())-//! }-//! ```-//!-//! ## v2 API (new)-//!//! ```no_run-//! use firecrawl::v2::Client;+//! use firecrawl::Client;//!//! #[tokio::main]//! async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -32,130 +17,24 @@//! }//! ```-use reqwest::{Client, Response};-use serde::de::DeserializeOwned;-use serde_json::Value;--// v1 modules (existing API)-pub mod batch_scrape;-pub mod crawl;-pub mod document;pub mod error;-pub mod extract;-pub mod llmstxt;-pub mod map;-pub mod scrape;-pub mod search;pub(crate) mod serde_helpers;-// v2 module (new API)-pub mod v2;--use error::FirecrawlAPIError;+mod agent;+mod batch_scrape;+mod client;+mod crawl;+mod map;+mod scrape;+mod search;+mod types;++pub use agent::*;+pub use batch_scrape::*;+pub use client::Client;+pub use crawl::*;pub use error::FirecrawlError;--#[derive(Clone, Debug)]-pub struct FirecrawlApp {- api_key: Option<String>,- api_url: String,- client: Client,-}--pub(crate) const API_VERSION: &str = "/v1";-const CLOUD_API_URL: &str = "https://api.firecrawl.dev";--impl FirecrawlApp {- pub fn new(api_key: impl AsRef<str>) -> Result<Self, FirecrawlError> {- FirecrawlApp::new_selfhosted(CLOUD_API_URL, Some(api_key))- }-- pub fn new_selfhosted(- api_url: impl AsRef<str>,- api_key: Option<impl AsRef<str>>,- ) -> Result<Self, FirecrawlError> {- let url = api_url.as_ref().to_string();-- if url == CLOUD_API_URL && api_key.is_none() {- return Err(FirecrawlError::APIError(- "Configuration".to_string(),- FirecrawlAPIError {- success: false,- error: "API key is required for cloud service".to_string(),- details: None,- },- ));- }-- Ok(FirecrawlApp {- api_key: api_key.map(|x| x.as_ref().to_string()),- api_url: url,- client: Client::new(),- })- }-- fn prepare_headers(&self, idempotency_key: Option<&String>) -> reqwest::header::HeaderMap {- let mut headers = reqwest::header::HeaderMap::new();- headers.insert("Content-Type", "application/json".parse().unwrap());- if let Some(api_key) = self.api_key.as_ref() {- headers.insert(- "Authorization",- format!("Bearer {}", api_key).parse().unwrap(),- );- }- if let Some(key) = idempotency_key {- headers.insert("x-idempotency-key", key.parse().unwrap());- }- headers- }-- async fn handle_response<'a, T: DeserializeOwned>(- &self,- response: Response,- action: impl AsRef<str>,- ) -> Result<T, FirecrawlError> {- let (is_success, status) = (response.status().is_success(), response.status());-- let response = response- .text()- .await- .map_err(|e| FirecrawlError::ResponseParseErrorText(e))- .and_then(|response_json| {- serde_json::from_str::<Value>(&response_json)- .map_err(|e| FirecrawlError::ResponseParseError(e))- .inspect(|data| {- tracing::debug!("Response JSON: {:#?}", data);- })- })- .and_then(|response_value| {- if action.as_ref().starts_with("crawl_") // no success in check/cancel crawl responses- || response_value["success"].as_bool().unwrap_or(false)- {- Ok(serde_json::from_value::<T>(response_value)- .map_err(|e| FirecrawlError::ResponseParseError(e))?)- } else {- Err(FirecrawlError::APIError(- action.as_ref().to_string(),- serde_json::from_value(response_value)- .map_err(|e| FirecrawlError::ResponseParseError(e))?,- ))- }- });-- match &response {- Ok(_) => response,- Err(FirecrawlError::ResponseParseError(_))- | Err(FirecrawlError::ResponseParseErrorText(_)) => {- if is_success {- response- } else {- Err(FirecrawlError::HttpRequestFailed(- action.as_ref().to_string(),- status.as_u16(),- status.as_str().to_string(),- ))- }- }- Err(_) => response,- }- }-}+pub use map::*;+pub use scrape::*;+pub use search::*;+pub use types::*;diff --git a/apps/rust-sdk/src/llmstxt.rs b/apps/rust-sdk/src/llmstxt.rsdeleted file mode 100644index f8fda7e91..000000000--- a/apps/rust-sdk/src/llmstxt.rs+++ /dev/null@@ -1,426 +0,0 @@-use serde::{Deserialize, Serialize};--use crate::{FirecrawlApp, FirecrawlError, API_VERSION};--/// Parameters for generating LLMs.txt-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct GenerateLLMsTextParams {- /// URL for which to generate LLMs.txt- pub url: String,-- /// Maximum number of URLs to process. Default: 10- pub max_urls: u32,-- /// Whether to show the full LLMs-full.txt in the response. Default: false- pub show_full_text: bool,-- /// Experimental streaming option- #[serde(rename = "__experimental_stream")]- pub experimental_stream: bool,-}--impl Default for GenerateLLMsTextParams {- fn default() -> Self {- Self {- url: String::new(),- max_urls: 1,- show_full_text: false,- experimental_stream: false,- }- }-}--/// Response from initiating a LLMs.txt generation job-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct GenerateLLMsTextResponse {- /// Whether the request was successful- pub success: bool,-- /// Job ID for the LLMs.txt generation- pub id: String,-}--#[derive(Deserialize, Serialize, Debug, Clone, Default)]-pub struct LLMTextData {- #[serde(rename = "llmstxt")]- pub compact: Option<String>,- #[serde(rename = "llmsfulltxt")]- pub full: Option<String>,-}--/// Response from checking the status of a LLMs.txt generation job-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct GenerateLLMsTextStatusResponse {- /// Whether the request was successful- pub success: bool,-- /// Status of the job: "pending", "processing", "completed", "failed"- pub status: String,-- /// Generated LLMs.txt data, present when status is "completed"- #[serde(default)]- pub data: LLMTextData,-- /// Error message if the job failed- pub error: Option<String>,-- /// Expiration timestamp for the data- pub expires_at: String,-}--impl FirecrawlApp {- /// Generates LLMs.txt for a given URL and polls until completion.- ///- /// # Arguments- ///- /// * `params` - Parameters for the LLMs.txt generation- ///- /// # Returns- ///- /// A response containing the generation results, or a FirecrawlError if the request fails.- pub async fn generate_llms_text(- &self,- params: impl Into<GenerateLLMsTextParams>,- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {- // Initiate the LLMs.txt generation job asynchronously- let response = self.async_generate_llms_text(params).await?;-- // Poll for the result- let poll_interval = 2000; // Default to 2 seconds- self.monitor_llms_text_job_status(&response.id, poll_interval)- .await- }-- /// Initiates an asynchronous LLMs.txt generation operation.- ///- /// # Arguments- ///- /// * `params` - Parameters for the LLMs.txt generation- ///- /// # Returns- ///- /// A response containing the generation job ID, or a FirecrawlError if the request fails.- pub async fn async_generate_llms_text(- &self,- params: impl Into<GenerateLLMsTextParams>,- ) -> Result<GenerateLLMsTextResponse, FirecrawlError> {- let params = params.into();-- // Validation: URL must be provided- if params.url.is_empty() {- return Err(FirecrawlError::APIError(- "Generate LLMs.txt validation".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: "URL must be provided".to_string(),- details: None,- },- ));- }-- let headers = self.prepare_headers(None);-- let response = self- .client- .post(format!("{}{}/llmstxt", self.api_url, API_VERSION))- .headers(headers)- .json(¶ms)- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError("Initiating LLMs.txt generation".to_string(), e)- })?;-- self.handle_response(response, "initiate LLMs.txt generation")- .await- }-- /// Checks the status of a LLMs.txt generation operation.- ///- /// # Arguments- ///- /// * `id` - The ID of the LLMs.txt generation operation- ///- /// # Returns- ///- /// A response containing the current status and results of the generation operation,- /// or a FirecrawlError if the request fails.- pub async fn check_generate_llms_text_status(- &self,- id: impl AsRef<str>,- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {- let response = self- .client- .get(format!(- "{}{}/llmstxt/{}",- self.api_url,- API_VERSION,- id.as_ref()- ))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(- format!("Checking status of LLMs.txt generation {}", id.as_ref()),- e,- )- })?;-- self.handle_response(- response,- format!("Checking status of LLMs.txt generation {}", id.as_ref()),- )- .await- }-- /// Helper function to poll for LLMs.txt generation job status until completion- async fn monitor_llms_text_job_status(- &self,- id: &str,- poll_interval: u64,- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {- loop {- let status_data = self.check_generate_llms_text_status(id).await?;-- match status_data.status.as_str() {- "completed" => {- break Ok(status_data);- }- "pending" | "processing" => {- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;- }- "failed" => {- let error_msg = status_data- .error- .clone()- .unwrap_or_else(|| "LLMs.txt generation failed".to_string());- break Err(FirecrawlError::APIError(- "LLMs.txt generation failed".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: error_msg,- details: None,- },- ));- }- _ => {- break Err(FirecrawlError::APIError(- "LLMs.txt generation status".to_string(),- crate::error::FirecrawlAPIError {- success: false,- error: format!("Unexpected status: {}", status_data.status),- details: None,- },- ));- }- }- }- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_generate_llms_text() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();-- let params = GenerateLLMsTextParams {- url: "https://example.com".to_string(),- max_urls: 5,- show_full_text: true,- ..Default::default()- };-- let response = app.async_generate_llms_text(params).await.unwrap();-- assert!(response.success);- assert!(!response.id.is_empty());- }-- #[tokio::test]- async fn test_async_generate_llms_text_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v1/llmstxt")- .match_body(mockito::Matcher::PartialJson(json!({- "url": "https://example.com",- "maxUrls": 5- })))- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "llmstxt-123"- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = GenerateLLMsTextParams {- url: "https://example.com".to_string(),- max_urls: 5,- ..Default::default()- };-- let response = app.async_generate_llms_text(params).await.unwrap();-- assert!(response.success);- assert_eq!(response.id, "llmstxt-123");- mock.assert();- }-- #[tokio::test]- async fn test_check_generate_llms_text_status_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("GET", "/v1/llmstxt/llmstxt-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "status": "processing",- "expiresAt": "2023-01-01T00:00:00Z"- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let status = app- .check_generate_llms_text_status("llmstxt-123")- .await- .unwrap();-- assert!(status.success);- assert_eq!(status.status, "processing");- assert_eq!(status.expires_at, "2023-01-01T00:00:00Z");- mock.assert();- }-- #[tokio::test]- async fn test_generate_llms_text_with_mock() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for the generate request- let mock = server- .mock("POST", "/v1/llmstxt")- .match_body(mockito::Matcher::PartialJson(json!({- "url": "https://example.com",- "showFullText": true- })))- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "llmstxt-123"- })- .to_string(),- )- .create();-- // Set up the mock for the status request- let status_mock = server- .mock("GET", "/v1/llmstxt/llmstxt-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "status": "completed",- "data": {- "llmstxt": "Allow: /about\nDisallow: /admin\n",- "llmsfulltxt": "# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n"- },- "expiresAt": "2023-01-01T00:00:00Z"- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = GenerateLLMsTextParams {- url: "https://example.com".to_string(),- show_full_text: true,- ..Default::default()- };-- let response = app.generate_llms_text(params).await.unwrap();-- assert!(response.success);- assert_eq!(response.status, "completed");-- let data = response.data;- assert_eq!(- data.compact,- Some("Allow: /about\nDisallow: /admin\n".into())- );- assert_eq!(- data.full,- Some("# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n".into())- );-- mock.assert();- status_mock.assert();- }-- #[tokio::test]- async fn test_generate_llms_text_validation_errors() {- let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();-- // Test missing URL- let params = GenerateLLMsTextParams {- url: "".to_string(),- ..Default::default()- };- let result = app.async_generate_llms_text(params).await;- assert!(result.is_err());- }-- #[tokio::test]- async fn test_generate_llms_text_api_error() {- let mut server = mockito::Server::new_async().await;-- // Set up the mock for an error response- let mock = server- .mock("POST", "/v1/llmstxt")- .with_status(400)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Invalid URL format"- })- .to_string(),- )- .create();-- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let params = GenerateLLMsTextParams {- url: "not-a-valid-url".to_string(),- ..Default::default()- };-- let result = app.async_generate_llms_text(params).await;- assert!(result.is_err());- mock.assert();- }-}diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 035785bf6..724e8bad2 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -1,63 +1,112 @@+//! Map endpoint for Firecrawl API v2.+use serde::{Deserialize, Serialize};-use crate::{FirecrawlApp, FirecrawlError, API_VERSION};+use crate::client::Client;+use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};+use crate::FirecrawlError;+/// Options for mapping a URL.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]-pub struct Location {- /// Country code (ISO 3166-1 alpha-2)- pub country: Option<String>,-- /// List of language codes- pub languages: Option<Vec<String>>,-}--#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]pub struct MapOptions {- /// Search query to use for mapping+ /// Search query to filter discovered links.pub search: Option<String>,- /// Ignore the website sitemap when crawling (default: `true`)- pub ignore_sitemap: Option<bool>,+ /// How to handle the sitemap.+ pub sitemap: Option<SitemapMode>,- /// Include subdomains of the website (default: `true`)+ /// Include subdomains in the mapping.pub include_subdomains: Option<bool>,- /// Maximum number of links to return (default: `5000`)+ /// Ignore query parameters when deduplicating URLs.+ pub ignore_query_parameters: Option<bool>,++ /// Maximum number of links to return.pub limit: Option<u32>,- /// Location configuration for proxy location- pub location: Option<Location>,+ /// Timeout in milliseconds.+ pub timeout: Option<u32>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Location configuration for proxy routing.+ pub location: Option<LocationConfig>,}+/// Request body for map endpoint.#[derive(Deserialize, Serialize, Debug, Default)]#[serde(rename_all = "camelCase")]-struct MapRequestBody {+struct MapRequest {url: String,-#[serde(flatten)]options: MapOptions,}-#[derive(Deserialize, Serialize, Debug, Default)]+/// Response from map endpoint.+#[derive(Deserialize, Serialize, Debug)]#[serde(rename_all = "camelCase")]-struct MapResponse {- success: bool,-- links: Vec<String>,+pub struct MapResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// Discovered links with metadata.+ pub links: Vec<SearchResultWeb>,+ /// Warning message if any.+ pub warning: Option<String>,}-impl FirecrawlApp {- /// Returns links from a URL using the Firecrawl API.- pub async fn map_url(+impl Client {+ /// Maps a URL to discover all associated links.+ ///+ /// This endpoint discovers links from a website's sitemap, page content,+ /// and other sources without fully scraping each page.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to map.+ /// * `options` - Optional mapping configuration.+ ///+ /// # Returns+ ///+ /// A `MapResponse` containing the discovered links.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, MapOptions, SitemapMode};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple map+ /// let response = client.map("https://example.com", None).await?;+ /// println!("Found {} links", response.links.len());+ ///+ /// // Map with options+ /// let options = MapOptions {+ /// sitemap: Some(SitemapMode::Include),+ /// include_subdomains: Some(true),+ /// limit: Some(1000),+ /// ..Default::default()+ /// };+ /// let response = client.map("https://example.com", options).await?;+ ///+ /// for link in response.links {+ /// println!("URL: {}, Title: {:?}", link.url, link.title);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn map(&self,url: impl AsRef<str>,options: impl Into<Option<MapOptions>>,- ) -> Result<Vec<String>, FirecrawlError> {- let body = MapRequestBody {+ ) -> Result<MapResponse, FirecrawlError> {+ let body = MapRequest {url: url.as_ref().to_string(),options: options.into().unwrap_or_default(),};@@ -66,17 +115,219 @@ impl FirecrawlApp {let response = self.client- .post(&format!("{}{}/map", self.api_url, API_VERSION))+ .post(self.url("/map")).headers(headers).json(&body).send().await.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;- let response = self- .handle_response::<MapResponse>(response, "scrape URL")- .await?;+ self.handle_response(response, "map").await+ }++ /// Maps a URL and returns just the list of URLs.+ ///+ /// This is a convenience method that returns only the URL strings.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to map.+ /// * `options` - Optional mapping configuration.+ ///+ /// # Returns+ ///+ /// A vector of discovered URL strings.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = client.map_urls("https://example.com", None).await?;+ /// for url in urls {+ /// println!("{}", url);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn map_urls(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<MapOptions>>,+ ) -> Result<Vec<String>, FirecrawlError> {+ let response = self.map(url, options).await?;+ Ok(response.links.into_iter().map(|link| link.url).collect())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_map_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ {+ "url": "https://example.com/",+ "title": "Example Domain",+ "description": "Home page"+ },+ {+ "url": "https://example.com/about",+ "title": "About Us"+ },+ {+ "url": "https://example.com/contact"+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.map("https://example.com", None).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 3);+ assert_eq!(response.links[0].url, "https://example.com/");+ assert_eq!(response.links[0].title, Some("Example Domain".to_string()));+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_with_options() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/page1" },+ { "url": "https://example.com/page2" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = MapOptions {+ sitemap: Some(SitemapMode::Include),+ include_subdomains: Some(true),+ limit: Some(100),+ ..Default::default()+ };++ let response = client.map("https://example.com", options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_urls() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/page1" },+ { "url": "https://example.com/page2" },+ { "url": "https://example.com/page3" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = client.map_urls("https://example.com", None).await.unwrap();++ assert_eq!(urls.len(), 3);+ assert_eq!(urls[0], "https://example.com/page1");+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_with_search_filter() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/blog/post1" },+ { "url": "https://example.com/blog/post2" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = MapOptions {+ search: Some("blog".to_string()),+ ..Default::default()+ };++ let response = client.map("https://example.com", options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid URL"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.map("invalid-url", None).await;- Ok(response.links)+ assert!(result.is_err());+ mock.assert();}}diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 5e2496fb9..2625c47bf 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -1,144 +1,244 @@-use std::collections::HashMap;+//! Scrape endpoint for Firecrawl API v2.use serde::{Deserialize, Serialize};use serde_json::Value;+use std::collections::HashMap;-use crate::{document::Document, FirecrawlApp, FirecrawlError, API_VERSION};+use crate::client::Client;+use crate::types::{+ Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,+ LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,+};+use crate::FirecrawlError;-#[derive(Deserialize, Serialize, Clone, Copy, Debug)]-pub enum ScrapeFormats {- /// Will result in a copy of the Markdown content of the page.- #[serde(rename = "markdown")]- Markdown,+/// Options for scraping a URL.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeOptions {+ /// Output formats to include in the response.+ pub formats: Option<Vec<Format>>,- /// Will result in a copy of the filtered, content-only HTML of the page.- #[serde(rename = "html")]- HTML,+ /// Additional HTTP headers to send with the request.+ pub headers: Option<HashMap<String, String>>,- /// Will result in a copy of the raw HTML of the page.- #[serde(rename = "rawHtml")]- RawHTML,+ /// HTML tags to exclusively include in the output.+ pub include_tags: Option<Vec<String>>,- /// Will result in a Vec of URLs found on the page.- #[serde(rename = "links")]- Links,+ /// HTML tags to exclude from the output.+ pub exclude_tags: Option<Vec<String>>,- /// Will result in a URL to a screenshot of the page.- ///- /// Can not be used in conjunction with `ScrapeFormats::ScreenshotFullPage`.- #[serde(rename = "screenshot")]- Screenshot,+ /// Only extract the main content of the page.+ pub only_main_content: Option<bool>,- /// Will result in a URL to a full-page screenshot of the page.- ///- /// Can not be used in conjunction with `ScrapeFormats::Screenshot`.- #[serde(rename = "screenshot@fullPage")]- ScreenshotFullPage,+ /// Timeout in milliseconds before returning an error.+ pub timeout: Option<u32>,- /// Will result in the results of an LLM extraction.- ///- /// See `ScrapeOptions.json_options` for more options.- #[serde(rename = "json")]- Json,-}+ /// Time to wait after page load before scraping (milliseconds).+ pub wait_for: Option<u32>,-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct AgentOptionsJson {- pub model: String,- pub prompt: Option<String>,-}+ /// Emulate a mobile device.+ pub mobile: Option<bool>,-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct JsonOptions {- /// Schema the output should adhere to, provided in JSON Schema format.- pub schema: Option<Value>,+ /// Parser configurations (e.g., for PDFs).+ pub parsers: Option<Vec<ParserConfig>>,- /// System prompt to send to the LLM agent along with the page content.- pub system_prompt: Option<String>,+ /// Browser automation actions to perform before scraping.+ pub actions: Option<Vec<Action>>,- /// Extraction prompt to send to the LLM agent along with the page content.- pub prompt: Option<String>,+ /// Location configuration for proxy routing.+ pub location: Option<LocationConfig>,- /// Agent options for JSON extraction.- pub agent: Option<AgentOptionsJson>,-}+ /// Skip TLS certificate verification.+ pub skip_tls_verification: Option<bool>,-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct AgentOptions {- pub model: String,- pub prompt: Option<String>,- pub session_id: Option<String>,- pub wait_before_closing_ms: Option<u32>,-}+ /// Remove base64-encoded images from the output.+ pub remove_base64_images: Option<bool>,-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ScrapeOptions {- /// Formats to extract from the page. (default: `[ Markdown ]`)- pub formats: Option<Vec<ScrapeFormats>>,+ /// Enable fast mode for quicker scrapes with reduced accuracy.+ pub fast_mode: Option<bool>,- /// Only extract the main content of the page, excluding navigation and other miscellaneous content. (default: `true`)- pub only_main_content: Option<bool>,+ /// Block advertisements on the page.+ pub block_ads: Option<bool>,- /// HTML tags to exclusively include.- ///- /// For example, if you pass `div`, you will only get content from `<div>`s and their children.- pub include_tags: Option<Vec<String>>,+ /// Proxy type to use.+ pub proxy: Option<ProxyType>,- /// HTML tags to exclude.- ///- /// For example, if you pass `img`, you will never get image URLs in your results.- pub exclude_tags: Option<Vec<String>>,+ /// Maximum age of cached content to accept (seconds).+ pub max_age: Option<u32>,- /// Additional HTTP headers to use when loading the page.- pub headers: Option<HashMap<String, String>>,+ /// Minimum age of cached content to accept (seconds).+ pub min_age: Option<u32>,- // Amount of time to wait after loading the page, and before grabbing the content, in milliseconds. (default: `0`)- pub wait_for: Option<u32>,+ /// Store the result in cache for future requests.+ pub store_in_cache: Option<bool>,- // Timeout before returning an error, in milliseconds. (default: `60000`)- pub timeout: Option<u32>,+ /// Persistent browser profile for maintaining state across scrapes.+ pub profile: Option<ProfileConfig>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,- /// JSON extraction options, to be used in conjunction with `ScrapeFormats::Json`.+ /// JSON extraction options.pub json_options: Option<JsonOptions>,- /// Agent options for smart scrape.- pub agent: Option<AgentOptions>,+ /// Screenshot options.+ pub screenshot_options: Option<ScreenshotOptions>,++ /// Change tracking options.+ pub change_tracking_options: Option<ChangeTrackingOptions>,++ /// Attribute selectors for extraction.+ pub attribute_selectors: Option<Vec<AttributeSelector>>,}+/// Parser configuration for document parsing.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(untagged)]+pub enum ParserConfig {+ /// Simple parser type string.+ Simple(String),+ /// PDF parser with options.+ Pdf {+ #[serde(rename = "type")]+ parser_type: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ max_pages: Option<u32>,+ },+}++/// Request body for scrape endpoint.#[derive(Deserialize, Serialize, Debug, Default)]#[serde(rename_all = "camelCase")]-struct ScrapeRequestBody {+struct ScrapeRequest {url: String,-#[serde(flatten)]options: ScrapeOptions,}-#[derive(Deserialize, Serialize, Debug, Default)]+/// Response from scrape endpoint.+#[derive(Deserialize, Serialize, Debug)]#[serde(rename_all = "camelCase")]struct ScrapeResponse {- /// This will always be `true` due to `FirecrawlApp::handle_response`.- /// No need to expose.success: bool,-- /// The resulting document.data: Document,+ #[serde(skip_serializing_if = "Option::is_none")]+ warning: Option<String>,+}++/// Supported languages for scrape-bound browser execution.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum ScrapeExecuteLanguage {+ Python,+ Node,+ Bash,+}++/// Options for executing code or a prompt in a scrape-bound browser session.+///+/// At least one of `code` or `prompt` must be provided.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeExecuteOptions {+ /// Code to execute (optional if `prompt` is provided).+ pub code: Option<String>,+ /// Natural-language instruction for the browser agent (optional if `code` is provided).+ pub prompt: Option<String>,+ /// Runtime language for the code.+ pub language: Option<ScrapeExecuteLanguage>,+ /// Execution timeout in seconds.+ pub timeout: Option<u32>,+ /// Optional origin tag for request attribution.+ pub origin: Option<String>,}-impl FirecrawlApp {- /// Scrapes a URL using the Firecrawl API.- pub async fn scrape_url(+/// Response from scrape-bound browser execution.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeExecuteResponse {+ /// Whether the request succeeded.+ pub success: bool,+ /// Live-view URL for the browser session.+ pub live_view_url: Option<String>,+ /// Interactive live-view URL for the browser session.+ pub interactive_live_view_url: Option<String>,+ /// Agent output when a prompt was used.+ pub output: Option<String>,+ /// Captured stdout from execution.+ pub stdout: Option<String>,+ /// Optional execution result payload.+ pub result: Option<String>,+ /// Captured stderr from execution.+ pub stderr: Option<String>,+ /// Process exit code.+ pub exit_code: Option<i32>,+ /// Whether execution was killed by timeout or system.+ pub killed: Option<bool>,+ /// Error message when execution fails.+ pub error: Option<String>,+}++/// Response from deleting a scrape-bound browser session.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeBrowserDeleteResponse {+ /// Whether the delete request succeeded.+ pub success: bool,+ /// Session duration in milliseconds when available.+ pub session_duration_ms: Option<u64>,+ /// Credits billed when available.+ pub credits_billed: Option<u32>,+ /// Error message when deletion fails.+ pub error: Option<String>,+}++impl Client {+ /// Scrapes a URL and returns the content in the requested formats.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to scrape.+ /// * `options` - Optional scrape configuration.+ ///+ /// # Returns+ ///+ /// A `Document` containing the scraped content.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::{Client, ScrapeOptions, Format};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple scrape+ /// let document = client.scrape("https://example.com", None).await?;+ /// println!("Markdown: {:?}", document.markdown);+ ///+ /// // Scrape with options+ /// let options = ScrapeOptions {+ /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ /// only_main_content: Some(true),+ /// ..Default::default()+ /// };+ /// let document = client.scrape("https://example.com", options).await?;+ /// println!("Links: {:?}", document.links);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn scrape(&self,url: impl AsRef<str>,options: impl Into<Option<ScrapeOptions>>,) -> Result<Document, FirecrawlError> {- let body = ScrapeRequestBody {+ let body = ScrapeRequest {url: url.as_ref().to_string(),options: options.into().unwrap_or_default(),};@@ -147,17 +247,485 @@ impl FirecrawlApp {let response = self.client- .post(&format!("{}{}/scrape", self.api_url, API_VERSION))+ .post(self.url("/scrape")).headers(headers).json(&body).send().await.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;- let response = self- .handle_response::<ScrapeResponse>(response, "scrape URL")- .await?;+ let response: ScrapeResponse = self.handle_response(response, "scrape").await?;Ok(response.data)}++ /// Scrapes a URL with a JSON schema for structured extraction.+ ///+ /// This is a convenience method that combines scraping with JSON extraction.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to scrape.+ /// * `schema` - JSON schema for the extraction.+ /// * `prompt` - Optional extraction prompt.+ ///+ /// # Returns+ ///+ /// The extracted JSON value matching the schema.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ /// use serde_json::json;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let schema = json!({+ /// "type": "object",+ /// "properties": {+ /// "title": { "type": "string" },+ /// "price": { "type": "number" }+ /// }+ /// });+ ///+ /// let data = client.scrape_with_schema(+ /// "https://example.com/product",+ /// schema,+ /// Some("Extract the product title and price")+ /// ).await?;+ ///+ /// println!("Extracted: {}", data);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn scrape_with_schema(+ &self,+ url: impl AsRef<str>,+ schema: Value,+ prompt: Option<impl AsRef<str>>,+ ) -> Result<Value, FirecrawlError> {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Json]),+ json_options: Some(JsonOptions {+ schema: Some(schema),+ prompt: prompt.map(|p| p.as_ref().to_string()),+ ..Default::default()+ }),+ ..Default::default()+ };++ let document = self.scrape(url, options).await?;+ Ok(document.json.unwrap_or(Value::Null))+ }++ /// Interacts with the browser session associated with a scrape job.+ ///+ /// # Arguments+ ///+ /// * `job_id` - The scrape job ID.+ /// * `options` - Execution options including code and runtime config.+ ///+ /// # Returns+ ///+ /// A `ScrapeExecuteResponse` containing execution output.+ pub async fn interact(+ &self,+ job_id: impl AsRef<str>,+ options: ScrapeExecuteOptions,+ ) -> Result<ScrapeExecuteResponse, FirecrawlError> {+ let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());+ let has_prompt = options+ .prompt+ .as_ref()+ .is_some_and(|p| !p.trim().is_empty());+ if !has_code && !has_prompt {+ return Err(FirecrawlError::Misuse(+ "Either 'code' or 'prompt' must be provided".into(),+ ));+ }++ let mut body = options;+ if body.language.is_none() {+ body.language = Some(ScrapeExecuteLanguage::Node);+ }++ let response = self+ .client+ .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))+ .headers(self.prepare_headers(None))+ .json(&body)+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Interacting with scrape browser for {}", job_id.as_ref()),+ e,+ )+ })?;++ self.handle_response(response, "scrape interact").await+ }++ /// Stops the interaction session associated with a scrape job.+ ///+ /// # Arguments+ ///+ /// * `job_id` - The scrape job ID.+ ///+ /// # Returns+ ///+ /// A `ScrapeBrowserDeleteResponse` indicating stop status.+ pub async fn stop_interaction(+ &self,+ job_id: impl AsRef<str>,+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {+ let response = self+ .client+ .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Stopping interaction for {}", job_id.as_ref()),+ e,+ )+ })?;++ self.handle_response(response, "stop interaction").await+ }++ /// Deprecated alias for [`Client::interact`].+ #[deprecated(note = "Use interact() instead")]+ pub async fn scrape_execute(+ &self,+ job_id: impl AsRef<str>,+ options: ScrapeExecuteOptions,+ ) -> Result<ScrapeExecuteResponse, FirecrawlError> {+ self.interact(job_id, options).await+ }++ /// Deprecated alias for [`Client::stop_interaction`].+ #[deprecated(note = "Use stop_interaction() instead")]+ pub async fn stop_interactive_browser(+ &self,+ job_id: impl AsRef<str>,+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {+ self.stop_interaction(job_id).await+ }++ /// Deprecated alias for [`Client::stop_interaction`].+ #[deprecated(note = "Use stop_interaction() instead")]+ pub async fn delete_scrape_browser(+ &self,+ job_id: impl AsRef<str>,+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {+ self.stop_interaction(job_id).await+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_scrape_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "markdown": "# Example Domain\n\nThis is an example.",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example Domain"+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let document = client.scrape("https://example.com", None).await.unwrap();++ assert!(document.markdown.is_some());+ assert!(document.markdown.unwrap().contains("Example Domain"));+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_with_options() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "markdown": "# Test",+ "html": "<h1>Test</h1>",+ "links": ["https://example.com/page1", "https://example.com/page2"],+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ only_main_content: Some(true),+ ..Default::default()+ };++ let document = client.scrape("https://example.com", options).await.unwrap();++ assert!(document.markdown.is_some());+ assert!(document.html.is_some());+ assert!(document.links.is_some());+ assert_eq!(document.links.unwrap().len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_with_schema() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "json": {+ "title": "Product Name",+ "price": 99.99+ },+ "metadata": {+ "sourceURL": "https://example.com/product",+ "statusCode": 200+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "price": { "type": "number" }+ }+ });++ let data = client+ .scrape_with_schema(+ "https://example.com/product",+ schema,+ Some("Extract product info"),+ )+ .await+ .unwrap();++ assert_eq!(data["title"], "Product Name");+ assert_eq!(data["price"], 99.99);+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid URL"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.scrape("invalid-url", None).await;++ assert!(result.is_err());+ mock.assert();+ }++ #[tokio::test]+ async fn test_interact_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape/job-123/interact")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "stdout": "ok",+ "result": "done",+ "stderr": "",+ "exitCode": 0,+ "killed": false+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client+ .interact(+ "job-123",+ ScrapeExecuteOptions {+ code: Some("console.log('ok')".to_string()),+ timeout: Some(30),+ ..Default::default()+ },+ )+ .await+ .unwrap();++ assert!(response.success);+ assert_eq!(response.exit_code, Some(0));+ assert_eq!(response.result, Some("done".to_string()));+ mock.assert();+ }++ #[tokio::test]+ async fn test_interact_with_prompt() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape/job-789/interact")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "output": "Clicked the login button",+ "liveViewUrl": "https://live.example.com/view",+ "interactiveLiveViewUrl": "https://live.example.com/interactive",+ "stdout": "",+ "exitCode": 0,+ "killed": false+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client+ .interact(+ "job-789",+ ScrapeExecuteOptions {+ prompt: Some("Click the login button".to_string()),+ ..Default::default()+ },+ )+ .await+ .unwrap();++ assert!(response.success);+ assert_eq!(+ response.output,+ Some("Clicked the login button".to_string())+ );+ assert_eq!(+ response.live_view_url,+ Some("https://live.example.com/view".to_string())+ );+ assert_eq!(+ response.interactive_live_view_url,+ Some("https://live.example.com/interactive".to_string())+ );+ mock.assert();+ }++ #[tokio::test]+ async fn test_stop_interaction_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("DELETE", "/v2/scrape/job-123/interact")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "sessionDurationMs": 1200,+ "creditsBilled": 3+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.stop_interaction("job-123").await.unwrap();++ assert!(response.success);+ assert_eq!(response.session_duration_ms, Some(1200));+ assert_eq!(response.credits_billed, Some(3));+ mock.assert();+ }++ #[tokio::test]+ async fn test_interact_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape/job-404/interact")+ .with_status(404)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Job not found."+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client+ .interact(+ "job-404",+ ScrapeExecuteOptions {+ code: Some("console.log('ok')".to_string()),+ ..Default::default()+ },+ )+ .await;++ assert!(result.is_err());+ mock.assert();+ }}diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rsindex 397bc8d16..534559bbb 100644--- a/apps/rust-sdk/src/search.rs+++ b/apps/rust-sdk/src/search.rs@@ -1,223 +1,386 @@-use crate::{scrape::ScrapeOptions, FirecrawlApp, FirecrawlError, API_VERSION};+//! Search endpoint for Firecrawl API v2.+use serde::{Deserialize, Serialize};-#[derive(Deserialize, Serialize, Debug, Clone)]+use crate::client::Client;+use crate::scrape::ScrapeOptions;+use crate::types::{+ Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,+};+use crate::FirecrawlError;++/// Options for search requests.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]-pub struct SearchParams {- /// The search query string- pub query: String,- /// Maximum number of results to return. Default: 5, Max: 20+pub struct SearchOptions {+ /// Maximum number of results to return. Default: 5, Max: 20.pub limit: Option<u32>,- /// Time-based search filter.- #[serde(skip_serializing_if = "Option::is_none")]++ /// Search sources to query (web, news, images).+ pub sources: Option<Vec<SearchSource>>,++ /// Categories to filter results (github, research, pdf).+ pub categories: Option<Vec<SearchCategory>>,++ /// Time-based search filter (e.g., "qdr:d" for past day).pub tbs: Option<String>,- /// Query string to filter search results. Example: "site:example.com"- #[serde(skip_serializing_if = "Option::is_none")]- pub filter: Option<String>,- /// Language code. Default: "en"- pub lang: Option<String>,- /// Country code. Default: "us"- pub country: Option<String>,- /// Geographic location string for local search results- #[serde(skip_serializing_if = "Option::is_none")]++ /// Geographic location string for local search results.pub location: Option<String>,- /// Origin identifier. Default: "api"- pub origin: Option<String>,- /// Timeout in milliseconds. Default: 60000++ /// Whether to ignore invalid URLs in results.+ pub ignore_invalid_urls: Option<bool>,++ /// Timeout in milliseconds.pub timeout: Option<u32>,- /// Additional options for webpage scraping behavior- #[serde(skip_serializing_if = "Option::is_none")]++ /// Scrape options to apply to each search result.pub scrape_options: Option<ScrapeOptions>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,+}++/// Request body for search endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct SearchRequest {+ query: String,+ #[serde(flatten)]+ options: SearchOptions,}-impl Default for SearchParams {- fn default() -> Self {- Self {- query: String::new(),- limit: Some(5),- tbs: None,- filter: None,- lang: Some("en".to_string()),- country: Some("us".to_string()),- location: None,- origin: Some("api".to_string()),- timeout: Some(60000),- scrape_options: None,+/// Search results data structure.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchData {+ /// Web search results (may include scraped documents).+ pub web: Option<Vec<SearchResultOrDocument>>,+ /// News search results.+ pub news: Option<Vec<SearchResultNews>>,+ /// Image search results.+ pub images: Option<Vec<SearchResultImage>>,+}++/// A search result that may be a simple result or a full document.+///+/// Uses custom deserialization to properly distinguish between web results+/// and scraped documents by checking for document-specific fields.+#[derive(Serialize, Debug, Clone)]+#[serde(untagged)]+pub enum SearchResultOrDocument {+ /// Simple web search result.+ WebResult(SearchResultWeb),+ /// Full scraped document.+ Document(Document),+}++impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ use serde_json::Value;++ let value = Value::deserialize(deserializer)?;++ // Check for document-specific fields that indicate scraped content+ // If any of these exist, it's a Document, not a simple WebResult+ let is_document = value.get("markdown").is_some()+ || value.get("html").is_some()+ || value.get("rawHtml").is_some()+ || value.get("metadata").is_some();++ if is_document {+ Document::deserialize(value)+ .map(SearchResultOrDocument::Document)+ .map_err(serde::de::Error::custom)+ } else {+ SearchResultWeb::deserialize(value)+ .map(SearchResultOrDocument::WebResult)+ .map_err(serde::de::Error::custom)}}}-#[derive(Deserialize, Serialize, Debug, Clone)]+/// Response from search endpoint.+#[derive(Deserialize, Serialize, Debug)]#[serde(rename_all = "camelCase")]pub struct SearchResponse {+ /// Whether the request was successful.pub success: bool,- pub data: Vec<SearchDocument>,+ /// Search results data.+ pub data: SearchData,+ /// Warning message if any.pub warning: Option<String>,}-// TODO: Consider merging fields into document::Document (url, title, description) while preserving optionality-/// A document returned from a search or scrape request-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct SearchDocument {- /// Document URL- pub url: String,- /// Document title- pub title: String,- /// Document description- pub description: String,-}--impl FirecrawlApp {- /// Search for content using the Firecrawl API.+impl Client {+ /// Searches the web and optionally scrapes the results.////// # Arguments///- /// * `query` - The search query string- /// * `params` - Optional parameters for the search request+ /// * `query` - The search query string.+ /// * `options` - Optional search configuration.////// # Returns///- /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.- pub async fn search(- &self,- query: impl AsRef<str>,- params: impl Into<Option<SearchParams>>,- ) -> Result<SearchResponse, FirecrawlError> {- let mut search_params = params.into().unwrap_or_default();- search_params.query = query.as_ref().to_string();-- self.search_with_params(search_params).await- }-- /// Alternative method that takes SearchParams directly+ /// A `SearchResponse` containing the search results.///- /// # Arguments+ /// # Example///- /// * `params` - Search parameters including the query+ /// ```no_run+ /// use firecrawl::{Client, SearchOptions, SearchSource};///- /// # Returns+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple search+ /// let results = client.search("rust programming", None).await?;+ /// for result in results.data.web.unwrap_or_default() {+ /// match result {+ /// firecrawl::SearchResultOrDocument::WebResult(r) => {+ /// println!("URL: {}", r.url);+ /// }+ /// firecrawl::SearchResultOrDocument::Document(d) => {+ /// println!("Content: {:?}", d.markdown);+ /// }+ /// }+ /// }///- /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.- pub async fn search_with_params(+ /// // Search with options+ /// let options = SearchOptions {+ /// limit: Some(10),+ /// sources: Some(vec![SearchSource::Web, SearchSource::News]),+ /// ..Default::default()+ /// };+ /// let results = client.search("rust programming", options).await?;+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn search(&self,- params: SearchParams,+ query: impl AsRef<str>,+ options: impl Into<Option<SearchOptions>>,) -> Result<SearchResponse, FirecrawlError> {+ let body = SearchRequest {+ query: query.as_ref().to_string(),+ options: options.into().unwrap_or_default(),+ };+let headers = self.prepare_headers(None);let response = self.client- .post(format!("{}{}/search", self.api_url, API_VERSION))+ .post(self.url("/search")).headers(headers)- .json(¶ms)+ .json(&body).send().await.map_err(|e| {- FirecrawlError::HttpError(format!("Searching with query: {:?}", params.query), e)+ FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)})?;- self.handle_response::<SearchResponse>(response, "search")- .await+ self.handle_response(response, "search").await+ }++ /// Searches the web and scrapes the results.+ ///+ /// This is a convenience method that enables scraping for all results.+ ///+ /// # Arguments+ ///+ /// * `query` - The search query string.+ /// * `limit` - Maximum number of results to return.+ ///+ /// # Returns+ ///+ /// A vector of scraped documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let documents = client.search_and_scrape("rust programming", 5).await?;+ /// for doc in documents {+ /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));+ /// println!("Content: {:?}", doc.markdown);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn search_and_scrape(+ &self,+ query: impl AsRef<str>,+ limit: u32,+ ) -> Result<Vec<Document>, FirecrawlError> {+ let options = SearchOptions {+ limit: Some(limit),+ scrape_options: Some(ScrapeOptions::default()),+ ..Default::default()+ };++ let response = self.search(query, options).await?;++ let documents: Vec<Document> = response+ .data+ .web+ .unwrap_or_default()+ .into_iter()+ .filter_map(|result| match result {+ SearchResultOrDocument::Document(doc) => Some(doc),+ SearchResultOrDocument::WebResult(_) => None,+ })+ .collect();++ Ok(documents)}}#[cfg(test)]-pub mod tests {+mod tests {use super::*;use serde_json::json;- #[tokio::test]- #[ignore = "Makes real network request"]- async fn test_real_search() {- let api_url = std::env::var("FIRECRAWL_API_URL")- .expect("Please set the FIRECRAWL_API_URL environment variable");- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();- let response = app.search("test query", None).await.unwrap();- assert!(response.success);- }-#[tokio::test]async fn test_search_with_mock() {let mut server = mockito::Server::new_async().await;let mock = server- .mock("POST", "/v1/search")+ .mock("POST", "/v2/search").with_status(200).with_header("content-type", "application/json").with_body(json!({"success": true,- "data": [{- "url": "https://example.com",- "title": "Example Domain",- "description": "...."- }],+ "data": {+ "web": [+ {+ "url": "https://example.com",+ "title": "Example Domain",+ "description": "This domain is for examples"+ },+ {+ "url": "https://example.org",+ "title": "Another Example",+ "description": "More examples"+ }+ ]+ }}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = app.search("test", None).await.unwrap();+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.search("test query", None).await.unwrap();assert!(response.success);- assert_eq!(response.data.len(), 1);- assert_eq!(response.data[0].url, "https://example.com");- assert_eq!(response.data[0].title, "Example Domain".to_string());- assert_eq!(response.data[0].description, "....".to_string());+ let web_results = response.data.web.unwrap();+ assert_eq!(web_results.len(), 2);mock.assert();}#[tokio::test]- async fn test_search_with_params() {+ async fn test_search_with_options() {let mut server = mockito::Server::new_async().await;+let mock = server- .mock("POST", "/v1/search")- .with_header("content-type", "application/json")- .match_body(mockito::Matcher::Json(json!({- "query": "test",- "limit": 10,- "lang": "fr",- "country": "fr",- "origin": "api",- "timeout": 30000- })))+ .mock("POST", "/v2/search").with_status(200).with_header("content-type", "application/json").with_body(json!({"success": true,- "data": [],- "warning": "No results found"+ "data": {+ "web": [+ {+ "url": "https://example.com",+ "title": "Test Result",+ "description": "A test result"+ }+ ],+ "news": [+ {+ "title": "Breaking News",+ "url": "https://news.example.com",+ "snippet": "Something happened",+ "date": "2024-01-01"+ }+ ]+ }}).to_string(),).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let params = SearchParams {- query: "test".to_string(),+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = SearchOptions {limit: Some(10),- lang: Some("fr".to_string()),- country: Some("fr".to_string()),- timeout: Some(30000),+ sources: Some(vec![SearchSource::Web, SearchSource::News]),..Default::default()};- let response = app.search_with_params(params).await.unwrap();+ let response = client.search("test", options).await.unwrap();assert!(response.success);- assert_eq!(response.data.len(), 0);- assert_eq!(response.warning, Some("No results found".to_string()));+ assert!(response.data.web.is_some());+ assert!(response.data.news.is_some());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_and_scrape() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ {+ "markdown": "# Example\n\nThis is the scraped content.",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example"+ }+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let documents = client.search_and_scrape("test", 5).await.unwrap();++ assert_eq!(documents.len(), 1);+ assert!(documents[0].markdown.is_some());mock.assert();}#[tokio::test]async fn test_search_error_response() {let mut server = mockito::Server::new_async().await;+let mock = server- .mock("POST", "/v1/search")+ .mock("POST", "/v2/search").with_status(400).with_header("content-type", "application/json").with_body(@@ -229,17 +392,75 @@ pub mod tests {).create();- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = app.search("", None).await;+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.search("", None).await;assert!(result.is_err());mock.assert();}#[tokio::test]- async fn test_search_network_error() {- let app = FirecrawlApp::new_selfhosted("http://invalid-url", Some("test_key")).unwrap();- let result = app.search("test", None).await;- assert!(result.is_err());+ async fn test_search_mixed_results_deserialization() {+ // Test that results with markdown/metadata are correctly identified as Documents+ // and simple results are identified as WebResults+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ // Simple web result (no markdown/metadata)+ {+ "url": "https://example.com/simple",+ "title": "Simple Result",+ "description": "Just a search result"+ },+ // Scraped document with url AND markdown+ {+ "url": "https://example.com/scraped",+ "markdown": "# Scraped Content\n\nThis has content.",+ "metadata": {+ "sourceURL": "https://example.com/scraped",+ "statusCode": 200,+ "title": "Scraped Page"+ }+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.search("test", None).await.unwrap();++ let web_results = response.data.web.unwrap();+ assert_eq!(web_results.len(), 2);++ // First should be WebResult+ match &web_results[0] {+ SearchResultOrDocument::WebResult(r) => {+ assert_eq!(r.url, "https://example.com/simple");+ assert_eq!(r.title, Some("Simple Result".to_string()));+ }+ SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),+ }++ // Second should be Document (has markdown)+ match &web_results[1] {+ SearchResultOrDocument::Document(d) => {+ assert!(d.markdown.is_some());+ assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));+ }+ SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),+ }++ mock.assert();}}diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/types.rssimilarity index 100%rename from apps/rust-sdk/src/v2/types.rsrename to apps/rust-sdk/src/types.rsdiff --git a/apps/rust-sdk/src/v2/batch_scrape.rs b/apps/rust-sdk/src/v2/batch_scrape.rsdeleted file mode 100644index cb831e787..000000000--- a/apps/rust-sdk/src/v2/batch_scrape.rs+++ /dev/null@@ -1,591 +0,0 @@-//! Batch scrape endpoint for Firecrawl API v2.--use serde::{Deserialize, Serialize};--use super::client::Client;-use super::scrape::ScrapeOptions;-use super::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};-use crate::FirecrawlError;--/// Options for batch scraping.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct BatchScrapeOptions {- /// Scrape options to apply to all URLs.- #[serde(flatten)]- pub options: Option<ScrapeOptions>,-- /// Webhook configuration for job notifications.- pub webhook: Option<WebhookConfig>,-- /// ID of an existing batch job to append URLs to.- pub append_to_id: Option<String>,-- /// Whether to ignore invalid URLs instead of failing.- #[serde(rename = "ignoreInvalidURLs")]- pub ignore_invalid_urls: Option<bool>,-- /// Maximum concurrent requests.- pub max_concurrency: Option<u32>,-- /// Enable zero data retention mode.- pub zero_data_retention: Option<bool>,-- /// Idempotency key for the request.- #[serde(skip)]- pub idempotency_key: Option<String>,-- /// Integration identifier for tracking.- pub integration: Option<String>,-- /// Poll interval for synchronous batch scrape (milliseconds).- #[serde(skip)]- pub poll_interval: Option<u64>,-}--/// Request body for batch scrape endpoint.-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct BatchScrapeRequest {- urls: Vec<String>,- #[serde(flatten)]- options: BatchScrapeOptions,-}--/// Response from starting a batch scrape job.-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct BatchScrapeResponse {- /// Whether the request was successful.- pub success: bool,- /// The batch scrape job ID.- pub id: String,- /// URL to check the batch scrape status.- pub url: String,- /// URLs that were invalid and ignored.- #[serde(rename = "invalidURLs")]- pub invalid_urls: Option<Vec<String>>,-}--/// Status of a batch scrape job.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct BatchScrapeJob {- /// Current status of the batch scrape job.- pub status: JobStatus,- /// Number of URLs completed.- pub completed: u32,- /// Total number of URLs to scrape.- pub total: u32,- /// Credits used by the batch scrape.- pub credits_used: Option<u32>,- /// Expiry time of the batch data.- pub expires_at: Option<String>,- /// URL for the next page of results.- pub next: Option<String>,- /// Scraped documents.- pub data: Vec<Document>,-}--impl Client {- /// Starts a batch scrape job asynchronously.- ///- /// Returns immediately with a job ID that can be used to check status.- ///- /// # Arguments- ///- /// * `urls` - List of URLs to scrape.- /// * `options` - Optional batch scrape configuration.- ///- /// # Returns- ///- /// A `BatchScrapeResponse` containing the job ID.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, BatchScrapeOptions};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let urls = vec![- /// "https://example.com".to_string(),- /// "https://example.org".to_string(),- /// ];- ///- /// let response = client.start_batch_scrape(urls, None).await?;- /// println!("Batch job started: {}", response.id);- ///- /// // Check status later- /// let status = client.get_batch_scrape_status(&response.id).await?;- /// println!("Completed: {}/{}", status.completed, status.total);- ///- /// Ok(())- /// }- /// ```- pub async fn start_batch_scrape(- &self,- urls: Vec<String>,- options: impl Into<Option<BatchScrapeOptions>>,- ) -> Result<BatchScrapeResponse, FirecrawlError> {- let options = options.into().unwrap_or_default();- let body = BatchScrapeRequest {- urls,- options: options.clone(),- };-- let headers = self.prepare_headers(options.idempotency_key.as_ref());-- let response = self- .client- .post(self.url("/batch/scrape"))- .headers(headers)- .json(&body)- .send()- .await- .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;-- self.handle_response(response, "start batch scrape").await- }-- /// Gets the status of a batch scrape job.- ///- /// If the job is completed, this will automatically fetch all pages of results.- ///- /// # Arguments- ///- /// * `id` - The batch scrape job ID.- ///- /// # Returns- ///- /// A `BatchScrapeJob` containing the current status and any available documents.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let status = client.get_batch_scrape_status("job-id").await?;- /// println!("Status: {:?}", status.status);- /// println!("Completed: {}/{}", status.completed, status.total);- /// println!("Documents: {}", status.data.len());- ///- /// Ok(())- /// }- /// ```- pub async fn get_batch_scrape_status(- &self,- id: impl AsRef<str>,- ) -> Result<BatchScrapeJob, FirecrawlError> {- let response = self- .client- .get(self.url(&format!("/batch/scrape/{}", id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(- format!("Checking batch scrape status {}", id.as_ref()),- e,- )- })?;-- let mut status: BatchScrapeJob = self- .handle_response(response, format!("batch scrape status {}", id.as_ref()))- .await?;-- // Auto-paginate if completed- if status.status == JobStatus::Completed {- while let Some(next) = status.next.take() {- let next_status = self.get_batch_scrape_status_next(&next).await?;- status.data.extend(next_status.data);- status.next = next_status.next;- }- }-- Ok(status)- }-- /// Fetches the next page of batch scrape results.- async fn get_batch_scrape_status_next(- &self,- next: &str,- ) -> Result<BatchScrapeJob, FirecrawlError> {- let response = self- .client- .get(next)- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)- })?;-- self.handle_response(response, "batch scrape pagination")- .await- }-- /// Scrapes multiple URLs and waits for completion.- ///- /// This method starts a batch scrape and polls until it completes or fails.- ///- /// # Arguments- ///- /// * `urls` - List of URLs to scrape.- /// * `options` - Optional batch scrape configuration.- ///- /// # Returns- ///- /// A `BatchScrapeJob` containing all scraped documents.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, BatchScrapeOptions, ScrapeOptions, Format};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let urls = vec![- /// "https://example.com/page1".to_string(),- /// "https://example.com/page2".to_string(),- /// "https://example.com/page3".to_string(),- /// ];- ///- /// let options = BatchScrapeOptions {- /// options: Some(ScrapeOptions {- /// formats: Some(vec![Format::Markdown, Format::Links]),- /// ..Default::default()- /// }),- /// ignore_invalid_urls: Some(true),- /// poll_interval: Some(3000),- /// ..Default::default()- /// };- ///- /// let result = client.batch_scrape(urls, options).await?;- /// println!("Scraped {} pages", result.data.len());- ///- /// for doc in result.data {- /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn batch_scrape(- &self,- urls: Vec<String>,- options: impl Into<Option<BatchScrapeOptions>>,- ) -> Result<BatchScrapeJob, FirecrawlError> {- let options = options.into().unwrap_or_default();- let poll_interval = options.poll_interval.unwrap_or(2000);-- let response = self.start_batch_scrape(urls, options).await?;- self.wait_for_batch_scrape(&response.id, poll_interval)- .await- }-- /// Waits for a batch scrape job to complete.- async fn wait_for_batch_scrape(- &self,- id: &str,- poll_interval: u64,- ) -> Result<BatchScrapeJob, FirecrawlError> {- loop {- let status = self.get_batch_scrape_status(id).await?;-- match status.status {- JobStatus::Completed => return Ok(status),- JobStatus::Scraping => {- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;- }- JobStatus::Failed => {- return Err(FirecrawlError::CrawlJobFailed(- "Batch scrape job failed".to_string(),- convert_batch_job_to_crawl_status(status),- ));- }- JobStatus::Cancelled => {- return Err(FirecrawlError::CrawlJobFailed(- "Batch scrape job was cancelled".to_string(),- convert_batch_job_to_crawl_status(status),- ));- }- }- }- }-- /// Gets errors from a batch scrape job.- ///- /// # Arguments- ///- /// * `id` - The batch scrape job ID.- ///- /// # Returns- ///- /// A `CrawlErrorsResponse` containing error details.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let errors = client.get_batch_scrape_errors("job-id").await?;- /// for error in errors.errors {- /// println!("Error on {}: {}", error.url, error.error);- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn get_batch_scrape_errors(- &self,- id: impl AsRef<str>,- ) -> Result<CrawlErrorsResponse, FirecrawlError> {- let response = self- .client- .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)- })?;-- self.handle_response(response, "batch scrape errors").await- }-}--/// Converts a BatchScrapeJob to CrawlStatus for error compatibility.-fn convert_batch_job_to_crawl_status(job: BatchScrapeJob) -> crate::crawl::CrawlStatus {- crate::crawl::CrawlStatus {- status: match job.status {- JobStatus::Completed => crate::crawl::CrawlStatusTypes::Completed,- JobStatus::Scraping => crate::crawl::CrawlStatusTypes::Scraping,- JobStatus::Failed => crate::crawl::CrawlStatusTypes::Failed,- JobStatus::Cancelled => crate::crawl::CrawlStatusTypes::Cancelled,- },- total: job.total,- completed: job.completed,- credits_used: job.credits_used.unwrap_or(0),- expires_at: job.expires_at.unwrap_or_default(),- next: job.next,- data: job- .data- .into_iter()- .map(|d| super::crawl::convert_v2_document_to_v1_pub(d))- .collect(),- }-}--impl super::crawl::CrawlJob {- /// Helper function to convert v2 document to v1 format.- pub(crate) fn _convert_document(doc: Document) -> crate::document::Document {- super::crawl::convert_v2_document_to_v1_pub(doc)- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- async fn test_start_batch_scrape_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/batch/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "batch-123",- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let urls = vec![- "https://example.com".to_string(),- "https://example.org".to_string(),- ];-- let response = client.start_batch_scrape(urls, None).await.unwrap();-- assert!(response.success);- assert_eq!(response.id, "batch-123");- mock.assert();- }-- #[tokio::test]- async fn test_get_batch_scrape_status_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("GET", "/v2/batch/scrape/batch-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "status": "completed",- "total": 3,- "completed": 3,- "creditsUsed": 3,- "data": [- {- "markdown": "# Page 1",- "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }- },- {- "markdown": "# Page 2",- "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }- }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let status = client.get_batch_scrape_status("batch-123").await.unwrap();-- assert_eq!(status.status, JobStatus::Completed);- assert_eq!(status.total, 3);- assert_eq!(status.completed, 3);- assert_eq!(status.data.len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_batch_scrape_with_invalid_urls() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/batch/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "batch-456",- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",- "invalidURLs": ["not-a-url"]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];-- let options = BatchScrapeOptions {- ignore_invalid_urls: Some(true),- ..Default::default()- };-- let response = client.start_batch_scrape(urls, options).await.unwrap();-- assert!(response.success);- assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));- mock.assert();- }-- #[tokio::test]- async fn test_batch_scrape_sync() {- let mut server = mockito::Server::new_async().await;-- // Mock the start endpoint- let start_mock = server- .mock("POST", "/v2/batch/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "batch-789",- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"- })- .to_string(),- )- .create();-- // Mock the status endpoint (completed immediately)- let status_mock = server- .mock("GET", "/v2/batch/scrape/batch-789")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "status": "completed",- "total": 2,- "completed": 2,- "data": [- {- "markdown": "# Content",- "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }- }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let urls = vec!["https://example.com".to_string()];-- let result = client.batch_scrape(urls, None).await.unwrap();-- assert_eq!(result.status, JobStatus::Completed);- assert_eq!(result.data.len(), 1);- start_mock.assert();- status_mock.assert();- }-- #[tokio::test]- async fn test_get_batch_scrape_errors() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("GET", "/v2/batch/scrape/batch-123/errors")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "errors": [- {- "id": "err-1",- "url": "https://example.com/broken",- "error": "Connection timeout"- }- ],- "robotsBlocked": []- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();-- assert_eq!(errors.errors.len(), 1);- assert_eq!(errors.errors[0].error, "Connection timeout");- mock.assert();- }-}diff --git a/apps/rust-sdk/src/v2/crawl.rs b/apps/rust-sdk/src/v2/crawl.rsdeleted file mode 100644index 4b773abe3..000000000--- a/apps/rust-sdk/src/v2/crawl.rs+++ /dev/null@@ -1,688 +0,0 @@-//! Crawl endpoint for Firecrawl API v2.--use serde::{Deserialize, Serialize};--use super::client::Client;-use super::scrape::ScrapeOptions;-use super::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};-use crate::FirecrawlError;--/// Options for crawling a website.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlOptions {- /// Natural language prompt to guide crawl behavior.- pub prompt: Option<String>,-- /// URL path patterns to exclude from crawling.- pub exclude_paths: Option<Vec<String>>,-- /// URL path patterns to include in crawling.- pub include_paths: Option<Vec<String>>,-- /// Maximum depth of links to follow from the initial URL.- pub max_discovery_depth: Option<u32>,-- /// How to handle the sitemap.- pub sitemap: Option<SitemapMode>,-- /// Ignore query parameters when deduplicating URLs.- pub ignore_query_parameters: Option<bool>,-- /// Maximum number of pages to crawl.- pub limit: Option<u32>,-- /// Crawl the entire domain regardless of path structure.- pub crawl_entire_domain: Option<bool>,-- /// Allow following links to external domains.- pub allow_external_links: Option<bool>,-- /// Allow following links to subdomains.- pub allow_subdomains: Option<bool>,-- /// Delay between requests in seconds.- pub delay: Option<u32>,-- /// Maximum concurrent requests.- pub max_concurrency: Option<u32>,-- /// Webhook configuration for job notifications.- pub webhook: Option<WebhookConfig>,-- /// Scrape options to apply to each page.- pub scrape_options: Option<ScrapeOptions>,-- /// Enable zero data retention mode.- pub zero_data_retention: Option<bool>,-- /// Integration identifier for tracking.- pub integration: Option<String>,-- /// Idempotency key for the request.- #[serde(skip)]- pub idempotency_key: Option<String>,-- /// Poll interval for synchronous crawl (milliseconds).- #[serde(skip)]- pub poll_interval: Option<u64>,-}--/// Request body for crawl endpoint.-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct CrawlRequest {- url: String,- #[serde(flatten)]- options: CrawlOptions,-}--/// Response from starting a crawl job.-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlResponse {- /// Whether the request was successful.- pub success: bool,- /// The crawl job ID.- pub id: String,- /// URL to check the crawl status.- pub url: String,-}--/// Status of a crawl job.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CrawlJob {- /// Current status of the crawl job.- pub status: JobStatus,- /// Total number of pages to crawl.- pub total: u32,- /// Number of pages completed.- pub completed: u32,- /// Credits used by the crawl.- pub credits_used: Option<u32>,- /// Expiry time of the crawl data.- pub expires_at: Option<String>,- /// URL for the next page of results.- pub next: Option<String>,- /// Crawled documents.- pub data: Vec<Document>,-}--/// Response from canceling a crawl.-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct CancelCrawlResponse {- /// Status of the cancellation.- pub status: String,-}--impl Client {- /// Starts a crawl job asynchronously.- ///- /// Returns immediately with a job ID that can be used to check status.- ///- /// # Arguments- ///- /// * `url` - The URL to start crawling from.- /// * `options` - Optional crawl configuration.- ///- /// # Returns- ///- /// A `CrawlResponse` containing the job ID.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, CrawlOptions};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let response = client.start_crawl("https://example.com", None).await?;- /// println!("Crawl job started: {}", response.id);- ///- /// // Check status later- /// let status = client.get_crawl_status(&response.id).await?;- /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);- ///- /// Ok(())- /// }- /// ```- pub async fn start_crawl(- &self,- url: impl AsRef<str>,- options: impl Into<Option<CrawlOptions>>,- ) -> Result<CrawlResponse, FirecrawlError> {- let options = options.into().unwrap_or_default();- let body = CrawlRequest {- url: url.as_ref().to_string(),- options: options.clone(),- };-- let headers = self.prepare_headers(options.idempotency_key.as_ref());-- let response = self- .client- .post(self.url("/crawl"))- .headers(headers)- .json(&body)- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)- })?;-- self.handle_response(response, "start crawl").await- }-- /// Gets the status of a crawl job.- ///- /// If the job is completed, this will automatically fetch all pages of results.- ///- /// # Arguments- ///- /// * `id` - The crawl job ID.- ///- /// # Returns- ///- /// A `CrawlJob` containing the current status and any available documents.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let status = client.get_crawl_status("job-id").await?;- /// println!("Status: {:?}", status.status);- /// println!("Completed: {}/{}", status.completed, status.total);- /// println!("Documents: {}", status.data.len());- ///- /// Ok(())- /// }- /// ```- pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {- let response = self- .client- .get(self.url(&format!("/crawl/{}", id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)- })?;-- let mut status: CrawlJob = self- .handle_response(response, format!("crawl status {}", id.as_ref()))- .await?;-- // Auto-paginate if completed- if status.status == JobStatus::Completed {- while let Some(next) = status.next.take() {- let next_status = self.get_crawl_status_next(&next).await?;- status.data.extend(next_status.data);- status.next = next_status.next;- }- }-- Ok(status)- }-- /// Fetches the next page of crawl results.- async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {- let response = self- .client- .get(next)- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;-- self.handle_response(response, "crawl pagination").await- }-- /// Crawls a website and waits for completion.- ///- /// This method starts a crawl and polls until it completes or fails.- ///- /// # Arguments- ///- /// * `url` - The URL to start crawling from.- /// * `options` - Optional crawl configuration.- ///- /// # Returns- ///- /// A `CrawlJob` containing all crawled documents.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, CrawlOptions, SitemapMode};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let options = CrawlOptions {- /// sitemap: Some(SitemapMode::Include),- /// limit: Some(100),- /// poll_interval: Some(5000), // Check every 5 seconds- /// ..Default::default()- /// };- ///- /// let result = client.crawl("https://example.com", options).await?;- /// println!("Crawled {} pages", result.data.len());- ///- /// for doc in result.data {- /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn crawl(- &self,- url: impl AsRef<str>,- options: impl Into<Option<CrawlOptions>>,- ) -> Result<CrawlJob, FirecrawlError> {- let options = options.into().unwrap_or_default();- let poll_interval = options.poll_interval.unwrap_or(2000);-- let response = self.start_crawl(url, options).await?;- self.wait_for_crawl(&response.id, poll_interval).await- }-- /// Waits for a crawl job to complete.- async fn wait_for_crawl(- &self,- id: &str,- poll_interval: u64,- ) -> Result<CrawlJob, FirecrawlError> {- loop {- let status = self.get_crawl_status(id).await?;-- match status.status {- JobStatus::Completed => return Ok(status),- JobStatus::Scraping => {- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;- }- JobStatus::Failed => {- return Err(FirecrawlError::CrawlJobFailed(- "Crawl job failed".to_string(),- crate::crawl::CrawlStatus {- status: crate::crawl::CrawlStatusTypes::Failed,- total: status.total,- completed: status.completed,- credits_used: status.credits_used.unwrap_or(0),- expires_at: status.expires_at.unwrap_or_default(),- next: status.next,- data: status- .data- .into_iter()- .map(|d| convert_v2_document_to_v1(d))- .collect(),- },- ));- }- JobStatus::Cancelled => {- return Err(FirecrawlError::CrawlJobFailed(- "Crawl job was cancelled".to_string(),- crate::crawl::CrawlStatus {- status: crate::crawl::CrawlStatusTypes::Cancelled,- total: status.total,- completed: status.completed,- credits_used: status.credits_used.unwrap_or(0),- expires_at: status.expires_at.unwrap_or_default(),- next: status.next,- data: status- .data- .into_iter()- .map(|d| convert_v2_document_to_v1(d))- .collect(),- },- ));- }- }- }- }-- /// Cancels a running crawl job.- ///- /// # Arguments- ///- /// * `id` - The crawl job ID to cancel.- ///- /// # Returns- ///- /// A `CancelCrawlResponse` indicating the cancellation status.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let response = client.cancel_crawl("job-id").await?;- /// println!("Cancellation status: {}", response.status);- ///- /// Ok(())- /// }- /// ```- pub async fn cancel_crawl(- &self,- id: impl AsRef<str>,- ) -> Result<CancelCrawlResponse, FirecrawlError> {- let response = self- .client- .delete(self.url(&format!("/crawl/{}", id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)- })?;-- self.handle_response(response, "cancel crawl").await- }-- /// Gets errors from a crawl job.- ///- /// # Arguments- ///- /// * `id` - The crawl job ID.- ///- /// # Returns- ///- /// A `CrawlErrorsResponse` containing error details.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let errors = client.get_crawl_errors("job-id").await?;- /// for error in errors.errors {- /// println!("Error on {}: {}", error.url, error.error);- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn get_crawl_errors(- &self,- id: impl AsRef<str>,- ) -> Result<CrawlErrorsResponse, FirecrawlError> {- let response = self- .client- .get(self.url(&format!("/crawl/{}/errors", id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)- })?;-- self.handle_response(response, "crawl errors").await- }-}--/// Converts a v2 Document to a v1 Document for error compatibility.-pub(crate) fn convert_v2_document_to_v1_pub(doc: Document) -> crate::document::Document {- convert_v2_document_to_v1(doc)-}--/// Converts a v2 Document to a v1 Document for error compatibility.-fn convert_v2_document_to_v1(doc: Document) -> crate::document::Document {- let metadata = doc.metadata.unwrap_or_default();- crate::document::Document {- markdown: doc.markdown,- html: doc.html,- raw_html: doc.raw_html,- screenshot: doc.screenshot,- links: doc.links,- extract: doc.json,- metadata: crate::document::DocumentMetadata {- source_url: metadata.source_url.unwrap_or_default(),- status_code: metadata.status_code.unwrap_or(0),- error: metadata.error,- title: metadata.title,- description: metadata.description,- language: metadata.language,- keywords: metadata.keywords,- robots: metadata.robots,- og_title: metadata.og_title,- og_description: metadata.og_description,- og_url: metadata.og_url,- og_image: metadata.og_image,- og_audio: metadata.og_audio,- og_determiner: metadata.og_determiner,- og_locale: metadata.og_locale,- og_locale_alternate: metadata.og_locale_alternate,- og_site_name: metadata.og_site_name,- og_video: metadata.og_video,- article_section: metadata.article_section,- article_tag: metadata.article_tag,- published_time: metadata.published_time,- modified_time: metadata.modified_time,- dcterms_keywords: metadata.dcterms_keywords,- dc_description: metadata.dc_description,- dc_subject: metadata.dc_subject,- dcterms_subject: metadata.dcterms_subject,- dcterms_audience: metadata.dcterms_audience,- dc_type: metadata.dc_type,- dcterms_type: metadata.dcterms_type,- dc_date: metadata.dc_date,- dc_date_created: metadata.dc_date_created,- dcterms_created: metadata.dcterms_created,- },- warning: doc.warning,- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- async fn test_start_crawl_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/crawl")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "crawl-123",- "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client- .start_crawl("https://example.com", None)- .await- .unwrap();-- assert!(response.success);- assert_eq!(response.id, "crawl-123");- mock.assert();- }-- #[tokio::test]- async fn test_get_crawl_status_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("GET", "/v2/crawl/crawl-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "status": "completed",- "total": 5,- "completed": 5,- "creditsUsed": 5,- "expiresAt": "2024-12-31T23:59:59Z",- "data": [- {- "markdown": "# Page 1",- "metadata": {- "sourceURL": "https://example.com/page1",- "statusCode": 200- }- },- {- "markdown": "# Page 2",- "metadata": {- "sourceURL": "https://example.com/page2",- "statusCode": 200- }- }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let status = client.get_crawl_status("crawl-123").await.unwrap();-- assert_eq!(status.status, JobStatus::Completed);- assert_eq!(status.total, 5);- assert_eq!(status.completed, 5);- assert_eq!(status.data.len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_cancel_crawl_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("DELETE", "/v2/crawl/crawl-123")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "status": "cancelled"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client.cancel_crawl("crawl-123").await.unwrap();-- assert_eq!(response.status, "cancelled");- mock.assert();- }-- #[tokio::test]- async fn test_get_crawl_errors_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("GET", "/v2/crawl/crawl-123/errors")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "errors": [- {- "id": "error-1",- "timestamp": "2024-01-01T00:00:00Z",- "url": "https://example.com/broken",- "error": "404 Not Found"- }- ],- "robotsBlocked": [- "https://example.com/admin"- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let errors = client.get_crawl_errors("crawl-123").await.unwrap();-- assert_eq!(errors.errors.len(), 1);- assert_eq!(errors.errors[0].url, "https://example.com/broken");- assert_eq!(errors.robots_blocked.len(), 1);- mock.assert();- }-- #[tokio::test]- async fn test_crawl_with_options() {- let mut server = mockito::Server::new_async().await;-- // Mock the start endpoint- let start_mock = server- .mock("POST", "/v2/crawl")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "id": "crawl-456",- "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"- })- .to_string(),- )- .create();-- // Mock the status endpoint (completed immediately)- let status_mock = server- .mock("GET", "/v2/crawl/crawl-456")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "status": "completed",- "total": 2,- "completed": 2,- "data": [- {- "markdown": "# Page 1",- "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }- }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let options = CrawlOptions {- limit: Some(10),- sitemap: Some(SitemapMode::Include),- ..Default::default()- };-- let result = client.crawl("https://example.com", options).await.unwrap();-- assert_eq!(result.status, JobStatus::Completed);- assert_eq!(result.data.len(), 1);- start_mock.assert();- status_mock.assert();- }-}diff --git a/apps/rust-sdk/src/v2/map.rs b/apps/rust-sdk/src/v2/map.rsdeleted file mode 100644index ce2dc54ca..000000000--- a/apps/rust-sdk/src/v2/map.rs+++ /dev/null@@ -1,333 +0,0 @@-//! Map endpoint for Firecrawl API v2.--use serde::{Deserialize, Serialize};--use super::client::Client;-use super::types::{LocationConfig, SearchResultWeb, SitemapMode};-use crate::FirecrawlError;--/// Options for mapping a URL.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct MapOptions {- /// Search query to filter discovered links.- pub search: Option<String>,-- /// How to handle the sitemap.- pub sitemap: Option<SitemapMode>,-- /// Include subdomains in the mapping.- pub include_subdomains: Option<bool>,-- /// Ignore query parameters when deduplicating URLs.- pub ignore_query_parameters: Option<bool>,-- /// Maximum number of links to return.- pub limit: Option<u32>,-- /// Timeout in milliseconds.- pub timeout: Option<u32>,-- /// Integration identifier for tracking.- pub integration: Option<String>,-- /// Location configuration for proxy routing.- pub location: Option<LocationConfig>,-}--/// Request body for map endpoint.-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct MapRequest {- url: String,- #[serde(flatten)]- options: MapOptions,-}--/// Response from map endpoint.-#[derive(Deserialize, Serialize, Debug)]-#[serde(rename_all = "camelCase")]-pub struct MapResponse {- /// Whether the request was successful.- pub success: bool,- /// Discovered links with metadata.- pub links: Vec<SearchResultWeb>,- /// Warning message if any.- pub warning: Option<String>,-}--impl Client {- /// Maps a URL to discover all associated links.- ///- /// This endpoint discovers links from a website's sitemap, page content,- /// and other sources without fully scraping each page.- ///- /// # Arguments- ///- /// * `url` - The URL to map.- /// * `options` - Optional mapping configuration.- ///- /// # Returns- ///- /// A `MapResponse` containing the discovered links.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, MapOptions, SitemapMode};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// // Simple map- /// let response = client.map("https://example.com", None).await?;- /// println!("Found {} links", response.links.len());- ///- /// // Map with options- /// let options = MapOptions {- /// sitemap: Some(SitemapMode::Include),- /// include_subdomains: Some(true),- /// limit: Some(1000),- /// ..Default::default()- /// };- /// let response = client.map("https://example.com", options).await?;- ///- /// for link in response.links {- /// println!("URL: {}, Title: {:?}", link.url, link.title);- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn map(- &self,- url: impl AsRef<str>,- options: impl Into<Option<MapOptions>>,- ) -> Result<MapResponse, FirecrawlError> {- let body = MapRequest {- url: url.as_ref().to_string(),- options: options.into().unwrap_or_default(),- };-- let headers = self.prepare_headers(None);-- let response = self- .client- .post(self.url("/map"))- .headers(headers)- .json(&body)- .send()- .await- .map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;-- self.handle_response(response, "map").await- }-- /// Maps a URL and returns just the list of URLs.- ///- /// This is a convenience method that returns only the URL strings.- ///- /// # Arguments- ///- /// * `url` - The URL to map.- /// * `options` - Optional mapping configuration.- ///- /// # Returns- ///- /// A vector of discovered URL strings.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let urls = client.map_urls("https://example.com", None).await?;- /// for url in urls {- /// println!("{}", url);- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn map_urls(- &self,- url: impl AsRef<str>,- options: impl Into<Option<MapOptions>>,- ) -> Result<Vec<String>, FirecrawlError> {- let response = self.map(url, options).await?;- Ok(response.links.into_iter().map(|link| link.url).collect())- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- async fn test_map_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/map")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "links": [- {- "url": "https://example.com/",- "title": "Example Domain",- "description": "Home page"- },- {- "url": "https://example.com/about",- "title": "About Us"- },- {- "url": "https://example.com/contact"- }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client.map("https://example.com", None).await.unwrap();-- assert!(response.success);- assert_eq!(response.links.len(), 3);- assert_eq!(response.links[0].url, "https://example.com/");- assert_eq!(response.links[0].title, Some("Example Domain".to_string()));- mock.assert();- }-- #[tokio::test]- async fn test_map_with_options() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/map")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "links": [- { "url": "https://example.com/page1" },- { "url": "https://example.com/page2" }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let options = MapOptions {- sitemap: Some(SitemapMode::Include),- include_subdomains: Some(true),- limit: Some(100),- ..Default::default()- };-- let response = client.map("https://example.com", options).await.unwrap();-- assert!(response.success);- assert_eq!(response.links.len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_map_urls() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/map")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "links": [- { "url": "https://example.com/page1" },- { "url": "https://example.com/page2" },- { "url": "https://example.com/page3" }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let urls = client.map_urls("https://example.com", None).await.unwrap();-- assert_eq!(urls.len(), 3);- assert_eq!(urls[0], "https://example.com/page1");- mock.assert();- }-- #[tokio::test]- async fn test_map_with_search_filter() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/map")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "links": [- { "url": "https://example.com/blog/post1" },- { "url": "https://example.com/blog/post2" }- ]- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let options = MapOptions {- search: Some("blog".to_string()),- ..Default::default()- };-- let response = client.map("https://example.com", options).await.unwrap();-- assert!(response.success);- assert_eq!(response.links.len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_map_error_response() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/map")- .with_status(400)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Invalid URL"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = client.map("invalid-url", None).await;-- assert!(result.is_err());- mock.assert();- }-}diff --git a/apps/rust-sdk/src/v2/mod.rs b/apps/rust-sdk/src/v2/mod.rsdeleted file mode 100644index caa1a4fcb..000000000--- a/apps/rust-sdk/src/v2/mod.rs+++ /dev/null@@ -1,40 +0,0 @@-//! Firecrawl API v2 client module.-//!-//! This module provides access to the v2 API endpoints while maintaining-//! backward compatibility with v1. The v2 API includes new features like-//! the Agent endpoint and improved options.-//!-//! # Example-//!-//! ```no_run-//! use firecrawl::v2::{Client, ScrapeOptions};-//!-//! #[tokio::main]-//! async fn main() -> Result<(), Box<dyn std::error::Error>> {-//! let client = Client::new("your-api-key")?;-//!-//! // Scrape a URL-//! let document = client.scrape("https://example.com", None).await?;-//! println!("Content: {:?}", document.markdown);-//!-//! Ok(())-//! }-//! ```--mod agent;-mod batch_scrape;-mod client;-mod crawl;-mod map;-mod scrape;-mod search;-mod types;--pub use agent::*;-pub use batch_scrape::*;-pub use client::Client;-pub use crawl::*;-pub use map::*;-pub use scrape::*;-pub use search::*;-pub use types::*;diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rsdeleted file mode 100644index ffbe8fe44..000000000--- a/apps/rust-sdk/src/v2/scrape.rs+++ /dev/null@@ -1,731 +0,0 @@-//! Scrape endpoint for Firecrawl API v2.--use serde::{Deserialize, Serialize};-use serde_json::Value;-use std::collections::HashMap;--use super::client::Client;-use super::types::{- Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,- LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,-};-use crate::FirecrawlError;--/// Options for scraping a URL.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ScrapeOptions {- /// Output formats to include in the response.- pub formats: Option<Vec<Format>>,-- /// Additional HTTP headers to send with the request.- pub headers: Option<HashMap<String, String>>,-- /// HTML tags to exclusively include in the output.- pub include_tags: Option<Vec<String>>,-- /// HTML tags to exclude from the output.- pub exclude_tags: Option<Vec<String>>,-- /// Only extract the main content of the page.- pub only_main_content: Option<bool>,-- /// Timeout in milliseconds before returning an error.- pub timeout: Option<u32>,-- /// Time to wait after page load before scraping (milliseconds).- pub wait_for: Option<u32>,-- /// Emulate a mobile device.- pub mobile: Option<bool>,-- /// Parser configurations (e.g., for PDFs).- pub parsers: Option<Vec<ParserConfig>>,-- /// Browser automation actions to perform before scraping.- pub actions: Option<Vec<Action>>,-- /// Location configuration for proxy routing.- pub location: Option<LocationConfig>,-- /// Skip TLS certificate verification.- pub skip_tls_verification: Option<bool>,-- /// Remove base64-encoded images from the output.- pub remove_base64_images: Option<bool>,-- /// Enable fast mode for quicker scrapes with reduced accuracy.- pub fast_mode: Option<bool>,-- /// Block advertisements on the page.- pub block_ads: Option<bool>,-- /// Proxy type to use.- pub proxy: Option<ProxyType>,-- /// Maximum age of cached content to accept (seconds).- pub max_age: Option<u32>,-- /// Minimum age of cached content to accept (seconds).- pub min_age: Option<u32>,-- /// Store the result in cache for future requests.- pub store_in_cache: Option<bool>,-- /// Persistent browser profile for maintaining state across scrapes.- pub profile: Option<ProfileConfig>,-- /// Integration identifier for tracking.- pub integration: Option<String>,-- /// JSON extraction options.- pub json_options: Option<JsonOptions>,-- /// Screenshot options.- pub screenshot_options: Option<ScreenshotOptions>,-- /// Change tracking options.- pub change_tracking_options: Option<ChangeTrackingOptions>,-- /// Attribute selectors for extraction.- pub attribute_selectors: Option<Vec<AttributeSelector>>,-}--/// Parser configuration for document parsing.-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(untagged)]-pub enum ParserConfig {- /// Simple parser type string.- Simple(String),- /// PDF parser with options.- Pdf {- #[serde(rename = "type")]- parser_type: String,- #[serde(skip_serializing_if = "Option::is_none")]- max_pages: Option<u32>,- },-}--/// Request body for scrape endpoint.-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct ScrapeRequest {- url: String,- #[serde(flatten)]- options: ScrapeOptions,-}--/// Response from scrape endpoint.-#[derive(Deserialize, Serialize, Debug)]-#[serde(rename_all = "camelCase")]-struct ScrapeResponse {- success: bool,- data: Document,- #[serde(skip_serializing_if = "Option::is_none")]- warning: Option<String>,-}--/// Supported languages for scrape-bound browser execution.-#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]-#[serde(rename_all = "lowercase")]-pub enum ScrapeExecuteLanguage {- Python,- Node,- Bash,-}--/// Options for executing code or a prompt in a scrape-bound browser session.-///-/// At least one of `code` or `prompt` must be provided.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ScrapeExecuteOptions {- /// Code to execute (optional if `prompt` is provided).- pub code: Option<String>,- /// Natural-language instruction for the browser agent (optional if `code` is provided).- pub prompt: Option<String>,- /// Runtime language for the code.- pub language: Option<ScrapeExecuteLanguage>,- /// Execution timeout in seconds.- pub timeout: Option<u32>,- /// Optional origin tag for request attribution.- pub origin: Option<String>,-}--/// Response from scrape-bound browser execution.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ScrapeExecuteResponse {- /// Whether the request succeeded.- pub success: bool,- /// Live-view URL for the browser session.- pub live_view_url: Option<String>,- /// Interactive live-view URL for the browser session.- pub interactive_live_view_url: Option<String>,- /// Agent output when a prompt was used.- pub output: Option<String>,- /// Captured stdout from execution.- pub stdout: Option<String>,- /// Optional execution result payload.- pub result: Option<String>,- /// Captured stderr from execution.- pub stderr: Option<String>,- /// Process exit code.- pub exit_code: Option<i32>,- /// Whether execution was killed by timeout or system.- pub killed: Option<bool>,- /// Error message when execution fails.- pub error: Option<String>,-}--/// Response from deleting a scrape-bound browser session.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Clone)]-#[serde(rename_all = "camelCase")]-pub struct ScrapeBrowserDeleteResponse {- /// Whether the delete request succeeded.- pub success: bool,- /// Session duration in milliseconds when available.- pub session_duration_ms: Option<u64>,- /// Credits billed when available.- pub credits_billed: Option<u32>,- /// Error message when deletion fails.- pub error: Option<String>,-}--impl Client {- /// Scrapes a URL and returns the content in the requested formats.- ///- /// # Arguments- ///- /// * `url` - The URL to scrape.- /// * `options` - Optional scrape configuration.- ///- /// # Returns- ///- /// A `Document` containing the scraped content.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, ScrapeOptions, Format};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// // Simple scrape- /// let document = client.scrape("https://example.com", None).await?;- /// println!("Markdown: {:?}", document.markdown);- ///- /// // Scrape with options- /// let options = ScrapeOptions {- /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),- /// only_main_content: Some(true),- /// ..Default::default()- /// };- /// let document = client.scrape("https://example.com", options).await?;- /// println!("Links: {:?}", document.links);- ///- /// Ok(())- /// }- /// ```- pub async fn scrape(- &self,- url: impl AsRef<str>,- options: impl Into<Option<ScrapeOptions>>,- ) -> Result<Document, FirecrawlError> {- let body = ScrapeRequest {- url: url.as_ref().to_string(),- options: options.into().unwrap_or_default(),- };-- let headers = self.prepare_headers(None);-- let response = self- .client- .post(self.url("/scrape"))- .headers(headers)- .json(&body)- .send()- .await- .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;-- let response: ScrapeResponse = self.handle_response(response, "scrape").await?;-- Ok(response.data)- }-- /// Scrapes a URL with a JSON schema for structured extraction.- ///- /// This is a convenience method that combines scraping with JSON extraction.- ///- /// # Arguments- ///- /// * `url` - The URL to scrape.- /// * `schema` - JSON schema for the extraction.- /// * `prompt` - Optional extraction prompt.- ///- /// # Returns- ///- /// The extracted JSON value matching the schema.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- /// use serde_json::json;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let schema = json!({- /// "type": "object",- /// "properties": {- /// "title": { "type": "string" },- /// "price": { "type": "number" }- /// }- /// });- ///- /// let data = client.scrape_with_schema(- /// "https://example.com/product",- /// schema,- /// Some("Extract the product title and price")- /// ).await?;- ///- /// println!("Extracted: {}", data);- ///- /// Ok(())- /// }- /// ```- pub async fn scrape_with_schema(- &self,- url: impl AsRef<str>,- schema: Value,- prompt: Option<impl AsRef<str>>,- ) -> Result<Value, FirecrawlError> {- let options = ScrapeOptions {- formats: Some(vec![Format::Json]),- json_options: Some(JsonOptions {- schema: Some(schema),- prompt: prompt.map(|p| p.as_ref().to_string()),- ..Default::default()- }),- ..Default::default()- };-- let document = self.scrape(url, options).await?;- Ok(document.json.unwrap_or(Value::Null))- }-- /// Interacts with the browser session associated with a scrape job.- ///- /// # Arguments- ///- /// * `job_id` - The scrape job ID.- /// * `options` - Execution options including code and runtime config.- ///- /// # Returns- ///- /// A `ScrapeExecuteResponse` containing execution output.- pub async fn interact(- &self,- job_id: impl AsRef<str>,- options: ScrapeExecuteOptions,- ) -> Result<ScrapeExecuteResponse, FirecrawlError> {- let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());- let has_prompt = options- .prompt- .as_ref()- .is_some_and(|p| !p.trim().is_empty());- if !has_code && !has_prompt {- return Err(FirecrawlError::Missuse(- "Either 'code' or 'prompt' must be provided".into(),- ));- }-- let mut body = options;- if body.language.is_none() {- body.language = Some(ScrapeExecuteLanguage::Node);- }-- let response = self- .client- .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))- .headers(self.prepare_headers(None))- .json(&body)- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(- format!("Interacting with scrape browser for {}", job_id.as_ref()),- e,- )- })?;-- self.handle_response(response, "scrape interact").await- }-- /// Stops the interaction session associated with a scrape job.- ///- /// # Arguments- ///- /// * `job_id` - The scrape job ID.- ///- /// # Returns- ///- /// A `ScrapeBrowserDeleteResponse` indicating stop status.- pub async fn stop_interaction(- &self,- job_id: impl AsRef<str>,- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {- let response = self- .client- .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))- .headers(self.prepare_headers(None))- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(- format!("Stopping interaction for {}", job_id.as_ref()),- e,- )- })?;-- self.handle_response(response, "stop interaction").await- }-- /// Deprecated alias for [`Client::interact`].- #[deprecated(note = "Use interact() instead")]- pub async fn scrape_execute(- &self,- job_id: impl AsRef<str>,- options: ScrapeExecuteOptions,- ) -> Result<ScrapeExecuteResponse, FirecrawlError> {- self.interact(job_id, options).await- }-- /// Deprecated alias for [`Client::stop_interaction`].- #[deprecated(note = "Use stop_interaction() instead")]- pub async fn stop_interactive_browser(- &self,- job_id: impl AsRef<str>,- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {- self.stop_interaction(job_id).await- }-- /// Deprecated alias for [`Client::stop_interaction`].- #[deprecated(note = "Use stop_interaction() instead")]- pub async fn delete_scrape_browser(- &self,- job_id: impl AsRef<str>,- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {- self.stop_interaction(job_id).await- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- async fn test_scrape_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "markdown": "# Example Domain\n\nThis is an example.",- "metadata": {- "sourceURL": "https://example.com",- "statusCode": 200,- "title": "Example Domain"- }- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let document = client.scrape("https://example.com", None).await.unwrap();-- assert!(document.markdown.is_some());- assert!(document.markdown.unwrap().contains("Example Domain"));- mock.assert();- }-- #[tokio::test]- async fn test_scrape_with_options() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "markdown": "# Test",- "html": "<h1>Test</h1>",- "links": ["https://example.com/page1", "https://example.com/page2"],- "metadata": {- "sourceURL": "https://example.com",- "statusCode": 200- }- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let options = ScrapeOptions {- formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),- only_main_content: Some(true),- ..Default::default()- };-- let document = client.scrape("https://example.com", options).await.unwrap();-- assert!(document.markdown.is_some());- assert!(document.html.is_some());- assert!(document.links.is_some());- assert_eq!(document.links.unwrap().len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_scrape_with_schema() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "json": {- "title": "Product Name",- "price": 99.99- },- "metadata": {- "sourceURL": "https://example.com/product",- "statusCode": 200- }- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();-- let schema = json!({- "type": "object",- "properties": {- "title": { "type": "string" },- "price": { "type": "number" }- }- });-- let data = client- .scrape_with_schema(- "https://example.com/product",- schema,- Some("Extract product info"),- )- .await- .unwrap();-- assert_eq!(data["title"], "Product Name");- assert_eq!(data["price"], 99.99);- mock.assert();- }-- #[tokio::test]- async fn test_scrape_error_response() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape")- .with_status(400)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Invalid URL"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = client.scrape("invalid-url", None).await;-- assert!(result.is_err());- mock.assert();- }-- #[tokio::test]- async fn test_interact_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape/job-123/interact")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "stdout": "ok",- "result": "done",- "stderr": "",- "exitCode": 0,- "killed": false- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client- .interact(- "job-123",- ScrapeExecuteOptions {- code: Some("console.log('ok')".to_string()),- timeout: Some(30),- ..Default::default()- },- )- .await- .unwrap();-- assert!(response.success);- assert_eq!(response.exit_code, Some(0));- assert_eq!(response.result, Some("done".to_string()));- mock.assert();- }-- #[tokio::test]- async fn test_interact_with_prompt() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape/job-789/interact")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "output": "Clicked the login button",- "liveViewUrl": "https://live.example.com/view",- "interactiveLiveViewUrl": "https://live.example.com/interactive",- "stdout": "",- "exitCode": 0,- "killed": false- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client- .interact(- "job-789",- ScrapeExecuteOptions {- prompt: Some("Click the login button".to_string()),- ..Default::default()- },- )- .await- .unwrap();-- assert!(response.success);- assert_eq!(- response.output,- Some("Clicked the login button".to_string())- );- assert_eq!(- response.live_view_url,- Some("https://live.example.com/view".to_string())- );- assert_eq!(- response.interactive_live_view_url,- Some("https://live.example.com/interactive".to_string())- );- mock.assert();- }-- #[tokio::test]- async fn test_stop_interaction_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("DELETE", "/v2/scrape/job-123/interact")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "sessionDurationMs": 1200,- "creditsBilled": 3- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client.stop_interaction("job-123").await.unwrap();-- assert!(response.success);- assert_eq!(response.session_duration_ms, Some(1200));- assert_eq!(response.credits_billed, Some(3));- mock.assert();- }-- #[tokio::test]- async fn test_interact_error_response() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/scrape/job-404/interact")- .with_status(404)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Job not found."- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = client- .interact(- "job-404",- ScrapeExecuteOptions {- code: Some("console.log('ok')".to_string()),- ..Default::default()- },- )- .await;-- assert!(result.is_err());- mock.assert();- }-}diff --git a/apps/rust-sdk/src/v2/search.rs b/apps/rust-sdk/src/v2/search.rsdeleted file mode 100644index ccca680f2..000000000--- a/apps/rust-sdk/src/v2/search.rs+++ /dev/null@@ -1,466 +0,0 @@-//! Search endpoint for Firecrawl API v2.--use serde::{Deserialize, Serialize};--use super::client::Client;-use super::scrape::ScrapeOptions;-use super::types::{- Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,-};-use crate::FirecrawlError;--/// Options for search requests.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct SearchOptions {- /// Maximum number of results to return. Default: 5, Max: 20.- pub limit: Option<u32>,-- /// Search sources to query (web, news, images).- pub sources: Option<Vec<SearchSource>>,-- /// Categories to filter results (github, research, pdf).- pub categories: Option<Vec<SearchCategory>>,-- /// Time-based search filter (e.g., "qdr:d" for past day).- pub tbs: Option<String>,-- /// Geographic location string for local search results.- pub location: Option<String>,-- /// Whether to ignore invalid URLs in results.- pub ignore_invalid_urls: Option<bool>,-- /// Timeout in milliseconds.- pub timeout: Option<u32>,-- /// Scrape options to apply to each search result.- pub scrape_options: Option<ScrapeOptions>,-- /// Integration identifier for tracking.- pub integration: Option<String>,-}--/// Request body for search endpoint.-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct SearchRequest {- query: String,- #[serde(flatten)]- options: SearchOptions,-}--/// Search results data structure.-#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]-pub struct SearchData {- /// Web search results (may include scraped documents).- pub web: Option<Vec<SearchResultOrDocument>>,- /// News search results.- pub news: Option<Vec<SearchResultNews>>,- /// Image search results.- pub images: Option<Vec<SearchResultImage>>,-}--/// A search result that may be a simple result or a full document.-///-/// Uses custom deserialization to properly distinguish between web results-/// and scraped documents by checking for document-specific fields.-#[derive(Serialize, Debug, Clone)]-#[serde(untagged)]-pub enum SearchResultOrDocument {- /// Simple web search result.- WebResult(SearchResultWeb),- /// Full scraped document.- Document(Document),-}--impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where- D: serde::Deserializer<'de>,- {- use serde_json::Value;-- let value = Value::deserialize(deserializer)?;-- // Check for document-specific fields that indicate scraped content- // If any of these exist, it's a Document, not a simple WebResult- let is_document = value.get("markdown").is_some()- || value.get("html").is_some()- || value.get("rawHtml").is_some()- || value.get("metadata").is_some();-- if is_document {- Document::deserialize(value)- .map(SearchResultOrDocument::Document)- .map_err(serde::de::Error::custom)- } else {- SearchResultWeb::deserialize(value)- .map(SearchResultOrDocument::WebResult)- .map_err(serde::de::Error::custom)- }- }-}--/// Response from search endpoint.-#[derive(Deserialize, Serialize, Debug)]-#[serde(rename_all = "camelCase")]-pub struct SearchResponse {- /// Whether the request was successful.- pub success: bool,- /// Search results data.- pub data: SearchData,- /// Warning message if any.- pub warning: Option<String>,-}--impl Client {- /// Searches the web and optionally scrapes the results.- ///- /// # Arguments- ///- /// * `query` - The search query string.- /// * `options` - Optional search configuration.- ///- /// # Returns- ///- /// A `SearchResponse` containing the search results.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::{Client, SearchOptions, SearchSource};- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// // Simple search- /// let results = client.search("rust programming", None).await?;- /// for result in results.data.web.unwrap_or_default() {- /// match result {- /// firecrawl::v2::SearchResultOrDocument::WebResult(r) => {- /// println!("URL: {}", r.url);- /// }- /// firecrawl::v2::SearchResultOrDocument::Document(d) => {- /// println!("Content: {:?}", d.markdown);- /// }- /// }- /// }- ///- /// // Search with options- /// let options = SearchOptions {- /// limit: Some(10),- /// sources: Some(vec![SearchSource::Web, SearchSource::News]),- /// ..Default::default()- /// };- /// let results = client.search("rust programming", options).await?;- ///- /// Ok(())- /// }- /// ```- pub async fn search(- &self,- query: impl AsRef<str>,- options: impl Into<Option<SearchOptions>>,- ) -> Result<SearchResponse, FirecrawlError> {- let body = SearchRequest {- query: query.as_ref().to_string(),- options: options.into().unwrap_or_default(),- };-- let headers = self.prepare_headers(None);-- let response = self- .client- .post(self.url("/search"))- .headers(headers)- .json(&body)- .send()- .await- .map_err(|e| {- FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)- })?;-- self.handle_response(response, "search").await- }-- /// Searches the web and scrapes the results.- ///- /// This is a convenience method that enables scraping for all results.- ///- /// # Arguments- ///- /// * `query` - The search query string.- /// * `limit` - Maximum number of results to return.- ///- /// # Returns- ///- /// A vector of scraped documents.- ///- /// # Example- ///- /// ```no_run- /// use firecrawl::v2::Client;- ///- /// #[tokio::main]- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {- /// let client = Client::new("your-api-key")?;- ///- /// let documents = client.search_and_scrape("rust programming", 5).await?;- /// for doc in documents {- /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));- /// println!("Content: {:?}", doc.markdown);- /// }- ///- /// Ok(())- /// }- /// ```- pub async fn search_and_scrape(- &self,- query: impl AsRef<str>,- limit: u32,- ) -> Result<Vec<Document>, FirecrawlError> {- let options = SearchOptions {- limit: Some(limit),- scrape_options: Some(ScrapeOptions::default()),- ..Default::default()- };-- let response = self.search(query, options).await?;-- let documents: Vec<Document> = response- .data- .web- .unwrap_or_default()- .into_iter()- .filter_map(|result| match result {- SearchResultOrDocument::Document(doc) => Some(doc),- SearchResultOrDocument::WebResult(_) => None,- })- .collect();-- Ok(documents)- }-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- #[tokio::test]- async fn test_search_with_mock() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/search")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "web": [- {- "url": "https://example.com",- "title": "Example Domain",- "description": "This domain is for examples"- },- {- "url": "https://example.org",- "title": "Another Example",- "description": "More examples"- }- ]- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client.search("test query", None).await.unwrap();-- assert!(response.success);- let web_results = response.data.web.unwrap();- assert_eq!(web_results.len(), 2);- mock.assert();- }-- #[tokio::test]- async fn test_search_with_options() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/search")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "web": [- {- "url": "https://example.com",- "title": "Test Result",- "description": "A test result"- }- ],- "news": [- {- "title": "Breaking News",- "url": "https://news.example.com",- "snippet": "Something happened",- "date": "2024-01-01"- }- ]- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let options = SearchOptions {- limit: Some(10),- sources: Some(vec![SearchSource::Web, SearchSource::News]),- ..Default::default()- };-- let response = client.search("test", options).await.unwrap();-- assert!(response.success);- assert!(response.data.web.is_some());- assert!(response.data.news.is_some());- mock.assert();- }-- #[tokio::test]- async fn test_search_and_scrape() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/search")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "web": [- {- "markdown": "# Example\n\nThis is the scraped content.",- "metadata": {- "sourceURL": "https://example.com",- "statusCode": 200,- "title": "Example"- }- }- ]- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let documents = client.search_and_scrape("test", 5).await.unwrap();-- assert_eq!(documents.len(), 1);- assert!(documents[0].markdown.is_some());- mock.assert();- }-- #[tokio::test]- async fn test_search_error_response() {- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/search")- .with_status(400)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": false,- "error": "Invalid query"- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let result = client.search("", None).await;-- assert!(result.is_err());- mock.assert();- }-- #[tokio::test]- async fn test_search_mixed_results_deserialization() {- // Test that results with markdown/metadata are correctly identified as Documents- // and simple results are identified as WebResults- let mut server = mockito::Server::new_async().await;-- let mock = server- .mock("POST", "/v2/search")- .with_status(200)- .with_header("content-type", "application/json")- .with_body(- json!({- "success": true,- "data": {- "web": [- // Simple web result (no markdown/metadata)- {- "url": "https://example.com/simple",- "title": "Simple Result",- "description": "Just a search result"- },- // Scraped document with url AND markdown- {- "url": "https://example.com/scraped",- "markdown": "# Scraped Content\n\nThis has content.",- "metadata": {- "sourceURL": "https://example.com/scraped",- "statusCode": 200,- "title": "Scraped Page"- }- }- ]- }- })- .to_string(),- )- .create();-- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();- let response = client.search("test", None).await.unwrap();-- let web_results = response.data.web.unwrap();- assert_eq!(web_results.len(), 2);-- // First should be WebResult- match &web_results[0] {- SearchResultOrDocument::WebResult(r) => {- assert_eq!(r.url, "https://example.com/simple");- assert_eq!(r.title, Some("Simple Result".to_string()));- }- SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),- }-- // Second should be Document (has markdown)- match &web_results[1] {- SearchResultOrDocument::Document(d) => {- assert!(d.markdown.is_some());- assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));- }- SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),- }-- mock.assert();- }-}diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rsdeleted file mode 100644index d6a09088b..000000000--- a/apps/rust-sdk/tests/e2e_with_auth.rs+++ /dev/null@@ -1,184 +0,0 @@-use dotenvy::dotenv;-use firecrawl::scrape::{JsonOptions, ScrapeFormats, ScrapeOptions};-use firecrawl::{FirecrawlApp, FirecrawlError};-use serde_json::json;-use std::env;--// #[tokio::test]-// async fn test_unsupported_url() {-// dotenv().ok();-// let api_url = env::var("API_URL").unwrap();-// let api_key = env::var("TEST_API_KEY").ok();-// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();-// let unsupported_url = "https://facebook.com/fake-test";-// let result = app.scrape_url(unsupported_url, None).await;--// assert_matches!(-// result,-// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")-// );-// }--#[tokio::test]-async fn test_successful_response_with_valid_preview_token() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let app =- FirecrawlApp::new_selfhosted(api_url, Some(env::var("PREVIEW_TOKEN").unwrap())).unwrap();- let result = app- .scrape_url("https://roastmywebsite.ai", None)- .await- .unwrap();- assert!(result.markdown.is_some());- assert!(result.markdown.unwrap().contains("_Roast_"));-}--#[tokio::test]-async fn test_scrape_url_e2e() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let result = app- .scrape_url("https://roastmywebsite.ai", None)- .await- .unwrap();- assert!(result.markdown.is_some());- assert!(result.markdown.unwrap().contains("_Roast_"));-}--#[tokio::test]-async fn test_successful_response_with_valid_api_key_and_include_html() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let params = ScrapeOptions {- formats: vec![ScrapeFormats::Markdown, ScrapeFormats::HTML].into(),- ..Default::default()- };- let result = app- .scrape_url("https://roastmywebsite.ai", params)- .await- .unwrap();- assert!(result.markdown.is_some());- assert!(result.html.is_some());- assert!(result.markdown.unwrap().contains("_Roast_"));- assert!(result.html.unwrap().contains("<h1"));-}--#[tokio::test]-async fn test_successful_response_for_valid_scrape_with_pdf_file() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let result = app- .scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None)- .await- .unwrap();- assert!(result.markdown.is_some());- assert!(result- .markdown- .unwrap()- .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));-}--#[tokio::test]-async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let result = app- .scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None)- .await- .unwrap();- assert!(result.markdown.is_some());- assert!(result- .markdown- .unwrap()- .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));-}--// #[tokio::test]-// async fn test_should_return_error_for_unsupported_url() {-// dotenv().ok();-// let api_url = env::var("API_URL").unwrap();-// let api_key = env::var("TEST_API_KEY").ok();-// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();-// let unsupported_url = "https://twitter.com/fake-test";-// let result = app.crawl_url(unsupported_url, None).await;--// assert_matches!(-// result,-// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.")-// );-// }--#[tokio::test]-async fn test_llm_extraction() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let options = ScrapeOptions {- formats: vec![ScrapeFormats::Json].into(),- json_options: JsonOptions {- prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source".to_string().into(),- schema: json!({- "type": "object",- "properties": {- "company_mission": {"type": "string"},- "supports_sso": {"type": "boolean"},- "is_open_source": {"type": "boolean"}- },- "required": ["company_mission", "supports_sso", "is_open_source"]- }).into(),- ..Default::default()- }.into(),- ..Default::default()- };- let result = app- .scrape_url("https://mendable.ai", options)- .await- .unwrap();- assert!(result.extract.is_some());- let llm_extraction = &result.extract.unwrap();- assert!(llm_extraction- .as_object()- .unwrap()- .contains_key("company_mission"));- assert!(llm_extraction["supports_sso"].is_boolean());- assert!(llm_extraction["is_open_source"].is_boolean());-}--#[test]-fn test_api_key_requirements() {- dotenv().ok();-- let api_url = env::var("API_URL").unwrap_or("http://localhost:3002".to_string());- let api_key = env::var("TEST_API_KEY").ok();-- match (api_url.contains("api.firecrawl.dev"), api_key) {- (false, _) => {- let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);- assert!(- result.is_ok(),- "Local setup failed: {:?}",- result.err().unwrap()- );- }- (true, None) => {- let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);- assert!(matches!(- result,- Err(FirecrawlError::APIError(msg, _)) if msg == "Configuration"- ));- }- (true, Some(key)) => {- let result = FirecrawlApp::new_selfhosted(&api_url, Some(&key));- assert!(result.is_ok());- }- }-}diff --git a/apps/rust-sdk/tests/v2_e2e.rs b/apps/rust-sdk/tests/v2_e2e.rsindex f91398206..619be32ec 100644--- a/apps/rust-sdk/tests/v2_e2e.rs+++ b/apps/rust-sdk/tests/v2_e2e.rs@@ -1,4 +1,4 @@-//! End-to-end tests for Firecrawl v2 API.+//! End-to-end tests for Firecrawl API.//!//! These tests require the following environment variables://! - API_URL: The Firecrawl API URL@@ -6,16 +6,14 @@//!//! Run with: cargo test --test v2_e2e -- --ignored-use dotenvy::dotenv;-use firecrawl::v2::{- AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions, ScrapeOptions,- SearchOptions, SitemapMode,+use firecrawl::{+ AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, JobStatus, MapOptions,+ ScrapeOptions, SearchOptions, SitemapMode,};use serde_json::json;use std::env;fn get_client() -> Client {- dotenv().ok();let api_url = env::var("API_URL").expect("API_URL environment variable is required");let api_key = env::var("TEST_API_KEY").ok();Client::new_selfhosted(api_url, api_key).expect("Failed to create client")@@ -23,7 +21,7 @@ fn get_client() -> Client {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_scrape() {+async fn test_scrape() {let client = get_client();let doc = client.scrape("https://example.com", None)@@ -35,7 +33,7 @@ async fn test_v2_scrape() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_scrape_with_options() {+async fn test_scrape_with_options() {let client = get_client();let options = ScrapeOptions {formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),@@ -54,7 +52,7 @@ async fn test_v2_scrape_with_options() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_scrape_with_schema() {+async fn test_scrape_with_schema() {let client = get_client();let schema = json!({@@ -75,7 +73,7 @@ async fn test_v2_scrape_with_schema() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_search() {+async fn test_search() {let client = get_client();let response = client.search("rust programming", None)@@ -87,7 +85,7 @@ async fn test_v2_search() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_search_with_options() {+async fn test_search_with_options() {let client = get_client();let options = SearchOptions {limit: Some(5),@@ -104,7 +102,7 @@ async fn test_v2_search_with_options() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_map() {+async fn test_map() {let client = get_client();let response = client.map("https://example.com", None)@@ -116,7 +114,7 @@ async fn test_v2_map() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_map_with_options() {+async fn test_map_with_options() {let client = get_client();let options = MapOptions {sitemap: Some(SitemapMode::Include),@@ -134,7 +132,7 @@ async fn test_v2_map_with_options() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_crawl_async() {+async fn test_crawl_async() {let client = get_client();let response = client.start_crawl("https://example.com", None)@@ -150,10 +148,7 @@ async fn test_v2_crawl_async() {.await.expect("Get crawl status should succeed");assert!(- matches!(- status.status,- firecrawl::v2::JobStatus::Scraping | firecrawl::v2::JobStatus::Completed- ),+ matches!(status.status, JobStatus::Scraping | JobStatus::Completed),"Status should be scraping or completed");@@ -163,7 +158,7 @@ async fn test_v2_crawl_async() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_crawl_sync() {+async fn test_crawl_sync() {let client = get_client();let options = CrawlOptions {limit: Some(2),@@ -177,14 +172,14 @@ async fn test_v2_crawl_sync() {.expect("Crawl should succeed");assert!(- job.status == firecrawl::v2::JobStatus::Completed,+ job.status == JobStatus::Completed,"Job should be completed");}#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_batch_scrape_async() {+async fn test_batch_scrape_async() {let client = get_client();let urls = vec!["https://example.com".to_string(),@@ -202,7 +197,7 @@ async fn test_v2_batch_scrape_async() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_batch_scrape_sync() {+async fn test_batch_scrape_sync() {let client = get_client();let urls = vec!["https://example.com".to_string()];@@ -221,14 +216,14 @@ async fn test_v2_batch_scrape_sync() {.expect("Batch scrape should succeed");assert!(- job.status == firecrawl::v2::JobStatus::Completed,+ job.status == JobStatus::Completed,"Job should be completed");}#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_agent_async() {+async fn test_agent_async() {let client = get_client();let options = AgentOptions {urls: Some(vec!["https://example.com".to_string()]),@@ -247,7 +242,7 @@ async fn test_v2_agent_async() {#[tokio::test]#[ignore = "Requires API access"]-async fn test_v2_agent_with_schema() {+async fn test_agent_with_schema() {let client = get_client();#[derive(Debug, serde::Deserialize)]@@ -280,10 +275,10 @@ async fn test_v2_agent_with_schema() {}}-// Test that the v2 client can be created with different configurations+// Test that the client can be created with different configurations// This test doesn't require API access#[test]-fn test_v2_client_creation() {+fn test_client_creation() {// Cloud client requires API keylet cloud_result = Client::new("test-key");assert!(cloud_result.is_ok());