Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(rust-sdk): upgrade to official v2 SDK

firecrawl-spring[bot]1c40db842426
diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.py
index 34c56d838..7c4f4a21e 100644
--- a/.github/scripts/check_version_has_incremented.py
+++ b/.github/scripts/check_version_has_incremented.py
@@ -82,18 +82,25 @@ def get_maven_central_version(package_name: str) -> str:
return version_match.group(1).strip()
return "0.0.0"
-# def get_rust_version(file_path: str) -> str:
-# """Extract version string from Cargo.toml."""
-# cargo_toml = toml.load(file_path)
-# if 'package' in cargo_toml and 'version' in cargo_toml['package']:
-# return cargo_toml['package']['version'].strip()
-# raise RuntimeError("Unable to find version string in Cargo.toml.")
-
-# def get_crates_version(package_name: str) -> str:
-# """Get latest version of Rust package from crates.io."""
-# response = requests.get(f"https://crates.io/api/v1/crates/{package_name}")
-# version = response.json()['crate']['newest_version']
-# return version.strip()
+def get_rust_version(file_path: str) -> str:
+ """Extract version string from Cargo.toml."""
+ import toml
+ cargo_toml = toml.load(file_path)
+ if 'package' in cargo_toml and 'version' in cargo_toml['package']:
+ return cargo_toml['package']['version'].strip()
+ raise RuntimeError("Unable to find version string in Cargo.toml.")
+
+def get_crates_version(package_name: str) -> str:
+ """Get latest version of Rust package from crates.io."""
+ response = requests.get(
+ f"https://crates.io/api/v1/crates/{package_name}",
+ headers={"User-Agent": "firecrawl-version-check"}
+ )
+ if response.status_code == 404:
+ return "0.0.0"
+ response.raise_for_status()
+ version = response.json()['crate']['newest_version']
+ return version.strip()
def is_version_incremented(local_version: str, published_version: str) -> bool:
"""Compare local and published versions."""
@@ -121,14 +128,14 @@ if __name__ == "__main__":
current_version = get_gradle_version(os.path.join(package_path, 'build.gradle.kts'))
# Get published version from Maven Central
published_version = get_maven_central_version(package_name)
- # if package_type == "rust":
- # # Get current version from Cargo.toml
- # current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))
- # # Get published version from crates.io
- # published_version = get_crates_version(package_name)
+ elif package_type == "rust":
+ # Get current version from Cargo.toml
+ current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))
+ # Get published version from crates.io
+ published_version = get_crates_version(package_name)
else:
- raise ValueError("Invalid package type. Use 'python', 'js', or 'java'.")
+ raise ValueError("Invalid package type. Use 'python', 'js', 'java', or 'rust'.")
# Print versions for debugging
# print(f"Local version: {current_version}")
diff --git a/.github/workflows/publish-rust-sdk.yml b/.github/workflows/publish-rust-sdk.yml
new file mode 100644
index 000000000..a005553e0
--- /dev/null
+++ b/.github/workflows/publish-rust-sdk.yml
@@ -0,0 +1,64 @@
+name: Publish Rust SDK
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - main
+ paths:
+ - 'apps/rust-sdk/**'
+
+jobs:
+ publish:
+ name: Publish to crates.io
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.x'
+
+ - name: Install version check dependencies
+ run: pip install requests packaging toml
+
+ - name: Run version check script
+ id: version_check
+ run: |
+ VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py rust ./apps/rust-sdk firecrawl)
+ echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+ - name: Install Rust toolchain
+ if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
+
+ - name: Cache cargo registry
+ if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ apps/rust-sdk/target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
+
+ - name: Build
+ if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ working-directory: ./apps/rust-sdk
+ run: cargo build --release
+
+ - name: Run tests
+ if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ working-directory: ./apps/rust-sdk
+ run: cargo test --lib
+
+ - name: Publish to crates.io
+ if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ working-directory: ./apps/rust-sdk
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+ run: cargo publish
diff --git a/README.md b/README.md
index 239d3577b..eb818202d 100644
--- a/README.md
+++ b/README.md
@@ -655,10 +655,43 @@ end
{:ok, response} = Firecrawl.map_urls(url: "https://example.com")
```
+### Rust
+
+Add the dependency:
+```toml
+[dependencies]
+firecrawl = "2"
+```
+```rust
+use firecrawl::{Client, ScrapeOptions, Format, CrawlOptions};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let client = Client::new("fc-YOUR_API_KEY")?;
+
+ // Scrape a URL
+ let document = client.scrape("https://firecrawl.dev", None).await?;
+ println!("{:?}", document.markdown);
+
+ // Crawl a website
+ let options = CrawlOptions {
+ limit: Some(50),
+ ..Default::default()
+ };
+ let result = client.crawl("https://docs.firecrawl.dev", options).await?;
+ println!("Crawled {} pages", result.data.len());
+
+ // Search the web
+ let response = client.search("best web scraping tools 2024", None).await?;
+ println!("{:?}", response.data);
+
+ Ok(())
+}
+```
+
### Community SDKs
- [Go SDK](https://github.com/mendableai/firecrawl-go)
-- [Rust SDK](https://docs.firecrawl.dev/sdks/rust)
---
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 120fe7e5e..701d04b9b 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -41,83 +41,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "ansi_colours"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14eec43e0298190790f41679fe69ef7a829d2a2ddd78c8c00339e84710e435fe"
-dependencies = [
- "rgb",
-]
-
-[[package]]
-name = "anstream"
-version = "0.6.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
-dependencies = [
- "windows-sys 0.60.2",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
-dependencies = [
- "anstyle",
- "once_cell_polyfill",
- "windows-sys 0.60.2",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.99"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
-
-[[package]]
-name = "arrayref"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
-
-[[package]]
-name = "arrayvec"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
-
[[package]]
name = "assert-json-diff"
version = "2.0.2"
@@ -128,12 +51,6 @@ dependencies = [
"serde_json",
]
-[[package]]
-name = "assert_matches"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9"
-
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -146,72 +63,6 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
-[[package]]
-name = "axum"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
-dependencies = [
- "axum-core",
- "axum-macros",
- "bytes",
- "form_urlencoded",
- "futures-util",
- "http",
- "http-body",
- "http-body-util",
- "hyper",
- "hyper-util",
- "itoa",
- "matchit",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "rustversion",
- "serde",
- "serde_json",
- "serde_path_to_error",
- "serde_urlencoded",
- "sync_wrapper",
- "tokio",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "axum-core"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
-dependencies = [
- "bytes",
- "futures-core",
- "http",
- "http-body",
- "http-body-util",
- "mime",
- "pin-project-lite",
- "rustversion",
- "sync_wrapper",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "axum-macros"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "backtrace"
version = "0.3.75"
@@ -227,154 +78,36 @@ dependencies = [
"windows-targets 0.52.6",
]
-[[package]]
-name = "base64"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
-
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-[[package]]
-name = "bat"
-version = "0.25.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab792c2ad113a666f08856c88cdec0a62d732559b1f3982eedf0142571e669a"
-dependencies = [
- "ansi_colours",
- "anyhow",
- "bincode",
- "bugreport",
- "bytesize",
- "clap",
- "clircle",
- "console",
- "content_inspector",
- "encoding_rs",
- "etcetera",
- "flate2",
- "git2",
- "globset",
- "grep-cli",
- "home",
- "indexmap 2.10.0",
- "itertools",
- "nu-ansi-term",
- "once_cell",
- "path_abs",
- "plist",
- "regex",
- "semver",
- "serde",
- "serde_derive",
- "serde_with",
- "serde_yaml",
- "shell-words",
- "syntect",
- "terminal-colorsaurus",
- "thiserror",
- "toml",
- "unicode-width 0.1.14",
- "walkdir",
- "wild",
-]
-
-[[package]]
-name = "bincode"
-version = "1.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
[[package]]
name = "bitflags"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
-[[package]]
-name = "blake2b_simd"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"
-dependencies = [
- "arrayref",
- "arrayvec",
- "constant_time_eq",
-]
-
-[[package]]
-name = "bstr"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
-dependencies = [
- "memchr",
- "regex-automata",
- "serde",
-]
-
-[[package]]
-name = "bugreport"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f280f65ce85b880919349bbfcb204930291251eedcb2e5f84ce2f51df969c162"
-dependencies = [
- "git-version",
- "shell-escape",
- "sysinfo",
-]
-
[[package]]
name = "bumpalo"
version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
-[[package]]
-name = "bytemuck"
-version = "1.23.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677"
-
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
[[package]]
name = "bytes"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
-[[package]]
-name = "bytesize"
-version = "1.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
-
[[package]]
name = "cc"
version = "1.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e"
dependencies = [
- "jobserver",
- "libc",
"shlex",
]
@@ -397,72 +130,6 @@ dependencies = [
"windows-link",
]
-[[package]]
-name = "clap"
-version = "4.5.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c1f056bae57e3e54c3375c41ff79619ddd13460a17d7438712bd0d83fda4ff8"
-dependencies = [
- "clap_builder",
- "clap_derive",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.5.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
- "terminal_size",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.5.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
-
-[[package]]
-name = "clippy"
-version = "0.0.302"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d911ee15579a3f50880d8c1d59ef6e79f9533127a3bd342462f5d584f5e8c294"
-dependencies = [
- "term",
-]
-
-[[package]]
-name = "clircle"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d9334f725b46fb9bed8580b9b47a932587e044fadb344ed7fa98774b067ac1a"
-dependencies = [
- "cfg-if",
- "windows 0.56.0",
-]
-
-[[package]]
-name = "colorchoice"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
-
[[package]]
name = "colored"
version = "3.0.0"
@@ -472,34 +139,6 @@ dependencies = [
"windows-sys 0.59.0",
]
-[[package]]
-name = "console"
-version = "0.15.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
-dependencies = [
- "encode_unicode",
- "libc",
- "once_cell",
- "unicode-width 0.2.1",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
-[[package]]
-name = "content_inspector"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -516,40 +155,6 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
-[[package]]
-name = "crc32fast"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
-dependencies = [
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
-
[[package]]
name = "darling"
version = "0.20.11"
@@ -595,17 +200,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "dirs"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
[[package]]
name = "displaydoc"
version = "0.2.5"
@@ -617,30 +211,12 @@ dependencies = [
"syn",
]
-[[package]]
-name = "dotenvy"
-version = "0.15.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
-
[[package]]
name = "dyn-clone"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
-[[package]]
-name = "either"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
-
-[[package]]
-name = "encode_unicode"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
-
[[package]]
name = "encoding_rs"
version = "0.8.35"
@@ -666,17 +242,6 @@ dependencies = [
"windows-sys 0.60.2",
]
-[[package]]
-name = "etcetera"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
-dependencies = [
- "cfg-if",
- "home",
- "windows-sys 0.48.0",
-]
-
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -685,36 +250,15 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "1.2.3"
-dependencies = [
- "assert_matches",
- "axum",
- "bat",
- "clap",
- "clippy",
- "dotenvy",
- "futures",
- "log",
+version = "2.0.0"
+dependencies = [
"mockito",
"reqwest",
- "schemars 0.8.22",
"serde",
"serde_json",
"serde_with",
"thiserror",
"tokio",
- "tracing",
- "uuid",
-]
-
-[[package]]
-name = "flate2"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d"
-dependencies = [
- "crc32fast",
- "miniz_oxide",
]
[[package]]
@@ -747,21 +291,6 @@ dependencies = [
"percent-encoding",
]
-[[package]]
-name = "futures"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-io",
- "futures-sink",
- "futures-task",
- "futures-util",
-]
-
[[package]]
name = "futures-channel"
version = "0.3.31"
@@ -769,7 +298,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
- "futures-sink",
]
[[package]]
@@ -778,34 +306,6 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
-[[package]]
-name = "futures-executor"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
-dependencies = [
- "futures-core",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-io"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
-
-[[package]]
-name = "futures-macro"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "futures-sink"
version = "0.3.31"
@@ -824,27 +324,10 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
- "futures-channel",
"futures-core",
- "futures-io",
- "futures-macro",
- "futures-sink",
"futures-task",
- "memchr",
"pin-project-lite",
"pin-utils",
- "slab",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
@@ -876,72 +359,6 @@ version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
-[[package]]
-name = "git-version"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19"
-dependencies = [
- "git-version-macro",
-]
-
-[[package]]
-name = "git-version-macro"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "git2"
-version = "0.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"
-dependencies = [
- "bitflags 2.9.1",
- "libc",
- "libgit2-sys",
- "log",
- "url",
-]
-
-[[package]]
-name = "glob"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
-
-[[package]]
-name = "globset"
-version = "0.4.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
-dependencies = [
- "aho-corasick",
- "bstr",
- "log",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "grep-cli"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47f1288f0e06f279f84926fa4c17e3fcd2a22b357927a82f2777f7be26e4cec0"
-dependencies = [
- "bstr",
- "globset",
- "libc",
- "log",
- "termcolor",
- "winapi-util",
-]
-
[[package]]
name = "h2"
version = "0.4.12"
@@ -973,27 +390,12 @@ version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-[[package]]
-name = "home"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
-dependencies = [
- "windows-sys 0.59.0",
-]
-
[[package]]
name = "http"
version = "1.3.1"
@@ -1099,7 +501,7 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
dependencies = [
- "base64 0.22.1",
+ "base64",
"bytes",
"futures-channel",
"futures-core",
@@ -1131,7 +533,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
- "windows-core 0.61.2",
+ "windows-core",
]
[[package]]
@@ -1284,7 +686,7 @@ version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"cfg-if",
"libc",
]
@@ -1305,37 +707,12 @@ dependencies = [
"serde",
]
-[[package]]
-name = "is_terminal_polyfill"
-version = "1.70.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
-
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
[[package]]
name = "itoa"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
-[[package]]
-name = "jobserver"
-version = "0.1.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
-dependencies = [
- "getrandom 0.3.3",
- "libc",
-]
-
[[package]]
name = "js-sys"
version = "0.3.77"
@@ -1352,36 +729,6 @@ version = "0.2.175"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
-[[package]]
-name = "libgit2-sys"
-version = "0.17.0+1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224"
-dependencies = [
- "cc",
- "libc",
- "libz-sys",
- "pkg-config",
-]
-
-[[package]]
-name = "libz-sys"
-version = "1.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
-[[package]]
-name = "linked-hash-map"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
-
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
@@ -1410,12 +757,6 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
-[[package]]
-name = "matchit"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
-
[[package]]
name = "memchr"
version = "2.7.5"
@@ -1489,24 +830,6 @@ dependencies = [
"tempfile",
]
-[[package]]
-name = "ntapi"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "nu-ansi-term"
-version = "0.50.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
-dependencies = [
- "windows-sys 0.52.0",
-]
-
[[package]]
name = "num-conv"
version = "0.1.0"
@@ -1537,41 +860,13 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
-[[package]]
-name = "once_cell_polyfill"
-version = "1.70.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
-
-[[package]]
-name = "onig"
-version = "6.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
-dependencies = [
- "bitflags 2.9.1",
- "libc",
- "once_cell",
- "onig_sys",
-]
-
-[[package]]
-name = "onig_sys"
-version = "69.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"
-dependencies = [
- "cc",
- "pkg-config",
-]
-
[[package]]
name = "openssl"
version = "0.10.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"cfg-if",
"foreign-types",
"libc",
@@ -1627,20 +922,11 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.17",
+ "redox_syscall",
"smallvec",
"windows-targets 0.52.6",
]
-[[package]]
-name = "path_abs"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3"
-dependencies = [
- "std_prelude",
-]
-
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -1665,19 +951,6 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
-[[package]]
-name = "plist"
-version = "1.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1"
-dependencies = [
- "base64 0.22.1",
- "indexmap 2.10.0",
- "quick-xml",
- "serde",
- "time",
-]
-
[[package]]
name = "potential_utf"
version = "0.1.2"
@@ -1711,15 +984,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "quick-xml"
-version = "0.38.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9845d9dccf565065824e69f9f235fafba1587031eda353c1f1561cd6a6be78f4"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "quote"
version = "1.0.40"
@@ -1764,50 +1028,13 @@ dependencies = [
"getrandom 0.3.3",
]
-[[package]]
-name = "rayon"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.1.57"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
-
[[package]]
name = "redox_syscall"
version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
dependencies = [
- "bitflags 2.9.1",
-]
-
-[[package]]
-name = "redox_users"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
-dependencies = [
- "getrandom 0.1.16",
- "redox_syscall 0.1.57",
- "rust-argon2",
+ "bitflags",
]
[[package]]
@@ -1865,12 +1092,10 @@ version = "0.12.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
dependencies = [
- "base64 0.22.1",
+ "base64",
"bytes",
"encoding_rs",
- "futures-channel",
"futures-core",
- "futures-util",
"h2",
"http",
"http-body",
@@ -1901,15 +1126,6 @@ dependencies = [
"web-sys",
]
-[[package]]
-name = "rgb"
-version = "0.8.52"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce"
-dependencies = [
- "bytemuck",
-]
-
[[package]]
name = "ring"
version = "0.17.14"
@@ -1924,18 +1140,6 @@ dependencies = [
"windows-sys 0.52.0",
]
-[[package]]
-name = "rust-argon2"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
-dependencies = [
- "base64 0.13.1",
- "blake2b_simd",
- "constant_time_eq",
- "crossbeam-utils",
-]
-
[[package]]
name = "rustc-demangle"
version = "0.1.26"
@@ -1948,7 +1152,7 @@ version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"errno",
"libc",
"linux-raw-sys",
@@ -2000,15 +1204,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
-[[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
-
[[package]]
name = "schannel"
version = "0.1.27"
@@ -2018,18 +1213,6 @@ dependencies = [
"windows-sys 0.59.0",
]
-[[package]]
-name = "schemars"
-version = "0.8.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
-dependencies = [
- "dyn-clone",
- "schemars_derive",
- "serde",
- "serde_json",
-]
-
[[package]]
name = "schemars"
version = "0.9.0"
@@ -2054,18 +1237,6 @@ dependencies = [
"serde_json",
]
-[[package]]
-name = "schemars_derive"
-version = "0.8.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
-dependencies = [
- "proc-macro2",
- "quote",
- "serde_derive_internals",
- "syn",
-]
-
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -2078,7 +1249,7 @@ version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
@@ -2095,12 +1266,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "semver"
-version = "1.0.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
-
[[package]]
name = "serde"
version = "1.0.219"
@@ -2121,17 +1286,6 @@ dependencies = [
"syn",
]
-[[package]]
-name = "serde_derive_internals"
-version = "0.29.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "serde_json"
version = "1.0.142"
@@ -2144,25 +1298,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "serde_path_to_error"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"
-dependencies = [
- "itoa",
- "serde",
-]
-
-[[package]]
-name = "serde_spanned"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
-dependencies = [
- "serde",
-]
-
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -2181,7 +1316,7 @@ version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5"
dependencies = [
- "base64 0.22.1",
+ "base64",
"chrono",
"hex",
"indexmap 1.9.3",
@@ -2207,31 +1342,6 @@ dependencies = [
"syn",
]
-[[package]]
-name = "serde_yaml"
-version = "0.9.34+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
-dependencies = [
- "indexmap 2.10.0",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
-[[package]]
-name = "shell-escape"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
-
-[[package]]
-name = "shell-words"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
-
[[package]]
name = "shlex"
version = "1.3.0"
@@ -2281,12 +1391,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-[[package]]
-name = "std_prelude"
-version = "0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe"
-
[[package]]
name = "strsim"
version = "0.11.1"
@@ -2323,47 +1427,11 @@ dependencies = [
name = "synstructure"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "syntect"
-version = "5.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
-dependencies = [
- "bincode",
- "bitflags 1.3.2",
- "flate2",
- "fnv",
- "once_cell",
- "onig",
- "plist",
- "regex-syntax",
- "serde",
- "serde_derive",
- "serde_json",
- "thiserror",
- "walkdir",
- "yaml-rust",
-]
-
-[[package]]
-name = "sysinfo"
-version = "0.33.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
-dependencies = [
- "core-foundation-sys",
- "libc",
- "memchr",
- "ntapi",
- "rayon",
- "windows 0.57.0",
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
@@ -2372,7 +1440,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"core-foundation",
"system-configuration-sys",
]
@@ -2400,62 +1468,6 @@ dependencies = [
"windows-sys 0.59.0",
]
-[[package]]
-name = "term"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42"
-dependencies = [
- "byteorder",
- "dirs",
- "winapi",
-]
-
-[[package]]
-name = "termcolor"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "terminal-colorsaurus"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7afe4c174a3cbfb52ebcb11b28965daf74fe9111d4e07e40689d05af06e26e8"
-dependencies = [
- "cfg-if",
- "libc",
- "memchr",
- "mio",
- "terminal-trx",
- "windows-sys 0.59.0",
- "xterm-color",
-]
-
-[[package]]
-name = "terminal-trx"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "975b4233aefa1b02456d5e53b22c61653c743e308c51cf4181191d8ce41753ab"
-dependencies = [
- "cfg-if",
- "libc",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "terminal_size"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"
-dependencies = [
- "rustix",
- "windows-sys 0.60.2",
-]
-
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -2581,48 +1593,6 @@ dependencies = [
"tokio",
]
-[[package]]
-name = "toml"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
-dependencies = [
- "indexmap 2.10.0",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
-dependencies = [
- "indexmap 2.10.0",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_write",
- "winnow",
-]
-
-[[package]]
-name = "toml_write"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
-
[[package]]
name = "tower"
version = "0.5.2"
@@ -2636,7 +1606,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
- "tracing",
]
[[package]]
@@ -2645,7 +1614,7 @@ version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
"bytes",
"futures-util",
"http",
@@ -2675,23 +1644,10 @@ version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
dependencies = [
- "log",
"pin-project-lite",
- "tracing-attributes",
"tracing-core",
]
-[[package]]
-name = "tracing-attributes"
-version = "0.1.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "tracing-core"
version = "0.1.34"
@@ -2713,24 +1669,6 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
-[[package]]
-name = "unicode-width"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
-
-[[package]]
-name = "unicode-width"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
-
-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -2754,39 +1692,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-[[package]]
-name = "utf8parse"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
-
-[[package]]
-name = "uuid"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be"
-dependencies = [
- "getrandom 0.3.3",
- "js-sys",
- "wasm-bindgen",
-]
-
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-[[package]]
-name = "walkdir"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
-dependencies = [
- "same-file",
- "winapi-util",
-]
-
[[package]]
name = "want"
version = "0.3.1"
@@ -2796,12 +1707,6 @@ dependencies = [
"try-lock",
]
-[[package]]
-name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
-
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -2898,125 +1803,19 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "wild"
-version = "2.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1"
-dependencies = [
- "glob",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-util"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
-dependencies = [
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows"
-version = "0.56.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132"
-dependencies = [
- "windows-core 0.56.0",
- "windows-targets 0.52.6",
-]
-
-[[package]]
-name = "windows"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
-dependencies = [
- "windows-core 0.57.0",
- "windows-targets 0.52.6",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.56.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6"
-dependencies = [
- "windows-implement 0.56.0",
- "windows-interface 0.56.0",
- "windows-result 0.1.2",
- "windows-targets 0.52.6",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
-dependencies = [
- "windows-implement 0.57.0",
- "windows-interface 0.57.0",
- "windows-result 0.1.2",
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows-core"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
- "windows-implement 0.60.0",
- "windows-interface 0.59.1",
+ "windows-implement",
+ "windows-interface",
"windows-link",
- "windows-result 0.3.4",
+ "windows-result",
"windows-strings",
]
-[[package]]
-name = "windows-implement"
-version = "0.56.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "windows-implement"
version = "0.60.0"
@@ -3028,28 +1827,6 @@ dependencies = [
"syn",
]
-[[package]]
-name = "windows-interface"
-version = "0.56.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "windows-interface"
version = "0.59.1"
@@ -3074,19 +1851,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e"
dependencies = [
"windows-link",
- "windows-result 0.3.4",
+ "windows-result",
"windows-strings",
]
-[[package]]
-name = "windows-result"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -3105,15 +1873,6 @@ dependencies = [
"windows-link",
]
-[[package]]
-name = "windows-sys"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
-dependencies = [
- "windows-targets 0.48.5",
-]
-
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -3141,21 +1900,6 @@ dependencies = [
"windows-targets 0.53.3",
]
-[[package]]
-name = "windows-targets"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
-dependencies = [
- "windows_aarch64_gnullvm 0.48.5",
- "windows_aarch64_msvc 0.48.5",
- "windows_i686_gnu 0.48.5",
- "windows_i686_msvc 0.48.5",
- "windows_x86_64_gnu 0.48.5",
- "windows_x86_64_gnullvm 0.48.5",
- "windows_x86_64_msvc 0.48.5",
-]
-
[[package]]
name = "windows-targets"
version = "0.52.6"
@@ -3189,12 +1933,6 @@ dependencies = [
"windows_x86_64_msvc 0.53.0",
]
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
-
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
@@ -3207,12 +1945,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
-
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
@@ -3225,12 +1957,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
-[[package]]
-name = "windows_i686_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
-
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@@ -3255,12 +1981,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
-[[package]]
-name = "windows_i686_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
-
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
@@ -3273,12 +1993,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
-
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
@@ -3291,12 +2005,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
-
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
@@ -3309,12 +2017,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
-
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
@@ -3327,22 +2029,13 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
-[[package]]
-name = "winnow"
-version = "0.7.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags",
]
[[package]]
@@ -3351,21 +2044,6 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
-[[package]]
-name = "xterm-color"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4de5f056fb9dc8b7908754867544e26145767187aaac5a98495e88ad7cb8a80f"
-
-[[package]]
-name = "yaml-rust"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
-dependencies = [
- "linked-hash-map",
-]
-
[[package]]
name = "yoke"
version = "0.8.0"
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index c47284047..b1b74cee5 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,41 +1,28 @@
[package]
name = "firecrawl"
-author= "Mendable.ai"
-version = "1.4.0"
+version = "2.0.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
-repository ="https://github.com/firecrawl/firecrawl"
-description = "Rust SDK for Firecrawl API."
+repository = "https://github.com/firecrawl/firecrawl"
+description = "Official Rust SDK for Firecrawl API v2."
authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>", "kkharji <[email protected]>"]
+keywords = ["firecrawl", "web-scraping", "crawling", "api"]
+categories = ["api-bindings", "web-programming"]
[lib]
path = "src/lib.rs"
name = "firecrawl"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-reqwest = { version = "0.12.22", features = ["json", "blocking"] }
-serde = { version = "^1.0", features = ["derive"] }
-serde_json = "^1.0"
-serde_with = "^3.9"
-log = "^0.4"
-thiserror = "^1.0"
-uuid = { version = "^1.10", features = ["v4"] }
-tokio = { version = "^1", features = ["full"] }
-futures = "0.3.31"
-schemars = "0.8.22"
-tracing = ">=0.1.0,<0.2.0"
+reqwest = { version = "0.12", features = ["json"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_with = "3"
+thiserror = "1"
+tokio = { version = "1", features = ["time"] }
[dev-dependencies]
-clippy = "^0.0.302"
-assert_matches = "^1.5"
-dotenvy = "^0.15"
-tokio = { version = "1", features = ["full"] }
-mockito = "1.7.0"
-clap = { version ="4.5.35", features = ["derive"] }
-axum = { version = "0.8.3", features = ["tokio", "macros"] }
-bat = "0.25.0"
-
-[build-dependencies]
tokio = { version = "1", features = ["full"] }
+mockito = "1.7"
+serde_json = "1"
diff --git a/apps/rust-sdk/examples/batch_scrape_example.rs b/apps/rust-sdk/examples/batch_scrape_example.rs
deleted file mode 100644
index 7db7856ec..000000000
--- a/apps/rust-sdk/examples/batch_scrape_example.rs
+++ /dev/null
@@ -1,175 +0,0 @@
-use clap::{Parser, Subcommand};
-use firecrawl::{
- batch_scrape::{BatchScrapeParams, WebhookOptions},
- map::MapOptions,
- scrape::{ScrapeFormats, ScrapeOptions},
- FirecrawlApp,
-};
-use serde_json::Value;
-use std::error::Error;
-use std::net::SocketAddr;
-use std::sync::Arc;
-use tokio::sync::Mutex;
-
-// Store webhook responses
-struct WebhookState {
- responses: Vec<Value>,
-}
-
-#[derive(Parser)]
-#[command(version, about, long_about = None)]
-struct Cli {
- #[command(subcommand)]
- command: Commands,
-}
-
-#[derive(Subcommand)]
-enum Commands {
- /// Multiple URL scraping with webhook monitoring
- Basic,
-}
-
-async fn create_firecrawl_app() -> Result<FirecrawlApp, Box<dyn Error>> {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- FirecrawlApp::new_selfhosted(api_url, None::<&str>).map_err(|e| e.into())
-}
-
-// Start webhook server and return its address
-async fn start_webhook_server(
- port: u16,
- state: Arc<Mutex<WebhookState>>,
-) -> Result<String, Box<dyn Error>> {
- let state = state.clone();
- use axum::routing::post;
- use axum::Json;
-
- let app = axum::Router::new().route(
- "/",
- post(move |body: Json<Value>| {
- let state = state.clone();
- async move {
- state.lock().await.responses.push(body.0.clone());
- match serde_json::to_string_pretty(&body.0) {
- Ok(data) => println!(
- "Received webhook: {}",
- serde_json::to_string_pretty(&data).unwrap()
- ),
- Err(_) => println!("Received webhook: {}", body.0),
- }
- "OK"
- }
- }),
- );
-
- let addr = SocketAddr::from(([0, 0, 0, 0], port));
- let webhook_url = format!("http://host.docker.internal:{}", port);
-
- tokio::spawn(async move {
- let listener = tokio::net::TcpListener::bind(addr)
- .await
- .inspect_err(|err| println!("{err:?}"))
- .unwrap();
-
- if let Err(e) = axum::serve(listener, app).await {
- eprintln!("Webhook server error: {}", e);
- }
- });
-
- println!("Webhook server running at {}", webhook_url);
-
- Ok(webhook_url)
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- let cli = Cli::parse();
- let firecrawl = create_firecrawl_app().await?;
-
- let state = Arc::new(Mutex::new(WebhookState { responses: vec![] }));
- let webhook_url = start_webhook_server(39120, state.clone()).await?;
-
- match cli.command {
- Commands::Basic => {
- let mut urls = Vec::new();
-
- let url_one = "https://invalid-url.url/";
- println!("Mapping: {}", url_one);
- match firecrawl.map_url(url_one, None).await {
- Ok(mapped_urls) => urls.extend(mapped_urls),
- Err(e) => println!("Error mapping {}: {}", url_one, e),
- }
-
- let url_two = "https://www.devjobsscanner.com";
- println!("Mapping: {}", url_two);
- match firecrawl
- .map_url(
- url_two,
- Some(MapOptions {
- search: Some("rust".into()),
- limit: Some(20),
- ..Default::default()
- }),
- )
- .await
- {
- Ok(mapped_urls) => urls.extend(mapped_urls),
- Err(e) => println!("Error mapping {}: {}", url_two, e),
- }
-
- test_multiple_urls(&firecrawl, urls, &webhook_url).await?;
-
- // Give time for webhooks to arrive
- tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
- println!(
- "Received {} webhook responses",
- state.lock().await.responses.len()
- );
- }
- }
-
- Ok(())
-}
-
-async fn test_multiple_urls(
- app: &FirecrawlApp,
- urls: Vec<String>,
- webhook_url: &str,
-) -> Result<(), Box<dyn Error>> {
- println!("Testing batch scraping of {} URLs", urls.len());
-
- let webhook = WebhookOptions {
- url: webhook_url.to_string(),
- headers: None,
- auth_token: None,
- };
-
- let params = BatchScrapeParams {
- urls,
- webhook: Some(webhook),
- ignore_invalid_urls: true,
- options: Some(ScrapeOptions {
- formats: Some(vec![ScrapeFormats::Markdown, ScrapeFormats::Links]),
- ..Default::default()
- }),
- ..Default::default()
- };
-
- let batch = app.async_batch_scrape_urls(params).await?;
- println!("Batch job started: {}", batch.id);
-
- // Poll status periodically
- loop {
- let status = app.check_batch_scrape_status(&batch.id).await?;
- println!("Progress: {}/{} pages", status.completed, status.total);
-
- if status.completed >= status.total {
- println!("Batch job completed!");
- break;
- }
-
- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
- }
-
- Ok(())
-}
diff --git a/apps/rust-sdk/examples/cancel_crawl_example.rs b/apps/rust-sdk/examples/cancel_crawl_example.rs
deleted file mode 100644
index 3451cc104..000000000
--- a/apps/rust-sdk/examples/cancel_crawl_example.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use firecrawl::FirecrawlApp;
-use std::error::Error;
-use std::time::Duration;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- // Get API URL from environment
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
-
- // Create the FirecrawlApp instance
- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
-
- // Start a crawl job
- println!("Starting a crawl job...");
- let crawl_response = firecrawl
- .crawl_url_async("https://example.com", None)
- .await?;
- println!("Crawl job started with ID: {}", crawl_response.id);
-
- // Wait for a moment to let the crawl job start
- println!("Waiting for a moment...");
- tokio::time::sleep(Duration::from_secs(2)).await;
-
- // Cancel the crawl job
- println!("Cancelling the crawl job...");
- let cancel_response = firecrawl.cancel_crawl(&crawl_response.id).await?;
-
- println!("Cancellation result:");
- println!(" Status: {:?}", cancel_response.status);
-
- Ok(())
-}
diff --git a/apps/rust-sdk/examples/check_crawl_errors_example.rs b/apps/rust-sdk/examples/check_crawl_errors_example.rs
deleted file mode 100644
index 7629d36cb..000000000
--- a/apps/rust-sdk/examples/check_crawl_errors_example.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use firecrawl::FirecrawlApp;
-use std::error::Error;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- // Get API URL from environment
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
-
- // Create the FirecrawlApp instance
- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
-
- // Start a crawl job that will likely have some errors (invalid URL format)
- println!("Starting a crawl job...");
- let crawl_response = firecrawl
- .crawl_url_async("https://no-wer-agg.invalid", None)
- .await?;
- println!("Crawl job started with ID: {}", crawl_response.id);
-
- println!("Let it do it's thing...");
- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
-
- // Check the crawl errors
- println!("Checking for crawl errors...");
- match firecrawl.check_crawl_errors(&crawl_response.id).await {
- Ok(error_response) => {
- println!("Crawl errors response:");
- println!(" Number of errors: {}", error_response.errors.len());
-
- if !error_response.errors.is_empty() {
- println!("\nDetailed errors:");
- for (i, error) in error_response.errors.iter().enumerate() {
- println!("Error #{}", i + 1);
- println!(" ID: {}", error.id);
- if let Some(timestamp) = &error.timestamp {
- println!(" Timestamp: {}", timestamp);
- }
- println!(" URL: {}", error.url);
- println!(" Error: {}", error.error);
- }
- }
-
- println!(
- "\nRobots.txt blocked URLs: {}",
- error_response.robots_blocked.len()
- );
- for (i, url) in error_response.robots_blocked.iter().enumerate() {
- println!(" {}. {}", i + 1, url);
- }
- }
- Err(e) => {
- println!("Failed to check crawl errors: {}", e);
- }
- }
- let cancel = firecrawl.cancel_crawl(&crawl_response.id).await?;
- println!("Cancel: {}", cancel.status);
-
- Ok(())
-}
diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs
index 37a3df7d2..57b418e93 100644
--- a/apps/rust-sdk/examples/example.rs
+++ b/apps/rust-sdk/examples/example.rs
@@ -1,90 +1,216 @@
+//! Example usage of Firecrawl API.
+//!
+//! Run with: cargo run --example example
+
use firecrawl::{
- crawl::CrawlOptions,
- scrape::{JsonOptions, ScrapeFormats, ScrapeOptions},
- FirecrawlApp,
+ AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,
+ ScrapeOptions, SearchOptions, SitemapMode,
};
+use serde::Deserialize;
use serde_json::json;
#[tokio::main]
async fn main() {
- // Initialize the FirecrawlApp with the API key
- let app = FirecrawlApp::new("fc-YOUR-API-KEY").expect("Failed to initialize FirecrawlApp");
+ // Initialize the Client with the API key
+ let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");
// Or, connect to a self-hosted instance:
- // let app = FirecrawlApp::new_selfhosted("http://localhost:3002", None).expect("Failed to initialize FirecrawlApp");
-
- // Scrape a website
- let scrape_result = app.scrape_url("https://firecrawl.dev", None).await;
+ // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)
+ // .expect("Failed to initialize Client");
- match scrape_result {
- Ok(data) => println!("Scrape Result:\n{}", data.markdown.unwrap()),
- Err(e) => eprintln!("Scrape failed: {:#?}", e),
+ // Example 1: Simple Scrape
+ println!("=== Example 1: Simple Scrape ===");
+ let result = client.scrape("https://example.com", None).await;
+ match result {
+ Ok(doc) => println!("Markdown: {:?}", doc.markdown),
+ Err(e) => eprintln!("Scrape failed: {}", e),
}
- // Crawl a website
- let crawl_options = CrawlOptions {
- exclude_paths: vec!["blog/*".into()].into(),
+ // Example 2: Scrape with Options
+ println!("\n=== Example 2: Scrape with Options ===");
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ only_main_content: Some(true),
..Default::default()
};
+ let result = client.scrape("https://example.com", options).await;
+ match result {
+ Ok(doc) => {
+ println!("Links: {:?}", doc.links);
+ }
+ Err(e) => eprintln!("Scrape with options failed: {}", e),
+ }
- let crawl_result = app.crawl_url("https://mendable.ai", crawl_options).await;
-
- match crawl_result {
+ // Example 3: Scrape with JSON Schema Extraction
+ println!("\n=== Example 3: JSON Schema Extraction ===");
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "description": { "type": "string" }
+ }
+ });
+ let result = client
+ .scrape_with_schema(
+ "https://example.com",
+ schema,
+ Some("Extract title and description"),
+ )
+ .await;
+ match result {
Ok(data) => println!(
- "Crawl Result (used {} credits):\n{:#?}",
- data.credits_used, data.data
+ "Extracted: {}",
+ serde_json::to_string_pretty(&data).unwrap()
),
- Err(e) => eprintln!("Crawl failed: {}", e),
+ Err(e) => eprintln!("Schema extraction failed: {}", e),
}
- // Scrape with Extract
- let json_schema = json!({
- "type": "object",
- "properties": {
- "top": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "title": {"type": "string"},
- "points": {"type": "number"},
- "by": {"type": "string"},
- "commentsURL": {"type": "string"}
- },
- "required": ["title", "points", "by", "commentsURL"]
- },
- "minItems": 5,
- "maxItems": 5,
- "description": "Top 5 stories on Hacker News"
+ // Example 4: Search
+ println!("\n=== Example 4: Search ===");
+ let options = SearchOptions {
+ limit: Some(5),
+ ..Default::default()
+ };
+ let result = client.search("rust programming", options).await;
+ match result {
+ Ok(response) => {
+ if let Some(web) = response.data.web {
+ for item in web {
+ match item {
+ firecrawl::SearchResultOrDocument::WebResult(r) => {
+ println!("Result: {} - {}", r.url, r.title.unwrap_or_default());
+ }
+ firecrawl::SearchResultOrDocument::Document(d) => {
+ if let Some(meta) = d.metadata {
+ println!("Document: {:?}", meta.title);
+ }
+ }
+ }
+ }
}
- },
- "required": ["top"]
- });
+ }
+ Err(e) => eprintln!("Search failed: {}", e),
+ }
+
+ // Example 5: Map a Website
+ println!("\n=== Example 5: Map ===");
+ let options = MapOptions {
+ sitemap: Some(SitemapMode::Include),
+ limit: Some(20),
+ ..Default::default()
+ };
+ let result = client.map("https://example.com", options).await;
+ match result {
+ Ok(response) => {
+ println!("Found {} links", response.links.len());
+ for link in response.links.iter().take(5) {
+ println!(" - {}", link.url);
+ }
+ }
+ Err(e) => eprintln!("Map failed: {}", e),
+ }
- let llm_extraction_options = ScrapeOptions {
- formats: vec![ScrapeFormats::Json].into(),
- json_options: JsonOptions {
- schema: json_schema.into(),
+ // Example 6: Crawl a Website
+ println!("\n=== Example 6: Crawl ===");
+ let options = CrawlOptions {
+ limit: Some(5),
+ sitemap: Some(SitemapMode::Include),
+ poll_interval: Some(3000),
+ ..Default::default()
+ };
+ let result = client.crawl("https://example.com", options).await;
+ match result {
+ Ok(job) => {
+ println!("Crawled {} pages", job.data.len());
+ for doc in job.data.iter().take(3) {
+ if let Some(meta) = &doc.metadata {
+ println!(" - {:?}", meta.source_url);
+ }
+ }
+ }
+ Err(e) => eprintln!("Crawl failed: {}", e),
+ }
+
+ // Example 7: Batch Scrape
+ println!("\n=== Example 7: Batch Scrape ===");
+ let urls = vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ];
+ let options = BatchScrapeOptions {
+ options: Some(ScrapeOptions {
+ formats: Some(vec![Format::Markdown]),
..Default::default()
+ }),
+ poll_interval: Some(2000),
+ ..Default::default()
+ };
+ let result = client.batch_scrape(urls, options).await;
+ match result {
+ Ok(job) => {
+ println!("Batch scraped {} pages", job.data.len());
}
- .into(),
+ Err(e) => eprintln!("Batch scrape failed: {}", e),
+ }
+
+ // Example 8: Agent (Autonomous Web Browsing)
+ println!("\n=== Example 8: Agent ===");
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Describe what this website is about and list any key features mentioned"
+ .to_string(),
+ model: Some(AgentModel::Spark1Pro),
+ timeout: Some(60),
..Default::default()
};
+ let result = client.agent(options).await;
+ match result {
+ Ok(response) => {
+ println!("Agent status: {:?}", response.status);
+ if let Some(data) = response.data {
+ println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());
+ }
+ }
+ Err(e) => eprintln!("Agent failed: {}", e),
+ }
- let llm_extraction_result = app
- .scrape_url("https://news.ycombinator.com", llm_extraction_options)
- .await;
+ // Example 9: Agent with Typed Schema
+ println!("\n=== Example 9: Agent with Typed Schema ===");
- match llm_extraction_result {
- Ok(data) => println!("LLM Extraction Result:\n{:#?}", data.extract.unwrap()),
- Err(e) => eprintln!("LLM Extraction failed: {}", e),
+ #[derive(Debug, Deserialize)]
+ struct CompanyInfo {
+ name: String,
+ description: Option<String>,
+ industry: Option<String>,
}
- // Map a website (Alpha)
- let map_result = app.map_url("https://firecrawl.dev", None).await;
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "description": { "type": "string" },
+ "industry": { "type": "string" }
+ },
+ "required": ["name"]
+ });
+
+ let result: Result<Option<CompanyInfo>, _> = client
+ .agent_with_schema(
+ vec!["https://firecrawl.dev".to_string()],
+ "Extract company information from this website",
+ schema,
+ )
+ .await;
- match map_result {
- Ok(data) => println!("Mapped URLs: {:#?}", data),
- Err(e) => eprintln!("Map failed: {}", e),
+ match result {
+ Ok(Some(info)) => {
+ println!("Company: {}", info.name);
+ println!("Description: {:?}", info.description);
+ println!("Industry: {:?}", info.industry);
+ }
+ Ok(None) => println!("No data extracted"),
+ Err(e) => eprintln!("Agent with schema failed: {}", e),
}
+
+ println!("\n=== All examples completed ===");
}
diff --git a/apps/rust-sdk/examples/extract_example.rs b/apps/rust-sdk/examples/extract_example.rs
deleted file mode 100644
index 47f446ae3..000000000
--- a/apps/rust-sdk/examples/extract_example.rs
+++ /dev/null
@@ -1,237 +0,0 @@
-use firecrawl::{extract::ExtractParams, FirecrawlApp};
-use serde_json::json;
-use std::error::Error;
-
-use clap::{Parser, ValueEnum};
-
-#[derive(Parser)]
-#[command(author, version, about, long_about = None)]
-struct Args {
- #[arg(value_enum)]
- command: Examples,
-}
-
-#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
-enum Examples {
- Basic,
- Schema,
- JsonSchema,
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- let args = Args::parse();
-
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
- let urls = vec![
- "https://www.firecrawl.dev/".to_string(),
- "https://betteruptime.com".to_string(),
- ];
-
- match args.command {
- Examples::Basic => {
- println!("Example 1: Extracting with URLs and prompt");
-
- let extract_params = ExtractParams {
- prompt: Some(
- "Extract Product promise, consice descirption and category".to_string(),
- ),
- url_trace: Some(true),
- ..Default::default()
- };
-
- println!("Starting asynchronous extraction job...");
- let response = firecrawl
- .async_extract(ExtractParams {
- urls: Some(urls.iter().map(|u| u.to_string()).collect()),
- prompt: extract_params.prompt.clone(),
- url_trace: extract_params.url_trace,
- ..Default::default()
- })
- .await?;
-
- println!("Extract job initiated:");
- println!(" Job ID: {}", response.id);
-
- println!("\nChecking extract status...");
- for _ in 0..5 {
- let response = firecrawl.get_extract_status(&response.id).await?;
-
- println!("Extract status: {}", response.status);
- if let Some(url_trace) = &response.url_trace {
- println!("URL traces:");
- for trace in url_trace {
- println!(" URL: {}", trace.url);
- println!(" Status: {}", trace.status);
- }
- }
- println!("Extract data: {:#?}", response.data);
- if response.status == "completed" {
- break;
- }
-
- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
- }
- }
- Examples::Schema => {
- println!("Example 2: Extracting with schema");
-
- let schema = json!({
- "type": "object",
- "properties": {
- "category": { "type": "string" },
- "promise": { "type": "string" },
- "descirption": { "type": "string" }
- },
- "required": ["category", "promise", "description"]
- });
-
- println!("Starting synchronous extraction job...");
-
- match firecrawl
- .extract(ExtractParams {
- urls: urls.into(),
- schema: Some(schema),
- ..Default::default()
- })
- .await
- {
- Ok(result) => {
- println!("Extraction completed successfully!");
- println!("Status: {}", result.status);
-
- if let Some(data) = result.data {
- println!("\nExtracted data:");
- println!(" Title: {}", data["title"]);
- if let Some(desc) = data.get("description") {
- println!(" Description: {}", desc);
- }
- println!(
- " Content (preview): {:.100}...",
- data["content"].as_str().unwrap_or("N/A")
- );
- }
-
- if let Some(sources) = result.sources {
- println!("\nSources:");
- for (field, urls) in sources {
- println!(" {}: {}", field, urls.join(", "));
- }
- }
- }
- Err(e) => {
- println!("Extraction failed: {}", e);
- }
- }
- }
- Examples::JsonSchema => {
- println!("Example 3: Using JsonSchema derive");
-
- /// A comprehensive analysis of given product
- #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
- struct ProductAnalysis {
- /// The full name of the product
- product_name: String,
- /// The company/brand behind the product
- brand: String,
- /// The general price range (e.g. "Premium", "$10-50", "Enterprise")
- price_range: String,
- /// The main customer segments this product targets
- target_audience: Vec<String>,
- /// Primary benefits and value propositions of the product
- key_benefits: Vec<String>,
- /// Distinctive features that set this product apart from competitors
- unique_selling_points: Vec<String>,
- /// Direct comparisons with competing products/services
- competitor_comparison: Vec<String>,
- /// Technologies, frameworks, or platforms used (if applicable)
- tech_stack: Option<Vec<String>>,
- /// Aggregated review data and sentiment analysis
- reviews_summary: ReviewsSummary,
- // /// Score from 0-10 indicating product-market fit based on analysis
- // market_fit_score: f32, // NOTE: Breaks
- /// Assessment of future growth prospects (e.g. "High", "Moderate", "Limited")
- growth_potential: String,
- /// Relevant compliance standards and certifications
- regulatory_compliance: Option<Vec<String>>,
- }
-
- /// Aggregated analysis of product reviews from multiple sources
- #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
- struct ReviewsSummary {
- /// Overall sentiment from review analysis (e.g. "Highly Positive", "Mixed", "Negative")
- sentiment_analysis: String,
- /// Most frequently mentioned positive aspects
- common_praises: Vec<String>,
- /// Most frequently mentioned criticisms or issues
- common_complaints: Vec<String>,
- /// Platforms or websites where reviews were sourced from
- review_sources: Vec<String>,
- }
- println!("Starting extraction with derived schema...");
- match firecrawl
- .extract_with_schemars::<ProductAnalysis>(ExtractParams {
- urls: urls.into(),
- ..Default::default()
- })
- .await
- {
- Ok(result) => {
- println!("Extraction completed!");
- println!("Status: {}", result.status);
-
- if let Some(data) = result.data {
- if let Ok(analysis) = serde_json::from_value::<ProductAnalysis>(data) {
- println!("\nExtracted Product Analysis:");
- println!(" Product: {}", analysis.product_name);
- println!(" Brand: {}", analysis.brand);
- println!(" Price Range: {}", analysis.price_range);
- println!(" Target Audience:");
- for audience in analysis.target_audience {
- println!(" - {}", audience);
- }
- println!(" Key Benefits:");
- for benefit in analysis.key_benefits {
- println!(" - {}", benefit);
- }
- println!(" USPs:");
- for usp in analysis.unique_selling_points {
- println!(" - {}", usp);
- }
-
- println!("\n Reviews Summary:");
- println!(
- " Sentiment: {}",
- analysis.reviews_summary.sentiment_analysis
- );
- println!(" Common Praises:");
- for praise in analysis.reviews_summary.common_praises {
- println!(" - {}", praise);
- }
- println!(" Common Complaints:");
- for complaint in analysis.reviews_summary.common_complaints {
- println!(" - {}", complaint);
- }
- } else {
- println!("Failed to parse extracted data");
- }
- }
-
- if let Some(sources) = result.sources {
- println!("\nSources:");
- for (field, urls) in sources {
- println!(" {}: {}", field, urls.join(", "));
- }
- }
- }
- Err(e) => {
- println!("Extraction failed: {}", e);
- }
- }
- }
- }
-
- Ok(())
-}
diff --git a/apps/rust-sdk/examples/llmstxt_example.rs b/apps/rust-sdk/examples/llmstxt_example.rs
deleted file mode 100644
index 64fb33179..000000000
--- a/apps/rust-sdk/examples/llmstxt_example.rs
+++ /dev/null
@@ -1,173 +0,0 @@
-#![allow(clippy::option_map_unit_fn)]
-use bat::{Input, PrettyPrinter};
-use firecrawl::{llmstxt::GenerateLLMsTextParams, FirecrawlApp};
-use std::error::Error;
-
-use clap::{Parser, ValueEnum};
-
-#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
-enum Mode {
- Basic,
- Pool,
- Fulltext,
-}
-
-#[derive(Parser)]
-#[command(author, version, about, long_about = None)]
-struct Args {
- /// URL for which to generate LLMs.txt
- #[arg(default_value = "https://www.firecrawl.dev/")]
- url: String,
-
- #[arg(long, short = 'm', value_enum, default_value = "Mode::Basic")]
- mode: Mode,
-
- /// Maximum number of URLs to process
- #[arg(long, short = 'd', default_value = "1")]
- max_urls: u32,
-
- /// Whether to show the full LLMs-full.txt in the response
- #[arg(long, short = 'f', default_value = "false")]
- full_text: bool,
-
- /// Experimental streaming option
- #[arg(long, short = 's', default_value = "false")]
- stream: bool,
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- let args = Args::parse();
-
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
-
- let params = GenerateLLMsTextParams {
- url: args.url.clone(),
- max_urls: args.max_urls,
- show_full_text: args.full_text,
- experimental_stream: args.stream,
- };
-
- match args.mode {
- Mode::Basic => {
- println!("Example 1: Basic LLMs.txt generation (synchronous)");
- println!("Generating LLMs.txt for {}...", args.url);
- firecrawl
- .generate_llms_text(params)
- .await
- .inspect(|result| {
- println!("Expires at: {}", result.expires_at);
- let text = (if args.full_text {
- result.data.full.as_ref()
- } else {
- result.data.compact.as_ref()
- })
- .expect("LLM Text");
-
- pretty_print_content("Firecrawl Result", text).expect("Print");
- })?;
- }
- Mode::Pool => {
- println!("Example 2: Asynchronous LLMs.txt generation with manual polling");
-
- println!("Starting asynchronous LLMs.txt generation job...");
- let response = firecrawl.async_generate_llms_text(params).await?;
-
- println!("LLMs.txt generation job initiated:");
- println!(" Job ID: {}", response.id);
- println!("\nManually polling for status...");
- for _ in 0..10 {
- let status = firecrawl
- .check_generate_llms_text_status(&response.id)
- .await?;
-
- match status.status.as_str() {
- "completed" => {
- println!("LLMs.txt generation completed!");
- let text = (if args.full_text {
- status.data.full.as_ref()
- } else {
- status.data.compact.as_ref()
- })
- .expect("LLM Text");
-
- pretty_print_content("Pool Result", text).expect("Print");
-
- break;
- }
- "failed" => {
- println!(
- "LLMs.txt generation failed: {}",
- status.error.unwrap_or_default()
- );
- break;
- }
- status => println!("Generation status: {}", status),
- }
-
- println!("Waiting 2 seconds before checking again...");
- tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
- }
- }
- Mode::Fulltext => {
- println!("Example 3: LLMs.txt generation with full text");
-
- println!("Generating LLMs.txt with full text...");
- match firecrawl.generate_llms_text(params).await {
- Ok(result) => {
- println!("LLMs.txt generation completed successfully!");
- let llmstxt = result.data.compact.expect("LLMs Text Expected");
- let fulltxt = result.data.full.expect("Full LLMs Text Expected");
-
- pretty_print_contents(&[
- ("LLMs.txt (compact)", llmstxt),
- ("LLMs.txt (full text)", fulltxt),
- ])
- .expect("Print")
- }
- Err(e) => {
- println!("LLMs.txt generation failed: {}", e);
- }
- }
- }
- }
-
- Ok(())
-}
-
-/// Pretty prints the provided content with syntax highlighting
-fn pretty_print_content(title: &str, content: &str) -> Result<(), Box<dyn Error>> {
- PrettyPrinter::new()
- .header(true)
- .grid(true)
- .input(
- Input::from_bytes(content.as_bytes())
- .title(title)
- .name("file.md"),
- )
- .print()?;
-
- Ok(())
-}
-
-/// Pretty prints multiple contents with syntax highlighting
-fn pretty_print_contents(title_contents: &[(&'static str, String)]) -> Result<(), Box<dyn Error>> {
- let mut inputs = Vec::new();
- for (title, content) in title_contents {
- inputs.push(
- Input::from_bytes(content.as_bytes())
- .title(*title)
- .name("file.md"),
- );
- }
-
- PrettyPrinter::new()
- .header(true)
- .grid(true)
- .inputs(inputs)
- .print()?;
-
- Ok(())
-}
diff --git a/apps/rust-sdk/examples/search_example.rs b/apps/rust-sdk/examples/search_example.rs
deleted file mode 100644
index 28437193a..000000000
--- a/apps/rust-sdk/examples/search_example.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-use clap::{Parser, ValueEnum};
-use firecrawl::{
- search::{SearchParams, SearchResponse},
- FirecrawlApp,
-};
-use std::error::Error;
-
-#[derive(Debug, Parser)]
-#[command(author, version, about, long_about = None)]
-struct Args {
- /// Which example to run
- #[arg(value_enum, default_value_t = Examples::All)]
- example: Examples,
-}
-
-#[derive(Debug, Clone, ValueEnum)]
-enum Examples {
- All,
- Basic,
- Advanced,
- Geo,
- Temporal,
- Social,
- News,
- Academic,
- Commercial,
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
- let args = Args::parse();
-
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
-
- match args.example {
- Examples::All => {
- run_basic_example(&firecrawl).await?;
- run_advanced_example(&firecrawl).await?;
- run_geographic_example(&firecrawl).await?;
- run_temporal_example(&firecrawl).await?;
- run_social_example(&firecrawl).await?;
- run_news_example(&firecrawl).await?;
- run_academic_example(&firecrawl).await?;
- run_commercial_example(&firecrawl).await?;
- }
- Examples::Basic => run_basic_example(&firecrawl).await?,
- Examples::Advanced => run_advanced_example(&firecrawl).await?,
- Examples::Geo => run_geographic_example(&firecrawl).await?,
- Examples::Temporal => run_temporal_example(&firecrawl).await?,
- Examples::Social => run_social_example(&firecrawl).await?,
- Examples::News => run_news_example(&firecrawl).await?,
- Examples::Academic => run_academic_example(&firecrawl).await?,
- Examples::Commercial => run_commercial_example(&firecrawl).await?,
- }
-
- Ok(())
-}
-async fn run_basic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "rust programming language";
- let results = firecrawl.search(query, None).await?;
- print_results("Basic Search", query, &results);
- Ok(())
-}
-
-async fn run_advanced_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "rust web framework site:github.com OR site:gitlab.com";
- let params = SearchParams {
- query: query.to_string(),
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Advanced Repository Search", query, &results);
- Ok(())
-}
-
-async fn run_geographic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "coworking space startup hub";
- let params = SearchParams {
- query: query.to_string(),
- // WARN: Doesn't work with searxng
- location: Some("Silicon Valley, CA".to_string()),
- // WARN: Doesn't work with searxng
- country: Some("us".to_string()),
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Geographic-Specific Search", query, &results);
- Ok(())
-}
-
-async fn run_temporal_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "artificial intelligence breakthroughs";
- let params = SearchParams {
- query: query.to_string(),
- // WARN: Doesn't work with searxng
- tbs: Some("qdr:m1".to_string()),
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Recent AI News", query, &results);
- Ok(())
-}
-
-async fn run_social_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "viral tech trends site:twitter.com";
- let params = SearchParams {
- query: query.to_string(),
- // WARN: Doesn't work. Maybe searxng related
- filter: Some("site:twitter.com OR site:linkedin.com".to_string()),
- // WARN: Doesn't work with searxng
- tbs: Some("qdr:w".to_string()), // Last week
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Social Media Tech Trends", query, &results);
- Ok(())
-}
-
-async fn run_news_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query =
- "cryptocurrency market analysis site:reuters.com OR site:bloomberg.com OR site:ft.com";
- let params = SearchParams {
- query: query.to_string(),
- // WARN: Doesn't work with searxng
- tbs: Some("qdr:d".to_string()), // Last 24 hours
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Financial News Search", query, &results);
- Ok(())
-}
-
-async fn run_academic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "quantum computing research papers site:arxiv.org OR site:scholar.google.com";
- let params = SearchParams {
- query: query.to_string(),
- // WARN: Doesn't work. Maybe searxng related
- // filter: Some("site:arxiv.org OR site:scholar.google.com".to_string()),
- // WARN: Doesn't work with searxng
- tbs: Some("qdr:y".to_string()), // Last year
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Academic Research Search", query, &results);
- Ok(())
-}
-
-async fn run_commercial_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
- let query = "enterprise cloud solutions reviews site:g2.com";
- let params = SearchParams {
- query: query.to_string(),
- limit: Some(5),
- ..Default::default()
- };
- let results = firecrawl.search_with_params(params).await?;
- print_results("Commercial Product Search", query, &results);
- Ok(())
-}
-
-fn print_results(name: &str, query: &str, results: &SearchResponse) {
- let sec = "=".repeat(70);
-
- println!("\n{sec}");
- println!("🔍 {name}");
- println!("🔎 Query: \"{query}\"");
- println!("{sec}");
-
- for (i, doc) in results.data.iter().enumerate() {
- println!("{}. 📌 Title: {}", i + 1, doc.title);
- println!(" - 🔗 URL: {}", doc.url);
- println!(" - 📝 Description: \"{:.40}\"...", doc.description);
- }
-
- if let Some(warning) = &results.warning {
- println!("\n⚠️ Warning: {warning}");
- }
- println!("{sec}\n");
-}
diff --git a/apps/rust-sdk/examples/v2_example.rs b/apps/rust-sdk/examples/v2_example.rs
deleted file mode 100644
index a5401c928..000000000
--- a/apps/rust-sdk/examples/v2_example.rs
+++ /dev/null
@@ -1,216 +0,0 @@
-//! Example usage of Firecrawl v2 API.
-//!
-//! Run with: cargo run --example v2_example
-
-use firecrawl::v2::{
- AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,
- ScrapeOptions, SearchOptions, SitemapMode,
-};
-use serde::Deserialize;
-use serde_json::json;
-
-#[tokio::main]
-async fn main() {
- // Initialize the v2 Client with the API key
- let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");
-
- // Or, connect to a self-hosted instance:
- // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)
- // .expect("Failed to initialize Client");
-
- // Example 1: Simple Scrape
- println!("=== Example 1: Simple Scrape ===");
- let result = client.scrape("https://example.com", None).await;
- match result {
- Ok(doc) => println!("Markdown: {:?}", doc.markdown),
- Err(e) => eprintln!("Scrape failed: {}", e),
- }
-
- // Example 2: Scrape with Options
- println!("\n=== Example 2: Scrape with Options ===");
- let options = ScrapeOptions {
- formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
- only_main_content: Some(true),
- ..Default::default()
- };
- let result = client.scrape("https://example.com", options).await;
- match result {
- Ok(doc) => {
- println!("Links: {:?}", doc.links);
- }
- Err(e) => eprintln!("Scrape with options failed: {}", e),
- }
-
- // Example 3: Scrape with JSON Schema Extraction
- println!("\n=== Example 3: JSON Schema Extraction ===");
- let schema = json!({
- "type": "object",
- "properties": {
- "title": { "type": "string" },
- "description": { "type": "string" }
- }
- });
- let result = client
- .scrape_with_schema(
- "https://example.com",
- schema,
- Some("Extract title and description"),
- )
- .await;
- match result {
- Ok(data) => println!(
- "Extracted: {}",
- serde_json::to_string_pretty(&data).unwrap()
- ),
- Err(e) => eprintln!("Schema extraction failed: {}", e),
- }
-
- // Example 4: Search
- println!("\n=== Example 4: Search ===");
- let options = SearchOptions {
- limit: Some(5),
- ..Default::default()
- };
- let result = client.search("rust programming", options).await;
- match result {
- Ok(response) => {
- if let Some(web) = response.data.web {
- for item in web {
- match item {
- firecrawl::v2::SearchResultOrDocument::WebResult(r) => {
- println!("Result: {} - {}", r.url, r.title.unwrap_or_default());
- }
- firecrawl::v2::SearchResultOrDocument::Document(d) => {
- if let Some(meta) = d.metadata {
- println!("Document: {:?}", meta.title);
- }
- }
- }
- }
- }
- }
- Err(e) => eprintln!("Search failed: {}", e),
- }
-
- // Example 5: Map a Website
- println!("\n=== Example 5: Map ===");
- let options = MapOptions {
- sitemap: Some(SitemapMode::Include),
- limit: Some(20),
- ..Default::default()
- };
- let result = client.map("https://example.com", options).await;
- match result {
- Ok(response) => {
- println!("Found {} links", response.links.len());
- for link in response.links.iter().take(5) {
- println!(" - {}", link.url);
- }
- }
- Err(e) => eprintln!("Map failed: {}", e),
- }
-
- // Example 6: Crawl a Website
- println!("\n=== Example 6: Crawl ===");
- let options = CrawlOptions {
- limit: Some(5),
- sitemap: Some(SitemapMode::Include),
- poll_interval: Some(3000),
- ..Default::default()
- };
- let result = client.crawl("https://example.com", options).await;
- match result {
- Ok(job) => {
- println!("Crawled {} pages", job.data.len());
- for doc in job.data.iter().take(3) {
- if let Some(meta) = &doc.metadata {
- println!(" - {:?}", meta.source_url);
- }
- }
- }
- Err(e) => eprintln!("Crawl failed: {}", e),
- }
-
- // Example 7: Batch Scrape
- println!("\n=== Example 7: Batch Scrape ===");
- let urls = vec![
- "https://example.com".to_string(),
- "https://example.org".to_string(),
- ];
- let options = BatchScrapeOptions {
- options: Some(ScrapeOptions {
- formats: Some(vec![Format::Markdown]),
- ..Default::default()
- }),
- poll_interval: Some(2000),
- ..Default::default()
- };
- let result = client.batch_scrape(urls, options).await;
- match result {
- Ok(job) => {
- println!("Batch scraped {} pages", job.data.len());
- }
- Err(e) => eprintln!("Batch scrape failed: {}", e),
- }
-
- // Example 8: Agent (Autonomous Web Browsing)
- println!("\n=== Example 8: Agent ===");
- let options = AgentOptions {
- urls: Some(vec!["https://example.com".to_string()]),
- prompt: "Describe what this website is about and list any key features mentioned"
- .to_string(),
- model: Some(AgentModel::Spark1Pro),
- timeout: Some(60),
- ..Default::default()
- };
- let result = client.agent(options).await;
- match result {
- Ok(response) => {
- println!("Agent status: {:?}", response.status);
- if let Some(data) = response.data {
- println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());
- }
- }
- Err(e) => eprintln!("Agent failed: {}", e),
- }
-
- // Example 9: Agent with Typed Schema
- println!("\n=== Example 9: Agent with Typed Schema ===");
-
- #[derive(Debug, Deserialize)]
- struct CompanyInfo {
- name: String,
- description: Option<String>,
- industry: Option<String>,
- }
-
- let schema = json!({
- "type": "object",
- "properties": {
- "name": { "type": "string" },
- "description": { "type": "string" },
- "industry": { "type": "string" }
- },
- "required": ["name"]
- });
-
- let result: Result<Option<CompanyInfo>, _> = client
- .agent_with_schema(
- vec!["https://firecrawl.dev".to_string()],
- "Extract company information from this website",
- schema,
- )
- .await;
-
- match result {
- Ok(Some(info)) => {
- println!("Company: {}", info.name);
- println!("Description: {:?}", info.description);
- println!("Industry: {:?}", info.industry);
- }
- Ok(None) => println!("No data extracted"),
- Err(e) => eprintln!("Agent with schema failed: {}", e),
- }
-
- println!("\n=== All examples completed ===");
-}
diff --git a/apps/rust-sdk/src/v2/agent.rs b/apps/rust-sdk/src/agent.rs
similarity index 98%
rename from apps/rust-sdk/src/v2/agent.rs
rename to apps/rust-sdk/src/agent.rs
index 47b4f734f..bd4e77796 100644
--- a/apps/rust-sdk/src/v2/agent.rs
+++ b/apps/rust-sdk/src/agent.rs
@@ -6,8 +6,8 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use super::client::Client;
-use super::types::{AgentModel, AgentWebhookConfig};
+use crate::client::Client;
+use crate::types::{AgentModel, AgentWebhookConfig};
use crate::FirecrawlError;
/// Options for running an agent task.
@@ -111,7 +111,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::{Client, AgentOptions};
+ /// use firecrawl::{Client, AgentOptions};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -160,7 +160,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::Client;
+ /// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -209,7 +209,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::{Client, AgentOptions, AgentModel};
+ /// use firecrawl::{Client, AgentOptions, AgentModel};
/// use serde_json::json;
///
/// #[tokio::main]
@@ -305,7 +305,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::Client;
+ /// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -358,7 +358,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::Client;
+ /// use firecrawl::Client;
/// use serde::Deserialize;
/// use serde_json::json;
///
diff --git a/apps/rust-sdk/src/batch_scrape.rs b/apps/rust-sdk/src/batch_scrape.rs
index 1429fbea6..2b2fef680 100644
--- a/apps/rust-sdk/src/batch_scrape.rs
+++ b/apps/rust-sdk/src/batch_scrape.rs
@@ -1,243 +1,370 @@
-use std::collections::HashMap;
+//! Batch scrape endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
-use crate::{
- crawl::{CrawlErrorsResponse, CrawlStatus, CrawlStatusTypes},
- scrape::ScrapeOptions,
- FirecrawlApp, FirecrawlError, API_VERSION,
-};
+use crate::client::Client;
+use crate::scrape::ScrapeOptions;
+use crate::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};
+use crate::FirecrawlError;
+/// Options for batch scraping.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
-pub struct BatchScrapeParams {
- /// List of URLs to scrape
- pub urls: Vec<String>,
- /// Scrape options to apply to all URLs
+pub struct BatchScrapeOptions {
+ /// Scrape options to apply to all URLs.
#[serde(flatten)]
pub options: Option<ScrapeOptions>,
- /// Whether to ignore invalid URLs
- #[serde(rename = "ignoreInvalidURLs")]
- pub ignore_invalid_urls: bool,
- /// ID of an existing job to append these URLs to
+
+ /// Webhook configuration for job notifications.
+ pub webhook: Option<WebhookConfig>,
+
+ /// ID of an existing batch job to append URLs to.
pub append_to_id: Option<String>,
- /// Webhook configuration
- pub webhook: Option<WebhookOptions>,
- /// Idempotency key to send to the crawl endpoint.
+ /// Whether to ignore invalid URLs instead of failing.
+ #[serde(rename = "ignoreInvalidURLs")]
+ pub ignore_invalid_urls: Option<bool>,
+
+ /// Maximum concurrent requests.
+ pub max_concurrency: Option<u32>,
+
+ /// Enable zero data retention mode.
+ pub zero_data_retention: Option<bool>,
+
+ /// Idempotency key for the request.
#[serde(skip)]
pub idempotency_key: Option<String>,
-}
-/// Options for webhook notifications
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct WebhookOptions {
- /// URL to send webhook notifications to
- pub url: String,
- /// Custom headers to include in webhook requests
- pub headers: Option<HashMap<String, String>>,
- /// Authentication token for the webhook
- pub auth_token: Option<String>,
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Poll interval for synchronous batch scrape (milliseconds).
+ #[serde(skip)]
+ pub poll_interval: Option<u64>,
}
-impl From<&str> for WebhookOptions {
- fn from(url: &str) -> Self {
- Self {
- url: url.to_string(),
- headers: None,
- auth_token: None,
- }
- }
+/// Request body for batch scrape endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct BatchScrapeRequest {
+ urls: Vec<String>,
+ #[serde(flatten)]
+ options: BatchScrapeOptions,
}
-/// Response from initiating a batch scrape job
+/// Response from starting a batch scrape job.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeResponse {
- /// Whether the request was successful
+ /// Whether the request was successful.
pub success: bool,
- /// The ID of the batch scrape job
+ /// The batch scrape job ID.
pub id: String,
- /// URL to get the status of the batch scrape job
+ /// URL to check the batch scrape status.
pub url: String,
- /// List of URLs that were invalid and could not be processed
+ /// URLs that were invalid and ignored.
+ #[serde(rename = "invalidURLs")]
pub invalid_urls: Option<Vec<String>>,
}
-impl From<String> for WebhookOptions {
- fn from(url: String) -> Self {
- Self {
- url,
- headers: None,
- auth_token: None,
- }
- }
+/// Status of a batch scrape job.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeJob {
+ /// Current status of the batch scrape job.
+ pub status: JobStatus,
+ /// Number of URLs completed.
+ pub completed: u32,
+ /// Total number of URLs to scrape.
+ pub total: u32,
+ /// Credits used by the batch scrape.
+ pub credits_used: Option<u32>,
+ /// Expiry time of the batch data.
+ pub expires_at: Option<String>,
+ /// URL for the next page of results.
+ pub next: Option<String>,
+ /// Scraped documents.
+ pub data: Vec<Document>,
}
-impl FirecrawlApp {
- /// Initiates an asynchronous batch scrape job
- pub async fn async_batch_scrape_urls(
+impl Client {
+ /// Starts a batch scrape job asynchronously.
+ ///
+ /// Returns immediately with a job ID that can be used to check status.
+ ///
+ /// # Arguments
+ ///
+ /// * `urls` - List of URLs to scrape.
+ /// * `options` - Optional batch scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeResponse` containing the job ID.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, BatchScrapeOptions};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = vec![
+ /// "https://example.com".to_string(),
+ /// "https://example.org".to_string(),
+ /// ];
+ ///
+ /// let response = client.start_batch_scrape(urls, None).await?;
+ /// println!("Batch job started: {}", response.id);
+ ///
+ /// // Check status later
+ /// let status = client.get_batch_scrape_status(&response.id).await?;
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn start_batch_scrape(
&self,
- params: BatchScrapeParams,
+ urls: Vec<String>,
+ options: impl Into<Option<BatchScrapeOptions>>,
) -> Result<BatchScrapeResponse, FirecrawlError> {
- let headers = self.prepare_headers(params.idempotency_key.as_ref());
+ let options = options.into().unwrap_or_default();
+ let body = BatchScrapeRequest {
+ urls,
+ options: options.clone(),
+ };
+
+ let headers = self.prepare_headers(options.idempotency_key.as_ref());
let response = self
.client
- .post(format!("{}{}/batch/scrape", self.api_url, API_VERSION))
+ .post(self.url("/batch/scrape"))
.headers(headers)
- .json(&params)
+ .json(&body)
.send()
.await
- .map_err(|e| FirecrawlError::HttpError("Initiating batch scrape job".to_string(), e))?;
-
- self.handle_response(response, "initiate batch scrape job")
- .await
- }
+ .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;
- /// Initiates a batch scrape job and waits for completion
- pub async fn batch_scrape_urls(
- &self,
- params: BatchScrapeParams,
- poll_interval: Option<u64>,
- ) -> Result<CrawlStatus, FirecrawlError> {
- let poll_interval_ms = poll_interval.unwrap_or(2000);
-
- let response = self.async_batch_scrape_urls(params).await?;
-
- self.monitor_batch_job_status(&response.id, poll_interval_ms)
- .await
+ self.handle_response(response, "start batch scrape").await
}
- /// Checks the status of a batch scrape job
- pub async fn check_batch_scrape_status(
+ /// Gets the status of a batch scrape job.
+ ///
+ /// If the job is completed, this will automatically fetch all pages of results.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The batch scrape job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeJob` containing the current status and any available documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let status = client.get_batch_scrape_status("job-id").await?;
+ /// println!("Status: {:?}", status.status);
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ /// println!("Documents: {}", status.data.len());
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_batch_scrape_status(
&self,
id: impl AsRef<str>,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
let response = self
.client
- .get(format!(
- "{}{}/batch/scrape/{}",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
+ .get(self.url(&format!("/batch/scrape/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
- format!("Checking status of batch scrape {}", id.as_ref()),
+ format!("Checking batch scrape status {}", id.as_ref()),
e,
)
})?;
- let mut status: CrawlStatus = self
- .handle_response(
- response,
- format!("Checking status of batch scrape {}", id.as_ref()),
- )
+ let mut status: BatchScrapeJob = self
+ .handle_response(response, format!("batch scrape status {}", id.as_ref()))
.await?;
- if status.status == CrawlStatusTypes::Completed {
- while let Some(next) = status.next.clone() {
- let new_status = self.check_batch_scrape_status_next(next).await?;
- status.data.extend_from_slice(&new_status.data);
- status.next = new_status.next;
+ // Auto-paginate if completed
+ if status.status == JobStatus::Completed {
+ while let Some(next) = status.next.take() {
+ let next_status = self.get_batch_scrape_status_next(&next).await?;
+ status.data.extend(next_status.data);
+ status.next = next_status.next;
}
}
Ok(status)
}
- /// Helper function to paginate through batch scrape status results
- async fn check_batch_scrape_status_next(
+ /// Fetches the next page of batch scrape results.
+ async fn get_batch_scrape_status_next(
&self,
- next: impl AsRef<str>,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ next: &str,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
let response = self
.client
- .get(next.as_ref())
+ .get(next)
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
- FirecrawlError::HttpError(
- format!("Paginating batch scrape using URL {:?}", next.as_ref()),
- e,
- )
+ FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)
})?;
- self.handle_response(
- response,
- format!("Paginating batch scrape using URL {:?}", next.as_ref()),
- )
- .await
+ self.handle_response(response, "batch scrape pagination")
+ .await
}
- /// Check for errors in a batch scrape job
- pub async fn check_batch_scrape_errors(
+ /// Scrapes multiple URLs and waits for completion.
+ ///
+ /// This method starts a batch scrape and polls until it completes or fails.
+ ///
+ /// # Arguments
+ ///
+ /// * `urls` - List of URLs to scrape.
+ /// * `options` - Optional batch scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeJob` containing all scraped documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, BatchScrapeOptions, ScrapeOptions, Format};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = vec![
+ /// "https://example.com/page1".to_string(),
+ /// "https://example.com/page2".to_string(),
+ /// "https://example.com/page3".to_string(),
+ /// ];
+ ///
+ /// let options = BatchScrapeOptions {
+ /// options: Some(ScrapeOptions {
+ /// formats: Some(vec![Format::Markdown, Format::Links]),
+ /// ..Default::default()
+ /// }),
+ /// ignore_invalid_urls: Some(true),
+ /// poll_interval: Some(3000),
+ /// ..Default::default()
+ /// };
+ ///
+ /// let result = client.batch_scrape(urls, options).await?;
+ /// println!("Scraped {} pages", result.data.len());
+ ///
+ /// for doc in result.data {
+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn batch_scrape(
&self,
- id: impl AsRef<str>,
- ) -> Result<CrawlErrorsResponse, FirecrawlError> {
- let response = self
- .client
- .get(format!(
- "{}{}/batch/scrape/{}/errors",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
- .headers(self.prepare_headers(None))
- .send()
+ urls: Vec<String>,
+ options: impl Into<Option<BatchScrapeOptions>>,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let poll_interval = options.poll_interval.unwrap_or(2000);
+
+ let response = self.start_batch_scrape(urls, options).await?;
+ self.wait_for_batch_scrape(&response.id, poll_interval)
.await
- .map_err(|e| {
- FirecrawlError::HttpError(
- format!("Checking errors for batch scrape {}", id.as_ref()),
- e,
- )
- })?;
-
- self.handle_response(
- response,
- format!("Checking errors for batch scrape {}", id.as_ref()),
- )
- .await
}
- /// Helper function to poll for batch job status until completion
- async fn monitor_batch_job_status(
+ /// Waits for a batch scrape job to complete.
+ async fn wait_for_batch_scrape(
&self,
id: &str,
poll_interval: u64,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
loop {
- let status_data = self.check_batch_scrape_status(id).await?;
- match status_data.status {
- CrawlStatusTypes::Completed => {
- break Ok(status_data);
- }
- CrawlStatusTypes::Scraping => {
+ let status = self.get_batch_scrape_status(id).await?;
+
+ match status.status {
+ JobStatus::Completed => return Ok(status),
+ JobStatus::Scraping => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
- CrawlStatusTypes::Failed => {
- break Err(FirecrawlError::CrawlJobFailed(
- "Batch scrape job failed".into(),
- status_data,
+ JobStatus::Failed => {
+ return Err(FirecrawlError::JobFailed(
+ "Batch scrape job failed".to_string(),
));
}
- CrawlStatusTypes::Cancelled => {
- break Err(FirecrawlError::CrawlJobFailed(
- "Batch scrape job was cancelled".into(),
- status_data,
+ JobStatus::Cancelled => {
+ return Err(FirecrawlError::JobFailed(
+ "Batch scrape job was cancelled".to_string(),
));
}
}
}
}
+
+ /// Gets errors from a batch scrape job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The batch scrape job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlErrorsResponse` containing error details.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let errors = client.get_batch_scrape_errors("job-id").await?;
+ /// for error in errors.errors {
+ /// println!("Error on {}: {}", error.url, error.error);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_batch_scrape_errors(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "batch scrape errors").await
+ }
}
#[cfg(test)]
@@ -246,98 +373,30 @@ mod tests {
use serde_json::json;
#[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_batch_scrape() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
-
- // Start a batch scrape job
- let params = BatchScrapeParams {
- urls: vec![
- "https://example.com".to_string(),
- "https://example.org".to_string(),
- ],
- ignore_invalid_urls: true,
- ..Default::default()
- };
-
- let response = app.async_batch_scrape_urls(params).await.unwrap();
-
- assert!(response.success);
- assert!(!response.id.is_empty());
- assert!(!response.url.is_empty());
- }
-
- #[tokio::test]
- async fn test_async_batch_scrape_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock
- let mock = server
- .mock("POST", "/v1/batch/scrape")
- // Remove the match_body expectation which might be causing issues
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "batch-123",
- "url": "https://api.example.com/v1/batch/batch-123",
- "invalidUrls": []
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = BatchScrapeParams {
- urls: vec![
- "https://example.com".to_string(),
- "https://example.org".to_string(),
- ],
- ignore_invalid_urls: true,
- ..Default::default()
- };
-
- let response = app.async_batch_scrape_urls(params).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.id, "batch-123");
- assert_eq!(response.url, "https://api.example.com/v1/batch/batch-123");
- assert!(response.invalid_urls.unwrap_or_default().is_empty());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_batch_scrape_with_webhook() {
+ async fn test_start_batch_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
- .mock("POST", "/v1/batch/scrape")
- // Remove the match_body expectation to simplify
+ .mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-123",
- "url": "https://api.example.com/v1/batch/batch-123"
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ];
- let params = BatchScrapeParams {
- urls: vec!["https://example.com".to_string()],
- webhook: Some("https://webhook.example.com/notify".into()),
- ..Default::default()
- };
-
- let response = app.async_batch_scrape_urls(params).await.unwrap();
+ let response = client.start_batch_scrape(urls, None).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "batch-123");
@@ -345,35 +404,27 @@ mod tests {
}
#[tokio::test]
- async fn test_check_batch_scrape_status_with_mock() {
+ async fn test_get_batch_scrape_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
- .mock("GET", "/v1/batch/scrape/batch-123")
+ .mock("GET", "/v2/batch/scrape/batch-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": true,
"status": "completed",
- "total": 2,
- "completed": 2,
- "creditsUsed": 2,
- "expiresAt": "2023-12-31T23:59:59Z",
+ "total": 3,
+ "completed": 3,
+ "creditsUsed": 3,
"data": [
{
- "metadata": {
- "sourceURL": "https://example.com",
- "statusCode": 200
- },
- "markdown": "Example Domain content"
+ "markdown": "# Page 1",
+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
},
{
- "metadata": {
- "sourceURL": "https://example.org",
- "statusCode": 200
- },
- "markdown": "Another example content"
+ "markdown": "# Page 2",
+ "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }
}
]
})
@@ -381,114 +432,129 @@ mod tests {
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let status = app.check_batch_scrape_status("batch-123").await.unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = client.get_batch_scrape_status("batch-123").await.unwrap();
- assert_eq!(status.total, 2);
- assert_eq!(status.completed, 2);
+ assert_eq!(status.status, JobStatus::Completed);
+ assert_eq!(status.total, 3);
+ assert_eq!(status.completed, 3);
assert_eq!(status.data.len(), 2);
- assert_eq!(status.data[0].metadata.source_url, "https://example.com");
- assert_eq!(status.data[1].metadata.source_url, "https://example.org");
mock.assert();
}
#[tokio::test]
- async fn test_check_batch_scrape_errors_with_mock() {
+ async fn test_batch_scrape_with_invalid_urls() {
let mut server = mockito::Server::new_async().await;
let mock = server
- .mock("GET", "/v1/batch/scrape/batch-123/errors")
+ .mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
- "errors": [
- {
- "id": "error1",
- "timestamp": "2023-01-01T00:00:00Z",
- "url": "https://invalid.example.com",
- "error": "Failed to load page"
- }
- ],
- "robotsBlocked": [
- "https://example.com/admin"
- ]
+ "id": "batch-456",
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",
+ "invalidURLs": ["not-a-url"]
})
.to_string(),
)
- .create_async()
- .await;
+ .create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let errors = app.check_batch_scrape_errors("batch-123").await.unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];
- assert_eq!(errors.errors.len(), 1);
- assert_eq!(errors.errors[0].url, "https://invalid.example.com");
- assert_eq!(errors.robots_blocked.len(), 1);
- assert_eq!(errors.robots_blocked[0], "https://example.com/admin");
+ let options = BatchScrapeOptions {
+ ignore_invalid_urls: Some(true),
+ ..Default::default()
+ };
+
+ let response = client.start_batch_scrape(urls, options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));
mock.assert();
}
#[tokio::test]
- async fn test_batch_scrape_with_invalid_urls() {
+ async fn test_batch_scrape_sync() {
let mut server = mockito::Server::new_async().await;
- let mock = server
- .mock("POST", "/v1/batch/scrape")
- // Remove the match_body expectation
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
- "id": "batch-123",
- "url": "https://api.example.com/v1/batch/batch-123",
- "invalidUrls": ["invalid-url"]
+ "id": "batch-789",
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ // Mock the status endpoint (completed immediately)
+ let status_mock = server
+ .mock("GET", "/v2/batch/scrape/batch-789")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "completed",
+ "total": 2,
+ "completed": 2,
+ "data": [
+ {
+ "markdown": "# Content",
+ "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
- let params = BatchScrapeParams {
- urls: vec!["https://example.com".to_string(), "invalid-url".to_string()],
- ignore_invalid_urls: true,
- ..Default::default()
- };
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec!["https://example.com".to_string()];
- let response = app.async_batch_scrape_urls(params).await.unwrap();
+ let result = client.batch_scrape(urls, None).await.unwrap();
- assert!(response.success);
- assert_eq!(response.id, "batch-123");
- assert_eq!(response.invalid_urls, Some(vec!["invalid-url".to_string()]));
- mock.assert();
+ assert_eq!(result.status, JobStatus::Completed);
+ assert_eq!(result.data.len(), 1);
+ start_mock.assert();
+ status_mock.assert();
}
#[tokio::test]
- async fn test_batch_scrape_error_response() {
+ async fn test_get_batch_scrape_errors() {
let mut server = mockito::Server::new_async().await;
let mock = server
- .mock("POST", "/v1/batch/scrape")
- .with_status(400)
+ .mock("GET", "/v2/batch/scrape/batch-123/errors")
+ .with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": false,
- "error": "No valid URLs provided"
+ "errors": [
+ {
+ "id": "err-1",
+ "url": "https://example.com/broken",
+ "error": "Connection timeout"
+ }
+ ],
+ "robotsBlocked": []
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();
- let params = BatchScrapeParams::default();
- let result = app.async_batch_scrape_urls(params).await;
-
- assert!(result.is_err());
+ assert_eq!(errors.errors.len(), 1);
+ assert_eq!(errors.errors[0].error, "Connection timeout");
mock.assert();
}
}
diff --git a/apps/rust-sdk/src/v2/client.rs b/apps/rust-sdk/src/client.rs
similarity index 97%
rename from apps/rust-sdk/src/v2/client.rs
rename to apps/rust-sdk/src/client.rs
index f3881d83d..67c0279ef 100644
--- a/apps/rust-sdk/src/v2/client.rs
+++ b/apps/rust-sdk/src/client.rs
@@ -17,7 +17,7 @@ const CLOUD_API_URL: &str = "https://api.firecrawl.dev";
/// # Example
///
/// ```no_run
-/// use firecrawl::v2::Client;
+/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -51,7 +51,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::Client;
+ /// use firecrawl::Client;
///
/// let client = Client::new("your-api-key").unwrap();
/// ```
@@ -73,7 +73,7 @@ impl Client {
/// # Example
///
/// ```no_run
- /// use firecrawl::v2::Client;
+ /// use firecrawl::Client;
///
/// // Self-hosted without authentication
/// let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();
@@ -164,9 +164,6 @@ impl Client {
.and_then(|response_json| {
serde_json::from_str::<Value>(&response_json)
.map_err(FirecrawlError::ResponseParseError)
- .inspect(|data| {
- tracing::debug!("Response JSON: {:#?}", data);
- })
})
.and_then(|response_value| {
// Check for success field, or allow responses without it for status checks
diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rs
index bd3b4273a..c9639b6ac 100644
--- a/apps/rust-sdk/src/crawl.rs
+++ b/apps/rust-sdk/src/crawl.rs
@@ -1,432 +1,364 @@
-use std::collections::HashMap;
+//! Crawl endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
-use crate::{
- document::Document,
- scrape::{ScrapeFormats, ScrapeOptions},
- FirecrawlApp, FirecrawlError, API_VERSION,
-};
-
-#[derive(Deserialize, Serialize, Clone, Copy, Debug)]
-pub enum CrawlScrapeFormats {
- /// Will result in a copy of the Markdown content of the page.
- #[serde(rename = "markdown")]
- Markdown,
-
- /// Will result in a copy of the filtered, content-only HTML of the page.
- #[serde(rename = "html")]
- HTML,
-
- /// Will result in a copy of the raw HTML of the page.
- #[serde(rename = "rawHtml")]
- RawHTML,
-
- /// Will result in a Vec of URLs found on the page.
- #[serde(rename = "links")]
- Links,
-
- /// Will result in a URL to a screenshot of the page.
- ///
- /// Can not be used in conjunction with `CrawlScrapeFormats::ScreenshotFullPage`.
- #[serde(rename = "screenshot")]
- Screenshot,
-
- /// Will result in a URL to a full-page screenshot of the page.
- ///
- /// Can not be used in conjunction with `CrawlScrapeFormats::Screenshot`.
- #[serde(rename = "screenshot@fullPage")]
- ScreenshotFullPage,
-}
-
-impl From<CrawlScrapeFormats> for ScrapeFormats {
- fn from(value: CrawlScrapeFormats) -> Self {
- match value {
- CrawlScrapeFormats::Markdown => Self::Markdown,
- CrawlScrapeFormats::HTML => Self::HTML,
- CrawlScrapeFormats::RawHTML => Self::RawHTML,
- CrawlScrapeFormats::Links => Self::Links,
- CrawlScrapeFormats::Screenshot => Self::Screenshot,
- CrawlScrapeFormats::ScreenshotFullPage => Self::ScreenshotFullPage,
- }
- }
-}
+use crate::client::Client;
+use crate::scrape::ScrapeOptions;
+use crate::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};
+use crate::FirecrawlError;
+/// Options for crawling a website.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
-pub struct CrawlScrapeOptions {
- /// Formats to extract from the page. (default: `[ Markdown ]`)
- pub formats: Option<Vec<CrawlScrapeFormats>>,
-
- /// Only extract the main content of the page, excluding navigation and other miscellaneous content. (default: `true`)
- pub only_main_content: Option<bool>,
-
- /// HTML tags to exclusively include.
- ///
- /// For example, if you pass `div`, you will only get content from `<div>`s and their children.
- pub include_tags: Option<Vec<String>>,
-
- /// HTML tags to exclude.
- ///
- /// For example, if you pass `img`, you will never get image URLs in your results.
- pub exclude_tags: Option<Vec<String>>,
-
- /// Additional HTTP headers to use when loading the page.
- pub headers: Option<HashMap<String, String>>,
-
- // Amount of time to wait after loading the page, and before grabbing the content, in milliseconds. (default: `0`)
- pub wait_for: Option<u32>,
-
- // Timeout before returning an error, in milliseconds. (default: `60000`)
- pub timeout: Option<u32>,
-}
-
-impl From<CrawlScrapeOptions> for ScrapeOptions {
- fn from(value: CrawlScrapeOptions) -> Self {
- ScrapeOptions {
- formats: value
- .formats
- .map(|formats| formats.into_iter().map(|x| x.into()).collect()),
- only_main_content: value.only_main_content,
- include_tags: value.include_tags,
- exclude_tags: value.exclude_tags,
- headers: value.headers,
- wait_for: value.wait_for,
- timeout: value.timeout,
- ..Default::default()
- }
- }
-}
-
-/// Options for webhook notifications
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct WebhookOptions {
- /// URL to send webhook notifications to
- pub url: String,
-
- /// Custom headers to include in webhook requests
- pub headers: Option<HashMap<String, String>>,
-
- /// Custom data included in all webhook payloads
- pub metadata: Option<HashMap<String, String>>,
+pub struct CrawlOptions {
+ /// Natural language prompt to guide crawl behavior.
+ pub prompt: Option<String>,
- /// Event types to receive
- pub events: Option<Vec<WebhookEvent>>,
-}
+ /// URL path patterns to exclude from crawling.
+ pub exclude_paths: Option<Vec<String>>,
-impl From<String> for WebhookOptions {
- fn from(value: String) -> Self {
- Self {
- url: value,
- ..Default::default()
- }
- }
-}
+ /// URL path patterns to include in crawling.
+ pub include_paths: Option<Vec<String>>,
-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
-#[serde(rename_all = "camelCase")]
-pub enum WebhookEvent {
- /// Crawl finished successfully
- Completed,
+ /// Maximum depth of links to follow from the initial URL.
+ pub max_discovery_depth: Option<u32>,
- /// Crawl encountered an error
- Failed,
+ /// How to handle the sitemap.
+ pub sitemap: Option<SitemapMode>,
- /// Individual page scraped
- Page,
+ /// Ignore query parameters when deduplicating URLs.
+ pub ignore_query_parameters: Option<bool>,
- /// Crawl job initiated
- Started,
-}
+ /// Maximum number of pages to crawl.
+ pub limit: Option<u32>,
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlOptions {
- /// Options to pass through to the scraper.
- pub scrape_options: Option<CrawlScrapeOptions>,
+ /// Crawl the entire domain regardless of path structure.
+ pub crawl_entire_domain: Option<bool>,
- /// URL RegEx patterns to (exclusively) include.
- ///
- /// For example, if you specified `"blog"`, only pages that have `blog` somewhere in the URL would be crawled.
- pub include_paths: Option<Vec<String>>,
+ /// Allow following links to external domains.
+ pub allow_external_links: Option<bool>,
- /// URL RegEx patterns to exclude.
- ///
- /// For example, if you specified `"blog"`, pages that have `blog` somewhere in the URL would not be crawled.
- pub exclude_paths: Option<Vec<String>>,
+ /// Allow following links to subdomains.
+ pub allow_subdomains: Option<bool>,
- /// Maximum URL depth to crawl, relative to the base URL. (default: `2`)
- pub max_depth: Option<u32>,
+ /// Delay between requests in seconds.
+ pub delay: Option<u32>,
- /// Tells the crawler to ignore the sitemap when crawling. (default: `true`)
- pub ignore_sitemap: Option<bool>,
+ /// Maximum concurrent requests.
+ pub max_concurrency: Option<u32>,
- /// Maximum number of pages to crawl. (default: `10`)
- pub limit: Option<u32>,
+ /// Webhook configuration for job notifications.
+ pub webhook: Option<WebhookConfig>,
- /// Allows the crawler to navigate links that are backwards in the URL hierarchy. (default: `false`)
- pub allow_backward_links: Option<bool>,
+ /// Scrape options to apply to each page.
+ pub scrape_options: Option<ScrapeOptions>,
- /// Allows the crawler to follow links to external URLs. (default: `false`)
- pub allow_external_links: Option<bool>,
+ /// Enable zero data retention mode.
+ pub zero_data_retention: Option<bool>,
- /// URL to send Webhook crawl events to.
- pub webhook: Option<WebhookOptions>,
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
- /// Idempotency key to send to the crawl endpoint.
+ /// Idempotency key for the request.
#[serde(skip)]
pub idempotency_key: Option<String>,
- pub delay: Option<u32>,
-
- /// When using `FirecrawlApp::crawl_url`, this is how often the status of the job should be checked, in milliseconds. (default: `2000`)
+ /// Poll interval for synchronous crawl (milliseconds).
#[serde(skip)]
pub poll_interval: Option<u64>,
}
+/// Request body for crawl endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
-struct CrawlRequestBody {
+struct CrawlRequest {
url: String,
-
#[serde(flatten)]
options: CrawlOptions,
}
-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
+/// Response from starting a crawl job.
+#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
-pub enum CrawlStatusTypes {
- /// The crawl job is in progress.
- Scraping,
-
- /// The crawl job has been completed successfully.
- Completed,
-
- /// The crawl job has failed.
- Failed,
-
- /// The crawl job has been cancelled.
- Cancelled,
+pub struct CrawlResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// The crawl job ID.
+ pub id: String,
+ /// URL to check the crawl status.
+ pub url: String,
}
+/// Status of a crawl job.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
-pub struct CrawlStatus {
- /// The status of the crawl.
- pub status: CrawlStatusTypes,
-
- /// Number of pages that will be scraped in total. This number may grow as the crawler discovers new pages.
+pub struct CrawlJob {
+ /// Current status of the crawl job.
+ pub status: JobStatus,
+ /// Total number of pages to crawl.
pub total: u32,
-
- /// Number of pages that have been successfully scraped.
+ /// Number of pages completed.
pub completed: u32,
-
- /// Amount of credits used by the crawl job.
- pub credits_used: u32,
-
- /// Expiry time of crawl data. After this date, the crawl data will be unavailable from the API.
- pub expires_at: String, // TODO: parse into date
-
- /// URL to call to get the next batch of documents.
- /// Unless you are sidestepping the SDK, you do not need to deal with this.
+ /// Credits used by the crawl.
+ pub credits_used: Option<u32>,
+ /// Expiry time of the crawl data.
+ pub expires_at: Option<String>,
+ /// URL for the next page of results.
pub next: Option<String>,
-
- /// List of documents returned by the crawl
+ /// Crawled documents.
pub data: Vec<Document>,
}
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlError {
- pub id: String,
- pub timestamp: Option<String>,
- pub url: String,
- pub error: String,
-}
-
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlErrorsResponse {
- pub errors: Vec<CrawlError>,
- #[serde(rename = "robotsBlocked")]
- pub robots_blocked: Vec<String>,
-}
-
+/// Response from canceling a crawl.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CancelCrawlResponse {
+ /// Status of the cancellation.
pub status: String,
}
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlAsyncResponse {
- success: bool,
-
- /// Crawl ID
- pub id: String,
-
- /// URL to get the status of the crawl job
- pub url: String,
-}
-
-impl FirecrawlApp {
- /// Initiates a crawl job for a URL using the Firecrawl API.
- pub async fn crawl_url_async(
+impl Client {
+ /// Starts a crawl job asynchronously.
+ ///
+ /// Returns immediately with a job ID that can be used to check status.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to start crawling from.
+ /// * `options` - Optional crawl configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlResponse` containing the job ID.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, CrawlOptions};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let response = client.start_crawl("https://example.com", None).await?;
+ /// println!("Crawl job started: {}", response.id);
+ ///
+ /// // Check status later
+ /// let status = client.get_crawl_status(&response.id).await?;
+ /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn start_crawl(
&self,
url: impl AsRef<str>,
- options: Option<CrawlOptions>,
- ) -> Result<CrawlAsyncResponse, FirecrawlError> {
- let body = CrawlRequestBody {
+ options: impl Into<Option<CrawlOptions>>,
+ ) -> Result<CrawlResponse, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let body = CrawlRequest {
url: url.as_ref().to_string(),
- options: options.unwrap_or_default(),
+ options: options.clone(),
};
- let headers = self.prepare_headers(body.options.idempotency_key.as_ref());
+ let headers = self.prepare_headers(options.idempotency_key.as_ref());
let response = self
.client
- .post(format!("{}{}/crawl", self.api_url, API_VERSION))
- .headers(headers.clone())
+ .post(self.url("/crawl"))
+ .headers(headers)
.json(&body)
.send()
.await
- .map_err(|e| FirecrawlError::HttpError(format!("Crawling {:?}", url.as_ref()), e))?;
-
- self.handle_response::<CrawlAsyncResponse>(response, "start crawl job")
- .await
- }
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)
+ })?;
- /// Performs a crawl job for a URL using the Firecrawl API, waiting for the end result. This may take a long time depending on the size of the target page and your options (namely `CrawlOptions.limit`).
- pub async fn crawl_url(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<CrawlOptions>>,
- ) -> Result<CrawlStatus, FirecrawlError> {
- let options = options.into();
- let poll_interval = options
- .as_ref()
- .and_then(|x| x.poll_interval)
- .unwrap_or(2000);
- let res = self.crawl_url_async(url, options).await?;
-
- self.monitor_job_status(&res.id, poll_interval).await
+ self.handle_response(response, "start crawl").await
}
- async fn check_crawl_status_next(
- &self,
- next: impl AsRef<str>,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ /// Gets the status of a crawl job.
+ ///
+ /// If the job is completed, this will automatically fetch all pages of results.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlJob` containing the current status and any available documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let status = client.get_crawl_status("job-id").await?;
+ /// println!("Status: {:?}", status.status);
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ /// println!("Documents: {}", status.data.len());
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {
let response = self
.client
- .get(next.as_ref())
+ .get(self.url(&format!("/crawl/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
- FirecrawlError::HttpError(
- format!("Paginating crawl using URL {:?}", next.as_ref()),
- e,
- )
+ FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)
})?;
- self.handle_response(
- response,
- format!("Paginating crawl using URL {:?}", next.as_ref()),
- )
- .await
+ let mut status: CrawlJob = self
+ .handle_response(response, format!("crawl status {}", id.as_ref()))
+ .await?;
+
+ // Auto-paginate if completed
+ if status.status == JobStatus::Completed {
+ while let Some(next) = status.next.take() {
+ let next_status = self.get_crawl_status_next(&next).await?;
+ status.data.extend(next_status.data);
+ status.next = next_status.next;
+ }
+ }
+
+ Ok(status)
}
- /// Checks for the status of a crawl, based on the crawl's ID. To be used in conjunction with `FirecrawlApp::crawl_url_async`.
- pub async fn check_crawl_status(
- &self,
- id: impl AsRef<str>,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ /// Fetches the next page of crawl results.
+ async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {
let response = self
.client
- .get(format!(
- "{}{}/crawl/{}",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
+ .get(next)
.headers(self.prepare_headers(None))
.send()
.await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e)
- })?;
+ .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;
- let mut status: CrawlStatus = self
- .handle_response(
- response,
- format!("Checking status of crawl {}", id.as_ref()),
- )
- .await?;
+ self.handle_response(response, "crawl pagination").await
+ }
- if status.status == CrawlStatusTypes::Completed {
- while let Some(next) = status.next {
- let new_status = self.check_crawl_status_next(next).await?;
- status.data.extend_from_slice(&new_status.data);
- status.next = new_status.next;
- }
- }
+ /// Crawls a website and waits for completion.
+ ///
+ /// This method starts a crawl and polls until it completes or fails.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to start crawling from.
+ /// * `options` - Optional crawl configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlJob` containing all crawled documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, CrawlOptions, SitemapMode};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let options = CrawlOptions {
+ /// sitemap: Some(SitemapMode::Include),
+ /// limit: Some(100),
+ /// poll_interval: Some(5000), // Check every 5 seconds
+ /// ..Default::default()
+ /// };
+ ///
+ /// let result = client.crawl("https://example.com", options).await?;
+ /// println!("Crawled {} pages", result.data.len());
+ ///
+ /// for doc in result.data {
+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn crawl(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<CrawlOptions>>,
+ ) -> Result<CrawlJob, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let poll_interval = options.poll_interval.unwrap_or(2000);
- Ok(status)
+ let response = self.start_crawl(url, options).await?;
+ self.wait_for_crawl(&response.id, poll_interval).await
}
- async fn monitor_job_status(
+ /// Waits for a crawl job to complete.
+ async fn wait_for_crawl(
&self,
id: &str,
poll_interval: u64,
- ) -> Result<CrawlStatus, FirecrawlError> {
+ ) -> Result<CrawlJob, FirecrawlError> {
loop {
- let status_data = self.check_crawl_status(id).await?;
- match status_data.status {
- CrawlStatusTypes::Completed => {
- break Ok(status_data);
- }
- CrawlStatusTypes::Scraping => {
+ let status = self.get_crawl_status(id).await?;
+
+ match status.status {
+ JobStatus::Completed => return Ok(status),
+ JobStatus::Scraping => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
- CrawlStatusTypes::Failed => {
- break Err(FirecrawlError::CrawlJobFailed(
- "Crawl job failed".into(),
- status_data,
+ JobStatus::Failed => {
+ return Err(FirecrawlError::JobFailed(
+ "Crawl job failed".to_string(),
));
}
- CrawlStatusTypes::Cancelled => {
- break Err(FirecrawlError::CrawlJobFailed(
- "Crawl job was cancelled.".into(),
- status_data,
+ JobStatus::Cancelled => {
+ return Err(FirecrawlError::JobFailed(
+ "Crawl job was cancelled".to_string(),
));
}
}
}
}
- /// Cancel an asynchronous crawl job using the Firecrawl API.
+ /// Cancels a running crawl job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID to cancel.
///
/// # Returns
///
- /// A response indicating whether the cancellation was successful, or a FirecrawlError if the request fails.
+ /// A `CancelCrawlResponse` indicating the cancellation status.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let response = client.cancel_crawl("job-id").await?;
+ /// println!("Cancellation status: {}", response.status);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
pub async fn cancel_crawl(
&self,
id: impl AsRef<str>,
) -> Result<CancelCrawlResponse, FirecrawlError> {
let response = self
.client
- .delete(format!(
- "{}{}/crawl/{}",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
+ .delete(self.url(&format!("/crawl/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
@@ -434,34 +366,51 @@ impl FirecrawlApp {
FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
})?;
- self.handle_response(response, "crawl_cancel").await
+ self.handle_response(response, "cancel crawl").await
}
- /// Returns information about crawl errors.
+ /// Gets errors from a crawl job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID.
///
/// # Returns
///
- /// A response containing information about crawl errors, or a FirecrawlError if the request fails.
- pub async fn check_crawl_errors(
+ /// A `CrawlErrorsResponse` containing error details.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let errors = client.get_crawl_errors("job-id").await?;
+ /// for error in errors.errors {
+ /// println!("Error on {}: {}", error.url, error.error);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_crawl_errors(
&self,
id: impl AsRef<str>,
) -> Result<CrawlErrorsResponse, FirecrawlError> {
let response = self
.client
- .get(format!(
- "{}{}/crawl/{}/errors",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
+ .get(self.url(&format!("/crawl/{}/errors", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
- FirecrawlError::HttpError(format!("Checking errors for crawl {}", id.as_ref()), e)
+ FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)
})?;
- self.handle_response(response, "crawl_check").await
+ self.handle_response(response, "crawl errors").await
}
}
@@ -471,164 +420,190 @@ mod tests {
use serde_json::json;
#[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_cancel_crawl() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
-
- // First start a crawl job
- let crawl_response = app
- .crawl_url_async("https://example.com", None)
- .await
- .unwrap();
-
- // Then cancel it
- let cancel_response = app.cancel_crawl(crawl_response.id).await.unwrap();
-
- assert_eq!(cancel_response.status, "cancelled");
- }
-
- #[tokio::test]
- async fn test_cancel_crawl_with_mock() {
+ async fn test_start_crawl_with_mock() {
let mut server = mockito::Server::new_async().await;
- // Set up the mock for the cancel request
let mock = server
- .mock("DELETE", "/v1/crawl/test-crawl-id")
+ .mock("POST", "/v2/crawl")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": null,
- "status": "cancelled"
+ "success": true,
+ "id": "crawl-123",
+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = app.cancel_crawl("test-crawl-id").await.unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client
+ .start_crawl("https://example.com", None)
+ .await
+ .unwrap();
- assert_eq!(response.status, "cancelled");
+ assert!(response.success);
+ assert_eq!(response.id, "crawl-123");
mock.assert();
}
#[tokio::test]
- async fn test_cancel_crawl_error_response() {
+ async fn test_get_crawl_status_with_mock() {
let mut server = mockito::Server::new_async().await;
- // Set up the mock for an error response
let mock = server
- .mock("DELETE", "/v1/crawl/invalid-id")
- .with_status(404)
+ .mock("GET", "/v2/crawl/crawl-123")
+ .with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": false,
- "error": "Crawl job not found"
+ "status": "completed",
+ "total": 5,
+ "completed": 5,
+ "creditsUsed": 5,
+ "expiresAt": "2024-12-31T23:59:59Z",
+ "data": [
+ {
+ "markdown": "# Page 1",
+ "metadata": {
+ "sourceURL": "https://example.com/page1",
+ "statusCode": 200
+ }
+ },
+ {
+ "markdown": "# Page 2",
+ "metadata": {
+ "sourceURL": "https://example.com/page2",
+ "statusCode": 200
+ }
+ }
+ ]
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = app.cancel_crawl("invalid-id").await;
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = client.get_crawl_status("crawl-123").await.unwrap();
- assert!(result.is_err());
+ assert_eq!(status.status, JobStatus::Completed);
+ assert_eq!(status.total, 5);
+ assert_eq!(status.completed, 5);
+ assert_eq!(status.data.len(), 2);
mock.assert();
}
#[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_check_crawl_errors() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
-
- // First start a crawl job
- let crawl_response = app
- .crawl_url_async("https://no-wer-agg.invalid", None)
- .await
- .unwrap();
+ async fn test_cancel_crawl_with_mock() {
+ let mut server = mockito::Server::new_async().await;
- // Check for errors
- let errors_response = app.check_crawl_errors(crawl_response.id).await.unwrap();
- println!("{errors_response:?}");
+ let mock = server
+ .mock("DELETE", "/v2/crawl/crawl-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "cancelled"
+ })
+ .to_string(),
+ )
+ .create();
- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.cancel_crawl("crawl-123").await.unwrap();
- assert!(
- !errors_response.errors.is_empty(),
- "WARN: Error returned related to Supabase not in my environment. It may fail"
- );
+ assert_eq!(response.status, "cancelled");
+ mock.assert();
}
#[tokio::test]
- async fn test_check_crawl_errors_with_mock() {
+ async fn test_get_crawl_errors_with_mock() {
let mut server = mockito::Server::new_async().await;
- // Set up the mock for the check errors request
let mock = server
- .mock("GET", "/v1/crawl/test-crawl-id/errors")
+ .mock("GET", "/v2/crawl/crawl-123/errors")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": true,
"errors": [
{
- "id": "error1",
- "timestamp": "2023-01-01T00:00:00Z",
- "url": "https://example.com/error-page",
- "error": "Failed to load page"
+ "id": "error-1",
+ "timestamp": "2024-01-01T00:00:00Z",
+ "url": "https://example.com/broken",
+ "error": "404 Not Found"
}
],
"robotsBlocked": [
- "https://example.com/blocked-by-robots"
+ "https://example.com/admin"
]
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = app.check_crawl_errors("test-crawl-id").await.unwrap();
-
- assert_eq!(response.errors.len(), 1);
- assert_eq!(response.errors[0].id, "error1");
- assert_eq!(response.errors[0].url, "https://example.com/error-page");
- assert_eq!(response.errors[0].error, "Failed to load page");
- assert_eq!(response.robots_blocked.len(), 1);
- assert_eq!(
- response.robots_blocked[0],
- "https://example.com/blocked-by-robots"
- );
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let errors = client.get_crawl_errors("crawl-123").await.unwrap();
+
+ assert_eq!(errors.errors.len(), 1);
+ assert_eq!(errors.errors[0].url, "https://example.com/broken");
+ assert_eq!(errors.robots_blocked.len(), 1);
mock.assert();
}
#[tokio::test]
- async fn test_check_crawl_errors_error_response() {
+ async fn test_crawl_with_options() {
let mut server = mockito::Server::new_async().await;
- // Set up the mock for an error response
- let mock = server
- .mock("GET", "/v1/crawl/invalid-id/errors")
- .with_status(404)
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/crawl")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "crawl-456",
+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Mock the status endpoint (completed immediately)
+ let status_mock = server
+ .mock("GET", "/v2/crawl/crawl-456")
+ .with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
- "success": false,
- "error": "Crawl job not found"
+ "status": "completed",
+ "total": 2,
+ "completed": 2,
+ "data": [
+ {
+ "markdown": "# Page 1",
+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
+ }
+ ]
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = app.check_crawl_errors("invalid-id").await;
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = CrawlOptions {
+ limit: Some(10),
+ sitemap: Some(SitemapMode::Include),
+ ..Default::default()
+ };
- assert!(result.is_err());
- mock.assert();
+ let result = client.crawl("https://example.com", options).await.unwrap();
+
+ assert_eq!(result.status, JobStatus::Completed);
+ assert_eq!(result.data.len(), 1);
+ start_mock.assert();
+ status_mock.assert();
}
}
diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rs
deleted file mode 100644
index 5dd2c1a47..000000000
--- a/apps/rust-sdk/src/document.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-
-use crate::serde_helpers::deserialize_string_or_array;
-
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct DocumentMetadata {
- // firecrawl specific
- #[serde(rename = "sourceURL")]
- pub source_url: String,
- pub status_code: u16,
- pub error: Option<String>,
-
- // basic meta tags
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub title: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub description: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub language: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub keywords: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub robots: Option<String>,
-
- // og: namespace
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_title: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_description: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_url: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_image: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_audio: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_determiner: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_locale: Option<String>,
- pub og_locale_alternate: Option<Vec<String>>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_site_name: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub og_video: Option<String>,
-
- // article: namespace
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub article_section: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub article_tag: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub published_time: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub modified_time: Option<String>,
-
- // dc./dcterms. namespace
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dcterms_keywords: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dc_description: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dc_subject: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dcterms_subject: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dcterms_audience: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dc_type: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dcterms_type: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dc_date: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dc_date_created: Option<String>,
- #[serde(default, deserialize_with = "deserialize_string_or_array")]
- pub dcterms_created: Option<String>,
-}
-
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct Document {
- /// A list of the links on the page, present if `ScrapeFormats::Markdown` is present in `ScrapeOptions.formats`. (default)
- pub markdown: Option<String>,
-
- /// The HTML of the page, present if `ScrapeFormats::HTML` is present in `ScrapeOptions.formats`.
- ///
- /// This contains HTML that has non-content tags removed. If you need the original HTML, use `ScrapeFormats::RawHTML`.
- pub html: Option<String>,
-
- /// The raw HTML of the page, present if `ScrapeFormats::RawHTML` is present in `ScrapeOptions.formats`.
- ///
- /// This contains the original, untouched HTML on the page. If you only need human-readable content, use `ScrapeFormats::HTML`.
- pub raw_html: Option<String>,
-
- /// The URL to the screenshot of the page, present if `ScrapeFormats::Screenshot` or `ScrapeFormats::ScreenshotFullPage` is present in `ScrapeOptions.formats`.
- pub screenshot: Option<String>,
-
- /// A list of the links on the page, present if `ScrapeFormats::Links` is present in `ScrapeOptions.formats`.
- pub links: Option<Vec<String>>,
-
- /// The extracted data from the page, present if `ScrapeFormats::Extract` is present in `ScrapeOptions.formats`.
- /// If `ScrapeOptions.extract.schema` is `Some`, this `Value` is guaranteed to match the provided schema.
- pub extract: Option<Value>,
-
- /// The metadata from the page.
- pub metadata: DocumentMetadata,
-
- /// Can be present if `ScrapeFormats::Extract` is present in `ScrapeOptions.formats`.
- /// The warning message will contain any errors encountered during the extraction.
- pub warning: Option<String>,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[test]
- fn test_full_document_with_array_metadata() {
- let json = json!({
- "markdown": "# Hello",
- "metadata": {
- "sourceURL": "https://example.com",
- "statusCode": 200,
- "title": "Example Page",
- "description": ["A great page", "with multiple descriptions"],
- "robots": ["index", "follow"],
- "ogImage": ["https://img.jpg"],
- "language": "en",
- "keywords": ["rust", "sdk", "firecrawl"]
- }
- });
- let doc: Document = serde_json::from_value(json).unwrap();
- assert_eq!(doc.markdown, Some("# Hello".to_string()));
- let meta = doc.metadata;
- assert_eq!(meta.title, Some("Example Page".to_string()));
- assert_eq!(
- meta.description,
- Some("A great page, with multiple descriptions".to_string())
- );
- assert_eq!(meta.robots, Some("index, follow".to_string()));
- assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));
- assert_eq!(meta.language, Some("en".to_string()));
- assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));
- }
-}
diff --git a/apps/rust-sdk/src/error.rs b/apps/rust-sdk/src/error.rs
index f452cd0e3..0f991503e 100644
--- a/apps/rust-sdk/src/error.rs
+++ b/apps/rust-sdk/src/error.rs
@@ -4,8 +4,6 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use thiserror::Error;
-use crate::crawl::CrawlStatus;
-
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct FirecrawlAPIError {
/// Always false.
@@ -40,8 +38,8 @@ pub enum FirecrawlError {
ResponseParseError(serde_json::Error),
#[error("{0} failed: {1}")]
APIError(String, FirecrawlAPIError),
- #[error("Crawl job failed: {0}")]
- CrawlJobFailed(String, CrawlStatus),
- #[error("Missuse: {0}")]
- Missuse(String),
+ #[error("Job failed: {0}")]
+ JobFailed(String),
+ #[error("Misuse: {0}")]
+ Misuse(String),
}
diff --git a/apps/rust-sdk/src/extract.rs b/apps/rust-sdk/src/extract.rs
deleted file mode 100644
index a8721ee08..000000000
--- a/apps/rust-sdk/src/extract.rs
+++ /dev/null
@@ -1,607 +0,0 @@
-use std::collections::HashMap;
-
-use schemars::schema_for;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-
-use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
-
-/// Agent options for extract requests
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct AgentOptionsExtract {
- /// Model to use for the agent
- pub model: String,
-}
-
-/// Parameters for extract requests
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ExtractParams {
- /// URLs to extract information from
- pub urls: Option<Vec<String>>,
-
- /// Extraction prompt
- pub prompt: Option<String>,
-
- /// Schema for structured output
- pub schema: Option<Value>,
-
- /// System prompt for the LLM
- pub system_prompt: Option<String>,
-
- /// Allow following external links
- pub allow_external_links: Option<bool>,
-
- /// Enable web search for additional information
- pub enable_web_search: Option<bool>,
-
- /// Show sources in the response
- pub show_sources: Option<bool>,
-
- /// Origin information, defaults to "api-sdk"
- pub origin: Option<String>,
-
- /// Timeout in milliseconds, defaults to 60000
- pub timeout: Option<u32>,
-
- /// Whether to include URL trace information, defaults to false
- pub url_trace: Option<bool>,
-
- /// Whether to ignore sitemap, defaults to false
- pub ignore_sitemap: Option<bool>,
-
- /// Whether to include subdomains, defaults to true
- pub include_subdomains: Option<bool>,
-
- /// Maximum number of URLs to process
- pub limit: Option<u32>,
-
- /// Agent options
- pub agent: Option<AgentOptionsExtract>,
-
- /// Experimental: Stream steps information
- #[serde(rename = "__experimental_streamSteps")]
- pub experimental_stream_steps: Option<bool>,
-
- /// Experimental: Include LLM usage information
- #[serde(rename = "__experimental_llmUsage")]
- pub experimental_llm_usage: Option<bool>,
-
- /// Experimental: Show sources information
- #[serde(rename = "__experimental_showSources")]
- pub experimental_show_sources: Option<bool>,
-
- /// Experimental: Cache key
- #[serde(rename = "__experimental_cacheKey")]
- pub experimental_cache_key: Option<String>,
-
- /// Experimental: Cache mode, defaults to "direct"
- #[serde(rename = "__experimental_cacheMode")]
- pub experimental_cache_mode: Option<String>,
-}
-
-/// Response from initiating an extract operation
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ExtractResponse {
- /// Whether the request was successful
- pub success: bool,
-
- /// The ID of the extract job
- pub id: String,
-
- /// URL trace information if requested
- pub url_trace: Option<Vec<URLTrace>>,
-}
-
-/// Information about URL processing during extraction
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct URLTrace {
- /// The URL being processed
- pub url: String,
-
- /// Status of processing this URL
- pub status: String,
-
- /// Timing information for URL processing
- pub timing: URLTraceTiming,
-
- /// Error message if processing failed
- pub error: Option<String>,
-
- /// Warning message if there were issues
- pub warning: Option<String>,
-
- /// Content statistics
- pub content_stats: Option<ContentStats>,
-
- /// Relevance score for this URL (0-1)
- pub relevance_score: Option<f64>,
-
- /// Whether this URL was used in the final completion
- pub used_in_completion: Option<bool>,
-
- /// Fields extracted from this URL
- pub extracted_fields: Option<Vec<String>>,
-}
-
-/// Timing information for URL processing
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct URLTraceTiming {
- /// When the URL was discovered
- pub discovered_at: String,
-
- /// When scraping began for this URL
- pub scraped_at: Option<String>,
-
- /// When processing was completed for this URL
- pub completed_at: Option<String>,
-}
-
-/// Statistics about processed content
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ContentStats {
- /// Length of the raw content in characters
- pub raw_content_length: u32,
-
- /// Length of the processed content in characters
- pub processed_content_length: u32,
-
- /// Number of tokens used for this content
- pub tokens_used: u32,
-}
-
-/// Response for extract status check
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ExtractStatusResponse {
- /// Whether the request was successful
- pub success: bool,
-
- /// Status of the extract job: "pending", "processing", "completed", "failed"
- pub status: String,
-
- /// Extracted data, present when status is "completed"
- pub data: Option<Value>,
-
- /// Error message if the job failed
- pub error: Option<String>,
-
- /// URL trace information if requested
- pub url_trace: Option<Vec<URLTrace>>,
-
- /// Sources information if requested
- pub sources: Option<HashMap<String, Vec<String>>>,
-}
-
-impl FirecrawlApp {
- /// Extracts information from URLs using the Firecrawl API.
- ///
- /// This is the synchronous version that polls until completion.
- ///
- /// Either `params.prompt` or `params.schema` must be provided.
- pub async fn extract(
- &self,
- params: impl Into<ExtractParams>,
- ) -> Result<ExtractStatusResponse, FirecrawlError> {
- let mut params = params.into();
- // Validation: Either prompt or schema must be provided
- if params.prompt.is_none() && params.schema.is_none() {
- return Err(FirecrawlError::APIError(
- "Extract validation".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: "Either prompt or schema must be provided".to_string(),
- details: None,
- },
- ));
- }
-
- // Set default origin if not provided
- if params.origin.is_none() {
- params.origin = Some("api-sdk".to_string());
- }
-
- // Initiate the extract job asynchronously
- let response = self.async_extract(params).await?;
-
- // Poll for the result
- let poll_interval = 2000; // Default to 2 seconds
- self.monitor_extract_job_status(&response.id, poll_interval)
- .await
- }
-
- pub async fn extract_with_schemars<T>(
- &self,
- params: impl Into<ExtractParams>,
- ) -> Result<ExtractStatusResponse, FirecrawlError>
- where
- T: schemars::JsonSchema,
- {
- let mut params = params.into();
- let schema = schema_for!(T);
- let schema_json = serde_json::to_value(schema).map_err(|e| {
- FirecrawlError::APIError(
- "Schema serialization".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: e.to_string(),
- details: None,
- },
- )
- })?;
- params.schema = Some(schema_json);
- self.extract(params).await
- }
-
- /// Initiates an asynchronous extract operation.
- ///
- /// # Arguments
- ///
- /// * `params` - Parameters for the extract request
- ///
- /// # Returns
- ///
- /// A response containing the extract job ID, or a FirecrawlError if the request fails.
- ///
- /// # Notes
- ///
- /// Either `params.urls` or `params.prompt` must be provided.
- /// Either `params.prompt` or `params.schema` must be provided.
- pub async fn async_extract(
- &self,
- params: impl Into<ExtractParams>,
- ) -> Result<ExtractResponse, FirecrawlError> {
- let params = params.into();
- // Validation: Either URLs or prompt must be provided
- if params.urls.is_none() && params.prompt.is_none() {
- return Err(FirecrawlError::APIError(
- "Extract validation".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: "Either URLs or prompt must be provided".to_string(),
- details: None,
- },
- ));
- }
-
- // Validation: Either prompt or schema must be provided
- if params.prompt.is_none() && params.schema.is_none() {
- return Err(FirecrawlError::APIError(
- "Extract validation".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: "Either prompt or schema must be provided".to_string(),
- details: None,
- },
- ));
- }
-
- let headers = self.prepare_headers(None);
-
- let response = self
- .client
- .post(format!("{}{}/extract", self.api_url, API_VERSION))
- .headers(headers)
- .json(&params)
- .send()
- .await
- .map_err(|e| FirecrawlError::HttpError("Initiating extract job".to_string(), e))?;
-
- self.handle_response(response, "initiate extract job").await
- }
-
- /// Checks the status of an extract job.
- ///
- /// # Arguments
- ///
- /// * `id` - The ID of the extract job
- ///
- /// # Returns
- ///
- /// A response containing the status of the extract job, or a FirecrawlError if the request fails.
- pub async fn get_extract_status(
- &self,
- id: impl AsRef<str>,
- ) -> Result<ExtractStatusResponse, FirecrawlError> {
- let response = self
- .client
- .get(format!(
- "{}{}/extract/{}",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Checking status of extract {}", id.as_ref()), e)
- })?;
-
- self.handle_response(
- response,
- format!("Checking status of extract {}", id.as_ref()),
- )
- .await
- }
-
- /// Helper function to poll for extract job status until completion
- async fn monitor_extract_job_status(
- &self,
- id: &str,
- poll_interval: u64,
- ) -> Result<ExtractStatusResponse, FirecrawlError> {
- loop {
- let status_data = self.get_extract_status(id).await?;
-
- match status_data.status.as_str() {
- "completed" => {
- break Ok(status_data);
- }
- "pending" | "processing" => {
- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
- }
- "failed" => {
- let error_msg = status_data
- .error
- .clone()
- .unwrap_or_else(|| "Extract job failed".to_string());
- break Err(FirecrawlError::APIError(
- "Extract job failed".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: error_msg,
- details: None,
- },
- ));
- }
- _ => {
- break Err(FirecrawlError::APIError(
- "Extract job status".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: format!("Unexpected status: {}", status_data.status),
- details: None,
- },
- ));
- }
- }
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_extract() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
-
- // Create extract params
- let params = ExtractParams {
- urls: Some(vec!["https://example.com".to_string()]),
- prompt: Some("Extract the title and main content from this page".to_string()),
- schema: None,
- origin: Some("test".to_string()),
- ..Default::default()
- };
-
- // Start an extract job
- let response = app.async_extract(params).await.unwrap();
-
- assert!(response.success);
- assert!(!response.id.is_empty());
- }
-
- #[tokio::test]
- async fn test_async_extract_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for the extract request
- let mock = server
- .mock("POST", "/v1/extract")
- .match_body(mockito::Matcher::PartialJson(json!({
- "urls": ["https://example.com"],
- "prompt": "Extract the title and main content"
- })))
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "extract-123",
- "urlTrace": []
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = ExtractParams {
- urls: Some(vec!["https://example.com".to_string()]),
- prompt: Some("Extract the title and main content".to_string()),
- schema: None,
- ..Default::default()
- };
-
- let response = app.async_extract(params).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.id, "extract-123");
- assert!(response.url_trace.unwrap_or_default().is_empty());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_extract_with_schema() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for the extract request with schema
- let mock = server
- .mock("POST", "/v1/extract")
- .match_body(mockito::Matcher::PartialJson(json!({
- "urls": ["https://example.com"],
- "schema": {
- "type": "object",
- "properties": {
- "title": { "type": "string" },
- "content": { "type": "string" }
- }
- }
- })))
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "extract-123"
- })
- .to_string(),
- )
- .create();
-
- // Set up the mock for the status request
- let status_mock = server
- .mock("GET", "/v1/extract/extract-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "status": "completed",
- "data": {
- "title": "Example Domain",
- "content": "This domain is for use in illustrative examples in documents."
- }
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let urls = Some(vec!["https://example.com".to_string()]);
- let params = ExtractParams {
- urls,
- schema: Some(json!({
- "type": "object",
- "properties": {
- "title": { "type": "string" },
- "content": { "type": "string" }
- }
- })),
- ..Default::default()
- };
-
- let response = app.extract(params).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.status, "completed");
-
- let data = response.data.unwrap();
- assert_eq!(data["title"], "Example Domain");
- assert_eq!(
- data["content"],
- "This domain is for use in illustrative examples in documents."
- );
-
- mock.assert();
- status_mock.assert();
- }
-
- #[tokio::test]
- async fn test_extract_status_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for the status check
- let mock = server
- .mock("GET", "/v1/extract/extract-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "status": "processing",
- "urlTrace": [
- {
- "url": "https://example.com",
- "status": "scraping",
- "timing": {
- "discoveredAt": "2023-01-01T00:00:00Z"
- }
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let status = app.get_extract_status("extract-123").await.unwrap();
-
- assert!(status.success);
- assert_eq!(status.status, "processing");
- assert_eq!(status.url_trace.unwrap()[0].url, "https://example.com");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_extract_validation_errors() {
- let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
-
- // Test missing both URLs and prompt
- let result = app.async_extract(ExtractParams::default()).await;
- assert!(result.is_err());
-
- // Test having URLs but missing both prompt and schema
- let params = ExtractParams {
- urls: Some(vec!["https://example.com".to_string()]),
- ..Default::default()
- };
- let result = app.async_extract(params).await;
- assert!(result.is_err());
- }
-
- #[tokio::test]
- async fn test_extract_api_error() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for an error response
- let mock = server
- .mock("POST", "/v1/extract")
- .with_status(400)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Invalid schema format"
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = ExtractParams {
- urls: Some(vec!["https://example.com".to_string()]),
- schema: Some(json!("invalid")), // Invalid schema format
- ..Default::default()
- };
-
- let result = app.async_extract(params).await;
- assert!(result.is_err());
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs
index 1b56c2e47..e439d2b8a 100644
--- a/apps/rust-sdk/src/lib.rs
+++ b/apps/rust-sdk/src/lib.rs
@@ -1,27 +1,12 @@
//! Firecrawl Rust SDK
//!
-//! This SDK provides access to the Firecrawl API for web scraping and crawling.
+//! This SDK provides access to the Firecrawl v2 API for web scraping, crawling,
+//! searching, mapping, batch scraping, and agent operations.
//!
//! # Quick Start
//!
-//! ## v1 API (existing)
-//!
-//! ```no_run
-//! use firecrawl::FirecrawlApp;
-//!
-//! #[tokio::main]
-//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
-//! let app = FirecrawlApp::new("your-api-key")?;
-//! let document = app.scrape_url("https://example.com", None).await?;
-//! println!("{:?}", document.markdown);
-//! Ok(())
-//! }
-//! ```
-//!
-//! ## v2 API (new)
-//!
//! ```no_run
-//! use firecrawl::v2::Client;
+//! use firecrawl::Client;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -32,130 +17,24 @@
//! }
//! ```
-use reqwest::{Client, Response};
-use serde::de::DeserializeOwned;
-use serde_json::Value;
-
-// v1 modules (existing API)
-pub mod batch_scrape;
-pub mod crawl;
-pub mod document;
pub mod error;
-pub mod extract;
-pub mod llmstxt;
-pub mod map;
-pub mod scrape;
-pub mod search;
pub(crate) mod serde_helpers;
-// v2 module (new API)
-pub mod v2;
-
-use error::FirecrawlAPIError;
+mod agent;
+mod batch_scrape;
+mod client;
+mod crawl;
+mod map;
+mod scrape;
+mod search;
+mod types;
+
+pub use agent::*;
+pub use batch_scrape::*;
+pub use client::Client;
+pub use crawl::*;
pub use error::FirecrawlError;
-
-#[derive(Clone, Debug)]
-pub struct FirecrawlApp {
- api_key: Option<String>,
- api_url: String,
- client: Client,
-}
-
-pub(crate) const API_VERSION: &str = "/v1";
-const CLOUD_API_URL: &str = "https://api.firecrawl.dev";
-
-impl FirecrawlApp {
- pub fn new(api_key: impl AsRef<str>) -> Result<Self, FirecrawlError> {
- FirecrawlApp::new_selfhosted(CLOUD_API_URL, Some(api_key))
- }
-
- pub fn new_selfhosted(
- api_url: impl AsRef<str>,
- api_key: Option<impl AsRef<str>>,
- ) -> Result<Self, FirecrawlError> {
- let url = api_url.as_ref().to_string();
-
- if url == CLOUD_API_URL && api_key.is_none() {
- return Err(FirecrawlError::APIError(
- "Configuration".to_string(),
- FirecrawlAPIError {
- success: false,
- error: "API key is required for cloud service".to_string(),
- details: None,
- },
- ));
- }
-
- Ok(FirecrawlApp {
- api_key: api_key.map(|x| x.as_ref().to_string()),
- api_url: url,
- client: Client::new(),
- })
- }
-
- fn prepare_headers(&self, idempotency_key: Option<&String>) -> reqwest::header::HeaderMap {
- let mut headers = reqwest::header::HeaderMap::new();
- headers.insert("Content-Type", "application/json".parse().unwrap());
- if let Some(api_key) = self.api_key.as_ref() {
- headers.insert(
- "Authorization",
- format!("Bearer {}", api_key).parse().unwrap(),
- );
- }
- if let Some(key) = idempotency_key {
- headers.insert("x-idempotency-key", key.parse().unwrap());
- }
- headers
- }
-
- async fn handle_response<'a, T: DeserializeOwned>(
- &self,
- response: Response,
- action: impl AsRef<str>,
- ) -> Result<T, FirecrawlError> {
- let (is_success, status) = (response.status().is_success(), response.status());
-
- let response = response
- .text()
- .await
- .map_err(|e| FirecrawlError::ResponseParseErrorText(e))
- .and_then(|response_json| {
- serde_json::from_str::<Value>(&response_json)
- .map_err(|e| FirecrawlError::ResponseParseError(e))
- .inspect(|data| {
- tracing::debug!("Response JSON: {:#?}", data);
- })
- })
- .and_then(|response_value| {
- if action.as_ref().starts_with("crawl_") // no success in check/cancel crawl responses
- || response_value["success"].as_bool().unwrap_or(false)
- {
- Ok(serde_json::from_value::<T>(response_value)
- .map_err(|e| FirecrawlError::ResponseParseError(e))?)
- } else {
- Err(FirecrawlError::APIError(
- action.as_ref().to_string(),
- serde_json::from_value(response_value)
- .map_err(|e| FirecrawlError::ResponseParseError(e))?,
- ))
- }
- });
-
- match &response {
- Ok(_) => response,
- Err(FirecrawlError::ResponseParseError(_))
- | Err(FirecrawlError::ResponseParseErrorText(_)) => {
- if is_success {
- response
- } else {
- Err(FirecrawlError::HttpRequestFailed(
- action.as_ref().to_string(),
- status.as_u16(),
- status.as_str().to_string(),
- ))
- }
- }
- Err(_) => response,
- }
- }
-}
+pub use map::*;
+pub use scrape::*;
+pub use search::*;
+pub use types::*;
diff --git a/apps/rust-sdk/src/llmstxt.rs b/apps/rust-sdk/src/llmstxt.rs
deleted file mode 100644
index f8fda7e91..000000000
--- a/apps/rust-sdk/src/llmstxt.rs
+++ /dev/null
@@ -1,426 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
-
-/// Parameters for generating LLMs.txt
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct GenerateLLMsTextParams {
- /// URL for which to generate LLMs.txt
- pub url: String,
-
- /// Maximum number of URLs to process. Default: 10
- pub max_urls: u32,
-
- /// Whether to show the full LLMs-full.txt in the response. Default: false
- pub show_full_text: bool,
-
- /// Experimental streaming option
- #[serde(rename = "__experimental_stream")]
- pub experimental_stream: bool,
-}
-
-impl Default for GenerateLLMsTextParams {
- fn default() -> Self {
- Self {
- url: String::new(),
- max_urls: 1,
- show_full_text: false,
- experimental_stream: false,
- }
- }
-}
-
-/// Response from initiating a LLMs.txt generation job
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct GenerateLLMsTextResponse {
- /// Whether the request was successful
- pub success: bool,
-
- /// Job ID for the LLMs.txt generation
- pub id: String,
-}
-
-#[derive(Deserialize, Serialize, Debug, Clone, Default)]
-pub struct LLMTextData {
- #[serde(rename = "llmstxt")]
- pub compact: Option<String>,
- #[serde(rename = "llmsfulltxt")]
- pub full: Option<String>,
-}
-
-/// Response from checking the status of a LLMs.txt generation job
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct GenerateLLMsTextStatusResponse {
- /// Whether the request was successful
- pub success: bool,
-
- /// Status of the job: "pending", "processing", "completed", "failed"
- pub status: String,
-
- /// Generated LLMs.txt data, present when status is "completed"
- #[serde(default)]
- pub data: LLMTextData,
-
- /// Error message if the job failed
- pub error: Option<String>,
-
- /// Expiration timestamp for the data
- pub expires_at: String,
-}
-
-impl FirecrawlApp {
- /// Generates LLMs.txt for a given URL and polls until completion.
- ///
- /// # Arguments
- ///
- /// * `params` - Parameters for the LLMs.txt generation
- ///
- /// # Returns
- ///
- /// A response containing the generation results, or a FirecrawlError if the request fails.
- pub async fn generate_llms_text(
- &self,
- params: impl Into<GenerateLLMsTextParams>,
- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
- // Initiate the LLMs.txt generation job asynchronously
- let response = self.async_generate_llms_text(params).await?;
-
- // Poll for the result
- let poll_interval = 2000; // Default to 2 seconds
- self.monitor_llms_text_job_status(&response.id, poll_interval)
- .await
- }
-
- /// Initiates an asynchronous LLMs.txt generation operation.
- ///
- /// # Arguments
- ///
- /// * `params` - Parameters for the LLMs.txt generation
- ///
- /// # Returns
- ///
- /// A response containing the generation job ID, or a FirecrawlError if the request fails.
- pub async fn async_generate_llms_text(
- &self,
- params: impl Into<GenerateLLMsTextParams>,
- ) -> Result<GenerateLLMsTextResponse, FirecrawlError> {
- let params = params.into();
-
- // Validation: URL must be provided
- if params.url.is_empty() {
- return Err(FirecrawlError::APIError(
- "Generate LLMs.txt validation".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: "URL must be provided".to_string(),
- details: None,
- },
- ));
- }
-
- let headers = self.prepare_headers(None);
-
- let response = self
- .client
- .post(format!("{}{}/llmstxt", self.api_url, API_VERSION))
- .headers(headers)
- .json(&params)
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError("Initiating LLMs.txt generation".to_string(), e)
- })?;
-
- self.handle_response(response, "initiate LLMs.txt generation")
- .await
- }
-
- /// Checks the status of a LLMs.txt generation operation.
- ///
- /// # Arguments
- ///
- /// * `id` - The ID of the LLMs.txt generation operation
- ///
- /// # Returns
- ///
- /// A response containing the current status and results of the generation operation,
- /// or a FirecrawlError if the request fails.
- pub async fn check_generate_llms_text_status(
- &self,
- id: impl AsRef<str>,
- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
- let response = self
- .client
- .get(format!(
- "{}{}/llmstxt/{}",
- self.api_url,
- API_VERSION,
- id.as_ref()
- ))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(
- format!("Checking status of LLMs.txt generation {}", id.as_ref()),
- e,
- )
- })?;
-
- self.handle_response(
- response,
- format!("Checking status of LLMs.txt generation {}", id.as_ref()),
- )
- .await
- }
-
- /// Helper function to poll for LLMs.txt generation job status until completion
- async fn monitor_llms_text_job_status(
- &self,
- id: &str,
- poll_interval: u64,
- ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
- loop {
- let status_data = self.check_generate_llms_text_status(id).await?;
-
- match status_data.status.as_str() {
- "completed" => {
- break Ok(status_data);
- }
- "pending" | "processing" => {
- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
- }
- "failed" => {
- let error_msg = status_data
- .error
- .clone()
- .unwrap_or_else(|| "LLMs.txt generation failed".to_string());
- break Err(FirecrawlError::APIError(
- "LLMs.txt generation failed".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: error_msg,
- details: None,
- },
- ));
- }
- _ => {
- break Err(FirecrawlError::APIError(
- "LLMs.txt generation status".to_string(),
- crate::error::FirecrawlAPIError {
- success: false,
- error: format!("Unexpected status: {}", status_data.status),
- details: None,
- },
- ));
- }
- }
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_generate_llms_text() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
-
- let params = GenerateLLMsTextParams {
- url: "https://example.com".to_string(),
- max_urls: 5,
- show_full_text: true,
- ..Default::default()
- };
-
- let response = app.async_generate_llms_text(params).await.unwrap();
-
- assert!(response.success);
- assert!(!response.id.is_empty());
- }
-
- #[tokio::test]
- async fn test_async_generate_llms_text_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v1/llmstxt")
- .match_body(mockito::Matcher::PartialJson(json!({
- "url": "https://example.com",
- "maxUrls": 5
- })))
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "llmstxt-123"
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = GenerateLLMsTextParams {
- url: "https://example.com".to_string(),
- max_urls: 5,
- ..Default::default()
- };
-
- let response = app.async_generate_llms_text(params).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.id, "llmstxt-123");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_check_generate_llms_text_status_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("GET", "/v1/llmstxt/llmstxt-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "status": "processing",
- "expiresAt": "2023-01-01T00:00:00Z"
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let status = app
- .check_generate_llms_text_status("llmstxt-123")
- .await
- .unwrap();
-
- assert!(status.success);
- assert_eq!(status.status, "processing");
- assert_eq!(status.expires_at, "2023-01-01T00:00:00Z");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_generate_llms_text_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for the generate request
- let mock = server
- .mock("POST", "/v1/llmstxt")
- .match_body(mockito::Matcher::PartialJson(json!({
- "url": "https://example.com",
- "showFullText": true
- })))
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "llmstxt-123"
- })
- .to_string(),
- )
- .create();
-
- // Set up the mock for the status request
- let status_mock = server
- .mock("GET", "/v1/llmstxt/llmstxt-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "status": "completed",
- "data": {
- "llmstxt": "Allow: /about\nDisallow: /admin\n",
- "llmsfulltxt": "# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n"
- },
- "expiresAt": "2023-01-01T00:00:00Z"
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = GenerateLLMsTextParams {
- url: "https://example.com".to_string(),
- show_full_text: true,
- ..Default::default()
- };
-
- let response = app.generate_llms_text(params).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.status, "completed");
-
- let data = response.data;
- assert_eq!(
- data.compact,
- Some("Allow: /about\nDisallow: /admin\n".into())
- );
- assert_eq!(
- data.full,
- Some("# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n".into())
- );
-
- mock.assert();
- status_mock.assert();
- }
-
- #[tokio::test]
- async fn test_generate_llms_text_validation_errors() {
- let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
-
- // Test missing URL
- let params = GenerateLLMsTextParams {
- url: "".to_string(),
- ..Default::default()
- };
- let result = app.async_generate_llms_text(params).await;
- assert!(result.is_err());
- }
-
- #[tokio::test]
- async fn test_generate_llms_text_api_error() {
- let mut server = mockito::Server::new_async().await;
-
- // Set up the mock for an error response
- let mock = server
- .mock("POST", "/v1/llmstxt")
- .with_status(400)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Invalid URL format"
- })
- .to_string(),
- )
- .create();
-
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let params = GenerateLLMsTextParams {
- url: "not-a-valid-url".to_string(),
- ..Default::default()
- };
-
- let result = app.async_generate_llms_text(params).await;
- assert!(result.is_err());
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rs
index 035785bf6..724e8bad2 100644
--- a/apps/rust-sdk/src/map.rs
+++ b/apps/rust-sdk/src/map.rs
@@ -1,63 +1,112 @@
+//! Map endpoint for Firecrawl API v2.
+
use serde::{Deserialize, Serialize};
-use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
+use crate::client::Client;
+use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};
+use crate::FirecrawlError;
+/// Options for mapping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
-pub struct Location {
- /// Country code (ISO 3166-1 alpha-2)
- pub country: Option<String>,
-
- /// List of language codes
- pub languages: Option<Vec<String>>,
-}
-
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
pub struct MapOptions {
- /// Search query to use for mapping
+ /// Search query to filter discovered links.
pub search: Option<String>,
- /// Ignore the website sitemap when crawling (default: `true`)
- pub ignore_sitemap: Option<bool>,
+ /// How to handle the sitemap.
+ pub sitemap: Option<SitemapMode>,
- /// Include subdomains of the website (default: `true`)
+ /// Include subdomains in the mapping.
pub include_subdomains: Option<bool>,
- /// Maximum number of links to return (default: `5000`)
+ /// Ignore query parameters when deduplicating URLs.
+ pub ignore_query_parameters: Option<bool>,
+
+ /// Maximum number of links to return.
pub limit: Option<u32>,
- /// Location configuration for proxy location
- pub location: Option<Location>,
+ /// Timeout in milliseconds.
+ pub timeout: Option<u32>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Location configuration for proxy routing.
+ pub location: Option<LocationConfig>,
}
+/// Request body for map endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
-struct MapRequestBody {
+struct MapRequest {
url: String,
-
#[serde(flatten)]
options: MapOptions,
}
-#[derive(Deserialize, Serialize, Debug, Default)]
+/// Response from map endpoint.
+#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
-struct MapResponse {
- success: bool,
-
- links: Vec<String>,
+pub struct MapResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// Discovered links with metadata.
+ pub links: Vec<SearchResultWeb>,
+ /// Warning message if any.
+ pub warning: Option<String>,
}
-impl FirecrawlApp {
- /// Returns links from a URL using the Firecrawl API.
- pub async fn map_url(
+impl Client {
+ /// Maps a URL to discover all associated links.
+ ///
+ /// This endpoint discovers links from a website's sitemap, page content,
+ /// and other sources without fully scraping each page.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to map.
+ /// * `options` - Optional mapping configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `MapResponse` containing the discovered links.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, MapOptions, SitemapMode};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple map
+ /// let response = client.map("https://example.com", None).await?;
+ /// println!("Found {} links", response.links.len());
+ ///
+ /// // Map with options
+ /// let options = MapOptions {
+ /// sitemap: Some(SitemapMode::Include),
+ /// include_subdomains: Some(true),
+ /// limit: Some(1000),
+ /// ..Default::default()
+ /// };
+ /// let response = client.map("https://example.com", options).await?;
+ ///
+ /// for link in response.links {
+ /// println!("URL: {}, Title: {:?}", link.url, link.title);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn map(
&self,
url: impl AsRef<str>,
options: impl Into<Option<MapOptions>>,
- ) -> Result<Vec<String>, FirecrawlError> {
- let body = MapRequestBody {
+ ) -> Result<MapResponse, FirecrawlError> {
+ let body = MapRequest {
url: url.as_ref().to_string(),
options: options.into().unwrap_or_default(),
};
@@ -66,17 +115,219 @@ impl FirecrawlApp {
let response = self
.client
- .post(&format!("{}{}/map", self.api_url, API_VERSION))
+ .post(self.url("/map"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
- let response = self
- .handle_response::<MapResponse>(response, "scrape URL")
- .await?;
+ self.handle_response(response, "map").await
+ }
+
+ /// Maps a URL and returns just the list of URLs.
+ ///
+ /// This is a convenience method that returns only the URL strings.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to map.
+ /// * `options` - Optional mapping configuration.
+ ///
+ /// # Returns
+ ///
+ /// A vector of discovered URL strings.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = client.map_urls("https://example.com", None).await?;
+ /// for url in urls {
+ /// println!("{}", url);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn map_urls(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<MapOptions>>,
+ ) -> Result<Vec<String>, FirecrawlError> {
+ let response = self.map(url, options).await?;
+ Ok(response.links.into_iter().map(|link| link.url).collect())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_map_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ {
+ "url": "https://example.com/",
+ "title": "Example Domain",
+ "description": "Home page"
+ },
+ {
+ "url": "https://example.com/about",
+ "title": "About Us"
+ },
+ {
+ "url": "https://example.com/contact"
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.map("https://example.com", None).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 3);
+ assert_eq!(response.links[0].url, "https://example.com/");
+ assert_eq!(response.links[0].title, Some("Example Domain".to_string()));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/page1" },
+ { "url": "https://example.com/page2" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = MapOptions {
+ sitemap: Some(SitemapMode::Include),
+ include_subdomains: Some(true),
+ limit: Some(100),
+ ..Default::default()
+ };
+
+ let response = client.map("https://example.com", options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_urls() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/page1" },
+ { "url": "https://example.com/page2" },
+ { "url": "https://example.com/page3" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = client.map_urls("https://example.com", None).await.unwrap();
+
+ assert_eq!(urls.len(), 3);
+ assert_eq!(urls[0], "https://example.com/page1");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_with_search_filter() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/blog/post1" },
+ { "url": "https://example.com/blog/post2" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = MapOptions {
+ search: Some("blog".to_string()),
+ ..Default::default()
+ };
+
+ let response = client.map("https://example.com", options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid URL"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.map("invalid-url", None).await;
- Ok(response.links)
+ assert!(result.is_err());
+ mock.assert();
}
}
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 5e2496fb9..2625c47bf 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -1,144 +1,244 @@
-use std::collections::HashMap;
+//! Scrape endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use std::collections::HashMap;
-use crate::{document::Document, FirecrawlApp, FirecrawlError, API_VERSION};
+use crate::client::Client;
+use crate::types::{
+ Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,
+ LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
+};
+use crate::FirecrawlError;
-#[derive(Deserialize, Serialize, Clone, Copy, Debug)]
-pub enum ScrapeFormats {
- /// Will result in a copy of the Markdown content of the page.
- #[serde(rename = "markdown")]
- Markdown,
+/// Options for scraping a URL.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeOptions {
+ /// Output formats to include in the response.
+ pub formats: Option<Vec<Format>>,
- /// Will result in a copy of the filtered, content-only HTML of the page.
- #[serde(rename = "html")]
- HTML,
+ /// Additional HTTP headers to send with the request.
+ pub headers: Option<HashMap<String, String>>,
- /// Will result in a copy of the raw HTML of the page.
- #[serde(rename = "rawHtml")]
- RawHTML,
+ /// HTML tags to exclusively include in the output.
+ pub include_tags: Option<Vec<String>>,
- /// Will result in a Vec of URLs found on the page.
- #[serde(rename = "links")]
- Links,
+ /// HTML tags to exclude from the output.
+ pub exclude_tags: Option<Vec<String>>,
- /// Will result in a URL to a screenshot of the page.
- ///
- /// Can not be used in conjunction with `ScrapeFormats::ScreenshotFullPage`.
- #[serde(rename = "screenshot")]
- Screenshot,
+ /// Only extract the main content of the page.
+ pub only_main_content: Option<bool>,
- /// Will result in a URL to a full-page screenshot of the page.
- ///
- /// Can not be used in conjunction with `ScrapeFormats::Screenshot`.
- #[serde(rename = "screenshot@fullPage")]
- ScreenshotFullPage,
+ /// Timeout in milliseconds before returning an error.
+ pub timeout: Option<u32>,
- /// Will result in the results of an LLM extraction.
- ///
- /// See `ScrapeOptions.json_options` for more options.
- #[serde(rename = "json")]
- Json,
-}
+ /// Time to wait after page load before scraping (milliseconds).
+ pub wait_for: Option<u32>,
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct AgentOptionsJson {
- pub model: String,
- pub prompt: Option<String>,
-}
+ /// Emulate a mobile device.
+ pub mobile: Option<bool>,
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct JsonOptions {
- /// Schema the output should adhere to, provided in JSON Schema format.
- pub schema: Option<Value>,
+ /// Parser configurations (e.g., for PDFs).
+ pub parsers: Option<Vec<ParserConfig>>,
- /// System prompt to send to the LLM agent along with the page content.
- pub system_prompt: Option<String>,
+ /// Browser automation actions to perform before scraping.
+ pub actions: Option<Vec<Action>>,
- /// Extraction prompt to send to the LLM agent along with the page content.
- pub prompt: Option<String>,
+ /// Location configuration for proxy routing.
+ pub location: Option<LocationConfig>,
- /// Agent options for JSON extraction.
- pub agent: Option<AgentOptionsJson>,
-}
+ /// Skip TLS certificate verification.
+ pub skip_tls_verification: Option<bool>,
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct AgentOptions {
- pub model: String,
- pub prompt: Option<String>,
- pub session_id: Option<String>,
- pub wait_before_closing_ms: Option<u32>,
-}
+ /// Remove base64-encoded images from the output.
+ pub remove_base64_images: Option<bool>,
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeOptions {
- /// Formats to extract from the page. (default: `[ Markdown ]`)
- pub formats: Option<Vec<ScrapeFormats>>,
+ /// Enable fast mode for quicker scrapes with reduced accuracy.
+ pub fast_mode: Option<bool>,
- /// Only extract the main content of the page, excluding navigation and other miscellaneous content. (default: `true`)
- pub only_main_content: Option<bool>,
+ /// Block advertisements on the page.
+ pub block_ads: Option<bool>,
- /// HTML tags to exclusively include.
- ///
- /// For example, if you pass `div`, you will only get content from `<div>`s and their children.
- pub include_tags: Option<Vec<String>>,
+ /// Proxy type to use.
+ pub proxy: Option<ProxyType>,
- /// HTML tags to exclude.
- ///
- /// For example, if you pass `img`, you will never get image URLs in your results.
- pub exclude_tags: Option<Vec<String>>,
+ /// Maximum age of cached content to accept (seconds).
+ pub max_age: Option<u32>,
- /// Additional HTTP headers to use when loading the page.
- pub headers: Option<HashMap<String, String>>,
+ /// Minimum age of cached content to accept (seconds).
+ pub min_age: Option<u32>,
- // Amount of time to wait after loading the page, and before grabbing the content, in milliseconds. (default: `0`)
- pub wait_for: Option<u32>,
+ /// Store the result in cache for future requests.
+ pub store_in_cache: Option<bool>,
- // Timeout before returning an error, in milliseconds. (default: `60000`)
- pub timeout: Option<u32>,
+ /// Persistent browser profile for maintaining state across scrapes.
+ pub profile: Option<ProfileConfig>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
- /// JSON extraction options, to be used in conjunction with `ScrapeFormats::Json`.
+ /// JSON extraction options.
pub json_options: Option<JsonOptions>,
- /// Agent options for smart scrape.
- pub agent: Option<AgentOptions>,
+ /// Screenshot options.
+ pub screenshot_options: Option<ScreenshotOptions>,
+
+ /// Change tracking options.
+ pub change_tracking_options: Option<ChangeTrackingOptions>,
+
+ /// Attribute selectors for extraction.
+ pub attribute_selectors: Option<Vec<AttributeSelector>>,
}
+/// Parser configuration for document parsing.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum ParserConfig {
+ /// Simple parser type string.
+ Simple(String),
+ /// PDF parser with options.
+ Pdf {
+ #[serde(rename = "type")]
+ parser_type: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ max_pages: Option<u32>,
+ },
+}
+
+/// Request body for scrape endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
-struct ScrapeRequestBody {
+struct ScrapeRequest {
url: String,
-
#[serde(flatten)]
options: ScrapeOptions,
}
-#[derive(Deserialize, Serialize, Debug, Default)]
+/// Response from scrape endpoint.
+#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ScrapeResponse {
- /// This will always be `true` due to `FirecrawlApp::handle_response`.
- /// No need to expose.
success: bool,
-
- /// The resulting document.
data: Document,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ warning: Option<String>,
+}
+
+/// Supported languages for scrape-bound browser execution.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ScrapeExecuteLanguage {
+ Python,
+ Node,
+ Bash,
+}
+
+/// Options for executing code or a prompt in a scrape-bound browser session.
+///
+/// At least one of `code` or `prompt` must be provided.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeExecuteOptions {
+ /// Code to execute (optional if `prompt` is provided).
+ pub code: Option<String>,
+ /// Natural-language instruction for the browser agent (optional if `code` is provided).
+ pub prompt: Option<String>,
+ /// Runtime language for the code.
+ pub language: Option<ScrapeExecuteLanguage>,
+ /// Execution timeout in seconds.
+ pub timeout: Option<u32>,
+ /// Optional origin tag for request attribution.
+ pub origin: Option<String>,
}
-impl FirecrawlApp {
- /// Scrapes a URL using the Firecrawl API.
- pub async fn scrape_url(
+/// Response from scrape-bound browser execution.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeExecuteResponse {
+ /// Whether the request succeeded.
+ pub success: bool,
+ /// Live-view URL for the browser session.
+ pub live_view_url: Option<String>,
+ /// Interactive live-view URL for the browser session.
+ pub interactive_live_view_url: Option<String>,
+ /// Agent output when a prompt was used.
+ pub output: Option<String>,
+ /// Captured stdout from execution.
+ pub stdout: Option<String>,
+ /// Optional execution result payload.
+ pub result: Option<String>,
+ /// Captured stderr from execution.
+ pub stderr: Option<String>,
+ /// Process exit code.
+ pub exit_code: Option<i32>,
+ /// Whether execution was killed by timeout or system.
+ pub killed: Option<bool>,
+ /// Error message when execution fails.
+ pub error: Option<String>,
+}
+
+/// Response from deleting a scrape-bound browser session.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeBrowserDeleteResponse {
+ /// Whether the delete request succeeded.
+ pub success: bool,
+ /// Session duration in milliseconds when available.
+ pub session_duration_ms: Option<u64>,
+ /// Credits billed when available.
+ pub credits_billed: Option<u32>,
+ /// Error message when deletion fails.
+ pub error: Option<String>,
+}
+
+impl Client {
+ /// Scrapes a URL and returns the content in the requested formats.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to scrape.
+ /// * `options` - Optional scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `Document` containing the scraped content.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::{Client, ScrapeOptions, Format};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple scrape
+ /// let document = client.scrape("https://example.com", None).await?;
+ /// println!("Markdown: {:?}", document.markdown);
+ ///
+ /// // Scrape with options
+ /// let options = ScrapeOptions {
+ /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ /// only_main_content: Some(true),
+ /// ..Default::default()
+ /// };
+ /// let document = client.scrape("https://example.com", options).await?;
+ /// println!("Links: {:?}", document.links);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn scrape(
&self,
url: impl AsRef<str>,
options: impl Into<Option<ScrapeOptions>>,
) -> Result<Document, FirecrawlError> {
- let body = ScrapeRequestBody {
+ let body = ScrapeRequest {
url: url.as_ref().to_string(),
options: options.into().unwrap_or_default(),
};
@@ -147,17 +247,485 @@ impl FirecrawlApp {
let response = self
.client
- .post(&format!("{}{}/scrape", self.api_url, API_VERSION))
+ .post(self.url("/scrape"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
- let response = self
- .handle_response::<ScrapeResponse>(response, "scrape URL")
- .await?;
+ let response: ScrapeResponse = self.handle_response(response, "scrape").await?;
Ok(response.data)
}
+
+ /// Scrapes a URL with a JSON schema for structured extraction.
+ ///
+ /// This is a convenience method that combines scraping with JSON extraction.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to scrape.
+ /// * `schema` - JSON schema for the extraction.
+ /// * `prompt` - Optional extraction prompt.
+ ///
+ /// # Returns
+ ///
+ /// The extracted JSON value matching the schema.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ /// use serde_json::json;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let schema = json!({
+ /// "type": "object",
+ /// "properties": {
+ /// "title": { "type": "string" },
+ /// "price": { "type": "number" }
+ /// }
+ /// });
+ ///
+ /// let data = client.scrape_with_schema(
+ /// "https://example.com/product",
+ /// schema,
+ /// Some("Extract the product title and price")
+ /// ).await?;
+ ///
+ /// println!("Extracted: {}", data);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn scrape_with_schema(
+ &self,
+ url: impl AsRef<str>,
+ schema: Value,
+ prompt: Option<impl AsRef<str>>,
+ ) -> Result<Value, FirecrawlError> {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Json]),
+ json_options: Some(JsonOptions {
+ schema: Some(schema),
+ prompt: prompt.map(|p| p.as_ref().to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let document = self.scrape(url, options).await?;
+ Ok(document.json.unwrap_or(Value::Null))
+ }
+
+ /// Interacts with the browser session associated with a scrape job.
+ ///
+ /// # Arguments
+ ///
+ /// * `job_id` - The scrape job ID.
+ /// * `options` - Execution options including code and runtime config.
+ ///
+ /// # Returns
+ ///
+ /// A `ScrapeExecuteResponse` containing execution output.
+ pub async fn interact(
+ &self,
+ job_id: impl AsRef<str>,
+ options: ScrapeExecuteOptions,
+ ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
+ let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
+ let has_prompt = options
+ .prompt
+ .as_ref()
+ .is_some_and(|p| !p.trim().is_empty());
+ if !has_code && !has_prompt {
+ return Err(FirecrawlError::Misuse(
+ "Either 'code' or 'prompt' must be provided".into(),
+ ));
+ }
+
+ let mut body = options;
+ if body.language.is_none() {
+ body.language = Some(ScrapeExecuteLanguage::Node);
+ }
+
+ let response = self
+ .client
+ .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Interacting with scrape browser for {}", job_id.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(response, "scrape interact").await
+ }
+
+ /// Stops the interaction session associated with a scrape job.
+ ///
+ /// # Arguments
+ ///
+ /// * `job_id` - The scrape job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `ScrapeBrowserDeleteResponse` indicating stop status.
+ pub async fn stop_interaction(
+ &self,
+ job_id: impl AsRef<str>,
+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
+ let response = self
+ .client
+ .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Stopping interaction for {}", job_id.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(response, "stop interaction").await
+ }
+
+ /// Deprecated alias for [`Client::interact`].
+ #[deprecated(note = "Use interact() instead")]
+ pub async fn scrape_execute(
+ &self,
+ job_id: impl AsRef<str>,
+ options: ScrapeExecuteOptions,
+ ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
+ self.interact(job_id, options).await
+ }
+
+ /// Deprecated alias for [`Client::stop_interaction`].
+ #[deprecated(note = "Use stop_interaction() instead")]
+ pub async fn stop_interactive_browser(
+ &self,
+ job_id: impl AsRef<str>,
+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
+ self.stop_interaction(job_id).await
+ }
+
+ /// Deprecated alias for [`Client::stop_interaction`].
+ #[deprecated(note = "Use stop_interaction() instead")]
+ pub async fn delete_scrape_browser(
+ &self,
+ job_id: impl AsRef<str>,
+ ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
+ self.stop_interaction(job_id).await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_scrape_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "markdown": "# Example Domain\n\nThis is an example.",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example Domain"
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let document = client.scrape("https://example.com", None).await.unwrap();
+
+ assert!(document.markdown.is_some());
+ assert!(document.markdown.unwrap().contains("Example Domain"));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "markdown": "# Test",
+ "html": "<h1>Test</h1>",
+ "links": ["https://example.com/page1", "https://example.com/page2"],
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ only_main_content: Some(true),
+ ..Default::default()
+ };
+
+ let document = client.scrape("https://example.com", options).await.unwrap();
+
+ assert!(document.markdown.is_some());
+ assert!(document.html.is_some());
+ assert!(document.links.is_some());
+ assert_eq!(document.links.unwrap().len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_with_schema() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "json": {
+ "title": "Product Name",
+ "price": 99.99
+ },
+ "metadata": {
+ "sourceURL": "https://example.com/product",
+ "statusCode": 200
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "price": { "type": "number" }
+ }
+ });
+
+ let data = client
+ .scrape_with_schema(
+ "https://example.com/product",
+ schema,
+ Some("Extract product info"),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(data["title"], "Product Name");
+ assert_eq!(data["price"], 99.99);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid URL"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.scrape("invalid-url", None).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_interact_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape/job-123/interact")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "stdout": "ok",
+ "result": "done",
+ "stderr": "",
+ "exitCode": 0,
+ "killed": false
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client
+ .interact(
+ "job-123",
+ ScrapeExecuteOptions {
+ code: Some("console.log('ok')".to_string()),
+ timeout: Some(30),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.exit_code, Some(0));
+ assert_eq!(response.result, Some("done".to_string()));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_interact_with_prompt() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape/job-789/interact")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "output": "Clicked the login button",
+ "liveViewUrl": "https://live.example.com/view",
+ "interactiveLiveViewUrl": "https://live.example.com/interactive",
+ "stdout": "",
+ "exitCode": 0,
+ "killed": false
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client
+ .interact(
+ "job-789",
+ ScrapeExecuteOptions {
+ prompt: Some("Click the login button".to_string()),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+
+ assert!(response.success);
+ assert_eq!(
+ response.output,
+ Some("Clicked the login button".to_string())
+ );
+ assert_eq!(
+ response.live_view_url,
+ Some("https://live.example.com/view".to_string())
+ );
+ assert_eq!(
+ response.interactive_live_view_url,
+ Some("https://live.example.com/interactive".to_string())
+ );
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_stop_interaction_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("DELETE", "/v2/scrape/job-123/interact")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "sessionDurationMs": 1200,
+ "creditsBilled": 3
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.stop_interaction("job-123").await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.session_duration_ms, Some(1200));
+ assert_eq!(response.credits_billed, Some(3));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_interact_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape/job-404/interact")
+ .with_status(404)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Job not found."
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client
+ .interact(
+ "job-404",
+ ScrapeExecuteOptions {
+ code: Some("console.log('ok')".to_string()),
+ ..Default::default()
+ },
+ )
+ .await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
}
diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs
index 397bc8d16..534559bbb 100644
--- a/apps/rust-sdk/src/search.rs
+++ b/apps/rust-sdk/src/search.rs
@@ -1,223 +1,386 @@
-use crate::{scrape::ScrapeOptions, FirecrawlApp, FirecrawlError, API_VERSION};
+//! Search endpoint for Firecrawl API v2.
+
use serde::{Deserialize, Serialize};
-#[derive(Deserialize, Serialize, Debug, Clone)]
+use crate::client::Client;
+use crate::scrape::ScrapeOptions;
+use crate::types::{
+ Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,
+};
+use crate::FirecrawlError;
+
+/// Options for search requests.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
-pub struct SearchParams {
- /// The search query string
- pub query: String,
- /// Maximum number of results to return. Default: 5, Max: 20
+pub struct SearchOptions {
+ /// Maximum number of results to return. Default: 5, Max: 20.
pub limit: Option<u32>,
- /// Time-based search filter.
- #[serde(skip_serializing_if = "Option::is_none")]
+
+ /// Search sources to query (web, news, images).
+ pub sources: Option<Vec<SearchSource>>,
+
+ /// Categories to filter results (github, research, pdf).
+ pub categories: Option<Vec<SearchCategory>>,
+
+ /// Time-based search filter (e.g., "qdr:d" for past day).
pub tbs: Option<String>,
- /// Query string to filter search results. Example: "site:example.com"
- #[serde(skip_serializing_if = "Option::is_none")]
- pub filter: Option<String>,
- /// Language code. Default: "en"
- pub lang: Option<String>,
- /// Country code. Default: "us"
- pub country: Option<String>,
- /// Geographic location string for local search results
- #[serde(skip_serializing_if = "Option::is_none")]
+
+ /// Geographic location string for local search results.
pub location: Option<String>,
- /// Origin identifier. Default: "api"
- pub origin: Option<String>,
- /// Timeout in milliseconds. Default: 60000
+
+ /// Whether to ignore invalid URLs in results.
+ pub ignore_invalid_urls: Option<bool>,
+
+ /// Timeout in milliseconds.
pub timeout: Option<u32>,
- /// Additional options for webpage scraping behavior
- #[serde(skip_serializing_if = "Option::is_none")]
+
+ /// Scrape options to apply to each search result.
pub scrape_options: Option<ScrapeOptions>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+}
+
+/// Request body for search endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct SearchRequest {
+ query: String,
+ #[serde(flatten)]
+ options: SearchOptions,
}
-impl Default for SearchParams {
- fn default() -> Self {
- Self {
- query: String::new(),
- limit: Some(5),
- tbs: None,
- filter: None,
- lang: Some("en".to_string()),
- country: Some("us".to_string()),
- location: None,
- origin: Some("api".to_string()),
- timeout: Some(60000),
- scrape_options: None,
+/// Search results data structure.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchData {
+ /// Web search results (may include scraped documents).
+ pub web: Option<Vec<SearchResultOrDocument>>,
+ /// News search results.
+ pub news: Option<Vec<SearchResultNews>>,
+ /// Image search results.
+ pub images: Option<Vec<SearchResultImage>>,
+}
+
+/// A search result that may be a simple result or a full document.
+///
+/// Uses custom deserialization to properly distinguish between web results
+/// and scraped documents by checking for document-specific fields.
+#[derive(Serialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum SearchResultOrDocument {
+ /// Simple web search result.
+ WebResult(SearchResultWeb),
+ /// Full scraped document.
+ Document(Document),
+}
+
+impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ use serde_json::Value;
+
+ let value = Value::deserialize(deserializer)?;
+
+ // Check for document-specific fields that indicate scraped content
+ // If any of these exist, it's a Document, not a simple WebResult
+ let is_document = value.get("markdown").is_some()
+ || value.get("html").is_some()
+ || value.get("rawHtml").is_some()
+ || value.get("metadata").is_some();
+
+ if is_document {
+ Document::deserialize(value)
+ .map(SearchResultOrDocument::Document)
+ .map_err(serde::de::Error::custom)
+ } else {
+ SearchResultWeb::deserialize(value)
+ .map(SearchResultOrDocument::WebResult)
+ .map_err(serde::de::Error::custom)
}
}
}
-#[derive(Deserialize, Serialize, Debug, Clone)]
+/// Response from search endpoint.
+#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct SearchResponse {
+ /// Whether the request was successful.
pub success: bool,
- pub data: Vec<SearchDocument>,
+ /// Search results data.
+ pub data: SearchData,
+ /// Warning message if any.
pub warning: Option<String>,
}
-// TODO: Consider merging fields into document::Document (url, title, description) while preserving optionality
-/// A document returned from a search or scrape request
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchDocument {
- /// Document URL
- pub url: String,
- /// Document title
- pub title: String,
- /// Document description
- pub description: String,
-}
-
-impl FirecrawlApp {
- /// Search for content using the Firecrawl API.
+impl Client {
+ /// Searches the web and optionally scrapes the results.
///
/// # Arguments
///
- /// * `query` - The search query string
- /// * `params` - Optional parameters for the search request
+ /// * `query` - The search query string.
+ /// * `options` - Optional search configuration.
///
/// # Returns
///
- /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
- pub async fn search(
- &self,
- query: impl AsRef<str>,
- params: impl Into<Option<SearchParams>>,
- ) -> Result<SearchResponse, FirecrawlError> {
- let mut search_params = params.into().unwrap_or_default();
- search_params.query = query.as_ref().to_string();
-
- self.search_with_params(search_params).await
- }
-
- /// Alternative method that takes SearchParams directly
+ /// A `SearchResponse` containing the search results.
///
- /// # Arguments
+ /// # Example
///
- /// * `params` - Search parameters including the query
+ /// ```no_run
+ /// use firecrawl::{Client, SearchOptions, SearchSource};
///
- /// # Returns
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple search
+ /// let results = client.search("rust programming", None).await?;
+ /// for result in results.data.web.unwrap_or_default() {
+ /// match result {
+ /// firecrawl::SearchResultOrDocument::WebResult(r) => {
+ /// println!("URL: {}", r.url);
+ /// }
+ /// firecrawl::SearchResultOrDocument::Document(d) => {
+ /// println!("Content: {:?}", d.markdown);
+ /// }
+ /// }
+ /// }
///
- /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
- pub async fn search_with_params(
+ /// // Search with options
+ /// let options = SearchOptions {
+ /// limit: Some(10),
+ /// sources: Some(vec![SearchSource::Web, SearchSource::News]),
+ /// ..Default::default()
+ /// };
+ /// let results = client.search("rust programming", options).await?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn search(
&self,
- params: SearchParams,
+ query: impl AsRef<str>,
+ options: impl Into<Option<SearchOptions>>,
) -> Result<SearchResponse, FirecrawlError> {
+ let body = SearchRequest {
+ query: query.as_ref().to_string(),
+ options: options.into().unwrap_or_default(),
+ };
+
let headers = self.prepare_headers(None);
let response = self
.client
- .post(format!("{}{}/search", self.api_url, API_VERSION))
+ .post(self.url("/search"))
.headers(headers)
- .json(&params)
+ .json(&body)
.send()
.await
.map_err(|e| {
- FirecrawlError::HttpError(format!("Searching with query: {:?}", params.query), e)
+ FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)
})?;
- self.handle_response::<SearchResponse>(response, "search")
- .await
+ self.handle_response(response, "search").await
+ }
+
+ /// Searches the web and scrapes the results.
+ ///
+ /// This is a convenience method that enables scraping for all results.
+ ///
+ /// # Arguments
+ ///
+ /// * `query` - The search query string.
+ /// * `limit` - Maximum number of results to return.
+ ///
+ /// # Returns
+ ///
+ /// A vector of scraped documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let documents = client.search_and_scrape("rust programming", 5).await?;
+ /// for doc in documents {
+ /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));
+ /// println!("Content: {:?}", doc.markdown);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn search_and_scrape(
+ &self,
+ query: impl AsRef<str>,
+ limit: u32,
+ ) -> Result<Vec<Document>, FirecrawlError> {
+ let options = SearchOptions {
+ limit: Some(limit),
+ scrape_options: Some(ScrapeOptions::default()),
+ ..Default::default()
+ };
+
+ let response = self.search(query, options).await?;
+
+ let documents: Vec<Document> = response
+ .data
+ .web
+ .unwrap_or_default()
+ .into_iter()
+ .filter_map(|result| match result {
+ SearchResultOrDocument::Document(doc) => Some(doc),
+ SearchResultOrDocument::WebResult(_) => None,
+ })
+ .collect();
+
+ Ok(documents)
}
}
#[cfg(test)]
-pub mod tests {
+mod tests {
use super::*;
use serde_json::json;
- #[tokio::test]
- #[ignore = "Makes real network request"]
- async fn test_real_search() {
- let api_url = std::env::var("FIRECRAWL_API_URL")
- .expect("Please set the FIRECRAWL_API_URL environment variable");
- let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
- let response = app.search("test query", None).await.unwrap();
- assert!(response.success);
- }
-
#[tokio::test]
async fn test_search_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
- .mock("POST", "/v1/search")
+ .mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
- "data": [{
- "url": "https://example.com",
- "title": "Example Domain",
- "description": "...."
- }],
+ "data": {
+ "web": [
+ {
+ "url": "https://example.com",
+ "title": "Example Domain",
+ "description": "This domain is for examples"
+ },
+ {
+ "url": "https://example.org",
+ "title": "Another Example",
+ "description": "More examples"
+ }
+ ]
+ }
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = app.search("test", None).await.unwrap();
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.search("test query", None).await.unwrap();
assert!(response.success);
- assert_eq!(response.data.len(), 1);
- assert_eq!(response.data[0].url, "https://example.com");
- assert_eq!(response.data[0].title, "Example Domain".to_string());
- assert_eq!(response.data[0].description, "....".to_string());
+ let web_results = response.data.web.unwrap();
+ assert_eq!(web_results.len(), 2);
mock.assert();
}
#[tokio::test]
- async fn test_search_with_params() {
+ async fn test_search_with_options() {
let mut server = mockito::Server::new_async().await;
+
let mock = server
- .mock("POST", "/v1/search")
- .with_header("content-type", "application/json")
- .match_body(mockito::Matcher::Json(json!({
- "query": "test",
- "limit": 10,
- "lang": "fr",
- "country": "fr",
- "origin": "api",
- "timeout": 30000
- })))
+ .mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
- "data": [],
- "warning": "No results found"
+ "data": {
+ "web": [
+ {
+ "url": "https://example.com",
+ "title": "Test Result",
+ "description": "A test result"
+ }
+ ],
+ "news": [
+ {
+ "title": "Breaking News",
+ "url": "https://news.example.com",
+ "snippet": "Something happened",
+ "date": "2024-01-01"
+ }
+ ]
+ }
})
.to_string(),
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let params = SearchParams {
- query: "test".to_string(),
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = SearchOptions {
limit: Some(10),
- lang: Some("fr".to_string()),
- country: Some("fr".to_string()),
- timeout: Some(30000),
+ sources: Some(vec![SearchSource::Web, SearchSource::News]),
..Default::default()
};
- let response = app.search_with_params(params).await.unwrap();
+ let response = client.search("test", options).await.unwrap();
assert!(response.success);
- assert_eq!(response.data.len(), 0);
- assert_eq!(response.warning, Some("No results found".to_string()));
+ assert!(response.data.web.is_some());
+ assert!(response.data.news.is_some());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_and_scrape() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ {
+ "markdown": "# Example\n\nThis is the scraped content.",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example"
+ }
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let documents = client.search_and_scrape("test", 5).await.unwrap();
+
+ assert_eq!(documents.len(), 1);
+ assert!(documents[0].markdown.is_some());
mock.assert();
}
#[tokio::test]
async fn test_search_error_response() {
let mut server = mockito::Server::new_async().await;
+
let mock = server
- .mock("POST", "/v1/search")
+ .mock("POST", "/v2/search")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
@@ -229,17 +392,75 @@ pub mod tests {
)
.create();
- let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = app.search("", None).await;
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.search("", None).await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
- async fn test_search_network_error() {
- let app = FirecrawlApp::new_selfhosted("http://invalid-url", Some("test_key")).unwrap();
- let result = app.search("test", None).await;
- assert!(result.is_err());
+ async fn test_search_mixed_results_deserialization() {
+ // Test that results with markdown/metadata are correctly identified as Documents
+ // and simple results are identified as WebResults
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ // Simple web result (no markdown/metadata)
+ {
+ "url": "https://example.com/simple",
+ "title": "Simple Result",
+ "description": "Just a search result"
+ },
+ // Scraped document with url AND markdown
+ {
+ "url": "https://example.com/scraped",
+ "markdown": "# Scraped Content\n\nThis has content.",
+ "metadata": {
+ "sourceURL": "https://example.com/scraped",
+ "statusCode": 200,
+ "title": "Scraped Page"
+ }
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.search("test", None).await.unwrap();
+
+ let web_results = response.data.web.unwrap();
+ assert_eq!(web_results.len(), 2);
+
+ // First should be WebResult
+ match &web_results[0] {
+ SearchResultOrDocument::WebResult(r) => {
+ assert_eq!(r.url, "https://example.com/simple");
+ assert_eq!(r.title, Some("Simple Result".to_string()));
+ }
+ SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),
+ }
+
+ // Second should be Document (has markdown)
+ match &web_results[1] {
+ SearchResultOrDocument::Document(d) => {
+ assert!(d.markdown.is_some());
+ assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));
+ }
+ SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),
+ }
+
+ mock.assert();
}
}
diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/types.rs
similarity index 100%
rename from apps/rust-sdk/src/v2/types.rs
rename to apps/rust-sdk/src/types.rs
diff --git a/apps/rust-sdk/src/v2/batch_scrape.rs b/apps/rust-sdk/src/v2/batch_scrape.rs
deleted file mode 100644
index cb831e787..000000000
--- a/apps/rust-sdk/src/v2/batch_scrape.rs
+++ /dev/null
@@ -1,591 +0,0 @@
-//! Batch scrape endpoint for Firecrawl API v2.
-
-use serde::{Deserialize, Serialize};
-
-use super::client::Client;
-use super::scrape::ScrapeOptions;
-use super::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};
-use crate::FirecrawlError;
-
-/// Options for batch scraping.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct BatchScrapeOptions {
- /// Scrape options to apply to all URLs.
- #[serde(flatten)]
- pub options: Option<ScrapeOptions>,
-
- /// Webhook configuration for job notifications.
- pub webhook: Option<WebhookConfig>,
-
- /// ID of an existing batch job to append URLs to.
- pub append_to_id: Option<String>,
-
- /// Whether to ignore invalid URLs instead of failing.
- #[serde(rename = "ignoreInvalidURLs")]
- pub ignore_invalid_urls: Option<bool>,
-
- /// Maximum concurrent requests.
- pub max_concurrency: Option<u32>,
-
- /// Enable zero data retention mode.
- pub zero_data_retention: Option<bool>,
-
- /// Idempotency key for the request.
- #[serde(skip)]
- pub idempotency_key: Option<String>,
-
- /// Integration identifier for tracking.
- pub integration: Option<String>,
-
- /// Poll interval for synchronous batch scrape (milliseconds).
- #[serde(skip)]
- pub poll_interval: Option<u64>,
-}
-
-/// Request body for batch scrape endpoint.
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct BatchScrapeRequest {
- urls: Vec<String>,
- #[serde(flatten)]
- options: BatchScrapeOptions,
-}
-
-/// Response from starting a batch scrape job.
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct BatchScrapeResponse {
- /// Whether the request was successful.
- pub success: bool,
- /// The batch scrape job ID.
- pub id: String,
- /// URL to check the batch scrape status.
- pub url: String,
- /// URLs that were invalid and ignored.
- #[serde(rename = "invalidURLs")]
- pub invalid_urls: Option<Vec<String>>,
-}
-
-/// Status of a batch scrape job.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct BatchScrapeJob {
- /// Current status of the batch scrape job.
- pub status: JobStatus,
- /// Number of URLs completed.
- pub completed: u32,
- /// Total number of URLs to scrape.
- pub total: u32,
- /// Credits used by the batch scrape.
- pub credits_used: Option<u32>,
- /// Expiry time of the batch data.
- pub expires_at: Option<String>,
- /// URL for the next page of results.
- pub next: Option<String>,
- /// Scraped documents.
- pub data: Vec<Document>,
-}
-
-impl Client {
- /// Starts a batch scrape job asynchronously.
- ///
- /// Returns immediately with a job ID that can be used to check status.
- ///
- /// # Arguments
- ///
- /// * `urls` - List of URLs to scrape.
- /// * `options` - Optional batch scrape configuration.
- ///
- /// # Returns
- ///
- /// A `BatchScrapeResponse` containing the job ID.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, BatchScrapeOptions};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let urls = vec![
- /// "https://example.com".to_string(),
- /// "https://example.org".to_string(),
- /// ];
- ///
- /// let response = client.start_batch_scrape(urls, None).await?;
- /// println!("Batch job started: {}", response.id);
- ///
- /// // Check status later
- /// let status = client.get_batch_scrape_status(&response.id).await?;
- /// println!("Completed: {}/{}", status.completed, status.total);
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn start_batch_scrape(
- &self,
- urls: Vec<String>,
- options: impl Into<Option<BatchScrapeOptions>>,
- ) -> Result<BatchScrapeResponse, FirecrawlError> {
- let options = options.into().unwrap_or_default();
- let body = BatchScrapeRequest {
- urls,
- options: options.clone(),
- };
-
- let headers = self.prepare_headers(options.idempotency_key.as_ref());
-
- let response = self
- .client
- .post(self.url("/batch/scrape"))
- .headers(headers)
- .json(&body)
- .send()
- .await
- .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;
-
- self.handle_response(response, "start batch scrape").await
- }
-
- /// Gets the status of a batch scrape job.
- ///
- /// If the job is completed, this will automatically fetch all pages of results.
- ///
- /// # Arguments
- ///
- /// * `id` - The batch scrape job ID.
- ///
- /// # Returns
- ///
- /// A `BatchScrapeJob` containing the current status and any available documents.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let status = client.get_batch_scrape_status("job-id").await?;
- /// println!("Status: {:?}", status.status);
- /// println!("Completed: {}/{}", status.completed, status.total);
- /// println!("Documents: {}", status.data.len());
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn get_batch_scrape_status(
- &self,
- id: impl AsRef<str>,
- ) -> Result<BatchScrapeJob, FirecrawlError> {
- let response = self
- .client
- .get(self.url(&format!("/batch/scrape/{}", id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(
- format!("Checking batch scrape status {}", id.as_ref()),
- e,
- )
- })?;
-
- let mut status: BatchScrapeJob = self
- .handle_response(response, format!("batch scrape status {}", id.as_ref()))
- .await?;
-
- // Auto-paginate if completed
- if status.status == JobStatus::Completed {
- while let Some(next) = status.next.take() {
- let next_status = self.get_batch_scrape_status_next(&next).await?;
- status.data.extend(next_status.data);
- status.next = next_status.next;
- }
- }
-
- Ok(status)
- }
-
- /// Fetches the next page of batch scrape results.
- async fn get_batch_scrape_status_next(
- &self,
- next: &str,
- ) -> Result<BatchScrapeJob, FirecrawlError> {
- let response = self
- .client
- .get(next)
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)
- })?;
-
- self.handle_response(response, "batch scrape pagination")
- .await
- }
-
- /// Scrapes multiple URLs and waits for completion.
- ///
- /// This method starts a batch scrape and polls until it completes or fails.
- ///
- /// # Arguments
- ///
- /// * `urls` - List of URLs to scrape.
- /// * `options` - Optional batch scrape configuration.
- ///
- /// # Returns
- ///
- /// A `BatchScrapeJob` containing all scraped documents.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, BatchScrapeOptions, ScrapeOptions, Format};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let urls = vec![
- /// "https://example.com/page1".to_string(),
- /// "https://example.com/page2".to_string(),
- /// "https://example.com/page3".to_string(),
- /// ];
- ///
- /// let options = BatchScrapeOptions {
- /// options: Some(ScrapeOptions {
- /// formats: Some(vec![Format::Markdown, Format::Links]),
- /// ..Default::default()
- /// }),
- /// ignore_invalid_urls: Some(true),
- /// poll_interval: Some(3000),
- /// ..Default::default()
- /// };
- ///
- /// let result = client.batch_scrape(urls, options).await?;
- /// println!("Scraped {} pages", result.data.len());
- ///
- /// for doc in result.data {
- /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn batch_scrape(
- &self,
- urls: Vec<String>,
- options: impl Into<Option<BatchScrapeOptions>>,
- ) -> Result<BatchScrapeJob, FirecrawlError> {
- let options = options.into().unwrap_or_default();
- let poll_interval = options.poll_interval.unwrap_or(2000);
-
- let response = self.start_batch_scrape(urls, options).await?;
- self.wait_for_batch_scrape(&response.id, poll_interval)
- .await
- }
-
- /// Waits for a batch scrape job to complete.
- async fn wait_for_batch_scrape(
- &self,
- id: &str,
- poll_interval: u64,
- ) -> Result<BatchScrapeJob, FirecrawlError> {
- loop {
- let status = self.get_batch_scrape_status(id).await?;
-
- match status.status {
- JobStatus::Completed => return Ok(status),
- JobStatus::Scraping => {
- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
- }
- JobStatus::Failed => {
- return Err(FirecrawlError::CrawlJobFailed(
- "Batch scrape job failed".to_string(),
- convert_batch_job_to_crawl_status(status),
- ));
- }
- JobStatus::Cancelled => {
- return Err(FirecrawlError::CrawlJobFailed(
- "Batch scrape job was cancelled".to_string(),
- convert_batch_job_to_crawl_status(status),
- ));
- }
- }
- }
- }
-
- /// Gets errors from a batch scrape job.
- ///
- /// # Arguments
- ///
- /// * `id` - The batch scrape job ID.
- ///
- /// # Returns
- ///
- /// A `CrawlErrorsResponse` containing error details.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let errors = client.get_batch_scrape_errors("job-id").await?;
- /// for error in errors.errors {
- /// println!("Error on {}: {}", error.url, error.error);
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn get_batch_scrape_errors(
- &self,
- id: impl AsRef<str>,
- ) -> Result<CrawlErrorsResponse, FirecrawlError> {
- let response = self
- .client
- .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)
- })?;
-
- self.handle_response(response, "batch scrape errors").await
- }
-}
-
-/// Converts a BatchScrapeJob to CrawlStatus for error compatibility.
-fn convert_batch_job_to_crawl_status(job: BatchScrapeJob) -> crate::crawl::CrawlStatus {
- crate::crawl::CrawlStatus {
- status: match job.status {
- JobStatus::Completed => crate::crawl::CrawlStatusTypes::Completed,
- JobStatus::Scraping => crate::crawl::CrawlStatusTypes::Scraping,
- JobStatus::Failed => crate::crawl::CrawlStatusTypes::Failed,
- JobStatus::Cancelled => crate::crawl::CrawlStatusTypes::Cancelled,
- },
- total: job.total,
- completed: job.completed,
- credits_used: job.credits_used.unwrap_or(0),
- expires_at: job.expires_at.unwrap_or_default(),
- next: job.next,
- data: job
- .data
- .into_iter()
- .map(|d| super::crawl::convert_v2_document_to_v1_pub(d))
- .collect(),
- }
-}
-
-impl super::crawl::CrawlJob {
- /// Helper function to convert v2 document to v1 format.
- pub(crate) fn _convert_document(doc: Document) -> crate::document::Document {
- super::crawl::convert_v2_document_to_v1_pub(doc)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- async fn test_start_batch_scrape_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/batch/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "batch-123",
- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let urls = vec![
- "https://example.com".to_string(),
- "https://example.org".to_string(),
- ];
-
- let response = client.start_batch_scrape(urls, None).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.id, "batch-123");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_get_batch_scrape_status_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("GET", "/v2/batch/scrape/batch-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "status": "completed",
- "total": 3,
- "completed": 3,
- "creditsUsed": 3,
- "data": [
- {
- "markdown": "# Page 1",
- "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
- },
- {
- "markdown": "# Page 2",
- "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let status = client.get_batch_scrape_status("batch-123").await.unwrap();
-
- assert_eq!(status.status, JobStatus::Completed);
- assert_eq!(status.total, 3);
- assert_eq!(status.completed, 3);
- assert_eq!(status.data.len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_batch_scrape_with_invalid_urls() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/batch/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "batch-456",
- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",
- "invalidURLs": ["not-a-url"]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];
-
- let options = BatchScrapeOptions {
- ignore_invalid_urls: Some(true),
- ..Default::default()
- };
-
- let response = client.start_batch_scrape(urls, options).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_batch_scrape_sync() {
- let mut server = mockito::Server::new_async().await;
-
- // Mock the start endpoint
- let start_mock = server
- .mock("POST", "/v2/batch/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "batch-789",
- "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"
- })
- .to_string(),
- )
- .create();
-
- // Mock the status endpoint (completed immediately)
- let status_mock = server
- .mock("GET", "/v2/batch/scrape/batch-789")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "status": "completed",
- "total": 2,
- "completed": 2,
- "data": [
- {
- "markdown": "# Content",
- "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let urls = vec!["https://example.com".to_string()];
-
- let result = client.batch_scrape(urls, None).await.unwrap();
-
- assert_eq!(result.status, JobStatus::Completed);
- assert_eq!(result.data.len(), 1);
- start_mock.assert();
- status_mock.assert();
- }
-
- #[tokio::test]
- async fn test_get_batch_scrape_errors() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("GET", "/v2/batch/scrape/batch-123/errors")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "errors": [
- {
- "id": "err-1",
- "url": "https://example.com/broken",
- "error": "Connection timeout"
- }
- ],
- "robotsBlocked": []
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();
-
- assert_eq!(errors.errors.len(), 1);
- assert_eq!(errors.errors[0].error, "Connection timeout");
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/v2/crawl.rs b/apps/rust-sdk/src/v2/crawl.rs
deleted file mode 100644
index 4b773abe3..000000000
--- a/apps/rust-sdk/src/v2/crawl.rs
+++ /dev/null
@@ -1,688 +0,0 @@
-//! Crawl endpoint for Firecrawl API v2.
-
-use serde::{Deserialize, Serialize};
-
-use super::client::Client;
-use super::scrape::ScrapeOptions;
-use super::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};
-use crate::FirecrawlError;
-
-/// Options for crawling a website.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlOptions {
- /// Natural language prompt to guide crawl behavior.
- pub prompt: Option<String>,
-
- /// URL path patterns to exclude from crawling.
- pub exclude_paths: Option<Vec<String>>,
-
- /// URL path patterns to include in crawling.
- pub include_paths: Option<Vec<String>>,
-
- /// Maximum depth of links to follow from the initial URL.
- pub max_discovery_depth: Option<u32>,
-
- /// How to handle the sitemap.
- pub sitemap: Option<SitemapMode>,
-
- /// Ignore query parameters when deduplicating URLs.
- pub ignore_query_parameters: Option<bool>,
-
- /// Maximum number of pages to crawl.
- pub limit: Option<u32>,
-
- /// Crawl the entire domain regardless of path structure.
- pub crawl_entire_domain: Option<bool>,
-
- /// Allow following links to external domains.
- pub allow_external_links: Option<bool>,
-
- /// Allow following links to subdomains.
- pub allow_subdomains: Option<bool>,
-
- /// Delay between requests in seconds.
- pub delay: Option<u32>,
-
- /// Maximum concurrent requests.
- pub max_concurrency: Option<u32>,
-
- /// Webhook configuration for job notifications.
- pub webhook: Option<WebhookConfig>,
-
- /// Scrape options to apply to each page.
- pub scrape_options: Option<ScrapeOptions>,
-
- /// Enable zero data retention mode.
- pub zero_data_retention: Option<bool>,
-
- /// Integration identifier for tracking.
- pub integration: Option<String>,
-
- /// Idempotency key for the request.
- #[serde(skip)]
- pub idempotency_key: Option<String>,
-
- /// Poll interval for synchronous crawl (milliseconds).
- #[serde(skip)]
- pub poll_interval: Option<u64>,
-}
-
-/// Request body for crawl endpoint.
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct CrawlRequest {
- url: String,
- #[serde(flatten)]
- options: CrawlOptions,
-}
-
-/// Response from starting a crawl job.
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlResponse {
- /// Whether the request was successful.
- pub success: bool,
- /// The crawl job ID.
- pub id: String,
- /// URL to check the crawl status.
- pub url: String,
-}
-
-/// Status of a crawl job.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CrawlJob {
- /// Current status of the crawl job.
- pub status: JobStatus,
- /// Total number of pages to crawl.
- pub total: u32,
- /// Number of pages completed.
- pub completed: u32,
- /// Credits used by the crawl.
- pub credits_used: Option<u32>,
- /// Expiry time of the crawl data.
- pub expires_at: Option<String>,
- /// URL for the next page of results.
- pub next: Option<String>,
- /// Crawled documents.
- pub data: Vec<Document>,
-}
-
-/// Response from canceling a crawl.
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct CancelCrawlResponse {
- /// Status of the cancellation.
- pub status: String,
-}
-
-impl Client {
- /// Starts a crawl job asynchronously.
- ///
- /// Returns immediately with a job ID that can be used to check status.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to start crawling from.
- /// * `options` - Optional crawl configuration.
- ///
- /// # Returns
- ///
- /// A `CrawlResponse` containing the job ID.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, CrawlOptions};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let response = client.start_crawl("https://example.com", None).await?;
- /// println!("Crawl job started: {}", response.id);
- ///
- /// // Check status later
- /// let status = client.get_crawl_status(&response.id).await?;
- /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn start_crawl(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<CrawlOptions>>,
- ) -> Result<CrawlResponse, FirecrawlError> {
- let options = options.into().unwrap_or_default();
- let body = CrawlRequest {
- url: url.as_ref().to_string(),
- options: options.clone(),
- };
-
- let headers = self.prepare_headers(options.idempotency_key.as_ref());
-
- let response = self
- .client
- .post(self.url("/crawl"))
- .headers(headers)
- .json(&body)
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)
- })?;
-
- self.handle_response(response, "start crawl").await
- }
-
- /// Gets the status of a crawl job.
- ///
- /// If the job is completed, this will automatically fetch all pages of results.
- ///
- /// # Arguments
- ///
- /// * `id` - The crawl job ID.
- ///
- /// # Returns
- ///
- /// A `CrawlJob` containing the current status and any available documents.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let status = client.get_crawl_status("job-id").await?;
- /// println!("Status: {:?}", status.status);
- /// println!("Completed: {}/{}", status.completed, status.total);
- /// println!("Documents: {}", status.data.len());
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {
- let response = self
- .client
- .get(self.url(&format!("/crawl/{}", id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)
- })?;
-
- let mut status: CrawlJob = self
- .handle_response(response, format!("crawl status {}", id.as_ref()))
- .await?;
-
- // Auto-paginate if completed
- if status.status == JobStatus::Completed {
- while let Some(next) = status.next.take() {
- let next_status = self.get_crawl_status_next(&next).await?;
- status.data.extend(next_status.data);
- status.next = next_status.next;
- }
- }
-
- Ok(status)
- }
-
- /// Fetches the next page of crawl results.
- async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {
- let response = self
- .client
- .get(next)
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;
-
- self.handle_response(response, "crawl pagination").await
- }
-
- /// Crawls a website and waits for completion.
- ///
- /// This method starts a crawl and polls until it completes or fails.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to start crawling from.
- /// * `options` - Optional crawl configuration.
- ///
- /// # Returns
- ///
- /// A `CrawlJob` containing all crawled documents.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, CrawlOptions, SitemapMode};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let options = CrawlOptions {
- /// sitemap: Some(SitemapMode::Include),
- /// limit: Some(100),
- /// poll_interval: Some(5000), // Check every 5 seconds
- /// ..Default::default()
- /// };
- ///
- /// let result = client.crawl("https://example.com", options).await?;
- /// println!("Crawled {} pages", result.data.len());
- ///
- /// for doc in result.data {
- /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn crawl(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<CrawlOptions>>,
- ) -> Result<CrawlJob, FirecrawlError> {
- let options = options.into().unwrap_or_default();
- let poll_interval = options.poll_interval.unwrap_or(2000);
-
- let response = self.start_crawl(url, options).await?;
- self.wait_for_crawl(&response.id, poll_interval).await
- }
-
- /// Waits for a crawl job to complete.
- async fn wait_for_crawl(
- &self,
- id: &str,
- poll_interval: u64,
- ) -> Result<CrawlJob, FirecrawlError> {
- loop {
- let status = self.get_crawl_status(id).await?;
-
- match status.status {
- JobStatus::Completed => return Ok(status),
- JobStatus::Scraping => {
- tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
- }
- JobStatus::Failed => {
- return Err(FirecrawlError::CrawlJobFailed(
- "Crawl job failed".to_string(),
- crate::crawl::CrawlStatus {
- status: crate::crawl::CrawlStatusTypes::Failed,
- total: status.total,
- completed: status.completed,
- credits_used: status.credits_used.unwrap_or(0),
- expires_at: status.expires_at.unwrap_or_default(),
- next: status.next,
- data: status
- .data
- .into_iter()
- .map(|d| convert_v2_document_to_v1(d))
- .collect(),
- },
- ));
- }
- JobStatus::Cancelled => {
- return Err(FirecrawlError::CrawlJobFailed(
- "Crawl job was cancelled".to_string(),
- crate::crawl::CrawlStatus {
- status: crate::crawl::CrawlStatusTypes::Cancelled,
- total: status.total,
- completed: status.completed,
- credits_used: status.credits_used.unwrap_or(0),
- expires_at: status.expires_at.unwrap_or_default(),
- next: status.next,
- data: status
- .data
- .into_iter()
- .map(|d| convert_v2_document_to_v1(d))
- .collect(),
- },
- ));
- }
- }
- }
- }
-
- /// Cancels a running crawl job.
- ///
- /// # Arguments
- ///
- /// * `id` - The crawl job ID to cancel.
- ///
- /// # Returns
- ///
- /// A `CancelCrawlResponse` indicating the cancellation status.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let response = client.cancel_crawl("job-id").await?;
- /// println!("Cancellation status: {}", response.status);
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn cancel_crawl(
- &self,
- id: impl AsRef<str>,
- ) -> Result<CancelCrawlResponse, FirecrawlError> {
- let response = self
- .client
- .delete(self.url(&format!("/crawl/{}", id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
- })?;
-
- self.handle_response(response, "cancel crawl").await
- }
-
- /// Gets errors from a crawl job.
- ///
- /// # Arguments
- ///
- /// * `id` - The crawl job ID.
- ///
- /// # Returns
- ///
- /// A `CrawlErrorsResponse` containing error details.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let errors = client.get_crawl_errors("job-id").await?;
- /// for error in errors.errors {
- /// println!("Error on {}: {}", error.url, error.error);
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn get_crawl_errors(
- &self,
- id: impl AsRef<str>,
- ) -> Result<CrawlErrorsResponse, FirecrawlError> {
- let response = self
- .client
- .get(self.url(&format!("/crawl/{}/errors", id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)
- })?;
-
- self.handle_response(response, "crawl errors").await
- }
-}
-
-/// Converts a v2 Document to a v1 Document for error compatibility.
-pub(crate) fn convert_v2_document_to_v1_pub(doc: Document) -> crate::document::Document {
- convert_v2_document_to_v1(doc)
-}
-
-/// Converts a v2 Document to a v1 Document for error compatibility.
-fn convert_v2_document_to_v1(doc: Document) -> crate::document::Document {
- let metadata = doc.metadata.unwrap_or_default();
- crate::document::Document {
- markdown: doc.markdown,
- html: doc.html,
- raw_html: doc.raw_html,
- screenshot: doc.screenshot,
- links: doc.links,
- extract: doc.json,
- metadata: crate::document::DocumentMetadata {
- source_url: metadata.source_url.unwrap_or_default(),
- status_code: metadata.status_code.unwrap_or(0),
- error: metadata.error,
- title: metadata.title,
- description: metadata.description,
- language: metadata.language,
- keywords: metadata.keywords,
- robots: metadata.robots,
- og_title: metadata.og_title,
- og_description: metadata.og_description,
- og_url: metadata.og_url,
- og_image: metadata.og_image,
- og_audio: metadata.og_audio,
- og_determiner: metadata.og_determiner,
- og_locale: metadata.og_locale,
- og_locale_alternate: metadata.og_locale_alternate,
- og_site_name: metadata.og_site_name,
- og_video: metadata.og_video,
- article_section: metadata.article_section,
- article_tag: metadata.article_tag,
- published_time: metadata.published_time,
- modified_time: metadata.modified_time,
- dcterms_keywords: metadata.dcterms_keywords,
- dc_description: metadata.dc_description,
- dc_subject: metadata.dc_subject,
- dcterms_subject: metadata.dcterms_subject,
- dcterms_audience: metadata.dcterms_audience,
- dc_type: metadata.dc_type,
- dcterms_type: metadata.dcterms_type,
- dc_date: metadata.dc_date,
- dc_date_created: metadata.dc_date_created,
- dcterms_created: metadata.dcterms_created,
- },
- warning: doc.warning,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- async fn test_start_crawl_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/crawl")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "crawl-123",
- "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client
- .start_crawl("https://example.com", None)
- .await
- .unwrap();
-
- assert!(response.success);
- assert_eq!(response.id, "crawl-123");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_get_crawl_status_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("GET", "/v2/crawl/crawl-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "status": "completed",
- "total": 5,
- "completed": 5,
- "creditsUsed": 5,
- "expiresAt": "2024-12-31T23:59:59Z",
- "data": [
- {
- "markdown": "# Page 1",
- "metadata": {
- "sourceURL": "https://example.com/page1",
- "statusCode": 200
- }
- },
- {
- "markdown": "# Page 2",
- "metadata": {
- "sourceURL": "https://example.com/page2",
- "statusCode": 200
- }
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let status = client.get_crawl_status("crawl-123").await.unwrap();
-
- assert_eq!(status.status, JobStatus::Completed);
- assert_eq!(status.total, 5);
- assert_eq!(status.completed, 5);
- assert_eq!(status.data.len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_cancel_crawl_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("DELETE", "/v2/crawl/crawl-123")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "status": "cancelled"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client.cancel_crawl("crawl-123").await.unwrap();
-
- assert_eq!(response.status, "cancelled");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_get_crawl_errors_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("GET", "/v2/crawl/crawl-123/errors")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "errors": [
- {
- "id": "error-1",
- "timestamp": "2024-01-01T00:00:00Z",
- "url": "https://example.com/broken",
- "error": "404 Not Found"
- }
- ],
- "robotsBlocked": [
- "https://example.com/admin"
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let errors = client.get_crawl_errors("crawl-123").await.unwrap();
-
- assert_eq!(errors.errors.len(), 1);
- assert_eq!(errors.errors[0].url, "https://example.com/broken");
- assert_eq!(errors.robots_blocked.len(), 1);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_crawl_with_options() {
- let mut server = mockito::Server::new_async().await;
-
- // Mock the start endpoint
- let start_mock = server
- .mock("POST", "/v2/crawl")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "id": "crawl-456",
- "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"
- })
- .to_string(),
- )
- .create();
-
- // Mock the status endpoint (completed immediately)
- let status_mock = server
- .mock("GET", "/v2/crawl/crawl-456")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "status": "completed",
- "total": 2,
- "completed": 2,
- "data": [
- {
- "markdown": "# Page 1",
- "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let options = CrawlOptions {
- limit: Some(10),
- sitemap: Some(SitemapMode::Include),
- ..Default::default()
- };
-
- let result = client.crawl("https://example.com", options).await.unwrap();
-
- assert_eq!(result.status, JobStatus::Completed);
- assert_eq!(result.data.len(), 1);
- start_mock.assert();
- status_mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/v2/map.rs b/apps/rust-sdk/src/v2/map.rs
deleted file mode 100644
index ce2dc54ca..000000000
--- a/apps/rust-sdk/src/v2/map.rs
+++ /dev/null
@@ -1,333 +0,0 @@
-//! Map endpoint for Firecrawl API v2.
-
-use serde::{Deserialize, Serialize};
-
-use super::client::Client;
-use super::types::{LocationConfig, SearchResultWeb, SitemapMode};
-use crate::FirecrawlError;
-
-/// Options for mapping a URL.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct MapOptions {
- /// Search query to filter discovered links.
- pub search: Option<String>,
-
- /// How to handle the sitemap.
- pub sitemap: Option<SitemapMode>,
-
- /// Include subdomains in the mapping.
- pub include_subdomains: Option<bool>,
-
- /// Ignore query parameters when deduplicating URLs.
- pub ignore_query_parameters: Option<bool>,
-
- /// Maximum number of links to return.
- pub limit: Option<u32>,
-
- /// Timeout in milliseconds.
- pub timeout: Option<u32>,
-
- /// Integration identifier for tracking.
- pub integration: Option<String>,
-
- /// Location configuration for proxy routing.
- pub location: Option<LocationConfig>,
-}
-
-/// Request body for map endpoint.
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct MapRequest {
- url: String,
- #[serde(flatten)]
- options: MapOptions,
-}
-
-/// Response from map endpoint.
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct MapResponse {
- /// Whether the request was successful.
- pub success: bool,
- /// Discovered links with metadata.
- pub links: Vec<SearchResultWeb>,
- /// Warning message if any.
- pub warning: Option<String>,
-}
-
-impl Client {
- /// Maps a URL to discover all associated links.
- ///
- /// This endpoint discovers links from a website's sitemap, page content,
- /// and other sources without fully scraping each page.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to map.
- /// * `options` - Optional mapping configuration.
- ///
- /// # Returns
- ///
- /// A `MapResponse` containing the discovered links.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, MapOptions, SitemapMode};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// // Simple map
- /// let response = client.map("https://example.com", None).await?;
- /// println!("Found {} links", response.links.len());
- ///
- /// // Map with options
- /// let options = MapOptions {
- /// sitemap: Some(SitemapMode::Include),
- /// include_subdomains: Some(true),
- /// limit: Some(1000),
- /// ..Default::default()
- /// };
- /// let response = client.map("https://example.com", options).await?;
- ///
- /// for link in response.links {
- /// println!("URL: {}, Title: {:?}", link.url, link.title);
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn map(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<MapOptions>>,
- ) -> Result<MapResponse, FirecrawlError> {
- let body = MapRequest {
- url: url.as_ref().to_string(),
- options: options.into().unwrap_or_default(),
- };
-
- let headers = self.prepare_headers(None);
-
- let response = self
- .client
- .post(self.url("/map"))
- .headers(headers)
- .json(&body)
- .send()
- .await
- .map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
-
- self.handle_response(response, "map").await
- }
-
- /// Maps a URL and returns just the list of URLs.
- ///
- /// This is a convenience method that returns only the URL strings.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to map.
- /// * `options` - Optional mapping configuration.
- ///
- /// # Returns
- ///
- /// A vector of discovered URL strings.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let urls = client.map_urls("https://example.com", None).await?;
- /// for url in urls {
- /// println!("{}", url);
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn map_urls(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<MapOptions>>,
- ) -> Result<Vec<String>, FirecrawlError> {
- let response = self.map(url, options).await?;
- Ok(response.links.into_iter().map(|link| link.url).collect())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- async fn test_map_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/map")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "links": [
- {
- "url": "https://example.com/",
- "title": "Example Domain",
- "description": "Home page"
- },
- {
- "url": "https://example.com/about",
- "title": "About Us"
- },
- {
- "url": "https://example.com/contact"
- }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client.map("https://example.com", None).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.links.len(), 3);
- assert_eq!(response.links[0].url, "https://example.com/");
- assert_eq!(response.links[0].title, Some("Example Domain".to_string()));
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_map_with_options() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/map")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "links": [
- { "url": "https://example.com/page1" },
- { "url": "https://example.com/page2" }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let options = MapOptions {
- sitemap: Some(SitemapMode::Include),
- include_subdomains: Some(true),
- limit: Some(100),
- ..Default::default()
- };
-
- let response = client.map("https://example.com", options).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.links.len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_map_urls() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/map")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "links": [
- { "url": "https://example.com/page1" },
- { "url": "https://example.com/page2" },
- { "url": "https://example.com/page3" }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let urls = client.map_urls("https://example.com", None).await.unwrap();
-
- assert_eq!(urls.len(), 3);
- assert_eq!(urls[0], "https://example.com/page1");
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_map_with_search_filter() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/map")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "links": [
- { "url": "https://example.com/blog/post1" },
- { "url": "https://example.com/blog/post2" }
- ]
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let options = MapOptions {
- search: Some("blog".to_string()),
- ..Default::default()
- };
-
- let response = client.map("https://example.com", options).await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.links.len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_map_error_response() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/map")
- .with_status(400)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Invalid URL"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = client.map("invalid-url", None).await;
-
- assert!(result.is_err());
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/v2/mod.rs b/apps/rust-sdk/src/v2/mod.rs
deleted file mode 100644
index caa1a4fcb..000000000
--- a/apps/rust-sdk/src/v2/mod.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-//! Firecrawl API v2 client module.
-//!
-//! This module provides access to the v2 API endpoints while maintaining
-//! backward compatibility with v1. The v2 API includes new features like
-//! the Agent endpoint and improved options.
-//!
-//! # Example
-//!
-//! ```no_run
-//! use firecrawl::v2::{Client, ScrapeOptions};
-//!
-//! #[tokio::main]
-//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
-//! let client = Client::new("your-api-key")?;
-//!
-//! // Scrape a URL
-//! let document = client.scrape("https://example.com", None).await?;
-//! println!("Content: {:?}", document.markdown);
-//!
-//! Ok(())
-//! }
-//! ```
-
-mod agent;
-mod batch_scrape;
-mod client;
-mod crawl;
-mod map;
-mod scrape;
-mod search;
-mod types;
-
-pub use agent::*;
-pub use batch_scrape::*;
-pub use client::Client;
-pub use crawl::*;
-pub use map::*;
-pub use scrape::*;
-pub use search::*;
-pub use types::*;
diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rs
deleted file mode 100644
index ffbe8fe44..000000000
--- a/apps/rust-sdk/src/v2/scrape.rs
+++ /dev/null
@@ -1,731 +0,0 @@
-//! Scrape endpoint for Firecrawl API v2.
-
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use std::collections::HashMap;
-
-use super::client::Client;
-use super::types::{
- Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,
- LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
-};
-use crate::FirecrawlError;
-
-/// Options for scraping a URL.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeOptions {
- /// Output formats to include in the response.
- pub formats: Option<Vec<Format>>,
-
- /// Additional HTTP headers to send with the request.
- pub headers: Option<HashMap<String, String>>,
-
- /// HTML tags to exclusively include in the output.
- pub include_tags: Option<Vec<String>>,
-
- /// HTML tags to exclude from the output.
- pub exclude_tags: Option<Vec<String>>,
-
- /// Only extract the main content of the page.
- pub only_main_content: Option<bool>,
-
- /// Timeout in milliseconds before returning an error.
- pub timeout: Option<u32>,
-
- /// Time to wait after page load before scraping (milliseconds).
- pub wait_for: Option<u32>,
-
- /// Emulate a mobile device.
- pub mobile: Option<bool>,
-
- /// Parser configurations (e.g., for PDFs).
- pub parsers: Option<Vec<ParserConfig>>,
-
- /// Browser automation actions to perform before scraping.
- pub actions: Option<Vec<Action>>,
-
- /// Location configuration for proxy routing.
- pub location: Option<LocationConfig>,
-
- /// Skip TLS certificate verification.
- pub skip_tls_verification: Option<bool>,
-
- /// Remove base64-encoded images from the output.
- pub remove_base64_images: Option<bool>,
-
- /// Enable fast mode for quicker scrapes with reduced accuracy.
- pub fast_mode: Option<bool>,
-
- /// Block advertisements on the page.
- pub block_ads: Option<bool>,
-
- /// Proxy type to use.
- pub proxy: Option<ProxyType>,
-
- /// Maximum age of cached content to accept (seconds).
- pub max_age: Option<u32>,
-
- /// Minimum age of cached content to accept (seconds).
- pub min_age: Option<u32>,
-
- /// Store the result in cache for future requests.
- pub store_in_cache: Option<bool>,
-
- /// Persistent browser profile for maintaining state across scrapes.
- pub profile: Option<ProfileConfig>,
-
- /// Integration identifier for tracking.
- pub integration: Option<String>,
-
- /// JSON extraction options.
- pub json_options: Option<JsonOptions>,
-
- /// Screenshot options.
- pub screenshot_options: Option<ScreenshotOptions>,
-
- /// Change tracking options.
- pub change_tracking_options: Option<ChangeTrackingOptions>,
-
- /// Attribute selectors for extraction.
- pub attribute_selectors: Option<Vec<AttributeSelector>>,
-}
-
-/// Parser configuration for document parsing.
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(untagged)]
-pub enum ParserConfig {
- /// Simple parser type string.
- Simple(String),
- /// PDF parser with options.
- Pdf {
- #[serde(rename = "type")]
- parser_type: String,
- #[serde(skip_serializing_if = "Option::is_none")]
- max_pages: Option<u32>,
- },
-}
-
-/// Request body for scrape endpoint.
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct ScrapeRequest {
- url: String,
- #[serde(flatten)]
- options: ScrapeOptions,
-}
-
-/// Response from scrape endpoint.
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct ScrapeResponse {
- success: bool,
- data: Document,
- #[serde(skip_serializing_if = "Option::is_none")]
- warning: Option<String>,
-}
-
-/// Supported languages for scrape-bound browser execution.
-#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "lowercase")]
-pub enum ScrapeExecuteLanguage {
- Python,
- Node,
- Bash,
-}
-
-/// Options for executing code or a prompt in a scrape-bound browser session.
-///
-/// At least one of `code` or `prompt` must be provided.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeExecuteOptions {
- /// Code to execute (optional if `prompt` is provided).
- pub code: Option<String>,
- /// Natural-language instruction for the browser agent (optional if `code` is provided).
- pub prompt: Option<String>,
- /// Runtime language for the code.
- pub language: Option<ScrapeExecuteLanguage>,
- /// Execution timeout in seconds.
- pub timeout: Option<u32>,
- /// Optional origin tag for request attribution.
- pub origin: Option<String>,
-}
-
-/// Response from scrape-bound browser execution.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeExecuteResponse {
- /// Whether the request succeeded.
- pub success: bool,
- /// Live-view URL for the browser session.
- pub live_view_url: Option<String>,
- /// Interactive live-view URL for the browser session.
- pub interactive_live_view_url: Option<String>,
- /// Agent output when a prompt was used.
- pub output: Option<String>,
- /// Captured stdout from execution.
- pub stdout: Option<String>,
- /// Optional execution result payload.
- pub result: Option<String>,
- /// Captured stderr from execution.
- pub stderr: Option<String>,
- /// Process exit code.
- pub exit_code: Option<i32>,
- /// Whether execution was killed by timeout or system.
- pub killed: Option<bool>,
- /// Error message when execution fails.
- pub error: Option<String>,
-}
-
-/// Response from deleting a scrape-bound browser session.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeBrowserDeleteResponse {
- /// Whether the delete request succeeded.
- pub success: bool,
- /// Session duration in milliseconds when available.
- pub session_duration_ms: Option<u64>,
- /// Credits billed when available.
- pub credits_billed: Option<u32>,
- /// Error message when deletion fails.
- pub error: Option<String>,
-}
-
-impl Client {
- /// Scrapes a URL and returns the content in the requested formats.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to scrape.
- /// * `options` - Optional scrape configuration.
- ///
- /// # Returns
- ///
- /// A `Document` containing the scraped content.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, ScrapeOptions, Format};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// // Simple scrape
- /// let document = client.scrape("https://example.com", None).await?;
- /// println!("Markdown: {:?}", document.markdown);
- ///
- /// // Scrape with options
- /// let options = ScrapeOptions {
- /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
- /// only_main_content: Some(true),
- /// ..Default::default()
- /// };
- /// let document = client.scrape("https://example.com", options).await?;
- /// println!("Links: {:?}", document.links);
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn scrape(
- &self,
- url: impl AsRef<str>,
- options: impl Into<Option<ScrapeOptions>>,
- ) -> Result<Document, FirecrawlError> {
- let body = ScrapeRequest {
- url: url.as_ref().to_string(),
- options: options.into().unwrap_or_default(),
- };
-
- let headers = self.prepare_headers(None);
-
- let response = self
- .client
- .post(self.url("/scrape"))
- .headers(headers)
- .json(&body)
- .send()
- .await
- .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
-
- let response: ScrapeResponse = self.handle_response(response, "scrape").await?;
-
- Ok(response.data)
- }
-
- /// Scrapes a URL with a JSON schema for structured extraction.
- ///
- /// This is a convenience method that combines scraping with JSON extraction.
- ///
- /// # Arguments
- ///
- /// * `url` - The URL to scrape.
- /// * `schema` - JSON schema for the extraction.
- /// * `prompt` - Optional extraction prompt.
- ///
- /// # Returns
- ///
- /// The extracted JSON value matching the schema.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- /// use serde_json::json;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let schema = json!({
- /// "type": "object",
- /// "properties": {
- /// "title": { "type": "string" },
- /// "price": { "type": "number" }
- /// }
- /// });
- ///
- /// let data = client.scrape_with_schema(
- /// "https://example.com/product",
- /// schema,
- /// Some("Extract the product title and price")
- /// ).await?;
- ///
- /// println!("Extracted: {}", data);
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn scrape_with_schema(
- &self,
- url: impl AsRef<str>,
- schema: Value,
- prompt: Option<impl AsRef<str>>,
- ) -> Result<Value, FirecrawlError> {
- let options = ScrapeOptions {
- formats: Some(vec![Format::Json]),
- json_options: Some(JsonOptions {
- schema: Some(schema),
- prompt: prompt.map(|p| p.as_ref().to_string()),
- ..Default::default()
- }),
- ..Default::default()
- };
-
- let document = self.scrape(url, options).await?;
- Ok(document.json.unwrap_or(Value::Null))
- }
-
- /// Interacts with the browser session associated with a scrape job.
- ///
- /// # Arguments
- ///
- /// * `job_id` - The scrape job ID.
- /// * `options` - Execution options including code and runtime config.
- ///
- /// # Returns
- ///
- /// A `ScrapeExecuteResponse` containing execution output.
- pub async fn interact(
- &self,
- job_id: impl AsRef<str>,
- options: ScrapeExecuteOptions,
- ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
- let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
- let has_prompt = options
- .prompt
- .as_ref()
- .is_some_and(|p| !p.trim().is_empty());
- if !has_code && !has_prompt {
- return Err(FirecrawlError::Missuse(
- "Either 'code' or 'prompt' must be provided".into(),
- ));
- }
-
- let mut body = options;
- if body.language.is_none() {
- body.language = Some(ScrapeExecuteLanguage::Node);
- }
-
- let response = self
- .client
- .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
- .headers(self.prepare_headers(None))
- .json(&body)
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(
- format!("Interacting with scrape browser for {}", job_id.as_ref()),
- e,
- )
- })?;
-
- self.handle_response(response, "scrape interact").await
- }
-
- /// Stops the interaction session associated with a scrape job.
- ///
- /// # Arguments
- ///
- /// * `job_id` - The scrape job ID.
- ///
- /// # Returns
- ///
- /// A `ScrapeBrowserDeleteResponse` indicating stop status.
- pub async fn stop_interaction(
- &self,
- job_id: impl AsRef<str>,
- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
- let response = self
- .client
- .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
- .headers(self.prepare_headers(None))
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(
- format!("Stopping interaction for {}", job_id.as_ref()),
- e,
- )
- })?;
-
- self.handle_response(response, "stop interaction").await
- }
-
- /// Deprecated alias for [`Client::interact`].
- #[deprecated(note = "Use interact() instead")]
- pub async fn scrape_execute(
- &self,
- job_id: impl AsRef<str>,
- options: ScrapeExecuteOptions,
- ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
- self.interact(job_id, options).await
- }
-
- /// Deprecated alias for [`Client::stop_interaction`].
- #[deprecated(note = "Use stop_interaction() instead")]
- pub async fn stop_interactive_browser(
- &self,
- job_id: impl AsRef<str>,
- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
- self.stop_interaction(job_id).await
- }
-
- /// Deprecated alias for [`Client::stop_interaction`].
- #[deprecated(note = "Use stop_interaction() instead")]
- pub async fn delete_scrape_browser(
- &self,
- job_id: impl AsRef<str>,
- ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
- self.stop_interaction(job_id).await
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- async fn test_scrape_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "markdown": "# Example Domain\n\nThis is an example.",
- "metadata": {
- "sourceURL": "https://example.com",
- "statusCode": 200,
- "title": "Example Domain"
- }
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let document = client.scrape("https://example.com", None).await.unwrap();
-
- assert!(document.markdown.is_some());
- assert!(document.markdown.unwrap().contains("Example Domain"));
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_scrape_with_options() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "markdown": "# Test",
- "html": "<h1>Test</h1>",
- "links": ["https://example.com/page1", "https://example.com/page2"],
- "metadata": {
- "sourceURL": "https://example.com",
- "statusCode": 200
- }
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let options = ScrapeOptions {
- formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
- only_main_content: Some(true),
- ..Default::default()
- };
-
- let document = client.scrape("https://example.com", options).await.unwrap();
-
- assert!(document.markdown.is_some());
- assert!(document.html.is_some());
- assert!(document.links.is_some());
- assert_eq!(document.links.unwrap().len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_scrape_with_schema() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "json": {
- "title": "Product Name",
- "price": 99.99
- },
- "metadata": {
- "sourceURL": "https://example.com/product",
- "statusCode": 200
- }
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
-
- let schema = json!({
- "type": "object",
- "properties": {
- "title": { "type": "string" },
- "price": { "type": "number" }
- }
- });
-
- let data = client
- .scrape_with_schema(
- "https://example.com/product",
- schema,
- Some("Extract product info"),
- )
- .await
- .unwrap();
-
- assert_eq!(data["title"], "Product Name");
- assert_eq!(data["price"], 99.99);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_scrape_error_response() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape")
- .with_status(400)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Invalid URL"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = client.scrape("invalid-url", None).await;
-
- assert!(result.is_err());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_interact_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape/job-123/interact")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "stdout": "ok",
- "result": "done",
- "stderr": "",
- "exitCode": 0,
- "killed": false
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client
- .interact(
- "job-123",
- ScrapeExecuteOptions {
- code: Some("console.log('ok')".to_string()),
- timeout: Some(30),
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert!(response.success);
- assert_eq!(response.exit_code, Some(0));
- assert_eq!(response.result, Some("done".to_string()));
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_interact_with_prompt() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape/job-789/interact")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "output": "Clicked the login button",
- "liveViewUrl": "https://live.example.com/view",
- "interactiveLiveViewUrl": "https://live.example.com/interactive",
- "stdout": "",
- "exitCode": 0,
- "killed": false
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client
- .interact(
- "job-789",
- ScrapeExecuteOptions {
- prompt: Some("Click the login button".to_string()),
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert!(response.success);
- assert_eq!(
- response.output,
- Some("Clicked the login button".to_string())
- );
- assert_eq!(
- response.live_view_url,
- Some("https://live.example.com/view".to_string())
- );
- assert_eq!(
- response.interactive_live_view_url,
- Some("https://live.example.com/interactive".to_string())
- );
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_stop_interaction_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("DELETE", "/v2/scrape/job-123/interact")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "sessionDurationMs": 1200,
- "creditsBilled": 3
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client.stop_interaction("job-123").await.unwrap();
-
- assert!(response.success);
- assert_eq!(response.session_duration_ms, Some(1200));
- assert_eq!(response.credits_billed, Some(3));
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_interact_error_response() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/scrape/job-404/interact")
- .with_status(404)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Job not found."
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = client
- .interact(
- "job-404",
- ScrapeExecuteOptions {
- code: Some("console.log('ok')".to_string()),
- ..Default::default()
- },
- )
- .await;
-
- assert!(result.is_err());
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/src/v2/search.rs b/apps/rust-sdk/src/v2/search.rs
deleted file mode 100644
index ccca680f2..000000000
--- a/apps/rust-sdk/src/v2/search.rs
+++ /dev/null
@@ -1,466 +0,0 @@
-//! Search endpoint for Firecrawl API v2.
-
-use serde::{Deserialize, Serialize};
-
-use super::client::Client;
-use super::scrape::ScrapeOptions;
-use super::types::{
- Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,
-};
-use crate::FirecrawlError;
-
-/// Options for search requests.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchOptions {
- /// Maximum number of results to return. Default: 5, Max: 20.
- pub limit: Option<u32>,
-
- /// Search sources to query (web, news, images).
- pub sources: Option<Vec<SearchSource>>,
-
- /// Categories to filter results (github, research, pdf).
- pub categories: Option<Vec<SearchCategory>>,
-
- /// Time-based search filter (e.g., "qdr:d" for past day).
- pub tbs: Option<String>,
-
- /// Geographic location string for local search results.
- pub location: Option<String>,
-
- /// Whether to ignore invalid URLs in results.
- pub ignore_invalid_urls: Option<bool>,
-
- /// Timeout in milliseconds.
- pub timeout: Option<u32>,
-
- /// Scrape options to apply to each search result.
- pub scrape_options: Option<ScrapeOptions>,
-
- /// Integration identifier for tracking.
- pub integration: Option<String>,
-}
-
-/// Request body for search endpoint.
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct SearchRequest {
- query: String,
- #[serde(flatten)]
- options: SearchOptions,
-}
-
-/// Search results data structure.
-#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchData {
- /// Web search results (may include scraped documents).
- pub web: Option<Vec<SearchResultOrDocument>>,
- /// News search results.
- pub news: Option<Vec<SearchResultNews>>,
- /// Image search results.
- pub images: Option<Vec<SearchResultImage>>,
-}
-
-/// A search result that may be a simple result or a full document.
-///
-/// Uses custom deserialization to properly distinguish between web results
-/// and scraped documents by checking for document-specific fields.
-#[derive(Serialize, Debug, Clone)]
-#[serde(untagged)]
-pub enum SearchResultOrDocument {
- /// Simple web search result.
- WebResult(SearchResultWeb),
- /// Full scraped document.
- Document(Document),
-}
-
-impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: serde::Deserializer<'de>,
- {
- use serde_json::Value;
-
- let value = Value::deserialize(deserializer)?;
-
- // Check for document-specific fields that indicate scraped content
- // If any of these exist, it's a Document, not a simple WebResult
- let is_document = value.get("markdown").is_some()
- || value.get("html").is_some()
- || value.get("rawHtml").is_some()
- || value.get("metadata").is_some();
-
- if is_document {
- Document::deserialize(value)
- .map(SearchResultOrDocument::Document)
- .map_err(serde::de::Error::custom)
- } else {
- SearchResultWeb::deserialize(value)
- .map(SearchResultOrDocument::WebResult)
- .map_err(serde::de::Error::custom)
- }
- }
-}
-
-/// Response from search endpoint.
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchResponse {
- /// Whether the request was successful.
- pub success: bool,
- /// Search results data.
- pub data: SearchData,
- /// Warning message if any.
- pub warning: Option<String>,
-}
-
-impl Client {
- /// Searches the web and optionally scrapes the results.
- ///
- /// # Arguments
- ///
- /// * `query` - The search query string.
- /// * `options` - Optional search configuration.
- ///
- /// # Returns
- ///
- /// A `SearchResponse` containing the search results.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::{Client, SearchOptions, SearchSource};
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// // Simple search
- /// let results = client.search("rust programming", None).await?;
- /// for result in results.data.web.unwrap_or_default() {
- /// match result {
- /// firecrawl::v2::SearchResultOrDocument::WebResult(r) => {
- /// println!("URL: {}", r.url);
- /// }
- /// firecrawl::v2::SearchResultOrDocument::Document(d) => {
- /// println!("Content: {:?}", d.markdown);
- /// }
- /// }
- /// }
- ///
- /// // Search with options
- /// let options = SearchOptions {
- /// limit: Some(10),
- /// sources: Some(vec![SearchSource::Web, SearchSource::News]),
- /// ..Default::default()
- /// };
- /// let results = client.search("rust programming", options).await?;
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn search(
- &self,
- query: impl AsRef<str>,
- options: impl Into<Option<SearchOptions>>,
- ) -> Result<SearchResponse, FirecrawlError> {
- let body = SearchRequest {
- query: query.as_ref().to_string(),
- options: options.into().unwrap_or_default(),
- };
-
- let headers = self.prepare_headers(None);
-
- let response = self
- .client
- .post(self.url("/search"))
- .headers(headers)
- .json(&body)
- .send()
- .await
- .map_err(|e| {
- FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)
- })?;
-
- self.handle_response(response, "search").await
- }
-
- /// Searches the web and scrapes the results.
- ///
- /// This is a convenience method that enables scraping for all results.
- ///
- /// # Arguments
- ///
- /// * `query` - The search query string.
- /// * `limit` - Maximum number of results to return.
- ///
- /// # Returns
- ///
- /// A vector of scraped documents.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use firecrawl::v2::Client;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
- /// let client = Client::new("your-api-key")?;
- ///
- /// let documents = client.search_and_scrape("rust programming", 5).await?;
- /// for doc in documents {
- /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));
- /// println!("Content: {:?}", doc.markdown);
- /// }
- ///
- /// Ok(())
- /// }
- /// ```
- pub async fn search_and_scrape(
- &self,
- query: impl AsRef<str>,
- limit: u32,
- ) -> Result<Vec<Document>, FirecrawlError> {
- let options = SearchOptions {
- limit: Some(limit),
- scrape_options: Some(ScrapeOptions::default()),
- ..Default::default()
- };
-
- let response = self.search(query, options).await?;
-
- let documents: Vec<Document> = response
- .data
- .web
- .unwrap_or_default()
- .into_iter()
- .filter_map(|result| match result {
- SearchResultOrDocument::Document(doc) => Some(doc),
- SearchResultOrDocument::WebResult(_) => None,
- })
- .collect();
-
- Ok(documents)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- #[tokio::test]
- async fn test_search_with_mock() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/search")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "web": [
- {
- "url": "https://example.com",
- "title": "Example Domain",
- "description": "This domain is for examples"
- },
- {
- "url": "https://example.org",
- "title": "Another Example",
- "description": "More examples"
- }
- ]
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client.search("test query", None).await.unwrap();
-
- assert!(response.success);
- let web_results = response.data.web.unwrap();
- assert_eq!(web_results.len(), 2);
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_search_with_options() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/search")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "web": [
- {
- "url": "https://example.com",
- "title": "Test Result",
- "description": "A test result"
- }
- ],
- "news": [
- {
- "title": "Breaking News",
- "url": "https://news.example.com",
- "snippet": "Something happened",
- "date": "2024-01-01"
- }
- ]
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let options = SearchOptions {
- limit: Some(10),
- sources: Some(vec![SearchSource::Web, SearchSource::News]),
- ..Default::default()
- };
-
- let response = client.search("test", options).await.unwrap();
-
- assert!(response.success);
- assert!(response.data.web.is_some());
- assert!(response.data.news.is_some());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_search_and_scrape() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/search")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "web": [
- {
- "markdown": "# Example\n\nThis is the scraped content.",
- "metadata": {
- "sourceURL": "https://example.com",
- "statusCode": 200,
- "title": "Example"
- }
- }
- ]
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let documents = client.search_and_scrape("test", 5).await.unwrap();
-
- assert_eq!(documents.len(), 1);
- assert!(documents[0].markdown.is_some());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_search_error_response() {
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/search")
- .with_status(400)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": false,
- "error": "Invalid query"
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let result = client.search("", None).await;
-
- assert!(result.is_err());
- mock.assert();
- }
-
- #[tokio::test]
- async fn test_search_mixed_results_deserialization() {
- // Test that results with markdown/metadata are correctly identified as Documents
- // and simple results are identified as WebResults
- let mut server = mockito::Server::new_async().await;
-
- let mock = server
- .mock("POST", "/v2/search")
- .with_status(200)
- .with_header("content-type", "application/json")
- .with_body(
- json!({
- "success": true,
- "data": {
- "web": [
- // Simple web result (no markdown/metadata)
- {
- "url": "https://example.com/simple",
- "title": "Simple Result",
- "description": "Just a search result"
- },
- // Scraped document with url AND markdown
- {
- "url": "https://example.com/scraped",
- "markdown": "# Scraped Content\n\nThis has content.",
- "metadata": {
- "sourceURL": "https://example.com/scraped",
- "statusCode": 200,
- "title": "Scraped Page"
- }
- }
- ]
- }
- })
- .to_string(),
- )
- .create();
-
- let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
- let response = client.search("test", None).await.unwrap();
-
- let web_results = response.data.web.unwrap();
- assert_eq!(web_results.len(), 2);
-
- // First should be WebResult
- match &web_results[0] {
- SearchResultOrDocument::WebResult(r) => {
- assert_eq!(r.url, "https://example.com/simple");
- assert_eq!(r.title, Some("Simple Result".to_string()));
- }
- SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),
- }
-
- // Second should be Document (has markdown)
- match &web_results[1] {
- SearchResultOrDocument::Document(d) => {
- assert!(d.markdown.is_some());
- assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));
- }
- SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),
- }
-
- mock.assert();
- }
-}
diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs
deleted file mode 100644
index d6a09088b..000000000
--- a/apps/rust-sdk/tests/e2e_with_auth.rs
+++ /dev/null
@@ -1,184 +0,0 @@
-use dotenvy::dotenv;
-use firecrawl::scrape::{JsonOptions, ScrapeFormats, ScrapeOptions};
-use firecrawl::{FirecrawlApp, FirecrawlError};
-use serde_json::json;
-use std::env;
-
-// #[tokio::test]
-// async fn test_unsupported_url() {
-// dotenv().ok();
-// let api_url = env::var("API_URL").unwrap();
-// let api_key = env::var("TEST_API_KEY").ok();
-// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
-// let unsupported_url = "https://facebook.com/fake-test";
-// let result = app.scrape_url(unsupported_url, None).await;
-
-// assert_matches!(
-// result,
-// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")
-// );
-// }
-
-#[tokio::test]
-async fn test_successful_response_with_valid_preview_token() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let app =
- FirecrawlApp::new_selfhosted(api_url, Some(env::var("PREVIEW_TOKEN").unwrap())).unwrap();
- let result = app
- .scrape_url("https://roastmywebsite.ai", None)
- .await
- .unwrap();
- assert!(result.markdown.is_some());
- assert!(result.markdown.unwrap().contains("_Roast_"));
-}
-
-#[tokio::test]
-async fn test_scrape_url_e2e() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let api_key = env::var("TEST_API_KEY").ok();
- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
- let result = app
- .scrape_url("https://roastmywebsite.ai", None)
- .await
- .unwrap();
- assert!(result.markdown.is_some());
- assert!(result.markdown.unwrap().contains("_Roast_"));
-}
-
-#[tokio::test]
-async fn test_successful_response_with_valid_api_key_and_include_html() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let api_key = env::var("TEST_API_KEY").ok();
- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
- let params = ScrapeOptions {
- formats: vec![ScrapeFormats::Markdown, ScrapeFormats::HTML].into(),
- ..Default::default()
- };
- let result = app
- .scrape_url("https://roastmywebsite.ai", params)
- .await
- .unwrap();
- assert!(result.markdown.is_some());
- assert!(result.html.is_some());
- assert!(result.markdown.unwrap().contains("_Roast_"));
- assert!(result.html.unwrap().contains("<h1"));
-}
-
-#[tokio::test]
-async fn test_successful_response_for_valid_scrape_with_pdf_file() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let api_key = env::var("TEST_API_KEY").ok();
- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
- let result = app
- .scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None)
- .await
- .unwrap();
- assert!(result.markdown.is_some());
- assert!(result
- .markdown
- .unwrap()
- .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
-}
-
-#[tokio::test]
-async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let api_key = env::var("TEST_API_KEY").ok();
- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
- let result = app
- .scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None)
- .await
- .unwrap();
- assert!(result.markdown.is_some());
- assert!(result
- .markdown
- .unwrap()
- .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
-}
-
-// #[tokio::test]
-// async fn test_should_return_error_for_unsupported_url() {
-// dotenv().ok();
-// let api_url = env::var("API_URL").unwrap();
-// let api_key = env::var("TEST_API_KEY").ok();
-// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
-// let unsupported_url = "https://twitter.com/fake-test";
-// let result = app.crawl_url(unsupported_url, None).await;
-
-// assert_matches!(
-// result,
-// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.")
-// );
-// }
-
-#[tokio::test]
-async fn test_llm_extraction() {
- dotenv().ok();
- let api_url = env::var("API_URL").unwrap();
- let api_key = env::var("TEST_API_KEY").ok();
- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
- let options = ScrapeOptions {
- formats: vec![ScrapeFormats::Json].into(),
- json_options: JsonOptions {
- prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source".to_string().into(),
- schema: json!({
- "type": "object",
- "properties": {
- "company_mission": {"type": "string"},
- "supports_sso": {"type": "boolean"},
- "is_open_source": {"type": "boolean"}
- },
- "required": ["company_mission", "supports_sso", "is_open_source"]
- }).into(),
- ..Default::default()
- }.into(),
- ..Default::default()
- };
- let result = app
- .scrape_url("https://mendable.ai", options)
- .await
- .unwrap();
- assert!(result.extract.is_some());
- let llm_extraction = &result.extract.unwrap();
- assert!(llm_extraction
- .as_object()
- .unwrap()
- .contains_key("company_mission"));
- assert!(llm_extraction["supports_sso"].is_boolean());
- assert!(llm_extraction["is_open_source"].is_boolean());
-}
-
-#[test]
-fn test_api_key_requirements() {
- dotenv().ok();
-
- let api_url = env::var("API_URL").unwrap_or("http://localhost:3002".to_string());
- let api_key = env::var("TEST_API_KEY").ok();
-
- match (api_url.contains("api.firecrawl.dev"), api_key) {
- (false, _) => {
- let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);
- assert!(
- result.is_ok(),
- "Local setup failed: {:?}",
- result.err().unwrap()
- );
- }
- (true, None) => {
- let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);
- assert!(matches!(
- result,
- Err(FirecrawlError::APIError(msg, _)) if msg == "Configuration"
- ));
- }
- (true, Some(key)) => {
- let result = FirecrawlApp::new_selfhosted(&api_url, Some(&key));
- assert!(result.is_ok());
- }
- }
-}
diff --git a/apps/rust-sdk/tests/v2_e2e.rs b/apps/rust-sdk/tests/v2_e2e.rs
index f91398206..619be32ec 100644
--- a/apps/rust-sdk/tests/v2_e2e.rs
+++ b/apps/rust-sdk/tests/v2_e2e.rs
@@ -1,4 +1,4 @@
-//! End-to-end tests for Firecrawl v2 API.
+//! End-to-end tests for Firecrawl API.
//!
//! These tests require the following environment variables:
//! - API_URL: The Firecrawl API URL
@@ -6,16 +6,14 @@
//!
//! Run with: cargo test --test v2_e2e -- --ignored
-use dotenvy::dotenv;
-use firecrawl::v2::{
- AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions, ScrapeOptions,
- SearchOptions, SitemapMode,
+use firecrawl::{
+ AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, JobStatus, MapOptions,
+ ScrapeOptions, SearchOptions, SitemapMode,
};
use serde_json::json;
use std::env;
fn get_client() -> Client {
- dotenv().ok();
let api_url = env::var("API_URL").expect("API_URL environment variable is required");
let api_key = env::var("TEST_API_KEY").ok();
Client::new_selfhosted(api_url, api_key).expect("Failed to create client")
@@ -23,7 +21,7 @@ fn get_client() -> Client {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_scrape() {
+async fn test_scrape() {
let client = get_client();
let doc = client
.scrape("https://example.com", None)
@@ -35,7 +33,7 @@ async fn test_v2_scrape() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_scrape_with_options() {
+async fn test_scrape_with_options() {
let client = get_client();
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
@@ -54,7 +52,7 @@ async fn test_v2_scrape_with_options() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_scrape_with_schema() {
+async fn test_scrape_with_schema() {
let client = get_client();
let schema = json!({
@@ -75,7 +73,7 @@ async fn test_v2_scrape_with_schema() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_search() {
+async fn test_search() {
let client = get_client();
let response = client
.search("rust programming", None)
@@ -87,7 +85,7 @@ async fn test_v2_search() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_search_with_options() {
+async fn test_search_with_options() {
let client = get_client();
let options = SearchOptions {
limit: Some(5),
@@ -104,7 +102,7 @@ async fn test_v2_search_with_options() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_map() {
+async fn test_map() {
let client = get_client();
let response = client
.map("https://example.com", None)
@@ -116,7 +114,7 @@ async fn test_v2_map() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_map_with_options() {
+async fn test_map_with_options() {
let client = get_client();
let options = MapOptions {
sitemap: Some(SitemapMode::Include),
@@ -134,7 +132,7 @@ async fn test_v2_map_with_options() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_crawl_async() {
+async fn test_crawl_async() {
let client = get_client();
let response = client
.start_crawl("https://example.com", None)
@@ -150,10 +148,7 @@ async fn test_v2_crawl_async() {
.await
.expect("Get crawl status should succeed");
assert!(
- matches!(
- status.status,
- firecrawl::v2::JobStatus::Scraping | firecrawl::v2::JobStatus::Completed
- ),
+ matches!(status.status, JobStatus::Scraping | JobStatus::Completed),
"Status should be scraping or completed"
);
@@ -163,7 +158,7 @@ async fn test_v2_crawl_async() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_crawl_sync() {
+async fn test_crawl_sync() {
let client = get_client();
let options = CrawlOptions {
limit: Some(2),
@@ -177,14 +172,14 @@ async fn test_v2_crawl_sync() {
.expect("Crawl should succeed");
assert!(
- job.status == firecrawl::v2::JobStatus::Completed,
+ job.status == JobStatus::Completed,
"Job should be completed"
);
}
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_batch_scrape_async() {
+async fn test_batch_scrape_async() {
let client = get_client();
let urls = vec![
"https://example.com".to_string(),
@@ -202,7 +197,7 @@ async fn test_v2_batch_scrape_async() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_batch_scrape_sync() {
+async fn test_batch_scrape_sync() {
let client = get_client();
let urls = vec!["https://example.com".to_string()];
@@ -221,14 +216,14 @@ async fn test_v2_batch_scrape_sync() {
.expect("Batch scrape should succeed");
assert!(
- job.status == firecrawl::v2::JobStatus::Completed,
+ job.status == JobStatus::Completed,
"Job should be completed"
);
}
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_agent_async() {
+async fn test_agent_async() {
let client = get_client();
let options = AgentOptions {
urls: Some(vec!["https://example.com".to_string()]),
@@ -247,7 +242,7 @@ async fn test_v2_agent_async() {
#[tokio::test]
#[ignore = "Requires API access"]
-async fn test_v2_agent_with_schema() {
+async fn test_agent_with_schema() {
let client = get_client();
#[derive(Debug, serde::Deserialize)]
@@ -280,10 +275,10 @@ async fn test_v2_agent_with_schema() {
}
}
-// Test that the v2 client can be created with different configurations
+// Test that the client can be created with different configurations
// This test doesn't require API access
#[test]
-fn test_v2_client_creation() {
+fn test_client_creation() {
// Cloud client requires API key
let cloud_result = Client::new("test-key");
assert!(cloud_result.is_ok());