Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(rust): update rust sdk to support new features (#1446)

kkharjif2c01340d19b
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 2ea5de692..d7a7b64e4 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -17,6 +17,21 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -32,6 +47,71 @@ dependencies = [
"libc",
]
+[[package]]
+name = "ansi_colours"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14eec43e0298190790f41679fe69ef7a829d2a2ddd78c8c00339e84710e435fe"
+dependencies = [
+ "rgb",
+]
+
+[[package]]
+name = "anstream"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
+dependencies = [
+ "anstyle",
+ "once_cell",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.97"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
+
[[package]]
name = "arrayref"
version = "0.3.7"
@@ -44,6 +124,16 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
+[[package]]
+name = "assert-json-diff"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "assert_matches"
version = "1.5.0"
@@ -62,6 +152,72 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+[[package]]
+name = "axum"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288"
+dependencies = [
+ "axum-core",
+ "axum-macros",
+ "bytes",
+ "form_urlencoded",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "serde_json",
+ "serde_path_to_error",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "tokio",
+ "tower 0.5.2",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "axum-macros"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "backtrace"
version = "0.3.73"
@@ -72,7 +228,7 @@ dependencies = [
"cc",
"cfg-if",
"libc",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
"object",
"rustc-demangle",
]
@@ -89,6 +245,59 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "bat"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ab792c2ad113a666f08856c88cdec0a62d732559b1f3982eedf0142571e669a"
+dependencies = [
+ "ansi_colours",
+ "anyhow",
+ "bincode",
+ "bugreport",
+ "bytesize",
+ "clap",
+ "clircle",
+ "console",
+ "content_inspector",
+ "encoding_rs",
+ "etcetera",
+ "flate2",
+ "git2",
+ "globset",
+ "grep-cli",
+ "home",
+ "indexmap 2.9.0",
+ "itertools",
+ "nu-ansi-term",
+ "once_cell",
+ "path_abs",
+ "plist",
+ "regex",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_with",
+ "serde_yaml",
+ "shell-words",
+ "syntect",
+ "terminal-colorsaurus",
+ "thiserror",
+ "toml",
+ "unicode-width 0.1.14",
+ "walkdir",
+ "wild",
+]
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -112,12 +321,40 @@ dependencies = [
"constant_time_eq",
]
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
+[[package]]
+name = "bugreport"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f280f65ce85b880919349bbfcb204930291251eedcb2e5f84ce2f51df969c162"
+dependencies = [
+ "git-version",
+ "shell-escape",
+ "sysinfo",
+]
+
[[package]]
name = "bumpalo"
version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+[[package]]
+name = "bytemuck"
+version = "1.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540"
+
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -130,11 +367,22 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+[[package]]
+name = "bytesize"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
+
[[package]]
name = "cc"
version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437"
+dependencies = [
+ "jobserver",
+ "libc",
+ "once_cell",
+]
[[package]]
name = "cfg-if"
@@ -155,6 +403,47 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "clap"
+version = "4.5.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+ "terminal_size",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.5.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+
[[package]]
name = "clippy"
version = "0.0.302"
@@ -164,12 +453,59 @@ dependencies = [
"term",
]
+[[package]]
+name = "clircle"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d9334f725b46fb9bed8580b9b47a932587e044fadb344ed7fa98774b067ac1a"
+dependencies = [
+ "cfg-if",
+ "windows 0.56.0",
+]
+
+[[package]]
+name = "colorchoice"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+
+[[package]]
+name = "colored"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width 0.2.0",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+[[package]]
+name = "content_inspector"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -182,9 +518,37 @@ dependencies = [
[[package]]
name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "crc32fast"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
[[package]]
name = "crossbeam-utils"
@@ -254,11 +618,29 @@ version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
+[[package]]
+name = "dyn-clone"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
+
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "encoding_rs"
-version = "0.8.34"
+version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
@@ -279,6 +661,17 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "etcetera"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
+dependencies = [
+ "cfg-if",
+ "home",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "fastrand"
version = "2.1.0"
@@ -287,13 +680,19 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
[[package]]
name = "firecrawl"
-version = "1.0.0"
+version = "1.1.0"
dependencies = [
"assert_matches",
+ "axum",
+ "bat",
+ "clap",
"clippy",
"dotenvy",
+ "futures",
"log",
+ "mockito",
"reqwest",
+ "schemars",
"serde",
"serde_json",
"serde_with",
@@ -302,6 +701,16 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "flate2"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide 0.8.8",
+]
+
[[package]]
name = "fnv"
version = "1.0.7"
@@ -332,11 +741,26 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-channel"
-version = "0.3.30"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
@@ -344,36 +768,60 @@ dependencies = [
[[package]]
name = "futures-core"
-version = "0.3.30"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
[[package]]
name = "futures-io"
-version = "0.3.30"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
[[package]]
name = "futures-sink"
-version = "0.3.30"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
[[package]]
name = "futures-task"
-version = "0.3.30"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
[[package]]
name = "futures-util"
-version = "0.3.30"
+version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
+ "futures-channel",
"futures-core",
"futures-io",
+ "futures-macro",
"futures-sink",
"futures-task",
"memchr",
@@ -386,29 +834,107 @@ dependencies = [
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi 0.14.2+wasi-0.2.4",
+]
+
+[[package]]
+name = "gimli"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+
+[[package]]
+name = "git-version"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19"
+dependencies = [
+ "git-version-macro",
+]
+
+[[package]]
+name = "git-version-macro"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "git2"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"
dependencies = [
- "cfg-if",
+ "bitflags 2.6.0",
"libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
+ "libgit2-sys",
+ "log",
+ "url",
]
[[package]]
-name = "getrandom"
-version = "0.2.15"
+name = "glob"
+version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+
+[[package]]
+name = "globset"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
]
[[package]]
-name = "gimli"
-version = "0.29.0"
+name = "grep-cli"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+checksum = "47f1288f0e06f279f84926fa4c17e3fcd2a22b357927a82f2777f7be26e4cec0"
+dependencies = [
+ "bstr",
+ "globset",
+ "libc",
+ "log",
+ "termcolor",
+ "winapi-util",
+]
[[package]]
name = "h2"
@@ -422,7 +948,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http",
- "indexmap 2.2.6",
+ "indexmap 2.9.0",
"slab",
"tokio",
"tokio-util",
@@ -437,9 +963,15 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
-version = "0.14.5"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
@@ -453,6 +985,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+[[package]]
+name = "home"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "http"
version = "1.1.0"
@@ -493,11 +1034,17 @@ version = "1.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
[[package]]
name = "hyper"
-version = "1.4.1"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
dependencies = [
"bytes",
"futures-channel",
@@ -506,6 +1053,7 @@ dependencies = [
"http",
"http-body",
"httparse",
+ "httpdate",
"itoa",
"pin-project-lite",
"smallvec",
@@ -561,7 +1109,7 @@ dependencies = [
"pin-project-lite",
"socket2",
"tokio",
- "tower",
+ "tower 0.4.13",
"tower-service",
"tracing",
]
@@ -577,7 +1125,7 @@ dependencies = [
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
- "windows-core",
+ "windows-core 0.52.0",
]
[[package]]
@@ -618,12 +1166,12 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.2.6"
+version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
dependencies = [
"equivalent",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
"serde",
]
@@ -633,12 +1181,37 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+[[package]]
+name = "jobserver"
+version = "0.1.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
+dependencies = [
+ "getrandom 0.3.2",
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.69"
@@ -650,9 +1223,39 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.155"
+version = "0.2.171"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
+
+[[package]]
+name = "libgit2-sys"
+version = "0.17.0+1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224"
+dependencies = [
+ "cc",
+ "libc",
+ "libz-sys",
+ "pkg-config",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
@@ -676,6 +1279,12 @@ version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+[[package]]
+name = "matchit"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
+
[[package]]
name = "memchr"
version = "2.7.4"
@@ -697,6 +1306,15 @@ dependencies = [
"adler",
]
+[[package]]
+name = "miniz_oxide"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
+dependencies = [
+ "adler2",
+]
+
[[package]]
name = "mio"
version = "1.0.2"
@@ -709,6 +1327,30 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "mockito"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"
+dependencies = [
+ "assert-json-diff",
+ "bytes",
+ "colored",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "log",
+ "rand",
+ "regex",
+ "serde_json",
+ "serde_urlencoded",
+ "similar",
+ "tokio",
+]
+
[[package]]
name = "native-tls"
version = "0.2.12"
@@ -726,6 +1368,24 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "nu-ansi-term"
+version = "0.50.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "num-conv"
version = "0.1.0"
@@ -752,9 +1412,31 @@ dependencies = [
[[package]]
name = "once_cell"
-version = "1.19.0"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
[[package]]
name = "openssl"
@@ -823,6 +1505,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "path_abs"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3"
+dependencies = [
+ "std_prelude",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -867,12 +1558,34 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+[[package]]
+name = "plist"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac26e981c03a6e53e0aee43c113e3202f5581d5360dae7bd2c70e800dd0451d"
+dependencies = [
+ "base64 0.22.1",
+ "indexmap 2.9.0",
+ "quick-xml",
+ "serde",
+ "time",
+]
+
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.86"
@@ -882,6 +1595,15 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "quick-xml"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d3a6e5838b60e0e8fa7a43f22ade549a37d61f8bdbe636d0d7816191de969c2"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "quote"
version = "1.0.36"
@@ -891,6 +1613,62 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
+[[package]]
+name = "rand"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+ "zerocopy",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom 0.3.2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
[[package]]
name = "redox_syscall"
version = "0.1.57"
@@ -917,6 +1695,35 @@ dependencies = [
"rust-argon2",
]
+[[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
+
[[package]]
name = "reqwest"
version = "0.12.5"
@@ -961,6 +1768,15 @@ dependencies = [
"winreg",
]
+[[package]]
+name = "rgb"
+version = "0.8.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"
+dependencies = [
+ "bytemuck",
+]
+
[[package]]
name = "ring"
version = "0.17.8"
@@ -1047,12 +1863,27 @@ dependencies = [
"untrusted",
]
+[[package]]
+name = "rustversion"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
+
[[package]]
name = "ryu"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
[[package]]
name = "schannel"
version = "0.1.23"
@@ -1062,6 +1893,30 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "schemars"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
+dependencies = [
+ "dyn-clone",
+ "schemars_derive",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars_derive"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn",
+]
+
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -1091,6 +1946,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
[[package]]
name = "serde"
version = "1.0.204"
@@ -1112,13 +1973,43 @@ dependencies = [
]
[[package]]
-name = "serde_json"
-version = "1.0.120"
+name = "serde_derive_internals"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.120"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"
+dependencies = [
+ "itoa",
+ "serde",
+]
+
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
dependencies = [
- "itoa",
- "ryu",
"serde",
]
@@ -1136,15 +2027,15 @@ dependencies = [
[[package]]
name = "serde_with"
-version = "3.9.0"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857"
+checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.2.6",
+ "indexmap 2.9.0",
"serde",
"serde_derive",
"serde_json",
@@ -1154,9 +2045,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
-version = "3.9.0"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350"
+checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
dependencies = [
"darling",
"proc-macro2",
@@ -1164,6 +2055,31 @@ dependencies = [
"syn",
]
+[[package]]
+name = "serde_yaml"
+version = "0.9.34+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+dependencies = [
+ "indexmap 2.9.0",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
+[[package]]
+name = "shell-escape"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
+
+[[package]]
+name = "shell-words"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
+
[[package]]
name = "signal-hook-registry"
version = "1.4.2"
@@ -1173,6 +2089,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "similar"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+
[[package]]
name = "slab"
version = "0.4.9"
@@ -1204,6 +2126,12 @@ version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+[[package]]
+name = "std_prelude"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe"
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -1233,6 +2161,42 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+[[package]]
+name = "syntect"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
+dependencies = [
+ "bincode",
+ "bitflags 1.3.2",
+ "flate2",
+ "fnv",
+ "once_cell",
+ "onig",
+ "plist",
+ "regex-syntax",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "walkdir",
+ "yaml-rust",
+]
+
+[[package]]
+name = "sysinfo"
+version = "0.33.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+ "memchr",
+ "ntapi",
+ "rayon",
+ "windows 0.57.0",
+]
+
[[package]]
name = "system-configuration"
version = "0.5.1"
@@ -1277,6 +2241,51 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "terminal-colorsaurus"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7afe4c174a3cbfb52ebcb11b28965daf74fe9111d4e07e40689d05af06e26e8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "memchr",
+ "mio",
+ "terminal-trx",
+ "windows-sys 0.59.0",
+ "xterm-color",
+]
+
+[[package]]
+name = "terminal-trx"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "975b4233aefa1b02456d5e53b22c61653c743e308c51cf4181191d8ce41753ab"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "terminal_size"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9"
+dependencies = [
+ "rustix",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "thiserror"
version = "1.0.61"
@@ -1345,9 +2354,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.42.0"
+version = "1.44.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
+checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
dependencies = [
"backtrace",
"bytes",
@@ -1363,9 +2372,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.4.0"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
@@ -1406,6 +2415,41 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "indexmap 2.9.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
+dependencies = [
+ "indexmap 2.9.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.4.13"
@@ -1421,17 +2465,33 @@ dependencies = [
"tower-service",
]
+[[package]]
+name = "tower"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
@@ -1439,6 +2499,7 @@ version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
+ "log",
"pin-project-lite",
"tracing-core",
]
@@ -1479,6 +2540,24 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "unicode-width"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
+
+[[package]]
+name = "unicode-width"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -1496,6 +2575,12 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
[[package]]
name = "uuid"
version = "1.10.0"
@@ -1511,6 +2596,16 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
[[package]]
name = "want"
version = "0.3.1"
@@ -1532,6 +2627,15 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+[[package]]
+name = "wasi"
+version = "0.14.2+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
@@ -1608,6 +2712,15 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "wild"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1"
+dependencies = [
+ "glob",
+]
+
[[package]]
name = "winapi"
version = "0.3.9"
@@ -1624,12 +2737,41 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+[[package]]
+name = "winapi-util"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+[[package]]
+name = "windows"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132"
+dependencies = [
+ "windows-core 0.56.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
+dependencies = [
+ "windows-core 0.57.0",
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-core"
version = "0.52.0"
@@ -1639,6 +2781,83 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-core"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6"
+dependencies = [
+ "windows-implement 0.56.0",
+ "windows-interface 0.56.0",
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
+dependencies = [
+ "windows-implement 0.57.0",
+ "windows-interface 0.57.0",
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-sys"
version = "0.48.0"
@@ -1657,6 +2876,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-targets"
version = "0.48.5"
@@ -1778,6 +3006,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "winnow"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "winreg"
version = "0.52.0"
@@ -1788,6 +3025,50 @@ dependencies = [
"windows-sys 0.48.0",
]
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags 2.6.0",
+]
+
+[[package]]
+name = "xterm-color"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4de5f056fb9dc8b7908754867544e26145767187aaac5a98495e88ad7cb8a80f"
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.8.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "zeroize"
version = "1.8.1"
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 6ea8d1790..3affd8640 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,13 +1,13 @@
[package]
name = "firecrawl"
author= "Mendable.ai"
-version = "1.0.0"
+version = "1.1.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
repository ="https://github.com/mendableai/firecrawl"
description = "Rust SDK for Firecrawl API."
-authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>"]
+authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>", "kkharji <[email protected]>"]
[lib]
path = "src/lib.rs"
@@ -23,12 +23,18 @@ log = "^0.4"
thiserror = "^1.0"
uuid = { version = "^1.10", features = ["v4"] }
tokio = { version = "^1", features = ["full"] }
+futures = "0.3.31"
+schemars = "0.8.22"
[dev-dependencies]
clippy = "^0.0.302"
assert_matches = "^1.5"
dotenvy = "^0.15"
tokio = { version = "1", features = ["full"] }
+mockito = "1.7.0"
+clap = { version ="4.5.35", features = ["derive"] }
+axum = { version = "0.8.3", features = ["tokio", "macros"] }
+bat = "0.25.0"
[build-dependencies]
tokio = { version = "1", features = ["full"] }
diff --git a/apps/rust-sdk/examples/batch_scrape_example.rs b/apps/rust-sdk/examples/batch_scrape_example.rs
new file mode 100644
index 000000000..7db7856ec
--- /dev/null
+++ b/apps/rust-sdk/examples/batch_scrape_example.rs
@@ -0,0 +1,175 @@
+use clap::{Parser, Subcommand};
+use firecrawl::{
+ batch_scrape::{BatchScrapeParams, WebhookOptions},
+ map::MapOptions,
+ scrape::{ScrapeFormats, ScrapeOptions},
+ FirecrawlApp,
+};
+use serde_json::Value;
+use std::error::Error;
+use std::net::SocketAddr;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+// Store webhook responses
+struct WebhookState {
+ responses: Vec<Value>,
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ #[command(subcommand)]
+ command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+ /// Multiple URL scraping with webhook monitoring
+ Basic,
+}
+
+async fn create_firecrawl_app() -> Result<FirecrawlApp, Box<dyn Error>> {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ FirecrawlApp::new_selfhosted(api_url, None::<&str>).map_err(|e| e.into())
+}
+
+// Start webhook server and return its address
+async fn start_webhook_server(
+ port: u16,
+ state: Arc<Mutex<WebhookState>>,
+) -> Result<String, Box<dyn Error>> {
+ let state = state.clone();
+ use axum::routing::post;
+ use axum::Json;
+
+ let app = axum::Router::new().route(
+ "/",
+ post(move |body: Json<Value>| {
+ let state = state.clone();
+ async move {
+ state.lock().await.responses.push(body.0.clone());
+ match serde_json::to_string_pretty(&body.0) {
+ Ok(data) => println!(
+ "Received webhook: {}",
+ serde_json::to_string_pretty(&data).unwrap()
+ ),
+ Err(_) => println!("Received webhook: {}", body.0),
+ }
+ "OK"
+ }
+ }),
+ );
+
+ let addr = SocketAddr::from(([0, 0, 0, 0], port));
+ let webhook_url = format!("http://host.docker.internal:{}", port);
+
+ tokio::spawn(async move {
+ let listener = tokio::net::TcpListener::bind(addr)
+ .await
+ .inspect_err(|err| println!("{err:?}"))
+ .unwrap();
+
+ if let Err(e) = axum::serve(listener, app).await {
+ eprintln!("Webhook server error: {}", e);
+ }
+ });
+
+ println!("Webhook server running at {}", webhook_url);
+
+ Ok(webhook_url)
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ let cli = Cli::parse();
+ let firecrawl = create_firecrawl_app().await?;
+
+ let state = Arc::new(Mutex::new(WebhookState { responses: vec![] }));
+ let webhook_url = start_webhook_server(39120, state.clone()).await?;
+
+ match cli.command {
+ Commands::Basic => {
+ let mut urls = Vec::new();
+
+ let url_one = "https://invalid-url.url/";
+ println!("Mapping: {}", url_one);
+ match firecrawl.map_url(url_one, None).await {
+ Ok(mapped_urls) => urls.extend(mapped_urls),
+ Err(e) => println!("Error mapping {}: {}", url_one, e),
+ }
+
+ let url_two = "https://www.devjobsscanner.com";
+ println!("Mapping: {}", url_two);
+ match firecrawl
+ .map_url(
+ url_two,
+ Some(MapOptions {
+ search: Some("rust".into()),
+ limit: Some(20),
+ ..Default::default()
+ }),
+ )
+ .await
+ {
+ Ok(mapped_urls) => urls.extend(mapped_urls),
+ Err(e) => println!("Error mapping {}: {}", url_two, e),
+ }
+
+ test_multiple_urls(&firecrawl, urls, &webhook_url).await?;
+
+ // Give time for webhooks to arrive
+ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+ println!(
+ "Received {} webhook responses",
+ state.lock().await.responses.len()
+ );
+ }
+ }
+
+ Ok(())
+}
+
+async fn test_multiple_urls(
+ app: &FirecrawlApp,
+ urls: Vec<String>,
+ webhook_url: &str,
+) -> Result<(), Box<dyn Error>> {
+ println!("Testing batch scraping of {} URLs", urls.len());
+
+ let webhook = WebhookOptions {
+ url: webhook_url.to_string(),
+ headers: None,
+ auth_token: None,
+ };
+
+ let params = BatchScrapeParams {
+ urls,
+ webhook: Some(webhook),
+ ignore_invalid_urls: true,
+ options: Some(ScrapeOptions {
+ formats: Some(vec![ScrapeFormats::Markdown, ScrapeFormats::Links]),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let batch = app.async_batch_scrape_urls(params).await?;
+ println!("Batch job started: {}", batch.id);
+
+ // Poll status periodically
+ loop {
+ let status = app.check_batch_scrape_status(&batch.id).await?;
+ println!("Progress: {}/{} pages", status.completed, status.total);
+
+ if status.completed >= status.total {
+ println!("Batch job completed!");
+ break;
+ }
+
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ }
+
+ Ok(())
+}
diff --git a/apps/rust-sdk/examples/cancel_crawl_example.rs b/apps/rust-sdk/examples/cancel_crawl_example.rs
new file mode 100644
index 000000000..3451cc104
--- /dev/null
+++ b/apps/rust-sdk/examples/cancel_crawl_example.rs
@@ -0,0 +1,33 @@
+use firecrawl::FirecrawlApp;
+use std::error::Error;
+use std::time::Duration;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ // Get API URL from environment
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+
+ // Create the FirecrawlApp instance
+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
+
+ // Start a crawl job
+ println!("Starting a crawl job...");
+ let crawl_response = firecrawl
+ .crawl_url_async("https://example.com", None)
+ .await?;
+ println!("Crawl job started with ID: {}", crawl_response.id);
+
+ // Wait for a moment to let the crawl job start
+ println!("Waiting for a moment...");
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ // Cancel the crawl job
+ println!("Cancelling the crawl job...");
+ let cancel_response = firecrawl.cancel_crawl(&crawl_response.id).await?;
+
+ println!("Cancellation result:");
+ println!(" Status: {:?}", cancel_response.status);
+
+ Ok(())
+}
diff --git a/apps/rust-sdk/examples/check_crawl_errors_example.rs b/apps/rust-sdk/examples/check_crawl_errors_example.rs
new file mode 100644
index 000000000..7629d36cb
--- /dev/null
+++ b/apps/rust-sdk/examples/check_crawl_errors_example.rs
@@ -0,0 +1,59 @@
+use firecrawl::FirecrawlApp;
+use std::error::Error;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ // Get API URL from environment
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+
+ // Create the FirecrawlApp instance
+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
+
+ // Start a crawl job that will likely have some errors (invalid URL format)
+ println!("Starting a crawl job...");
+ let crawl_response = firecrawl
+ .crawl_url_async("https://no-wer-agg.invalid", None)
+ .await?;
+ println!("Crawl job started with ID: {}", crawl_response.id);
+
+ println!("Let it do it's thing...");
+ tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+
+ // Check the crawl errors
+ println!("Checking for crawl errors...");
+ match firecrawl.check_crawl_errors(&crawl_response.id).await {
+ Ok(error_response) => {
+ println!("Crawl errors response:");
+ println!(" Number of errors: {}", error_response.errors.len());
+
+ if !error_response.errors.is_empty() {
+ println!("\nDetailed errors:");
+ for (i, error) in error_response.errors.iter().enumerate() {
+ println!("Error #{}", i + 1);
+ println!(" ID: {}", error.id);
+ if let Some(timestamp) = &error.timestamp {
+ println!(" Timestamp: {}", timestamp);
+ }
+ println!(" URL: {}", error.url);
+ println!(" Error: {}", error.error);
+ }
+ }
+
+ println!(
+ "\nRobots.txt blocked URLs: {}",
+ error_response.robots_blocked.len()
+ );
+ for (i, url) in error_response.robots_blocked.iter().enumerate() {
+ println!(" {}. {}", i + 1, url);
+ }
+ }
+ Err(e) => {
+ println!("Failed to check crawl errors: {}", e);
+ }
+ }
+ let cancel = firecrawl.cancel_crawl(&crawl_response.id).await?;
+ println!("Cancel: {}", cancel.status);
+
+ Ok(())
+}
diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs
index 0dcb0d466..51592f3f1 100644
--- a/apps/rust-sdk/examples/example.rs
+++ b/apps/rust-sdk/examples/example.rs
@@ -1,4 +1,8 @@
-use firecrawl::{crawl::CrawlOptions, scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions}, FirecrawlApp};
+use firecrawl::{
+ crawl::CrawlOptions,
+ scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions},
+ FirecrawlApp,
+};
use serde_json::json;
#[tokio::main]
@@ -19,19 +23,20 @@ async fn main() {
// Crawl a website
let crawl_options = CrawlOptions {
- exclude_paths: vec![ "blog/*".into() ].into(),
+ exclude_paths: vec!["blog/*".into()].into(),
..Default::default()
};
-
- let crawl_result = app
- .crawl_url("https://mendable.ai", crawl_options)
- .await;
+
+ let crawl_result = app.crawl_url("https://mendable.ai", crawl_options).await;
match crawl_result {
- Ok(data) => println!("Crawl Result (used {} credits):\n{:#?}", data.credits_used, data.data),
+ Ok(data) => println!(
+ "Crawl Result (used {} credits):\n{:#?}",
+ data.credits_used, data.data
+ ),
Err(e) => eprintln!("Crawl failed: {}", e),
}
-
+
// Scrape with Extract
let json_schema = json!({
"type": "object",
@@ -57,11 +62,12 @@ async fn main() {
});
let llm_extraction_options = ScrapeOptions {
- formats: vec![ ScrapeFormats::Extract ].into(),
+ formats: vec![ScrapeFormats::Extract].into(),
extract: ExtractOptions {
schema: json_schema.into(),
..Default::default()
- }.into(),
+ }
+ .into(),
..Default::default()
};
@@ -75,9 +81,7 @@ async fn main() {
}
// Map a website (Alpha)
- let map_result = app
- .map_url("https://firecrawl.dev", None)
- .await;
+ let map_result = app.map_url("https://firecrawl.dev", None).await;
match map_result {
Ok(data) => println!("Mapped URLs: {:#?}", data),
diff --git a/apps/rust-sdk/examples/extract_example.rs b/apps/rust-sdk/examples/extract_example.rs
new file mode 100644
index 000000000..47f446ae3
--- /dev/null
+++ b/apps/rust-sdk/examples/extract_example.rs
@@ -0,0 +1,237 @@
+use firecrawl::{extract::ExtractParams, FirecrawlApp};
+use serde_json::json;
+use std::error::Error;
+
+use clap::{Parser, ValueEnum};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+ #[arg(value_enum)]
+ command: Examples,
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
+enum Examples {
+ Basic,
+ Schema,
+ JsonSchema,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ let args = Args::parse();
+
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
+ let urls = vec![
+ "https://www.firecrawl.dev/".to_string(),
+ "https://betteruptime.com".to_string(),
+ ];
+
+ match args.command {
+ Examples::Basic => {
+ println!("Example 1: Extracting with URLs and prompt");
+
+ let extract_params = ExtractParams {
+ prompt: Some(
+ "Extract Product promise, consice descirption and category".to_string(),
+ ),
+ url_trace: Some(true),
+ ..Default::default()
+ };
+
+ println!("Starting asynchronous extraction job...");
+ let response = firecrawl
+ .async_extract(ExtractParams {
+ urls: Some(urls.iter().map(|u| u.to_string()).collect()),
+ prompt: extract_params.prompt.clone(),
+ url_trace: extract_params.url_trace,
+ ..Default::default()
+ })
+ .await?;
+
+ println!("Extract job initiated:");
+ println!(" Job ID: {}", response.id);
+
+ println!("\nChecking extract status...");
+ for _ in 0..5 {
+ let response = firecrawl.get_extract_status(&response.id).await?;
+
+ println!("Extract status: {}", response.status);
+ if let Some(url_trace) = &response.url_trace {
+ println!("URL traces:");
+ for trace in url_trace {
+ println!(" URL: {}", trace.url);
+ println!(" Status: {}", trace.status);
+ }
+ }
+ println!("Extract data: {:#?}", response.data);
+ if response.status == "completed" {
+ break;
+ }
+
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ }
+ }
+ Examples::Schema => {
+ println!("Example 2: Extracting with schema");
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "category": { "type": "string" },
+ "promise": { "type": "string" },
+ "descirption": { "type": "string" }
+ },
+ "required": ["category", "promise", "description"]
+ });
+
+ println!("Starting synchronous extraction job...");
+
+ match firecrawl
+ .extract(ExtractParams {
+ urls: urls.into(),
+ schema: Some(schema),
+ ..Default::default()
+ })
+ .await
+ {
+ Ok(result) => {
+ println!("Extraction completed successfully!");
+ println!("Status: {}", result.status);
+
+ if let Some(data) = result.data {
+ println!("\nExtracted data:");
+ println!(" Title: {}", data["title"]);
+ if let Some(desc) = data.get("description") {
+ println!(" Description: {}", desc);
+ }
+ println!(
+ " Content (preview): {:.100}...",
+ data["content"].as_str().unwrap_or("N/A")
+ );
+ }
+
+ if let Some(sources) = result.sources {
+ println!("\nSources:");
+ for (field, urls) in sources {
+ println!(" {}: {}", field, urls.join(", "));
+ }
+ }
+ }
+ Err(e) => {
+ println!("Extraction failed: {}", e);
+ }
+ }
+ }
+ Examples::JsonSchema => {
+ println!("Example 3: Using JsonSchema derive");
+
+ /// A comprehensive analysis of given product
+ #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
+ struct ProductAnalysis {
+ /// The full name of the product
+ product_name: String,
+ /// The company/brand behind the product
+ brand: String,
+ /// The general price range (e.g. "Premium", "$10-50", "Enterprise")
+ price_range: String,
+ /// The main customer segments this product targets
+ target_audience: Vec<String>,
+ /// Primary benefits and value propositions of the product
+ key_benefits: Vec<String>,
+ /// Distinctive features that set this product apart from competitors
+ unique_selling_points: Vec<String>,
+ /// Direct comparisons with competing products/services
+ competitor_comparison: Vec<String>,
+ /// Technologies, frameworks, or platforms used (if applicable)
+ tech_stack: Option<Vec<String>>,
+ /// Aggregated review data and sentiment analysis
+ reviews_summary: ReviewsSummary,
+ // /// Score from 0-10 indicating product-market fit based on analysis
+ // market_fit_score: f32, // NOTE: Breaks
+ /// Assessment of future growth prospects (e.g. "High", "Moderate", "Limited")
+ growth_potential: String,
+ /// Relevant compliance standards and certifications
+ regulatory_compliance: Option<Vec<String>>,
+ }
+
+ /// Aggregated analysis of product reviews from multiple sources
+ #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
+ struct ReviewsSummary {
+ /// Overall sentiment from review analysis (e.g. "Highly Positive", "Mixed", "Negative")
+ sentiment_analysis: String,
+ /// Most frequently mentioned positive aspects
+ common_praises: Vec<String>,
+ /// Most frequently mentioned criticisms or issues
+ common_complaints: Vec<String>,
+ /// Platforms or websites where reviews were sourced from
+ review_sources: Vec<String>,
+ }
+ println!("Starting extraction with derived schema...");
+ match firecrawl
+ .extract_with_schemars::<ProductAnalysis>(ExtractParams {
+ urls: urls.into(),
+ ..Default::default()
+ })
+ .await
+ {
+ Ok(result) => {
+ println!("Extraction completed!");
+ println!("Status: {}", result.status);
+
+ if let Some(data) = result.data {
+ if let Ok(analysis) = serde_json::from_value::<ProductAnalysis>(data) {
+ println!("\nExtracted Product Analysis:");
+ println!(" Product: {}", analysis.product_name);
+ println!(" Brand: {}", analysis.brand);
+ println!(" Price Range: {}", analysis.price_range);
+ println!(" Target Audience:");
+ for audience in analysis.target_audience {
+ println!(" - {}", audience);
+ }
+ println!(" Key Benefits:");
+ for benefit in analysis.key_benefits {
+ println!(" - {}", benefit);
+ }
+ println!(" USPs:");
+ for usp in analysis.unique_selling_points {
+ println!(" - {}", usp);
+ }
+
+ println!("\n Reviews Summary:");
+ println!(
+ " Sentiment: {}",
+ analysis.reviews_summary.sentiment_analysis
+ );
+ println!(" Common Praises:");
+ for praise in analysis.reviews_summary.common_praises {
+ println!(" - {}", praise);
+ }
+ println!(" Common Complaints:");
+ for complaint in analysis.reviews_summary.common_complaints {
+ println!(" - {}", complaint);
+ }
+ } else {
+ println!("Failed to parse extracted data");
+ }
+ }
+
+ if let Some(sources) = result.sources {
+ println!("\nSources:");
+ for (field, urls) in sources {
+ println!(" {}: {}", field, urls.join(", "));
+ }
+ }
+ }
+ Err(e) => {
+ println!("Extraction failed: {}", e);
+ }
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/apps/rust-sdk/examples/llmstxt_example.rs b/apps/rust-sdk/examples/llmstxt_example.rs
new file mode 100644
index 000000000..64fb33179
--- /dev/null
+++ b/apps/rust-sdk/examples/llmstxt_example.rs
@@ -0,0 +1,173 @@
+#![allow(clippy::option_map_unit_fn)]
+use bat::{Input, PrettyPrinter};
+use firecrawl::{llmstxt::GenerateLLMsTextParams, FirecrawlApp};
+use std::error::Error;
+
+use clap::{Parser, ValueEnum};
+
+#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
+enum Mode {
+ Basic,
+ Pool,
+ Fulltext,
+}
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+ /// URL for which to generate LLMs.txt
+ #[arg(default_value = "https://www.firecrawl.dev/")]
+ url: String,
+
+ #[arg(long, short = 'm', value_enum, default_value = "Mode::Basic")]
+ mode: Mode,
+
+ /// Maximum number of URLs to process
+ #[arg(long, short = 'd', default_value = "1")]
+ max_urls: u32,
+
+ /// Whether to show the full LLMs-full.txt in the response
+ #[arg(long, short = 'f', default_value = "false")]
+ full_text: bool,
+
+ /// Experimental streaming option
+ #[arg(long, short = 's', default_value = "false")]
+ stream: bool,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ let args = Args::parse();
+
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
+
+ let params = GenerateLLMsTextParams {
+ url: args.url.clone(),
+ max_urls: args.max_urls,
+ show_full_text: args.full_text,
+ experimental_stream: args.stream,
+ };
+
+ match args.mode {
+ Mode::Basic => {
+ println!("Example 1: Basic LLMs.txt generation (synchronous)");
+ println!("Generating LLMs.txt for {}...", args.url);
+ firecrawl
+ .generate_llms_text(params)
+ .await
+ .inspect(|result| {
+ println!("Expires at: {}", result.expires_at);
+ let text = (if args.full_text {
+ result.data.full.as_ref()
+ } else {
+ result.data.compact.as_ref()
+ })
+ .expect("LLM Text");
+
+ pretty_print_content("Firecrawl Result", text).expect("Print");
+ })?;
+ }
+ Mode::Pool => {
+ println!("Example 2: Asynchronous LLMs.txt generation with manual polling");
+
+ println!("Starting asynchronous LLMs.txt generation job...");
+ let response = firecrawl.async_generate_llms_text(params).await?;
+
+ println!("LLMs.txt generation job initiated:");
+ println!(" Job ID: {}", response.id);
+ println!("\nManually polling for status...");
+ for _ in 0..10 {
+ let status = firecrawl
+ .check_generate_llms_text_status(&response.id)
+ .await?;
+
+ match status.status.as_str() {
+ "completed" => {
+ println!("LLMs.txt generation completed!");
+ let text = (if args.full_text {
+ status.data.full.as_ref()
+ } else {
+ status.data.compact.as_ref()
+ })
+ .expect("LLM Text");
+
+ pretty_print_content("Pool Result", text).expect("Print");
+
+ break;
+ }
+ "failed" => {
+ println!(
+ "LLMs.txt generation failed: {}",
+ status.error.unwrap_or_default()
+ );
+ break;
+ }
+ status => println!("Generation status: {}", status),
+ }
+
+ println!("Waiting 2 seconds before checking again...");
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ }
+ }
+ Mode::Fulltext => {
+ println!("Example 3: LLMs.txt generation with full text");
+
+ println!("Generating LLMs.txt with full text...");
+ match firecrawl.generate_llms_text(params).await {
+ Ok(result) => {
+ println!("LLMs.txt generation completed successfully!");
+ let llmstxt = result.data.compact.expect("LLMs Text Expected");
+ let fulltxt = result.data.full.expect("Full LLMs Text Expected");
+
+ pretty_print_contents(&[
+ ("LLMs.txt (compact)", llmstxt),
+ ("LLMs.txt (full text)", fulltxt),
+ ])
+ .expect("Print")
+ }
+ Err(e) => {
+ println!("LLMs.txt generation failed: {}", e);
+ }
+ }
+ }
+ }
+
+ Ok(())
+}
+
+/// Pretty prints the provided content with syntax highlighting
+fn pretty_print_content(title: &str, content: &str) -> Result<(), Box<dyn Error>> {
+ PrettyPrinter::new()
+ .header(true)
+ .grid(true)
+ .input(
+ Input::from_bytes(content.as_bytes())
+ .title(title)
+ .name("file.md"),
+ )
+ .print()?;
+
+ Ok(())
+}
+
+/// Pretty prints multiple contents with syntax highlighting
+fn pretty_print_contents(title_contents: &[(&'static str, String)]) -> Result<(), Box<dyn Error>> {
+ let mut inputs = Vec::new();
+ for (title, content) in title_contents {
+ inputs.push(
+ Input::from_bytes(content.as_bytes())
+ .title(*title)
+ .name("file.md"),
+ );
+ }
+
+ PrettyPrinter::new()
+ .header(true)
+ .grid(true)
+ .inputs(inputs)
+ .print()?;
+
+ Ok(())
+}
diff --git a/apps/rust-sdk/examples/search_example.rs b/apps/rust-sdk/examples/search_example.rs
new file mode 100644
index 000000000..28437193a
--- /dev/null
+++ b/apps/rust-sdk/examples/search_example.rs
@@ -0,0 +1,186 @@
+use clap::{Parser, ValueEnum};
+use firecrawl::{
+ search::{SearchParams, SearchResponse},
+ FirecrawlApp,
+};
+use std::error::Error;
+
+#[derive(Debug, Parser)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+ /// Which example to run
+ #[arg(value_enum, default_value_t = Examples::All)]
+ example: Examples,
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+enum Examples {
+ All,
+ Basic,
+ Advanced,
+ Geo,
+ Temporal,
+ Social,
+ News,
+ Academic,
+ Commercial,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+ let args = Args::parse();
+
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;
+
+ match args.example {
+ Examples::All => {
+ run_basic_example(&firecrawl).await?;
+ run_advanced_example(&firecrawl).await?;
+ run_geographic_example(&firecrawl).await?;
+ run_temporal_example(&firecrawl).await?;
+ run_social_example(&firecrawl).await?;
+ run_news_example(&firecrawl).await?;
+ run_academic_example(&firecrawl).await?;
+ run_commercial_example(&firecrawl).await?;
+ }
+ Examples::Basic => run_basic_example(&firecrawl).await?,
+ Examples::Advanced => run_advanced_example(&firecrawl).await?,
+ Examples::Geo => run_geographic_example(&firecrawl).await?,
+ Examples::Temporal => run_temporal_example(&firecrawl).await?,
+ Examples::Social => run_social_example(&firecrawl).await?,
+ Examples::News => run_news_example(&firecrawl).await?,
+ Examples::Academic => run_academic_example(&firecrawl).await?,
+ Examples::Commercial => run_commercial_example(&firecrawl).await?,
+ }
+
+ Ok(())
+}
+async fn run_basic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "rust programming language";
+ let results = firecrawl.search(query, None).await?;
+ print_results("Basic Search", query, &results);
+ Ok(())
+}
+
+async fn run_advanced_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "rust web framework site:github.com OR site:gitlab.com";
+ let params = SearchParams {
+ query: query.to_string(),
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Advanced Repository Search", query, &results);
+ Ok(())
+}
+
+async fn run_geographic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "coworking space startup hub";
+ let params = SearchParams {
+ query: query.to_string(),
+ // WARN: Doesn't work with searxng
+ location: Some("Silicon Valley, CA".to_string()),
+ // WARN: Doesn't work with searxng
+ country: Some("us".to_string()),
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Geographic-Specific Search", query, &results);
+ Ok(())
+}
+
+async fn run_temporal_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "artificial intelligence breakthroughs";
+ let params = SearchParams {
+ query: query.to_string(),
+ // WARN: Doesn't work with searxng
+ tbs: Some("qdr:m1".to_string()),
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Recent AI News", query, &results);
+ Ok(())
+}
+
+async fn run_social_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "viral tech trends site:twitter.com";
+ let params = SearchParams {
+ query: query.to_string(),
+ // WARN: Doesn't work. Maybe searxng related
+ filter: Some("site:twitter.com OR site:linkedin.com".to_string()),
+ // WARN: Doesn't work with searxng
+ tbs: Some("qdr:w".to_string()), // Last week
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Social Media Tech Trends", query, &results);
+ Ok(())
+}
+
+async fn run_news_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query =
+ "cryptocurrency market analysis site:reuters.com OR site:bloomberg.com OR site:ft.com";
+ let params = SearchParams {
+ query: query.to_string(),
+ // WARN: Doesn't work with searxng
+ tbs: Some("qdr:d".to_string()), // Last 24 hours
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Financial News Search", query, &results);
+ Ok(())
+}
+
+async fn run_academic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "quantum computing research papers site:arxiv.org OR site:scholar.google.com";
+ let params = SearchParams {
+ query: query.to_string(),
+ // WARN: Doesn't work. Maybe searxng related
+ // filter: Some("site:arxiv.org OR site:scholar.google.com".to_string()),
+ // WARN: Doesn't work with searxng
+ tbs: Some("qdr:y".to_string()), // Last year
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Academic Research Search", query, &results);
+ Ok(())
+}
+
+async fn run_commercial_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {
+ let query = "enterprise cloud solutions reviews site:g2.com";
+ let params = SearchParams {
+ query: query.to_string(),
+ limit: Some(5),
+ ..Default::default()
+ };
+ let results = firecrawl.search_with_params(params).await?;
+ print_results("Commercial Product Search", query, &results);
+ Ok(())
+}
+
+fn print_results(name: &str, query: &str, results: &SearchResponse) {
+ let sec = "=".repeat(70);
+
+ println!("\n{sec}");
+ println!("🔍 {name}");
+ println!("🔎 Query: \"{query}\"");
+ println!("{sec}");
+
+ for (i, doc) in results.data.iter().enumerate() {
+ println!("{}. 📌 Title: {}", i + 1, doc.title);
+ println!(" - 🔗 URL: {}", doc.url);
+ println!(" - 📝 Description: \"{:.40}\"...", doc.description);
+ }
+
+ if let Some(warning) = &results.warning {
+ println!("\n⚠️ Warning: {warning}");
+ }
+ println!("{sec}\n");
+}
diff --git a/apps/rust-sdk/src/batch_scrape.rs b/apps/rust-sdk/src/batch_scrape.rs
new file mode 100644
index 000000000..1429fbea6
--- /dev/null
+++ b/apps/rust-sdk/src/batch_scrape.rs
@@ -0,0 +1,494 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ crawl::{CrawlErrorsResponse, CrawlStatus, CrawlStatusTypes},
+ scrape::ScrapeOptions,
+ FirecrawlApp, FirecrawlError, API_VERSION,
+};
+
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeParams {
+ /// List of URLs to scrape
+ pub urls: Vec<String>,
+ /// Scrape options to apply to all URLs
+ #[serde(flatten)]
+ pub options: Option<ScrapeOptions>,
+ /// Whether to ignore invalid URLs
+ #[serde(rename = "ignoreInvalidURLs")]
+ pub ignore_invalid_urls: bool,
+ /// ID of an existing job to append these URLs to
+ pub append_to_id: Option<String>,
+ /// Webhook configuration
+ pub webhook: Option<WebhookOptions>,
+
+ /// Idempotency key to send to the crawl endpoint.
+ #[serde(skip)]
+ pub idempotency_key: Option<String>,
+}
+
+/// Options for webhook notifications
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct WebhookOptions {
+ /// URL to send webhook notifications to
+ pub url: String,
+ /// Custom headers to include in webhook requests
+ pub headers: Option<HashMap<String, String>>,
+ /// Authentication token for the webhook
+ pub auth_token: Option<String>,
+}
+
+impl From<&str> for WebhookOptions {
+ fn from(url: &str) -> Self {
+ Self {
+ url: url.to_string(),
+ headers: None,
+ auth_token: None,
+ }
+ }
+}
+
+/// Response from initiating a batch scrape job
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeResponse {
+ /// Whether the request was successful
+ pub success: bool,
+ /// The ID of the batch scrape job
+ pub id: String,
+ /// URL to get the status of the batch scrape job
+ pub url: String,
+ /// List of URLs that were invalid and could not be processed
+ pub invalid_urls: Option<Vec<String>>,
+}
+
+impl From<String> for WebhookOptions {
+ fn from(url: String) -> Self {
+ Self {
+ url,
+ headers: None,
+ auth_token: None,
+ }
+ }
+}
+
+impl FirecrawlApp {
+ /// Initiates an asynchronous batch scrape job
+ pub async fn async_batch_scrape_urls(
+ &self,
+ params: BatchScrapeParams,
+ ) -> Result<BatchScrapeResponse, FirecrawlError> {
+ let headers = self.prepare_headers(params.idempotency_key.as_ref());
+
+ let response = self
+ .client
+ .post(format!("{}{}/batch/scrape", self.api_url, API_VERSION))
+ .headers(headers)
+ .json(&params)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Initiating batch scrape job".to_string(), e))?;
+
+ self.handle_response(response, "initiate batch scrape job")
+ .await
+ }
+
+ /// Initiates a batch scrape job and waits for completion
+ pub async fn batch_scrape_urls(
+ &self,
+ params: BatchScrapeParams,
+ poll_interval: Option<u64>,
+ ) -> Result<CrawlStatus, FirecrawlError> {
+ let poll_interval_ms = poll_interval.unwrap_or(2000);
+
+ let response = self.async_batch_scrape_urls(params).await?;
+
+ self.monitor_batch_job_status(&response.id, poll_interval_ms)
+ .await
+ }
+
+ /// Checks the status of a batch scrape job
+ pub async fn check_batch_scrape_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlStatus, FirecrawlError> {
+ let response = self
+ .client
+ .get(format!(
+ "{}{}/batch/scrape/{}",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Checking status of batch scrape {}", id.as_ref()),
+ e,
+ )
+ })?;
+
+ let mut status: CrawlStatus = self
+ .handle_response(
+ response,
+ format!("Checking status of batch scrape {}", id.as_ref()),
+ )
+ .await?;
+
+ if status.status == CrawlStatusTypes::Completed {
+ while let Some(next) = status.next.clone() {
+ let new_status = self.check_batch_scrape_status_next(next).await?;
+ status.data.extend_from_slice(&new_status.data);
+ status.next = new_status.next;
+ }
+ }
+
+ Ok(status)
+ }
+
+ /// Helper function to paginate through batch scrape status results
+ async fn check_batch_scrape_status_next(
+ &self,
+ next: impl AsRef<str>,
+ ) -> Result<CrawlStatus, FirecrawlError> {
+ let response = self
+ .client
+ .get(next.as_ref())
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Paginating batch scrape using URL {:?}", next.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(
+ response,
+ format!("Paginating batch scrape using URL {:?}", next.as_ref()),
+ )
+ .await
+ }
+
+ /// Check for errors in a batch scrape job
+ pub async fn check_batch_scrape_errors(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(format!(
+ "{}{}/batch/scrape/{}/errors",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Checking errors for batch scrape {}", id.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(
+ response,
+ format!("Checking errors for batch scrape {}", id.as_ref()),
+ )
+ .await
+ }
+
+ /// Helper function to poll for batch job status until completion
+ async fn monitor_batch_job_status(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ ) -> Result<CrawlStatus, FirecrawlError> {
+ loop {
+ let status_data = self.check_batch_scrape_status(id).await?;
+ match status_data.status {
+ CrawlStatusTypes::Completed => {
+ break Ok(status_data);
+ }
+ CrawlStatusTypes::Scraping => {
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ CrawlStatusTypes::Failed => {
+ break Err(FirecrawlError::CrawlJobFailed(
+ "Batch scrape job failed".into(),
+ status_data,
+ ));
+ }
+ CrawlStatusTypes::Cancelled => {
+ break Err(FirecrawlError::CrawlJobFailed(
+ "Batch scrape job was cancelled".into(),
+ status_data,
+ ));
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_batch_scrape() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+
+ // Start a batch scrape job
+ let params = BatchScrapeParams {
+ urls: vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ],
+ ignore_invalid_urls: true,
+ ..Default::default()
+ };
+
+ let response = app.async_batch_scrape_urls(params).await.unwrap();
+
+ assert!(response.success);
+ assert!(!response.id.is_empty());
+ assert!(!response.url.is_empty());
+ }
+
+ #[tokio::test]
+ async fn test_async_batch_scrape_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock
+ let mock = server
+ .mock("POST", "/v1/batch/scrape")
+ // Remove the match_body expectation which might be causing issues
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-123",
+ "url": "https://api.example.com/v1/batch/batch-123",
+ "invalidUrls": []
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = BatchScrapeParams {
+ urls: vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ],
+ ignore_invalid_urls: true,
+ ..Default::default()
+ };
+
+ let response = app.async_batch_scrape_urls(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "batch-123");
+ assert_eq!(response.url, "https://api.example.com/v1/batch/batch-123");
+ assert!(response.invalid_urls.unwrap_or_default().is_empty());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_batch_scrape_with_webhook() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v1/batch/scrape")
+ // Remove the match_body expectation to simplify
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-123",
+ "url": "https://api.example.com/v1/batch/batch-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = BatchScrapeParams {
+ urls: vec!["https://example.com".to_string()],
+ webhook: Some("https://webhook.example.com/notify".into()),
+ ..Default::default()
+ };
+
+ let response = app.async_batch_scrape_urls(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "batch-123");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_check_batch_scrape_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v1/batch/scrape/batch-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "total": 2,
+ "completed": 2,
+ "creditsUsed": 2,
+ "expiresAt": "2023-12-31T23:59:59Z",
+ "data": [
+ {
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200
+ },
+ "markdown": "Example Domain content"
+ },
+ {
+ "metadata": {
+ "sourceURL": "https://example.org",
+ "statusCode": 200
+ },
+ "markdown": "Another example content"
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = app.check_batch_scrape_status("batch-123").await.unwrap();
+
+ assert_eq!(status.total, 2);
+ assert_eq!(status.completed, 2);
+ assert_eq!(status.data.len(), 2);
+ assert_eq!(status.data[0].metadata.source_url, "https://example.com");
+ assert_eq!(status.data[1].metadata.source_url, "https://example.org");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_check_batch_scrape_errors_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v1/batch/scrape/batch-123/errors")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "errors": [
+ {
+ "id": "error1",
+ "timestamp": "2023-01-01T00:00:00Z",
+ "url": "https://invalid.example.com",
+ "error": "Failed to load page"
+ }
+ ],
+ "robotsBlocked": [
+ "https://example.com/admin"
+ ]
+ })
+ .to_string(),
+ )
+ .create_async()
+ .await;
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let errors = app.check_batch_scrape_errors("batch-123").await.unwrap();
+
+ assert_eq!(errors.errors.len(), 1);
+ assert_eq!(errors.errors[0].url, "https://invalid.example.com");
+ assert_eq!(errors.robots_blocked.len(), 1);
+ assert_eq!(errors.robots_blocked[0], "https://example.com/admin");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_batch_scrape_with_invalid_urls() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v1/batch/scrape")
+ // Remove the match_body expectation
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-123",
+ "url": "https://api.example.com/v1/batch/batch-123",
+ "invalidUrls": ["invalid-url"]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = BatchScrapeParams {
+ urls: vec!["https://example.com".to_string(), "invalid-url".to_string()],
+ ignore_invalid_urls: true,
+ ..Default::default()
+ };
+
+ let response = app.async_batch_scrape_urls(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "batch-123");
+ assert_eq!(response.invalid_urls, Some(vec!["invalid-url".to_string()]));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_batch_scrape_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v1/batch/scrape")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "No valid URLs provided"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = BatchScrapeParams::default();
+ let result = app.async_batch_scrape_urls(params).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rs
index 2860d24ac..a5f30f40b 100644
--- a/apps/rust-sdk/src/crawl.rs
+++ b/apps/rust-sdk/src/crawl.rs
@@ -2,7 +2,11 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
-use crate::{document::Document, scrape::{ScrapeFormats, ScrapeOptions}, FirecrawlApp, FirecrawlError, API_VERSION};
+use crate::{
+ document::Document,
+ scrape::{ScrapeFormats, ScrapeOptions},
+ FirecrawlApp, FirecrawlError, API_VERSION,
+};
#[derive(Deserialize, Serialize, Clone, Copy, Debug)]
pub enum CrawlScrapeFormats {
@@ -23,13 +27,13 @@ pub enum CrawlScrapeFormats {
Links,
/// Will result in a URL to a screenshot of the page.
- ///
+ ///
/// Can not be used in conjunction with `CrawlScrapeFormats::ScreenshotFullPage`.
#[serde(rename = "screenshot")]
Screenshot,
/// Will result in a URL to a full-page screenshot of the page.
- ///
+ ///
/// Can not be used in conjunction with `CrawlScrapeFormats::Screenshot`.
#[serde(rename = "screenshot@fullPage")]
ScreenshotFullPage,
@@ -59,12 +63,12 @@ pub struct CrawlScrapeOptions {
pub only_main_content: Option<bool>,
/// HTML tags to exclusively include.
- ///
+ ///
/// For example, if you pass `div`, you will only get content from `<div>`s and their children.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude.
- ///
+ ///
/// For example, if you pass `img`, you will never get image URLs in your results.
pub exclude_tags: Option<Vec<String>>,
@@ -81,7 +85,9 @@ pub struct CrawlScrapeOptions {
impl From<CrawlScrapeOptions> for ScrapeOptions {
fn from(value: CrawlScrapeOptions) -> Self {
ScrapeOptions {
- formats: value.formats.map(|formats| formats.into_iter().map(|x| x.into()).collect()),
+ formats: value
+ .formats
+ .map(|formats| formats.into_iter().map(|x| x.into()).collect()),
only_main_content: value.only_main_content,
include_tags: value.include_tags,
exclude_tags: value.exclude_tags,
@@ -101,12 +107,12 @@ pub struct CrawlOptions {
pub scrape_options: Option<CrawlScrapeOptions>,
/// URL RegEx patterns to (exclusively) include.
- ///
+ ///
/// For example, if you specified `"blog"`, only pages that have `blog` somewhere in the URL would be crawled.
pub include_paths: Option<Vec<String>>,
/// URL RegEx patterns to exclude.
- ///
+ ///
/// For example, if you specified `"blog"`, pages that have `blog` somewhere in the URL would not be crawled.
pub exclude_paths: Option<Vec<String>>,
@@ -200,6 +206,29 @@ pub struct CrawlStatus {
pub data: Vec<Document>,
}
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlError {
+ pub id: String,
+ pub timestamp: Option<String>,
+ pub url: String,
+ pub error: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlErrorsResponse {
+ pub errors: Vec<CrawlError>,
+ #[serde(rename = "robotsBlocked")]
+ pub robots_blocked: Vec<String>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CancelCrawlResponse {
+ pub status: String,
+}
+
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlAsyncResponse {
@@ -223,19 +252,20 @@ impl FirecrawlApp {
url: url.as_ref().to_string(),
options: options.unwrap_or_default(),
};
-
+
let headers = self.prepare_headers(body.options.idempotency_key.as_ref());
let response = self
.client
- .post(&format!("{}{}/crawl", self.api_url, API_VERSION))
+ .post(format!("{}{}/crawl", self.api_url, API_VERSION))
.headers(headers.clone())
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Crawling {:?}", url.as_ref()), e))?;
- self.handle_response::<CrawlAsyncResponse>(response, "start crawl job").await
+ self.handle_response::<CrawlAsyncResponse>(response, "start crawl job")
+ .await
}
/// Performs a crawl job for a URL using the Firecrawl API, waiting for the end result. This may take a long time depending on the size of the target page and your options (namely `CrawlOptions.limit`).
@@ -245,38 +275,65 @@ impl FirecrawlApp {
options: impl Into<Option<CrawlOptions>>,
) -> Result<CrawlStatus, FirecrawlError> {
let options = options.into();
- let poll_interval = options.as_ref().and_then(|x| x.poll_interval).unwrap_or(2000);
+ let poll_interval = options
+ .as_ref()
+ .and_then(|x| x.poll_interval)
+ .unwrap_or(2000);
let res = self.crawl_url_async(url, options).await?;
self.monitor_job_status(&res.id, poll_interval).await
}
- async fn check_crawl_status_next(&self, next: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {
+ async fn check_crawl_status_next(
+ &self,
+ next: impl AsRef<str>,
+ ) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
.get(next.as_ref())
.headers(self.prepare_headers(None))
.send()
.await
- .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl using URL {:?}", next.as_ref()), e))?;
-
- self.handle_response(response, format!("Paginating crawl using URL {:?}", next.as_ref())).await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Paginating crawl using URL {:?}", next.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(
+ response,
+ format!("Paginating crawl using URL {:?}", next.as_ref()),
+ )
+ .await
}
/// Checks for the status of a crawl, based on the crawl's ID. To be used in conjunction with `FirecrawlApp::crawl_url_async`.
- pub async fn check_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {
+ pub async fn check_crawl_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
- .get(&format!(
+ .get(format!(
"{}{}/crawl/{}",
- self.api_url, API_VERSION, id.as_ref()
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
- .map_err(|e| FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e))?;
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e)
+ })?;
- let mut status: CrawlStatus = self.handle_response(response, format!("Checking status of crawl {}", id.as_ref())).await?;
+ let mut status: CrawlStatus = self
+ .handle_response(
+ response,
+ format!("Checking status of crawl {}", id.as_ref()),
+ )
+ .await?;
if status.status == CrawlStatusTypes::Completed {
while let Some(next) = status.next {
@@ -304,16 +361,240 @@ impl FirecrawlApp {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
CrawlStatusTypes::Failed => {
- break Err(FirecrawlError::CrawlJobFailed(format!(
- "Crawl job failed."
- ), status_data));
+ break Err(FirecrawlError::CrawlJobFailed(
+ "Crawl job failed".into(),
+ status_data,
+ ));
}
CrawlStatusTypes::Cancelled => {
- break Err(FirecrawlError::CrawlJobFailed(format!(
- "Crawl job was cancelled."
- ), status_data));
+ break Err(FirecrawlError::CrawlJobFailed(
+ "Crawl job was cancelled.".into(),
+ status_data,
+ ));
}
}
}
}
+
+ /// Cancel an asynchronous crawl job using the Firecrawl API.
+ ///
+ /// # Returns
+ ///
+ /// A response indicating whether the cancellation was successful, or a FirecrawlError if the request fails.
+ pub async fn cancel_crawl(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CancelCrawlResponse, FirecrawlError> {
+ let response = self
+ .client
+ .delete(format!(
+ "{}{}/crawl/{}",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "crawl_cancel").await
+ }
+
+ /// Returns information about crawl errors.
+ ///
+ /// # Returns
+ ///
+ /// A response containing information about crawl errors, or a FirecrawlError if the request fails.
+ pub async fn check_crawl_errors(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(format!(
+ "{}{}/crawl/{}/errors",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Checking errors for crawl {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "crawl_check").await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_cancel_crawl() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+
+ // First start a crawl job
+ let crawl_response = app
+ .crawl_url_async("https://example.com", None)
+ .await
+ .unwrap();
+
+ // Then cancel it
+ let cancel_response = app.cancel_crawl(crawl_response.id).await.unwrap();
+
+ assert_eq!(cancel_response.status, "cancelled");
+ }
+
+ #[tokio::test]
+ async fn test_cancel_crawl_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the cancel request
+ let mock = server
+ .mock("DELETE", "/v1/crawl/test-crawl-id")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": null,
+ "status": "cancelled"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = app.cancel_crawl("test-crawl-id").await.unwrap();
+
+ assert_eq!(response.status, "cancelled");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_cancel_crawl_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for an error response
+ let mock = server
+ .mock("DELETE", "/v1/crawl/invalid-id")
+ .with_status(404)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Crawl job not found"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = app.cancel_crawl("invalid-id").await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_check_crawl_errors() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+
+ // First start a crawl job
+ let crawl_response = app
+ .crawl_url_async("https://no-wer-agg.invalid", None)
+ .await
+ .unwrap();
+
+ // Check for errors
+ let errors_response = app.check_crawl_errors(crawl_response.id).await.unwrap();
+ println!("{errors_response:?}");
+
+ tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+
+ assert!(
+ !errors_response.errors.is_empty(),
+ "WARN: Error returned related to Supabase not in my environment. It may fail"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_check_crawl_errors_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the check errors request
+ let mock = server
+ .mock("GET", "/v1/crawl/test-crawl-id/errors")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "errors": [
+ {
+ "id": "error1",
+ "timestamp": "2023-01-01T00:00:00Z",
+ "url": "https://example.com/error-page",
+ "error": "Failed to load page"
+ }
+ ],
+ "robotsBlocked": [
+ "https://example.com/blocked-by-robots"
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = app.check_crawl_errors("test-crawl-id").await.unwrap();
+
+ assert_eq!(response.errors.len(), 1);
+ assert_eq!(response.errors[0].id, "error1");
+ assert_eq!(response.errors[0].url, "https://example.com/error-page");
+ assert_eq!(response.errors[0].error, "Failed to load page");
+ assert_eq!(response.robots_blocked.len(), 1);
+ assert_eq!(
+ response.robots_blocked[0],
+ "https://example.com/blocked-by-robots"
+ );
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_check_crawl_errors_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for an error response
+ let mock = server
+ .mock("GET", "/v1/crawl/invalid-id/errors")
+ .with_status(404)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Crawl job not found"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = app.check_crawl_errors("invalid-id").await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
}
diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rs
index 1948a4ce6..38c7fcc15 100644
--- a/apps/rust-sdk/src/document.rs
+++ b/apps/rust-sdk/src/document.rs
@@ -57,12 +57,12 @@ pub struct Document {
pub markdown: Option<String>,
/// The HTML of the page, present if `ScrapeFormats::HTML` is present in `ScrapeOptions.formats`.
- ///
+ ///
/// This contains HTML that has non-content tags removed. If you need the original HTML, use `ScrapeFormats::RawHTML`.
pub html: Option<String>,
/// The raw HTML of the page, present if `ScrapeFormats::RawHTML` is present in `ScrapeOptions.formats`.
- ///
+ ///
/// This contains the original, untouched HTML on the page. If you only need human-readable content, use `ScrapeFormats::HTML`.
pub raw_html: Option<String>,
@@ -83,4 +83,3 @@ pub struct Document {
/// The warning message will contain any errors encountered during the extraction.
pub warning: Option<String>,
}
-
diff --git a/apps/rust-sdk/src/error.rs b/apps/rust-sdk/src/error.rs
index 33e4edc61..f452cd0e3 100644
--- a/apps/rust-sdk/src/error.rs
+++ b/apps/rust-sdk/src/error.rs
@@ -42,4 +42,6 @@ pub enum FirecrawlError {
APIError(String, FirecrawlAPIError),
#[error("Crawl job failed: {0}")]
CrawlJobFailed(String, CrawlStatus),
+ #[error("Missuse: {0}")]
+ Missuse(String),
}
diff --git a/apps/rust-sdk/src/extract.rs b/apps/rust-sdk/src/extract.rs
new file mode 100644
index 000000000..a1dd2ef92
--- /dev/null
+++ b/apps/rust-sdk/src/extract.rs
@@ -0,0 +1,596 @@
+use std::collections::HashMap;
+
+use schemars::schema_for;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
+
+/// Parameters for extract requests
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ExtractParams {
+ /// URLs to extract information from
+ pub urls: Option<Vec<String>>,
+
+ /// Extraction prompt
+ pub prompt: Option<String>,
+
+ /// Schema for structured output
+ pub schema: Option<Value>,
+
+ /// System prompt for the LLM
+ pub system_prompt: Option<String>,
+
+ /// Allow following external links
+ pub allow_external_links: Option<bool>,
+
+ /// Enable web search for additional information
+ pub enable_web_search: Option<bool>,
+
+ /// Show sources in the response
+ pub show_sources: Option<bool>,
+
+ /// Origin information, defaults to "api-sdk"
+ pub origin: Option<String>,
+
+ /// Timeout in milliseconds, defaults to 60000
+ pub timeout: Option<u32>,
+
+ /// Whether to include URL trace information, defaults to false
+ pub url_trace: Option<bool>,
+
+ /// Whether to ignore sitemap, defaults to false
+ pub ignore_sitemap: Option<bool>,
+
+ /// Whether to include subdomains, defaults to true
+ pub include_subdomains: Option<bool>,
+
+ /// Maximum number of URLs to process
+ pub limit: Option<u32>,
+
+ /// Experimental: Stream steps information
+ #[serde(rename = "__experimental_streamSteps")]
+ pub experimental_stream_steps: Option<bool>,
+
+ /// Experimental: Include LLM usage information
+ #[serde(rename = "__experimental_llmUsage")]
+ pub experimental_llm_usage: Option<bool>,
+
+ /// Experimental: Show sources information
+ #[serde(rename = "__experimental_showSources")]
+ pub experimental_show_sources: Option<bool>,
+
+ /// Experimental: Cache key
+ #[serde(rename = "__experimental_cacheKey")]
+ pub experimental_cache_key: Option<String>,
+
+ /// Experimental: Cache mode, defaults to "direct"
+ #[serde(rename = "__experimental_cacheMode")]
+ pub experimental_cache_mode: Option<String>,
+}
+
+/// Response from initiating an extract operation
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ExtractResponse {
+ /// Whether the request was successful
+ pub success: bool,
+
+ /// The ID of the extract job
+ pub id: String,
+
+ /// URL trace information if requested
+ pub url_trace: Option<Vec<URLTrace>>,
+}
+
+/// Information about URL processing during extraction
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct URLTrace {
+ /// The URL being processed
+ pub url: String,
+
+ /// Status of processing this URL
+ pub status: String,
+
+ /// Timing information for URL processing
+ pub timing: URLTraceTiming,
+
+ /// Error message if processing failed
+ pub error: Option<String>,
+
+ /// Warning message if there were issues
+ pub warning: Option<String>,
+
+ /// Content statistics
+ pub content_stats: Option<ContentStats>,
+
+ /// Relevance score for this URL (0-1)
+ pub relevance_score: Option<f64>,
+
+ /// Whether this URL was used in the final completion
+ pub used_in_completion: Option<bool>,
+
+ /// Fields extracted from this URL
+ pub extracted_fields: Option<Vec<String>>,
+}
+
+/// Timing information for URL processing
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct URLTraceTiming {
+ /// When the URL was discovered
+ pub discovered_at: String,
+
+ /// When scraping began for this URL
+ pub scraped_at: Option<String>,
+
+ /// When processing was completed for this URL
+ pub completed_at: Option<String>,
+}
+
+/// Statistics about processed content
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ContentStats {
+ /// Length of the raw content in characters
+ pub raw_content_length: u32,
+
+ /// Length of the processed content in characters
+ pub processed_content_length: u32,
+
+ /// Number of tokens used for this content
+ pub tokens_used: u32,
+}
+
+/// Response for extract status check
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ExtractStatusResponse {
+ /// Whether the request was successful
+ pub success: bool,
+
+ /// Status of the extract job: "pending", "processing", "completed", "failed"
+ pub status: String,
+
+ /// Extracted data, present when status is "completed"
+ pub data: Option<Value>,
+
+ /// Error message if the job failed
+ pub error: Option<String>,
+
+ /// URL trace information if requested
+ pub url_trace: Option<Vec<URLTrace>>,
+
+ /// Sources information if requested
+ pub sources: Option<HashMap<String, Vec<String>>>,
+}
+
+impl FirecrawlApp {
+ /// Extracts information from URLs using the Firecrawl API.
+ ///
+ /// This is the synchronous version that polls until completion.
+ ///
+ /// Either `params.prompt` or `params.schema` must be provided.
+ pub async fn extract(
+ &self,
+ params: impl Into<ExtractParams>,
+ ) -> Result<ExtractStatusResponse, FirecrawlError> {
+ let mut params = params.into();
+ // Validation: Either prompt or schema must be provided
+ if params.prompt.is_none() && params.schema.is_none() {
+ return Err(FirecrawlError::APIError(
+ "Extract validation".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: "Either prompt or schema must be provided".to_string(),
+ details: None,
+ },
+ ));
+ }
+
+ // Set default origin if not provided
+ if params.origin.is_none() {
+ params.origin = Some("api-sdk".to_string());
+ }
+
+ // Initiate the extract job asynchronously
+ let response = self.async_extract(params).await?;
+
+ // Poll for the result
+ let poll_interval = 2000; // Default to 2 seconds
+ self.monitor_extract_job_status(&response.id, poll_interval)
+ .await
+ }
+
+ pub async fn extract_with_schemars<T>(
+ &self,
+ params: impl Into<ExtractParams>,
+ ) -> Result<ExtractStatusResponse, FirecrawlError>
+ where
+ T: schemars::JsonSchema,
+ {
+ let mut params = params.into();
+ let schema = schema_for!(T);
+ let schema_json = serde_json::to_value(schema).map_err(|e| {
+ FirecrawlError::APIError(
+ "Schema serialization".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: e.to_string(),
+ details: None,
+ },
+ )
+ })?;
+ params.schema = Some(schema_json);
+ self.extract(params).await
+ }
+
+ /// Initiates an asynchronous extract operation.
+ ///
+ /// # Arguments
+ ///
+ /// * `params` - Parameters for the extract request
+ ///
+ /// # Returns
+ ///
+ /// A response containing the extract job ID, or a FirecrawlError if the request fails.
+ ///
+ /// # Notes
+ ///
+ /// Either `params.urls` or `params.prompt` must be provided.
+ /// Either `params.prompt` or `params.schema` must be provided.
+ pub async fn async_extract(
+ &self,
+ params: impl Into<ExtractParams>,
+ ) -> Result<ExtractResponse, FirecrawlError> {
+ let params = params.into();
+ // Validation: Either URLs or prompt must be provided
+ if params.urls.is_none() && params.prompt.is_none() {
+ return Err(FirecrawlError::APIError(
+ "Extract validation".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: "Either URLs or prompt must be provided".to_string(),
+ details: None,
+ },
+ ));
+ }
+
+ // Validation: Either prompt or schema must be provided
+ if params.prompt.is_none() && params.schema.is_none() {
+ return Err(FirecrawlError::APIError(
+ "Extract validation".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: "Either prompt or schema must be provided".to_string(),
+ details: None,
+ },
+ ));
+ }
+
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(format!("{}{}/extract", self.api_url, API_VERSION))
+ .headers(headers)
+ .json(&params)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Initiating extract job".to_string(), e))?;
+
+ self.handle_response(response, "initiate extract job").await
+ }
+
+ /// Checks the status of an extract job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The ID of the extract job
+ ///
+ /// # Returns
+ ///
+ /// A response containing the status of the extract job, or a FirecrawlError if the request fails.
+ pub async fn get_extract_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<ExtractStatusResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(format!(
+ "{}{}/extract/{}",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Checking status of extract {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(
+ response,
+ format!("Checking status of extract {}", id.as_ref()),
+ )
+ .await
+ }
+
+ /// Helper function to poll for extract job status until completion
+ async fn monitor_extract_job_status(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ ) -> Result<ExtractStatusResponse, FirecrawlError> {
+ loop {
+ let status_data = self.get_extract_status(id).await?;
+
+ match status_data.status.as_str() {
+ "completed" => {
+ break Ok(status_data);
+ }
+ "pending" | "processing" => {
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ "failed" => {
+ let error_msg = status_data
+ .error
+ .clone()
+ .unwrap_or_else(|| "Extract job failed".to_string());
+ break Err(FirecrawlError::APIError(
+ "Extract job failed".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: error_msg,
+ details: None,
+ },
+ ));
+ }
+ _ => {
+ break Err(FirecrawlError::APIError(
+ "Extract job status".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: format!("Unexpected status: {}", status_data.status),
+ details: None,
+ },
+ ));
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_extract() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+
+ // Create extract params
+ let params = ExtractParams {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: Some("Extract the title and main content from this page".to_string()),
+ schema: None,
+ origin: Some("test".to_string()),
+ ..Default::default()
+ };
+
+ // Start an extract job
+ let response = app.async_extract(params).await.unwrap();
+
+ assert!(response.success);
+ assert!(!response.id.is_empty());
+ }
+
+ #[tokio::test]
+ async fn test_async_extract_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the extract request
+ let mock = server
+ .mock("POST", "/v1/extract")
+ .match_body(mockito::Matcher::PartialJson(json!({
+ "urls": ["https://example.com"],
+ "prompt": "Extract the title and main content"
+ })))
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "extract-123",
+ "urlTrace": []
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = ExtractParams {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: Some("Extract the title and main content".to_string()),
+ schema: None,
+ ..Default::default()
+ };
+
+ let response = app.async_extract(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "extract-123");
+ assert!(response.url_trace.unwrap_or_default().is_empty());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_extract_with_schema() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the extract request with schema
+ let mock = server
+ .mock("POST", "/v1/extract")
+ .match_body(mockito::Matcher::PartialJson(json!({
+ "urls": ["https://example.com"],
+ "schema": {
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "content": { "type": "string" }
+ }
+ }
+ })))
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "extract-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Set up the mock for the status request
+ let status_mock = server
+ .mock("GET", "/v1/extract/extract-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "data": {
+ "title": "Example Domain",
+ "content": "This domain is for use in illustrative examples in documents."
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let urls = Some(vec!["https://example.com".to_string()]);
+ let params = ExtractParams {
+ urls,
+ schema: Some(json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "content": { "type": "string" }
+ }
+ })),
+ ..Default::default()
+ };
+
+ let response = app.extract(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.status, "completed");
+
+ let data = response.data.unwrap();
+ assert_eq!(data["title"], "Example Domain");
+ assert_eq!(
+ data["content"],
+ "This domain is for use in illustrative examples in documents."
+ );
+
+ mock.assert();
+ status_mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_extract_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the status check
+ let mock = server
+ .mock("GET", "/v1/extract/extract-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "processing",
+ "urlTrace": [
+ {
+ "url": "https://example.com",
+ "status": "scraping",
+ "timing": {
+ "discoveredAt": "2023-01-01T00:00:00Z"
+ }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = app.get_extract_status("extract-123").await.unwrap();
+
+ assert!(status.success);
+ assert_eq!(status.status, "processing");
+ assert_eq!(status.url_trace.unwrap()[0].url, "https://example.com");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_extract_validation_errors() {
+ let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
+
+ // Test missing both URLs and prompt
+ let result = app.async_extract(ExtractParams::default()).await;
+ assert!(result.is_err());
+
+ // Test having URLs but missing both prompt and schema
+ let params = ExtractParams {
+ urls: Some(vec!["https://example.com".to_string()]),
+ ..Default::default()
+ };
+ let result = app.async_extract(params).await;
+ assert!(result.is_err());
+ }
+
+ #[tokio::test]
+ async fn test_extract_api_error() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for an error response
+ let mock = server
+ .mock("POST", "/v1/extract")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid schema format"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = ExtractParams {
+ urls: Some(vec!["https://example.com".to_string()]),
+ schema: Some(json!("invalid")), // Invalid schema format
+ ..Default::default()
+ };
+
+ let result = app.async_extract(params).await;
+ assert!(result.is_err());
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs
index 5d95cc7d2..80bffc468 100644
--- a/apps/rust-sdk/src/lib.rs
+++ b/apps/rust-sdk/src/lib.rs
@@ -2,14 +2,18 @@ use reqwest::{Client, Response};
use serde::de::DeserializeOwned;
use serde_json::Value;
+pub mod batch_scrape;
pub mod crawl;
pub mod document;
mod error;
+pub mod extract;
+pub mod llmstxt;
pub mod map;
pub mod scrape;
+pub mod search;
-pub use error::FirecrawlError;
use error::FirecrawlAPIError;
+pub use error::FirecrawlError;
#[derive(Clone, Debug)]
pub struct FirecrawlApp {
@@ -26,9 +30,12 @@ impl FirecrawlApp {
FirecrawlApp::new_selfhosted(CLOUD_API_URL, Some(api_key))
}
- pub fn new_selfhosted(api_url: impl AsRef<str>, api_key: Option<impl AsRef<str>>) -> Result<Self, FirecrawlError> {
+ pub fn new_selfhosted(
+ api_url: impl AsRef<str>,
+ api_key: Option<impl AsRef<str>>,
+ ) -> Result<Self, FirecrawlError> {
let url = api_url.as_ref().to_string();
-
+
if url == CLOUD_API_URL && api_key.is_none() {
return Err(FirecrawlError::APIError(
"Configuration".to_string(),
@@ -36,7 +43,7 @@ impl FirecrawlApp {
success: false,
error: "API key is required for cloud service".to_string(),
details: None,
- }
+ },
));
}
@@ -73,27 +80,43 @@ impl FirecrawlApp {
.text()
.await
.map_err(|e| FirecrawlError::ResponseParseErrorText(e))
- .and_then(|response_json| serde_json::from_str::<Value>(&response_json).map_err(|e| FirecrawlError::ResponseParseError(e)))
+ .and_then(|response_json| {
+ serde_json::from_str::<Value>(&response_json)
+ .map_err(|e| FirecrawlError::ResponseParseError(e))
+ .inspect(|data| {
+ #[cfg(debug_assertions)]
+ println!("Response JSON: {:#?}", data);
+ })
+ })
.and_then(|response_value| {
- if response_value["success"].as_bool().unwrap_or(false) {
- Ok(serde_json::from_value::<T>(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?)
+ if action.as_ref().starts_with("crawl_") // no success in check/cancel crawl responses
+ || response_value["success"].as_bool().unwrap_or(false)
+ {
+ Ok(serde_json::from_value::<T>(response_value)
+ .map_err(|e| FirecrawlError::ResponseParseError(e))?)
} else {
Err(FirecrawlError::APIError(
action.as_ref().to_string(),
- serde_json::from_value(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?
+ serde_json::from_value(response_value)
+ .map_err(|e| FirecrawlError::ResponseParseError(e))?,
))
}
});
match &response {
Ok(_) => response,
- Err(FirecrawlError::ResponseParseError(_)) | Err(FirecrawlError::ResponseParseErrorText(_)) => {
+ Err(FirecrawlError::ResponseParseError(_))
+ | Err(FirecrawlError::ResponseParseErrorText(_)) => {
if is_success {
response
} else {
- Err(FirecrawlError::HttpRequestFailed(action.as_ref().to_string(), status.as_u16(), status.as_str().to_string()))
+ Err(FirecrawlError::HttpRequestFailed(
+ action.as_ref().to_string(),
+ status.as_u16(),
+ status.as_str().to_string(),
+ ))
}
- },
+ }
Err(_) => response,
}
}
diff --git a/apps/rust-sdk/src/llmstxt.rs b/apps/rust-sdk/src/llmstxt.rs
new file mode 100644
index 000000000..f8fda7e91
--- /dev/null
+++ b/apps/rust-sdk/src/llmstxt.rs
@@ -0,0 +1,426 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
+
+/// Parameters for generating LLMs.txt
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct GenerateLLMsTextParams {
+ /// URL for which to generate LLMs.txt
+ pub url: String,
+
+ /// Maximum number of URLs to process. Default: 10
+ pub max_urls: u32,
+
+ /// Whether to show the full LLMs-full.txt in the response. Default: false
+ pub show_full_text: bool,
+
+ /// Experimental streaming option
+ #[serde(rename = "__experimental_stream")]
+ pub experimental_stream: bool,
+}
+
+impl Default for GenerateLLMsTextParams {
+ fn default() -> Self {
+ Self {
+ url: String::new(),
+ max_urls: 1,
+ show_full_text: false,
+ experimental_stream: false,
+ }
+ }
+}
+
+/// Response from initiating a LLMs.txt generation job
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct GenerateLLMsTextResponse {
+ /// Whether the request was successful
+ pub success: bool,
+
+ /// Job ID for the LLMs.txt generation
+ pub id: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, Default)]
+pub struct LLMTextData {
+ #[serde(rename = "llmstxt")]
+ pub compact: Option<String>,
+ #[serde(rename = "llmsfulltxt")]
+ pub full: Option<String>,
+}
+
+/// Response from checking the status of a LLMs.txt generation job
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct GenerateLLMsTextStatusResponse {
+ /// Whether the request was successful
+ pub success: bool,
+
+ /// Status of the job: "pending", "processing", "completed", "failed"
+ pub status: String,
+
+ /// Generated LLMs.txt data, present when status is "completed"
+ #[serde(default)]
+ pub data: LLMTextData,
+
+ /// Error message if the job failed
+ pub error: Option<String>,
+
+ /// Expiration timestamp for the data
+ pub expires_at: String,
+}
+
+impl FirecrawlApp {
+ /// Generates LLMs.txt for a given URL and polls until completion.
+ ///
+ /// # Arguments
+ ///
+ /// * `params` - Parameters for the LLMs.txt generation
+ ///
+ /// # Returns
+ ///
+ /// A response containing the generation results, or a FirecrawlError if the request fails.
+ pub async fn generate_llms_text(
+ &self,
+ params: impl Into<GenerateLLMsTextParams>,
+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
+ // Initiate the LLMs.txt generation job asynchronously
+ let response = self.async_generate_llms_text(params).await?;
+
+ // Poll for the result
+ let poll_interval = 2000; // Default to 2 seconds
+ self.monitor_llms_text_job_status(&response.id, poll_interval)
+ .await
+ }
+
+ /// Initiates an asynchronous LLMs.txt generation operation.
+ ///
+ /// # Arguments
+ ///
+ /// * `params` - Parameters for the LLMs.txt generation
+ ///
+ /// # Returns
+ ///
+ /// A response containing the generation job ID, or a FirecrawlError if the request fails.
+ pub async fn async_generate_llms_text(
+ &self,
+ params: impl Into<GenerateLLMsTextParams>,
+ ) -> Result<GenerateLLMsTextResponse, FirecrawlError> {
+ let params = params.into();
+
+ // Validation: URL must be provided
+ if params.url.is_empty() {
+ return Err(FirecrawlError::APIError(
+ "Generate LLMs.txt validation".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: "URL must be provided".to_string(),
+ details: None,
+ },
+ ));
+ }
+
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(format!("{}{}/llmstxt", self.api_url, API_VERSION))
+ .headers(headers)
+ .json(&params)
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError("Initiating LLMs.txt generation".to_string(), e)
+ })?;
+
+ self.handle_response(response, "initiate LLMs.txt generation")
+ .await
+ }
+
+ /// Checks the status of a LLMs.txt generation operation.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The ID of the LLMs.txt generation operation
+ ///
+ /// # Returns
+ ///
+ /// A response containing the current status and results of the generation operation,
+ /// or a FirecrawlError if the request fails.
+ pub async fn check_generate_llms_text_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(format!(
+ "{}{}/llmstxt/{}",
+ self.api_url,
+ API_VERSION,
+ id.as_ref()
+ ))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Checking status of LLMs.txt generation {}", id.as_ref()),
+ e,
+ )
+ })?;
+
+ self.handle_response(
+ response,
+ format!("Checking status of LLMs.txt generation {}", id.as_ref()),
+ )
+ .await
+ }
+
+ /// Helper function to poll for LLMs.txt generation job status until completion
+ async fn monitor_llms_text_job_status(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
+ loop {
+ let status_data = self.check_generate_llms_text_status(id).await?;
+
+ match status_data.status.as_str() {
+ "completed" => {
+ break Ok(status_data);
+ }
+ "pending" | "processing" => {
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ "failed" => {
+ let error_msg = status_data
+ .error
+ .clone()
+ .unwrap_or_else(|| "LLMs.txt generation failed".to_string());
+ break Err(FirecrawlError::APIError(
+ "LLMs.txt generation failed".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: error_msg,
+ details: None,
+ },
+ ));
+ }
+ _ => {
+ break Err(FirecrawlError::APIError(
+ "LLMs.txt generation status".to_string(),
+ crate::error::FirecrawlAPIError {
+ success: false,
+ error: format!("Unexpected status: {}", status_data.status),
+ details: None,
+ },
+ ));
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_generate_llms_text() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+
+ let params = GenerateLLMsTextParams {
+ url: "https://example.com".to_string(),
+ max_urls: 5,
+ show_full_text: true,
+ ..Default::default()
+ };
+
+ let response = app.async_generate_llms_text(params).await.unwrap();
+
+ assert!(response.success);
+ assert!(!response.id.is_empty());
+ }
+
+ #[tokio::test]
+ async fn test_async_generate_llms_text_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v1/llmstxt")
+ .match_body(mockito::Matcher::PartialJson(json!({
+ "url": "https://example.com",
+ "maxUrls": 5
+ })))
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "llmstxt-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = GenerateLLMsTextParams {
+ url: "https://example.com".to_string(),
+ max_urls: 5,
+ ..Default::default()
+ };
+
+ let response = app.async_generate_llms_text(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "llmstxt-123");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_check_generate_llms_text_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v1/llmstxt/llmstxt-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "processing",
+ "expiresAt": "2023-01-01T00:00:00Z"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = app
+ .check_generate_llms_text_status("llmstxt-123")
+ .await
+ .unwrap();
+
+ assert!(status.success);
+ assert_eq!(status.status, "processing");
+ assert_eq!(status.expires_at, "2023-01-01T00:00:00Z");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_generate_llms_text_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for the generate request
+ let mock = server
+ .mock("POST", "/v1/llmstxt")
+ .match_body(mockito::Matcher::PartialJson(json!({
+ "url": "https://example.com",
+ "showFullText": true
+ })))
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "llmstxt-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Set up the mock for the status request
+ let status_mock = server
+ .mock("GET", "/v1/llmstxt/llmstxt-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "data": {
+ "llmstxt": "Allow: /about\nDisallow: /admin\n",
+ "llmsfulltxt": "# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n"
+ },
+ "expiresAt": "2023-01-01T00:00:00Z"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = GenerateLLMsTextParams {
+ url: "https://example.com".to_string(),
+ show_full_text: true,
+ ..Default::default()
+ };
+
+ let response = app.generate_llms_text(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.status, "completed");
+
+ let data = response.data;
+ assert_eq!(
+ data.compact,
+ Some("Allow: /about\nDisallow: /admin\n".into())
+ );
+ assert_eq!(
+ data.full,
+ Some("# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n".into())
+ );
+
+ mock.assert();
+ status_mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_generate_llms_text_validation_errors() {
+ let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
+
+ // Test missing URL
+ let params = GenerateLLMsTextParams {
+ url: "".to_string(),
+ ..Default::default()
+ };
+ let result = app.async_generate_llms_text(params).await;
+ assert!(result.is_err());
+ }
+
+ #[tokio::test]
+ async fn test_generate_llms_text_api_error() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Set up the mock for an error response
+ let mock = server
+ .mock("POST", "/v1/llmstxt")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid URL format"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let params = GenerateLLMsTextParams {
+ url: "not-a-valid-url".to_string(),
+ ..Default::default()
+ };
+
+ let result = app.async_generate_llms_text(params).await;
+ assert!(result.is_err());
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rs
index 7c3b3a438..44016064d 100644
--- a/apps/rust-sdk/src/map.rs
+++ b/apps/rust-sdk/src/map.rs
@@ -16,7 +16,7 @@ pub struct MapOptions {
pub include_subdomains: Option<bool>,
/// Maximum number of links to return (default: `5000`)
- pub exclude_tags: Option<u32>,
+ pub limit: Option<u32>,
}
#[derive(Deserialize, Serialize, Debug, Default)]
@@ -59,7 +59,9 @@ impl FirecrawlApp {
.await
.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
- let response = self.handle_response::<MapResponse>(response, "scrape URL").await?;
+ let response = self
+ .handle_response::<MapResponse>(response, "scrape URL")
+ .await?;
Ok(response.links)
}
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index b879fdaf3..6432b04a7 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -24,26 +24,26 @@ pub enum ScrapeFormats {
Links,
/// Will result in a URL to a screenshot of the page.
- ///
+ ///
/// Can not be used in conjunction with `ScrapeFormats::ScreenshotFullPage`.
#[serde(rename = "screenshot")]
Screenshot,
/// Will result in a URL to a full-page screenshot of the page.
- ///
+ ///
/// Can not be used in conjunction with `ScrapeFormats::Screenshot`.
#[serde(rename = "screenshot@fullPage")]
ScreenshotFullPage,
/// Will result in the results of an LLM extraction.
- ///
+ ///
/// See `ScrapeOptions.extract` for more options.
#[serde(rename = "extract")]
Extract,
}
#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default)]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ExtractOptions {
/// Schema the output should adhere to, provided in JSON Schema format.
@@ -56,7 +56,7 @@ pub struct ExtractOptions {
}
#[serde_with::skip_serializing_none]
-#[derive(Deserialize, Serialize, Debug, Default)]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeOptions {
/// Formats to extract from the page. (default: `[ Markdown ]`)
@@ -66,12 +66,12 @@ pub struct ScrapeOptions {
pub only_main_content: Option<bool>,
/// HTML tags to exclusively include.
- ///
+ ///
/// For example, if you pass `div`, you will only get content from `<div>`s and their children.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude.
- ///
+ ///
/// For example, if you pass `img`, you will never get image URLs in your results.
pub exclude_tags: Option<Vec<String>>,
@@ -131,7 +131,9 @@ impl FirecrawlApp {
.await
.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
- let response = self.handle_response::<ScrapeResponse>(response, "scrape URL").await?;
+ let response = self
+ .handle_response::<ScrapeResponse>(response, "scrape URL")
+ .await?;
Ok(response.data)
}
diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs
new file mode 100644
index 000000000..397bc8d16
--- /dev/null
+++ b/apps/rust-sdk/src/search.rs
@@ -0,0 +1,245 @@
+use crate::{scrape::ScrapeOptions, FirecrawlApp, FirecrawlError, API_VERSION};
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchParams {
+ /// The search query string
+ pub query: String,
+ /// Maximum number of results to return. Default: 5, Max: 20
+ pub limit: Option<u32>,
+ /// Time-based search filter.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub tbs: Option<String>,
+ /// Query string to filter search results. Example: "site:example.com"
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub filter: Option<String>,
+ /// Language code. Default: "en"
+ pub lang: Option<String>,
+ /// Country code. Default: "us"
+ pub country: Option<String>,
+ /// Geographic location string for local search results
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub location: Option<String>,
+ /// Origin identifier. Default: "api"
+ pub origin: Option<String>,
+ /// Timeout in milliseconds. Default: 60000
+ pub timeout: Option<u32>,
+ /// Additional options for webpage scraping behavior
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub scrape_options: Option<ScrapeOptions>,
+}
+
+impl Default for SearchParams {
+ fn default() -> Self {
+ Self {
+ query: String::new(),
+ limit: Some(5),
+ tbs: None,
+ filter: None,
+ lang: Some("en".to_string()),
+ country: Some("us".to_string()),
+ location: None,
+ origin: Some("api".to_string()),
+ timeout: Some(60000),
+ scrape_options: None,
+ }
+ }
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchResponse {
+ pub success: bool,
+ pub data: Vec<SearchDocument>,
+ pub warning: Option<String>,
+}
+
+// TODO: Consider merging fields into document::Document (url, title, description) while preserving optionality
+/// A document returned from a search or scrape request
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchDocument {
+ /// Document URL
+ pub url: String,
+ /// Document title
+ pub title: String,
+ /// Document description
+ pub description: String,
+}
+
+impl FirecrawlApp {
+ /// Search for content using the Firecrawl API.
+ ///
+ /// # Arguments
+ ///
+ /// * `query` - The search query string
+ /// * `params` - Optional parameters for the search request
+ ///
+ /// # Returns
+ ///
+ /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
+ pub async fn search(
+ &self,
+ query: impl AsRef<str>,
+ params: impl Into<Option<SearchParams>>,
+ ) -> Result<SearchResponse, FirecrawlError> {
+ let mut search_params = params.into().unwrap_or_default();
+ search_params.query = query.as_ref().to_string();
+
+ self.search_with_params(search_params).await
+ }
+
+ /// Alternative method that takes SearchParams directly
+ ///
+ /// # Arguments
+ ///
+ /// * `params` - Search parameters including the query
+ ///
+ /// # Returns
+ ///
+ /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
+ pub async fn search_with_params(
+ &self,
+ params: SearchParams,
+ ) -> Result<SearchResponse, FirecrawlError> {
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(format!("{}{}/search", self.api_url, API_VERSION))
+ .headers(headers)
+ .json(&params)
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Searching with query: {:?}", params.query), e)
+ })?;
+
+ self.handle_response::<SearchResponse>(response, "search")
+ .await
+ }
+}
+
+#[cfg(test)]
+pub mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ #[ignore = "Makes real network request"]
+ async fn test_real_search() {
+ let api_url = std::env::var("FIRECRAWL_API_URL")
+ .expect("Please set the FIRECRAWL_API_URL environment variable");
+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
+ let response = app.search("test query", None).await.unwrap();
+ assert!(response.success);
+ }
+
+ #[tokio::test]
+ async fn test_search_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v1/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": [{
+ "url": "https://example.com",
+ "title": "Example Domain",
+ "description": "...."
+ }],
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = app.search("test", None).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.data.len(), 1);
+ assert_eq!(response.data[0].url, "https://example.com");
+ assert_eq!(response.data[0].title, "Example Domain".to_string());
+ assert_eq!(response.data[0].description, "....".to_string());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_with_params() {
+ let mut server = mockito::Server::new_async().await;
+ let mock = server
+ .mock("POST", "/v1/search")
+ .with_header("content-type", "application/json")
+ .match_body(mockito::Matcher::Json(json!({
+ "query": "test",
+ "limit": 10,
+ "lang": "fr",
+ "country": "fr",
+ "origin": "api",
+ "timeout": 30000
+ })))
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": [],
+ "warning": "No results found"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let params = SearchParams {
+ query: "test".to_string(),
+ limit: Some(10),
+ lang: Some("fr".to_string()),
+ country: Some("fr".to_string()),
+ timeout: Some(30000),
+ ..Default::default()
+ };
+
+ let response = app.search_with_params(params).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.data.len(), 0);
+ assert_eq!(response.warning, Some("No results found".to_string()));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_error_response() {
+ let mut server = mockito::Server::new_async().await;
+ let mock = server
+ .mock("POST", "/v1/search")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid query"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = app.search("", None).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_network_error() {
+ let app = FirecrawlApp::new_selfhosted("http://invalid-url", Some("test_key")).unwrap();
+ let result = app.search("test", None).await;
+ assert!(result.is_err());
+ }
+}
diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs
index 882a29413..071537f50 100644
--- a/apps/rust-sdk/tests/e2e_with_auth.rs
+++ b/apps/rust-sdk/tests/e2e_with_auth.rs
@@ -1,4 +1,3 @@
-use assert_matches::assert_matches;
use dotenvy::dotenv;
use firecrawl::scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions};
use firecrawl::{FirecrawlApp, FirecrawlError};
@@ -24,11 +23,8 @@ use std::env;
async fn test_successful_response_with_valid_preview_token() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
- let app = FirecrawlApp::new_selfhosted(
- api_url,
- Some(env::var("PREVIEW_TOKEN").unwrap()),
- )
- .unwrap();
+ let app =
+ FirecrawlApp::new_selfhosted(api_url, Some(env::var("PREVIEW_TOKEN").unwrap())).unwrap();
let result = app
.scrape_url("https://roastmywebsite.ai", None)
.await
@@ -58,7 +54,7 @@ async fn test_successful_response_with_valid_api_key_and_include_html() {
let api_key = env::var("TEST_API_KEY").ok();
let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
let params = ScrapeOptions {
- formats: vec! [ ScrapeFormats::Markdown, ScrapeFormats::HTML ].into(),
+ formats: vec![ScrapeFormats::Markdown, ScrapeFormats::HTML].into(),
..Default::default()
};
let result = app
@@ -82,7 +78,8 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file() {
.await
.unwrap();
assert!(result.markdown.is_some());
- assert!(result.markdown
+ assert!(result
+ .markdown
.unwrap()
.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
}
@@ -98,12 +95,12 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici
.await
.unwrap();
assert!(result.markdown.is_some());
- assert!(result.markdown
+ assert!(result
+ .markdown
.unwrap()
.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
}
-
// #[tokio::test]
// async fn test_should_return_error_for_blocklisted_url() {
// dotenv().ok();
@@ -159,14 +156,18 @@ async fn test_llm_extraction() {
#[test]
fn test_api_key_requirements() {
dotenv().ok();
-
+
let api_url = env::var("API_URL").unwrap_or("http://localhost:3002".to_string());
let api_key = env::var("TEST_API_KEY").ok();
match (api_url.contains("api.firecrawl.dev"), api_key) {
(false, _) => {
let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);
- assert!(result.is_ok(), "Local setup failed: {:?}", result.err().unwrap());
+ assert!(
+ result.is_ok(),
+ "Local setup failed: {:?}",
+ result.err().unwrap()
+ );
}
(true, None) => {
let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);