Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(rust): update rust sdk to support new features (#1446)
kkharjif2c01340d19b
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 2ea5de692..d7a7b64e4 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -17,6 +17,21 @@ version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"+[[package]]+name = "adler2"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"++[[package]]+name = "aho-corasick"+version = "1.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"+dependencies = [+ "memchr",+]+[[package]]name = "android-tzdata"version = "0.1.1"@@ -32,6 +47,71 @@ dependencies = ["libc",]+[[package]]+name = "ansi_colours"+version = "1.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "14eec43e0298190790f41679fe69ef7a829d2a2ddd78c8c00339e84710e435fe"+dependencies = [+ "rgb",+]++[[package]]+name = "anstream"+version = "0.6.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"+dependencies = [+ "anstyle",+ "anstyle-parse",+ "anstyle-query",+ "anstyle-wincon",+ "colorchoice",+ "is_terminal_polyfill",+ "utf8parse",+]++[[package]]+name = "anstyle"+version = "1.0.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"++[[package]]+name = "anstyle-parse"+version = "0.2.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"+dependencies = [+ "utf8parse",+]++[[package]]+name = "anstyle-query"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"+dependencies = [+ "windows-sys 0.59.0",+]++[[package]]+name = "anstyle-wincon"+version = "3.0.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"+dependencies = [+ "anstyle",+ "once_cell",+ "windows-sys 0.59.0",+]++[[package]]+name = "anyhow"+version = "1.0.97"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"+[[package]]name = "arrayref"version = "0.3.7"@@ -44,6 +124,16 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"+[[package]]+name = "assert-json-diff"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "assert_matches"version = "1.5.0"@@ -62,6 +152,72 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"+[[package]]+name = "axum"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288"+dependencies = [+ "axum-core",+ "axum-macros",+ "bytes",+ "form_urlencoded",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-util",+ "itoa",+ "matchit",+ "memchr",+ "mime",+ "percent-encoding",+ "pin-project-lite",+ "rustversion",+ "serde",+ "serde_json",+ "serde_path_to_error",+ "serde_urlencoded",+ "sync_wrapper",+ "tokio",+ "tower 0.5.2",+ "tower-layer",+ "tower-service",+ "tracing",+]++[[package]]+name = "axum-core"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "http-body-util",+ "mime",+ "pin-project-lite",+ "rustversion",+ "sync_wrapper",+ "tower-layer",+ "tower-service",+ "tracing",+]++[[package]]+name = "axum-macros"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "backtrace"version = "0.3.73"@@ -72,7 +228,7 @@ dependencies = ["cc","cfg-if","libc",- "miniz_oxide",+ "miniz_oxide 0.7.4","object","rustc-demangle",]@@ -89,6 +245,59 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "bat"+version = "0.25.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ab792c2ad113a666f08856c88cdec0a62d732559b1f3982eedf0142571e669a"+dependencies = [+ "ansi_colours",+ "anyhow",+ "bincode",+ "bugreport",+ "bytesize",+ "clap",+ "clircle",+ "console",+ "content_inspector",+ "encoding_rs",+ "etcetera",+ "flate2",+ "git2",+ "globset",+ "grep-cli",+ "home",+ "indexmap 2.9.0",+ "itertools",+ "nu-ansi-term",+ "once_cell",+ "path_abs",+ "plist",+ "regex",+ "semver",+ "serde",+ "serde_derive",+ "serde_with",+ "serde_yaml",+ "shell-words",+ "syntect",+ "terminal-colorsaurus",+ "thiserror",+ "toml",+ "unicode-width 0.1.14",+ "walkdir",+ "wild",+]++[[package]]+name = "bincode"+version = "1.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"+dependencies = [+ "serde",+]+[[package]]name = "bitflags"version = "1.3.2"@@ -112,12 +321,40 @@ dependencies = ["constant_time_eq",]+[[package]]+name = "bstr"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"+dependencies = [+ "memchr",+ "regex-automata",+ "serde",+]++[[package]]+name = "bugreport"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f280f65ce85b880919349bbfcb204930291251eedcb2e5f84ce2f51df969c162"+dependencies = [+ "git-version",+ "shell-escape",+ "sysinfo",+]+[[package]]name = "bumpalo"version = "3.16.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"+[[package]]+name = "bytemuck"+version = "1.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540"+[[package]]name = "byteorder"version = "1.5.0"@@ -130,11 +367,22 @@ version = "1.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"+[[package]]+name = "bytesize"+version = "1.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"+[[package]]name = "cc"version = "1.0.105"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437"+dependencies = [+ "jobserver",+ "libc",+ "once_cell",+][[package]]name = "cfg-if"@@ -155,6 +403,47 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "clap"+version = "4.5.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"+dependencies = [+ "clap_builder",+ "clap_derive",+]++[[package]]+name = "clap_builder"+version = "4.5.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"+dependencies = [+ "anstream",+ "anstyle",+ "clap_lex",+ "strsim",+ "terminal_size",+]++[[package]]+name = "clap_derive"+version = "4.5.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7"+dependencies = [+ "heck",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "clap_lex"+version = "0.7.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"+[[package]]name = "clippy"version = "0.0.302"@@ -164,12 +453,59 @@ dependencies = ["term",]+[[package]]+name = "clircle"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7d9334f725b46fb9bed8580b9b47a932587e044fadb344ed7fa98774b067ac1a"+dependencies = [+ "cfg-if",+ "windows 0.56.0",+]++[[package]]+name = "colorchoice"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"++[[package]]+name = "colored"+version = "3.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e"+dependencies = [+ "windows-sys 0.59.0",+]++[[package]]+name = "console"+version = "0.15.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"+dependencies = [+ "encode_unicode",+ "libc",+ "once_cell",+ "unicode-width 0.2.0",+ "windows-sys 0.59.0",+]+[[package]]name = "constant_time_eq"version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"+[[package]]+name = "content_inspector"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38"+dependencies = [+ "memchr",+]+[[package]]name = "core-foundation"version = "0.9.4"@@ -182,9 +518,37 @@ dependencies = [[[package]]name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"++[[package]]+name = "crc32fast"+version = "1.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"+dependencies = [+ "cfg-if",+]++[[package]]+name = "crossbeam-deque"version = "0.8.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"+dependencies = [+ "crossbeam-epoch",+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-epoch"+version = "0.9.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"+dependencies = [+ "crossbeam-utils",+][[package]]name = "crossbeam-utils"@@ -254,11 +618,29 @@ version = "0.15.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"+[[package]]+name = "dyn-clone"+version = "1.0.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"++[[package]]+name = "either"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"++[[package]]+name = "encode_unicode"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"+[[package]]name = "encoding_rs"-version = "0.8.34"+version = "0.8.35"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"dependencies = ["cfg-if",]@@ -279,6 +661,17 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "etcetera"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"+dependencies = [+ "cfg-if",+ "home",+ "windows-sys 0.48.0",+]+[[package]]name = "fastrand"version = "2.1.0"@@ -287,13 +680,19 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"[[package]]name = "firecrawl"-version = "1.0.0"+version = "1.1.0"dependencies = ["assert_matches",+ "axum",+ "bat",+ "clap","clippy","dotenvy",+ "futures","log",+ "mockito","reqwest",+ "schemars","serde","serde_json","serde_with",@@ -302,6 +701,16 @@ dependencies = ["uuid",]+[[package]]+name = "flate2"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"+dependencies = [+ "crc32fast",+ "miniz_oxide 0.8.8",+]+[[package]]name = "fnv"version = "1.0.7"@@ -332,11 +741,26 @@ dependencies = ["percent-encoding",]+[[package]]+name = "futures"+version = "0.3.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"+dependencies = [+ "futures-channel",+ "futures-core",+ "futures-executor",+ "futures-io",+ "futures-sink",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-channel"-version = "0.3.30"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"dependencies = ["futures-core","futures-sink",@@ -344,36 +768,60 @@ dependencies = [[[package]]name = "futures-core"-version = "0.3.30"+version = "0.3.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"++[[package]]+name = "futures-executor"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+][[package]]name = "futures-io"-version = "0.3.30"+version = "0.3.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"++[[package]]+name = "futures-macro"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+][[package]]name = "futures-sink"-version = "0.3.30"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"[[package]]name = "futures-task"-version = "0.3.30"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"[[package]]name = "futures-util"-version = "0.3.30"+version = "0.3.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"dependencies = [+ "futures-channel","futures-core","futures-io",+ "futures-macro","futures-sink","futures-task","memchr",@@ -386,29 +834,107 @@ dependencies = [name = "getrandom"version = "0.1.16"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"+dependencies = [+ "cfg-if",+ "libc",+ "wasi 0.9.0+wasi-snapshot-preview1",+]++[[package]]+name = "getrandom"+version = "0.2.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"+dependencies = [+ "cfg-if",+ "libc",+ "wasi 0.11.0+wasi-snapshot-preview1",+]++[[package]]+name = "getrandom"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi",+ "wasi 0.14.2+wasi-0.2.4",+]++[[package]]+name = "gimli"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"++[[package]]+name = "git-version"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19"+dependencies = [+ "git-version-macro",+]++[[package]]+name = "git-version-macro"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "git2"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"dependencies = [- "cfg-if",+ "bitflags 2.6.0","libc",- "wasi 0.9.0+wasi-snapshot-preview1",+ "libgit2-sys",+ "log",+ "url",][[package]]-name = "getrandom"-version = "0.2.15"+name = "glob"+version = "0.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"++[[package]]+name = "globset"+version = "0.4.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"dependencies = [- "cfg-if",- "libc",- "wasi 0.11.0+wasi-snapshot-preview1",+ "aho-corasick",+ "bstr",+ "log",+ "regex-automata",+ "regex-syntax",][[package]]-name = "gimli"-version = "0.29.0"+name = "grep-cli"+version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"+checksum = "47f1288f0e06f279f84926fa4c17e3fcd2a22b357927a82f2777f7be26e4cec0"+dependencies = [+ "bstr",+ "globset",+ "libc",+ "log",+ "termcolor",+ "winapi-util",+][[package]]name = "h2"@@ -422,7 +948,7 @@ dependencies = ["futures-core","futures-sink","http",- "indexmap 2.2.6",+ "indexmap 2.9.0","slab","tokio","tokio-util",@@ -437,9 +963,15 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"[[package]]name = "hashbrown"-version = "0.14.5"+version = "0.15.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"++[[package]]+name = "heck"+version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"[[package]]name = "hermit-abi"@@ -453,6 +985,15 @@ version = "0.4.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"+[[package]]+name = "home"+version = "0.5.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"+dependencies = [+ "windows-sys 0.59.0",+]+[[package]]name = "http"version = "1.1.0"@@ -493,11 +1034,17 @@ version = "1.9.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"+[[package]]+name = "httpdate"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"+[[package]]name = "hyper"-version = "1.4.1"+version = "1.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"dependencies = ["bytes","futures-channel",@@ -506,6 +1053,7 @@ dependencies = ["http","http-body","httparse",+ "httpdate","itoa","pin-project-lite","smallvec",@@ -561,7 +1109,7 @@ dependencies = ["pin-project-lite","socket2","tokio",- "tower",+ "tower 0.4.13","tower-service","tracing",]@@ -577,7 +1125,7 @@ dependencies = ["iana-time-zone-haiku","js-sys","wasm-bindgen",- "windows-core",+ "windows-core 0.52.0",][[package]]@@ -618,12 +1166,12 @@ dependencies = [[[package]]name = "indexmap"-version = "2.2.6"+version = "2.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"dependencies = ["equivalent",- "hashbrown 0.14.5",+ "hashbrown 0.15.2","serde",]@@ -633,12 +1181,37 @@ version = "2.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"+[[package]]+name = "is_terminal_polyfill"+version = "1.70.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"++[[package]]+name = "itertools"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"+[[package]]+name = "jobserver"+version = "0.1.33"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"+dependencies = [+ "getrandom 0.3.2",+ "libc",+]+[[package]]name = "js-sys"version = "0.3.69"@@ -650,9 +1223,39 @@ dependencies = [[[package]]name = "libc"-version = "0.2.155"+version = "0.2.171"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"++[[package]]+name = "libgit2-sys"+version = "0.17.0+1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224"+dependencies = [+ "cc",+ "libc",+ "libz-sys",+ "pkg-config",+]++[[package]]+name = "libz-sys"+version = "1.1.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+ "vcpkg",+]++[[package]]+name = "linked-hash-map"+version = "0.5.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"[[package]]name = "linux-raw-sys"@@ -676,6 +1279,12 @@ version = "0.4.22"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"+[[package]]+name = "matchit"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"+[[package]]name = "memchr"version = "2.7.4"@@ -697,6 +1306,15 @@ dependencies = ["adler",]+[[package]]+name = "miniz_oxide"+version = "0.8.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"+dependencies = [+ "adler2",+]+[[package]]name = "mio"version = "1.0.2"@@ -709,6 +1327,30 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "mockito"+version = "1.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"+dependencies = [+ "assert-json-diff",+ "bytes",+ "colored",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-util",+ "log",+ "rand",+ "regex",+ "serde_json",+ "serde_urlencoded",+ "similar",+ "tokio",+]+[[package]]name = "native-tls"version = "0.2.12"@@ -726,6 +1368,24 @@ dependencies = ["tempfile",]+[[package]]+name = "ntapi"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"+dependencies = [+ "winapi",+]++[[package]]+name = "nu-ansi-term"+version = "0.50.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"+dependencies = [+ "windows-sys 0.52.0",+]+[[package]]name = "num-conv"version = "0.1.0"@@ -752,9 +1412,31 @@ dependencies = [[[package]]name = "once_cell"-version = "1.19.0"+version = "1.21.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"++[[package]]+name = "onig"+version = "6.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"+dependencies = [+ "bitflags 1.3.2",+ "libc",+ "once_cell",+ "onig_sys",+]++[[package]]+name = "onig_sys"+version = "69.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"+dependencies = [+ "cc",+ "pkg-config",+][[package]]name = "openssl"@@ -823,6 +1505,15 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "path_abs"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3"+dependencies = [+ "std_prelude",+]+[[package]]name = "percent-encoding"version = "2.3.1"@@ -867,12 +1558,34 @@ version = "0.3.30"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"+[[package]]+name = "plist"+version = "1.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eac26e981c03a6e53e0aee43c113e3202f5581d5360dae7bd2c70e800dd0451d"+dependencies = [+ "base64 0.22.1",+ "indexmap 2.9.0",+ "quick-xml",+ "serde",+ "time",+]+[[package]]name = "powerfmt"version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"+[[package]]+name = "ppv-lite86"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"+dependencies = [+ "zerocopy",+]+[[package]]name = "proc-macro2"version = "1.0.86"@@ -882,6 +1595,15 @@ dependencies = ["unicode-ident",]+[[package]]+name = "quick-xml"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d3a6e5838b60e0e8fa7a43f22ade549a37d61f8bdbe636d0d7816191de969c2"+dependencies = [+ "memchr",+]+[[package]]name = "quote"version = "1.0.36"@@ -891,6 +1613,62 @@ dependencies = ["proc-macro2",]+[[package]]+name = "r-efi"+version = "5.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"++[[package]]+name = "rand"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"+dependencies = [+ "rand_chacha",+ "rand_core",+ "zerocopy",+]++[[package]]+name = "rand_chacha"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"+dependencies = [+ "ppv-lite86",+ "rand_core",+]++[[package]]+name = "rand_core"+version = "0.9.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"+dependencies = [+ "getrandom 0.3.2",+]++[[package]]+name = "rayon"+version = "1.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"+dependencies = [+ "either",+ "rayon-core",+]++[[package]]+name = "rayon-core"+version = "1.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"+dependencies = [+ "crossbeam-deque",+ "crossbeam-utils",+]+[[package]]name = "redox_syscall"version = "0.1.57"@@ -917,6 +1695,35 @@ dependencies = ["rust-argon2",]+[[package]]+name = "regex"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.4.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"+[[package]]name = "reqwest"version = "0.12.5"@@ -961,6 +1768,15 @@ dependencies = ["winreg",]+[[package]]+name = "rgb"+version = "0.8.50"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"+dependencies = [+ "bytemuck",+]+[[package]]name = "ring"version = "0.17.8"@@ -1047,12 +1863,27 @@ dependencies = ["untrusted",]+[[package]]+name = "rustversion"+version = "1.0.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"+[[package]]name = "ryu"version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"+[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]+[[package]]name = "schannel"version = "0.1.23"@@ -1062,6 +1893,30 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "schemars"+version = "0.8.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"+dependencies = [+ "dyn-clone",+ "schemars_derive",+ "serde",+ "serde_json",+]++[[package]]+name = "schemars_derive"+version = "0.8.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"+dependencies = [+ "proc-macro2",+ "quote",+ "serde_derive_internals",+ "syn",+]+[[package]]name = "scopeguard"version = "1.2.0"@@ -1091,6 +1946,12 @@ dependencies = ["libc",]+[[package]]+name = "semver"+version = "1.0.26"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"+[[package]]name = "serde"version = "1.0.204"@@ -1112,13 +1973,43 @@ dependencies = [][[package]]-name = "serde_json"-version = "1.0.120"+name = "serde_derive_internals"+version = "0.29.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "serde_json"+version = "1.0.120"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"+dependencies = [+ "itoa",+ "ryu",+ "serde",+]++[[package]]+name = "serde_path_to_error"+version = "0.1.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"+dependencies = [+ "itoa",+ "serde",+]++[[package]]+name = "serde_spanned"+version = "0.6.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"dependencies = [- "itoa",- "ryu","serde",]@@ -1136,15 +2027,15 @@ dependencies = [[[package]]name = "serde_with"-version = "3.9.0"+version = "3.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857"+checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"dependencies = ["base64 0.22.1","chrono","hex","indexmap 1.9.3",- "indexmap 2.2.6",+ "indexmap 2.9.0","serde","serde_derive","serde_json",@@ -1154,9 +2045,9 @@ dependencies = [[[package]]name = "serde_with_macros"-version = "3.9.0"+version = "3.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350"+checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"dependencies = ["darling","proc-macro2",@@ -1164,6 +2055,31 @@ dependencies = ["syn",]+[[package]]+name = "serde_yaml"+version = "0.9.34+deprecated"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"+dependencies = [+ "indexmap 2.9.0",+ "itoa",+ "ryu",+ "serde",+ "unsafe-libyaml",+]++[[package]]+name = "shell-escape"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"++[[package]]+name = "shell-words"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"+[[package]]name = "signal-hook-registry"version = "1.4.2"@@ -1173,6 +2089,12 @@ dependencies = ["libc",]+[[package]]+name = "similar"+version = "2.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"+[[package]]name = "slab"version = "0.4.9"@@ -1204,6 +2126,12 @@ version = "0.9.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"+[[package]]+name = "std_prelude"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe"+[[package]]name = "strsim"version = "0.11.1"@@ -1233,6 +2161,42 @@ version = "1.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"+[[package]]+name = "syntect"+version = "5.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"+dependencies = [+ "bincode",+ "bitflags 1.3.2",+ "flate2",+ "fnv",+ "once_cell",+ "onig",+ "plist",+ "regex-syntax",+ "serde",+ "serde_derive",+ "serde_json",+ "thiserror",+ "walkdir",+ "yaml-rust",+]++[[package]]+name = "sysinfo"+version = "0.33.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"+dependencies = [+ "core-foundation-sys",+ "libc",+ "memchr",+ "ntapi",+ "rayon",+ "windows 0.57.0",+]+[[package]]name = "system-configuration"version = "0.5.1"@@ -1277,6 +2241,51 @@ dependencies = ["winapi",]+[[package]]+name = "termcolor"+version = "1.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"+dependencies = [+ "winapi-util",+]++[[package]]+name = "terminal-colorsaurus"+version = "0.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7afe4c174a3cbfb52ebcb11b28965daf74fe9111d4e07e40689d05af06e26e8"+dependencies = [+ "cfg-if",+ "libc",+ "memchr",+ "mio",+ "terminal-trx",+ "windows-sys 0.59.0",+ "xterm-color",+]++[[package]]+name = "terminal-trx"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "975b4233aefa1b02456d5e53b22c61653c743e308c51cf4181191d8ce41753ab"+dependencies = [+ "cfg-if",+ "libc",+ "windows-sys 0.59.0",+]++[[package]]+name = "terminal_size"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9"+dependencies = [+ "rustix",+ "windows-sys 0.59.0",+]+[[package]]name = "thiserror"version = "1.0.61"@@ -1345,9 +2354,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"[[package]]name = "tokio"-version = "1.42.0"+version = "1.44.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"+checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"dependencies = ["backtrace","bytes",@@ -1363,9 +2372,9 @@ dependencies = [[[package]]name = "tokio-macros"-version = "2.4.0"+version = "2.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"dependencies = ["proc-macro2","quote",@@ -1406,6 +2415,41 @@ dependencies = ["tokio",]+[[package]]+name = "toml"+version = "0.8.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"+dependencies = [+ "indexmap 2.9.0",+ "serde",+ "serde_spanned",+ "toml_datetime",+ "toml_edit",+]++[[package]]+name = "toml_datetime"+version = "0.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"+dependencies = [+ "serde",+]++[[package]]+name = "toml_edit"+version = "0.22.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"+dependencies = [+ "indexmap 2.9.0",+ "serde",+ "serde_spanned",+ "toml_datetime",+ "winnow",+]+[[package]]name = "tower"version = "0.4.13"@@ -1421,17 +2465,33 @@ dependencies = ["tower-service",]+[[package]]+name = "tower"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"+dependencies = [+ "futures-core",+ "futures-util",+ "pin-project-lite",+ "sync_wrapper",+ "tokio",+ "tower-layer",+ "tower-service",+ "tracing",+]+[[package]]name = "tower-layer"-version = "0.3.2"+version = "0.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"[[package]]name = "tower-service"-version = "0.3.2"+version = "0.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"[[package]]name = "tracing"@@ -1439,6 +2499,7 @@ version = "0.1.40"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"dependencies = [+ "log","pin-project-lite","tracing-core",]@@ -1479,6 +2540,24 @@ dependencies = ["tinyvec",]+[[package]]+name = "unicode-width"+version = "0.1.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"++[[package]]+name = "unicode-width"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"++[[package]]+name = "unsafe-libyaml"+version = "0.2.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"+[[package]]name = "untrusted"version = "0.9.0"@@ -1496,6 +2575,12 @@ dependencies = ["percent-encoding",]+[[package]]+name = "utf8parse"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"+[[package]]name = "uuid"version = "1.10.0"@@ -1511,6 +2596,16 @@ version = "0.2.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"+[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]+[[package]]name = "want"version = "0.3.1"@@ -1532,6 +2627,15 @@ version = "0.11.0+wasi-snapshot-preview1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"+[[package]]+name = "wasi"+version = "0.14.2+wasi-0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"+dependencies = [+ "wit-bindgen-rt",+]+[[package]]name = "wasm-bindgen"version = "0.2.92"@@ -1608,6 +2712,15 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "wild"+version = "2.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1"+dependencies = [+ "glob",+]+[[package]]name = "winapi"version = "0.3.9"@@ -1624,12 +2737,41 @@ version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"+[[package]]+name = "winapi-util"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"+dependencies = [+ "windows-sys 0.59.0",+]+[[package]]name = "winapi-x86_64-pc-windows-gnu"version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"+[[package]]+name = "windows"+version = "0.56.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132"+dependencies = [+ "windows-core 0.56.0",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows"+version = "0.57.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"+dependencies = [+ "windows-core 0.57.0",+ "windows-targets 0.52.6",+]+[[package]]name = "windows-core"version = "0.52.0"@@ -1639,6 +2781,83 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "windows-core"+version = "0.56.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6"+dependencies = [+ "windows-implement 0.56.0",+ "windows-interface 0.56.0",+ "windows-result",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-core"+version = "0.57.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"+dependencies = [+ "windows-implement 0.57.0",+ "windows-interface 0.57.0",+ "windows-result",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-implement"+version = "0.56.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-implement"+version = "0.57.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.56.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.57.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-result"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"+dependencies = [+ "windows-targets 0.52.6",+]+[[package]]name = "windows-sys"version = "0.48.0"@@ -1657,6 +2876,15 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "windows-sys"+version = "0.59.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"+dependencies = [+ "windows-targets 0.52.6",+]+[[package]]name = "windows-targets"version = "0.48.5"@@ -1778,6 +3006,15 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"+[[package]]+name = "winnow"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"+dependencies = [+ "memchr",+]+[[package]]name = "winreg"version = "0.52.0"@@ -1788,6 +3025,50 @@ dependencies = ["windows-sys 0.48.0",]+[[package]]+name = "wit-bindgen-rt"+version = "0.39.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"+dependencies = [+ "bitflags 2.6.0",+]++[[package]]+name = "xterm-color"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4de5f056fb9dc8b7908754867544e26145767187aaac5a98495e88ad7cb8a80f"++[[package]]+name = "yaml-rust"+version = "0.4.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"+dependencies = [+ "linked-hash-map",+]++[[package]]+name = "zerocopy"+version = "0.8.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"+dependencies = [+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.8.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "zeroize"version = "1.8.1"diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 6ea8d1790..3affd8640 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,13 +1,13 @@[package]name = "firecrawl"author= "Mendable.ai"-version = "1.0.0"+version = "1.1.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"repository ="https://github.com/mendableai/firecrawl"description = "Rust SDK for Firecrawl API."-authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>"]+authors = ["Gergő Móricz <[email protected]>", "sanix-darker <[email protected]>", "kkharji <[email protected]>"][lib]path = "src/lib.rs"@@ -23,12 +23,18 @@ log = "^0.4"thiserror = "^1.0"uuid = { version = "^1.10", features = ["v4"] }tokio = { version = "^1", features = ["full"] }+futures = "0.3.31"+schemars = "0.8.22"[dev-dependencies]clippy = "^0.0.302"assert_matches = "^1.5"dotenvy = "^0.15"tokio = { version = "1", features = ["full"] }+mockito = "1.7.0"+clap = { version ="4.5.35", features = ["derive"] }+axum = { version = "0.8.3", features = ["tokio", "macros"] }+bat = "0.25.0"[build-dependencies]tokio = { version = "1", features = ["full"] }diff --git a/apps/rust-sdk/examples/batch_scrape_example.rs b/apps/rust-sdk/examples/batch_scrape_example.rsnew file mode 100644index 000000000..7db7856ec--- /dev/null+++ b/apps/rust-sdk/examples/batch_scrape_example.rs@@ -0,0 +1,175 @@+use clap::{Parser, Subcommand};+use firecrawl::{+ batch_scrape::{BatchScrapeParams, WebhookOptions},+ map::MapOptions,+ scrape::{ScrapeFormats, ScrapeOptions},+ FirecrawlApp,+};+use serde_json::Value;+use std::error::Error;+use std::net::SocketAddr;+use std::sync::Arc;+use tokio::sync::Mutex;++// Store webhook responses+struct WebhookState {+ responses: Vec<Value>,+}++#[derive(Parser)]+#[command(version, about, long_about = None)]+struct Cli {+ #[command(subcommand)]+ command: Commands,+}++#[derive(Subcommand)]+enum Commands {+ /// Multiple URL scraping with webhook monitoring+ Basic,+}++async fn create_firecrawl_app() -> Result<FirecrawlApp, Box<dyn Error>> {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ FirecrawlApp::new_selfhosted(api_url, None::<&str>).map_err(|e| e.into())+}++// Start webhook server and return its address+async fn start_webhook_server(+ port: u16,+ state: Arc<Mutex<WebhookState>>,+) -> Result<String, Box<dyn Error>> {+ let state = state.clone();+ use axum::routing::post;+ use axum::Json;++ let app = axum::Router::new().route(+ "/",+ post(move |body: Json<Value>| {+ let state = state.clone();+ async move {+ state.lock().await.responses.push(body.0.clone());+ match serde_json::to_string_pretty(&body.0) {+ Ok(data) => println!(+ "Received webhook: {}",+ serde_json::to_string_pretty(&data).unwrap()+ ),+ Err(_) => println!("Received webhook: {}", body.0),+ }+ "OK"+ }+ }),+ );++ let addr = SocketAddr::from(([0, 0, 0, 0], port));+ let webhook_url = format!("http://host.docker.internal:{}", port);++ tokio::spawn(async move {+ let listener = tokio::net::TcpListener::bind(addr)+ .await+ .inspect_err(|err| println!("{err:?}"))+ .unwrap();++ if let Err(e) = axum::serve(listener, app).await {+ eprintln!("Webhook server error: {}", e);+ }+ });++ println!("Webhook server running at {}", webhook_url);++ Ok(webhook_url)+}++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ let cli = Cli::parse();+ let firecrawl = create_firecrawl_app().await?;++ let state = Arc::new(Mutex::new(WebhookState { responses: vec![] }));+ let webhook_url = start_webhook_server(39120, state.clone()).await?;++ match cli.command {+ Commands::Basic => {+ let mut urls = Vec::new();++ let url_one = "https://invalid-url.url/";+ println!("Mapping: {}", url_one);+ match firecrawl.map_url(url_one, None).await {+ Ok(mapped_urls) => urls.extend(mapped_urls),+ Err(e) => println!("Error mapping {}: {}", url_one, e),+ }++ let url_two = "https://www.devjobsscanner.com";+ println!("Mapping: {}", url_two);+ match firecrawl+ .map_url(+ url_two,+ Some(MapOptions {+ search: Some("rust".into()),+ limit: Some(20),+ ..Default::default()+ }),+ )+ .await+ {+ Ok(mapped_urls) => urls.extend(mapped_urls),+ Err(e) => println!("Error mapping {}: {}", url_two, e),+ }++ test_multiple_urls(&firecrawl, urls, &webhook_url).await?;++ // Give time for webhooks to arrive+ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;+ println!(+ "Received {} webhook responses",+ state.lock().await.responses.len()+ );+ }+ }++ Ok(())+}++async fn test_multiple_urls(+ app: &FirecrawlApp,+ urls: Vec<String>,+ webhook_url: &str,+) -> Result<(), Box<dyn Error>> {+ println!("Testing batch scraping of {} URLs", urls.len());++ let webhook = WebhookOptions {+ url: webhook_url.to_string(),+ headers: None,+ auth_token: None,+ };++ let params = BatchScrapeParams {+ urls,+ webhook: Some(webhook),+ ignore_invalid_urls: true,+ options: Some(ScrapeOptions {+ formats: Some(vec![ScrapeFormats::Markdown, ScrapeFormats::Links]),+ ..Default::default()+ }),+ ..Default::default()+ };++ let batch = app.async_batch_scrape_urls(params).await?;+ println!("Batch job started: {}", batch.id);++ // Poll status periodically+ loop {+ let status = app.check_batch_scrape_status(&batch.id).await?;+ println!("Progress: {}/{} pages", status.completed, status.total);++ if status.completed >= status.total {+ println!("Batch job completed!");+ break;+ }++ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;+ }++ Ok(())+}diff --git a/apps/rust-sdk/examples/cancel_crawl_example.rs b/apps/rust-sdk/examples/cancel_crawl_example.rsnew file mode 100644index 000000000..3451cc104--- /dev/null+++ b/apps/rust-sdk/examples/cancel_crawl_example.rs@@ -0,0 +1,33 @@+use firecrawl::FirecrawlApp;+use std::error::Error;+use std::time::Duration;++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ // Get API URL from environment+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");++ // Create the FirecrawlApp instance+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;++ // Start a crawl job+ println!("Starting a crawl job...");+ let crawl_response = firecrawl+ .crawl_url_async("https://example.com", None)+ .await?;+ println!("Crawl job started with ID: {}", crawl_response.id);++ // Wait for a moment to let the crawl job start+ println!("Waiting for a moment...");+ tokio::time::sleep(Duration::from_secs(2)).await;++ // Cancel the crawl job+ println!("Cancelling the crawl job...");+ let cancel_response = firecrawl.cancel_crawl(&crawl_response.id).await?;++ println!("Cancellation result:");+ println!(" Status: {:?}", cancel_response.status);++ Ok(())+}diff --git a/apps/rust-sdk/examples/check_crawl_errors_example.rs b/apps/rust-sdk/examples/check_crawl_errors_example.rsnew file mode 100644index 000000000..7629d36cb--- /dev/null+++ b/apps/rust-sdk/examples/check_crawl_errors_example.rs@@ -0,0 +1,59 @@+use firecrawl::FirecrawlApp;+use std::error::Error;++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ // Get API URL from environment+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");++ // Create the FirecrawlApp instance+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;++ // Start a crawl job that will likely have some errors (invalid URL format)+ println!("Starting a crawl job...");+ let crawl_response = firecrawl+ .crawl_url_async("https://no-wer-agg.invalid", None)+ .await?;+ println!("Crawl job started with ID: {}", crawl_response.id);++ println!("Let it do it's thing...");+ tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;++ // Check the crawl errors+ println!("Checking for crawl errors...");+ match firecrawl.check_crawl_errors(&crawl_response.id).await {+ Ok(error_response) => {+ println!("Crawl errors response:");+ println!(" Number of errors: {}", error_response.errors.len());++ if !error_response.errors.is_empty() {+ println!("\nDetailed errors:");+ for (i, error) in error_response.errors.iter().enumerate() {+ println!("Error #{}", i + 1);+ println!(" ID: {}", error.id);+ if let Some(timestamp) = &error.timestamp {+ println!(" Timestamp: {}", timestamp);+ }+ println!(" URL: {}", error.url);+ println!(" Error: {}", error.error);+ }+ }++ println!(+ "\nRobots.txt blocked URLs: {}",+ error_response.robots_blocked.len()+ );+ for (i, url) in error_response.robots_blocked.iter().enumerate() {+ println!(" {}. {}", i + 1, url);+ }+ }+ Err(e) => {+ println!("Failed to check crawl errors: {}", e);+ }+ }+ let cancel = firecrawl.cancel_crawl(&crawl_response.id).await?;+ println!("Cancel: {}", cancel.status);++ Ok(())+}diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rsindex 0dcb0d466..51592f3f1 100644--- a/apps/rust-sdk/examples/example.rs+++ b/apps/rust-sdk/examples/example.rs@@ -1,4 +1,8 @@-use firecrawl::{crawl::CrawlOptions, scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions}, FirecrawlApp};+use firecrawl::{+ crawl::CrawlOptions,+ scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions},+ FirecrawlApp,+};use serde_json::json;#[tokio::main]@@ -19,19 +23,20 @@ async fn main() {// Crawl a websitelet crawl_options = CrawlOptions {- exclude_paths: vec![ "blog/*".into() ].into(),+ exclude_paths: vec!["blog/*".into()].into(),..Default::default()};-- let crawl_result = app- .crawl_url("https://mendable.ai", crawl_options)- .await;++ let crawl_result = app.crawl_url("https://mendable.ai", crawl_options).await;match crawl_result {- Ok(data) => println!("Crawl Result (used {} credits):\n{:#?}", data.credits_used, data.data),+ Ok(data) => println!(+ "Crawl Result (used {} credits):\n{:#?}",+ data.credits_used, data.data+ ),Err(e) => eprintln!("Crawl failed: {}", e),}-+// Scrape with Extractlet json_schema = json!({"type": "object",@@ -57,11 +62,12 @@ async fn main() {});let llm_extraction_options = ScrapeOptions {- formats: vec![ ScrapeFormats::Extract ].into(),+ formats: vec![ScrapeFormats::Extract].into(),extract: ExtractOptions {schema: json_schema.into(),..Default::default()- }.into(),+ }+ .into(),..Default::default()};@@ -75,9 +81,7 @@ async fn main() {}// Map a website (Alpha)- let map_result = app- .map_url("https://firecrawl.dev", None)- .await;+ let map_result = app.map_url("https://firecrawl.dev", None).await;match map_result {Ok(data) => println!("Mapped URLs: {:#?}", data),diff --git a/apps/rust-sdk/examples/extract_example.rs b/apps/rust-sdk/examples/extract_example.rsnew file mode 100644index 000000000..47f446ae3--- /dev/null+++ b/apps/rust-sdk/examples/extract_example.rs@@ -0,0 +1,237 @@+use firecrawl::{extract::ExtractParams, FirecrawlApp};+use serde_json::json;+use std::error::Error;++use clap::{Parser, ValueEnum};++#[derive(Parser)]+#[command(author, version, about, long_about = None)]+struct Args {+ #[arg(value_enum)]+ command: Examples,+}++#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]+enum Examples {+ Basic,+ Schema,+ JsonSchema,+}++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ let args = Args::parse();++ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;+ let urls = vec![+ "https://www.firecrawl.dev/".to_string(),+ "https://betteruptime.com".to_string(),+ ];++ match args.command {+ Examples::Basic => {+ println!("Example 1: Extracting with URLs and prompt");++ let extract_params = ExtractParams {+ prompt: Some(+ "Extract Product promise, consice descirption and category".to_string(),+ ),+ url_trace: Some(true),+ ..Default::default()+ };++ println!("Starting asynchronous extraction job...");+ let response = firecrawl+ .async_extract(ExtractParams {+ urls: Some(urls.iter().map(|u| u.to_string()).collect()),+ prompt: extract_params.prompt.clone(),+ url_trace: extract_params.url_trace,+ ..Default::default()+ })+ .await?;++ println!("Extract job initiated:");+ println!(" Job ID: {}", response.id);++ println!("\nChecking extract status...");+ for _ in 0..5 {+ let response = firecrawl.get_extract_status(&response.id).await?;++ println!("Extract status: {}", response.status);+ if let Some(url_trace) = &response.url_trace {+ println!("URL traces:");+ for trace in url_trace {+ println!(" URL: {}", trace.url);+ println!(" Status: {}", trace.status);+ }+ }+ println!("Extract data: {:#?}", response.data);+ if response.status == "completed" {+ break;+ }++ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;+ }+ }+ Examples::Schema => {+ println!("Example 2: Extracting with schema");++ let schema = json!({+ "type": "object",+ "properties": {+ "category": { "type": "string" },+ "promise": { "type": "string" },+ "descirption": { "type": "string" }+ },+ "required": ["category", "promise", "description"]+ });++ println!("Starting synchronous extraction job...");++ match firecrawl+ .extract(ExtractParams {+ urls: urls.into(),+ schema: Some(schema),+ ..Default::default()+ })+ .await+ {+ Ok(result) => {+ println!("Extraction completed successfully!");+ println!("Status: {}", result.status);++ if let Some(data) = result.data {+ println!("\nExtracted data:");+ println!(" Title: {}", data["title"]);+ if let Some(desc) = data.get("description") {+ println!(" Description: {}", desc);+ }+ println!(+ " Content (preview): {:.100}...",+ data["content"].as_str().unwrap_or("N/A")+ );+ }++ if let Some(sources) = result.sources {+ println!("\nSources:");+ for (field, urls) in sources {+ println!(" {}: {}", field, urls.join(", "));+ }+ }+ }+ Err(e) => {+ println!("Extraction failed: {}", e);+ }+ }+ }+ Examples::JsonSchema => {+ println!("Example 3: Using JsonSchema derive");++ /// A comprehensive analysis of given product+ #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]+ struct ProductAnalysis {+ /// The full name of the product+ product_name: String,+ /// The company/brand behind the product+ brand: String,+ /// The general price range (e.g. "Premium", "$10-50", "Enterprise")+ price_range: String,+ /// The main customer segments this product targets+ target_audience: Vec<String>,+ /// Primary benefits and value propositions of the product+ key_benefits: Vec<String>,+ /// Distinctive features that set this product apart from competitors+ unique_selling_points: Vec<String>,+ /// Direct comparisons with competing products/services+ competitor_comparison: Vec<String>,+ /// Technologies, frameworks, or platforms used (if applicable)+ tech_stack: Option<Vec<String>>,+ /// Aggregated review data and sentiment analysis+ reviews_summary: ReviewsSummary,+ // /// Score from 0-10 indicating product-market fit based on analysis+ // market_fit_score: f32, // NOTE: Breaks+ /// Assessment of future growth prospects (e.g. "High", "Moderate", "Limited")+ growth_potential: String,+ /// Relevant compliance standards and certifications+ regulatory_compliance: Option<Vec<String>>,+ }++ /// Aggregated analysis of product reviews from multiple sources+ #[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]+ struct ReviewsSummary {+ /// Overall sentiment from review analysis (e.g. "Highly Positive", "Mixed", "Negative")+ sentiment_analysis: String,+ /// Most frequently mentioned positive aspects+ common_praises: Vec<String>,+ /// Most frequently mentioned criticisms or issues+ common_complaints: Vec<String>,+ /// Platforms or websites where reviews were sourced from+ review_sources: Vec<String>,+ }+ println!("Starting extraction with derived schema...");+ match firecrawl+ .extract_with_schemars::<ProductAnalysis>(ExtractParams {+ urls: urls.into(),+ ..Default::default()+ })+ .await+ {+ Ok(result) => {+ println!("Extraction completed!");+ println!("Status: {}", result.status);++ if let Some(data) = result.data {+ if let Ok(analysis) = serde_json::from_value::<ProductAnalysis>(data) {+ println!("\nExtracted Product Analysis:");+ println!(" Product: {}", analysis.product_name);+ println!(" Brand: {}", analysis.brand);+ println!(" Price Range: {}", analysis.price_range);+ println!(" Target Audience:");+ for audience in analysis.target_audience {+ println!(" - {}", audience);+ }+ println!(" Key Benefits:");+ for benefit in analysis.key_benefits {+ println!(" - {}", benefit);+ }+ println!(" USPs:");+ for usp in analysis.unique_selling_points {+ println!(" - {}", usp);+ }++ println!("\n Reviews Summary:");+ println!(+ " Sentiment: {}",+ analysis.reviews_summary.sentiment_analysis+ );+ println!(" Common Praises:");+ for praise in analysis.reviews_summary.common_praises {+ println!(" - {}", praise);+ }+ println!(" Common Complaints:");+ for complaint in analysis.reviews_summary.common_complaints {+ println!(" - {}", complaint);+ }+ } else {+ println!("Failed to parse extracted data");+ }+ }++ if let Some(sources) = result.sources {+ println!("\nSources:");+ for (field, urls) in sources {+ println!(" {}: {}", field, urls.join(", "));+ }+ }+ }+ Err(e) => {+ println!("Extraction failed: {}", e);+ }+ }+ }+ }++ Ok(())+}diff --git a/apps/rust-sdk/examples/llmstxt_example.rs b/apps/rust-sdk/examples/llmstxt_example.rsnew file mode 100644index 000000000..64fb33179--- /dev/null+++ b/apps/rust-sdk/examples/llmstxt_example.rs@@ -0,0 +1,173 @@+#![allow(clippy::option_map_unit_fn)]+use bat::{Input, PrettyPrinter};+use firecrawl::{llmstxt::GenerateLLMsTextParams, FirecrawlApp};+use std::error::Error;++use clap::{Parser, ValueEnum};++#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]+enum Mode {+ Basic,+ Pool,+ Fulltext,+}++#[derive(Parser)]+#[command(author, version, about, long_about = None)]+struct Args {+ /// URL for which to generate LLMs.txt+ #[arg(default_value = "https://www.firecrawl.dev/")]+ url: String,++ #[arg(long, short = 'm', value_enum, default_value = "Mode::Basic")]+ mode: Mode,++ /// Maximum number of URLs to process+ #[arg(long, short = 'd', default_value = "1")]+ max_urls: u32,++ /// Whether to show the full LLMs-full.txt in the response+ #[arg(long, short = 'f', default_value = "false")]+ full_text: bool,++ /// Experimental streaming option+ #[arg(long, short = 's', default_value = "false")]+ stream: bool,+}++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ let args = Args::parse();++ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;++ let params = GenerateLLMsTextParams {+ url: args.url.clone(),+ max_urls: args.max_urls,+ show_full_text: args.full_text,+ experimental_stream: args.stream,+ };++ match args.mode {+ Mode::Basic => {+ println!("Example 1: Basic LLMs.txt generation (synchronous)");+ println!("Generating LLMs.txt for {}...", args.url);+ firecrawl+ .generate_llms_text(params)+ .await+ .inspect(|result| {+ println!("Expires at: {}", result.expires_at);+ let text = (if args.full_text {+ result.data.full.as_ref()+ } else {+ result.data.compact.as_ref()+ })+ .expect("LLM Text");++ pretty_print_content("Firecrawl Result", text).expect("Print");+ })?;+ }+ Mode::Pool => {+ println!("Example 2: Asynchronous LLMs.txt generation with manual polling");++ println!("Starting asynchronous LLMs.txt generation job...");+ let response = firecrawl.async_generate_llms_text(params).await?;++ println!("LLMs.txt generation job initiated:");+ println!(" Job ID: {}", response.id);+ println!("\nManually polling for status...");+ for _ in 0..10 {+ let status = firecrawl+ .check_generate_llms_text_status(&response.id)+ .await?;++ match status.status.as_str() {+ "completed" => {+ println!("LLMs.txt generation completed!");+ let text = (if args.full_text {+ status.data.full.as_ref()+ } else {+ status.data.compact.as_ref()+ })+ .expect("LLM Text");++ pretty_print_content("Pool Result", text).expect("Print");++ break;+ }+ "failed" => {+ println!(+ "LLMs.txt generation failed: {}",+ status.error.unwrap_or_default()+ );+ break;+ }+ status => println!("Generation status: {}", status),+ }++ println!("Waiting 2 seconds before checking again...");+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;+ }+ }+ Mode::Fulltext => {+ println!("Example 3: LLMs.txt generation with full text");++ println!("Generating LLMs.txt with full text...");+ match firecrawl.generate_llms_text(params).await {+ Ok(result) => {+ println!("LLMs.txt generation completed successfully!");+ let llmstxt = result.data.compact.expect("LLMs Text Expected");+ let fulltxt = result.data.full.expect("Full LLMs Text Expected");++ pretty_print_contents(&[+ ("LLMs.txt (compact)", llmstxt),+ ("LLMs.txt (full text)", fulltxt),+ ])+ .expect("Print")+ }+ Err(e) => {+ println!("LLMs.txt generation failed: {}", e);+ }+ }+ }+ }++ Ok(())+}++/// Pretty prints the provided content with syntax highlighting+fn pretty_print_content(title: &str, content: &str) -> Result<(), Box<dyn Error>> {+ PrettyPrinter::new()+ .header(true)+ .grid(true)+ .input(+ Input::from_bytes(content.as_bytes())+ .title(title)+ .name("file.md"),+ )+ .print()?;++ Ok(())+}++/// Pretty prints multiple contents with syntax highlighting+fn pretty_print_contents(title_contents: &[(&'static str, String)]) -> Result<(), Box<dyn Error>> {+ let mut inputs = Vec::new();+ for (title, content) in title_contents {+ inputs.push(+ Input::from_bytes(content.as_bytes())+ .title(*title)+ .name("file.md"),+ );+ }++ PrettyPrinter::new()+ .header(true)+ .grid(true)+ .inputs(inputs)+ .print()?;++ Ok(())+}diff --git a/apps/rust-sdk/examples/search_example.rs b/apps/rust-sdk/examples/search_example.rsnew file mode 100644index 000000000..28437193a--- /dev/null+++ b/apps/rust-sdk/examples/search_example.rs@@ -0,0 +1,186 @@+use clap::{Parser, ValueEnum};+use firecrawl::{+ search::{SearchParams, SearchResponse},+ FirecrawlApp,+};+use std::error::Error;++#[derive(Debug, Parser)]+#[command(author, version, about, long_about = None)]+struct Args {+ /// Which example to run+ #[arg(value_enum, default_value_t = Examples::All)]+ example: Examples,+}++#[derive(Debug, Clone, ValueEnum)]+enum Examples {+ All,+ Basic,+ Advanced,+ Geo,+ Temporal,+ Social,+ News,+ Academic,+ Commercial,+}++#[tokio::main]+async fn main() -> Result<(), Box<dyn Error>> {+ let args = Args::parse();++ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let firecrawl = FirecrawlApp::new_selfhosted(api_url, None::<&str>)?;++ match args.example {+ Examples::All => {+ run_basic_example(&firecrawl).await?;+ run_advanced_example(&firecrawl).await?;+ run_geographic_example(&firecrawl).await?;+ run_temporal_example(&firecrawl).await?;+ run_social_example(&firecrawl).await?;+ run_news_example(&firecrawl).await?;+ run_academic_example(&firecrawl).await?;+ run_commercial_example(&firecrawl).await?;+ }+ Examples::Basic => run_basic_example(&firecrawl).await?,+ Examples::Advanced => run_advanced_example(&firecrawl).await?,+ Examples::Geo => run_geographic_example(&firecrawl).await?,+ Examples::Temporal => run_temporal_example(&firecrawl).await?,+ Examples::Social => run_social_example(&firecrawl).await?,+ Examples::News => run_news_example(&firecrawl).await?,+ Examples::Academic => run_academic_example(&firecrawl).await?,+ Examples::Commercial => run_commercial_example(&firecrawl).await?,+ }++ Ok(())+}+async fn run_basic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "rust programming language";+ let results = firecrawl.search(query, None).await?;+ print_results("Basic Search", query, &results);+ Ok(())+}++async fn run_advanced_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "rust web framework site:github.com OR site:gitlab.com";+ let params = SearchParams {+ query: query.to_string(),+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Advanced Repository Search", query, &results);+ Ok(())+}++async fn run_geographic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "coworking space startup hub";+ let params = SearchParams {+ query: query.to_string(),+ // WARN: Doesn't work with searxng+ location: Some("Silicon Valley, CA".to_string()),+ // WARN: Doesn't work with searxng+ country: Some("us".to_string()),+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Geographic-Specific Search", query, &results);+ Ok(())+}++async fn run_temporal_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "artificial intelligence breakthroughs";+ let params = SearchParams {+ query: query.to_string(),+ // WARN: Doesn't work with searxng+ tbs: Some("qdr:m1".to_string()),+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Recent AI News", query, &results);+ Ok(())+}++async fn run_social_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "viral tech trends site:twitter.com";+ let params = SearchParams {+ query: query.to_string(),+ // WARN: Doesn't work. Maybe searxng related+ filter: Some("site:twitter.com OR site:linkedin.com".to_string()),+ // WARN: Doesn't work with searxng+ tbs: Some("qdr:w".to_string()), // Last week+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Social Media Tech Trends", query, &results);+ Ok(())+}++async fn run_news_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query =+ "cryptocurrency market analysis site:reuters.com OR site:bloomberg.com OR site:ft.com";+ let params = SearchParams {+ query: query.to_string(),+ // WARN: Doesn't work with searxng+ tbs: Some("qdr:d".to_string()), // Last 24 hours+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Financial News Search", query, &results);+ Ok(())+}++async fn run_academic_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "quantum computing research papers site:arxiv.org OR site:scholar.google.com";+ let params = SearchParams {+ query: query.to_string(),+ // WARN: Doesn't work. Maybe searxng related+ // filter: Some("site:arxiv.org OR site:scholar.google.com".to_string()),+ // WARN: Doesn't work with searxng+ tbs: Some("qdr:y".to_string()), // Last year+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Academic Research Search", query, &results);+ Ok(())+}++async fn run_commercial_example(firecrawl: &FirecrawlApp) -> Result<(), Box<dyn Error>> {+ let query = "enterprise cloud solutions reviews site:g2.com";+ let params = SearchParams {+ query: query.to_string(),+ limit: Some(5),+ ..Default::default()+ };+ let results = firecrawl.search_with_params(params).await?;+ print_results("Commercial Product Search", query, &results);+ Ok(())+}++fn print_results(name: &str, query: &str, results: &SearchResponse) {+ let sec = "=".repeat(70);++ println!("\n{sec}");+ println!("🔍 {name}");+ println!("🔎 Query: \"{query}\"");+ println!("{sec}");++ for (i, doc) in results.data.iter().enumerate() {+ println!("{}. 📌 Title: {}", i + 1, doc.title);+ println!(" - 🔗 URL: {}", doc.url);+ println!(" - 📝 Description: \"{:.40}\"...", doc.description);+ }++ if let Some(warning) = &results.warning {+ println!("\n⚠️ Warning: {warning}");+ }+ println!("{sec}\n");+}diff --git a/apps/rust-sdk/src/batch_scrape.rs b/apps/rust-sdk/src/batch_scrape.rsnew file mode 100644index 000000000..1429fbea6--- /dev/null+++ b/apps/rust-sdk/src/batch_scrape.rs@@ -0,0 +1,494 @@+use std::collections::HashMap;++use serde::{Deserialize, Serialize};++use crate::{+ crawl::{CrawlErrorsResponse, CrawlStatus, CrawlStatusTypes},+ scrape::ScrapeOptions,+ FirecrawlApp, FirecrawlError, API_VERSION,+};++#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeParams {+ /// List of URLs to scrape+ pub urls: Vec<String>,+ /// Scrape options to apply to all URLs+ #[serde(flatten)]+ pub options: Option<ScrapeOptions>,+ /// Whether to ignore invalid URLs+ #[serde(rename = "ignoreInvalidURLs")]+ pub ignore_invalid_urls: bool,+ /// ID of an existing job to append these URLs to+ pub append_to_id: Option<String>,+ /// Webhook configuration+ pub webhook: Option<WebhookOptions>,++ /// Idempotency key to send to the crawl endpoint.+ #[serde(skip)]+ pub idempotency_key: Option<String>,+}++/// Options for webhook notifications+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct WebhookOptions {+ /// URL to send webhook notifications to+ pub url: String,+ /// Custom headers to include in webhook requests+ pub headers: Option<HashMap<String, String>>,+ /// Authentication token for the webhook+ pub auth_token: Option<String>,+}++impl From<&str> for WebhookOptions {+ fn from(url: &str) -> Self {+ Self {+ url: url.to_string(),+ headers: None,+ auth_token: None,+ }+ }+}++/// Response from initiating a batch scrape job+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeResponse {+ /// Whether the request was successful+ pub success: bool,+ /// The ID of the batch scrape job+ pub id: String,+ /// URL to get the status of the batch scrape job+ pub url: String,+ /// List of URLs that were invalid and could not be processed+ pub invalid_urls: Option<Vec<String>>,+}++impl From<String> for WebhookOptions {+ fn from(url: String) -> Self {+ Self {+ url,+ headers: None,+ auth_token: None,+ }+ }+}++impl FirecrawlApp {+ /// Initiates an asynchronous batch scrape job+ pub async fn async_batch_scrape_urls(+ &self,+ params: BatchScrapeParams,+ ) -> Result<BatchScrapeResponse, FirecrawlError> {+ let headers = self.prepare_headers(params.idempotency_key.as_ref());++ let response = self+ .client+ .post(format!("{}{}/batch/scrape", self.api_url, API_VERSION))+ .headers(headers)+ .json(¶ms)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Initiating batch scrape job".to_string(), e))?;++ self.handle_response(response, "initiate batch scrape job")+ .await+ }++ /// Initiates a batch scrape job and waits for completion+ pub async fn batch_scrape_urls(+ &self,+ params: BatchScrapeParams,+ poll_interval: Option<u64>,+ ) -> Result<CrawlStatus, FirecrawlError> {+ let poll_interval_ms = poll_interval.unwrap_or(2000);++ let response = self.async_batch_scrape_urls(params).await?;++ self.monitor_batch_job_status(&response.id, poll_interval_ms)+ .await+ }++ /// Checks the status of a batch scrape job+ pub async fn check_batch_scrape_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlStatus, FirecrawlError> {+ let response = self+ .client+ .get(format!(+ "{}{}/batch/scrape/{}",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Checking status of batch scrape {}", id.as_ref()),+ e,+ )+ })?;++ let mut status: CrawlStatus = self+ .handle_response(+ response,+ format!("Checking status of batch scrape {}", id.as_ref()),+ )+ .await?;++ if status.status == CrawlStatusTypes::Completed {+ while let Some(next) = status.next.clone() {+ let new_status = self.check_batch_scrape_status_next(next).await?;+ status.data.extend_from_slice(&new_status.data);+ status.next = new_status.next;+ }+ }++ Ok(status)+ }++ /// Helper function to paginate through batch scrape status results+ async fn check_batch_scrape_status_next(+ &self,+ next: impl AsRef<str>,+ ) -> Result<CrawlStatus, FirecrawlError> {+ let response = self+ .client+ .get(next.as_ref())+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Paginating batch scrape using URL {:?}", next.as_ref()),+ e,+ )+ })?;++ self.handle_response(+ response,+ format!("Paginating batch scrape using URL {:?}", next.as_ref()),+ )+ .await+ }++ /// Check for errors in a batch scrape job+ pub async fn check_batch_scrape_errors(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {+ let response = self+ .client+ .get(format!(+ "{}{}/batch/scrape/{}/errors",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Checking errors for batch scrape {}", id.as_ref()),+ e,+ )+ })?;++ self.handle_response(+ response,+ format!("Checking errors for batch scrape {}", id.as_ref()),+ )+ .await+ }++ /// Helper function to poll for batch job status until completion+ async fn monitor_batch_job_status(+ &self,+ id: &str,+ poll_interval: u64,+ ) -> Result<CrawlStatus, FirecrawlError> {+ loop {+ let status_data = self.check_batch_scrape_status(id).await?;+ match status_data.status {+ CrawlStatusTypes::Completed => {+ break Ok(status_data);+ }+ CrawlStatusTypes::Scraping => {+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ CrawlStatusTypes::Failed => {+ break Err(FirecrawlError::CrawlJobFailed(+ "Batch scrape job failed".into(),+ status_data,+ ));+ }+ CrawlStatusTypes::Cancelled => {+ break Err(FirecrawlError::CrawlJobFailed(+ "Batch scrape job was cancelled".into(),+ status_data,+ ));+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_batch_scrape() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();++ // Start a batch scrape job+ let params = BatchScrapeParams {+ urls: vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ],+ ignore_invalid_urls: true,+ ..Default::default()+ };++ let response = app.async_batch_scrape_urls(params).await.unwrap();++ assert!(response.success);+ assert!(!response.id.is_empty());+ assert!(!response.url.is_empty());+ }++ #[tokio::test]+ async fn test_async_batch_scrape_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock+ let mock = server+ .mock("POST", "/v1/batch/scrape")+ // Remove the match_body expectation which might be causing issues+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-123",+ "url": "https://api.example.com/v1/batch/batch-123",+ "invalidUrls": []+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = BatchScrapeParams {+ urls: vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ],+ ignore_invalid_urls: true,+ ..Default::default()+ };++ let response = app.async_batch_scrape_urls(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "batch-123");+ assert_eq!(response.url, "https://api.example.com/v1/batch/batch-123");+ assert!(response.invalid_urls.unwrap_or_default().is_empty());+ mock.assert();+ }++ #[tokio::test]+ async fn test_batch_scrape_with_webhook() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v1/batch/scrape")+ // Remove the match_body expectation to simplify+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-123",+ "url": "https://api.example.com/v1/batch/batch-123"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = BatchScrapeParams {+ urls: vec!["https://example.com".to_string()],+ webhook: Some("https://webhook.example.com/notify".into()),+ ..Default::default()+ };++ let response = app.async_batch_scrape_urls(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "batch-123");+ mock.assert();+ }++ #[tokio::test]+ async fn test_check_batch_scrape_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v1/batch/scrape/batch-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "total": 2,+ "completed": 2,+ "creditsUsed": 2,+ "expiresAt": "2023-12-31T23:59:59Z",+ "data": [+ {+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200+ },+ "markdown": "Example Domain content"+ },+ {+ "metadata": {+ "sourceURL": "https://example.org",+ "statusCode": 200+ },+ "markdown": "Another example content"+ }+ ]+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = app.check_batch_scrape_status("batch-123").await.unwrap();++ assert_eq!(status.total, 2);+ assert_eq!(status.completed, 2);+ assert_eq!(status.data.len(), 2);+ assert_eq!(status.data[0].metadata.source_url, "https://example.com");+ assert_eq!(status.data[1].metadata.source_url, "https://example.org");+ mock.assert();+ }++ #[tokio::test]+ async fn test_check_batch_scrape_errors_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v1/batch/scrape/batch-123/errors")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "errors": [+ {+ "id": "error1",+ "timestamp": "2023-01-01T00:00:00Z",+ "url": "https://invalid.example.com",+ "error": "Failed to load page"+ }+ ],+ "robotsBlocked": [+ "https://example.com/admin"+ ]+ })+ .to_string(),+ )+ .create_async()+ .await;++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let errors = app.check_batch_scrape_errors("batch-123").await.unwrap();++ assert_eq!(errors.errors.len(), 1);+ assert_eq!(errors.errors[0].url, "https://invalid.example.com");+ assert_eq!(errors.robots_blocked.len(), 1);+ assert_eq!(errors.robots_blocked[0], "https://example.com/admin");+ mock.assert();+ }++ #[tokio::test]+ async fn test_batch_scrape_with_invalid_urls() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v1/batch/scrape")+ // Remove the match_body expectation+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-123",+ "url": "https://api.example.com/v1/batch/batch-123",+ "invalidUrls": ["invalid-url"]+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = BatchScrapeParams {+ urls: vec!["https://example.com".to_string(), "invalid-url".to_string()],+ ignore_invalid_urls: true,+ ..Default::default()+ };++ let response = app.async_batch_scrape_urls(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "batch-123");+ assert_eq!(response.invalid_urls, Some(vec!["invalid-url".to_string()]));+ mock.assert();+ }++ #[tokio::test]+ async fn test_batch_scrape_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v1/batch/scrape")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "No valid URLs provided"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = BatchScrapeParams::default();+ let result = app.async_batch_scrape_urls(params).await;++ assert!(result.is_err());+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rsindex 2860d24ac..a5f30f40b 100644--- a/apps/rust-sdk/src/crawl.rs+++ b/apps/rust-sdk/src/crawl.rs@@ -2,7 +2,11 @@ use std::collections::HashMap;use serde::{Deserialize, Serialize};-use crate::{document::Document, scrape::{ScrapeFormats, ScrapeOptions}, FirecrawlApp, FirecrawlError, API_VERSION};+use crate::{+ document::Document,+ scrape::{ScrapeFormats, ScrapeOptions},+ FirecrawlApp, FirecrawlError, API_VERSION,+};#[derive(Deserialize, Serialize, Clone, Copy, Debug)]pub enum CrawlScrapeFormats {@@ -23,13 +27,13 @@ pub enum CrawlScrapeFormats {Links,/// Will result in a URL to a screenshot of the page.- ///+ ////// Can not be used in conjunction with `CrawlScrapeFormats::ScreenshotFullPage`.#[serde(rename = "screenshot")]Screenshot,/// Will result in a URL to a full-page screenshot of the page.- ///+ ////// Can not be used in conjunction with `CrawlScrapeFormats::Screenshot`.#[serde(rename = "screenshot@fullPage")]ScreenshotFullPage,@@ -59,12 +63,12 @@ pub struct CrawlScrapeOptions {pub only_main_content: Option<bool>,/// HTML tags to exclusively include.- ///+ ////// For example, if you pass `div`, you will only get content from `<div>`s and their children.pub include_tags: Option<Vec<String>>,/// HTML tags to exclude.- ///+ ////// For example, if you pass `img`, you will never get image URLs in your results.pub exclude_tags: Option<Vec<String>>,@@ -81,7 +85,9 @@ pub struct CrawlScrapeOptions {impl From<CrawlScrapeOptions> for ScrapeOptions {fn from(value: CrawlScrapeOptions) -> Self {ScrapeOptions {- formats: value.formats.map(|formats| formats.into_iter().map(|x| x.into()).collect()),+ formats: value+ .formats+ .map(|formats| formats.into_iter().map(|x| x.into()).collect()),only_main_content: value.only_main_content,include_tags: value.include_tags,exclude_tags: value.exclude_tags,@@ -101,12 +107,12 @@ pub struct CrawlOptions {pub scrape_options: Option<CrawlScrapeOptions>,/// URL RegEx patterns to (exclusively) include.- ///+ ////// For example, if you specified `"blog"`, only pages that have `blog` somewhere in the URL would be crawled.pub include_paths: Option<Vec<String>>,/// URL RegEx patterns to exclude.- ///+ ////// For example, if you specified `"blog"`, pages that have `blog` somewhere in the URL would not be crawled.pub exclude_paths: Option<Vec<String>>,@@ -200,6 +206,29 @@ pub struct CrawlStatus {pub data: Vec<Document>,}+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlError {+ pub id: String,+ pub timestamp: Option<String>,+ pub url: String,+ pub error: String,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlErrorsResponse {+ pub errors: Vec<CrawlError>,+ #[serde(rename = "robotsBlocked")]+ pub robots_blocked: Vec<String>,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CancelCrawlResponse {+ pub status: String,+}+#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(rename_all = "camelCase")]pub struct CrawlAsyncResponse {@@ -223,19 +252,20 @@ impl FirecrawlApp {url: url.as_ref().to_string(),options: options.unwrap_or_default(),};-+let headers = self.prepare_headers(body.options.idempotency_key.as_ref());let response = self.client- .post(&format!("{}{}/crawl", self.api_url, API_VERSION))+ .post(format!("{}{}/crawl", self.api_url, API_VERSION)).headers(headers.clone()).json(&body).send().await.map_err(|e| FirecrawlError::HttpError(format!("Crawling {:?}", url.as_ref()), e))?;- self.handle_response::<CrawlAsyncResponse>(response, "start crawl job").await+ self.handle_response::<CrawlAsyncResponse>(response, "start crawl job")+ .await}/// Performs a crawl job for a URL using the Firecrawl API, waiting for the end result. This may take a long time depending on the size of the target page and your options (namely `CrawlOptions.limit`).@@ -245,38 +275,65 @@ impl FirecrawlApp {options: impl Into<Option<CrawlOptions>>,) -> Result<CrawlStatus, FirecrawlError> {let options = options.into();- let poll_interval = options.as_ref().and_then(|x| x.poll_interval).unwrap_or(2000);+ let poll_interval = options+ .as_ref()+ .and_then(|x| x.poll_interval)+ .unwrap_or(2000);let res = self.crawl_url_async(url, options).await?;self.monitor_job_status(&res.id, poll_interval).await}- async fn check_crawl_status_next(&self, next: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {+ async fn check_crawl_status_next(+ &self,+ next: impl AsRef<str>,+ ) -> Result<CrawlStatus, FirecrawlError> {let response = self.client.get(next.as_ref()).headers(self.prepare_headers(None)).send().await- .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl using URL {:?}", next.as_ref()), e))?;-- self.handle_response(response, format!("Paginating crawl using URL {:?}", next.as_ref())).await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Paginating crawl using URL {:?}", next.as_ref()),+ e,+ )+ })?;++ self.handle_response(+ response,+ format!("Paginating crawl using URL {:?}", next.as_ref()),+ )+ .await}/// Checks for the status of a crawl, based on the crawl's ID. To be used in conjunction with `FirecrawlApp::crawl_url_async`.- pub async fn check_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {+ pub async fn check_crawl_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlStatus, FirecrawlError> {let response = self.client- .get(&format!(+ .get(format!("{}{}/crawl/{}",- self.api_url, API_VERSION, id.as_ref()+ self.api_url,+ API_VERSION,+ id.as_ref())).headers(self.prepare_headers(None)).send().await- .map_err(|e| FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e))?;+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e)+ })?;- let mut status: CrawlStatus = self.handle_response(response, format!("Checking status of crawl {}", id.as_ref())).await?;+ let mut status: CrawlStatus = self+ .handle_response(+ response,+ format!("Checking status of crawl {}", id.as_ref()),+ )+ .await?;if status.status == CrawlStatusTypes::Completed {while let Some(next) = status.next {@@ -304,16 +361,240 @@ impl FirecrawlApp {tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;}CrawlStatusTypes::Failed => {- break Err(FirecrawlError::CrawlJobFailed(format!(- "Crawl job failed."- ), status_data));+ break Err(FirecrawlError::CrawlJobFailed(+ "Crawl job failed".into(),+ status_data,+ ));}CrawlStatusTypes::Cancelled => {- break Err(FirecrawlError::CrawlJobFailed(format!(- "Crawl job was cancelled."- ), status_data));+ break Err(FirecrawlError::CrawlJobFailed(+ "Crawl job was cancelled.".into(),+ status_data,+ ));}}}}++ /// Cancel an asynchronous crawl job using the Firecrawl API.+ ///+ /// # Returns+ ///+ /// A response indicating whether the cancellation was successful, or a FirecrawlError if the request fails.+ pub async fn cancel_crawl(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CancelCrawlResponse, FirecrawlError> {+ let response = self+ .client+ .delete(format!(+ "{}{}/crawl/{}",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "crawl_cancel").await+ }++ /// Returns information about crawl errors.+ ///+ /// # Returns+ ///+ /// A response containing information about crawl errors, or a FirecrawlError if the request fails.+ pub async fn check_crawl_errors(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {+ let response = self+ .client+ .get(format!(+ "{}{}/crawl/{}/errors",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Checking errors for crawl {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "crawl_check").await+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_cancel_crawl() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();++ // First start a crawl job+ let crawl_response = app+ .crawl_url_async("https://example.com", None)+ .await+ .unwrap();++ // Then cancel it+ let cancel_response = app.cancel_crawl(crawl_response.id).await.unwrap();++ assert_eq!(cancel_response.status, "cancelled");+ }++ #[tokio::test]+ async fn test_cancel_crawl_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the cancel request+ let mock = server+ .mock("DELETE", "/v1/crawl/test-crawl-id")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": null,+ "status": "cancelled"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = app.cancel_crawl("test-crawl-id").await.unwrap();++ assert_eq!(response.status, "cancelled");+ mock.assert();+ }++ #[tokio::test]+ async fn test_cancel_crawl_error_response() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for an error response+ let mock = server+ .mock("DELETE", "/v1/crawl/invalid-id")+ .with_status(404)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Crawl job not found"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = app.cancel_crawl("invalid-id").await;++ assert!(result.is_err());+ mock.assert();+ }++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_check_crawl_errors() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();++ // First start a crawl job+ let crawl_response = app+ .crawl_url_async("https://no-wer-agg.invalid", None)+ .await+ .unwrap();++ // Check for errors+ let errors_response = app.check_crawl_errors(crawl_response.id).await.unwrap();+ println!("{errors_response:?}");++ tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;++ assert!(+ !errors_response.errors.is_empty(),+ "WARN: Error returned related to Supabase not in my environment. It may fail"+ );+ }++ #[tokio::test]+ async fn test_check_crawl_errors_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the check errors request+ let mock = server+ .mock("GET", "/v1/crawl/test-crawl-id/errors")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "errors": [+ {+ "id": "error1",+ "timestamp": "2023-01-01T00:00:00Z",+ "url": "https://example.com/error-page",+ "error": "Failed to load page"+ }+ ],+ "robotsBlocked": [+ "https://example.com/blocked-by-robots"+ ]+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = app.check_crawl_errors("test-crawl-id").await.unwrap();++ assert_eq!(response.errors.len(), 1);+ assert_eq!(response.errors[0].id, "error1");+ assert_eq!(response.errors[0].url, "https://example.com/error-page");+ assert_eq!(response.errors[0].error, "Failed to load page");+ assert_eq!(response.robots_blocked.len(), 1);+ assert_eq!(+ response.robots_blocked[0],+ "https://example.com/blocked-by-robots"+ );+ mock.assert();+ }++ #[tokio::test]+ async fn test_check_crawl_errors_error_response() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for an error response+ let mock = server+ .mock("GET", "/v1/crawl/invalid-id/errors")+ .with_status(404)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Crawl job not found"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = app.check_crawl_errors("invalid-id").await;++ assert!(result.is_err());+ mock.assert();+ }}diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rsindex 1948a4ce6..38c7fcc15 100644--- a/apps/rust-sdk/src/document.rs+++ b/apps/rust-sdk/src/document.rs@@ -57,12 +57,12 @@ pub struct Document {pub markdown: Option<String>,/// The HTML of the page, present if `ScrapeFormats::HTML` is present in `ScrapeOptions.formats`.- ///+ ////// This contains HTML that has non-content tags removed. If you need the original HTML, use `ScrapeFormats::RawHTML`.pub html: Option<String>,/// The raw HTML of the page, present if `ScrapeFormats::RawHTML` is present in `ScrapeOptions.formats`.- ///+ ////// This contains the original, untouched HTML on the page. If you only need human-readable content, use `ScrapeFormats::HTML`.pub raw_html: Option<String>,@@ -83,4 +83,3 @@ pub struct Document {/// The warning message will contain any errors encountered during the extraction.pub warning: Option<String>,}-diff --git a/apps/rust-sdk/src/error.rs b/apps/rust-sdk/src/error.rsindex 33e4edc61..f452cd0e3 100644--- a/apps/rust-sdk/src/error.rs+++ b/apps/rust-sdk/src/error.rs@@ -42,4 +42,6 @@ pub enum FirecrawlError {APIError(String, FirecrawlAPIError),#[error("Crawl job failed: {0}")]CrawlJobFailed(String, CrawlStatus),+ #[error("Missuse: {0}")]+ Missuse(String),}diff --git a/apps/rust-sdk/src/extract.rs b/apps/rust-sdk/src/extract.rsnew file mode 100644index 000000000..a1dd2ef92--- /dev/null+++ b/apps/rust-sdk/src/extract.rs@@ -0,0 +1,596 @@+use std::collections::HashMap;++use schemars::schema_for;+use serde::{Deserialize, Serialize};+use serde_json::Value;++use crate::{FirecrawlApp, FirecrawlError, API_VERSION};++/// Parameters for extract requests+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ExtractParams {+ /// URLs to extract information from+ pub urls: Option<Vec<String>>,++ /// Extraction prompt+ pub prompt: Option<String>,++ /// Schema for structured output+ pub schema: Option<Value>,++ /// System prompt for the LLM+ pub system_prompt: Option<String>,++ /// Allow following external links+ pub allow_external_links: Option<bool>,++ /// Enable web search for additional information+ pub enable_web_search: Option<bool>,++ /// Show sources in the response+ pub show_sources: Option<bool>,++ /// Origin information, defaults to "api-sdk"+ pub origin: Option<String>,++ /// Timeout in milliseconds, defaults to 60000+ pub timeout: Option<u32>,++ /// Whether to include URL trace information, defaults to false+ pub url_trace: Option<bool>,++ /// Whether to ignore sitemap, defaults to false+ pub ignore_sitemap: Option<bool>,++ /// Whether to include subdomains, defaults to true+ pub include_subdomains: Option<bool>,++ /// Maximum number of URLs to process+ pub limit: Option<u32>,++ /// Experimental: Stream steps information+ #[serde(rename = "__experimental_streamSteps")]+ pub experimental_stream_steps: Option<bool>,++ /// Experimental: Include LLM usage information+ #[serde(rename = "__experimental_llmUsage")]+ pub experimental_llm_usage: Option<bool>,++ /// Experimental: Show sources information+ #[serde(rename = "__experimental_showSources")]+ pub experimental_show_sources: Option<bool>,++ /// Experimental: Cache key+ #[serde(rename = "__experimental_cacheKey")]+ pub experimental_cache_key: Option<String>,++ /// Experimental: Cache mode, defaults to "direct"+ #[serde(rename = "__experimental_cacheMode")]+ pub experimental_cache_mode: Option<String>,+}++/// Response from initiating an extract operation+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ExtractResponse {+ /// Whether the request was successful+ pub success: bool,++ /// The ID of the extract job+ pub id: String,++ /// URL trace information if requested+ pub url_trace: Option<Vec<URLTrace>>,+}++/// Information about URL processing during extraction+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct URLTrace {+ /// The URL being processed+ pub url: String,++ /// Status of processing this URL+ pub status: String,++ /// Timing information for URL processing+ pub timing: URLTraceTiming,++ /// Error message if processing failed+ pub error: Option<String>,++ /// Warning message if there were issues+ pub warning: Option<String>,++ /// Content statistics+ pub content_stats: Option<ContentStats>,++ /// Relevance score for this URL (0-1)+ pub relevance_score: Option<f64>,++ /// Whether this URL was used in the final completion+ pub used_in_completion: Option<bool>,++ /// Fields extracted from this URL+ pub extracted_fields: Option<Vec<String>>,+}++/// Timing information for URL processing+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct URLTraceTiming {+ /// When the URL was discovered+ pub discovered_at: String,++ /// When scraping began for this URL+ pub scraped_at: Option<String>,++ /// When processing was completed for this URL+ pub completed_at: Option<String>,+}++/// Statistics about processed content+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ContentStats {+ /// Length of the raw content in characters+ pub raw_content_length: u32,++ /// Length of the processed content in characters+ pub processed_content_length: u32,++ /// Number of tokens used for this content+ pub tokens_used: u32,+}++/// Response for extract status check+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ExtractStatusResponse {+ /// Whether the request was successful+ pub success: bool,++ /// Status of the extract job: "pending", "processing", "completed", "failed"+ pub status: String,++ /// Extracted data, present when status is "completed"+ pub data: Option<Value>,++ /// Error message if the job failed+ pub error: Option<String>,++ /// URL trace information if requested+ pub url_trace: Option<Vec<URLTrace>>,++ /// Sources information if requested+ pub sources: Option<HashMap<String, Vec<String>>>,+}++impl FirecrawlApp {+ /// Extracts information from URLs using the Firecrawl API.+ ///+ /// This is the synchronous version that polls until completion.+ ///+ /// Either `params.prompt` or `params.schema` must be provided.+ pub async fn extract(+ &self,+ params: impl Into<ExtractParams>,+ ) -> Result<ExtractStatusResponse, FirecrawlError> {+ let mut params = params.into();+ // Validation: Either prompt or schema must be provided+ if params.prompt.is_none() && params.schema.is_none() {+ return Err(FirecrawlError::APIError(+ "Extract validation".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: "Either prompt or schema must be provided".to_string(),+ details: None,+ },+ ));+ }++ // Set default origin if not provided+ if params.origin.is_none() {+ params.origin = Some("api-sdk".to_string());+ }++ // Initiate the extract job asynchronously+ let response = self.async_extract(params).await?;++ // Poll for the result+ let poll_interval = 2000; // Default to 2 seconds+ self.monitor_extract_job_status(&response.id, poll_interval)+ .await+ }++ pub async fn extract_with_schemars<T>(+ &self,+ params: impl Into<ExtractParams>,+ ) -> Result<ExtractStatusResponse, FirecrawlError>+ where+ T: schemars::JsonSchema,+ {+ let mut params = params.into();+ let schema = schema_for!(T);+ let schema_json = serde_json::to_value(schema).map_err(|e| {+ FirecrawlError::APIError(+ "Schema serialization".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: e.to_string(),+ details: None,+ },+ )+ })?;+ params.schema = Some(schema_json);+ self.extract(params).await+ }++ /// Initiates an asynchronous extract operation.+ ///+ /// # Arguments+ ///+ /// * `params` - Parameters for the extract request+ ///+ /// # Returns+ ///+ /// A response containing the extract job ID, or a FirecrawlError if the request fails.+ ///+ /// # Notes+ ///+ /// Either `params.urls` or `params.prompt` must be provided.+ /// Either `params.prompt` or `params.schema` must be provided.+ pub async fn async_extract(+ &self,+ params: impl Into<ExtractParams>,+ ) -> Result<ExtractResponse, FirecrawlError> {+ let params = params.into();+ // Validation: Either URLs or prompt must be provided+ if params.urls.is_none() && params.prompt.is_none() {+ return Err(FirecrawlError::APIError(+ "Extract validation".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: "Either URLs or prompt must be provided".to_string(),+ details: None,+ },+ ));+ }++ // Validation: Either prompt or schema must be provided+ if params.prompt.is_none() && params.schema.is_none() {+ return Err(FirecrawlError::APIError(+ "Extract validation".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: "Either prompt or schema must be provided".to_string(),+ details: None,+ },+ ));+ }++ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(format!("{}{}/extract", self.api_url, API_VERSION))+ .headers(headers)+ .json(¶ms)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Initiating extract job".to_string(), e))?;++ self.handle_response(response, "initiate extract job").await+ }++ /// Checks the status of an extract job.+ ///+ /// # Arguments+ ///+ /// * `id` - The ID of the extract job+ ///+ /// # Returns+ ///+ /// A response containing the status of the extract job, or a FirecrawlError if the request fails.+ pub async fn get_extract_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<ExtractStatusResponse, FirecrawlError> {+ let response = self+ .client+ .get(format!(+ "{}{}/extract/{}",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Checking status of extract {}", id.as_ref()), e)+ })?;++ self.handle_response(+ response,+ format!("Checking status of extract {}", id.as_ref()),+ )+ .await+ }++ /// Helper function to poll for extract job status until completion+ async fn monitor_extract_job_status(+ &self,+ id: &str,+ poll_interval: u64,+ ) -> Result<ExtractStatusResponse, FirecrawlError> {+ loop {+ let status_data = self.get_extract_status(id).await?;++ match status_data.status.as_str() {+ "completed" => {+ break Ok(status_data);+ }+ "pending" | "processing" => {+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ "failed" => {+ let error_msg = status_data+ .error+ .clone()+ .unwrap_or_else(|| "Extract job failed".to_string());+ break Err(FirecrawlError::APIError(+ "Extract job failed".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: error_msg,+ details: None,+ },+ ));+ }+ _ => {+ break Err(FirecrawlError::APIError(+ "Extract job status".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: format!("Unexpected status: {}", status_data.status),+ details: None,+ },+ ));+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_extract() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();++ // Create extract params+ let params = ExtractParams {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: Some("Extract the title and main content from this page".to_string()),+ schema: None,+ origin: Some("test".to_string()),+ ..Default::default()+ };++ // Start an extract job+ let response = app.async_extract(params).await.unwrap();++ assert!(response.success);+ assert!(!response.id.is_empty());+ }++ #[tokio::test]+ async fn test_async_extract_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the extract request+ let mock = server+ .mock("POST", "/v1/extract")+ .match_body(mockito::Matcher::PartialJson(json!({+ "urls": ["https://example.com"],+ "prompt": "Extract the title and main content"+ })))+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "extract-123",+ "urlTrace": []+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = ExtractParams {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: Some("Extract the title and main content".to_string()),+ schema: None,+ ..Default::default()+ };++ let response = app.async_extract(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "extract-123");+ assert!(response.url_trace.unwrap_or_default().is_empty());+ mock.assert();+ }++ #[tokio::test]+ async fn test_extract_with_schema() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the extract request with schema+ let mock = server+ .mock("POST", "/v1/extract")+ .match_body(mockito::Matcher::PartialJson(json!({+ "urls": ["https://example.com"],+ "schema": {+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "content": { "type": "string" }+ }+ }+ })))+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "extract-123"+ })+ .to_string(),+ )+ .create();++ // Set up the mock for the status request+ let status_mock = server+ .mock("GET", "/v1/extract/extract-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "data": {+ "title": "Example Domain",+ "content": "This domain is for use in illustrative examples in documents."+ }+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let urls = Some(vec!["https://example.com".to_string()]);+ let params = ExtractParams {+ urls,+ schema: Some(json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "content": { "type": "string" }+ }+ })),+ ..Default::default()+ };++ let response = app.extract(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.status, "completed");++ let data = response.data.unwrap();+ assert_eq!(data["title"], "Example Domain");+ assert_eq!(+ data["content"],+ "This domain is for use in illustrative examples in documents."+ );++ mock.assert();+ status_mock.assert();+ }++ #[tokio::test]+ async fn test_extract_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the status check+ let mock = server+ .mock("GET", "/v1/extract/extract-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "processing",+ "urlTrace": [+ {+ "url": "https://example.com",+ "status": "scraping",+ "timing": {+ "discoveredAt": "2023-01-01T00:00:00Z"+ }+ }+ ]+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = app.get_extract_status("extract-123").await.unwrap();++ assert!(status.success);+ assert_eq!(status.status, "processing");+ assert_eq!(status.url_trace.unwrap()[0].url, "https://example.com");+ mock.assert();+ }++ #[tokio::test]+ async fn test_extract_validation_errors() {+ let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();++ // Test missing both URLs and prompt+ let result = app.async_extract(ExtractParams::default()).await;+ assert!(result.is_err());++ // Test having URLs but missing both prompt and schema+ let params = ExtractParams {+ urls: Some(vec!["https://example.com".to_string()]),+ ..Default::default()+ };+ let result = app.async_extract(params).await;+ assert!(result.is_err());+ }++ #[tokio::test]+ async fn test_extract_api_error() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for an error response+ let mock = server+ .mock("POST", "/v1/extract")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid schema format"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = ExtractParams {+ urls: Some(vec!["https://example.com".to_string()]),+ schema: Some(json!("invalid")), // Invalid schema format+ ..Default::default()+ };++ let result = app.async_extract(params).await;+ assert!(result.is_err());+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex 5d95cc7d2..80bffc468 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -2,14 +2,18 @@ use reqwest::{Client, Response};use serde::de::DeserializeOwned;use serde_json::Value;+pub mod batch_scrape;pub mod crawl;pub mod document;mod error;+pub mod extract;+pub mod llmstxt;pub mod map;pub mod scrape;+pub mod search;-pub use error::FirecrawlError;use error::FirecrawlAPIError;+pub use error::FirecrawlError;#[derive(Clone, Debug)]pub struct FirecrawlApp {@@ -26,9 +30,12 @@ impl FirecrawlApp {FirecrawlApp::new_selfhosted(CLOUD_API_URL, Some(api_key))}- pub fn new_selfhosted(api_url: impl AsRef<str>, api_key: Option<impl AsRef<str>>) -> Result<Self, FirecrawlError> {+ pub fn new_selfhosted(+ api_url: impl AsRef<str>,+ api_key: Option<impl AsRef<str>>,+ ) -> Result<Self, FirecrawlError> {let url = api_url.as_ref().to_string();-+if url == CLOUD_API_URL && api_key.is_none() {return Err(FirecrawlError::APIError("Configuration".to_string(),@@ -36,7 +43,7 @@ impl FirecrawlApp {success: false,error: "API key is required for cloud service".to_string(),details: None,- }+ },));}@@ -73,27 +80,43 @@ impl FirecrawlApp {.text().await.map_err(|e| FirecrawlError::ResponseParseErrorText(e))- .and_then(|response_json| serde_json::from_str::<Value>(&response_json).map_err(|e| FirecrawlError::ResponseParseError(e)))+ .and_then(|response_json| {+ serde_json::from_str::<Value>(&response_json)+ .map_err(|e| FirecrawlError::ResponseParseError(e))+ .inspect(|data| {+ #[cfg(debug_assertions)]+ println!("Response JSON: {:#?}", data);+ })+ }).and_then(|response_value| {- if response_value["success"].as_bool().unwrap_or(false) {- Ok(serde_json::from_value::<T>(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?)+ if action.as_ref().starts_with("crawl_") // no success in check/cancel crawl responses+ || response_value["success"].as_bool().unwrap_or(false)+ {+ Ok(serde_json::from_value::<T>(response_value)+ .map_err(|e| FirecrawlError::ResponseParseError(e))?)} else {Err(FirecrawlError::APIError(action.as_ref().to_string(),- serde_json::from_value(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?+ serde_json::from_value(response_value)+ .map_err(|e| FirecrawlError::ResponseParseError(e))?,))}});match &response {Ok(_) => response,- Err(FirecrawlError::ResponseParseError(_)) | Err(FirecrawlError::ResponseParseErrorText(_)) => {+ Err(FirecrawlError::ResponseParseError(_))+ | Err(FirecrawlError::ResponseParseErrorText(_)) => {if is_success {response} else {- Err(FirecrawlError::HttpRequestFailed(action.as_ref().to_string(), status.as_u16(), status.as_str().to_string()))+ Err(FirecrawlError::HttpRequestFailed(+ action.as_ref().to_string(),+ status.as_u16(),+ status.as_str().to_string(),+ ))}- },+ }Err(_) => response,}}diff --git a/apps/rust-sdk/src/llmstxt.rs b/apps/rust-sdk/src/llmstxt.rsnew file mode 100644index 000000000..f8fda7e91--- /dev/null+++ b/apps/rust-sdk/src/llmstxt.rs@@ -0,0 +1,426 @@+use serde::{Deserialize, Serialize};++use crate::{FirecrawlApp, FirecrawlError, API_VERSION};++/// Parameters for generating LLMs.txt+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct GenerateLLMsTextParams {+ /// URL for which to generate LLMs.txt+ pub url: String,++ /// Maximum number of URLs to process. Default: 10+ pub max_urls: u32,++ /// Whether to show the full LLMs-full.txt in the response. Default: false+ pub show_full_text: bool,++ /// Experimental streaming option+ #[serde(rename = "__experimental_stream")]+ pub experimental_stream: bool,+}++impl Default for GenerateLLMsTextParams {+ fn default() -> Self {+ Self {+ url: String::new(),+ max_urls: 1,+ show_full_text: false,+ experimental_stream: false,+ }+ }+}++/// Response from initiating a LLMs.txt generation job+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct GenerateLLMsTextResponse {+ /// Whether the request was successful+ pub success: bool,++ /// Job ID for the LLMs.txt generation+ pub id: String,+}++#[derive(Deserialize, Serialize, Debug, Clone, Default)]+pub struct LLMTextData {+ #[serde(rename = "llmstxt")]+ pub compact: Option<String>,+ #[serde(rename = "llmsfulltxt")]+ pub full: Option<String>,+}++/// Response from checking the status of a LLMs.txt generation job+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct GenerateLLMsTextStatusResponse {+ /// Whether the request was successful+ pub success: bool,++ /// Status of the job: "pending", "processing", "completed", "failed"+ pub status: String,++ /// Generated LLMs.txt data, present when status is "completed"+ #[serde(default)]+ pub data: LLMTextData,++ /// Error message if the job failed+ pub error: Option<String>,++ /// Expiration timestamp for the data+ pub expires_at: String,+}++impl FirecrawlApp {+ /// Generates LLMs.txt for a given URL and polls until completion.+ ///+ /// # Arguments+ ///+ /// * `params` - Parameters for the LLMs.txt generation+ ///+ /// # Returns+ ///+ /// A response containing the generation results, or a FirecrawlError if the request fails.+ pub async fn generate_llms_text(+ &self,+ params: impl Into<GenerateLLMsTextParams>,+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {+ // Initiate the LLMs.txt generation job asynchronously+ let response = self.async_generate_llms_text(params).await?;++ // Poll for the result+ let poll_interval = 2000; // Default to 2 seconds+ self.monitor_llms_text_job_status(&response.id, poll_interval)+ .await+ }++ /// Initiates an asynchronous LLMs.txt generation operation.+ ///+ /// # Arguments+ ///+ /// * `params` - Parameters for the LLMs.txt generation+ ///+ /// # Returns+ ///+ /// A response containing the generation job ID, or a FirecrawlError if the request fails.+ pub async fn async_generate_llms_text(+ &self,+ params: impl Into<GenerateLLMsTextParams>,+ ) -> Result<GenerateLLMsTextResponse, FirecrawlError> {+ let params = params.into();++ // Validation: URL must be provided+ if params.url.is_empty() {+ return Err(FirecrawlError::APIError(+ "Generate LLMs.txt validation".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: "URL must be provided".to_string(),+ details: None,+ },+ ));+ }++ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(format!("{}{}/llmstxt", self.api_url, API_VERSION))+ .headers(headers)+ .json(¶ms)+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError("Initiating LLMs.txt generation".to_string(), e)+ })?;++ self.handle_response(response, "initiate LLMs.txt generation")+ .await+ }++ /// Checks the status of a LLMs.txt generation operation.+ ///+ /// # Arguments+ ///+ /// * `id` - The ID of the LLMs.txt generation operation+ ///+ /// # Returns+ ///+ /// A response containing the current status and results of the generation operation,+ /// or a FirecrawlError if the request fails.+ pub async fn check_generate_llms_text_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {+ let response = self+ .client+ .get(format!(+ "{}{}/llmstxt/{}",+ self.api_url,+ API_VERSION,+ id.as_ref()+ ))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Checking status of LLMs.txt generation {}", id.as_ref()),+ e,+ )+ })?;++ self.handle_response(+ response,+ format!("Checking status of LLMs.txt generation {}", id.as_ref()),+ )+ .await+ }++ /// Helper function to poll for LLMs.txt generation job status until completion+ async fn monitor_llms_text_job_status(+ &self,+ id: &str,+ poll_interval: u64,+ ) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {+ loop {+ let status_data = self.check_generate_llms_text_status(id).await?;++ match status_data.status.as_str() {+ "completed" => {+ break Ok(status_data);+ }+ "pending" | "processing" => {+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ "failed" => {+ let error_msg = status_data+ .error+ .clone()+ .unwrap_or_else(|| "LLMs.txt generation failed".to_string());+ break Err(FirecrawlError::APIError(+ "LLMs.txt generation failed".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: error_msg,+ details: None,+ },+ ));+ }+ _ => {+ break Err(FirecrawlError::APIError(+ "LLMs.txt generation status".to_string(),+ crate::error::FirecrawlAPIError {+ success: false,+ error: format!("Unexpected status: {}", status_data.status),+ details: None,+ },+ ));+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_generate_llms_text() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();++ let params = GenerateLLMsTextParams {+ url: "https://example.com".to_string(),+ max_urls: 5,+ show_full_text: true,+ ..Default::default()+ };++ let response = app.async_generate_llms_text(params).await.unwrap();++ assert!(response.success);+ assert!(!response.id.is_empty());+ }++ #[tokio::test]+ async fn test_async_generate_llms_text_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v1/llmstxt")+ .match_body(mockito::Matcher::PartialJson(json!({+ "url": "https://example.com",+ "maxUrls": 5+ })))+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "llmstxt-123"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = GenerateLLMsTextParams {+ url: "https://example.com".to_string(),+ max_urls: 5,+ ..Default::default()+ };++ let response = app.async_generate_llms_text(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "llmstxt-123");+ mock.assert();+ }++ #[tokio::test]+ async fn test_check_generate_llms_text_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v1/llmstxt/llmstxt-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "processing",+ "expiresAt": "2023-01-01T00:00:00Z"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = app+ .check_generate_llms_text_status("llmstxt-123")+ .await+ .unwrap();++ assert!(status.success);+ assert_eq!(status.status, "processing");+ assert_eq!(status.expires_at, "2023-01-01T00:00:00Z");+ mock.assert();+ }++ #[tokio::test]+ async fn test_generate_llms_text_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for the generate request+ let mock = server+ .mock("POST", "/v1/llmstxt")+ .match_body(mockito::Matcher::PartialJson(json!({+ "url": "https://example.com",+ "showFullText": true+ })))+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "llmstxt-123"+ })+ .to_string(),+ )+ .create();++ // Set up the mock for the status request+ let status_mock = server+ .mock("GET", "/v1/llmstxt/llmstxt-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "data": {+ "llmstxt": "Allow: /about\nDisallow: /admin\n",+ "llmsfulltxt": "# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n"+ },+ "expiresAt": "2023-01-01T00:00:00Z"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = GenerateLLMsTextParams {+ url: "https://example.com".to_string(),+ show_full_text: true,+ ..Default::default()+ };++ let response = app.generate_llms_text(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.status, "completed");++ let data = response.data;+ assert_eq!(+ data.compact,+ Some("Allow: /about\nDisallow: /admin\n".into())+ );+ assert_eq!(+ data.full,+ Some("# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n".into())+ );++ mock.assert();+ status_mock.assert();+ }++ #[tokio::test]+ async fn test_generate_llms_text_validation_errors() {+ let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();++ // Test missing URL+ let params = GenerateLLMsTextParams {+ url: "".to_string(),+ ..Default::default()+ };+ let result = app.async_generate_llms_text(params).await;+ assert!(result.is_err());+ }++ #[tokio::test]+ async fn test_generate_llms_text_api_error() {+ let mut server = mockito::Server::new_async().await;++ // Set up the mock for an error response+ let mock = server+ .mock("POST", "/v1/llmstxt")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid URL format"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let params = GenerateLLMsTextParams {+ url: "not-a-valid-url".to_string(),+ ..Default::default()+ };++ let result = app.async_generate_llms_text(params).await;+ assert!(result.is_err());+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 7c3b3a438..44016064d 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -16,7 +16,7 @@ pub struct MapOptions {pub include_subdomains: Option<bool>,/// Maximum number of links to return (default: `5000`)- pub exclude_tags: Option<u32>,+ pub limit: Option<u32>,}#[derive(Deserialize, Serialize, Debug, Default)]@@ -59,7 +59,9 @@ impl FirecrawlApp {.await.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;- let response = self.handle_response::<MapResponse>(response, "scrape URL").await?;+ let response = self+ .handle_response::<MapResponse>(response, "scrape URL")+ .await?;Ok(response.links)}diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex b879fdaf3..6432b04a7 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -24,26 +24,26 @@ pub enum ScrapeFormats {Links,/// Will result in a URL to a screenshot of the page.- ///+ ////// Can not be used in conjunction with `ScrapeFormats::ScreenshotFullPage`.#[serde(rename = "screenshot")]Screenshot,/// Will result in a URL to a full-page screenshot of the page.- ///+ ////// Can not be used in conjunction with `ScrapeFormats::Screenshot`.#[serde(rename = "screenshot@fullPage")]ScreenshotFullPage,/// Will result in the results of an LLM extraction.- ///+ ////// See `ScrapeOptions.extract` for more options.#[serde(rename = "extract")]Extract,}#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default)]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]pub struct ExtractOptions {/// Schema the output should adhere to, provided in JSON Schema format.@@ -56,7 +56,7 @@ pub struct ExtractOptions {}#[serde_with::skip_serializing_none]-#[derive(Deserialize, Serialize, Debug, Default)]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]pub struct ScrapeOptions {/// Formats to extract from the page. (default: `[ Markdown ]`)@@ -66,12 +66,12 @@ pub struct ScrapeOptions {pub only_main_content: Option<bool>,/// HTML tags to exclusively include.- ///+ ////// For example, if you pass `div`, you will only get content from `<div>`s and their children.pub include_tags: Option<Vec<String>>,/// HTML tags to exclude.- ///+ ////// For example, if you pass `img`, you will never get image URLs in your results.pub exclude_tags: Option<Vec<String>>,@@ -131,7 +131,9 @@ impl FirecrawlApp {.await.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;- let response = self.handle_response::<ScrapeResponse>(response, "scrape URL").await?;+ let response = self+ .handle_response::<ScrapeResponse>(response, "scrape URL")+ .await?;Ok(response.data)}diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rsnew file mode 100644index 000000000..397bc8d16--- /dev/null+++ b/apps/rust-sdk/src/search.rs@@ -0,0 +1,245 @@+use crate::{scrape::ScrapeOptions, FirecrawlApp, FirecrawlError, API_VERSION};+use serde::{Deserialize, Serialize};++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchParams {+ /// The search query string+ pub query: String,+ /// Maximum number of results to return. Default: 5, Max: 20+ pub limit: Option<u32>,+ /// Time-based search filter.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub tbs: Option<String>,+ /// Query string to filter search results. Example: "site:example.com"+ #[serde(skip_serializing_if = "Option::is_none")]+ pub filter: Option<String>,+ /// Language code. Default: "en"+ pub lang: Option<String>,+ /// Country code. Default: "us"+ pub country: Option<String>,+ /// Geographic location string for local search results+ #[serde(skip_serializing_if = "Option::is_none")]+ pub location: Option<String>,+ /// Origin identifier. Default: "api"+ pub origin: Option<String>,+ /// Timeout in milliseconds. Default: 60000+ pub timeout: Option<u32>,+ /// Additional options for webpage scraping behavior+ #[serde(skip_serializing_if = "Option::is_none")]+ pub scrape_options: Option<ScrapeOptions>,+}++impl Default for SearchParams {+ fn default() -> Self {+ Self {+ query: String::new(),+ limit: Some(5),+ tbs: None,+ filter: None,+ lang: Some("en".to_string()),+ country: Some("us".to_string()),+ location: None,+ origin: Some("api".to_string()),+ timeout: Some(60000),+ scrape_options: None,+ }+ }+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchResponse {+ pub success: bool,+ pub data: Vec<SearchDocument>,+ pub warning: Option<String>,+}++// TODO: Consider merging fields into document::Document (url, title, description) while preserving optionality+/// A document returned from a search or scrape request+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchDocument {+ /// Document URL+ pub url: String,+ /// Document title+ pub title: String,+ /// Document description+ pub description: String,+}++impl FirecrawlApp {+ /// Search for content using the Firecrawl API.+ ///+ /// # Arguments+ ///+ /// * `query` - The search query string+ /// * `params` - Optional parameters for the search request+ ///+ /// # Returns+ ///+ /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.+ pub async fn search(+ &self,+ query: impl AsRef<str>,+ params: impl Into<Option<SearchParams>>,+ ) -> Result<SearchResponse, FirecrawlError> {+ let mut search_params = params.into().unwrap_or_default();+ search_params.query = query.as_ref().to_string();++ self.search_with_params(search_params).await+ }++ /// Alternative method that takes SearchParams directly+ ///+ /// # Arguments+ ///+ /// * `params` - Search parameters including the query+ ///+ /// # Returns+ ///+ /// A SearchResponse containing the search results, or a FirecrawlError if the request fails.+ pub async fn search_with_params(+ &self,+ params: SearchParams,+ ) -> Result<SearchResponse, FirecrawlError> {+ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(format!("{}{}/search", self.api_url, API_VERSION))+ .headers(headers)+ .json(¶ms)+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Searching with query: {:?}", params.query), e)+ })?;++ self.handle_response::<SearchResponse>(response, "search")+ .await+ }+}++#[cfg(test)]+pub mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ #[ignore = "Makes real network request"]+ async fn test_real_search() {+ let api_url = std::env::var("FIRECRAWL_API_URL")+ .expect("Please set the FIRECRAWL_API_URL environment variable");+ let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();+ let response = app.search("test query", None).await.unwrap();+ assert!(response.success);+ }++ #[tokio::test]+ async fn test_search_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v1/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": [{+ "url": "https://example.com",+ "title": "Example Domain",+ "description": "...."+ }],+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = app.search("test", None).await.unwrap();++ assert!(response.success);+ assert_eq!(response.data.len(), 1);+ assert_eq!(response.data[0].url, "https://example.com");+ assert_eq!(response.data[0].title, "Example Domain".to_string());+ assert_eq!(response.data[0].description, "....".to_string());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_with_params() {+ let mut server = mockito::Server::new_async().await;+ let mock = server+ .mock("POST", "/v1/search")+ .with_header("content-type", "application/json")+ .match_body(mockito::Matcher::Json(json!({+ "query": "test",+ "limit": 10,+ "lang": "fr",+ "country": "fr",+ "origin": "api",+ "timeout": 30000+ })))+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": [],+ "warning": "No results found"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let params = SearchParams {+ query: "test".to_string(),+ limit: Some(10),+ lang: Some("fr".to_string()),+ country: Some("fr".to_string()),+ timeout: Some(30000),+ ..Default::default()+ };++ let response = app.search_with_params(params).await.unwrap();++ assert!(response.success);+ assert_eq!(response.data.len(), 0);+ assert_eq!(response.warning, Some("No results found".to_string()));+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_error_response() {+ let mut server = mockito::Server::new_async().await;+ let mock = server+ .mock("POST", "/v1/search")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid query"+ })+ .to_string(),+ )+ .create();++ let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = app.search("", None).await;++ assert!(result.is_err());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_network_error() {+ let app = FirecrawlApp::new_selfhosted("http://invalid-url", Some("test_key")).unwrap();+ let result = app.search("test", None).await;+ assert!(result.is_err());+ }+}diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rsindex 882a29413..071537f50 100644--- a/apps/rust-sdk/tests/e2e_with_auth.rs+++ b/apps/rust-sdk/tests/e2e_with_auth.rs@@ -1,4 +1,3 @@-use assert_matches::assert_matches;use dotenvy::dotenv;use firecrawl::scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions};use firecrawl::{FirecrawlApp, FirecrawlError};@@ -24,11 +23,8 @@ use std::env;async fn test_successful_response_with_valid_preview_token() {dotenv().ok();let api_url = env::var("API_URL").unwrap();- let app = FirecrawlApp::new_selfhosted(- api_url,- Some(env::var("PREVIEW_TOKEN").unwrap()),- )- .unwrap();+ let app =+ FirecrawlApp::new_selfhosted(api_url, Some(env::var("PREVIEW_TOKEN").unwrap())).unwrap();let result = app.scrape_url("https://roastmywebsite.ai", None).await@@ -58,7 +54,7 @@ async fn test_successful_response_with_valid_api_key_and_include_html() {let api_key = env::var("TEST_API_KEY").ok();let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();let params = ScrapeOptions {- formats: vec! [ ScrapeFormats::Markdown, ScrapeFormats::HTML ].into(),+ formats: vec![ScrapeFormats::Markdown, ScrapeFormats::HTML].into(),..Default::default()};let result = app@@ -82,7 +78,8 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file() {.await.unwrap();assert!(result.markdown.is_some());- assert!(result.markdown+ assert!(result+ .markdown.unwrap().contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));}@@ -98,12 +95,12 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici.await.unwrap();assert!(result.markdown.is_some());- assert!(result.markdown+ assert!(result+ .markdown.unwrap().contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));}-// #[tokio::test]// async fn test_should_return_error_for_blocklisted_url() {// dotenv().ok();@@ -159,14 +156,18 @@ async fn test_llm_extraction() {#[test]fn test_api_key_requirements() {dotenv().ok();-+let api_url = env::var("API_URL").unwrap_or("http://localhost:3002".to_string());let api_key = env::var("TEST_API_KEY").ok();match (api_url.contains("api.firecrawl.dev"), api_key) {(false, _) => {let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);- assert!(result.is_ok(), "Local setup failed: {:?}", result.err().unwrap());+ assert!(+ result.is_ok(),+ "Local setup failed: {:?}",+ result.err().unwrap()+ );}(true, None) => {let result = FirecrawlApp::new_selfhosted(&api_url, None::<String>);