Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
//! Hand-off to Firecrawl cloud for what a laptop cannot do: web search, pages
//! that block a local fetch, and Alexandria data providers.
//!
//! Nothing here runs unless the caller asks for search or the cloud setting
//! allows a fallback. Every result says it came from the cloud and what it
//! cost, as data fields.
use std::time::Duration;
use serde_json::{json, Value};
pub const DEFAULT_API_URL: &str = "https://api.firecrawl.dev";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CloudMode {
/// Never contact Firecrawl cloud.
Off,
/// Search goes to the cloud; scrapes stay local.
SearchOnly,
/// Search goes to the cloud, and a scrape the local engine could not
/// complete is retried there.
Fallback,
}
impl CloudMode {
pub fn parse(s: &str) -> Option<CloudMode> {
match s.trim().to_ascii_lowercase().as_str() {
"off" | "false" | "0" | "none" | "no" | "disabled" | "disable" => Some(CloudMode::Off),
"search" | "search-only" => Some(CloudMode::SearchOnly),
"on" | "auto" | "fallback" | "true" | "1" | "yes" | "enabled" | "enable" => {
Some(CloudMode::Fallback)
}
_ => None,
}
}
/// The mode a `FIRECRAWL_CLOUD` value asks for. Unset or empty asks for
/// nothing. A value that is not understood asks for off: someone who set
/// it was restricting the cloud, and a typo must not open it up.
pub fn requested(raw: Option<&str>) -> Option<CloudMode> {
let raw = raw.map(str::trim).filter(|r| !r.is_empty())?;
Some(CloudMode::parse(raw).unwrap_or(CloudMode::Off))
}
}
/// Whether a path has a `.` or `..` segment, plain or percent-encoded. The
/// HTTP client resolves those before sending, so `/v2/search/../agent` would
/// arrive as `/v2/agent`. Such paths are never passed on.
pub fn has_dot_segment(path: &str) -> bool {
path.split(['/', '\\']).any(|segment| {
let segment = segment.to_ascii_lowercase().replace("%2e", ".");
segment == "." || segment == ".."
})
}
/// Say once that `FIRECRAWL_CLOUD` holds a value this build does not know.
fn warn_unknown_mode(raw: &str) {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
eprintln!(
"FIRECRAWL_CLOUD={raw} is not a setting Firecrawl Local knows (off, search or auto). The cloud stays off."
);
});
}
#[derive(Clone)]
pub struct Cloud {
client: reqwest::Client,
base: String,
api_key: Option<String>,
pub mode: CloudMode,
/// Random ID for this install, so Firecrawl can count installs, not IP
/// addresses. Created on the first cloud call; never tied to a person.
install_id: Option<String>,
/// The agent harness that launched the engine (from the MCP handshake).
harness: Option<String>,
origin: String,
/// Whether to use the login saved by `firecrawl-local login` or the
/// Firecrawl CLI. It is read again on every call, so signing in while an
/// MCP server or `serve` is running takes effect without a restart.
saved_login: bool,
}
pub const SIGN_IN_HINT: &str = "Run `firecrawl-local login`, or set FIRECRAWL_API_KEY.";
/// Reword a refusal Firecrawl cloud sent to a caller with no key, so it says
/// what to do on this machine. The cloud writes these for people calling the
/// API directly ("authenticate with: Authorization: Bearer ..."); someone
/// using a local MCP server or `serve` signs in with one command instead.
pub fn local_sign_in_message(error: &str) -> String {
let lower = error.to_ascii_lowercase();
let mut text = match lower.find("then authenticate with") {
Some(at) => error[..at].trim_end().to_string(),
None => error.trim().to_string(),
};
if lower.contains("alexandria") && lower.contains("api key") {
text = "Alexandria data providers need a free Firecrawl account.".to_string();
}
if text.contains("firecrawl-local login") {
return text;
}
if text.contains("http") {
// Keep the sign-up link on its own line so nothing is glued to it.
format!(
"{text}\n\nOr run `firecrawl-local login`, which signs you in and saves the key for you."
)
} else {
format!("{text} Run `firecrawl-local login` to create one or sign in. It is one step in the browser and the key is saved for you, then the same call works.")
}
}
/// Whether a refusal is about the caller having no account, as opposed to a
/// route's own rule or an ordinary rate limit that signing in would not lift.
fn is_sign_in_refusal(status: u16, payload: &Value, error: &str) -> bool {
if status == 401 || payload.get("signup_url").is_some() || payload.get("signupUrl").is_some() {
return true;
}
let lower = error.to_ascii_lowercase();
["keyless", "api key", "unauthorized", "authenticate with"]
.iter()
.any(|sign| lower.contains(sign))
}
/// Apply `local_sign_in_message` to the `error` field of a JSON answer, when
/// the answer is a refusal for want of an account.
fn with_sign_in_step(status: u16, body: &[u8]) -> Option<bytes::Bytes> {
let mut payload: Value = serde_json::from_slice(body).ok()?;
let error = payload.get("error")?.as_str()?.to_string();
if !is_sign_in_refusal(status, &payload, &error) {
return None;
}
payload["error"] = Value::String(local_sign_in_message(&error));
serde_json::to_vec(&payload).ok().map(bytes::Bytes::from)
}
fn config_dir() -> Option<std::path::PathBuf> {
if let Some(dir) = std::env::var_os("FIRECRAWL_LOCAL_HOME") {
return Some(std::path::PathBuf::from(dir));
}
std::env::var_os("HOME").map(|h| {
std::path::PathBuf::from(h)
.join(".config")
.join("firecrawl-local")
})
}
/// Where the Firecrawl CLI keeps the login from `firecrawl login`. Same
/// paths as the CLI's own `credentials.ts`.
pub fn cli_credentials_path() -> Option<std::path::PathBuf> {
if let Some(p) = std::env::var_os("FIRECRAWL_CLI_CREDENTIALS") {
return Some(std::path::PathBuf::from(p));
}
let home =
std::path::PathBuf::from(std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE"))?);
let dir = if cfg!(target_os = "macos") {
home
.join("Library")
.join("Application Support")
.join("firecrawl-cli")
} else if cfg!(target_os = "windows") {
home.join("AppData").join("Roaming").join("firecrawl-cli")
} else {
home.join(".config").join("firecrawl-cli")
};
Some(dir.join("credentials.json"))
}
/// The API key and API URL saved by `firecrawl login`, if the user has
/// logged in with the CLI. This is what makes sign-in one step: log in once
/// in the browser and the engine picks it up.
fn cli_credentials() -> (Option<String>, Option<String>) {
let Some(path) = cli_credentials_path() else {
return (None, None);
};
let Ok(text) = std::fs::read_to_string(path) else {
return (None, None);
};
let Ok(v) = serde_json::from_str::<Value>(&text) else {
return (None, None);
};
let field = |name: &str| {
v.get(name)
.and_then(Value::as_str)
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
};
(field("apiKey"), field("apiUrl"))
}
/// Read this install's ID, creating it on first use.
fn load_install_id() -> Option<String> {
let dir = config_dir()?;
let path = dir.join("install-id");
if let Ok(existing) = std::fs::read_to_string(&path) {
let id = existing.trim().to_string();
if !id.is_empty() {
return Some(id);
}
}
let id = uuid::Uuid::new_v4().to_string();
std::fs::create_dir_all(&dir).ok()?;
std::fs::write(&path, &id).ok()?;
Some(id)
}
/// How the engine was started, which decides the origin its cloud calls carry.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Surface {
/// A command typed in a terminal, or run by an AI agent through one.
Cli,
/// The stdio MCP server.
Mcp,
/// The local HTTP API.
Serve,
}
/// The origin for a surface, in the forms Firecrawl already uses:
/// `cli@<version>` for the CLI and `mcp-<client>@<version>` for an MCP
/// server, where the client is the name the MCP client gave at initialize.
/// The version is marked `local-` so Firecrawl Local can be told apart from the
/// Node CLI and the hosted MCP server.
fn origin_for(surface: Surface, client: Option<&str>) -> String {
let version = concat!("local-", env!("CARGO_PKG_VERSION"));
match surface {
Surface::Cli => format!("cli@{version}"),
Surface::Serve => format!("firecrawl-local@{version}"),
Surface::Mcp => {
let token: String = client
.unwrap_or("")
.to_ascii_lowercase()
.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
.collect();
let token: String = token
.split('-')
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join("-")
.chars()
.take(48)
.collect();
if token.is_empty() {
format!("mcp-local@{version}")
} else {
format!("mcp-{token}@{version}")
}
}
}
}
/// Firecrawl cloud's answer to a forwarded request, passed back untouched.
pub struct Forwarded {
pub status: u16,
pub content_type: Option<String>,
pub body: bytes::Bytes,
}
#[derive(Debug)]
pub struct CloudError {
pub status: u16,
pub message: String,
}
impl std::fmt::Display for CloudError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
impl Cloud {
/// Reads `FIRECRAWL_API_KEY`, `FIRECRAWL_API_URL` and `FIRECRAWL_CLOUD`,
/// falling back to the login saved by `firecrawl-local login` or the
/// Firecrawl CLI. With no key, cloud calls go out keyless and Firecrawl
/// cloud decides what a caller without an account may do.
pub fn from_env() -> Self {
let (_, saved_url) = cli_credentials();
let api_key = std::env::var("FIRECRAWL_API_KEY")
.ok()
.filter(|k| !k.trim().is_empty());
let base = std::env::var("FIRECRAWL_API_URL")
.ok()
.filter(|u| !u.trim().is_empty())
.or(saved_url)
.unwrap_or_else(|| DEFAULT_API_URL.to_string());
let setting = std::env::var("FIRECRAWL_CLOUD").ok();
if let Some(raw) = setting.as_deref().map(str::trim) {
if !raw.is_empty() && CloudMode::parse(raw).is_none() {
warn_unknown_mode(raw);
}
}
let mode = CloudMode::requested(setting.as_deref()).unwrap_or(CloudMode::Fallback);
let mut cloud = Self::new(base, api_key, mode);
cloud.saved_login = true;
cloud
}
/// The key cloud calls go out with: the one given explicitly, or else the
/// saved login as it is on disk right now.
fn key(&self) -> Option<String> {
self.api_key.clone().or_else(|| {
if self.saved_login {
cli_credentials().0
} else {
None
}
})
}
/// Give the cloud a fixed install ID, so a test never writes one to the
/// developer's config folder.
#[cfg(test)]
pub(crate) fn with_install_id(mut self, id: &str) -> Self {
self.install_id = Some(id.to_string());
self
}
/// Use a key obtained mid-session (after a one-step sign-in).
pub fn with_key(mut self, api_key: String) -> Self {
self.api_key = Some(api_key);
self
}
/// Tag cloud calls with the harness that launched the engine.
pub fn with_harness(mut self, harness: Option<String>) -> Self {
self.harness = harness.map(|h| {
h.chars()
.filter(|c| c.is_ascii_graphic() || *c == ' ')
.take(64)
.collect()
});
self
}
pub fn new(base: String, api_key: Option<String>, mode: CloudMode) -> Self {
let client = reqwest::Client::builder()
.user_agent(concat!("firecrawl-local/", env!("CARGO_PKG_VERSION")))
.connect_timeout(Duration::from_secs(10))
.build()
.unwrap_or_default();
Self {
client,
base: base.trim_end_matches('/').to_string(),
api_key,
mode,
install_id: None,
harness: None,
origin: origin_for(Surface::Cli, None),
saved_login: false,
}
}
/// Stamp cloud calls with the surface the engine is running as, so the
/// request is attributed the way the Firecrawl CLI and MCP server are.
pub fn with_surface(mut self, surface: Surface, client: Option<&str>) -> Self {
self.origin = origin_for(surface, client);
self
}
pub fn origin(&self) -> &str {
&self.origin
}
pub fn has_key(&self) -> bool {
self.key().is_some()
}
/// The key in use right now, for handing to the MCP server this process
/// starts. Never printed.
pub fn api_key(&self) -> Option<String> {
self.key()
}
/// Use the origin a client stamped on its own request, so a call the
/// engine passes on is attributed to that client and not to the engine.
pub fn with_origin(mut self, origin: &str) -> Self {
let clean: String = origin
.chars()
.filter(|c| c.is_ascii_graphic())
.take(100)
.collect();
if !clean.is_empty() {
self.origin = clean;
}
self
}
/// Whether a request for `path` may be passed on to Firecrawl cloud.
pub fn allows(&self, path: &str) -> bool {
if has_dot_segment(path) {
return false;
}
match self.mode {
CloudMode::Off => false,
CloudMode::SearchOnly => path == "/v2/search" || path.starts_with("/v2/search/"),
CloudMode::Fallback => true,
}
}
/// Pass a request Firecrawl Local does not serve itself on to Firecrawl
/// cloud and return the answer. Its own key is used when it has one;
/// otherwise the caller's, and with neither the call is keyless. A keyless
/// call that the cloud refuses comes back with the sign-in step added.
pub async fn forward(
&self,
method: reqwest::Method,
path_and_query: &str,
content_type: Option<&str>,
body: bytes::Bytes,
client_auth: Option<&str>,
client_origin: Option<&str>,
) -> Result<Forwarded, CloudError> {
let path = path_and_query.split('?').next().unwrap_or(path_and_query);
if !self.allows(path) {
return Err(CloudError {
status: 400,
message: format!("{path} is not a path Firecrawl Local passes on to Firecrawl cloud."),
});
}
let origin = match client_origin {
Some(o) => self.clone().with_origin(o).origin,
None => self.origin.clone(),
};
let mut req = self
.client
.request(method, format!("{}{}", self.base, path_and_query))
.timeout(Duration::from_secs(190))
.header("x-origin", origin)
.body(body);
if let Some(ct) = content_type {
req = req.header(reqwest::header::CONTENT_TYPE, ct);
}
if let Some(id) = self.install_id.clone().or_else(load_install_id) {
req = req.header("x-firecrawl-install", id);
}
let own_key = self.key();
let key = own_key.as_deref().or(client_auth);
let keyless = key.is_none();
if let Some(key) = key {
req = req.bearer_auth(key);
}
let resp = req.send().await.map_err(|e| CloudError {
status: 502,
message: format!("Could not reach Firecrawl cloud: {e}"),
})?;
let status = resp.status().as_u16();
let content_type = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let mut body = resp.bytes().await.map_err(|e| CloudError {
status: 502,
message: format!("Firecrawl cloud's answer was cut short: {e}"),
})?;
if keyless && matches!(status, 401 | 402 | 403 | 429) {
if let Some(reworded) = with_sign_in_step(status, &body) {
body = reworded;
}
}
Ok(Forwarded {
status,
content_type,
body,
})
}
async fn post(
&self,
path: &str,
mut body: Value,
timeout: Duration,
) -> Result<Value, CloudError> {
// The scrape route reads the origin from the body only, so it travels
// there as well as in the header.
if let Some(obj) = body.as_object_mut() {
obj
.entry("origin")
.or_insert_with(|| Value::String(self.origin.clone()));
}
let mut req = self
.client
.post(format!("{}{}", self.base, path))
.timeout(timeout)
.header("x-origin", &self.origin)
.json(&body);
if let Some(id) = self.install_id.clone().or_else(load_install_id) {
req = req.header("x-firecrawl-install", id);
}
if let Some(h) = &self.harness {
req = req.header("x-firecrawl-harness", h);
}
let key = self.key();
let signed_in = key.is_some();
if let Some(key) = &key {
req = req.bearer_auth(key);
}
let resp = req.send().await.map_err(|e| CloudError {
status: 502,
message: format!("Could not reach Firecrawl cloud: {e}"),
})?;
let status = resp.status().as_u16();
let payload: Value = resp.json().await.unwrap_or(Value::Null);
if !(200..300).contains(&status) {
let detail = payload
.get("error")
.and_then(Value::as_str)
.unwrap_or("request failed")
.to_string();
let message = match status {
401 | 402 | 403 | 429 if !signed_in => {
// The cloud sends a signup link made for this caller; pass it on.
let link = payload
.get("signupUrl")
.and_then(Value::as_str)
.map(|u| format!(" Or sign up at {u}"))
.unwrap_or_default();
let reason = detail.lines().next().unwrap_or("").trim();
format!("Firecrawl cloud needs a sign-in for this ({reason}). {SIGN_IN_HINT}{link}")
}
401 if signed_in => format!(
"Firecrawl rejected the saved login ({detail}). It may have been rotated or revoked. Run `firecrawl-local login` again, or update FIRECRAWL_API_KEY."
),
_ => format!("Firecrawl cloud returned {status}: {detail}"),
};
return Err(CloudError { status, message });
}
Ok(payload)
}
/// Web search. Always a cloud call: there is no index on a laptop.
pub async fn search(&self, query: &str, limit: u32) -> Result<Value, CloudError> {
if self.mode == CloudMode::Off {
return Err(CloudError {
status: 403,
message:
"Web search runs on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off)."
.into(),
});
}
let mut payload = self
.post(
"/v2/search",
json!({ "query": query, "limit": limit.clamp(1, 50) }),
Duration::from_secs(60),
)
.await?;
annotate(&mut payload);
Ok(payload)
}
/// Retry a scrape in the cloud. Only called when the mode is `Fallback`.
pub async fn scrape(&self, url: &str, options: &Value) -> Result<Value, CloudError> {
let mut body = options.clone();
if !body.is_object() {
body = json!({});
}
body["url"] = Value::String(url.to_string());
let mut payload = self
.post("/v2/scrape", body, Duration::from_secs(120))
.await?;
annotate(&mut payload);
Ok(payload)
}
}
impl Cloud {
/// Run one or more Alexandria provider calls. `calls` is one call object
/// or an array of up to ten: `{provider, capability, options}`. Needs an
/// account: providers cost Firecrawl money per call.
pub async fn alexandria(&self, calls: &Value) -> Result<Value, CloudError> {
if self.mode == CloudMode::Off {
return Err(CloudError {
status: 403,
message: "Alexandria data providers run on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off).".into(),
});
}
if !self.has_key() {
return Err(CloudError {
status: 401,
message: format!("Alexandria data providers need a free Firecrawl account. {SIGN_IN_HINT} The same call works after that."),
});
}
let calls = match calls {
Value::Array(_) => calls.clone(),
Value::Object(_) => Value::Array(vec![calls.clone()]),
_ => return Err(CloudError {
status: 400,
message:
"'alexandria' must be a call object or an array of them: {provider, capability, options}."
.into(),
}),
};
let mut payload = self
.post(
"/v2/scrape",
json!({ "alexandria": calls, "origin": self.origin }),
Duration::from_secs(190),
)
.await?;
annotate(&mut payload);
Ok(payload)
}
/// Browse or search the Alexandria catalogue. Tried without a key first,
/// since browsing costs nothing; the cloud decides whether to allow it.
pub async fn find_tools(&self, options: &Value) -> Result<Value, CloudError> {
if self.mode == CloudMode::Off {
return Err(CloudError {
status: 403,
message: "The Alexandria catalogue lives on Firecrawl cloud, and the cloud is turned off (FIRECRAWL_CLOUD=off).".into(),
});
}
let call = json!([{ "provider": "firecrawl", "capability": "find-tools", "options": options }]);
let mut payload = self
.post(
"/v2/scrape",
json!({ "alexandria": call, "origin": self.origin }),
Duration::from_secs(60),
)
.await?;
annotate(&mut payload);
Ok(payload)
}
}
/// Scrape locally; if the page was refused and the cloud fallback is on,
/// retry it through Firecrawl. The result says where it ran.
pub async fn scrape_with_fallback(
engine: &crate::scrape::Engine,
cloud: &Cloud,
url: &str,
opts: &crate::scrape::ScrapeOptions,
) -> Result<Value, crate::scrape::ScrapeError> {
let local = engine.scrape(url, opts).await;
let mut status = 0u16;
let blocked = match &local {
Ok(doc) => {
status = doc
.pointer("/metadata/statusCode")
.and_then(Value::as_u64)
.unwrap_or(200) as u16;
let md = doc.get("markdown").and_then(Value::as_str).unwrap_or("");
looks_blocked(status, md)
}
Err(e) => matches!(e.code, "FETCH_FAILED" | "SCRAPE_TIMEOUT"),
};
// Why the cloud did not take over, when it was asked and could not.
let mut cloud_refusal: Option<String> = None;
if blocked && cloud.mode == CloudMode::Fallback {
let cloud_opts = json!({ "formats": ["markdown"], "onlyMainContent": opts.only_main_content });
match cloud.scrape(url, &cloud_opts).await {
Ok(payload) => {
if let Some(mut data) = payload.get("data").cloned() {
if let Some(obj) = data.as_object_mut() {
obj.insert("source".into(), Value::String("firecrawl-cloud".into()));
}
return Ok(data);
}
}
Err(e) => cloud_refusal = Some(e.message),
}
}
match local {
Ok(mut doc) => {
if let Some(obj) = doc.as_object_mut() {
obj.insert("source".into(), Value::String("local".into()));
if blocked {
obj.insert("blocked".into(), Value::Bool(true));
crate::scrape::add_warning(
obj,
blocked_warning(status, cloud.mode, cloud_refusal.as_deref()),
);
}
}
Ok(doc)
}
Err(e) => Err(e),
}
}
/// What to tell someone whose page came back refused, so an empty result is
/// never mistaken for an empty page.
fn blocked_warning(status: u16, mode: CloudMode, cloud_refusal: Option<&str>) -> String {
let lead = format!(
"This site did not serve the page to this machine (status {status}), so the content is missing or is a bot check."
);
match (cloud_refusal, mode) {
(Some(refusal), _) => format!("{lead} Firecrawl cloud did not fetch it either. {refusal}"),
(None, CloudMode::Fallback) => lead,
(None, _) => {
format!("{lead} Firecrawl cloud can fetch pages like this when FIRECRAWL_CLOUD is auto.")
}
}
}
/// Mark a cloud response so callers can tell where it ran and what it cost.
fn annotate(payload: &mut Value) {
if let Some(obj) = payload.as_object_mut() {
obj.insert("source".into(), Value::String("firecrawl-cloud".into()));
}
}
/// Signs that a local fetch was refused rather than served.
pub fn looks_blocked(status: u16, markdown: &str) -> bool {
matches!(status, 401 | 403 | 407 | 429 | 503) || looks_like_bot_check(markdown)
}
/// True when a short page is a bot check or a refusal, not content.
pub fn looks_like_bot_check(markdown: &str) -> bool {
if markdown.len() > 2000 {
return false;
}
let lower = markdown.to_ascii_lowercase();
[
"verify you are human",
"verifying you are human",
"checking your browser",
"verifying your browser",
"quick browser check",
"security checkpoint",
"performing security verification",
"enable javascript and cookies to continue",
"attention required! | cloudflare",
"access denied",
"are you a robot",
"captcha",
"unusual traffic",
"pardon our interruption",
]
.iter()
.any(|needle| lower.contains(needle))
}
/// A stand-in for Firecrawl cloud that returns what it was sent.
#[cfg(test)]
pub(crate) async fn echo_cloud() -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
while let Ok((mut socket, _)) = listener.accept().await {
tokio::spawn(async move {
// Read the whole request, keeping only its first 8 KB to echo.
let mut seen: Vec<u8> = Vec::new();
let mut buf = vec![0u8; 64 * 1024];
let mut wanted: Option<usize> = None;
let mut total = 0usize;
loop {
let n = socket.read(&mut buf).await.unwrap_or(0);
if n == 0 {
break;
}
total += n;
if seen.len() < 8192 {
let room = 8192 - seen.len();
seen.extend_from_slice(&buf[..n.min(room)]);
}
if wanted.is_none() {
let text = String::from_utf8_lossy(&seen).to_lowercase();
if let Some(head_end) = text.find("\r\n\r\n") {
let length = text[..head_end]
.lines()
.find_map(|l| l.strip_prefix("content-length:"))
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(0);
wanted = Some(head_end + 4 + length);
}
}
if wanted.is_some_and(|w| total >= w) {
break;
}
}
let request = String::from_utf8_lossy(&seen).to_string();
let body = serde_json::json!({ "request": request }).to_string();
let answer = format!(
"HTTP/1.1 207 Multi-Status\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = socket.write_all(answer.as_bytes()).await;
});
}
});
base
}
/// A stand-in for Firecrawl cloud that refuses every call the way the
/// keyless limit does.
#[cfg(test)]
pub(crate) async fn refusing_cloud() -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
while let Ok((mut socket, _)) = listener.accept().await {
tokio::spawn(async move {
let mut buf = vec![0u8; 8192];
let _ = socket.read(&mut buf).await;
let body = serde_json::json!({
"success": false,
"error": "You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://firecrawl.dev/k/abc123\n\nThen authenticate with:\nAuthorization: Bearer YOUR_API_KEY",
"signup_url": "https://firecrawl.dev/k/abc123"
})
.to_string();
let answer = format!(
"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = socket.write_all(answer.as_bytes()).await;
});
}
});
base
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cli_login_is_picked_up() {
let dir = std::env::temp_dir().join(format!("fc-cred-test-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("credentials.json");
std::fs::write(
&path,
r#"{"apiKey":"fc-test-key","apiUrl":"https://api.example.test"}"#,
)
.unwrap();
std::env::set_var("FIRECRAWL_CLI_CREDENTIALS", &path);
let (key, url) = cli_credentials();
assert_eq!(key.as_deref(), Some("fc-test-key"));
assert_eq!(url.as_deref(), Some("https://api.example.test"));
std::fs::write(&path, "not json").unwrap();
assert_eq!(cli_credentials(), (None, None));
// A login made while the process is running is picked up on the next call.
let mut running = Cloud::new("https://api.example.test".into(), None, CloudMode::Fallback);
running.saved_login = true;
assert!(!running.has_key());
std::fs::write(&path, r#"{"apiKey":"fc-signed-in-later"}"#).unwrap();
assert_eq!(running.key().as_deref(), Some("fc-signed-in-later"));
// A key given explicitly wins, and a cloud that was not told to use the
// saved login never reads it.
assert_eq!(
running
.clone()
.with_key("fc-explicit".into())
.key()
.as_deref(),
Some("fc-explicit")
);
let plain = Cloud::new("https://api.example.test".into(), None, CloudMode::Fallback);
assert!(!plain.has_key());
std::env::remove_var("FIRECRAWL_CLI_CREDENTIALS");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn origins_follow_the_cli_and_mcp_conventions() {
let v = concat!("local-", env!("CARGO_PKG_VERSION"));
assert_eq!(origin_for(Surface::Cli, None), format!("cli@{v}"));
assert_eq!(
origin_for(Surface::Serve, None),
format!("firecrawl-local@{v}")
);
assert_eq!(
origin_for(Surface::Mcp, Some("Claude Code")),
format!("mcp-claude-code@{v}")
);
assert_eq!(
origin_for(Surface::Mcp, Some(" ")),
format!("mcp-local@{v}")
);
assert_eq!(origin_for(Surface::Mcp, None), format!("mcp-local@{v}"));
assert!(origin_for(Surface::Mcp, Some("cursor/1.0 (x)")).starts_with("mcp-cursor-1-0-x@"));
}
#[tokio::test]
async fn forward_passes_the_request_and_the_answer_through() {
let base = echo_cloud().await;
let keyless =
Cloud::new(base.clone(), None, CloudMode::Fallback).with_install_id("test-install");
let answer = keyless
.forward(
reqwest::Method::POST,
"/v2/search?x=1",
Some("application/json"),
bytes::Bytes::from_static(b"{\"query\":\"q\"}"),
None,
Some("[email protected]"),
)
.await
.unwrap();
assert_eq!(answer.status, 207);
assert_eq!(answer.content_type.as_deref(), Some("application/json"));
let seen = String::from_utf8_lossy(&answer.body).to_lowercase();
assert!(seen.contains("post /v2/search?x=1"), "{seen}");
assert!(seen.contains("x-origin: [email protected]"), "{seen}");
assert!(seen.contains("{\\\"query\\\":\\\"q\\\"}"), "{seen}");
assert!(
!seen.contains("authorization:"),
"a keyless call carries no key: {seen}"
);
// The caller's key is used when the engine has none, and the engine's wins when it has one.
let with_client_key = keyless
.forward(
reqwest::Method::GET,
"/v2/team/credit-usage",
None,
bytes::Bytes::new(),
Some("fc-client"),
None,
)
.await
.unwrap();
assert!(String::from_utf8_lossy(&with_client_key.body).contains("Bearer fc-client"));
let own = Cloud::new(base, Some("fc-engine".into()), CloudMode::Fallback)
.with_install_id("test-install");
let with_own_key = own
.forward(
reqwest::Method::GET,
"/v2/team/credit-usage",
None,
bytes::Bytes::new(),
Some("fc-client"),
None,
)
.await
.unwrap();
let seen = String::from_utf8_lossy(&with_own_key.body).to_string();
assert!(
seen.contains("Bearer fc-engine") && !seen.contains("fc-client"),
"{seen}"
);
}
#[test]
fn refusals_are_reworded_for_a_local_user() {
let limit = "You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://firecrawl.dev/k/abc123\n\nThen authenticate with:\nAuthorization: Bearer YOUR_API_KEY";
let said = local_sign_in_message(limit);
assert!(said.contains("https://firecrawl.dev/k/abc123\n"), "{said}");
assert!(said.contains("firecrawl-local login"), "{said}");
assert!(!said.contains("Authorization"), "{said}");
let alexandria =
local_sign_in_message("Alexandria requires an API key on a team with Alexandria access");
assert!(alexandria.starts_with("Alexandria data providers need a free Firecrawl account."));
assert!(alexandria.contains("firecrawl-local login"), "{alexandria}");
// Said once, not twice.
assert_eq!(local_sign_in_message(&said), said);
let body = with_sign_in_step(
429,
br#"{"success":false,"error":"Daily limit reached","signup_url":"x"}"#,
)
.unwrap();
let payload: Value = serde_json::from_slice(&body).unwrap();
assert!(payload["error"]
.as_str()
.unwrap()
.contains("firecrawl-local login"));
assert_eq!(payload["signup_url"], "x");
assert!(with_sign_in_step(401, br#"{"error":"Unauthorized"}"#).is_some());
assert!(with_sign_in_step(403, br#"{"error":"Alexandria requires an API key"}"#).is_some());
// A route's own rule, or a plain rate limit, is not a request to sign in.
assert!(with_sign_in_step(429, br#"{"error":"Rate limit exceeded. Retry in 10s."}"#).is_none());
assert!(with_sign_in_step(
403,
br#"{"error":"This route is not enabled for your team."}"#
)
.is_none());
assert!(with_sign_in_step(402, br#"{"error":"Insufficient credits."}"#).is_none());
assert!(with_sign_in_step(401, b"not json").is_none());
assert!(with_sign_in_step(401, br#"{"error":{"code":1}}"#).is_none());
}
#[tokio::test]
async fn a_keyless_refusal_gains_the_sign_in_step_and_a_keyed_one_does_not() {
let base = refusing_cloud().await;
let keyless = Cloud::new(base.clone(), None, CloudMode::Fallback).with_install_id("test");
let answer = keyless
.forward(
reqwest::Method::POST,
"/v2/search",
Some("application/json"),
bytes::Bytes::from_static(b"{}"),
None,
None,
)
.await
.unwrap();
assert_eq!(answer.status, 429);
let text = String::from_utf8_lossy(&answer.body).to_string();
assert!(text.contains("firecrawl-local login"), "{text}");
assert!(!text.contains("Authorization"), "{text}");
// A caller who sent a key gets the cloud's words as they came.
let with_client_key = keyless
.forward(
reqwest::Method::POST,
"/v2/search",
Some("application/json"),
bytes::Bytes::from_static(b"{}"),
Some("fc-client"),
None,
)
.await
.unwrap();
let text = String::from_utf8_lossy(&with_client_key.body).to_string();
assert!(text.contains("Authorization"), "{text}");
assert!(!text.contains("firecrawl-local login"), "{text}");
}
#[test]
fn a_blocked_page_says_why_it_is_empty() {
let off = blocked_warning(403, CloudMode::Off, None);
assert!(
off.contains("status 403") && off.contains("FIRECRAWL_CLOUD"),
"{off}"
);
let refused = blocked_warning(
403,
CloudMode::Fallback,
Some("Firecrawl cloud needs a sign-in for this. Run `firecrawl-local login`."),
);
assert!(refused.contains("firecrawl-local login"), "{refused}");
assert!(refused.contains("did not fetch it either"), "{refused}");
assert!(!blocked_warning(503, CloudMode::Fallback, None).contains("FIRECRAWL_CLOUD"));
}
#[test]
fn modes_decide_what_is_passed_on() {
let cloud = |mode| Cloud::new("https://api.example.test".into(), None, mode);
assert!(!cloud(CloudMode::Off).allows("/v2/search"));
assert!(cloud(CloudMode::SearchOnly).allows("/v2/search"));
assert!(cloud(CloudMode::SearchOnly).allows("/v2/search/research/papers"));
assert!(!cloud(CloudMode::SearchOnly).allows("/v2/agent"));
assert!(!cloud(CloudMode::SearchOnly).allows("/v2/searching"));
assert!(cloud(CloudMode::Fallback).allows("/v2/agent"));
}
#[test]
fn dot_segments_are_never_passed_on() {
let cloud = |mode| Cloud::new("https://api.example.test".into(), None, mode);
for path in [
"/v2/search/../agent",
"/v2/search/%2e%2e/agent",
"/v2/search/%2E./agent",
"/v2/search/./x",
"/v2/search\\..\\agent",
"/v2/search/..",
] {
assert!(has_dot_segment(path), "{path}");
assert!(!cloud(CloudMode::SearchOnly).allows(path), "{path}");
assert!(!cloud(CloudMode::Fallback).allows(path), "{path}");
}
assert!(!has_dot_segment("/v2/search/v1.2/..hidden"));
assert!(!has_dot_segment("/v2/search/a.b"));
}
#[tokio::test]
async fn forward_refuses_a_path_it_would_not_send_as_written() {
let cloud =
Cloud::new(echo_cloud().await, None, CloudMode::SearchOnly).with_install_id("test-install");
let refused = cloud
.forward(
reqwest::Method::POST,
"/v2/search/../agent?x=1",
Some("application/json"),
bytes::Bytes::from_static(b"{}"),
None,
None,
)
.await;
assert_eq!(refused.err().map(|e| e.status), Some(400));
}
#[test]
fn a_client_origin_replaces_the_engine_origin() {
let cloud = Cloud::new("https://api.example.test".into(), None, CloudMode::Fallback);
assert_eq!(
cloud.clone().with_origin("[email protected]").origin(),
"[email protected]"
);
assert_eq!(cloud.clone().with_origin(" ").origin(), cloud.origin());
assert_eq!(cloud.with_origin("a b\nc").origin(), "abc");
}
#[test]
fn modes_parse() {
assert_eq!(CloudMode::parse("off"), Some(CloudMode::Off));
assert_eq!(CloudMode::parse("search"), Some(CloudMode::SearchOnly));
assert_eq!(CloudMode::parse("auto"), Some(CloudMode::Fallback));
assert_eq!(CloudMode::parse("nonsense"), None);
assert_eq!(CloudMode::parse(" Disabled "), Some(CloudMode::Off));
assert_eq!(CloudMode::parse("no"), Some(CloudMode::Off));
assert_eq!(CloudMode::parse("yes"), Some(CloudMode::Fallback));
}
#[test]
fn a_setting_that_is_not_understood_turns_the_cloud_off() {
assert_eq!(CloudMode::requested(None), None);
assert_eq!(CloudMode::requested(Some(" ")), None);
assert_eq!(
CloudMode::requested(Some("search")),
Some(CloudMode::SearchOnly)
);
assert_eq!(
CloudMode::requested(Some("auto")),
Some(CloudMode::Fallback)
);
assert_eq!(CloudMode::requested(Some("offf")), Some(CloudMode::Off));
assert_eq!(CloudMode::requested(Some("nope")), Some(CloudMode::Off));
}
#[test]
fn blocked_detection() {
assert!(looks_blocked(403, "anything"));
assert!(looks_blocked(
200,
"Please verify you are human to continue"
));
assert!(!looks_blocked(
200,
"A normal article about captcha history. "
.repeat(100)
.as_str()
));
assert!(!looks_blocked(200, "Hello world"));
assert!(looks_blocked(200, "We're verifying your browser\n\n[Website owner? Click here to fix](https://vercel.link/security-checkpoint)"));
assert!(looks_blocked(
200,
"We're performing a quick browser check to help us fight bots."
));
assert!(looks_like_bot_check("## Performing security verification"));
assert!(!looks_like_bot_check(
"A long guide to browser checks. ".repeat(100).as_str()
));
}
}