Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: better observability for keyless (#4634)

Thomas Kosmas51bdd34f1ae5
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 97cabfde5..aeaad01e8 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -22,6 +22,7 @@ import {
normalizeKeylessIpv4,
} from "../lib/keyless";
import { isKeylessIpSuspicious } from "../lib/spur";
+import { keylessAuthTotal } from "../lib/keyless-metrics";
import { checkIpRestriction } from "../lib/ip-restriction";
import { checkKeyEndpointRestriction } from "../lib/key-restriction";
import { deleteKey, getValue, setValue } from "../services/redis";
@@ -537,6 +538,7 @@ async function handleKeylessAuth(
// main way the per-IP caps get bypassed. Fails open on any Spur error, and
// runs before consuming quota so a flagged IP doesn't burn a request slot.
if (await isKeylessIpSuspicious(ip)) {
+ keylessAuthTotal.inc({ mode, outcome: "suspicious" });
logger.warn("Keyless request blocked: suspicious IP", {
canonicalLog: "keyless/consume",
ip,
@@ -570,6 +572,7 @@ async function handleKeylessAuth(
try {
result = await consumeKeylessRequest(ip);
} catch (error) {
+ keylessAuthTotal.inc({ mode, outcome: "error" });
// Limiter store (Redis) unavailable — fail closed with a controlled auth
// response instead of surfacing a 500, and shed the free traffic while the
// limiter can't enforce quotas.
@@ -594,6 +597,7 @@ async function handleKeylessAuth(
};
if (!result.ok) {
+ keylessAuthTotal.inc({ mode, outcome: result.reason ?? "error" });
logger.warn("Keyless request blocked", {
...baseLog,
blocked: true,
@@ -614,6 +618,7 @@ async function handleKeylessAuth(
}
logger.debug("Keyless request consumed", { ...baseLog, blocked: false });
+ keylessAuthTotal.inc({ mode, outcome: "allowed" });
// Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence
// are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.
diff --git a/apps/api/src/lib/keyless-metrics.ts b/apps/api/src/lib/keyless-metrics.ts
new file mode 100644
index 000000000..d0ec3ff4d
--- /dev/null
+++ b/apps/api/src/lib/keyless-metrics.ts
@@ -0,0 +1,30 @@
+import { Counter } from "prom-client";
+
+// Fixed labels only: never put IPs, team IDs, or caller-provided values here.
+export const keylessAuthTotal = new Counter({
+ name: "firecrawl_keyless_auth_total",
+ help: "Keyless auth decisions for eligible IPs on supported endpoints with the tier configured; allowed does not imply downstream success",
+ labelNames: ["mode", "outcome"],
+});
+
+export const keylessCreditBlocksTotal = new Counter({
+ name: "firecrawl_keyless_credit_blocks_total",
+ help: "Projected-credit quota blocks after keyless auth allowed a request",
+});
+
+export const keylessCreditsTotal = new Counter({
+ name: "firecrawl_keyless_credits_total",
+ help: "Actual keyless credits consumed, excluding provisional reservations",
+});
+
+export const spurEventsTotal = new Counter({
+ name: "firecrawl_spur_events_total",
+ help: "Spur lookup, cache, suspicious verdict and error events; a check may emit multiple events and includes eligibility probes",
+ labelNames: ["event"],
+});
+
+export const spurBypassesTotal = new Counter({
+ name: "firecrawl_spur_bypasses_total",
+ help: "Spur checks skipped or failed open without a reputation result; includes eligibility probes, not necessarily admitted requests",
+ labelNames: ["reason"],
+});
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index 823993f02..7b6cd9d0a 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -7,6 +7,10 @@ import * as schema from "../db/schema";
import { redisRateLimitClient } from "../services/rate-limiter";
import { isKeylessIpSuspicious } from "./spur";
import { logger } from "./logger";
+import {
+ keylessCreditBlocksTotal,
+ keylessCreditsTotal,
+} from "./keyless-metrics";
// Keyless free tier: scrape, search, and interact can be used without an API key
// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO
@@ -270,6 +274,7 @@ return {1, total}
DAY_SECONDS,
)) as [number, number];
+ if (result[0] !== 1) keylessCreditBlocksTotal.inc();
return {
ok: result[0] === 1,
creditsUsed: Number(result[1] ?? 0),
@@ -408,6 +413,7 @@ export async function logKeylessCreditUsage(
// record: self-hosted deployments without DB auth track nothing.
if (config.USE_DB_AUTHENTICATION !== true) return;
+ keylessCreditsTotal.inc(creditsUsed);
if (creditsUsed <= 0) {
// TODO(firecrawl-db): switch to a `keyless_credit_usage` row once the
// zero-credit usage migration is merged. The IP is repeated in the
diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.ts
index c6117f526..afa92dff4 100644
--- a/apps/api/src/lib/spur.ts
+++ b/apps/api/src/lib/spur.ts
@@ -14,6 +14,7 @@ import { isIPv4 } from "net";
import { config } from "../config";
import { redisSpurClient } from "../services/spur-redis";
import { logger } from "./logger";
+import { spurBypassesTotal, spurEventsTotal } from "./keyless-metrics";
const FETCH_TIMEOUT_MS = 5000;
// Headroom over the fetch timeout for the Redis round-trips around it.
@@ -89,6 +90,7 @@ function isSuspicious(ctx: SpurContext): boolean {
function verdict(ip: string, ctx: SpurContext | null): boolean {
if (!ctx || !isSuspicious(ctx)) return false;
+ spurEventsTotal.inc({ event: "suspicious" });
logger.info("Keyless IP flagged suspicious by Spur", {
...meta(ip),
suspicious: true,
@@ -99,6 +101,12 @@ function verdict(ip: string, ctx: SpurContext | null): boolean {
// Verdict for a cache entry that is not a miss: a failure marker fails open.
function cachedVerdict(ip: string, cached: CacheState): boolean {
+ spurEventsTotal.inc({
+ event: cached.state === "hit" ? "cache_hit" : "cached_failure",
+ });
+ if (cached.state === "failed") {
+ spurBypassesTotal.inc({ reason: "cached_failure" });
+ }
return verdict(ip, cached.state === "hit" ? cached.ctx : null);
}
@@ -118,6 +126,7 @@ async function readCache(ip: string): Promise<CacheState> {
[raw, failed] = await redisSpurClient.mget(contextKey(ip), failedKey(ip));
} catch (error) {
logger.warn("Spur cache read failed", { ...meta(ip), error });
+ spurEventsTotal.inc({ event: "cache_error" });
return { state: "miss" };
}
const ctx = parseContext(raw);
@@ -136,6 +145,7 @@ async function writeCache(
await redisSpurClient.set(key, value, "EX", ttlSec);
} catch (error) {
logger.warn("Spur cache write failed", { ...meta(ip), key, error });
+ spurEventsTotal.inc({ event: "cache_error" });
}
}
@@ -144,6 +154,7 @@ async function releaseLock(ip: string, token: string) {
await redisSpurClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);
} catch (error) {
logger.warn("Spur lock release failed", { ...meta(ip), error });
+ spurEventsTotal.inc({ event: "cache_error" });
}
}
@@ -152,6 +163,7 @@ async function fetchContext(
apiKey: string,
): Promise<SpurContext | null> {
logger.info("Spur Context API request (cache miss)", meta(ip));
+ spurEventsTotal.inc({ event: "lookup" });
const res = await fetch(
`https://api.spur.us/v2/context/${encodeURIComponent(ip)}`,
{
@@ -190,6 +202,8 @@ async function lookup(ip: string, apiKey: string): Promise<boolean> {
if (ctx) {
await writeCache(ip, contextKey(ip), JSON.stringify(ctx), CONTEXT_TTL_SEC);
} else {
+ spurEventsTotal.inc({ event: "lookup_error" });
+ spurBypassesTotal.inc({ reason: "lookup_error" });
await writeCache(ip, failedKey(ip), "1", FAILED_TTL_SEC);
}
return verdict(ip, ctx);
@@ -211,12 +225,21 @@ async function waitForResult(ip: string): Promise<boolean> {
...meta(ip),
savedApiCall: false,
});
+ spurEventsTotal.inc({ event: "wait_timeout" });
+ spurBypassesTotal.inc({ reason: "wait_timeout" });
return false;
}
export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {
const apiKey = config.SPUR_API_KEY;
- if (!apiKey || !isIPv4(ip)) return false;
+ if (!apiKey) {
+ spurBypassesTotal.inc({ reason: "disabled" });
+ return false;
+ }
+ if (!isIPv4(ip)) {
+ spurBypassesTotal.inc({ reason: "non_ipv4" });
+ return false;
+ }
const cached = await readCache(ip);
if (cached.state !== "miss") return cachedVerdict(ip, cached);
@@ -233,6 +256,8 @@ export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {
"NX",
)) === "OK";
} catch (error) {
+ spurEventsTotal.inc({ event: "cache_error" });
+ spurBypassesTotal.inc({ reason: "lock_error" });
logger.warn("Spur context lookup failed; failing open", {
...meta(ip),
timedOut: false,