Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: better observability for keyless (#4634)
Thomas Kosmas51bdd34f1ae5
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 97cabfde5..aeaad01e8 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -22,6 +22,7 @@ import {normalizeKeylessIpv4,} from "../lib/keyless";import { isKeylessIpSuspicious } from "../lib/spur";+import { keylessAuthTotal } from "../lib/keyless-metrics";import { checkIpRestriction } from "../lib/ip-restriction";import { checkKeyEndpointRestriction } from "../lib/key-restriction";import { deleteKey, getValue, setValue } from "../services/redis";@@ -537,6 +538,7 @@ async function handleKeylessAuth(// main way the per-IP caps get bypassed. Fails open on any Spur error, and// runs before consuming quota so a flagged IP doesn't burn a request slot.if (await isKeylessIpSuspicious(ip)) {+ keylessAuthTotal.inc({ mode, outcome: "suspicious" });logger.warn("Keyless request blocked: suspicious IP", {canonicalLog: "keyless/consume",ip,@@ -570,6 +572,7 @@ async function handleKeylessAuth(try {result = await consumeKeylessRequest(ip);} catch (error) {+ keylessAuthTotal.inc({ mode, outcome: "error" });// Limiter store (Redis) unavailable — fail closed with a controlled auth// response instead of surfacing a 500, and shed the free traffic while the// limiter can't enforce quotas.@@ -594,6 +597,7 @@ async function handleKeylessAuth(};if (!result.ok) {+ keylessAuthTotal.inc({ mode, outcome: result.reason ?? "error" });logger.warn("Keyless request blocked", {...baseLog,blocked: true,@@ -614,6 +618,7 @@ async function handleKeylessAuth(}logger.debug("Keyless request consumed", { ...baseLog, blocked: false });+ keylessAuthTotal.inc({ mode, outcome: "allowed" });// Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence// are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.diff --git a/apps/api/src/lib/keyless-metrics.ts b/apps/api/src/lib/keyless-metrics.tsnew file mode 100644index 000000000..d0ec3ff4d--- /dev/null+++ b/apps/api/src/lib/keyless-metrics.ts@@ -0,0 +1,30 @@+import { Counter } from "prom-client";++// Fixed labels only: never put IPs, team IDs, or caller-provided values here.+export const keylessAuthTotal = new Counter({+ name: "firecrawl_keyless_auth_total",+ help: "Keyless auth decisions for eligible IPs on supported endpoints with the tier configured; allowed does not imply downstream success",+ labelNames: ["mode", "outcome"],+});++export const keylessCreditBlocksTotal = new Counter({+ name: "firecrawl_keyless_credit_blocks_total",+ help: "Projected-credit quota blocks after keyless auth allowed a request",+});++export const keylessCreditsTotal = new Counter({+ name: "firecrawl_keyless_credits_total",+ help: "Actual keyless credits consumed, excluding provisional reservations",+});++export const spurEventsTotal = new Counter({+ name: "firecrawl_spur_events_total",+ help: "Spur lookup, cache, suspicious verdict and error events; a check may emit multiple events and includes eligibility probes",+ labelNames: ["event"],+});++export const spurBypassesTotal = new Counter({+ name: "firecrawl_spur_bypasses_total",+ help: "Spur checks skipped or failed open without a reputation result; includes eligibility probes, not necessarily admitted requests",+ labelNames: ["reason"],+});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 823993f02..7b6cd9d0a 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -7,6 +7,10 @@ import * as schema from "../db/schema";import { redisRateLimitClient } from "../services/rate-limiter";import { isKeylessIpSuspicious } from "./spur";import { logger } from "./logger";+import {+ keylessCreditBlocksTotal,+ keylessCreditsTotal,+} from "./keyless-metrics";// Keyless free tier: scrape, search, and interact can be used without an API key// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO@@ -270,6 +274,7 @@ return {1, total}DAY_SECONDS,)) as [number, number];+ if (result[0] !== 1) keylessCreditBlocksTotal.inc();return {ok: result[0] === 1,creditsUsed: Number(result[1] ?? 0),@@ -408,6 +413,7 @@ export async function logKeylessCreditUsage(// record: self-hosted deployments without DB auth track nothing.if (config.USE_DB_AUTHENTICATION !== true) return;+ keylessCreditsTotal.inc(creditsUsed);if (creditsUsed <= 0) {// TODO(firecrawl-db): switch to a `keyless_credit_usage` row once the// zero-credit usage migration is merged. The IP is repeated in thediff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.tsindex c6117f526..afa92dff4 100644--- a/apps/api/src/lib/spur.ts+++ b/apps/api/src/lib/spur.ts@@ -14,6 +14,7 @@ import { isIPv4 } from "net";import { config } from "../config";import { redisSpurClient } from "../services/spur-redis";import { logger } from "./logger";+import { spurBypassesTotal, spurEventsTotal } from "./keyless-metrics";const FETCH_TIMEOUT_MS = 5000;// Headroom over the fetch timeout for the Redis round-trips around it.@@ -89,6 +90,7 @@ function isSuspicious(ctx: SpurContext): boolean {function verdict(ip: string, ctx: SpurContext | null): boolean {if (!ctx || !isSuspicious(ctx)) return false;+ spurEventsTotal.inc({ event: "suspicious" });logger.info("Keyless IP flagged suspicious by Spur", {...meta(ip),suspicious: true,@@ -99,6 +101,12 @@ function verdict(ip: string, ctx: SpurContext | null): boolean {// Verdict for a cache entry that is not a miss: a failure marker fails open.function cachedVerdict(ip: string, cached: CacheState): boolean {+ spurEventsTotal.inc({+ event: cached.state === "hit" ? "cache_hit" : "cached_failure",+ });+ if (cached.state === "failed") {+ spurBypassesTotal.inc({ reason: "cached_failure" });+ }return verdict(ip, cached.state === "hit" ? cached.ctx : null);}@@ -118,6 +126,7 @@ async function readCache(ip: string): Promise<CacheState> {[raw, failed] = await redisSpurClient.mget(contextKey(ip), failedKey(ip));} catch (error) {logger.warn("Spur cache read failed", { ...meta(ip), error });+ spurEventsTotal.inc({ event: "cache_error" });return { state: "miss" };}const ctx = parseContext(raw);@@ -136,6 +145,7 @@ async function writeCache(await redisSpurClient.set(key, value, "EX", ttlSec);} catch (error) {logger.warn("Spur cache write failed", { ...meta(ip), key, error });+ spurEventsTotal.inc({ event: "cache_error" });}}@@ -144,6 +154,7 @@ async function releaseLock(ip: string, token: string) {await redisSpurClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);} catch (error) {logger.warn("Spur lock release failed", { ...meta(ip), error });+ spurEventsTotal.inc({ event: "cache_error" });}}@@ -152,6 +163,7 @@ async function fetchContext(apiKey: string,): Promise<SpurContext | null> {logger.info("Spur Context API request (cache miss)", meta(ip));+ spurEventsTotal.inc({ event: "lookup" });const res = await fetch(`https://api.spur.us/v2/context/${encodeURIComponent(ip)}`,{@@ -190,6 +202,8 @@ async function lookup(ip: string, apiKey: string): Promise<boolean> {if (ctx) {await writeCache(ip, contextKey(ip), JSON.stringify(ctx), CONTEXT_TTL_SEC);} else {+ spurEventsTotal.inc({ event: "lookup_error" });+ spurBypassesTotal.inc({ reason: "lookup_error" });await writeCache(ip, failedKey(ip), "1", FAILED_TTL_SEC);}return verdict(ip, ctx);@@ -211,12 +225,21 @@ async function waitForResult(ip: string): Promise<boolean> {...meta(ip),savedApiCall: false,});+ spurEventsTotal.inc({ event: "wait_timeout" });+ spurBypassesTotal.inc({ reason: "wait_timeout" });return false;}export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {const apiKey = config.SPUR_API_KEY;- if (!apiKey || !isIPv4(ip)) return false;+ if (!apiKey) {+ spurBypassesTotal.inc({ reason: "disabled" });+ return false;+ }+ if (!isIPv4(ip)) {+ spurBypassesTotal.inc({ reason: "non_ipv4" });+ return false;+ }const cached = await readCache(ip);if (cached.state !== "miss") return cachedVerdict(ip, cached);@@ -233,6 +256,8 @@ export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {"NX",)) === "OK";} catch (error) {+ spurEventsTotal.inc({ event: "cache_error" });+ spurBypassesTotal.inc({ reason: "lock_error" });logger.warn("Spur context lookup failed; failing open", {...meta(ip),timedOut: false,