Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

18.2 KB
Raw Blame
import {
  CLIENT_BOT_SIGNAL_MASKS,
  CLIENT_BOT_SIGNAL_WEIGHTS,
  ClientBotSignalName,
  getClientBotSignalNames,
  getScreenDimensionSignals,
  MAX_CLIENT_BOT_SCORE,
  scoreFromMask,
  STRONG_CLIENT_BOT_SIGNAL_BITS,
} from "@rybbit/shared";
import type { IncomingHttpHeaders } from "http";
import { lookupAsn, type AsnInfo, type AsnLookup } from "../../../db/geolocation/asn.js";
import { logger } from "../../../lib/logger/logger.js";
import type { AnomalyCounters } from "./anomalyScorer.js";
import { observeTrackingAnomaly } from "./anomalyScorer.js";
import type { BotDetectionMethod } from "./botDetectionStats.js";
import { recordBotBlockingRequest, recordBotDetections } from "./botDetectionStats.js";
import { classifyBotAsn } from "./botProviderAsns.js";
import { CLIENT_BOT_SCORE_THRESHOLD } from "./config.js";
import { detectBot } from "./headerHeuristics.js";
import { classifyStaleBrowserVersion } from "./staleBrowserVersion.js";
import { classifyUA } from "./uaBots/index.js";

// Per-detection logging is verbose and costly at high traffic; off by default.
const LOG_BOT_DETECTIONS = false;

interface BotBlockingPayload {
  /** Numeric Site id β€” the one ingestion uses everywhere else. */
  siteId: number;
  userAgent?: string;
  clientBotScore?: number;
  clientBotSignalMask?: number;
  screenWidth?: number;
  screenHeight?: number;
  language?: string;
  hostname?: string;
  pathname?: string;
  eventType?: string;
  /** As it arrived, before self-referrers are cleared for storage. */
  referrer?: string;
  ipAddress: string;
}

interface BotBlockingInput {
  /**
   * The request's headers. Header heuristics are the only thing this needs from
   * the HTTP request, so it takes headers rather than a `FastifyRequest` β€”
   * ingestion can then be driven from a plain value.
   */
  headers: IncomingHttpHeaders;
  blockBots: boolean;
  trustedServerSideIngestion?: boolean;
  /**
   * App/mobile site. The UA-pattern and header-heuristic layers are
   * browser-shaped and produce false positives for native SDK traffic, so they
   * are skipped for mobile sites (see checkBotBlocking).
   */
  isMobileSite?: boolean;
  payload: BotBlockingPayload;
  /** Request-scoped ASN resolver, shared with the rest of the ingestion path. */
  lookupAsn?: AsnLookup;
}

interface AnomalyReason {
  rule: string;
  score: number;
  value: number;
  threshold: number;
  windowSeconds: number;
}

export interface BotBlockingDetection {
  layer: BotDetectionMethod;
  botCategory?: string | null;
  matchedPattern?: string | null;
  botName?: string | null;
  botOperator?: string | null;
  botPurpose?: string | null;
  reason?: string;
  score?: number;
  clientBotScore?: number;
  clientBotSignalMask?: number;
  clientSignals?: ClientBotSignalName[];
  ip?: string;
  asn?: number;
  asnOrg?: string;
  asnProvider?: string;
  asnCategory?: string;
  asnNote?: string;
  anomalyReasons?: AnomalyReason[];
  anomalyCounters?: AnomalyCounters;
}

export interface BotEventProperties {
  isBot: true;
  botAsn?: number;
  botAsnOrg?: string;
  detectedUaPattern: boolean;
  detectedHeaderHeuristics: boolean;
  detectedClientSignals: boolean;
  detectedBotAsn: boolean;
  detectedRateAnomaly: boolean;
  matchedUaPattern: string;
  botCategory: string;
  /**
   * Published identity of the bot, when the matched pattern is one of the
   * curated ones. `matchedUaPattern` is the evidence (a regex source); these
   * are the answer to "who is this and what do they want" β€” which is the only
   * form in which crawl volume can be attributed to an operator or split into
   * training, answer-engine indexing, and user-triggered agent fetches.
   * Empty for a bot matched only by a generic upstream pattern.
   */
  botName: string;
  botOperator: string;
  botPurpose: string;
  /**
   * Curated provider behind the request's ASN. A user agent can claim any
   * name, so this is the independent half of an attribution claim.
   */
  asnProvider: string;
  clientBotScore?: number;
  clientSignalMask?: number;
  /**
   * Which anomaly rules fired, comma-separated, and what they summed to. The
   * other layers name themselves β€” a UA pattern or an ASN is its own
   * explanation β€” but "rate anomaly" covers a dozen rules with very different
   * meanings, and without these an audit row cannot answer why the request was
   * convicted. That question is the entire reason `bot_observations` exists.
   */
  anomalyReasons: string;
  anomalyScore: number;
}

export interface BotDetectionResult {
  isBot: true;
  /**
   * Whether this detection is acted on. Detection and enforcement are separate
   * concerns: every site is evaluated, but only a site with bot blocking on has
   * its bot traffic diverted out of `events`. A detection with `enforced: false`
   * is recorded to `bot_observations` and the event is tracked as normal, so a
   * site that has opted out still produces evidence of what it is receiving.
   */
  enforced: boolean;
  message: string;
  detections: BotBlockingDetection[];
  eventProperties: BotEventProperties;
}

function buildBotEventProperties(
  detections: BotBlockingDetection[],
  asnInfo: AsnInfo | null,
  clientSignalResult: ReturnType<typeof getClientSignalResult>
): BotEventProperties {
  const detectionLayers = new Set(detections.map(detection => detection.layer));
  const uaDetection = detections.find(detection => detection.layer === "ua_pattern");
  const anomalyDetection = detections.find(detection => detection.layer === "rate_anomaly");
  const asnDetection = detections.find(detection => detection.layer === "bot_asn");

  return {
    isBot: true,
    botAsn: asnInfo?.asn,
    botAsnOrg: asnInfo?.organization ?? "",
    detectedUaPattern: detectionLayers.has("ua_pattern"),
    detectedHeaderHeuristics: detectionLayers.has("header_heuristics"),
    detectedClientSignals: detectionLayers.has("client_signals"),
    detectedBotAsn: detectionLayers.has("bot_asn"),
    detectedRateAnomaly: detectionLayers.has("rate_anomaly"),
    matchedUaPattern: uaDetection?.matchedPattern ?? "",
    botCategory: uaDetection?.botCategory ?? "",
    botName: uaDetection?.botName ?? "",
    botOperator: uaDetection?.botOperator ?? "",
    botPurpose: uaDetection?.botPurpose ?? "",
    asnProvider: asnDetection?.asnProvider ?? "",
    clientBotScore: clientSignalResult.scoreForStats,
    clientSignalMask: clientSignalResult.maskForStats,
    anomalyReasons: (anomalyDetection?.anomalyReasons ?? []).map(reason => reason.rule).join(","),
    anomalyScore: anomalyDetection?.score ?? 0,
  };
}

function getClientSignalResult(
  payload: BotBlockingPayload,
  userAgent: string,
  hasReportableScreen: boolean
) {
  const hasClientScore = typeof payload.clientBotScore === "number" && Number.isFinite(payload.clientBotScore);
  const hasClientMask = typeof payload.clientBotSignalMask === "number" && Number.isFinite(payload.clientBotSignalMask);
  const rawMask = hasClientMask ? payload.clientBotSignalMask! : 0;
  let mask = rawMask;
  let inferredScore = 0;

  function addInferredSignal(name: ClientBotSignalName) {
    const bit = CLIENT_BOT_SIGNAL_MASKS[name];
    if ((mask & bit) === 0) {
      mask |= bit;
    }

    if (!hasClientScore || (rawMask & bit) === 0) {
      inferredScore += CLIENT_BOT_SIGNAL_WEIGHTS[name];
    }
  }

  // Re-derived server-side rather than trusted from the client mask, so the
  // rules apply to every hit regardless of which tracker version sent it. An
  // event that reports no dimensions at all says nothing about its display, so
  // the geometry rules are skipped β€” but the skip itself is recorded, weakly, so
  // that "no dimensions" is visible in the mask instead of passing silently. It
  // is only raised where a screen was expected: a native SDK and a trusted
  // server-side integration both legitimately have none, and counting them would
  // make the signal useless for reading adoption or confirming a deploy.
  const { screenWidth, screenHeight } = payload;
  if (screenWidth !== undefined || screenHeight !== undefined) {
    for (const signal of getScreenDimensionSignals(screenWidth ?? NaN, screenHeight ?? NaN, userAgent)) {
      addInferredSignal(signal);
    }
  } else if (hasReportableScreen) {
    addInferredSignal("missingScreenDimensions");
  }

  const score = Math.min((hasClientScore ? payload.clientBotScore! : 0) + inferredScore, MAX_CLIENT_BOT_SCORE);

  // Score contributed by strong (convicting) signals only. Derived from the
  // mask, not the client's opaque score β€” a score sent without a mask cannot be
  // decomposed, so it can only ever corroborate.
  const strongScore = scoreFromMask(mask & STRONG_CLIENT_BOT_SIGNAL_BITS);

  return {
    score,
    strongScore,
    mask,
    signalNames: getClientBotSignalNames(mask),
    scoreForStats: hasClientScore || inferredScore > 0 ? score : undefined,
    maskForStats: hasClientMask || mask !== 0 ? mask : undefined,
    hasClientMask,
    hasClientScore,
  };
}

/**
 * Bot detection for trusted server-side ingestion: the user-agent layer alone.
 *
 * The stale-browser-version sub-rule is left out too. It convicts a UA claiming
 * Chrome older than the tracker that reported the hit β€” sound reasoning for a
 * browser, meaningless for a server relaying someone else's user agent, and it
 * would start convicting traffic that existing API users are sending today.
 *
 * Enforcement is not special-cased: the result goes back as an ordinary
 * detection, so `blockBots` decides its fate exactly as it does for browser
 * traffic and the row lands in `bot_events` or `bot_observations` accordingly.
 */
function classifyTrustedIngestion(
  userAgent: string,
  blockBots: boolean,
  clientSignalResult: ReturnType<typeof getClientSignalResult>
): BotDetectionResult | null {
  const uaClassification = classifyUA(userAgent);
  if (!uaClassification.isBot) {
    return null;
  }

  const detections: BotBlockingDetection[] = [
    {
      layer: "ua_pattern",
      botCategory: uaClassification.category,
      matchedPattern: uaClassification.matchedPattern,
      botName: uaClassification.name,
      botOperator: uaClassification.operator,
      botPurpose: uaClassification.purpose,
    },
  ];

  recordBotDetections(["ua_pattern"], blockBots);

  return {
    isBot: true,
    enforced: blockBots,
    message: "Bot detected using ua-pattern",
    detections,
    // No ASN: the address belongs to the reporting server, not to the bot, so
    // attributing it would be worse than leaving it empty.
    eventProperties: buildBotEventProperties(detections, null, clientSignalResult),
  };
}

export async function checkBotBlocking({
  headers,
  blockBots,
  trustedServerSideIngestion = false,
  isMobileSite = false,
  payload,
  lookupAsn: asnLookup = lookupAsn,
}: BotBlockingInput): Promise<BotDetectionResult | null> {
  const userAgent = payload.userAgent || (headers["user-agent"] as string) || "";
  const clientSignalResult = getClientSignalResult(payload, userAgent, !isMobileSite && !trustedServerSideIngestion);
  recordBotBlockingRequest(
    clientSignalResult.scoreForStats,
    clientSignalResult.maskForStats,
    clientSignalResult.hasClientMask,
    clientSignalResult.hasClientScore
  );

  // Trusted server-side ingestion is authenticated first-party traffic that
  // reports its own IP and user agent on someone else's behalf. Four of the
  // five layers are meaningless against it and several would convict it
  // outright: header heuristics read the reporting server's headers, client
  // signals require a browser that never ran, the ASN belongs to whoever is
  // doing the reporting, and rate anomaly would see one origin standing in for
  // its entire audience.
  //
  // The UA layer is the exception, and the only way Rybbit can see a crawler
  // that does not execute JavaScript. `POST /api/track` is the sole ingestion
  // path, so a training crawler that fetches HTML and runs nothing is invisible
  // unless the origin forwards it β€” and a forwarded `GPTBot` used to land in
  // `events` as a human, because detection was skipped wholesale. Classifying
  // the reported user agent routes it to `bot_events` with full identity
  // instead.
  //
  // `blockBots` deliberately does NOT skip detection. It decides what happens to
  // a detection, not whether one is looked for β€” a site that has turned blocking
  // off used to get no evaluation at all, which left it with neither protection
  // nor any record of what it was receiving.
  if (trustedServerSideIngestion) {
    return classifyTrustedIngestion(userAgent, blockBots, clientSignalResult);
  }

  const detections: BotBlockingDetection[] = [];
  let blockMessage: string | null = null;

  function addDetection(message: string, detection: BotBlockingDetection) {
    blockMessage ??= message;
    detections.push(detection);
  }

  // Layers 1 and 2 are browser-shaped: they flag native HTTP clients (okhttp,
  // CFNetwork, Cronet) as scripting frameworks and treat the absence of
  // browser-only headers (Accept-Language, sec-fetch-*) as suspicious. A
  // first-party mobile SDK legitimately looks exactly like that, so skip these
  // layers for app/mobile sites and rely on the client-signal, ASN, and
  // rate-anomaly layers below, which apply equally to native traffic.
  if (!isMobileSite) {
    // Layer 1: User-agent classification (vendored from isbot patterns, with categories)
    const uaClassification = classifyUA(userAgent);
    if (uaClassification.isBot) {
      addDetection("Bot detected using ua-pattern", {
        layer: "ua_pattern",
        botCategory: uaClassification.category,
        matchedPattern: uaClassification.matchedPattern,
        botName: uaClassification.name,
        botOperator: uaClassification.operator,
        botPurpose: uaClassification.purpose,
      });
    } else {
      // Layer 1b: a browser release too old to have run the tracker that
      // reported the hit. Shares the ua_pattern layer so it lands in the
      // existing bot_category/matched_ua_pattern columns for auditing.
      const staleBrowser = classifyStaleBrowserVersion(userAgent);
      if (staleBrowser.isStale) {
        addDetection("Bot detected using ua-pattern", {
          layer: "ua_pattern",
          botCategory: "stale_version",
          matchedPattern: staleBrowser.matchedVersion,
        });
      }
    }

    // Layer 2: Header heuristic bot detection
    const detection = detectBot(headers, userAgent);
    if (detection.isBot) {
      addDetection("Bot detected using header heuristics", {
        layer: "header_heuristics",
        reason: detection.reason,
        score: detection.score,
      });
    }
  }

  // Layer 3: Client-side and client-derived bot signal score check. Only
  // strong signals convict on their own; weak signals reaching the threshold
  // (or a score the client sent without a mask) corroborate other layers but
  // never convict, mirroring the generic-hosting-ASN rule below.
  let supportingClientSignalDetection: BotBlockingDetection | null = null;
  if (Math.max(clientSignalResult.score, clientSignalResult.strongScore) >= CLIENT_BOT_SCORE_THRESHOLD) {
    const clientSignalDetection: BotBlockingDetection = {
      layer: "client_signals",
      clientBotScore: clientSignalResult.score,
      clientBotSignalMask: clientSignalResult.mask,
      clientSignals: clientSignalResult.signalNames,
    };

    if (clientSignalResult.strongScore >= CLIENT_BOT_SCORE_THRESHOLD) {
      addDetection("Bot detected using client signals", clientSignalDetection);
    } else {
      supportingClientSignalDetection = clientSignalDetection;
    }
  }

  // Layer 4: ASN check β€” IP belongs to hosting/cloud or curated bot provider infrastructure.
  const ipForAsn = payload.ipAddress;
  let asnInfo: AsnInfo | null = null;
  let supportingHostingAsnDetection: BotBlockingDetection | null = null;
  let isHostingAsn = false;
  if (ipForAsn) {
    asnInfo = asnLookup(ipForAsn);
    const botAsnMatch = classifyBotAsn(asnInfo?.asn);
    isHostingAsn = asnInfo !== null && botAsnMatch.isBotInfrastructure;
    if (asnInfo && botAsnMatch.isBotInfrastructure) {
      const asnDetection: BotBlockingDetection = {
        layer: "bot_asn",
        ip: ipForAsn,
        asn: asnInfo.asn,
        asnOrg: asnInfo.organization,
        asnProvider: botAsnMatch.provider,
        asnCategory: botAsnMatch.category,
        asnNote: botAsnMatch.note,
      };

      if (botAsnMatch.source === "curated_bot_provider") {
        addDetection("Bot detected using bot asn", asnDetection);
      } else {
        supportingHostingAsnDetection = asnDetection;
      }
    }
  }

  // Layer 5: Request-rate and crawl-shape anomaly detection.
  const anomaly = await observeTrackingAnomaly({
    siteId: payload.siteId,
    ipAddress: payload.ipAddress,
    userAgent,
    hostname: payload.hostname,
    pathname: payload.pathname,
    eventType: payload.eventType,
    referrer: payload.referrer,
    hasClientBotScore: typeof payload.clientBotScore === "number",
    screenWidth: payload.screenWidth,
    screenHeight: payload.screenHeight,
    language: payload.language,
    isHostingAsn,
    asn: asnInfo?.asn,
  });
  if (anomaly.isAnomalous) {
    addDetection("Bot detected using rate anomaly", {
      layer: "rate_anomaly",
      score: anomaly.score,
      anomalyReasons: anomaly.reasons,
      anomalyCounters: anomaly.counters,
    });
  }

  // Supporting evidence attaches only when a convicting layer already fired β€”
  // two supporting signals must not convict each other.
  const hasConvictingDetection = detections.length > 0;
  if (hasConvictingDetection && supportingClientSignalDetection) {
    addDetection("Bot detected using client signals", supportingClientSignalDetection);
  }
  if (hasConvictingDetection && supportingHostingAsnDetection) {
    addDetection("Bot detected using bot asn", supportingHostingAsnDetection);
  }

  if (detections.length === 0) {
    return null;
  }

  if (LOG_BOT_DETECTIONS) {
    logger.info(
      {
        siteId: payload.siteId,
        detectionCount: detections.length,
        detectionLayers: detections.map(detection => detection.layer),
        detections,
      },
      "Bot request detected"
    );
  }

  recordBotDetections(
    detections.map(detection => detection.layer),
    blockBots
  );

  return {
    isBot: true,
    enforced: blockBots,
    message: blockMessage ?? "Bot detected",
    detections,
    eventProperties: buildBotEventProperties(detections, asnInfo, clientSignalResult),
  };
}