Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

9.1 KB
Raw Blame
import { redis } from "../../../db/redis/redis.js";
import { createServiceLogger } from "../../../lib/logger/logger.js";

/**
 * The enumeration observer β€” shadow mode.
 *
 * A distributed crawler that walks a site's URL space leaves no per-visitor
 * evidence: it paces itself under every rate threshold, sends one well-formed
 * event per identity, and runs a real browser. Nothing in the per-request layers
 * can see it. What it cannot hide is the *shape* of what a cohort requests: it
 * enumerates permutations, so nearly every path it asks for is one nobody has
 * asked for before, and it burns a fresh identity per hit.
 *
 * That shape is measured here, per (cohort, 15-minute bucket):
 *
 *   path novelty      distinct paths / events        β€” enumeration, not reading
 *   events per actor  events / distinct actors       β€” an identity per request
 *   direct share      events with no referrer        β€” arrived at, not navigated to
 *
 * Two properties keep this honest. First, it is an aggregate statement β€” it
 * describes a cohort, not the request in front of us β€” so it is recorded and
 * never convicts. `observeEnumeration` returns an observation; no caller turns
 * it into a score. Promoting it later is a deliberate act, informed by what
 * shadow mode collects, not a threshold nudge.
 *
 * Second, a cohort must qualify in two consecutive buckets before it is marked,
 * which filters a one-off migration crawl or sitemap ping.
 *
 * The direct share is measured on the RAW referrer, before self-referrers are
 * cleared for storage. A stored empty referrer means "direct or internal
 * navigation", which is a different and far more common thing β€” measuring the
 * stored value would put most ordinary browsing above the threshold.
 *
 * Two known weaknesses, recorded because they are the reason this is not a rule
 * yet and must be resolved before it becomes one:
 *
 *   1. The streak is evaluated against the CURRENT bucket's running counters,
 *      not a closed one. A cohort that qualifies transiently early in a bucket
 *      stays marked even if the full bucket would not have qualified, and 300
 *      events either side of a bucket boundary reads as "sustained" on two
 *      seconds of traffic. Promotion needs closed-bucket evaluation.
 *   2. The direct share does not separate a crawler from a genuinely direct
 *      campaign. Email, in-app and messenger traffic arrives with no referrer
 *      on personalized one-hit-per-recipient URLs β€” the crawler's shape in every
 *      measured respect. Only UTM-tagged campaigns are distinguishable today,
 *      and only because they tend to be referred, which is not something to rely
 *      on.
 */

const logger = createServiceLogger("enumeration-observer");

export const ENUMERATION_BUCKET_MS = 15 * 60 * 1000;

/**
 * Gates, set well outside anything organic traffic produces. The crawler that
 * motivated them ran at 0.96 novelty and 1.17 events per actor sustained over
 * hours; a busy content site sits near 0.02 novelty with several events per
 * visitor. The volume floor keeps small cohorts β€” where these ratios are noise β€”
 * out of it entirely.
 */
const MIN_EVENTS = 300;
const MIN_PATH_NOVELTY = 0.9;
const MAX_EVENTS_PER_ACTOR = 1.25;
const MIN_DIRECT_SHARE = 0.98;

/** Raw counter readings for one cohort's current bucket. */
export interface EnumerationReading {
  events: number;
  distinctPaths: number;
  distinctActors: number;
  directEvents: number;
}

export interface EnumerationObservation {
  /** This bucket meets every gate. */
  qualifies: boolean;
  /** This bucket and the one before it both qualified. */
  sustained: boolean;
  pathNovelty: number;
  eventsPerActor: number;
  directShare: number;
  events: number;
}

// Mirrors the Redis streak markers for when Redis is unavailable, one entry per
// (cohort, bucket) exactly as Redis holds one key per (cohort, bucket). Storing
// only the latest bucket per cohort would not do: every later hit in the same
// bucket would then read its own marker back and see no streak. Bounded because
// a cohort only lands here after clearing the volume and shape gates, which
// almost nothing does.
const localQualifiedBuckets = new Set<string>();
const MAX_LOCAL_STREAK_ENTRIES = 1024;

/**
 * The streak verdict for each (cohort, bucket) this process has already
 * resolved. Without it the cost of observing scales with the crawler: a cohort
 * qualifies at its 300th event, and every event after that in the same bucket
 * would repeat the round-trip and re-log the same finding β€” thousands of both,
 * on exactly the traffic being measured. The marker only needs writing once, so
 * resolve once per bucket and serve the answer from here afterwards.
 */
const resolvedBuckets = new Map<string, boolean>();
const MAX_RESOLVED_BUCKETS = 1024;

function ratio(numerator: number, denominator: number): number {
  return denominator > 0 ? numerator / denominator : 0;
}

export function evaluateEnumerationReading(reading: EnumerationReading) {
  const pathNovelty = ratio(reading.distinctPaths, reading.events);
  const eventsPerActor = ratio(reading.events, reading.distinctActors);
  const directShare = ratio(reading.directEvents, reading.events);

  const qualifies =
    reading.events >= MIN_EVENTS &&
    pathNovelty >= MIN_PATH_NOVELTY &&
    // An actor count above the event count is not meaningful; HyperLogLog is
    // approximate, so this can happen by a hair on small buckets.
    eventsPerActor > 0 &&
    eventsPerActor <= MAX_EVENTS_PER_ACTOR &&
    directShare >= MIN_DIRECT_SHARE;

  return { qualifies, pathNovelty, eventsPerActor, directShare, events: reading.events };
}

function markLocally(cohortKey: string, bucket: number): boolean {
  const previousQualified = localQualifiedBuckets.has(`${cohortKey}:${bucket - 1}`);
  if (localQualifiedBuckets.size >= MAX_LOCAL_STREAK_ENTRIES) {
    localQualifiedBuckets.clear();
  }
  localQualifiedBuckets.add(`${cohortKey}:${bucket}`);
  return previousQualified;
}

/**
 * Record that this bucket qualified and report whether the previous one did too.
 * Resolved at most once per (cohort, bucket) per process β€” see `resolvedBuckets`
 * β€” so the extra round-trip is charged per qualifying bucket, not per event in
 * one.
 *
 * `useLocalState` follows whichever backend the counters themselves used. Going
 * to Redis for the marker while the counters are in-process would mean paying a
 * connection timeout on every qualifying bucket during an outage, which is the
 * moment the hot path can least afford it.
 */
async function resolveStreak(cohortKey: string, bucket: number, useLocalState: boolean): Promise<boolean> {
  const bucketKey = `${cohortKey}:${bucket}`;
  const resolved = resolvedBuckets.get(bucketKey);
  if (resolved !== undefined) {
    return resolved;
  }

  let sustained: boolean;
  if (useLocalState) {
    sustained = markLocally(cohortKey, bucket);
  } else {
    try {
      const results = await redis
        .multi()
        .set(`bot:enum:q:${bucketKey}`, "1", "PX", ENUMERATION_BUCKET_MS * 3)
        .exists(`bot:enum:q:${cohortKey}:${bucket - 1}`)
        .exec();

      // `exec` returns [error, value] pairs in command order; the second is ours.
      sustained = results?.[1]?.[1] === 1;
    } catch (error) {
      logger.error({ err: error }, "Enumeration streak check failed; using in-process fallback");
      // The marker for the previous bucket may only exist in Redis, so a streak
      // spanning an outage is lost. Shadow mode tolerates that; it under-reports
      // rather than inventing a finding.
      sustained = markLocally(cohortKey, bucket);
    }
  }

  if (resolvedBuckets.size >= MAX_RESOLVED_BUCKETS) {
    resolvedBuckets.clear();
  }
  resolvedBuckets.set(bucketKey, sustained);
  return sustained;
}

/**
 * Evaluate one cohort bucket. Returns null when the cohort's counters were not
 * collected (no cohort dimensions, or no path to measure novelty against).
 */
export async function observeEnumeration(
  cohortKey: string,
  bucket: number,
  reading: EnumerationReading,
  useLocalState = false
): Promise<EnumerationObservation | null> {
  const evaluated = evaluateEnumerationReading(reading);
  if (!evaluated.qualifies) {
    return { ...evaluated, sustained: false };
  }

  const alreadyResolved = resolvedBuckets.has(`${cohortKey}:${bucket}`);
  const sustained = await resolveStreak(cohortKey, bucket, useLocalState);

  if (sustained && !alreadyResolved) {
    // The only output of shadow mode. It is deliberately a log line and not a
    // score: promoting this rule should follow from reading these, not from it
    // having quietly been convicting all along.
    logger.warn(
      {
        cohortKey,
        bucket,
        events: evaluated.events,
        pathNovelty: Number(evaluated.pathNovelty.toFixed(3)),
        eventsPerActor: Number(evaluated.eventsPerActor.toFixed(3)),
        directShare: Number(evaluated.directShare.toFixed(3)),
      },
      "Enumeration cohort sustained across two buckets (shadow mode, not enforced)"
    );
  }

  return { ...evaluated, sustained };
}

export function resetEnumerationObserverForTests() {
  localQualifiedBuckets.clear();
  resolvedBuckets.clear();
}