Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

18.9 KB
Raw Blame
import { Redis } from "ioredis";
import { createServiceLogger } from "../../lib/logger/logger.js";
import { API_RATE_LIMIT_LUA } from "./apiRateLimitLua.js";
import { IMPORT_LOCK_RELEASE_LUA, IMPORT_QUOTA_RESERVE_LUA } from "./importQuotaLua.js";
import { REPLAY_BUDGET_LUA } from "./replayBudgetLua.js";
import { STICKY_RESOLVE_LUA } from "./stickyResolveLua.js";

const logger = createServiceLogger("redis");

// Fail fast so callers can fall back rather than hanging during an outage.
const REDIS_COMMAND_TIMEOUT_MS = 1000;

// ioredis pipelines commands over a single connection, so a slow command holds
// up everything queued behind it (head-of-line blocking). Build each hot-path
// concern its own connection so, for example, session resolution never waits on
// the heavier multi-key anomaly Lua script that runs on the same event.
function createRedisClient(label: string): Redis {
  const client = new Redis({
    host: process.env.REDIS_HOST || "localhost",
    port: parseInt(process.env.REDIS_PORT || "6379", 10),
    ...(process.env.REDIS_PASSWORD ? { password: process.env.REDIS_PASSWORD } : {}),
    commandTimeout: REDIS_COMMAND_TIMEOUT_MS,
    maxRetriesPerRequest: 3,
    retryStrategy: (times: number) => Math.min(times * 200, 5000),
  });

  // ioredis emits an error on every failed reconnect attempt, which floods the
  // logs while Redis is unreachable (e.g. local dev without Redis). Log only
  // when the error changes; a successful connect re-arms logging so the next
  // outage is still reported.
  let lastLoggedError: string | undefined;
  client.on("error", (error: Error) => {
    const signature = `${(error as NodeJS.ErrnoException).code ?? ""}:${error.message}`;
    if (signature === lastLoggedError) return;
    lastLoggedError = signature;
    logger.error(error, `Redis client error (${label}); suppressing repeats`);
  });
  client.on("connect", () => {
    lastLoggedError = undefined;
    logger.info(`Redis connected (${label})`);
  });

  return client;
}

// Shared request/response client for bot detection (anomaly scoring + stats) and
// general use.
export const redis = createRedisClient("main");

// Dashboard JSON responses must not queue behind tracking/session commands.
export const dashboardCacheRedis = createRedisClient("dashboard-cache");

// Dedicated connection for per-event session resolution. Isolated from `redis`
// so a session GET/SET is never head-of-line blocked behind the anomaly scorer's
// Lua call β€” that blocking timed sessions out, and a timed-out session forced the
// fallback path, which used to fracture one visitor into multiple sessions.
export const sessionRedis = createRedisClient("session");

// Atomic get-or-create with a sliding TTL. If the key exists we refresh its TTL
// and return the stored id; otherwise we store the candidate and return it. Run
// as a single Lua script so concurrent workers can never create duplicate
// sessions for the same (siteId, userId). defineCommand uses EVALSHA with an
// EVAL fallback, so the script body is only sent over the wire once.
sessionRedis.defineCommand("sessionGetOrCreate", {
  numberOfKeys: 1,
  lua: `
    local existing = redis.call('GET', KEYS[1])
    if existing then
      redis.call('PEXPIRE', KEYS[1], ARGV[2])
      return existing
    end
    redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
    return ARGV[1]
  `,
});

interface SessionRedis extends Redis {
  sessionGetOrCreate(key: string, candidateId: string, ttlMs: number): Promise<string>;
}

/**
 * Return the session id for `key`, creating it from `candidateId` if absent, and
 * (re)setting its TTL to `ttlMs`. Atomic across all workers.
 */
export function sessionGetOrCreate(key: string, candidateId: string, ttlMs: number): Promise<string> {
  return (sessionRedis as SessionRedis).sessionGetOrCreate(key, candidateId, ttlMs);
}

// Dedicated connection for sticky identity re-attachment, isolated for the same
// reason as `sessionRedis`: a slow call on a shared connection would head-of-line
// block session resolution for unrelated requests.
export const identityRedis = createRedisClient("identity");

// Sticky identity re-attachment: when a visitor behind a rotating proxy egress
// gets a brand-new fingerprint hash mid-visit, re-attach them to their previous
// identity instead of minting a phantom user β€” but only when the evidence is
// unambiguous (exactly one recently-active candidate under the same user agent).
//
// State per site:
//   seen:<userId>   flag: this fingerprint is a known identity (sliding TTL)
//   cand (ZSET)     identities recently seen through *datacenter egress* under
//                   one (site, UA[, salt day]), scored by last-seen ms β€” only
//                   eligible requests register, so a residential visitor can
//                   never be the lone candidate a datacenter arrival merges into
//   alias (chain)   durable re-attachment decision: raw fingerprint -> canonical;
//                   resolution follows chains to the terminal id and compresses
//
// Flow: alias hit -> follow it; known fingerprint -> keep it; new fingerprint ->
// attach iff eligible (datacenter egress) and exactly one candidate, else abstain
// and register as a new identity. Abstention means the failure mode stays "split"
// (status quo), never "merge". Runs as one Lua call so decisions are atomic
// across workers. Keys for the seen/alias touches of *other* identities are
// composed inside the script from prefix ARGVs β€” fine on single-node Redis, not
// cluster-safe. Script body lives in stickyResolveLua.ts so the behavioral tests
// exercise the identical Lua.
identityRedis.defineCommand("stickyResolve", {
  numberOfKeys: 3,
  lua: STICKY_RESOLVE_LUA,
});

export type StickyResolveOutcome = "alias" | "known" | "attach" | "abstain_none" | "abstain_multi" | "new";

export interface StickyResolveInput {
  seenKey: string;
  candidatesKey: string;
  aliasKey: string;
  rawUserId: string;
  nowMs: number;
  candidateWindowMs: number;
  seenTtlMs: number;
  aliasTtlMs: number;
  /** Whether this request may attempt re-attachment (datacenter-egress IPs only). */
  eligible: boolean;
  seenKeyPrefix: string;
  aliasKeyPrefix: string;
  maxCandidates: number;
}

interface IdentityRedis extends Redis {
  stickyResolve(...args: (string | number)[]): Promise<[string, StickyResolveOutcome]>;
}

/**
 * Resolve the canonical user id for a raw fingerprint hash, re-attaching rotated
 * proxy identities when unambiguous. Atomic across all workers; one round-trip.
 */
export function stickyResolve(input: StickyResolveInput): Promise<[string, StickyResolveOutcome]> {
  return (identityRedis as IdentityRedis).stickyResolve(
    input.seenKey,
    input.candidatesKey,
    input.aliasKey,
    input.rawUserId,
    input.nowMs,
    input.candidateWindowMs,
    input.seenTtlMs,
    input.aliasTtlMs,
    input.eligible ? "1" : "0",
    input.seenKeyPrefix,
    input.aliasKeyPrefix,
    input.maxCandidates
  );
}

// Dedicated connection for the per-session replay byte budget. Every replay
// batch runs this script, so it gets its own connection for the same reason as
// `sessionRedis`.
export const replayBudgetRedis = createRedisClient("replay-budget");

replayBudgetRedis.defineCommand("replayBudgetCharge", {
  numberOfKeys: 1,
  lua: REPLAY_BUDGET_LUA,
});

interface ReplayBudgetRedis extends Redis {
  replayBudgetCharge(key: string, bytes: number, capBytes: number, ttlMs: number): Promise<[number, number]>;
}

/**
 * Charge `bytes` against a session's replay budget. Accepted while the session
 * is under `capBytes`; the TTL slides on every call. Atomic across all workers.
 */
export async function replayBudgetCharge(
  key: string,
  bytes: number,
  capBytes: number,
  ttlMs: number
): Promise<{ accepted: boolean; usedBytes: number }> {
  const [accepted, usedBytes] = await (replayBudgetRedis as ReplayBudgetRedis).replayBudgetCharge(
    key,
    bytes,
    capBytes,
    ttlMs
  );
  return { accepted: accepted === 1, usedBytes };
}

// Dedicated connection for API-key rate limiting, isolated for the same reason
// as `sessionRedis`: every authenticated API request runs this script, and a
// shared connection would put it behind the heavier anomaly Lua call.
export const apiRateLimitRedis = createRedisClient("api-rate-limit");

apiRateLimitRedis.defineCommand("apiRateLimit", {
  numberOfKeys: 2,
  lua: API_RATE_LIMIT_LUA,
});

interface ApiRateLimitRedis extends Redis {
  apiRateLimit(...args: (string | number)[]): Promise<[number, number, number, number, number]>;
}

export interface ApiRateLimitInput {
  burstKey: string;
  dailyKey: string;
  burstCapacity: number;
  /** Tokens restored per second; capacity/refillPerSecond is the drain time. */
  burstRefillPerSecond: number;
  /** 0 = unlimited (self-hosted). */
  dailyLimit: number;
  dailyTtlSeconds: number;
  observeOnly: boolean;
}

export interface ApiRateLimitResult {
  allowed: boolean;
  /** 0 = within limits, 1 = burst tier, 2 = daily tier. */
  scope: number;
  burstRemaining: number;
  /** -1 when the daily tier is unlimited. */
  dailyRemaining: number;
  retryAfterMs: number;
}

/**
 * Charge one request against an owner's burst bucket and daily quota. Atomic
 * across all workers and replicas; one round-trip.
 */
export async function apiRateLimitConsume(input: ApiRateLimitInput): Promise<ApiRateLimitResult> {
  const [allowed, scope, burstRemaining, dailyRemaining, retryAfterMs] = await (
    apiRateLimitRedis as ApiRateLimitRedis
  ).apiRateLimit(
    input.burstKey,
    input.dailyKey,
    input.burstCapacity,
    input.burstRefillPerSecond,
    input.dailyLimit,
    input.dailyTtlSeconds,
    input.observeOnly ? 1 : 0
  );
  return { allowed: allowed === 1, scope, burstRemaining, dailyRemaining, retryAfterMs };
}

// Rolling-window counters for the bot anomaly scorer. Each counter is a sorted
// set: member -> observation, score -> time (ms). For event-rate counters the
// member is a unique per-event token (so cardinality == event count); for
// distinct counters the member is the observed value (so cardinality == distinct
// count, and re-observing refreshes its score). For every counter we add the
// observation, drop anything older than its window, cap the set size, refresh the
// key TTL, and read back the cardinality.
//
// All of a request's counters run in one Lua call so anomaly scoring costs a
// single round-trip, and because the state lives in Redis every worker/replica
// shares one view β€” an in-process Map would only ever see 1/N of the traffic
// behind a load balancer, diluting the thresholds by the worker count.
//
// A second counter kind ("distribution") answers a question a sorted set cannot:
// how concentrated is a cohort's traffic across some dimension? It is a hash of
// value -> count inside a fixed time bucket (the bucket is encoded in the key by
// the caller, so the hash simply expires), and it returns the bucket total, the
// largest single value's count, and the number of distinct values. The ratio
// top/total is the modal share, which separates an organic population (one
// dominant browser version) from a fleet rotating a fixed list of user agents.
//
// Two further kinds serve aggregate rules that ask about a whole cohort rather
// than one visitor, where per-member bookkeeping would be far too expensive to
// keep. Both live inside a fixed time bucket encoded in the key by the caller,
// exactly like a distribution, so the key simply expires. "counter" is a plain
// monotonic INCR β€” the cheapest possible answer to "how many events did this
// cohort produce this bucket?" β€” and ignores the member entirely. "cardinality"
// is a HyperLogLog: an approximate distinct count for dimensions such as distinct
// paths or distinct actors, where the true cardinality can reach tens of
// thousands and a sorted set holding every member would be far too large.
//
// numberOfKeys is omitted so the key count can be passed per call. KEYS holds the
// counter keys; ARGV is [nowMs, then (kind, member, windowMs, maxSize) per key].
// Every key yields three results β€” [count, 0, 0] for rolling counters, plain
// counters and cardinality estimates, and [total, top, distinct] for
// distributions β€” so the caller can index by position.
redis.defineCommand("anomalyObserve", {
  lua: `
    local now = tonumber(ARGV[1])
    local results = {}
    for i = 1, #KEYS do
      local key = KEYS[i]
      local base = (i - 1) * 4 + 1
      local kind = ARGV[base + 1]
      local member = ARGV[base + 2]
      local windowMs = tonumber(ARGV[base + 3])
      local maxSize = tonumber(ARGV[base + 4])
      local out = (i - 1) * 3
      if kind == 'h' then
        -- Only admit a new field while under the cap; already-tracked fields keep
        -- counting, so a high-cardinality flood cannot grow the hash without bound.
        if maxSize <= 0 or redis.call('HEXISTS', key, member) == 1 or redis.call('HLEN', key) < maxSize then
          redis.call('HINCRBY', key, member, 1)
        end
        redis.call('PEXPIRE', key, windowMs)
        local total = 0
        local top = 0
        local values = redis.call('HVALS', key)
        for j = 1, #values do
          local value = tonumber(values[j])
          total = total + value
          if value > top then top = value end
        end
        results[out + 1] = total
        results[out + 2] = top
        results[out + 3] = #values
      elseif kind == 'c' then
        -- Plain monotonic counter inside the caller-encoded bucket; member unused.
        results[out + 1] = redis.call('INCR', key)
        redis.call('PEXPIRE', key, windowMs)
        results[out + 2] = 0
        results[out + 3] = 0
      elseif kind == 'p' then
        -- HyperLogLog: an approximate distinct count for dimensions whose true
        -- cardinality can reach tens of thousands (distinct paths, distinct
        -- actors), where a sorted set holding every member would be far too large.
        redis.call('PFADD', key, member)
        redis.call('PEXPIRE', key, windowMs)
        results[out + 1] = redis.call('PFCOUNT', key)
        results[out + 2] = 0
        results[out + 3] = 0
      else
        redis.call('ZADD', key, now, member)
        redis.call('ZREMRANGEBYSCORE', key, '-inf', '(' .. (now - windowMs))
        if maxSize > 0 then
          redis.call('ZREMRANGEBYRANK', key, 0, -maxSize - 1)
        end
        redis.call('PEXPIRE', key, windowMs)
        results[out + 1] = redis.call('ZCARD', key)
        results[out + 2] = 0
        results[out + 3] = 0
      end
    end
    return results
  `,
});

export interface AnomalyCounterSpec {
  /** Fully-namespaced Redis key for this counter. */
  key: string;
  /**
   * How this counter stores its observations:
   * - `rolling` (default) β€” sorted set over a sliding `windowMs`; reports exact
   *   cardinality.
   * - `distribution` β€” hash of value -> count; reports total, top and distinct.
   * - `counter` β€” plain monotonic INCR; reports the incremented value. `member`
   *   and `maxSize` are unused.
   * - `cardinality` β€” HyperLogLog; reports an approximate distinct count.
   *   `maxSize` is unused.
   *
   * `rolling` manages its own window, so its key may be stable. The other three
   * kinds are fixed-bucket: the key must already encode its time bucket, since
   * they only expire rather than trim.
   */
  kind?: "rolling" | "distribution" | "counter" | "cardinality";
  /** Unique per-event token (rate counters) or observed value (distinct/distribution/cardinality counters). */
  member: string;
  /** Rolling window for `rolling`; key TTL for the fixed-bucket kinds. */
  windowMs: number;
  /** Hard cap on stored members/fields; 0 disables trimming. Unused by `counter` and `cardinality`. */
  maxSize: number;
}

/** Bucket total, the largest single value's count, and the distinct value count. */
export interface AnomalyDistribution {
  total: number;
  top: number;
  distinct: number;
}

interface AnomalyRedis extends Redis {
  anomalyObserve(...args: (string | number)[]): Promise<number[]>;
}

// Wire codes the Lua script dispatches on. Written as an exhaustive map rather
// than a ternary so a new kind cannot silently fall through to a rolling counter.
const ANOMALY_KIND_CODES: Record<NonNullable<AnomalyCounterSpec["kind"]>, string> = {
  rolling: "z",
  distribution: "h",
  counter: "c",
  cardinality: "p",
};

/**
 * Observe one event against a batch of counters and return each counter's
 * current reading, in the same order as `specs`. Atomic across all workers; a
 * single round-trip regardless of how many counters are passed. Distributions
 * fill all three fields; every other kind reports its single number in `total`
 * and leaves `top`/`distinct` at zero.
 */
export async function anomalyObserve(nowMs: number, specs: AnomalyCounterSpec[]): Promise<AnomalyDistribution[]> {
  const keys = specs.map(spec => spec.key);
  const args: (string | number)[] = [specs.length, ...keys, nowMs];
  for (const spec of specs) {
    args.push(ANOMALY_KIND_CODES[spec.kind ?? "rolling"], spec.member, spec.windowMs, spec.maxSize);
  }
  const flat = await (redis as AnomalyRedis).anomalyObserve(...args);
  return specs.map((_, index) => ({
    total: flat[index * 3] ?? 0,
    top: flat[index * 3 + 1] ?? 0,
    distinct: flat[index * 3 + 2] ?? 0,
  }));
}

// Data import lock and monthly quota (importQuotaManager), shared so every cluster
// worker enforces one import per organization and one quota. Imports are rare and
// each call is a single small script, so they use the main connection.
redis.defineCommand("importLockRelease", { numberOfKeys: 1, lua: IMPORT_LOCK_RELEASE_LUA });
redis.defineCommand("importQuotaReserve", { numberOfKeys: 1, lua: IMPORT_QUOTA_RESERVE_LUA });

interface ImportRedis extends Redis {
  importLockRelease(key: string, owner: string): Promise<number>;
  importQuotaReserve(key: string, ...args: (string | number)[]): Promise<number[]>;
}

/** Takes the lock for `owner` unless someone holds it. Atomic across all workers. */
export async function importLockAcquire(key: string, owner: string, ttlMs: number): Promise<boolean> {
  return (await redis.set(key, owner, "PX", ttlMs, "NX")) === "OK";
}

/** Releases the lock if `owner` still holds it. */
export async function importLockRelease(key: string, owner: string): Promise<boolean> {
  return (await (redis as ImportRedis).importLockRelease(key, owner)) === 1;
}

export interface ImportQuotaRequest {
  /** yyyyMM */
  month: string;
  /** Events the caller knows are counted for the month; the shared count never drops below it. */
  baseline: number;
  requested: number;
}

/**
 * Reserves up to `requested` events per month against `monthlyLimit` and returns how many
 * were granted, in the same order. Atomic across all workers; one round-trip.
 */
export async function importQuotaReserve(
  key: string,
  monthlyLimit: number,
  ttlMs: number,
  requests: ImportQuotaRequest[]
): Promise<number[]> {
  const args: (string | number)[] = [monthlyLimit, ttlMs];
  for (const request of requests) {
    args.push(request.month, request.baseline, request.requested);
  }
  const granted = await (redis as ImportRedis).importQuotaReserve(key, ...args);
  return requests.map((_, index) => Number(granted[index] ?? 0));
}