Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

5.4 KB
Raw Blame
import { createServiceLogger } from "../../lib/logger/logger.js";
import { sessionsService } from "../sessions/sessionsService.js";
import { decideSiteExclusion, type SiteExclusionDecision } from "../sites/siteExclusionDecision.js";
import { usageService } from "../usageService.js";
import { botEventQueue, botObservationQueue } from "./botBlocking/botEventQueue.js";
import { checkBotBlocking } from "./botBlocking/index.js";
import { pageviewQueue } from "./pageviewQueue.js";
import { truncateUrlForStorage } from "./trackingPayload.js";
import type { TrackingRequest } from "./trackingRequest.js";
import { createBasePayload } from "./utils.js";

const logger = createServiceLogger("ingest-event");

type MatchedExclusion = Extract<SiteExclusionDecision, { excluded: true }>;

/**
 * What ingestion did with an event. Every outcome is a success from the
 * caller's point of view β€” the HTTP layer turns these into status codes and
 * bodies, and holds no ingestion logic of its own.
 */
export type IngestOutcome =
  | { status: "excluded"; exclusion: MatchedExclusion }
  | { status: "over_limit" }
  | { status: "no_plan" }
  | { status: "bot" }
  | { status: "tracked"; sessionId: string };

/**
 * Take one resolved Tracking Request through the ingestion pipeline.
 *
 * The order of the stages is the interesting part, and it is the reason this is
 * a module rather than a handler body: each stage can veto, and until they sat
 * together the order was an accident of how the handler had been edited.
 *
 *   1. Site Exclusion Decision β€” first, so traffic the Site owner has disowned
 *      (their own office IP, a staging hostname) leaves no trace at all. It
 *      used to run third, after bot detection, which meant an excluded visitor
 *      still moved every anomaly counter for the Site.
 *   2. Over-limit or no plan β€” before any Redis or ClickHouse work, since the
 *      event is going to be dropped regardless.
 *   3. Bot detection β€” needs a real event; charges a Redis round-trip. It runs
 *      for every Site, whether or not the Site blocks bots; `blockBots` decides
 *      only where a detection is sent, at stage 5.
 *   4. Identity and session β€” the only stage that hashes a fingerprint.
 *   5. Enqueue β€” to `bot_events` if the detection is enforced, otherwise to
 *      `events` plus, when something was detected, `bot_observations`.
 */
export async function ingestEvent(trackingRequest: TrackingRequest): Promise<IngestOutcome> {
  const { site } = trackingRequest;

  // Matched against the URL whole: a query string cut to its column limit
  // first can lose the very parameter an exclusion names.
  const exclusionDecision = await decideSiteExclusion(site, {
    ipAddress: trackingRequest.ipAddress,
    candidateIps: trackingRequest.candidateIps,
    pathname: trackingRequest.payload.pathname,
    querystring: trackingRequest.payload.querystring,
    hostname: trackingRequest.payload.hostname,
    userAgent: trackingRequest.userAgent,
    lookupAsn: trackingRequest.lookupAsn,
  });

  // Everything past the exclusion works on the event as it will be stored.
  const payload = truncateUrlForStorage(trackingRequest.payload);

  if (exclusionDecision.excluded) {
    logger.debug({ siteId: payload.site_id, exclusionReason: exclusionDecision.reason }, "Site exclusion matched");
    return { status: "excluded", exclusion: exclusionDecision };
  }

  if (usageService.isSiteOverLimit(site.siteId)) {
    logger.debug({ siteId: payload.site_id }, "Skipping event - site over monthly limit");
    return { status: "over_limit" };
  }

  if (usageService.isSiteWithoutPlan(site.siteId)) {
    return { status: "no_plan" };
  }

  const botDetectionResult = await checkBotBlocking({
    headers: trackingRequest.headers,
    blockBots: site.blockBots,
    trustedServerSideIngestion: trackingRequest.trustedServerSideIngestion,
    isMobileSite: site.type === "mobile",
    lookupAsn: trackingRequest.lookupAsn,
    payload: {
      siteId: site.siteId,
      userAgent: trackingRequest.userAgent,
      clientBotScore: payload._bs,
      clientBotSignalMask: payload._bsm,
      screenWidth: payload.screenWidth,
      screenHeight: payload.screenHeight,
      language: payload.language,
      hostname: payload.hostname,
      pathname: payload.pathname,
      eventType: payload.type,
      referrer: payload.referrer,
      ipAddress: trackingRequest.ipAddress,
    },
  });

  const eventPayload = await createBasePayload({ ...trackingRequest, payload });

  if (botDetectionResult?.enforced) {
    await botEventQueue.add({
      ...eventPayload,
      ...botDetectionResult.eventProperties,
      sessionId: `bot:${eventPayload.userId}`,
    });

    return { status: "bot" };
  }

  const { sessionId } = await sessionsService.updateSession({
    userId: eventPayload.userId,
    identifiedUserId: eventPayload.identifiedUserId,
    siteId: site.siteId,
  });

  await pageviewQueue.add({
    ...eventPayload,
    sessionId,
  });

  // The Site has bot blocking turned off, so this hit is tracked like any other
  // β€” but it was detected, and the detection is worth keeping. Mirroring it to
  // bot_observations is what lets the Site's owner see what turning blocking on
  // would have removed, which is otherwise unanswerable: before this, a Site
  // that opted out was never evaluated at all.
  if (botDetectionResult) {
    await botObservationQueue.add({
      ...eventPayload,
      ...botDetectionResult.eventProperties,
      sessionId,
    });
  }

  return { status: "tracked", sessionId };
}