Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

20.3 KB
Raw Blame
import { getRedisCache, type Redis } from '@openpanel/redis';
import { DateTime, getSafeJson } from '@openpanel/shared';
import type { CoreConfig } from '../config';
import type { IServiceCreateEventPayload } from '../modules/event/event.service';
import type { IClickhouseSession } from '../modules/session/session.service';
import { TABLE_NAMES } from '../shared/ch-tables';
import { BaseBuffer, type BufferDeps } from './base-buffer';
import { defineRedisScript } from './redis-script';

// 30min of idle in event-time → session ends.
const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;

// The profile pointer is refreshed on every persist, so it only has to outlive
// the idle timeout plus the reaper's lag. It sits in no index, so a TTL is the
// only thing that collects a pointer `cleanup()` never gets handed.
const PROFILE_POINTER_TTL_FACTOR = 2;

/**
 * Idle window for a session (boundary detection + the reaper's default
 * deadman). Overridable through SESSION_TIMEOUT_MS so the E2E harness can
 * shrink it from 30min to a few seconds — the harness and the stack must be
 * given the same value (apps/api/e2e/README.md).
 */
export function resolveSessionTimeoutMs(config: CoreConfig): number {
  return config.session.timeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
}

/** A session row is rewritten on every event, so its group union is capped. */
const MAX_SESSION_GROUPS = 100;

const sessionKey = (projectId: string, deviceId: string) =>
  `session:${projectId}:${deviceId}`;
const profileIndexKey = (projectId: string, profileId: string) =>
  `session:profile:${projectId}:${profileId}`;
const wallclockSetKey = (projectId: string) => `session:wallclock:${projectId}`;
const PROJECTS_SET_KEY = 'session:projects';

// Atomic id-gated cleanup. Only deletes the blob + sorted-set entry +
// profile pointer if the live blob's session id matches the one we're
// closing. Prevents races where a boundary has already overwritten the slot
// with a new session.
const cleanupScript = defineRedisScript(`
local cur = redis.call('GET', KEYS[1])
if cur then
  local ok, parsed = pcall(cjson.decode, cur)
  if ok and parsed and parsed.id ~= ARGV[1] then
    return 0
  end
end
redis.call('DEL', KEYS[1])
redis.call('ZREM', KEYS[2], ARGV[2])
if KEYS[3] ~= '' then
  redis.call('DEL', KEYS[3])
end
return 1
`);

export type SessionIngestResult =
  | { kind: 'new'; current: IClickhouseSession }
  | { kind: 'extend'; current: IClickhouseSession }
  | {
      kind: 'boundary';
      current: IClickhouseSession;
      closed: IClickhouseSession;
    };

function toClickhouseDate(date: Date): string {
  return DateTime.fromJSDate(date)
    .setZone('UTC')
    .toFormat('yyyy-MM-dd HH:mm:ss.SSS');
}

function fromClickhouseDate(s: string): Date {
  return DateTime.fromFormat(s, 'yyyy-MM-dd HH:mm:ss.SSS', {
    zone: 'UTC',
  }).toJSDate();
}

function pickUtm(
  payload: IServiceCreateEventPayload,
  key: 'utm_medium' | 'utm_source' | 'utm_campaign' | 'utm_content' | 'utm_term'
): string {
  const query = (
    payload.properties as { __query?: Record<string, unknown> } | undefined
  )?.__query;
  const v = query?.[key];
  return v ? String(v) : '';
}

// Large enough that a busy session's versions share a batch and squash.
const DEFAULT_BATCH_SIZE = 10_000;

export class SessionBuffer extends BaseBuffer {
  private readonly batchSize =
    this.deps.config.buffers.session.batchSize ?? DEFAULT_BATCH_SIZE;
  // One insert per batch by default: a failed chunk keeps the whole batch
  // queued, so chunks that already landed would be inserted again, and a
  // duplicated sign row corrupts the collapsed session.
  private readonly chunkSize =
    this.deps.config.buffers.session.chunkSize ?? this.batchSize;

  private readonly squashEnabled = this.deps.config.buffers.session.squash;
  private readonly idleTimeoutMs = resolveSessionTimeoutMs(this.deps.config);
  private readonly profilePointerTtlMs =
    this.idleTimeoutMs * PROFILE_POINTER_TTL_FACTOR;

  private readonly redisKey = 'session-buffer';
  private readonly redis: Redis;
  constructor(deps: BufferDeps) {
    super(deps, {
      name: 'session',
      onFlush: async () => {
        await this.processBuffer();
      },
    });
    this.redis = getRedisCache();
  }

  /**
   * Look up the active session for a device, either directly by deviceId or
   * via the profile index (`session:profile:{pid}:{profileId}` → deviceId).
   */
  public async getExistingSession(
    options:
      | { projectId: string; deviceId: string }
      | { projectId: string; profileId: string }
  ): Promise<IClickhouseSession | null> {
    if ('deviceId' in options) {
      const hit = await this.redis.get(
        sessionKey(options.projectId, options.deviceId)
      );
      return hit ? getSafeJson<IClickhouseSession>(hit) : null;
    }
    const deviceId = await this.redis.get(
      profileIndexKey(options.projectId, options.profileId)
    );
    if (!deviceId) {
      return null;
    }
    const hit = await this.redis.get(sessionKey(options.projectId, deviceId));
    return hit ? getSafeJson<IClickhouseSession>(hit) : null;
  }

  /** The live sessions of several devices in one MGET, in the given order. */
  public async getExistingSessions(
    projectId: string,
    deviceIds: string[]
  ): Promise<(IClickhouseSession | null)[]> {
    if (deviceIds.length === 0) {
      return [];
    }
    const hits = await this.redis.mget(
      deviceIds.map((deviceId) => sessionKey(projectId, deviceId))
    );
    return hits.map((hit) =>
      hit ? getSafeJson<IClickhouseSession>(hit) : null
    );
  }

  /**
   * Ingest one event into the session lifecycle. Returns the action taken so
   * the caller can drive session_start / session_end event emission —
   * `session_start` / `session_end` events themselves are skipped here, since
   * they are derived signals, not session activity.
   */
  async ingest(
    payload: IServiceCreateEventPayload
  ): Promise<SessionIngestResult | null> {
    if (!(payload.projectId && payload.deviceId)) {
      return null;
    }
    if (payload.name === 'session_start' || payload.name === 'session_end') {
      return null;
    }

    try {
      const existing = await this.getExistingSession({
        projectId: payload.projectId,
        deviceId: payload.deviceId,
      });

      const eventTimeMs = payload.createdAt.getTime();
      const isBoundary =
        existing &&
        eventTimeMs - fromClickhouseDate(existing.ended_at).getTime() >
          this.idleTimeoutMs;

      if (existing && !isBoundary) {
        const { current, chRows } = this.extendSession(existing, payload);
        await this.persist(current, chRows, existing);
        return { kind: 'extend', current };
      }

      const current = this.newSession(payload);
      await this.persist(current, [current]);

      if (existing && isBoundary) {
        return { kind: 'boundary', current, closed: existing };
      }
      return { kind: 'new', current };
    } catch (error) {
      this.logger.error({ err: error }, 'Failed to ingest session');
      return null;
    }
  }

  /**
   * Remove a closed session from Redis. Atomically gated on `sessionId` via
   * Lua: if a different session now occupies the `(projectId, deviceId)`
   * slot (because a boundary already overwrote it with a new session), the
   * call is a no-op — the new session still needs its blob and sorted-set
   * entry.
   *
   * Blobs have no Redis TTL, so this is the only path that removes them. The
   * daily vacuum cron is a backstop for when it fails (worker crash, network
   * partition). The profile pointer additionally expires on its own: it is
   * indexed nowhere, so a pointer this call is not handed (a boundary that
   * gated the script out, a session re-identified before it closed) would
   * otherwise stay forever.
   */
  async cleanup({
    projectId,
    deviceId,
    sessionId,
    profileId,
  }: {
    projectId: string;
    deviceId: string;
    sessionId: string;
    profileId?: string | null;
  }): Promise<void> {
    const pointerKey =
      profileId && profileId !== deviceId
        ? profileIndexKey(projectId, profileId)
        : '';

    await cleanupScript(
      this.redis,
      [sessionKey(projectId, deviceId), wallclockSetKey(projectId), pointerKey],
      [sessionId, deviceId]
    );
  }

  /**
   * Extend an in-flight session with a new event. Returns the updated
   * session AND the pair of CollapsingMergeTree rows that ClickHouse needs
   * to replace the previous row (sign=-1 cancels, sign=+1 inserts).
   */
  private extendSession(
    existing: IClickhouseSession,
    payload: IServiceCreateEventPayload
  ): {
    current: IClickhouseSession;
    chRows: [IClickhouseSession, IClickhouseSession];
  } {
    const oldRow: IClickhouseSession = { ...existing, sign: -1 };
    const current: IClickhouseSession = {
      ...existing,
      sign: 1,
      version: existing.version + 1,
    };

    // Out-of-order safety: treat the session window as [min(event ts),
    // max(event ts)]. Late-arriving events (offline mobile flush, retries)
    // can't drag ended_at backwards or push duration negative.
    const eventTimeMs = payload.createdAt.getTime();
    const startMs = fromClickhouseDate(current.created_at).getTime();
    const endMs = fromClickhouseDate(current.ended_at).getTime();
    const eventCh = toClickhouseDate(payload.createdAt);

    if (eventTimeMs >= endMs) {
      current.ended_at = eventCh;
      if (payload.path) {
        current.exit_path = payload.path;
      }
      if (payload.origin) {
        current.exit_origin = payload.origin;
      }
    }

    if (eventTimeMs < startMs) {
      current.created_at = eventCh;
      if (payload.path) {
        current.entry_path = payload.path;
      }
      if (payload.origin) {
        current.entry_origin = payload.origin;
      }
    } else {
      if (!current.entry_path && payload.path) {
        current.entry_path = payload.path;
      }
      if (!current.entry_origin && payload.origin) {
        current.entry_origin = payload.origin;
      }
    }

    current.duration =
      fromClickhouseDate(current.ended_at).getTime() -
      fromClickhouseDate(current.created_at).getTime();

    if (payload.name === 'revenue') {
      current.revenue = (current.revenue ?? 0) + (payload.revenue ?? 0);
    }

    if (payload.name === 'screen_view' && payload.path) {
      current.screen_view_count += 1;
    } else {
      current.event_count += 1;
    }

    if (current.screen_view_count > 1) {
      current.is_bounce = false;
    }

    if (payload.profileId && payload.profileId !== payload.deviceId) {
      current.profile_id = payload.profileId;
    }

    if (payload.groups?.length) {
      current.groups = [
        ...new Set([...(current.groups ?? []), ...payload.groups]),
      ].slice(0, MAX_SESSION_GROUPS);
    }

    return { current, chRows: [current, oldRow] };
  }

  /**
   * Open a brand-new session from this event's payload. Called for the first
   * event of a device AND on boundary (gap > 30min closing the old session).
   */
  private newSession(payload: IServiceCreateEventPayload): IClickhouseSession {
    const createdAt = toClickhouseDate(payload.createdAt);
    // For anonymous traffic the SDK doesn't send a profileId — fall back to
    // the deviceId so each anonymous device shows up as a unique visitor in
    // the dashboard. Matches what `createEvent` does for the events table.
    const profileId = payload.profileId || payload.deviceId;
    return {
      id: payload.sessionId,
      project_id: payload.projectId,
      device_id: payload.deviceId,
      profile_id: profileId,
      is_bounce: true,
      created_at: createdAt,
      ended_at: createdAt,
      event_count: payload.name === 'screen_view' ? 0 : 1,
      screen_view_count: payload.name === 'screen_view' ? 1 : 0,
      entry_path: payload.path ?? '',
      entry_origin: payload.origin ?? '',
      exit_path: payload.path ?? '',
      exit_origin: payload.origin ?? '',
      revenue: payload.name === 'revenue' ? (payload.revenue ?? 0) : 0,
      referrer: payload.referrer ?? '',
      referrer_name: payload.referrerName ?? '',
      referrer_type: payload.referrerType ?? '',
      os: payload.os ?? '',
      os_version: payload.osVersion ?? '',
      browser: payload.browser ?? '',
      browser_version: payload.browserVersion ?? '',
      device: payload.device ?? '',
      brand: payload.brand ?? '',
      model: payload.model ?? '',
      country: payload.country ?? '',
      region: payload.region ?? '',
      city: payload.city ?? '',
      longitude: payload.longitude ?? null,
      latitude: payload.latitude ?? null,
      duration: payload.duration ?? 0,
      utm_medium: pickUtm(payload, 'utm_medium'),
      utm_source: pickUtm(payload, 'utm_source'),
      utm_campaign: pickUtm(payload, 'utm_campaign'),
      utm_content: pickUtm(payload, 'utm_content'),
      utm_term: pickUtm(payload, 'utm_term'),
      groups: (payload.groups ?? []).slice(0, MAX_SESSION_GROUPS),
      sign: 1,
      version: 1,
    };
  }

  /**
   * Write-back in one MULTI: session blob + wallclock ZSET + projects SET +
   * profile index + ClickHouse buffer rows. Append only; the cron drains the
   * buffer, so ingest never waits on a ClickHouse insert.
   *
   * No TTL on the blob — it is removed exclusively by `cleanup()` after
   * `session_end` emission, so the reaper can always find a session's state
   * regardless of how long the worker has been down. The reaper never goes
   * through the profile pointer, so that one may expire; the only reader is
   * server-side ingest looking for a live client session to ride along on.
   */
  private async persist(
    current: IClickhouseSession,
    chRows: IClickhouseSession[],
    previous?: IClickhouseSession
  ) {
    const projectId = current.project_id;
    const deviceId = current.device_id;
    const hasProfilePointer =
      current.profile_id && current.profile_id !== current.device_id;
    const previousProfileId = previous?.profile_id;
    const hasStalePointer =
      previousProfileId &&
      previousProfileId !== deviceId &&
      previousProfileId !== current.profile_id;

    const multi = this.redis
      .multi()
      .set(sessionKey(projectId, deviceId), JSON.stringify(current))
      .zadd(wallclockSetKey(projectId), Date.now(), deviceId)
      .sadd(PROJECTS_SET_KEY, projectId);
    if (hasStalePointer) {
      multi.del(profileIndexKey(projectId, previousProfileId));
    }
    if (hasProfilePointer) {
      multi.set(
        profileIndexKey(projectId, current.profile_id),
        deviceId,
        'PX',
        this.profilePointerTtlMs
      );
    }
    multi.rpush(this.redisKey, ...chRows.map((row) => JSON.stringify(row)));

    // Ioredis resolves a MULTI whose commands failed, with the error per entry.
    const results = await multi.exec();
    if (results === null) {
      throw new Error('session write-back transaction was aborted');
    }
    const rejected = results.find(([commandError]) => commandError !== null);
    if (rejected?.[0]) {
      throw rejected[0];
    }
  }

  /**
   * Squash the (sign=-1, sign=+1) rows for each session id down to the
   * minimal set that produces the same collapsed state in ClickHouse, by
   * netting the signs per version. Single-row sessions pass through
   * unchanged.
   *
   * `sessions` uses VersionedCollapsingMergeTree(sign, version): a (-1, V)
   * row only collapses against a (+1, V) row of the SAME version. So the
   * net effect of a batch on a session is computed per-version — sum the
   * signs at each version and emit only the rows needed to realise that
   * net. Versions that fully cancel (net 0) are dropped; the surviving
   * rows are normally the final (+1, maxVersion) plus, for a mid-life
   * session, the (-1, V) that cancels CH's resident row.
   *
   * This per-version netting is required because a session's creation row
   * (+1 at the base version) can land in the SAME batch as its updates.
   * The first update emits a (-1, baseVersion) whose only partner is that
   * in-batch creation (+1, baseVersion). They must cancel each other here;
   * naively keeping "oldest -1 + newest +1" would drop the creation +1
   * while keeping its canceller, leaving a permanently un-collapsible (-1)
   * row in CH (duplicate session rows + negative bounce counts).
   */
  private squashSessionsByVersion(
    sessions: IClickhouseSession[]
  ): IClickhouseSession[] {
    if (sessions.length <= 1) {
      return sessions;
    }

    const grouped = new Map<string, IClickhouseSession[]>();
    for (const s of sessions) {
      const arr = grouped.get(s.id);
      if (arr) {
        arr.push(s);
      } else {
        grouped.set(s.id, [s]);
      }
    }

    let squashedCount = 0;
    const out: IClickhouseSession[] = [];

    for (const entries of grouped.values()) {
      if (entries.length === 1) {
        out.push(entries[0]!);
        continue;
      }

      const netByVersion = new Map<
        number,
        { net: number; pos?: IClickhouseSession; neg?: IClickhouseSession }
      >();
      for (const e of entries) {
        const slot = netByVersion.get(e.version) ?? { net: 0 };
        slot.net += e.sign;
        if (e.sign === 1) {
          slot.pos = e;
        } else if (e.sign === -1) {
          slot.neg = e;
        }
        netByVersion.set(e.version, slot);
      }

      const emitted: IClickhouseSession[] = [];
      for (const slot of netByVersion.values()) {
        if (slot.net === 0) {
          continue;
        }
        const row = slot.net > 0 ? slot.pos : slot.neg;
        if (!row) {
          continue;
        }
        for (let i = 0; i < Math.abs(slot.net); i++) {
          emitted.push(row);
        }
      }

      squashedCount += entries.length - emitted.length;
      out.push(...emitted);
    }

    if (squashedCount > 0) {
      this.logger.debug(
        {
          inputRows: sessions.length,
          outputRows: out.length,
          dropped: squashedCount,
          uniqueSessions: grouped.size,
        },
        'Session batch squashed'
      );
    }

    return out;
  }

  async processBuffer() {
    await this.drainBatches(this.batchSize, () => this.processBatch());
  }

  /** Inserts and trims one batch from the head of the list; returns its size. */
  private async processBatch(): Promise<number> {
    const lrangeStart = performance.now();
    const events = await this.redis.lrange(
      this.redisKey,
      0,
      this.batchSize - 1
    );
    const lrangeMs = performance.now() - lrangeStart;
    if (events.length === 0) {
      this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
      return 0;
    }

    const transformStart = performance.now();
    const parsed: IClickhouseSession[] = [];
    for (const e of events) {
      const s = getSafeJson<IClickhouseSession>(e);
      if (!s) {
        continue;
      }
      // Freshly parsed object — safe to clamp in place, no clone needed.
      s.duration = Math.max(0, s.duration || 0);
      parsed.push(s);
    }

    // A busy session appends a (-1, +1) pair per extend within one flush.
    // VersionedCollapsingMergeTree would collapse them at merge time anyway,
    // but inserting them all costs network, gzip CPU and merge work. Squash
    // per version to insert only the boundary rows.
    //
    // Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.
    const sessions = this.squashEnabled
      ? this.squashSessionsByVersion(parsed)
      : parsed;
    const transformMs = performance.now() - transformStart;

    // Chunks may land in any order: VersionedCollapsingMergeTree collapses by
    // (sign, version), not by insert order.
    const ch = this.resolveCh();
    const chStart = performance.now();
    await this.parallelLimit(this.chunks(sessions, this.chunkSize), (chunk) =>
      ch.insert({
        table: TABLE_NAMES.sessions,
        values: chunk,
        format: 'JSONEachRow',
      })
    );
    const chInsertMs = performance.now() - chStart;

    // Trim only after a successful insert — on failure the rows stay queued.
    // Don't swallow: let it propagate so the base flush records result:'error'
    // (and the next flush retries the still-queued rows).
    const trimStart = performance.now();
    await this.redis.ltrim(this.redisKey, events.length, -1);
    const trimMs = performance.now() - trimStart;

    this.reportFlushStats({
      rowsProcessed: events.length,
      phases: { lrangeMs, transformMs, chInsertMs, trimMs },
    });
    return events.length;
  }

  protected getRedisListKey(): string {
    return this.redisKey;
  }
}