Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

16.5 KB
Raw Blame
import { sql } from '@openpanel/db/src/clickhouse/sql';
import { getRedisCache, publishEvent } from '@openpanel/redis';
import type { IClickhouseEvent } from '../modules/event/event.service';
import { BaseBuffer, type BufferDeps } from './base-buffer';

const PROJECT_ID_NEEDLE = '"project_id":"';

/**
 * Extract the top-level `project_id` from a JSONEachRow event line without a
 * full JSON.parse. Returns `null` if absent or malformed.
 *
 * Fast path: `indexOf` on the needle. A nested `project_id` inside a string
 * value is escaped (`\"project_id\"`), so it cannot match; exactly one hit is
 * the top-level field. Two or more hits mean a real nested key (a user
 * property of that name), so fall back to JSON.parse rather than misattribute.
 */
export function extractProjectId(line: string): string | null {
  const first = line.indexOf(PROJECT_ID_NEEDLE);
  if (first < 0) {
    return null;
  }

  const second = line.indexOf(
    PROJECT_ID_NEEDLE,
    first + PROJECT_ID_NEEDLE.length
  );
  if (second >= 0) {
    try {
      const obj = JSON.parse(line) as { project_id?: unknown };
      return typeof obj.project_id === 'string' ? obj.project_id : null;
    } catch {
      return null;
    }
  }

  const valueStart = first + PROJECT_ID_NEEDLE.length;
  const valueEnd = line.indexOf('"', valueStart);
  // An empty value is treated as missing so it does not pollute pub/sub
  // counts with an empty key.
  if (valueEnd <= valueStart) {
    return null;
  }
  return line.slice(valueStart, valueEnd);
}

const DEFAULT_BATCH_SIZE = 4000;
const DEFAULT_CHUNK_SIZE = 1000;
const DEFAULT_MICRO_BATCH_MS = 10;
const DEFAULT_MICRO_BATCH_SIZE = 100;

export class EventBuffer extends BaseBuffer {
  private readonly batchSize =
    this.deps.config.buffers.event.batchSize ?? DEFAULT_BATCH_SIZE;
  private readonly chunkSize =
    this.deps.config.buffers.event.chunkSize ?? DEFAULT_CHUNK_SIZE;

  private readonly microBatchIntervalMs =
    this.deps.config.buffers.event.microBatchMs ?? DEFAULT_MICRO_BATCH_MS;
  private readonly microBatchMaxSize =
    this.deps.config.buffers.event.microBatchSize ?? DEFAULT_MICRO_BATCH_SIZE;

  private pendingEvents: IClickhouseEvent[] = [];
  private flushTimer: ReturnType<typeof setTimeout> | null = null;

  /**
   * The durability watermark. Every event gets a monotonically increasing
   * sequence number in `add`; `lastDurableSeq` is the highest sequence a
   * COMPLETED rpush covered. A caller's events are in Redis once
   * `lastDurableSeq` reaches the sequence of its last event — whoever wrote
   * them. That is the question the Kafka batch handler and shutdown need
   * answered ("are my events in Redis?"), and it is not the same question as
   * "did I start the write?".
   */
  private lastQueuedSeq = 0;
  private lastDurableSeq = 0;

  /**
   * The rpush in flight and the highest sequence it will make durable. A write
   * always takes the whole pending array, so a caller whose target sequence is
   * at or below `coversSeq` is answered by this write and must not start
   * another.
   */
  private inFlightWrite: {
    coversSeq: number;
    done: Promise<unknown>;
  } | null = null;
  /** Tracks consecutive flush failures for observability; reset on success. */
  private flushRetryCount = 0;

  /**
   * Events whose producer re-produces them when this buffer reports a failed
   * flush. They are DROPPED by a failed write instead of being re-queued —
   * see `addRedeliverable`.
   */
  private readonly redeliverableEvents = new WeakSet<IClickhouseEvent>();

  /**
   * The highest sequence a failed write dropped, and the failure it dropped
   * it with. `lastDurableSeq` answers "is everything up to here in Redis?",
   * which stopped being the same question once a write could drop events: a
   * later write can carry the watermark past a sequence that was dropped
   * rather than written. A durability window (`openDurabilityWindow`) compares
   * against this so the producer that owns the dropped events is the one told
   * about it.
   */
  private droppedThroughSeq = 0;
  private lastDropFailure: unknown = null;

  /** Built once, on first use; see `asRedeliverable`. */
  private redeliverableView: EventBuffer | null = null;

  private queueKey = 'event_buffer:queue';

  constructor(deps: BufferDeps) {
    super(deps, {
      name: 'event',
      onFlush: async () => {
        await this.processBuffer();
      },
    });
  }

  bulkAdd(events: IClickhouseEvent[]) {
    for (const event of events) {
      this.add(event);
    }
  }

  add(event: IClickhouseEvent) {
    const start = performance.now();
    this.lastQueuedSeq += 1;
    this.pendingEvents.push(event);

    if (this.pendingEvents.length >= this.microBatchMaxSize) {
      this.flushLocalBuffer();
    } else if (!this.flushTimer) {
      this.flushTimer = setTimeout(() => {
        this.flushTimer = null;
        this.flushLocalBuffer();
      }, this.microBatchIntervalMs);
    }

    try {
      this.addObserver?.({
        buffer: this.name,
        durationMs: performance.now() - start,
      });
    } catch {
      // never break add on observer failure
    }
  }

  /** Number of events buffered locally in process memory, before Redis. */
  public getPendingLocalCount(): number {
    return this.pendingEvents.length;
  }

  public async flush() {
    try {
      await this.flushPendingOrThrow();
    } catch {
      // The fire-and-forget path swallows a Redis failure on purpose: the
      // events are re-queued and the micro-batch timer tries again, which is
      // the right answer while the process lives.
    }
  }

  /**
   * The same flush, but it THROWS when the rpush did not land, and it does not
   * resolve until every event accepted before the call is in Redis.
   *
   * `flush` swallows a Redis failure on purpose. The two callers here have no
   * next attempt: the Kafka batch handler is about to resolve offsets that
   * kafkajs commits as soon as it returns, and shutdown is about to exit. Both
   * must be able to SEE the failure and decline to commit — a redelivered
   * duplicate is recoverable, a dropped `pendingEvents` array is not.
   */
  public async flushPendingOrThrow(
    sinceSeq: number = this.droppedThroughSeq
  ): Promise<void> {
    const target = this.lastQueuedSeq;

    // Not a retry loop: each pass either waits out a write that is already
    // running or starts the one write still missing, and a write covers
    // everything pending when it starts. So a concurrent write that happens to
    // carry our events resolves us — losing the race to start a write is a
    // SUCCESS, which is the whole correctness point.
    while (this.lastDurableSeq < target) {
      const inFlight = this.inFlightWrite;

      if (inFlight === null) {
        if (this.pendingEvents.length === 0) {
          // Nothing pending and nothing in flight: every sequence queued is
          // SETTLED — written, or dropped for the producer that owns it to
          // produce again. Settled is not durable, which the check below is
          // what separates.
          break;
        }
        this.throwIfFailed(await this.flushLocalBuffer());
        continue;
      }

      const failure = await inFlight.done;
      // A write that did not carry our events is not our failure: its events
      // are re-queued at the front and the write we start next takes them too.
      if (inFlight.coversSeq >= target) {
        this.throwIfFailed(failure);
      }
    }

    // A write that dropped events did not make them durable, and the watermark
    // alone cannot say so: it moves on with the NEXT write, which carries none
    // of them. Without this a batch whose events were dropped by a micro-batch
    // timer it never awaited would be told they are safe, commit its offsets,
    // and lose them for good.
    if (this.droppedThroughSeq > sinceSeq) {
      this.throwIfFailed(this.lastDropFailure);
    }
  }

  /**
   * Buffer an event whose producer produces it AGAIN when this buffer reports a
   * failed flush — today the Kafka consumer, whose batch is left uncommitted
   * and redelivered (`modules/ingest/src/consumer.ts`).
   *
   * Such an event is not re-queued by a failed write. Re-queueing it was right
   * while the buffer owned the retry, and became harmful the moment Kafka took
   * that ownership over: the redelivery buffers a fresh copy, so the old one is
   * a SECOND copy of one event, and every further redelivery adds another
   * (a real run once saw 4,759 ClickHouse rows for 230 events this way).
   *
   * Producers Kafka does not redeliver — the session-end job, whose Redis `SET
   * NX` claim makes a job retry a no-op — keep the safety net by using plain
   * `add`.
   */
  public addRedeliverable(event: IClickhouseEvent): void {
    this.redeliverableEvents.add(event);
    this.add(event);
  }

  /**
   * A view of this buffer whose `add()` is `addRedeliverable()`; everything
   * else on it is this buffer's own.
   *
   * The ownership has to ride in on the SCOPE because `createEvent()` reaches
   * the buffer as `deps.buffers.event` and takes no ownership argument — so
   * the consumer hands its handlers a scope carrying this view
   * (`modules/ingest/src/consumer-handler.ts`) and every other transport keeps
   * the buffer itself.
   */
  public asRedeliverable(): EventBuffer {
    this.redeliverableView ??= new Proxy(this, {
      get: (target, property) =>
        property === 'add'
          ? (event: IClickhouseEvent) => target.addRedeliverable(event)
          : Reflect.get(target, property, target),
    });
    return this.redeliverableView;
  }

  /**
   * Opens a durability window and returns the gate that closes it: a promise
   * that resolves once everything buffered after this call is in Redis, and
   * rejects when any of it was dropped instead.
   *
   * The LOWER bound is the point. A caller that asks "is everything queued in
   * Redis?" cannot tell its own dropped events from another producer's, and a
   * failed write can drop events the caller never awaited. Opening the window
   * before the first event is buffered is what makes the answer that caller's
   * own.
   */
  public openDurabilityWindow(): () => Promise<void> {
    const sinceSeq = this.lastQueuedSeq;
    return () => this.flushPendingOrThrow(sinceSeq);
  }

  private throwIfFailed(failure: unknown): void {
    if (failure === null) {
      return;
    }
    throw failure instanceof Error ? failure : new Error(String(failure));
  }

  /**
   * Starts a write unless one is already running, and returns the write that
   * makes everything currently pending durable.
   *
   * Never rejects — `add()` calls it without awaiting. The failure comes back
   * as the resolved value instead, for the callers that must act on it.
   */
  private flushLocalBuffer(): Promise<unknown> {
    if (this.inFlightWrite) {
      return this.inFlightWrite.done;
    }
    if (this.pendingEvents.length === 0) {
      return Promise.resolve(null);
    }

    // The write takes everything the timer would have taken.
    if (this.flushTimer) {
      clearTimeout(this.flushTimer);
      this.flushTimer = null;
    }

    const eventsToFlush = this.pendingEvents;
    this.pendingEvents = [];
    // The whole pending array goes in one write, and `add()` assigns sequences
    // in order, so this write covers every sequence assigned so far.
    const coversSeq = this.lastQueuedSeq;

    const done = this.writeToRedis(eventsToFlush, coversSeq).then((failure) => {
      this.inFlightWrite = null;
      this.scheduleFlushIfPending();
      return failure;
    });
    this.inFlightWrite = { coversSeq, done };
    return done;
  }

  /** @returns the failure the rpush ended with, or `null` on success. */
  private async writeToRedis(
    eventsToFlush: IClickhouseEvent[],
    coversSeq: number
  ): Promise<unknown> {
    try {
      // One RPUSH for the whole batch: atomic on its own, and it rejects on
      // WRONGTYPE or out-of-memory, which the batch handler relies on before
      // resolving Kafka offsets.
      await getRedisCache().rpush(
        this.queueKey,
        ...eventsToFlush.map((event) => JSON.stringify(event))
      );

      // The durability boundary: past this point the events survive any
      // process death, so everyone waiting on a sequence in this write is done.
      this.lastDurableSeq = Math.max(this.lastDurableSeq, coversSeq);
      this.flushRetryCount = 0;
      return null;
    } catch (error) {
      // Re-queue only the events nothing else will produce again, at the front
      // to preserve order. A redeliverable event is already on its way back —
      // keeping it here would hold a second copy of it until Redis returns,
      // and one more for every redelivery in between (see `addRedeliverable`).
      const orphaned = eventsToFlush.filter(
        (event) => !this.redeliverableEvents.has(event)
      );
      const droppedCount = eventsToFlush.length - orphaned.length;
      if (droppedCount > 0) {
        // `coversSeq` over-states which sequences were dropped when the write
        // mixed the two kinds. Erring high costs an extra redelivery for a
        // window that straddled the failure; erring low would cost the events.
        this.droppedThroughSeq = Math.max(this.droppedThroughSeq, coversSeq);
        this.lastDropFailure = error;
      }
      this.pendingEvents = orphaned.concat(this.pendingEvents);

      this.flushRetryCount += 1;
      this.logger.warn(
        {
          err: error,
          eventCount: eventsToFlush.length,
          requeuedCount: orphaned.length,
          droppedForRedeliveryCount: droppedCount,
          flushRetryCount: this.flushRetryCount,
        },
        'Failed to flush local buffer to Redis; events re-queued'
      );
      return error;
    }
  }

  /** Events may arrive while a write is running; they get the next micro-batch. */
  private scheduleFlushIfPending(): void {
    if (this.pendingEvents.length === 0 || this.flushTimer) {
      return;
    }
    this.flushTimer = setTimeout(() => {
      this.flushTimer = null;
      this.flushLocalBuffer();
    }, this.microBatchIntervalMs);
  }

  protected getRedisListKey(): string {
    return this.queueKey;
  }

  async processBuffer() {
    await this.drainBatches(this.batchSize, () => this.processBatch());
  }

  /** Inserts and trims one batch from the head of the queue; returns its size. */
  private async processBatch(): Promise<number> {
    const redis = getRedisCache();

    const lrangeStart = performance.now();
    const queueEvents = await redis.lrange(
      this.queueKey,
      0,
      this.batchSize - 1
    );
    const lrangeMs = performance.now() - lrangeStart;

    if (queueEvents.length === 0) {
      this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
      return 0;
    }

    // The queued lines are already JSONEachRow, and the client's custom
    // `json.stringify` (CLICKHOUSE_OPTIONS) passes strings through unchanged,
    // so the bytes go from Redis to the CH HTTP body without a parse.
    const countByProject = new Map<string, number>();
    const yieldEvery = this.getYieldInterval(queueEvents.length, {
      min: 1000,
      max: 5000,
    });
    for (let i = 0; i < queueEvents.length; i++) {
      const projectId = extractProjectId(queueEvents[i]!);
      if (projectId) {
        countByProject.set(projectId, (countByProject.get(projectId) ?? 0) + 1);
      }
      if ((i + 1) % yieldEvery === 0) {
        await this.yieldToEventLoop();
      }
    }

    const ch = this.resolveCh();
    const chStart = performance.now();
    await this.parallelLimit(
      this.chunks(queueEvents, this.chunkSize),
      (chunk) =>
        ch.insert({
          table: 'events',
          values: this.jsonEachRowStream(chunk),
          format: 'JSONEachRow',
          clickhouse_settings: this.getClickhouseSettings(),
        })
    );
    const chInsertMs = performance.now() - chStart;

    for (const [projectId, count] of countByProject) {
      publishEvent('events', 'batch', { projectId, count });
    }

    const trimStart = performance.now();
    await redis.ltrim(this.queueKey, queueEvents.length, -1);
    const trimMs = performance.now() - trimStart;

    this.reportFlushStats({
      rowsProcessed: queueEvents.length,
      phases: { lrangeMs, chInsertMs, trimMs },
    });
    return queueEvents.length;
  }

  public async getActiveVisitorCount(projectId: string): Promise<number> {
    const rows = await this.chQuery<{ count: number }>(sql`
      SELECT uniq(profile_id) AS count
      FROM events
      WHERE project_id = ${sql.string(projectId)}
        AND profile_id != ''
        AND created_at >= now() - INTERVAL 5 MINUTE
    `);
    return rows[0]?.count ?? 0;
  }
}