Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

3.0 KB
Raw Blame
// Batches rows per table and inserts them as JSONEachRow through the shared
// ClickHouse client, so the seed lands in whatever database CLICKHOUSE_URL names.
// Adds are synchronous (the visitor pool evicts from inside a sync path); the
// generator awaits `flushFull()` between sessions.

import { ch, TABLE_NAMES } from '@openpanel/db';
import type {
  ClickhouseAiEventRow,
  ClickhouseEventRow,
  ClickhouseProfileRow,
  ClickhouseSessionRow,
} from './clickhouse-rows';
import { INSERT_BATCH_ROWS } from './seed.constants';

export interface SinkCounts {
  events: number;
  aiEvents: number;
  sessions: number;
  profiles: number;
}

class TableBuffer<Row> {
  private rows: Row[] = [];
  count = 0;

  constructor(private readonly table: string) {}

  add(rows: readonly Row[]): void {
    for (const row of rows) {
      this.rows.push(row);
    }
    this.count += rows.length;
  }

  get full(): boolean {
    return this.rows.length >= INSERT_BATCH_ROWS;
  }

  async flush(): Promise<void> {
    if (this.rows.length === 0) {
      return;
    }
    const batch = this.rows;
    this.rows = [];
    await ch.insert({
      table: this.table,
      values: batch,
      format: 'JSONEachRow',
    });
  }
}

/** What the generator needs from a sink; tests substitute an in-memory one. */
export interface RowSink {
  addEvents(rows: readonly ClickhouseEventRow[]): void;
  addAiEvents(rows: readonly ClickhouseAiEventRow[]): void;
  addSession(row: ClickhouseSessionRow): void;
  addProfiles(rows: readonly ClickhouseProfileRow[]): void;
  flushFull(): Promise<void>;
  flush(): Promise<void>;
  counts(): SinkCounts;
}

export class ClickhouseSink implements RowSink {
  private readonly events = new TableBuffer<ClickhouseEventRow>(
    TABLE_NAMES.events
  );
  private readonly aiEvents = new TableBuffer<ClickhouseAiEventRow>(
    TABLE_NAMES.ai_events
  );
  private readonly sessions = new TableBuffer<ClickhouseSessionRow>(
    TABLE_NAMES.sessions
  );
  private readonly profiles = new TableBuffer<ClickhouseProfileRow>(
    TABLE_NAMES.profiles
  );

  private get buffers(): TableBuffer<unknown>[] {
    return [this.events, this.aiEvents, this.sessions, this.profiles];
  }

  addEvents(rows: readonly ClickhouseEventRow[]): void {
    this.events.add(rows);
  }

  addAiEvents(rows: readonly ClickhouseAiEventRow[]): void {
    this.aiEvents.add(rows);
  }

  addSession(row: ClickhouseSessionRow): void {
    this.sessions.add([row]);
  }

  addProfiles(rows: readonly ClickhouseProfileRow[]): void {
    this.profiles.add(rows);
  }

  async flushFull(): Promise<void> {
    for (const buffer of this.buffers) {
      if (buffer.full) {
        await buffer.flush();
      }
    }
  }

  async flush(): Promise<void> {
    for (const buffer of this.buffers) {
      await buffer.flush();
    }
  }

  counts(): SinkCounts {
    return {
      events: this.events.count,
      aiEvents: this.aiEvents.count,
      sessions: this.sessions.count,
      profiles: this.profiles.count,
    };
  }
}