Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

20.8 KB
Raw Blame
import { beforeEach, describe, expect, it, mock } from 'bun:test';
import { getRedisCache } from '@openpanel/redis';
import { bufferDepsWithCh } from '../../test/buffer-fixtures';
import { testCoreConfig } from '../../test/config-fixture';
import type { IServiceCreateEventPayload } from '../modules/event/event.service';
import type { IClickhouseSession } from '../modules/session/session.service';
import type { FlushObservation } from './base-buffer';

// The client comes in as `BufferDeps.ch` — no module mock needed for the flush
// path at all. See event-buffer.test.ts's header.
const chInsert = mock(
  async (_options: {
    table: string;
    format: string;
    values: IClickhouseSession[];
  }): Promise<unknown> => undefined
);

const { SessionBuffer } = await import('./session-buffer');
type SessionBuffer = InstanceType<typeof SessionBuffer>;

const redis = getRedisCache();

const projectId = 'project-1';
const deviceId = 'device-1';
const sessionId = 'session-1';

function makePayload(
  overrides: Partial<IServiceCreateEventPayload> = {}
): IServiceCreateEventPayload {
  return {
    name: 'screen_view',
    projectId,
    deviceId,
    sessionId,
    profileId: 'profile-1',
    properties: {},
    groups: [],
    createdAt: new Date(),
    duration: 0,
    sdkName: 'web',
    sdkVersion: '1.0.0',
    city: '',
    country: '',
    region: '',
    longitude: 0,
    latitude: 0,
    path: '/home',
    origin: '',
    referrer: '',
    referrerName: '',
    referrerType: '',
    os: '',
    osVersion: '',
    browser: '',
    browserVersion: '',
    device: '',
    brand: '',
    model: '',
    ...overrides,
  };
}

beforeEach(async () => {
  const keys = [
    ...(await redis.keys('session*')),
    ...(await redis.keys('lock:session')),
  ];
  // Spread breaks at ~100k args — del in chunks so old test state can't kill us.
  const BATCH = 1000;
  for (let i = 0; i < keys.length; i += BATCH) {
    await redis.del(...keys.slice(i, i + BATCH));
  }
  chInsert.mockReset();
  chInsert.mockResolvedValue(undefined);
});

// No `afterAll` on purpose: the shared `getRedisCache()` client must NOT be
// quit, because bun runs every file in one process and closing the singleton
// takes it away from the files that run next.

describe('SessionBuffer', () => {
  let sessionBuffer: SessionBuffer;

  beforeEach(() => {
    sessionBuffer = new SessionBuffer(bufferDepsWithCh({ insert: chInsert }));
  });

  it('opens a new session and stores it at session:{pid}:{did}', async () => {
    const result = await sessionBuffer.ingest(makePayload());

    expect(result?.kind).toBe('new');

    const blob = await redis.get(`session:${projectId}:${deviceId}`);
    expect(blob).not.toBeNull();
    const parsed = JSON.parse(blob ?? '');
    expect(parsed.id).toBe(sessionId);
    expect(parsed.project_id).toBe(projectId);
    expect(parsed.device_id).toBe(deviceId);
  });

  it('getExistingSessions reads several devices in order, null when absent', async () => {
    await sessionBuffer.ingest(makePayload({ deviceId: 'device-A' }));
    await sessionBuffer.ingest(
      makePayload({ deviceId: 'device-C', sessionId: 'session-C' })
    );

    const sessions = await sessionBuffer.getExistingSessions(projectId, [
      'device-A',
      'device-B',
      'device-C',
    ]);

    expect(sessions.map((session) => session?.id ?? null)).toEqual([
      sessionId,
      null,
      'session-C',
    ]);
  });

  it('registers the device in the wallclock sorted set and the projects set', async () => {
    await sessionBuffer.ingest(makePayload());

    expect(
      await redis.zscore(`session:wallclock:${projectId}`, deviceId)
    ).not.toBeNull();
    expect(await redis.sismember('session:projects', projectId)).toBe(1);
  });

  it('writes no TTL on the session blob, but one on the profile pointer', async () => {
    await sessionBuffer.ingest(
      makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
    );

    // The blob must survive arbitrarily long reaper outages. The pointer is
    // in no index, so its TTL is what collects it when cleanup() misses it.
    expect(await redis.ttl(`session:${projectId}:device-Y`)).toBe(-1);
    const pointerTtl = await redis.pttl(
      `session:profile:${projectId}:profile-X`
    );
    expect(pointerTtl).toBeGreaterThan(30 * 60 * 1000);
    expect(pointerTtl).toBeLessThanOrEqual(60 * 60 * 1000);
  });

  it('drops the previous profile pointer when a session is re-identified', async () => {
    await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
    await sessionBuffer.ingest(
      makePayload({
        profileId: 'profile-B',
        createdAt: new Date(Date.now() + 1000),
      })
    );

    expect(
      await redis.get(`session:profile:${projectId}:profile-A`)
    ).toBeNull();
    expect(await redis.get(`session:profile:${projectId}:profile-B`)).toBe(
      deviceId
    );
  });

  it('keeps the profile pointer when an extend repeats the same profile', async () => {
    await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
    await sessionBuffer.ingest(
      makePayload({
        profileId: 'profile-A',
        createdAt: new Date(Date.now() + 1000),
      })
    );

    expect(await redis.get(`session:profile:${projectId}:profile-A`)).toBe(
      deviceId
    );
  });

  it('writes profile→device pointer when profile_id differs from device_id', async () => {
    await sessionBuffer.ingest(
      makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
    );
    expect(await redis.get(`session:profile:${projectId}:profile-X`)).toBe(
      'device-Y'
    );
  });

  it('skips session_start and session_end events', async () => {
    const sizeBefore = await sessionBuffer.getBufferSize();
    expect(
      await sessionBuffer.ingest(makePayload({ name: 'session_start' }))
    ).toBeNull();
    expect(
      await sessionBuffer.ingest(makePayload({ name: 'session_end' }))
    ).toBeNull();
    expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore);
  });

  it('emits sign=-1 + sign=+1 CH rows when extending within timeout', async () => {
    const t0 = new Date();
    await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
    const sizeBefore = await sessionBuffer.getBufferSize();

    const result = await sessionBuffer.ingest(
      makePayload({ createdAt: new Date(t0.getTime() + 5000) })
    );

    expect(result?.kind).toBe('extend');
    expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore + 2);
  });

  it('returns kind=boundary when gap > 30min, with closed + current populated', async () => {
    const t0 = new Date();
    const first = await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
    expect(first?.kind).toBe('new');

    const second = await sessionBuffer.ingest(
      makePayload({
        createdAt: new Date(t0.getTime() + 35 * 60 * 1000),
        sessionId: 'session-2',
      })
    );

    expect(second?.kind).toBe('boundary');
    if (second?.kind === 'boundary') {
      expect(second.closed.id).toBe(sessionId);
      expect(second.current.id).toBe('session-2');
    }
  });

  it('invariant: one visit keeps one id across extends; a >30min gap splits and closes the first', async () => {
    // This ties the layers together against real Redis: ingest() drives the
    // state machine, and getExistingSession() is what the API reads on each
    // request to resolve the id — they must agree throughout a visit.
    const t0 = new Date('2026-06-08T12:00:00.000Z');
    const at = (mins: number) => new Date(t0.getTime() + mins * 60_000);

    const e1 = await sessionBuffer.ingest(
      makePayload({ createdAt: t0, sessionId: 'S1' })
    );
    expect(e1?.kind).toBe('new');
    expect(e1?.current.id).toBe('S1');
    // The API's next lookup must see the live session.
    expect(
      (await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id
    ).toBe('S1');

    // Events within the idle window extend the SAME session.
    for (const mins of [10, 20]) {
      const ext = await sessionBuffer.ingest(
        makePayload({ createdAt: at(mins), sessionId: 'S1' })
      );
      expect(ext?.kind).toBe('extend');
      expect(ext?.current.id).toBe('S1');
    }

    // A >30min gap: ingest detects the boundary, closes S1, opens S2.
    const e4 = await sessionBuffer.ingest(
      makePayload({ createdAt: at(55), sessionId: 'S2' })
    );
    expect(e4?.kind).toBe('boundary');
    if (e4?.kind === 'boundary') {
      expect(e4.closed.id).toBe('S1');
      expect(e4.current.id).toBe('S2');
    }

    // The slot now holds the new session — the API reads S2 from here on.
    expect(
      (await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id
    ).toBe('S2');
  });

  it('inherits utm_* fields from event.properties.__query', async () => {
    await sessionBuffer.ingest(
      makePayload({
        properties: {
          __query: {
            utm_medium: 'cpc',
            utm_source: 'google',
            utm_campaign: 'spring',
          },
        },
      })
    );
    const blob = JSON.parse(
      (await redis.get(`session:${projectId}:${deviceId}`)) ?? ''
    );
    expect(blob.utm_medium).toBe('cpc');
    expect(blob.utm_source).toBe('google');
    expect(blob.utm_campaign).toBe('spring');
  });

  it('out-of-order events: older event extends backwards, never produces negative duration', async () => {
    const t0 = new Date();
    await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
    // Late-arriving event with an older timestamp.
    await sessionBuffer.ingest(
      makePayload({ createdAt: new Date(t0.getTime() - 30_000) })
    );
    const blob = JSON.parse(
      (await redis.get(`session:${projectId}:${deviceId}`)) ?? ''
    );
    expect(blob.duration).toBeGreaterThanOrEqual(0);
  });

  it('cleanup() removes blob, wallclock entry, and profile pointer', async () => {
    await sessionBuffer.ingest(
      makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
    );

    await sessionBuffer.cleanup({
      projectId,
      deviceId: 'device-Y',
      sessionId,
      profileId: 'profile-X',
    });

    expect(await redis.get(`session:${projectId}:device-Y`)).toBeNull();
    expect(
      await redis.zscore(`session:wallclock:${projectId}`, 'device-Y')
    ).toBeNull();
    expect(
      await redis.get(`session:profile:${projectId}:profile-X`)
    ).toBeNull();
  });

  it('cleanup() is a no-op when slot is owned by a different session (boundary)', async () => {
    // Open session S1
    await sessionBuffer.ingest(makePayload({ sessionId: 's1' }));
    // 35min later — boundary, opens S2 at same (pid, did)
    await sessionBuffer.ingest(
      makePayload({
        sessionId: 's2',
        createdAt: new Date(Date.now() + 35 * 60 * 1000),
      })
    );

    // Try to cleanup S1 — should not touch S2's data. The Lua script in
    // cleanup() detects the id mismatch and returns early.
    await sessionBuffer.cleanup({
      projectId,
      deviceId,
      sessionId: 's1',
    });

    const blob = await redis.get(`session:${projectId}:${deviceId}`);
    expect(blob).not.toBeNull();
    const parsed = JSON.parse(blob ?? '');
    expect(parsed.id).toBe('s2');
    expect(
      await redis.zscore(`session:wallclock:${projectId}`, deviceId)
    ).not.toBeNull();
  });

  it('processes buffer and inserts sessions into ClickHouse', async () => {
    await sessionBuffer.ingest(makePayload());

    await sessionBuffer.processBuffer();

    expect(chInsert).toHaveBeenCalledWith(
      expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
    );
    expect(await sessionBuffer.getBufferSize()).toBe(0);
  });

  it('squash does not orphan the creation row when create + updates share a batch', async () => {
    // Regression: a brand-new session whose creation event and subsequent
    // updates all land in one flush window. ingest emits (+1,v1) for the
    // creation then (-1,vN)/(+1,vN+1) pairs for each update. The squash must
    // net per version so nothing is left un-collapsible: a (-1,V) row only
    // collapses against a (+1,V) of the SAME version in CH.
    const t0 = Date.now();
    const EVENTS = 8;
    for (let i = 0; i < EVENTS; i++) {
      await sessionBuffer.ingest(
        makePayload({ createdAt: new Date(t0 + i * 1000) })
      );
    }

    const inserted: Array<{ sign: number; version: number }> = [];
    chInsert.mockImplementation(async ({ values }) => {
      for (const v of values) {
        inserted.push({ sign: v.sign, version: v.version });
      }
      return await Promise.resolve(undefined);
    });

    await sessionBuffer.processBuffer();

    // Squash must actually fire. The 8 events queue 1 creation + 7
    // (-1,vN)/(+1,vN+1) pairs = 15 raw rows; for a fresh session with no
    // CH-resident row they all net down to the single final (+1, v8). If
    // squash is disabled/removed, every raw row is inserted and this is 15
    // — this assertion is the guard that the optimisation is wired in.
    expect(inserted.length).toBe(1);

    // Net the inserted rows per version exactly as CH's
    // VersionedCollapsingMergeTree(sign, version) would: a row survives only
    // if positives and negatives at its version don't cancel.
    const netByVersion = new Map<number, number>();
    for (const { sign, version } of inserted) {
      netByVersion.set(version, (netByVersion.get(version) ?? 0) + sign);
    }
    const survivors = [...netByVersion.entries()].filter(
      ([, net]) => net !== 0
    );

    // A fresh session must collapse to exactly one final +1 row, with no
    // orphaned negatives (the bug left a permanent (-1, v1)).
    expect(survivors).toEqual([[EVENTS, 1]]);
  });

  it('retains sessions in queue when ClickHouse insert fails', async () => {
    await sessionBuffer.ingest(makePayload());

    chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));

    // Errors propagate to tryFlush; the queue is preserved.
    await expect(sessionBuffer.processBuffer()).rejects.toThrow(
      'ClickHouse unavailable'
    );
    expect(await sessionBuffer.getBufferSize()).toBe(1);
  });

  // The write-back runs once per client event, so every extra command on it is
  // multiplied by the whole ingest rate.
  it('ingest sends no TYPE, EVAL or EVALSHA to Redis', async () => {
    const monitor = await redis.duplicate().monitor();
    const commands: string[] = [];
    let sentinelSeen = false;
    // A dev stack may share this Redis, so only commands on this test's
    // project count.
    monitor.on('monitor', (_time: string, args: string[]) => {
      if (args.includes(MONITOR_SENTINEL)) {
        sentinelSeen = true;
      } else if (args.some((arg) => arg.includes(projectId))) {
        commands.push(String(args[0]).toLowerCase());
      }
    });

    const t0 = Date.now();
    await sessionBuffer.ingest(makePayload({ createdAt: new Date(t0) }));
    await sessionBuffer.ingest(makePayload({ createdAt: new Date(t0 + 1000) }));
    await redis.ping(MONITOR_SENTINEL);
    while (!sentinelSeen) {
      await Bun.sleep(MONITOR_POLL_MS);
    }
    monitor.disconnect();

    expect(commands).toContain('rpush');
    expect(commands).not.toContain('type');
    expect(commands).not.toContain('eval');
    expect(commands).not.toContain('evalsha');
  });

  it('ingest never flushes, even past the batch size', async () => {
    const buffer = bufferWithBatchSize(2);
    const t0 = Date.now();
    for (let i = 0; i < 5; i++) {
      await buffer.ingest(makePayload({ createdAt: new Date(t0 + i * 1000) }));
    }

    expect(chInsert).not.toHaveBeenCalled();
    expect(await buffer.getBufferSize()).toBe(9);
  });

  it('a flush drains a backlog larger than one batch', async () => {
    const buffer = bufferWithBatchSize(3);
    const t0 = Date.now();
    for (let i = 0; i < 10; i++) {
      await buffer.ingest(
        makePayload({ deviceId: `device-${i}`, createdAt: new Date(t0) })
      );
    }

    await buffer.tryFlush({ trigger: 'cron' });

    expect(chInsert).toHaveBeenCalledTimes(4);
    expect(await buffer.getBufferSize()).toBe(0);
  });

  it('inserts the chunks of one batch concurrently', async () => {
    const config = testCoreConfig();
    config.buffers.session.chunkSize = 2;
    const buffer = new SessionBuffer(
      bufferDepsWithCh({ insert: chInsert }, config)
    );
    const t0 = Date.now();
    for (let i = 0; i < 10; i++) {
      await buffer.ingest(
        makePayload({
          deviceId: `device-${i}`,
          sessionId: `session-${i}`,
          createdAt: new Date(t0),
        })
      );
    }

    let inFlight = 0;
    let maxInFlight = 0;
    const insertedIds: string[] = [];
    chInsert.mockImplementation(async ({ values }) => {
      inFlight += 1;
      maxInFlight = Math.max(maxInFlight, inFlight);
      await Bun.sleep(INSERT_LATENCY_MS);
      insertedIds.push(...values.map((row) => row.device_id));
      inFlight -= 1;
    });

    await buffer.processBuffer();

    expect(chInsert).toHaveBeenCalledTimes(5);
    expect(maxInFlight).toBeGreaterThan(1);
    expect(insertedIds.sort()).toEqual(
      Array.from({ length: 10 }, (_, i) => `device-${i}`).sort()
    );
    expect(await buffer.getBufferSize()).toBe(0);
  });

  it('keeps the whole batch queued when one concurrent chunk fails', async () => {
    const config = testCoreConfig();
    config.buffers.session.chunkSize = 2;
    const buffer = new SessionBuffer(
      bufferDepsWithCh({ insert: chInsert }, config)
    );
    const t0 = Date.now();
    for (let i = 0; i < 10; i++) {
      await buffer.ingest(
        makePayload({
          deviceId: `device-${i}`,
          sessionId: `session-${i}`,
          createdAt: new Date(t0),
        })
      );
    }

    chInsert.mockImplementation(async ({ values }) => {
      await Bun.sleep(INSERT_LATENCY_MS);
      if (values.some((row) => row.device_id === 'device-5')) {
        throw new Error('ClickHouse unavailable');
      }
    });

    await expect(buffer.processBuffer()).rejects.toThrow(
      'ClickHouse unavailable'
    );
    expect(await buffer.getBufferSize()).toBe(10);
  });

  it('inserts a whole batch in one insert by default', async () => {
    const buffer = new SessionBuffer(bufferDepsWithCh({ insert: chInsert }));
    // More rows than any fixed chunk size short of the batch size would allow
    // in one insert.
    const rows = Array.from({ length: ROWS_OVER_OLD_CHUNK_SIZE }, (_, i) =>
      JSON.stringify({ id: `session-${i}`, sign: 1, version: 1, duration: 0 })
    );
    await redis.rpush('session-buffer', ...rows);

    await buffer.processBuffer();

    expect(chInsert).toHaveBeenCalledTimes(1);
    expect(chInsert.mock.calls[0]?.[0].values).toHaveLength(
      ROWS_OVER_OLD_CHUNK_SIZE
    );
  });

  it('after a failed chunk starts no new chunk and rejects once the running ones finish', async () => {
    const config = testCoreConfig();
    config.buffers.session.chunkSize = 2;
    const buffer = new SessionBuffer(
      bufferDepsWithCh({ insert: chInsert }, config)
    );
    const t0 = Date.now();
    for (let i = 0; i < 20; i++) {
      await buffer.ingest(
        makePayload({
          deviceId: `device-${i}`,
          sessionId: `session-${i}`,
          createdAt: new Date(t0),
        })
      );
    }

    let started = 0;
    let finished = 0;
    chInsert.mockImplementation(async ({ values }) => {
      started += 1;
      if (values.some((row) => row.device_id === 'device-0')) {
        throw new Error('ClickHouse unavailable');
      }
      await Bun.sleep(INSERT_LATENCY_MS);
      finished += 1;
    });

    await expect(buffer.processBuffer()).rejects.toThrow(
      'ClickHouse unavailable'
    );
    // Ten chunks, five at a time: the first wave runs, nothing after it.
    expect(started).toBe(BUFFER_CH_INSERT_CONCURRENCY_DEFAULT);
    expect(finished).toBe(BUFFER_CH_INSERT_CONCURRENCY_DEFAULT - 1);
    expect(await buffer.getBufferSize()).toBe(20);
  });

  it('reports read, transform, insert and trim timings', async () => {
    const buffer = bufferWithBatchSize(1000);
    const observations: FlushObservation[] = [];
    buffer.flushObserver = (observation) => observations.push(observation);
    await buffer.ingest(makePayload());

    await buffer.tryFlush({ trigger: 'cron' });

    expect(observations).toHaveLength(1);
    const [observation] = observations;
    expect(observation?.result).toBe('success');
    expect(observation?.rowsProcessed).toBe(1);
    for (const phase of [
      'lrangeMs',
      'transformMs',
      'chInsertMs',
      'trimMs',
    ] as const) {
      expect(observation?.phases?.[phase]).toBeGreaterThanOrEqual(0);
    }
  });
});

const INSERT_LATENCY_MS = 10;
const ROWS_OVER_OLD_CHUNK_SIZE = 1500;
/** `testCoreConfig()` leaves it unset, so the base buffer's default of 5 applies. */
const BUFFER_CH_INSERT_CONCURRENCY_DEFAULT = 5;

const MONITOR_SENTINEL = 'session-buffer-test-done';
const MONITOR_POLL_MS = 5;

function bufferWithBatchSize(batchSize: number) {
  const config = testCoreConfig();
  config.buffers.session.batchSize = batchSize;
  return new SessionBuffer(bufferDepsWithCh({ insert: chInsert }, config));
}