OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
import { toStatement } from '@openpanel/db/src/clickhouse/sql';
import { getRedisCache } from '@openpanel/redis';
import { bufferDepsWithCh } from '../../test/buffer-fixtures';
import { testCoreConfig } from '../../test/config-fixture';
import type { IClickhouseProfile } from '../modules/profile/profile.service';
// The client comes in as `BufferDeps.ch` and reads go through core's own
// `chQuery` — see event-buffer.test.ts's header.
const realChQuery = { ...(await import('../ch-query')) };
const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);
const chQuery = mock(
async (_sql: unknown): Promise<IClickhouseProfile[]> => []
);
mock.module('../ch-query', () => ({
...realChQuery,
chQuery: (_scope: unknown, sql: unknown) => chQuery(sql),
}));
const { ProfileBuffer } = await import('./profile-buffer');
type ProfileBuffer = InstanceType<typeof ProfileBuffer>;
const redis = getRedisCache();
function makeProfile(
overrides: Partial<IClickhouseProfile>
): IClickhouseProfile {
const now = new Date().toISOString();
return {
id: 'profile-1',
project_id: 'project-1',
first_name: '',
last_name: '',
email: '',
avatar: '',
properties: {},
is_external: true,
created_at: now,
last_seen_at: now,
groups: [],
...overrides,
};
}
beforeEach(async () => {
const keys = [
...(await redis.keys('profile*')),
...(await redis.keys('lock:profile')),
];
if (keys.length > 0) {
await redis.del(...keys);
}
chInsert.mockClear();
chQuery.mockReset();
chQuery.mockResolvedValue([]);
});
// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs
// every file in one process, and closing the singleton takes it away from the
// files that run next.
afterAll(() => {
mock.module('../ch-query', () => realChQuery);
});
describe('ProfileBuffer', () => {
let profileBuffer: ProfileBuffer;
beforeEach(() => {
profileBuffer = new ProfileBuffer(bufferDepsWithCh({ insert: chInsert }));
});
it('adds a profile to the buffer', async () => {
const profile = makeProfile({
first_name: 'John',
email: '[email protected]',
});
const sizeBefore = await profileBuffer.getBufferSize();
await profileBuffer.add(profile);
const sizeAfter = await profileBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('add never flushes; a cron flush drains past one batch', async () => {
const config = testCoreConfig();
config.buffers.profile.batchSize = 2;
const buffer = new ProfileBuffer(
bufferDepsWithCh({ insert: chInsert }, config)
);
for (let i = 0; i < 5; i++) {
await buffer.add(makeProfile({ id: `profile-${i}` }));
}
expect(chInsert).not.toHaveBeenCalled();
await buffer.tryFlush({ trigger: 'cron' });
expect(chInsert).toHaveBeenCalledTimes(3);
expect(await buffer.getBufferSize()).toBe(0);
});
it('concurrent adds: both raw profiles are queued', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: '[email protected]',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
const sizeBefore = await profileBuffer.getBufferSize();
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const sizeAfter = await profileBuffer.getBufferSize();
// Both raw profiles are queued; merge happens at flush time
expect(sizeAfter).toBe(sizeBefore + 2);
});
it('merges sequential updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: '[email protected]',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('[email protected]');
expect(cached?.groups).toContain('group-abc');
});
it('merges concurrent updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: '[email protected]',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('[email protected]');
expect(cached?.groups).toContain('group-abc');
});
it('uses existing ClickHouse data for cache misses when merging', async () => {
const existingInClickhouse = makeProfile({
first_name: 'Jane',
email: '[email protected]',
groups: ['existing-group'],
});
chQuery.mockResolvedValue([existingInClickhouse]);
const incomingProfile = makeProfile({
first_name: '',
email: '',
groups: ['new-group'],
});
await profileBuffer.add(incomingProfile);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('Jane');
expect(cached?.email).toBe('[email protected]');
expect(cached?.groups).toContain('existing-group');
expect(cached?.groups).toContain('new-group');
});
it('finds an anonymous profile last seen days ago and keeps its created_at', async () => {
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)
.toISOString()
.replace('T', ' ')
.replace('Z', '');
const existingInClickhouse = makeProfile({
id: 'device-uuid',
is_external: false,
created_at: weekAgo,
last_seen_at: weekAgo,
properties: { os: 'iOS', first_referrer: 'google' },
});
chQuery.mockResolvedValue([existingInClickhouse]);
await profileBuffer.add(
makeProfile({
id: 'device-uuid',
is_external: false,
properties: { os: 'iOS' },
})
);
await profileBuffer.processBuffer();
// A recency window on the lookup would hide this profile and make it new.
const lookup = toStatement(chQuery.mock.calls[0]?.[0] as never).query;
expect(lookup).not.toContain('INTERVAL');
const cached = await profileBuffer.fetchFromCache(
'device-uuid',
'project-1'
);
expect(cached?.created_at).toBe(weekAgo);
expect(cached?.properties.first_referrer).toBe('google');
});
it('buffer is empty after flush', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
expect(await profileBuffer.getBufferSize()).toBe(1);
await profileBuffer.processBuffer();
expect(await profileBuffer.getBufferSize()).toBe(0);
});
it('retains profiles in queue when ClickHouse insert fails', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
// Errors propagate to tryFlush; the queue is preserved on CH failure.
await expect(profileBuffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
expect(await profileBuffer.getBufferSize()).toBe(1);
});
it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
chQuery.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await profileBuffer.add(makeProfile({ first_name: 'John' }));
await profileBuffer.processBuffer();
// Insert must still have been called — no data loss even when fetch fails
expect(chInsert).toHaveBeenCalled();
expect(await profileBuffer.getBufferSize()).toBe(0);
});
});