OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(ingest): returning anonymous visitors keep their profile (#590)
Carl-Gerhard Lindesvärdcc84b2741bfb
diff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.tsindex 6784083e3..bf46b9952 100644--- a/packages/core/src/buffers/profile-buffer.test.ts+++ b/packages/core/src/buffers/profile-buffer.test.ts@@ -1,4 +1,5 @@import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';+import { toStatement } from '@openpanel/db/src/clickhouse/sql';import { getRedisCache } from '@openpanel/redis';import { bufferDepsWithCh } from '../../test/buffer-fixtures';import { testCoreConfig } from '../../test/config-fixture';@@ -9,11 +10,13 @@ import type { IClickhouseProfile } from '../modules/profile/profile.service';const realChQuery = { ...(await import('../ch-query')) };const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);-const chQuery = mock(async (_sql: string): Promise<IClickhouseProfile[]> => []);+const chQuery = mock(+ async (_sql: unknown): Promise<IClickhouseProfile[]> => []+);mock.module('../ch-query', () => ({...realChQuery,- chQuery: (_scope: unknown, sql: string) => chQuery(sql),+ chQuery: (_scope: unknown, sql: unknown) => chQuery(sql),}));const { ProfileBuffer } = await import('./profile-buffer');@@ -192,6 +195,40 @@ describe('ProfileBuffer', () => {expect(cached?.groups).toContain('new-group');});+ it('finds an anonymous profile last seen days ago and keeps its created_at', async () => {+ const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)+ .toISOString()+ .replace('T', ' ')+ .replace('Z', '');+ const existingInClickhouse = makeProfile({+ id: 'device-uuid',+ is_external: false,+ created_at: weekAgo,+ last_seen_at: weekAgo,+ properties: { os: 'iOS', first_referrer: 'google' },+ });+ chQuery.mockResolvedValue([existingInClickhouse]);++ await profileBuffer.add(+ makeProfile({+ id: 'device-uuid',+ is_external: false,+ properties: { os: 'iOS' },+ })+ );+ await profileBuffer.processBuffer();++ // A recency window on the lookup would hide this profile and make it new.+ const lookup = toStatement(chQuery.mock.calls[0]?.[0] as never).query;+ expect(lookup).not.toContain('INTERVAL');+ const cached = await profileBuffer.fetchFromCache(+ 'device-uuid',+ 'project-1'+ );+ expect(cached?.created_at).toBe(weekAgo);+ expect(cached?.properties.first_referrer).toBe('google');+ });+it('buffer is empty after flush', async () => {await profileBuffer.add(makeProfile({ first_name: 'John' }));expect(await profileBuffer.getBufferSize()).toBe(1);diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.tsindex 3660cf2a3..0b4fb379e 100644--- a/packages/core/src/buffers/profile-buffer.ts+++ b/packages/core/src/buffers/profile-buffer.ts@@ -42,6 +42,8 @@ const DEFAULT_BATCH_SIZE = 200;const DEFAULT_CHUNK_SIZE = 1000;const DEFAULT_TTL_IN_SECONDS = 60 * 60;const DEFAULT_FETCH_CHUNK_SIZE = 50;+// Matches the two lookups (identified, anonymous) that used to run side by side.+const FETCH_CONCURRENCY = 2;export class ProfileBuffer extends BaseBuffer {private readonly batchSize =@@ -175,16 +177,14 @@ export class ProfileBuffer extends BaseBuffer {): Promise<Map<string, IClickhouseProfile>> {const result = new Map<string, IClickhouseProfile>();- // Non-external (anonymous/device) profiles get a 2-day recency filter to- // avoid pulling stale anonymous sessions from far back.- const external = profiles.filter((p) => p.is_external !== false);- const nonExternal = profiles.filter((p) => p.is_external === false);-- const fetchGroup = async (- group: IClickhouseProfile[],- withDateFilter: boolean- ) => {- for (const chunk of this.chunks(group, this.fetchChunkSize)) {+ // No recency window. SDKs that send their own device id keep an anonymous+ // profile for weeks; one that comes back and is not found is written as+ // new, with a new created_at, so its rows split across monthly partitions+ // and its earlier properties are lost. The window pruned nothing: the+ // table is partitioned by created_at, not last_seen_at.+ await this.parallelLimit(+ this.chunks(profiles, this.fetchChunkSize),+ async (chunk) => {const tuples = sql.join(chunk.map((p) =>@@ -193,13 +193,10 @@ export class ProfileBuffer extends BaseBuffer {', ');try {- // Qualify with `p.`: a bare `last_seen_at` resolves to the aggregate- // alias and is illegal in WHERE (ILLEGAL_AGGREGATION).const rows = await this.chQuery<IClickhouseProfile>(sql`SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}FROM ${sql.id(TABLE_NAMES.profiles)} AS pWHERE (p.id, p.project_id) IN (${tuples})- ${withDateFilter ? sql`AND p.last_seen_at > now() - INTERVAL 2 DAY` : sql.empty}GROUP BY p.id, p.project_id`);for (const row of rows) {@@ -211,13 +208,9 @@ export class ProfileBuffer extends BaseBuffer {'Failed to batch fetch profiles from Clickhouse, proceeding without existing data');}- }- };-- await Promise.all([- fetchGroup(external, false),- fetchGroup(nonExternal, true),- ]);+ },+ FETCH_CONCURRENCY+ );return result;}