Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(ingest): returning anonymous visitors keep their profile (#590)

Carl-Gerhard Lindesvärdcc84b2741bfb
diff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.ts
index 6784083e3..bf46b9952 100644
--- a/packages/core/src/buffers/profile-buffer.test.ts
+++ b/packages/core/src/buffers/profile-buffer.test.ts
@@ -1,4 +1,5 @@
import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
+import { toStatement } from '@openpanel/db/src/clickhouse/sql';
import { getRedisCache } from '@openpanel/redis';
import { bufferDepsWithCh } from '../../test/buffer-fixtures';
import { testCoreConfig } from '../../test/config-fixture';
@@ -9,11 +10,13 @@ import type { IClickhouseProfile } from '../modules/profile/profile.service';
const realChQuery = { ...(await import('../ch-query')) };
const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);
-const chQuery = mock(async (_sql: string): Promise<IClickhouseProfile[]> => []);
+const chQuery = mock(
+ async (_sql: unknown): Promise<IClickhouseProfile[]> => []
+);
mock.module('../ch-query', () => ({
...realChQuery,
- chQuery: (_scope: unknown, sql: string) => chQuery(sql),
+ chQuery: (_scope: unknown, sql: unknown) => chQuery(sql),
}));
const { ProfileBuffer } = await import('./profile-buffer');
@@ -192,6 +195,40 @@ describe('ProfileBuffer', () => {
expect(cached?.groups).toContain('new-group');
});
+ it('finds an anonymous profile last seen days ago and keeps its created_at', async () => {
+ const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', '');
+ const existingInClickhouse = makeProfile({
+ id: 'device-uuid',
+ is_external: false,
+ created_at: weekAgo,
+ last_seen_at: weekAgo,
+ properties: { os: 'iOS', first_referrer: 'google' },
+ });
+ chQuery.mockResolvedValue([existingInClickhouse]);
+
+ await profileBuffer.add(
+ makeProfile({
+ id: 'device-uuid',
+ is_external: false,
+ properties: { os: 'iOS' },
+ })
+ );
+ await profileBuffer.processBuffer();
+
+ // A recency window on the lookup would hide this profile and make it new.
+ const lookup = toStatement(chQuery.mock.calls[0]?.[0] as never).query;
+ expect(lookup).not.toContain('INTERVAL');
+ const cached = await profileBuffer.fetchFromCache(
+ 'device-uuid',
+ 'project-1'
+ );
+ expect(cached?.created_at).toBe(weekAgo);
+ expect(cached?.properties.first_referrer).toBe('google');
+ });
+
it('buffer is empty after flush', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
expect(await profileBuffer.getBufferSize()).toBe(1);
diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.ts
index 3660cf2a3..0b4fb379e 100644
--- a/packages/core/src/buffers/profile-buffer.ts
+++ b/packages/core/src/buffers/profile-buffer.ts
@@ -42,6 +42,8 @@ const DEFAULT_BATCH_SIZE = 200;
const DEFAULT_CHUNK_SIZE = 1000;
const DEFAULT_TTL_IN_SECONDS = 60 * 60;
const DEFAULT_FETCH_CHUNK_SIZE = 50;
+// Matches the two lookups (identified, anonymous) that used to run side by side.
+const FETCH_CONCURRENCY = 2;
export class ProfileBuffer extends BaseBuffer {
private readonly batchSize =
@@ -175,16 +177,14 @@ export class ProfileBuffer extends BaseBuffer {
): Promise<Map<string, IClickhouseProfile>> {
const result = new Map<string, IClickhouseProfile>();
- // Non-external (anonymous/device) profiles get a 2-day recency filter to
- // avoid pulling stale anonymous sessions from far back.
- const external = profiles.filter((p) => p.is_external !== false);
- const nonExternal = profiles.filter((p) => p.is_external === false);
-
- const fetchGroup = async (
- group: IClickhouseProfile[],
- withDateFilter: boolean
- ) => {
- for (const chunk of this.chunks(group, this.fetchChunkSize)) {
+ // No recency window. SDKs that send their own device id keep an anonymous
+ // profile for weeks; one that comes back and is not found is written as
+ // new, with a new created_at, so its rows split across monthly partitions
+ // and its earlier properties are lost. The window pruned nothing: the
+ // table is partitioned by created_at, not last_seen_at.
+ await this.parallelLimit(
+ this.chunks(profiles, this.fetchChunkSize),
+ async (chunk) => {
const tuples = sql.join(
chunk.map(
(p) =>
@@ -193,13 +193,10 @@ export class ProfileBuffer extends BaseBuffer {
', '
);
try {
- // Qualify with `p.`: a bare `last_seen_at` resolves to the aggregate
- // alias and is illegal in WHERE (ILLEGAL_AGGREGATION).
const rows = await this.chQuery<IClickhouseProfile>(
sql`SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}
FROM ${sql.id(TABLE_NAMES.profiles)} AS p
WHERE (p.id, p.project_id) IN (${tuples})
- ${withDateFilter ? sql`AND p.last_seen_at > now() - INTERVAL 2 DAY` : sql.empty}
GROUP BY p.id, p.project_id`
);
for (const row of rows) {
@@ -211,13 +208,9 @@ export class ProfileBuffer extends BaseBuffer {
'Failed to batch fetch profiles from Clickhouse, proceeding without existing data'
);
}
- }
- };
-
- await Promise.all([
- fetchGroup(external, false),
- fetchGroup(nonExternal, true),
- ]);
+ },
+ FETCH_CONCURRENCY
+ );
return result;
}