Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Enhance user identification process by implementing retroactive backfill for identified user IDs in ClickHouse. Added important notes on shared devices, cross-device tracking, and `clearUserId()` behavior to documentation. Removed redundant anonymous ID from pageview queue and updated tracking payload structure for clarity. (#879)

Goldflag5ab43e55bbf5
diff --git a/docs/content/docs/identify-users.mdx b/docs/content/docs/identify-users.mdx
index 0cf67c5ca..477fd917e 100644
--- a/docs/content/docs/identify-users.mdx
+++ b/docs/content/docs/identify-users.mdx
@@ -20,6 +20,8 @@ By default, Rybbit assigns an anonymous ID to each visitor based on their device
- Store user metadata (traits) like email, name, and custom fields
- View identified users in your dashboard with their real names
+When a new alias is created (i.e., the first time `identify()` is called for a given device), Rybbit retroactively updates all past anonymous events from that device to include the identified user ID. This means pre-login pageviews and events are attributed to the user once they identify themselves.
+
## Special Trait Fields
When identifying users, certain trait fields are treated specially by Rybbit:
@@ -106,6 +108,13 @@ if (currentUserId) {
}
```
+## Important Notes
+
+- **Shared devices**: If multiple users log in from the same device (same IP address and user agent), calling `identify()` with a new user ID updates the alias to point to the new user. Future events will be attributed to the most recently identified user. Past events that were already backfilled retain their original attribution.
+- **Cross-device tracking**: Identifying the same user on different devices creates separate aliases. Events from each device are linked to the user independently via the `identified_user_id` field.
+- **Retroactive backfill**: The backfill runs asynchronously when a new alias is first created. ClickHouse mutations may take a short time to propagate depending on table size. Subsequent `identify()` calls from the same device (even with a different user ID) do not re-backfill old events.
+- **`clearUserId()` behavior**: Calling `clearUserId()` removes the user ID from localStorage so future events are anonymous. It does not delete the alias in the database or modify previously attributed events.
+
## Complete Example
Here's a typical workflow for user identification:
diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.ts
index 662561ac7..cac24beb4 100644
--- a/server/src/api/analytics/users/getUserInfo.ts
+++ b/server/src/api/analytics/users/getUserInfo.ts
@@ -54,7 +54,6 @@ export async function getUserInfo(
const numericSiteId = Number(siteId);
try {
- // Run ClickHouse query and Postgres queries in parallel
const [queryResult, profileResult, aliasesResult] = await Promise.all([
clickhouse.query({
query: `
diff --git a/server/src/services/tracker/identifyService.ts b/server/src/services/tracker/identifyService.ts
index bb1940c06..daf26f6de 100644
--- a/server/src/services/tracker/identifyService.ts
+++ b/server/src/services/tracker/identifyService.ts
@@ -2,6 +2,7 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { eq, and } from "drizzle-orm";
import { z } from "zod";
import { db } from "../../db/postgres/postgres.js";
+import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { userProfiles, userAliases } from "../../db/postgres/schema.js";
import { siteConfig } from "../../lib/siteConfig.js";
import { userIdService } from "../userId/userIdService.js";
@@ -31,6 +32,29 @@ const identifyPayloadSchema = z.object({
is_new_identify: z.boolean().default(true),
});
+// Backfill window limits partition scanning to recent data only.
+// Anonymous events older than this are unlikely to belong to the identifying user.
+const BACKFILL_DAYS = 30;
+
+async function backfillIdentifiedUserId(
+ siteId: number,
+ anonymousId: string,
+ userId: string
+) {
+ try {
+ const tables = ["events", "session_replay_events", "session_replay_metadata"];
+ for (const table of tables) {
+ await clickhouse.command({
+ query: `ALTER TABLE ${table} UPDATE identified_user_id = {userId: String} WHERE site_id = {siteId: UInt16} AND user_id = {anonymousId: String} AND identified_user_id = '' AND timestamp >= now() - INTERVAL {days: UInt16} DAY`,
+ query_params: { userId, siteId, anonymousId, days: BACKFILL_DAYS },
+ });
+ }
+ logger.info({ siteId, anonymousId, userId }, "Backfilled identified_user_id in ClickHouse");
+ } catch (error) {
+ logger.error({ siteId, anonymousId, userId, error }, "Error backfilling identified_user_id");
+ }
+}
+
export async function handleIdentify(request: FastifyRequest, reply: FastifyReply) {
try {
const validationResult = identifyPayloadSchema.safeParse(request.body);
@@ -78,17 +102,14 @@ export async function handleIdentify(request: FastifyRequest, reply: FastifyRepl
anonymousId,
userId: user_id,
});
+ // Fire-and-forget: backfill identified_user_id on past anonymous events
+ backfillIdentifiedUserId(siteId, anonymousId, user_id);
} else if (existingAlias[0].userId !== user_id) {
- // Anonymous ID already linked to a different user - log but don't error
- // logger.warn(
- // {
- // siteId,
- // anonymousId,
- // existingUserId: existingAlias[0].userId,
- // newUserId: user_id,
- // },
- // "Anonymous ID already linked to different user"
- // );
+ // Update alias to point to new user
+ await db
+ .update(userAliases)
+ .set({ userId: user_id })
+ .where(and(eq(userAliases.siteId, siteId), eq(userAliases.anonymousId, anonymousId)));
}
} catch (error) {
// Handle unique constraint violation gracefully (race condition)
diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.ts
index 485ed00ef..bddedb7df 100644
--- a/server/src/services/tracker/pageviewQueue.ts
+++ b/server/src/services/tracker/pageviewQueue.ts
@@ -66,7 +66,6 @@ class PageviewQueue {
timestamp: DateTime.fromISO(pv.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"),
session_id: pv.sessionId,
user_id: pv.userId, // Always the device fingerprint
- anonymous_id: pv.anonymousId,
identified_user_id: pv.identifiedUserId || "", // Custom user ID when identified
hostname: pv.hostname || "",
pathname: pv.pathname || "",
diff --git a/server/src/services/tracker/utils.ts b/server/src/services/tracker/utils.ts
index 9b240d6a8..567d17515 100644
--- a/server/src/services/tracker/utils.ts
+++ b/server/src/services/tracker/utils.ts
@@ -8,8 +8,7 @@ import { TrackingPayload } from "./types.js";
import { SiteConfigData } from "../../lib/siteConfig.js";
export type TotalTrackingPayload = TrackingPayload & {
- userId: string; // Always the device fingerprint (same as anonymousId)
- anonymousId: string; // Always the hash of IP+UserAgent (device fingerprint)
+ userId: string; // Always the device fingerprint
identifiedUserId: string; // Custom user ID when identified, empty string otherwise
timestamp: string;
type?: string;
@@ -129,7 +128,6 @@ export async function createBasePayload(
timestamp: new Date().toISOString(),
ua: userAgentParser(userAgent),
userId: anonymousId, // Always the device fingerprint
- anonymousId: anonymousId,
identifiedUserId: identifiedUserId, // Custom user ID when identified
storeIp: siteConfiguration.trackIp,
} as any;