OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: add last seen column to profile and remove stale time
Carl-Gerhard Lindesvärd739c899af59c
diff --git a/apps/start/src/components/profiles/profile-properties.tsx b/apps/start/src/components/profiles/profile-properties.tsxindex 0ad4fbf88..16528729d 100644--- a/apps/start/src/components/profiles/profile-properties.tsx+++ b/apps/start/src/components/profiles/profile-properties.tsx@@ -60,6 +60,10 @@ export const ProfileProperties = ({ profile }: Props) => {name: 'createdAt',value: formatDateTime(new Date(profile.createdAt)),},+ {+ name: 'lastSeenAt',+ value: formatDateTime(new Date(profile.lastSeenAt)),+ },...(profile.properties.country? [{ name: 'country', value: profile.properties.country }]: []),diff --git a/apps/start/src/components/profiles/table/columns.tsx b/apps/start/src/components/profiles/table/columns.tsxindex e5f90e39d..63c741e0f 100644--- a/apps/start/src/components/profiles/table/columns.tsx+++ b/apps/start/src/components/profiles/table/columns.tsx@@ -102,6 +102,15 @@ export function useColumns(type: 'profiles' | 'power-users') {return <ColumnCreatedAt>{item.createdAt}</ColumnCreatedAt>;},},+ {+ accessorKey: 'lastSeenAt',+ header: 'Last seen',+ size: ColumnCreatedAt.size,+ cell: ({ row }) => {+ const item = row.original;+ return <ColumnCreatedAt>{item.lastSeenAt}</ColumnCreatedAt>;+ },+ },{accessorKey: 'groups',header: 'Groups',diff --git a/apps/start/src/components/report-chart/area/chart.tsx b/apps/start/src/components/report-chart/area/chart.tsxindex 644c66dc1..70adba4be 100644--- a/apps/start/src/components/report-chart/area/chart.tsx+++ b/apps/start/src/components/report-chart/area/chart.tsx@@ -62,17 +62,12 @@ export function Chart({ data }: Props) {const dispatch = useDispatch();const trpc = useTRPC();const references = useQuery(- trpc.reference.getChartReferences.queryOptions(- {- projectId,- startDate,- endDate,- range,- },- {- staleTime: 1000 * 60 * 10,- },- ),+ trpc.reference.getChartReferences.queryOptions({+ projectId,+ startDate,+ endDate,+ range,+ }),);const { series, setVisibleSeries } = useVisibleSeries(data, {savedVisibleSeries,diff --git a/apps/start/src/components/report-chart/area/index.tsx b/apps/start/src/components/report-chart/area/index.tsxindex 8b5946b88..ca97c2d8a 100644--- a/apps/start/src/components/report-chart/area/index.tsx+++ b/apps/start/src/components/report-chart/area/index.tsx@@ -21,7 +21,6 @@ export function ReportAreaChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/bar/index.tsx b/apps/start/src/components/report-chart/bar/index.tsxindex f17f5f1a3..b5778de42 100644--- a/apps/start/src/components/report-chart/bar/index.tsx+++ b/apps/start/src/components/report-chart/bar/index.tsx@@ -21,7 +21,6 @@ export function ReportBarChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/conversion/chart.tsx b/apps/start/src/components/report-chart/conversion/chart.tsxindex 1be114c19..7ce1cf83e 100644--- a/apps/start/src/components/report-chart/conversion/chart.tsx+++ b/apps/start/src/components/report-chart/conversion/chart.tsx@@ -65,17 +65,12 @@ export function Chart({ data }: Props) {const rechartData = useConversionRechartDataModel(series);const trpc = useTRPC();const references = useQuery(- trpc.reference.getChartReferences.queryOptions(- {- projectId,- startDate,- endDate,- range,- },- {- staleTime: 1000 * 60 * 10,- },- ),+ trpc.reference.getChartReferences.queryOptions({+ projectId,+ startDate,+ endDate,+ range,+ }),);const xAxisProps = useXAxisProps({ interval, hide: hideXAxis });diff --git a/apps/start/src/components/report-chart/conversion/index.tsx b/apps/start/src/components/report-chart/conversion/index.tsxindex 2081d3a0d..9d82da56e 100644--- a/apps/start/src/components/report-chart/conversion/index.tsx+++ b/apps/start/src/components/report-chart/conversion/index.tsx@@ -22,7 +22,6 @@ export function ReportConversionChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/histogram/chart.tsx b/apps/start/src/components/report-chart/histogram/chart.tsxindex 3e36e0f37..2acaed660 100644--- a/apps/start/src/components/report-chart/histogram/chart.tsx+++ b/apps/start/src/components/report-chart/histogram/chart.tsx@@ -75,17 +75,12 @@ export function Chart({ data }: Props) {const isStacked = histogramOptions?.stacked ?? false;const trpc = useTRPC();const references = useQuery(- trpc.reference.getChartReferences.queryOptions(- {- projectId,- startDate,- endDate,- range,- },- {- staleTime: 1000 * 60 * 10,- },- ),+ trpc.reference.getChartReferences.queryOptions({+ projectId,+ startDate,+ endDate,+ range,+ }),);const { series, setVisibleSeries } = useVisibleSeries(data, {savedVisibleSeries,diff --git a/apps/start/src/components/report-chart/histogram/index.tsx b/apps/start/src/components/report-chart/histogram/index.tsxindex 219e853e0..67862bd73 100644--- a/apps/start/src/components/report-chart/histogram/index.tsx+++ b/apps/start/src/components/report-chart/histogram/index.tsx@@ -21,7 +21,6 @@ export function ReportHistogramChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/line/chart.tsx b/apps/start/src/components/report-chart/line/chart.tsxindex 4f7979271..c78097ae6 100644--- a/apps/start/src/components/report-chart/line/chart.tsx+++ b/apps/start/src/components/report-chart/line/chart.tsx@@ -62,17 +62,12 @@ export function Chart({ data }: Props) {const dataLength = data.series[0]?.data?.length || 0;const trpc = useTRPC();const references = useQuery(- trpc.reference.getChartReferences.queryOptions(- {- projectId,- startDate,- endDate,- range,- },- {- staleTime: 1000 * 60 * 10,- },- ),+ trpc.reference.getChartReferences.queryOptions({+ projectId,+ startDate,+ endDate,+ range,+ }),);const { series, setVisibleSeries } = useVisibleSeries(data, {savedVisibleSeries,diff --git a/apps/start/src/components/report-chart/line/index.tsx b/apps/start/src/components/report-chart/line/index.tsxindex 56f74b163..ddb3bd053 100644--- a/apps/start/src/components/report-chart/line/index.tsx+++ b/apps/start/src/components/report-chart/line/index.tsx@@ -20,7 +20,6 @@ export function ReportLineChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/map/index.tsx b/apps/start/src/components/report-chart/map/index.tsxindex 39912235e..a3ffa45d9 100644--- a/apps/start/src/components/report-chart/map/index.tsx+++ b/apps/start/src/components/report-chart/map/index.tsx@@ -21,7 +21,6 @@ export function ReportMapChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/metric/index.tsx b/apps/start/src/components/report-chart/metric/index.tsxindex 47aad7d43..94f086702 100644--- a/apps/start/src/components/report-chart/metric/index.tsx+++ b/apps/start/src/components/report-chart/metric/index.tsx@@ -20,7 +20,6 @@ export function ReportMetricChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/pie/index.tsx b/apps/start/src/components/report-chart/pie/index.tsxindex 0f87c78c8..feae7e443 100644--- a/apps/start/src/components/report-chart/pie/index.tsx+++ b/apps/start/src/components/report-chart/pie/index.tsx@@ -21,7 +21,6 @@ export function ReportPieChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: !isLazyLoading,},),diff --git a/apps/start/src/components/report-chart/retention/index.tsx b/apps/start/src/components/report-chart/retention/index.tsxindex aa52fe3d8..ba1fc0c8e 100644--- a/apps/start/src/components/report-chart/retention/index.tsx+++ b/apps/start/src/components/report-chart/retention/index.tsx@@ -37,7 +37,6 @@ export function ReportRetentionChart() {},{placeholderData: keepPreviousData,- staleTime: 1000 * 60 * 1,enabled: isEnabled,},),diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.dashboards_.$dashboardId.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.dashboards_.$dashboardId.tsxindex c7c6143a3..60e23b6f3 100644--- a/apps/start/src/routes/_app.$organizationId.$projectId.dashboards_.$dashboardId.tsx+++ b/apps/start/src/routes/_app.$organizationId.$projectId.dashboards_.$dashboardId.tsx@@ -14,6 +14,7 @@ import {MoreHorizontal,PlusIcon,RotateCcw,+ SearchIcon,ShareIcon,TrashIcon,} from 'lucide-react';@@ -33,6 +34,7 @@ import {ReportItem,ReportItemSkeleton,} from '@/components/report/report-item';+import { Input } from '@/components/ui/input';import { useDashboardPageContext } from '@/hooks/use-page-context-helpers';import { handleErrorToastOptions, useTRPC } from '@/integrations/trpc/react';import { pushModal, showConfirm } from '@/modals';@@ -124,6 +126,13 @@ function Component() {const dashboard = dashboardQuery.data;const [isGridReady, setIsGridReady] = useState(false);const [enableTransitions, setEnableTransitions] = useState(false);+ const [search, setSearch] = useState('');++ const filteredReports = useMemo(() => {+ const q = search.trim().toLowerCase();+ if (!q) return reports;+ return reports.filter((r) => r.name?.toLowerCase().includes(q));+ }, [reports, search]);// Wait for initial render to ensure grid has proper dimensionsuseEffect(() => {@@ -181,7 +190,7 @@ function Component() {);// Convert reports to grid layout format for all breakpoints- const layouts = useReportLayouts(reports);+ const layouts = useReportLayouts(filteredReports);const dashboardPrimer = useMemo(() => ({@@ -280,6 +289,18 @@ function Component() {className="mb-4"actions={<>+ {reports.length > 0 && (+ <div className="relative">+ <SearchIcon className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground pointer-events-none" />+ <Input+ type="search"+ placeholder="Search reports..."+ value={search}+ onChange={(e) => setSearch(e.target.value)}+ className="pl-8 w-[180px] sm:w-[220px]"+ />+ </div>+ )}<OverviewRange /><OverviewInterval /><LinkButton@@ -361,6 +382,10 @@ function Component() {<ReportItemSkeleton /><ReportItemSkeleton /></div>+ ) : filteredReports.length === 0 ? (+ <FullPageEmptyState title="No matching reports" icon={SearchIcon}>+ <p>No reports match "{search}". Try a different search.</p>+ </FullPageEmptyState>) : (<GrafanaGridtransitions={enableTransitions}@@ -368,10 +393,10 @@ function Component() {onLayoutChange={handleLayoutChange}onDragStop={handleDragStop}onResizeStop={handleResizeStop}- isDraggable={true}- isResizable={true}+ isDraggable={!search}+ isResizable={!search}>- {reports.map((report) => (+ {filteredReports.map((report) => (<div key={report.id}><ReportItemreport={report}diff --git a/packages/db/code-migrations/16-restructure-profiles.ts b/packages/db/code-migrations/16-restructure-profiles.tsnew file mode 100644index 000000000..7e0b54266--- /dev/null+++ b/packages/db/code-migrations/16-restructure-profiles.ts@@ -0,0 +1,210 @@+import fs from 'node:fs';+import path from 'node:path';+import {+ chMigrationClient,+ createTable,+ renameTable,+ runClickhouseMigrationCommands,+} from '../src/clickhouse/migration';+import { getIsCluster } from './helpers';++// Restructures the `profiles` table so the column names actually mean what+// they say:+//+// - `created_at` → first time this profile was seen (preserved across+// upserts, like a real created_at). NOT the RMT version.+// - `last_seen_at` → most recent activity. NEW column. ReplacingMergeTree+// uses this as its version, so dedup naturally keeps the+// latest.+//+// Before this migration we abused `created_at` as both the version column and+// (de facto, due to the buffer's merge logic) the first-seen marker. That+// double duty was confusing and fragile.+//+// Strategy mirrors `8-order-keys.ts`:+// 1. Create `profiles_new_<DATE>` with the new schema.+// 2. Batch-copy month-by-month from the existing `profiles` (FINAL),+// seeding `last_seen_at` from `created_at`. The buffer's session-boundary+// upserts will start moving `last_seen_at` forward immediately after+// cutover; profiles that go inactive simply keep `last_seen_at = created_at`,+// which is honest — they haven't been seen since.+// 3. Rename current `profiles` → `profiles_<DATE>` and the new table into+// place as `profiles`.+//+// If a deployment specifically wants accurate `last_seen_at` for historically+// active profiles right after the migration, run a one-off backfill against+// events:+//+// INSERT INTO profiles SELECT+// id, is_external, first_name, last_name, email, avatar, properties,+// project_id, groups, created_at,+// greatest(last_seen_at, e.last_seen_at) AS last_seen_at+// FROM profiles AS p+// ANY LEFT JOIN (+// SELECT project_id, profile_id, max(created_at) AS last_seen_at+// FROM events+// WHERE profile_id != '' AND device_id != profile_id+// GROUP BY project_id, profile_id+// ) AS e+// ON e.project_id = p.project_id AND e.profile_id = p.id+// SETTINGS distributed_product_mode = 'allow';+//+// (RMT's version column resolves the dedup — the row with the higher+// `last_seen_at` wins.) Skip this on small/self-hosted deployments; it's not+// required for correctness.+//+// During the migration window, profile upserts hitting the renamed-away+// `profiles` table will be lost. Stop the worker (or accept brief data loss)+// before running in production.++const SUFFIX = '20260504';+const NEW_TABLE = `profiles_new_${SUFFIX}`;+const OLD_TABLE = `profiles_${SUFFIX}`;++const replicated = (table: string) => `${table}_replicated`;++export async function up() {+ const isClustered = getIsCluster();+ const sqls: string[] = [];++ // 1. New profiles table. ReplacingMergeTree's version is now `last_seen_at`,+ // so the latest activity timestamp wins on dedup — and `created_at` is+ // preserved naturally (no longer the version, no special merge handling).+ const profileTables = createTable({+ name: NEW_TABLE,+ columns: [+ '`id` String CODEC(ZSTD(3))',+ '`is_external` Bool',+ '`first_name` String CODEC(ZSTD(3))',+ '`last_name` String CODEC(ZSTD(3))',+ '`email` String CODEC(ZSTD(3))',+ '`avatar` String CODEC(ZSTD(3))',+ '`properties` Map(String, String) CODEC(ZSTD(3))',+ '`project_id` String CODEC(ZSTD(3))',+ '`groups` Array(String) DEFAULT [] CODEC(ZSTD(3))',+ '`created_at` DateTime64(3) CODEC(Delta(4), LZ4)',+ '`last_seen_at` DateTime64(3) CODEC(Delta(4), LZ4)',+ ],+ indices: [+ 'INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1',+ 'INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1',+ 'INDEX idx_email email TYPE bloom_filter GRANULARITY 1',+ ],+ engine: 'ReplacingMergeTree(last_seen_at)',+ orderBy: ['project_id', 'id'],+ partitionBy: 'toYYYYMM(created_at)',+ settings: { index_granularity: 8192 },+ distributionHash: 'cityHash64(project_id)',+ replicatedVersion: '2',+ isClustered,+ });+ sqls.push(...profileTables);++ // 2. Find the oldest profile so we know how far back to batch.+ const firstProfileResp = await chMigrationClient.query({+ query: 'SELECT min(created_at) AS created_at FROM profiles',+ format: 'JSONEachRow',+ });+ const firstProfileJson = await firstProfileResp.json<{+ created_at: string;+ }>();+ const firstDate = firstProfileJson[0]?.created_at;++ if (firstDate && !firstDate.startsWith('1970')) {+ const startDate = new Date(firstDate);+ const endDate = new Date();+ endDate.setMonth(endDate.getMonth() + 1);+ endDate.setDate(1);++ const monthBoundary = (d: Date) =>+ `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-01`;++ let cursor = new Date(endDate);+ while (true) {+ const lower = new Date(cursor);+ lower.setMonth(lower.getMonth() - 1);+ // Don't go below the first profile's month.+ if (+ lower.getFullYear() < startDate.getFullYear() ||+ (lower.getFullYear() === startDate.getFullYear() &&+ lower.getMonth() < startDate.getMonth())+ ) {+ lower.setFullYear(startDate.getFullYear());+ lower.setMonth(startDate.getMonth());+ lower.setDate(1);+ }++ sqls.push(+ // Aliases are intentionally distinct from column names. ClickHouse+ // resolves identifiers against SELECT-list aliases first, so naming an+ // alias `created_at` would shadow the source column inside `argMax(..., created_at)`+ // and trigger ILLEGAL_AGGREGATION ("aggregate inside another aggregate").+ // INSERT INTO ... (col list) SELECT maps positionally, so the alias+ // names don't need to match the target columns.+ `INSERT INTO ${NEW_TABLE}+ (id, is_external, first_name, last_name, email, avatar, properties,+ project_id, groups, created_at, last_seen_at)+ SELECT+ id,+ argMax(is_external, created_at) as v_is_external,+ argMax(nullIf(first_name, ''), created_at) as v_first_name,+ argMax(nullIf(last_name, ''), created_at) as v_last_name,+ argMax(nullIf(email, ''), created_at) as v_email,+ argMax(nullIf(avatar, ''), created_at) as v_avatar,+ argMax(properties, created_at) as v_properties,+ project_id,+ argMax(groups, created_at) as v_groups,+ min(created_at) as v_created_at,+ min(created_at) as v_last_seen_at+ FROM profiles FINAL+ WHERE profiles.created_at >= toDateTime('${monthBoundary(lower)}')+ AND profiles.created_at < toDateTime('${monthBoundary(cursor)}')+ GROUP BY id, project_id+ SETTINGS insert_distributed_sync = 1`+ );++ // Stop once we've processed the month containing the oldest profile.+ if (+ lower.getFullYear() === startDate.getFullYear() &&+ lower.getMonth() === startDate.getMonth()+ ) {+ break;+ }+ cursor = lower;+ }+ }++ // 3. Atomic-ish swap. Same dance as 8-order-keys.ts: rename the existing+ // table aside, then promote the new one in place. In cluster mode we+ // rename the underlying replicated table and rebuild the Distributed+ // wrapper.+ sqls.push(...renameTable({ from: 'profiles', to: OLD_TABLE, isClustered }));++ if (isClustered && profileTables[1]) {+ sqls.push(+ `DROP TABLE IF EXISTS ${NEW_TABLE} ON CLUSTER '{cluster}'`,+ `RENAME TABLE ${replicated(NEW_TABLE)} TO ${replicated('profiles')} ON CLUSTER '{cluster}'`,+ // Rebuild the distributed wrapper pointing at the new replicated table.+ profileTables[1].replaceAll(NEW_TABLE, 'profiles')+ );+ } else {+ sqls.push(...renameTable({ from: NEW_TABLE, to: 'profiles', isClustered }));+ }++ fs.writeFileSync(+ path.join(import.meta.filename.replace('.ts', '.sql')),+ sqls+ .map((sql) =>+ sql+ .trim()+ .replace(/;$/, '')+ .replace(/\n{2,}/g, '\n')+ .concat(';')+ )+ .join('\n\n---\n\n')+ );++ if (!process.argv.includes('--dry')) {+ await runClickhouseMigrationCommands(sqls);+ }+}diff --git a/packages/db/scripts/backfill-profile-seen.ts b/packages/db/scripts/backfill-profile-seen.tsnew file mode 100644index 000000000..6349b9a5d--- /dev/null+++ b/packages/db/scripts/backfill-profile-seen.ts@@ -0,0 +1,171 @@+// One-off backfill that reconciles `profiles.created_at` (first seen) and+// `profiles.last_seen_at` (last seen) against actual event activity.+//+// After migration `16-restructure-profiles.ts` runs, every profile has+// `last_seen_at = created_at`. From cutover onward the buffer's session-boundary+// upserts move `last_seen_at` forward — but historically active profiles with no+// new sessions stay at their migrated value until they're seen again.+//+// Run this once (per environment) when you want post-migration `last_seen_at`+// (and, defensively, `created_at`) to match what the events table already+// records.+//+// How it works:+// For each project, INSERT one new row per profile with:+// created_at = least(existing, min(events.created_at))+// last_seen_at = greatest(existing, max(events.created_at))+// ReplacingMergeTree(last_seen_at) dedups on the next merge, keeping the row+// with the higher `last_seen_at` — i.e. our updated row wins whenever it has+// newer activity, otherwise the existing row wins (greatest = old value).+//+// Per-project iteration is the trick that avoids needing a temp table:+// - The events table is sharded/sorted by `project_id` first, so the WHERE+// clause partition-prunes the events scan to that project.+// - Memory is bounded by one project's profile count, not the whole table.+// - Each project is one round-trip, so progress is observable.+//+// Usage:+// cd packages/db+// pnpm with-env tsx scripts/backfill-profile-seen.ts # all projects+// pnpm with-env tsx scripts/backfill-profile-seen.ts --dry-run # print SQL, no writes+// pnpm with-env tsx scripts/backfill-profile-seen.ts --project=abc123 # one project+// pnpm with-env tsx scripts/backfill-profile-seen.ts --concurrency=4 # N projects in parallel++import { parseArgs } from 'node:util';+import { ch, chQuery, TABLE_NAMES } from '../src/clickhouse/client';++const { values } = parseArgs({+ args: process.argv.slice(2),+ options: {+ 'dry-run': { type: 'boolean', default: false },+ project: { type: 'string' },+ concurrency: { type: 'string', default: '1' },+ },+ strict: false,+});++const dryRun = Boolean(values['dry-run']);+const onlyProject =+ typeof values.project === 'string' ? values.project : undefined;+const concurrency = Math.max(+ 1,+ Number.parseInt(+ typeof values.concurrency === 'string' ? values.concurrency : '1',+ 10,+ ) || 1,+);++function buildSql(): string {+ // `join_use_nulls = 1` makes unmatched LEFT JOIN columns NULL (instead of+ // 1970-01-01 defaults), so `ifNull` cleanly falls back to the profile's+ // existing values for profiles that have no events.+ //+ // `device_id != profile_id` excludes anonymous device-only events from the+ // lookup (matches the convention used in our other profile/event queries).+ return `+ INSERT INTO ${TABLE_NAMES.profiles}+ (id, is_external, first_name, last_name, email, avatar, properties,+ project_id, groups, created_at, last_seen_at)+ SELECT+ p.id,+ p.is_external,+ p.first_name,+ p.last_name,+ p.email,+ p.avatar,+ p.properties,+ p.project_id,+ p.groups,+ least(p.created_at, ifNull(e.first_seen, p.created_at)) AS created_at,+ greatest(p.last_seen_at, ifNull(e.last_seen, p.last_seen_at)) AS last_seen_at+ FROM ${TABLE_NAMES.profiles} AS p FINAL+ LEFT JOIN (+ SELECT+ profile_id,+ min(created_at) AS first_seen,+ max(created_at) AS last_seen+ FROM ${TABLE_NAMES.events}+ WHERE project_id = {projectId:String}+ AND profile_id != ''+ AND device_id != profile_id+ GROUP BY profile_id+ ) AS e ON e.profile_id = p.id+ WHERE p.project_id = {projectId:String}+ SETTINGS+ join_use_nulls = 1,+ -- 'global' rewrites the subquery JOIN to GLOBAL JOIN in cluster mode,+ -- so the events aggregate is gathered across all shards on the+ -- initiator and broadcast. Without it, each shard would only see its+ -- local events slice and the last_seen lookup could undercount. No-op+ -- in non-clustered setups.+ distributed_product_mode = 'global',+ insert_distributed_sync = 1+ `;+}++async function backfillProject(projectId: string, index: number, total: number) {+ const t0 = Date.now();+ const sql = buildSql();++ if (dryRun) {+ console.log(+ `[backfill] [${index}/${total}] DRY RUN project=${projectId}\n${sql}\n`,+ );+ return;+ }++ await ch.command({+ query: sql,+ query_params: { projectId },+ });++ const ms = Date.now() - t0;+ console.log(+ `[backfill] [${index}/${total}] project=${projectId} done in ${ms}ms`,+ );+}++async function main() {+ const start = Date.now();++ let projectIds: string[];+ if (onlyProject) {+ projectIds = [onlyProject];+ } else {+ const rows = await chQuery<{ project_id: string }>(+ `SELECT DISTINCT project_id FROM ${TABLE_NAMES.profiles}+ ORDER BY project_id`,+ );+ projectIds = rows.map((r) => r.project_id);+ }++ console.log(+ `[backfill] ${projectIds.length} project(s) · concurrency=${concurrency}${dryRun ? ' · DRY RUN' : ''}`,+ );++ // Simple worker-pool: each worker drains a shared cursor.+ let cursor = 0;+ await Promise.all(+ Array.from({ length: Math.min(concurrency, projectIds.length) }, async () => {+ while (cursor < projectIds.length) {+ const i = cursor++;+ const id = projectIds[i]!;+ try {+ await backfillProject(id, i + 1, projectIds.length);+ } catch (err) {+ console.error(+ `[backfill] [${i + 1}/${projectIds.length}] project=${id} FAILED`,+ err,+ );+ }+ }+ }),+ );++ console.log(`[backfill] Done in ${((Date.now() - start) / 1000).toFixed(1)}s`);+}++main().catch((err) => {+ console.error('[backfill] Fatal error:', err);+ process.exit(1);+});diff --git a/packages/db/src/buffers/profile-buffer.test.ts b/packages/db/src/buffers/profile-buffer.test.tsindex 2dc757f97..070b758ec 100644--- a/packages/db/src/buffers/profile-buffer.test.ts+++ b/packages/db/src/buffers/profile-buffer.test.ts@@ -19,6 +19,7 @@ import { chQuery } from '../clickhouse/client';const redis = getRedisCache();function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile {+ const now = new Date().toISOString();return {id: 'profile-1',project_id: 'project-1',@@ -28,7 +29,8 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfileavatar: '',properties: {},is_external: true,- created_at: new Date().toISOString(),+ created_at: now,+ last_seen_at: now,groups: [],...overrides,};diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 5ae61c8ce..53f14b256 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -7,6 +7,12 @@ import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';import type { IClickhouseProfile } from '../services/profile.service';import { BaseBuffer } from './base-buffer';+// Inlined to avoid a circular value-import with `profile.service.ts`+// (which imports `profileBuffer` from `../buffers`). Keep this in sync+// with the `PROFILE_COLUMNS` constant there.+const PROFILE_COLUMNS =+ 'id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, last_seen_at, groups';+export class ProfileBuffer extends BaseBuffer {private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)@@ -59,6 +65,19 @@ export class ProfileBuffer extends BaseBuffer {return getSafeJson<IClickhouseProfile>(cached);}+ public async setCache(profile: IClickhouseProfile): Promise<void> {+ const cacheKey = this.getProfileCacheKey({+ profileId: profile.id,+ projectId: profile.project_id,+ });+ await this.redis.set(+ cacheKey,+ JSON.stringify(profile),+ 'EX',+ this.ttlInSeconds+ );+ }+async add(profile: IClickhouseProfile, isFromEvent = false) {try {if (isFromEvent) {@@ -129,6 +148,9 @@ export class ProfileBuffer extends BaseBuffer {}return {+ // `created_at` is preserved (existing wins → first seen stays put).+ // `last_seen_at` flows through from incoming so the RMT version advances.+ // `groups` get unioned below....deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),};@@ -157,20 +179,10 @@ export class ProfileBuffer extends BaseBuffer {.join(', ');try {const rows = await chQuery<IClickhouseProfile>(- `SELECT- id,- project_id,- argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,- argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,- argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,- argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,- argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,- argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,- max(created_at) as created_at- FROM ${TABLE_NAMES.profiles}+ `SELECT ${PROFILE_COLUMNS}+ FROM ${TABLE_NAMES.profiles} FINALWHERE (id, project_id) IN (${tuples})- ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}- GROUP BY id, project_id`+ ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.last_seen_at > now() - INTERVAL 2 DAY` : ''}`);for (const row of rows) {result.set(`${row.project_id}:${row.id}`, row);@@ -178,7 +190,7 @@ export class ProfileBuffer extends BaseBuffer {} catch (error) {this.logger.warn({ err: error, chunkSize: chunk.length },- 'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',+ 'Failed to batch fetch profiles from Clickhouse, proceeding without existing data');}}@@ -294,7 +306,7 @@ export class ProfileBuffer extends BaseBuffer {totalProfiles: rawProfiles.length,uniqueProfiles: uniqueProfiles.length,},- 'Successfully completed profile processing',+ 'Successfully completed profile processing');} catch (error) {this.logger.error({ err: error }, 'Failed to process buffer');diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex 7a4aa2cf0..ee7a86f19 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -202,6 +202,12 @@ export function getProfilePropertySelect(property: string): string {if (withoutPrefix === 'avatar') {return 'avatar';}+ if (withoutPrefix === 'created_at') {+ return 'created_at';+ }+ if (withoutPrefix === 'last_seen_at') {+ return 'last_seen_at';+ }if (withoutPrefix.startsWith('properties.')) {const propKey = withoutPrefix.replace(/^properties\./, '');return `properties[${sqlstring.escape(propKey)}]`;@@ -370,7 +376,13 @@ export async function getChartSql({fields.add('properties');} else if (fieldName &&- ['email', 'first_name', 'last_name'].includes(fieldName)+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)) {fields.add(fieldName);}@@ -385,7 +397,13 @@ export async function getChartSql({fields.add('properties');} else if (fieldName &&- ['email', 'first_name', 'last_name'].includes(fieldName)+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)) {fields.add(fieldName);}@@ -419,6 +437,12 @@ export async function getChartSql({if (field === 'last_name') {return 'last_name as "profile.last_name"';}+ if (field === 'created_at') {+ return 'created_at as "profile.created_at"';+ }+ if (field === 'last_seen_at') {+ return 'last_seen_at as "profile.last_seen_at"';+ }return field;});@@ -720,7 +744,13 @@ export async function getAggregateChartSql({fields.add('properties');} else if (fieldName &&- ['email', 'first_name', 'last_name'].includes(fieldName)+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)) {fields.add(fieldName);}@@ -735,7 +765,13 @@ export async function getAggregateChartSql({fields.add('properties');} else if (fieldName &&- ['email', 'first_name', 'last_name'].includes(fieldName)+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)) {fields.add(fieldName);}@@ -768,6 +804,12 @@ export async function getAggregateChartSql({if (field === 'last_name') {return 'last_name as "profile.last_name"';}+ if (field === 'created_at') {+ return 'created_at as "profile.created_at"';+ }+ if (field === 'last_seen_at') {+ return 'last_seen_at as "profile.last_seen_at"';+ }return field;});diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.tsindex 89ac8e7a8..ba1ce8e89 100644--- a/packages/db/src/services/conversion.service.ts+++ b/packages/db/src/services/conversion.service.ts@@ -70,7 +70,15 @@ export class ConversionService {const fieldName = b.name.replace('profile.', '').split('.')[0];if (fieldName === 'properties') {profileFields.add('properties');- } else if (['email', 'first_name', 'last_name'].includes(fieldName!)) {+ } else if (+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName!)+ ) {profileFields.add(fieldName!);}}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex d50c686b1..bc320dc23 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -332,6 +332,7 @@ export async function getEvents(firstName: '',lastName: '',createdAt: new Date(),+ lastSeenAt: new Date(),projectId,isExternal: false,properties: {},@@ -421,11 +422,17 @@ export async function createEvent(payload: IServiceCreateEventPayload) {},};- if (- profile.isExternal ||- (profile.isExternal === false && payload.name === 'session_start')- ) {- promises.push(upsertProfile(profile, true));+ // Only upsert the profile on session boundaries.+ // - session_start covers fresh activity (drives `last_seen_at` via+ // `created_at`, our ReplacingMergeTree version column).+ // - session_end is synthesized server-side by the worker, so we get a+ // reliable end-of-session bump even when the SDK doesn't fire it.+ // Per-event upserts for identified users are skipped — `op.identify()` /+ // `op.setProfile()` go through the controller path directly and stay fresh.+ // `isFromEvent=false` so the buffer's cache-shortcut doesn't suppress the+ // write — we want every session boundary to bump `last_seen_at`.+ if (payload.name === 'session_start' || payload.name === 'session_end') {+ promises.push(upsertProfile(profile));}}diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.tsindex f84704e3b..f3a939ffd 100644--- a/packages/db/src/services/funnel.service.ts+++ b/packages/db/src/services/funnel.service.ts@@ -283,7 +283,15 @@ export class FunnelService {const fieldName = b.name.replace('profile.', '').split('.')[0];if (fieldName === 'properties') {profileFields.add('properties');- } else if (['email', 'first_name', 'last_name'].includes(fieldName!)) {+ } else if (+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName!)+ ) {profileFields.add(fieldName!);}}diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.tsindex f32a50098..66a04222a 100644--- a/packages/db/src/services/import.service.ts+++ b/packages/db/src/services/import.service.ts@@ -161,6 +161,7 @@ export async function insertProfilesBatch()) as Record<string, string>,created_at: p.created_at,+ last_seen_at: p.last_seen_at ?? p.created_at,}));await ch.insert({diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 7464b0bfc..47ec7a7f6 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -39,11 +39,11 @@ export function getProfileMetrics(profileId: string, projectId: string) {firstSeen: string;}>(`- WITH lastSeen AS (- SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}- ),- firstSeen AS (- SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}+ WITH profileSeen AS (+ SELECT created_at as firstSeen, last_seen_at as lastSeen+ FROM ${TABLE_NAMES.profiles} FINAL+ WHERE id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}+ LIMIT 1),screenViews AS (SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}@@ -77,15 +77,15 @@ export function getProfileMetrics(profileId: string, projectId: string) {SELECTCASEWHEN (SELECT sessions FROM sessions) <= 1 THEN 0- ELSE round(dateDiff('second', (SELECT firstSeen FROM firstSeen), (SELECT lastSeen FROM lastSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)+ ELSE round(dateDiff('second', (SELECT firstSeen FROM profileSeen), (SELECT lastSeen FROM profileSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)END as avgTimeBetweenSessions),revenue AS (SELECT sum(revenue) as revenue FROM ${TABLE_NAMES.events} WHERE name = 'revenue' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)})SELECT- (SELECT lastSeen FROM lastSeen) as lastSeen,- (SELECT firstSeen FROM firstSeen) as firstSeen,+ (SELECT lastSeen FROM profileSeen) as lastSeen,+ (SELECT firstSeen FROM profileSeen) as firstSeen,(SELECT screenViews FROM screenViews) as screenViews,(SELECT sessions FROM sessions) as sessions,(SELECT durationAvg FROM duration) as durationAvg,@@ -123,17 +123,10 @@ export async function getProfileById(id: string, projectId: string) {}const [profile] = await chQuery<IClickhouseProfile>(- `SELECT- id,- project_id,- last_value(nullIf(first_name, '')) as first_name,- last_value(nullIf(last_name, '')) as last_name,- last_value(nullIf(email, '')) as email,- last_value(nullIf(avatar, '')) as avatar,- last_value(is_external) as is_external,- last_value(properties) as properties,- last_value(created_at) as created_at- FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`+ `SELECT ${PROFILE_COLUMNS}+ FROM ${TABLE_NAMES.profiles} FINAL+ WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)}+ LIMIT 1`);if (!profile) {@@ -160,22 +153,11 @@ export async function getProfiles(ids: string[], projectId: string) {}const data = await chQuery<IClickhouseProfile>(- `SELECT- id,- project_id,- any(nullIf(first_name, '')) as first_name,- any(nullIf(last_name, '')) as last_name,- any(nullIf(email, '')) as email,- any(nullIf(avatar, '')) as avatar,- last_value(is_external) as is_external,- any(properties) as properties,- any(created_at) as created_at,- any(groups) as groups- FROM ${TABLE_NAMES.profiles}- WHERE+ `SELECT ${PROFILE_COLUMNS}+ FROM ${TABLE_NAMES.profiles} FINAL+ WHEREproject_id = ${sqlstring.escape(projectId)} ANDid IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})- GROUP BY id, project_id`);@@ -234,7 +216,10 @@ export interface IServiceProfile {avatar: string;firstName: string;lastName: string;+ /** First time this profile was seen — preserved across upserts. */createdAt: Date;+ /** Most recent activity. ReplacingMergeTree version column. */+ lastSeenAt: Date;isExternal: boolean;projectId: string;groups: string[];@@ -264,7 +249,10 @@ export interface IClickhouseProfile {properties: Record<string, string | undefined>;project_id: string;is_external: boolean;+ /** First time this profile was seen — preserved across upserts. */created_at: string;+ /** Most recent activity. ReplacingMergeTree version column. */+ last_seen_at: string;groups: string[];}@@ -282,16 +270,21 @@ export interface IServiceUpsertProfile {export function transformProfile({created_at,+ last_seen_at,first_name,last_name,...profile}: IClickhouseProfile): IServiceProfile {+ const createdAtJs = convertClickhouseDateToJs(created_at);return {firstName: first_name,lastName: last_name,isExternal: profile.is_external,properties: toObject(profile.properties),- createdAt: convertClickhouseDateToJs(created_at),+ createdAt: createdAtJs,+ lastSeenAt: last_seen_at+ ? convertClickhouseDateToJs(last_seen_at)+ : createdAtJs,projectId: profile.project_id,id: profile.id,email: profile.email,@@ -314,6 +307,7 @@ export function upsertProfile(}: IServiceUpsertProfile,isFromEvent = false) {+ const now = formatClickhouseDate(new Date());const profile: IClickhouseProfile = {id: String(id),first_name: firstName || '',@@ -322,7 +316,12 @@ export function upsertProfile(avatar: avatar || '',properties: strip((properties as Record<string, string | undefined>) || {}),project_id: projectId,- created_at: formatClickhouseDate(new Date()),+ // First-seen value for brand-new profiles. The buffer's mergeProfiles+ // omits `created_at` from incoming, so for existing profiles the original+ // value is carried forward.+ created_at: now,+ // RMT version column — must advance on every write so the latest row wins.+ last_seen_at: now,is_external: isExternal,groups: groups ?? [],};@@ -330,8 +329,8 @@ export function upsertProfile(return profileBuffer.add(profile, isFromEvent);}-const PROFILE_COLUMNS =- 'id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, groups';+export const PROFILE_COLUMNS =+ 'id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, last_seen_at, groups';export interface FindProfilesInput {projectId: string;@@ -416,7 +415,7 @@ export function findProfilesCore(const sql = `SELECT ${PROFILE_COLUMNS}- FROM ${TABLE_NAMES.profiles}+ FROM ${TABLE_NAMES.profiles} FINALWHERE ${conditions.join(' AND ')}ORDER BY created_at ${orderDir}LIMIT ${limit}@@ -436,7 +435,7 @@ export async function getProfileWithEvents(const [profiles, recent_events] = await Promise.all([chQuery<IClickhouseProfile>(`SELECT ${PROFILE_COLUMNS}- FROM ${TABLE_NAMES.profiles}+ FROM ${TABLE_NAMES.profiles} FINALWHERE project_id = ${sqlstring.escape(projectId)} AND id = ${sqlstring.escape(profileId)}LIMIT 1`),diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex 8d5e8866d..77614bb32 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -292,6 +292,7 @@ export async function getSessionList(options: GetSessionListOptions) {firstName: '',lastName: '',createdAt: new Date(),+ lastSeenAt: new Date(),projectId,isExternal: false,properties: {},diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.tsindex caf11681a..99dafc177 100644--- a/packages/importer/src/providers/mixpanel.ts+++ b/packages/importer/src/providers/mixpanel.ts@@ -401,6 +401,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {avatar: String(props.$avatar ?? props.$image ?? ''),properties,created_at: createdAt,+ last_seen_at: createdAt,is_external: true,groups: [],};diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex c191d9f47..9fe876a0b 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -304,6 +304,8 @@ export const chartRouter = createTRPCRouter({'profile.first_name','profile.last_name','profile.email',+ 'profile.created_at',+ 'profile.last_seen_at',];const properties = [diff --git a/test/fixtures.ts b/test/fixtures.tsindex f85d901d6..3cf300d97 100644--- a/test/fixtures.ts+++ b/test/fixtures.ts@@ -197,6 +197,7 @@ async function insertFixtures(client: ChClient, projectId: string) {properties: { browser: 'Chrome', country: 'US', device: 'desktop' },groups: [],created_at: timeAgo(now, 60),+ last_seen_at: timeAgo(now, 1),},{id: FIXTURE.profiles.bob,@@ -210,6 +211,7 @@ async function insertFixtures(client: ChClient, projectId: string) {properties: { browser: 'Chrome', country: 'SE', device: 'desktop' },groups: [],created_at: timeAgo(now, 90),+ last_seen_at: timeAgo(now, 90),},{id: FIXTURE.profiles.charlie,@@ -222,6 +224,7 @@ async function insertFixtures(client: ChClient, projectId: string) {properties: { browser: 'Firefox', country: 'US', device: 'desktop' },groups: [],created_at: timeAgo(now, 30),+ last_seen_at: timeAgo(now, 5),},],format: 'JSONEachRow',