OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
prep events partition
Carl-Gerhard Lindesvärd3993b493e380
diff --git a/apps/api/scripts/migrate-origins.ts b/apps/api/scripts/migrate-origins.tsindex 7f365e856..0c8698bd9 100644--- a/apps/api/scripts/migrate-origins.ts+++ b/apps/api/scripts/migrate-origins.ts@@ -1,8 +1,8 @@-import { ch, chQuery } from '@openpanel/db';+import { ch, chQuery, TABLE_NAMES } from '@openpanel/db';async function main() {const projects = await chQuery(- `SELECT distinct project_id FROM events ORDER BY project_id`+ `SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`);const withOrigin = [];@@ -10,10 +10,10 @@ async function main() {try {const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([await chQuery(- `SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`+ `SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`),await chQuery(- `SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`+ `SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`),]);@@ -22,7 +22,7 @@ async function main() {console.log(`- Origin: ${eventWithOrigin[0].origin}`);withOrigin.push(project.project_id);const events = await chQuery(- `SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`+ `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`);console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);await ch.command({diff --git a/apps/api/src/controllers/live.controller.ts b/apps/api/src/controllers/live.controller.tsindex e0e691025..ba783fe22 100644--- a/apps/api/src/controllers/live.controller.ts+++ b/apps/api/src/controllers/live.controller.ts@@ -10,6 +10,7 @@ import {getEvents,getLiveVisitors,getProfileById,+ TABLE_NAMES,transformMinimalEvent,} from '@openpanel/db';import { redis, redisPub, redisSub } from '@openpanel/redis';@@ -28,8 +29,7 @@ export async function testVisitors(reply: FastifyReply) {const events = await getEvents(- `SELECT * FROM events LIMIT 500`- // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`+ `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`);const event = events[Math.floor(Math.random() * events.length)];if (!event) {@@ -55,8 +55,7 @@ export async function testEvents(reply: FastifyReply) {const events = await getEvents(- `SELECT * FROM events LIMIT 500`- // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`+ `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`);const event = events[Math.floor(Math.random() * events.length)];if (!event) {diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 9d918d731..57e596bc4 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -7,7 +7,7 @@ import Fastify from 'fastify';import metricsPlugin from 'fastify-metrics';import { round } from '@openpanel/common';-import { chQuery, db } from '@openpanel/db';+import { chQuery, db, TABLE_NAMES } from '@openpanel/db';import type { IServiceClient } from '@openpanel/db';import { eventsQueue } from '@openpanel/queue';import { redis, redisPub } from '@openpanel/redis';@@ -101,7 +101,9 @@ const startServer = async () => {const redisRes = await withTimings(redis.keys('*'));const dbRes = await withTimings(db.project.findFirst());const queueRes = await withTimings(eventsQueue.getCompleted());- const chRes = await withTimings(chQuery('SELECT * FROM events LIMIT 1'));+ const chRes = await withTimings(+ chQuery(`SELECT * FROM ${TABLE_NAMES.events} LIMIT 1`)+ );const status = redisRes && dbRes && queueRes && chRes ? 200 : 500;reply.status(status).send({diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsxindex 338ae0ac5..fb204124b 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx@@ -1,7 +1,7 @@import withLoadingWidget from '@/hocs/with-loading-widget';import { escape } from 'sqlstring';-import { db, getEvents } from '@openpanel/db';+import { db, getEvents, TABLE_NAMES } from '@openpanel/db';import { EventConversionsList } from './event-conversions-list';@@ -22,7 +22,7 @@ async function EventConversionsListServer({ projectId }: Props) {}const events = await getEvents(- `SELECT * FROM events WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,+ `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,{profile: true,meta: true,diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsxindex f52cd4117..8be37ccdc 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx@@ -1,7 +1,7 @@import withLoadingWidget from '@/hocs/with-loading-widget';import { escape } from 'sqlstring';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';import MostEvents from './most-events';@@ -12,7 +12,7 @@ type Props = {const MostEventsServer = async ({ projectId, profileId }: Props) => {const data = await chQuery<{ count: number; name: string }>(- `SELECT count(*) as count, name FROM events WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`+ `SELECT count(*) as count, name FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`);return <MostEvents data={data} />;};diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsxindex 64980052e..66ea3fb44 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx@@ -1,7 +1,7 @@import withLoadingWidget from '@/hocs/with-loading-widget';import { escape } from 'sqlstring';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';import PopularRoutes from './popular-routes';@@ -12,7 +12,7 @@ type Props = {const PopularRoutesServer = async ({ projectId, profileId }: Props) => {const data = await chQuery<{ count: number; path: string }>(- `SELECT count(*) as count, path FROM events WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`+ `SELECT count(*) as count, path FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`);return <PopularRoutes data={data} />;};diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsxindex 0bd724a03..37784b7a2 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx@@ -1,7 +1,7 @@import withLoadingWidget from '@/hocs/with-loading-widget';import { escape } from 'sqlstring';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';import ProfileActivity from './profile-activity';@@ -12,7 +12,7 @@ type Props = {const ProfileActivityServer = async ({ projectId, profileId }: Props) => {const data = await chQuery<{ count: number; date: string }>(- `SELECT count(*) as count, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`+ `SELECT count(*) as count, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`);return <ProfileActivity data={data} />;};diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsxindex 5887484f8..6c5bb1c5e 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx@@ -7,7 +7,7 @@ import { Widget, WidgetBody, WidgetHead } from '@/components/widget';import { cn } from '@/utils/cn';import { escape } from 'sqlstring';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';interface Props {projectId: string;@@ -21,7 +21,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {// Days since last event from users// group by daysconst res = await chQuery<Row>(- `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM events where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`+ `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM ${TABLE_NAMES.events} where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`);const maxValue = Math.max(...res.map((x) => x.count));@@ -37,7 +37,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {<Tooltip><TooltipTrigger asChild><div- className={cn('bg-highlight aspect-square w-full shrink-0 rounded')}+ className={cn('aspect-square w-full shrink-0 rounded bg-highlight')}style={{opacity: calculateRatio(item.count),}}diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsxindex 570122fc6..337edc1e7 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsx@@ -7,7 +7,7 @@ import { getProfileName } from '@/utils/getters';import Link from 'next/link';import { escape } from 'sqlstring';-import { chQuery, getProfiles } from '@openpanel/db';+import { chQuery, getProfiles, TABLE_NAMES } from '@openpanel/db';interface Props {projectId: string;@@ -18,7 +18,7 @@ async function ProfileTopServer({ organizationSlug, projectId }: Props) {// Days since last event from users// group by daysconst res = await chQuery<{ profile_id: string; count: number }>(- `SELECT profile_id, count(*) as count from events where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`+ `SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`);const profiles = await getProfiles(res.map((r) => r.profile_id));const list = res.map((item) => {diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsxindex 23fb18727..a0f183fad 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx@@ -1,7 +1,7 @@import { subMinutes } from 'date-fns';import { escape } from 'sqlstring';-import { chQuery, formatClickhouseDate } from '@openpanel/db';+import { chQuery, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db';import type { Coordinate } from './coordinates';import Map from './map';@@ -11,7 +11,7 @@ type Props = {};const RealtimeMap = async ({ projectId }: Props) => {const res = await chQuery<Coordinate>(- `SELECT DISTINCT city, longitude as long, latitude as lat FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`+ `SELECT DISTINCT city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`);return <Map markers={res} />;diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsxindex 3421ec39d..b85700fc3 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx@@ -1,6 +1,6 @@import { escape } from 'sqlstring';-import { getEvents } from '@openpanel/db';+import { getEvents, TABLE_NAMES } from '@openpanel/db';import LiveEvents from './live-events';@@ -10,7 +10,7 @@ type Props = {};const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => {const events = await getEvents(- `SELECT * FROM events WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,+ `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,{profile: true,}diff --git a/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx b/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsxindex 3e7243233..8a4c1c6bd 100644--- a/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx+++ b/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx@@ -5,6 +5,7 @@ import {getCurrentOrganizations,getEvents,getProjectWithClients,+ TABLE_NAMES,} from '@openpanel/db';import OnboardingVerify from './onboarding-verify';@@ -24,7 +25,7 @@ const Verify = async ({ params: { projectId } }: Props) => {const [project, events] = await Promise.all([await getProjectWithClients(projectId),getEvents(- `SELECT * FROM events WHERE project_id = ${escape(projectId)} LIMIT 100`+ `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} LIMIT 100`),]);diff --git a/apps/dashboard/src/components/projects/project-card.tsx b/apps/dashboard/src/components/projects/project-card.tsxindex 9a0186673..11715e437 100644--- a/apps/dashboard/src/components/projects/project-card.tsx+++ b/apps/dashboard/src/components/projects/project-card.tsx@@ -3,7 +3,7 @@ import { shortNumber } from '@/hooks/useNumerFormatter';import { escape } from 'sqlstring';import type { IServiceProject } from '@openpanel/db';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';import { ChartSSR } from '../chart-ssr';import { FadeIn } from '../fade-in';@@ -34,7 +34,7 @@ function ProjectCard({ id, name, organizationSlug }: IServiceProject) {async function ProjectChart({ id }: { id: string }) {const chart = await chQuery<{ value: number; date: string }>(- `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`+ `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`);return (@@ -62,13 +62,13 @@ async function ProjectMetrics({ id }: { id: string }) {`SELECT(- SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)}+ SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)}) as total,(- SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'+ SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month') as month,(- SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'+ SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day') as day`);diff --git a/apps/public/src/app/hero.tsx b/apps/public/src/app/hero.tsxindex 615e919ed..64354c109 100644--- a/apps/public/src/app/hero.tsx+++ b/apps/public/src/app/hero.tsx@@ -1,6 +1,6 @@import { ALink } from '@/components/ui/button';-import { chQuery } from '@openpanel/db';+import { chQuery, TABLE_NAMES } from '@openpanel/db';import AnimatedText from './animated-text';import { Heading1, Lead2 } from './copy';@@ -15,7 +15,7 @@ function shortNumber(num: number) {export async function Hero() {const projects = await chQuery<{ project_id: string; count: number }>(- 'SELECT project_id, count(*) as count from events GROUP by project_id order by count()'+ `SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()`);const projectCount = projects.length;const eventCount = projects.reduce((acc, { count }) => acc + count, 0);diff --git a/apps/worker/scripts/debug.ts b/apps/worker/scripts/debug.tsindex da0a81ed3..41817eef7 100644--- a/apps/worker/scripts/debug.ts+++ b/apps/worker/scripts/debug.ts@@ -1,7 +1,7 @@import { escape } from 'sqlstring';import type { IClickhouseEvent } from '@openpanel/db';-import { chQuery, eventBuffer } from '@openpanel/db';+import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db';import { sessionsQueue } from '@openpanel/queue/src/queues';import { redis } from '@openpanel/redis';@@ -70,7 +70,7 @@ async function debugStalledEvents() {if (stalledEvents.length > 0) {const res = await chQuery(- `SELECT * FROM events WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`+ `SELECT * FROM ${TABLE_NAMES.events} WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`);stalledEvents.forEach((item) => {diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 22a451820..0e1e0d455 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -1,7 +1,12 @@import type { Job } from 'bullmq';import { getTime } from '@openpanel/common';-import { createEvent, eventBuffer, getEvents } from '@openpanel/db';+import {+ createEvent,+ eventBuffer,+ getEvents,+ TABLE_NAMES,+} from '@openpanel/db';import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';export async function createSessionEnd(@@ -13,12 +18,12 @@ export async function createSessionEnd();const sql = `- SELECT * FROM events+ SELECT * FROM ${TABLE_NAMES.events}WHEREsession_id = '${payload.sessionId}'AND created_at >= (SELECT created_at- FROM events+ FROM ${TABLE_NAMES.events}WHEREsession_id = '${payload.sessionId}'AND name = 'session_start'diff --git a/apps/worker/src/jobs/events.ts b/apps/worker/src/jobs/events.tsindex c784f79d9..a94220020 100644--- a/apps/worker/src/jobs/events.ts+++ b/apps/worker/src/jobs/events.ts@@ -1,12 +1,13 @@import type { Job } from 'bullmq';import { escape } from 'sqlstring';-import { chQuery, db } from '@openpanel/db';+import { chQuery, db, TABLE_NAMES } from '@openpanel/db';import type {EventsQueuePayload,EventsQueuePayloadCreateSessionEnd,EventsQueuePayloadIncomingEvent,} from '@openpanel/queue';+import { redis } from '@openpanel/redis';import { createSessionEnd } from './events.create-session-end';import { incomingEvent } from './events.incoming-event';@@ -26,7 +27,7 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {async function updateEventsCount(projectId: string) {const res = await chQuery<{ count: number }>(- `SELECT count(*) as count FROM events WHERE project_id = ${escape(projectId)}`+ `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`);const count = res[0]?.count;if (count) {diff --git a/packages/db/clickhouse_init.sql b/packages/db/clickhouse_init.sqlindex 6c71cdb06..4339d4a26 100644--- a/packages/db/clickhouse_init.sql+++ b/packages/db/clickhouse_init.sql@@ -34,6 +34,51 @@ CREATE TABLE IF NOT EXISTS openpanel.events (ORDER BY(project_id, created_at, profile_id) SETTINGS index_granularity = 8192;+CREATE TABLE IF NOT EXISTS openpanel.events_v2 (+ `id` UUID DEFAULT generateUUIDv4(),+ `name` String,+ `device_id` String,+ `profile_id` String,+ `project_id` String,+ `session_id` String,+ `path` String,+ `origin` String,+ `referrer` String,+ `referrer_name` String,+ `referrer_type` String,+ `duration` UInt64,+ `properties` Map(String, String),+ `created_at` DateTime64(3),+ `country` String,+ `city` String,+ `region` String,+ `longitude` Nullable(Float32),+ `latitude` Nullable(Float32),+ `os` String,+ `os_version` String,+ `browser` String,+ `browser_version` String,+ -- device: mobile/desktop/tablet+ `device` String,+ -- brand: (Samsung, OnePlus)+ `brand` String,+ -- model: (Samsung Galaxy, iPhone X)+ `model` String+) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)+ORDER BY+ (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;++ALTER TABLE+ events DROP COLUMN utm_source,+ DROP COLUMN utm_medium,+ DROP COLUMN utm_campaign,+ DROP COLUMN utm_term,+ DROP COLUMN utm_content,+ DROP COLUMN sdk,+ DROP COLUMN sdk_version,+ DROP COLUMN client_type,+ DROP COLUMN continent;+CREATE TABLE IF NOT EXISTS openpanel.events_bots (`id` UUID DEFAULT generateUUIDv4(),`project_id` String,diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex c902cf470..78ff5caaa 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -4,7 +4,7 @@ import SuperJSON from 'superjson';import { deepMergeObjects } from '@openpanel/common';import { redis, redisPub } from '@openpanel/redis';-import { ch } from '../clickhouse-client';+import { ch, TABLE_NAMES } from '../clickhouse-client';import { transformEvent } from '../services/event.service';import type {IClickhouseEvent,@@ -30,7 +30,7 @@ const sortOldestFirst = (export class EventBuffer extends RedisBuffer<IClickhouseEvent> {constructor() {super({- table: 'events',+ table: TABLE_NAMES.events,redis,});}@@ -176,7 +176,7 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {}await ch.insert({- table: 'events',+ table: TABLE_NAMES.events,values: Array.from(itemsToClickhouse).map((item) => item.event),format: 'JSONEachRow',});diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 0581fcf18..a49f2384d 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -1,6 +1,11 @@import type { ResponseJSON } from '@clickhouse/client';import { createClient } from '@clickhouse/client';+export const TABLE_NAMES = {+ events: 'events',+ profiles: 'profiles',+};+export const originalCh = createClient({url: process.env.CLICKHOUSE_URL,username: process.env.CLICKHOUSE_USER,diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex a5568bbd5..e84a2e85c 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -6,7 +6,7 @@ import type {IGetChartDataInput,} from '@openpanel/validation';-import { formatClickhouseDate } from '../clickhouse-client';+import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';import { createSqlBuilder } from '../sql-builder';export function getChartSql({@@ -94,7 +94,7 @@ export function getChartSql({if (event.segment === 'one_event_per_user') {sb.from = `(- SELECT DISTINCT ON (profile_id) * from events WHERE ${join(+ SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join(sb.where,' AND ')}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex e8e3c0f89..3802929bc 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -12,6 +12,7 @@ import {chQuery,convertClickhouseDateToJs,formatClickhouseDate,+ TABLE_NAMES,} from '../clickhouse-client';import type { EventMeta, Prisma } from '../prisma-client';import { db } from '../prisma-client';@@ -323,7 +324,7 @@ export async function getEventList({sb.where.projectId = `project_id = ${escape(projectId)}`;if (profileId) {- sb.where.deviceId = `device_id IN (SELECT device_id as did FROM events WHERE profile_id = ${escape(profileId)} group by did)`;+ sb.where.deviceId = `device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} group by did)`;}if (startDate && endDate) {@@ -448,7 +449,7 @@ export async function getLastScreenViewFromProfileId({const [eventInDb] = profileId? await getEvents(- `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1`+ `SELECT * FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1`): [];diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex fa7cbaaea..425375a3e 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -4,7 +4,11 @@ import { toObject } from '@openpanel/common';import type { IChartEventFilter } from '@openpanel/validation';import { profileBuffer } from '../buffers';-import { chQuery, formatClickhouseDate } from '../clickhouse-client';+import {+ chQuery,+ formatClickhouseDate,+ TABLE_NAMES,+} from '../clickhouse-client';import { createSqlBuilder } from '../sql-builder';export type IProfileMetrics = {@@ -18,19 +22,19 @@ export type IProfileMetrics = {export function getProfileMetrics(profileId: string, projectId: string) {return chQuery<IProfileMetrics>(`WITH lastSeen AS (- SELECT max(created_at) as lastSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}+ SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}),firstSeen AS (- SELECT min(created_at) as firstSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}+ SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}),screenViews AS (- SELECT count(*) as screenViews FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}+ SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}),sessions AS (- SELECT count(*) as sessions FROM events WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}+ SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}),duration AS (- SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM events WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}+ SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)})SELECT lastSeen, firstSeen, screenViews, sessions, durationAvg, durationP90 FROM lastSeen, firstSeen, screenViews,sessions, duration`).then((data) => data[0]!);diff --git a/packages/db/src/services/retention.service.ts b/packages/db/src/services/retention.service.tsindex e2bc195d9..ed9acb180 100644--- a/packages/db/src/services/retention.service.ts+++ b/packages/db/src/services/retention.service.ts@@ -1,6 +1,6 @@import { escape } from 'sqlstring';-import { chQuery } from '../clickhouse-client';+import { chQuery, TABLE_NAMES } from '../clickhouse-client';type IGetWeekRetentionInput = {projectId: string;@@ -15,7 +15,7 @@ WITHSELECTprofile_id,max(toWeek(created_at)) AS last_seen- FROM events+ FROM ${TABLE_NAMES.events}WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)GROUP BY profile_id),@@ -24,7 +24,7 @@ WITHSELECTprofile_id,min(toWeek(created_at)) AS first_seen- FROM events+ FROM ${TABLE_NAMES.events}WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)GROUP BY profile_id),@@ -79,8 +79,8 @@ export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {countDistinct(events.profile_id) AS active_users,countDistinct(future_events.profile_id) AS retained_users,(100 * (countDistinct(future_events.profile_id) / CAST(countDistinct(events.profile_id), 'float'))) AS retention- FROM events- LEFT JOIN events AS future_events ON+ FROM ${TABLE_NAMES.events} as events+ LEFT JOIN ${TABLE_NAMES.events} AS future_events ONevents.profile_id = future_events.profile_idAND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1))AND future_events.profile_id != future_events.device_id@@ -140,7 +140,7 @@ export function getRetentionLastSeenSeries({SELECTmax(created_at) AS last_active,profile_id- FROM events+ FROM ${TABLE_NAMES.events}WHERE (project_id = ${escape(projectId)}) AND (device_id != profile_id)GROUP BY profile_id)diff --git a/packages/db/src/sql-builder.ts b/packages/db/src/sql-builder.tsindex 7bf0019ef..da625fdcf 100644--- a/packages/db/src/sql-builder.ts+++ b/packages/db/src/sql-builder.ts@@ -1,3 +1,5 @@+import { TABLE_NAMES } from './clickhouse-client';+export interface SqlBuilderObject {where: Record<string, string>;having: Record<string, string>;@@ -15,7 +17,7 @@ export function createSqlBuilder() {const sb: SqlBuilderObject = {where: {},- from: 'events',+ from: TABLE_NAMES.events,select: {},groupBy: {},orderBy: {},diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.tsindex 73cac51c6..2fe752c39 100644--- a/packages/trpc/src/routers/chart.helpers.ts+++ b/packages/trpc/src/routers/chart.helpers.ts@@ -34,6 +34,7 @@ import {getChartSql,getEventFiltersWhereClause,getProfiles,+ TABLE_NAMES,} from '@openpanel/db';import type {FinalChart,@@ -293,7 +294,7 @@ export async function getFunnelData({const innerSql = `SELECTsession_id,windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level- FROM events+ FROM ${TABLE_NAMES.events}WHEREproject_id = ${escape(projectId)} ANDcreated_at >= '${formatClickhouseDate(startDate)}' AND@@ -306,7 +307,7 @@ export async function getFunnelData({const [funnelRes, sessionRes] = await Promise.all([chQuery<{ level: number; count: number }>(sql),chQuery<{ count: number }>(- `SELECT count(name) as count FROM events WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`+ `SELECT count(name) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`),]);@@ -409,7 +410,7 @@ export async function getFunnelStep({const innerSql = `SELECTsession_id,windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level- FROM events+ FROM ${TABLE_NAMES.events}WHEREproject_id = ${escape(projectId)} ANDcreated_at >= '${formatClickhouseDate(startDate)}' AND@@ -421,7 +422,7 @@ export async function getFunnelStep({SELECTDISTINCT e.profile_id as idFROM sessions s- JOIN events e ON s.session_id = e.session_id+ JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_idWHEREs.level = ${step} ANDe.project_id = ${escape(projectId)} ANDdiff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 707bdea09..2b26ddbb8 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -3,7 +3,7 @@ import { escape } from 'sqlstring';import { z } from 'zod';import { average, max, min, round, slug, sum } from '@openpanel/common';-import { chQuery, createSqlBuilder, db } from '@openpanel/db';+import { chQuery, createSqlBuilder, db, TABLE_NAMES } from '@openpanel/db';import { zChartInput } from '@openpanel/validation';import type {FinalChart,@@ -17,7 +17,6 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';import {getChart,getChartPrevStartEndDate,- getChartSeries,getChartStartEndDate,getFunnelData,getFunnelStep,@@ -28,7 +27,7 @@ export const chartRouter = createTRPCRouter({.input(z.object({ projectId: z.string() })).query(async ({ input: { projectId } }) => {const events = await chQuery<{ name: string }>(- `SELECT DISTINCT name FROM events WHERE project_id = ${escape(projectId)}`+ `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`);return [@@ -43,7 +42,7 @@ export const chartRouter = createTRPCRouter({.input(z.object({ event: z.string().optional(), projectId: z.string() })).query(async ({ input: { projectId, event } }) => {const events = await chQuery<{ keys: string[] }>(- `SELECT distinct mapKeys(properties) as keys from events where ${+ `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${event && event !== '*' ? `name = ${escape(event)} AND ` : ''} project_id = ${escape(projectId)};`);