OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: change order keys for clickhouse tables
Carl-Gerhard Lindesvärdd7c6e88adc1f
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex b5d00528e..2ee11fc98 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -1,6 +1,7 @@import type { FastifyReply, FastifyRequest } from 'fastify';import { assocPath, pathOr, pick } from 'ramda';+import { HttpError } from '@/utils/errors';import { generateId, slug } from '@openpanel/common';import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';@@ -187,9 +188,16 @@ export async function handler(break;}case 'identify': {+ const payload = request.body.payload;const geo = await getGeoLocation(ip);+ if (!payload.profileId) {+ throw new HttpError('Missing profileId', {+ status: 400,+ });+ }+await identify({- payload: request.body.payload,+ payload,projectId,geo,ua,diff --git a/apps/start/src/hooks/use-app-context.ts b/apps/start/src/hooks/use-app-context.tsindex c00230fbf..6bf67d0bd 100644--- a/apps/start/src/hooks/use-app-context.ts+++ b/apps/start/src/hooks/use-app-context.ts@@ -17,5 +17,6 @@ export function useAppContext() {apiUrl: params.apiUrl,dashboardUrl: params.dashboardUrl,isSelfHosted: params.isSelfHosted,+ isMaintenance: params.isMaintenance ?? false,};}diff --git a/apps/start/src/routes/__root.tsx b/apps/start/src/routes/__root.tsxindex 33adeb6fb..c76afa363 100644--- a/apps/start/src/routes/__root.tsx+++ b/apps/start/src/routes/__root.tsx@@ -34,6 +34,7 @@ interface MyRouterContext {apiUrl: string;dashboardUrl: string;isSelfHosted: boolean;+ isMaintenance: boolean;}export const Route = createRootRouteWithContext<MyRouterContext>()({diff --git a/apps/start/src/routes/_app.tsx b/apps/start/src/routes/_app.tsxindex d06f21d17..2c4139d7e 100644--- a/apps/start/src/routes/_app.tsx+++ b/apps/start/src/routes/_app.tsx@@ -1,5 +1,10 @@+import { FullPageEmptyState } from '@/components/full-page-empty-state';import { Sidebar } from '@/components/sidebar';+import { Button, LinkButton, buttonVariants } from '@/components/ui/button';+import { useAppContext } from '@/hooks/use-app-context';+import { cn } from '@/utils/cn';import { Outlet, createFileRoute, redirect } from '@tanstack/react-router';+import { ConstructionIcon } from 'lucide-react';export const Route = createFileRoute('/_app')({beforeLoad: async ({ context }) => {@@ -11,6 +16,28 @@ export const Route = createFileRoute('/_app')({});function AppLayout() {+ const { isMaintenance } = useAppContext();++ if (isMaintenance) {+ return (+ <FullPageEmptyState+ icon={ConstructionIcon}+ className="min-h-screen"+ title="Maintenance mode"+ description="We are currently performing maintenance on the system. Please check back later."+ >+ <a+ href="https://status.openpanel.dev/"+ className={cn(buttonVariants())}+ target="_blank"+ rel="noopener noreferrer"+ >+ Check out our status page+ </a>+ </FullPageEmptyState>+ );+ }+return (<div className="flex h-screen w-full"><Sidebar />diff --git a/apps/start/src/server/get-envs.ts b/apps/start/src/server/get-envs.tsindex f921daefb..55a8fb610 100644--- a/apps/start/src/server/get-envs.ts+++ b/apps/start/src/server/get-envs.ts@@ -8,6 +8,7 @@ export const getServerEnvs = createServerFn().handler(async () => {process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL,),isSelfHosted: process.env.SELF_HOSTED !== undefined,+ isMaintenance: process.env.MAINTENANCE === '1',};return envs;diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex bcce6a802..55652de8d 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -117,7 +117,7 @@ export async function bootWorkers() {const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({queue,concurrency,- logger: queueLogger,+ logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,blockingTimeoutSec: Number.parseFloat(process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',),diff --git a/apps/worker/src/jobs/cron.ping.ts b/apps/worker/src/jobs/cron.ping.tsindex 8252702ba..547ce83ae 100644--- a/apps/worker/src/jobs/cron.ping.ts+++ b/apps/worker/src/jobs/cron.ping.ts@@ -1,6 +1,10 @@import { TABLE_NAMES, chQuery } from '@openpanel/db';export async function ping() {+ if (process.env.DISABLE_PING) {+ return;+ }+const [res] = await chQuery<{ count: number }>(`SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`,);diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 83fac06a1..d24e554eb 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -96,7 +96,6 @@ export async function createSessionEnd(...payload,properties: {...payload.properties,- ...(session?.properties ?? {}),__bounce: session.is_bounce,},name: 'session_end',diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsindex 9b3a54c70..1ee1ebb90 100644--- a/apps/worker/src/jobs/events.incoming-events.test.ts+++ b/apps/worker/src/jobs/events.incoming-events.test.ts@@ -319,7 +319,6 @@ describe('incomingEvent', () => {utm_content: '',utm_medium: '',revenue: 0,- properties: {},project_id: projectId,device_id: 'last-device-123',profile_id: 'profile-123',diff --git a/packages/db/code-migrations/8-order-keys.ts b/packages/db/code-migrations/8-order-keys.tsnew file mode 100644index 000000000..cb2b5bf29--- /dev/null+++ b/packages/db/code-migrations/8-order-keys.ts@@ -0,0 +1,296 @@+import fs from 'node:fs';+import path from 'node:path';+import {+ chMigrationClient,+ createTable,+ moveDataBetweenTables,+ renameTable,+ runClickhouseMigrationCommands,+} from '../src/clickhouse/migration';+import { getIsCluster } from './helpers';++export async function up() {+ const isClustered = getIsCluster();++ const sqls: string[] = [];++ const eventTables = createTable({+ name: 'events_new_20251123',+ columns: [+ '`id` UUID DEFAULT generateUUIDv4()',+ '`name` LowCardinality(String)',+ '`sdk_name` LowCardinality(String)',+ '`sdk_version` LowCardinality(String)',+ '`device_id` String CODEC(ZSTD(3))',+ '`profile_id` String CODEC(ZSTD(3))',+ '`project_id` String CODEC(ZSTD(3))',+ '`session_id` String CODEC(LZ4)',+ '`path` String CODEC(ZSTD(3))',+ '`origin` String CODEC(ZSTD(3))',+ '`referrer` String CODEC(ZSTD(3))',+ '`referrer_name` String CODEC(ZSTD(3))',+ '`referrer_type` LowCardinality(String)',+ '`revenue` UInt64',+ '`duration` UInt64 CODEC(Delta(4), LZ4)',+ '`properties` Map(String, String) CODEC(ZSTD(3))',+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',+ '`country` LowCardinality(FixedString(2))',+ '`city` String',+ '`region` LowCardinality(String)',+ '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`os` LowCardinality(String)',+ '`os_version` LowCardinality(String)',+ '`browser` LowCardinality(String)',+ '`browser_version` LowCardinality(String)',+ '`device` LowCardinality(String)',+ '`brand` LowCardinality(String)',+ '`model` LowCardinality(String)',+ '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)',+ ],+ indices: [+ 'INDEX idx_name name TYPE bloom_filter GRANULARITY 1',+ "INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1",+ 'INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1',+ 'INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1',+ ],+ // New ORDER BY: project_id, toDate(created_at), created_at, name+ // Removed profile_id, added created_at for better ordering within same day+ orderBy: ['project_id', 'toDate(created_at)', 'created_at', 'name'],+ partitionBy: 'toYYYYMM(created_at)',+ settings: {+ index_granularity: 8192,+ // For lightweight updates+ enable_block_offset_column: 1,+ enable_block_number_column: 1,+ },+ distributionHash:+ 'cityHash64(project_id, toString(toStartOfHour(created_at)))',+ replicatedVersion: '1',+ isClustered,+ });++ // Step 1: Create temporary tables with new ORDER BY keys+ // Events table with new ORDER BY+ sqls.push(...eventTables);++ const sessionTables = createTable({+ name: 'sessions_new_20251123',+ engine: 'VersionedCollapsingMergeTree(sign, version)',+ columns: [+ '`id` String',+ '`project_id` String CODEC(ZSTD(3))',+ '`profile_id` String CODEC(ZSTD(3))',+ '`device_id` String CODEC(ZSTD(3))',+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',+ '`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',+ '`is_bounce` Bool',+ '`entry_origin` LowCardinality(String)',+ '`entry_path` String CODEC(ZSTD(3))',+ '`exit_origin` LowCardinality(String)',+ '`exit_path` String CODEC(ZSTD(3))',+ '`screen_view_count` Int32',+ '`revenue` Float64',+ '`event_count` Int32',+ '`duration` UInt32',+ '`country` LowCardinality(FixedString(2))',+ '`region` LowCardinality(String)',+ '`city` String',+ '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',+ '`device` LowCardinality(String)',+ '`brand` LowCardinality(String)',+ '`model` LowCardinality(String)',+ '`browser` LowCardinality(String)',+ '`browser_version` LowCardinality(String)',+ '`os` LowCardinality(String)',+ '`os_version` LowCardinality(String)',+ '`utm_medium` String CODEC(ZSTD(3))',+ '`utm_source` String CODEC(ZSTD(3))',+ '`utm_campaign` String CODEC(ZSTD(3))',+ '`utm_content` String CODEC(ZSTD(3))',+ '`utm_term` String CODEC(ZSTD(3))',+ '`referrer` String CODEC(ZSTD(3))',+ '`referrer_name` String CODEC(ZSTD(3))',+ '`referrer_type` LowCardinality(String)',+ '`sign` Int8',+ '`version` UInt64',+ ],+ // New ORDER BY: project_id, toDate(created_at), created_at, id+ // Removed profile_id, reordered to match query patterns (date first, then id)+ orderBy: ['project_id', 'toDate(created_at)', 'created_at'],+ partitionBy: 'toYYYYMM(created_at)',+ settings: {+ index_granularity: 8192,+ },+ distributionHash:+ 'cityHash64(project_id, toString(toStartOfHour(created_at)))',+ replicatedVersion: '1',+ isClustered,+ });++ // Sessions table with new ORDER BY+ sqls.push(...sessionTables);++ const firstEventDateResponse = await chMigrationClient.query({+ query: 'SELECT min(created_at) as created_at FROM events',+ format: 'JSONEachRow',+ });+ const firstEventDateJson = await firstEventDateResponse.json<{+ created_at: string;+ }>();+ if (firstEventDateJson[0]?.created_at) {+ const firstEventDate = new Date(firstEventDateJson[0]?.created_at);+ // Step 2: Copy data from old tables to new tables (partitioned by month for efficiency)+ // Set endDate to first of next month to ensure we capture all data in the current month+ const endDate = new Date();+ endDate.setMonth(endDate.getMonth() + 1);+ endDate.setDate(1);++ sqls.push(+ ...moveDataBetweenTables({+ from: 'events',+ to: 'events_new_20251123',+ batch: {+ startDate: firstEventDate,+ endDate: endDate,+ column: 'toDate(created_at)',+ interval: 'month',+ transform: (date: Date) => {+ const year = date.getFullYear();+ const month = String(date.getMonth() + 1).padStart(2, '0');+ return `${year}-${month}-01`;+ },+ },+ }),+ );+ }++ const firstSessionDateResponse = await chMigrationClient.query({+ query: 'SELECT min(created_at) as created_at FROM sessions',+ format: 'JSONEachRow',+ });+ const firstSessionDateJson = await firstSessionDateResponse.json<{+ created_at: string;+ }>();++ if (firstSessionDateJson[0]?.created_at) {+ const firstSessionDate = new Date(+ firstSessionDateJson[0]?.created_at ?? '',+ );+ // Set endDate to first of next month to ensure we capture all data in the current month+ const endDate = new Date();+ endDate.setMonth(endDate.getMonth() + 1);+ endDate.setDate(1);++ sqls.push(+ ...moveDataBetweenTables({+ from: 'sessions',+ to: 'sessions_new_20251123',+ columns: [+ 'id',+ 'project_id',+ 'profile_id',+ 'device_id',+ 'created_at',+ 'ended_at',+ 'is_bounce',+ 'entry_origin',+ 'entry_path',+ 'exit_origin',+ 'exit_path',+ 'screen_view_count',+ 'revenue',+ 'event_count',+ 'duration',+ 'country',+ 'region',+ 'city',+ 'longitude',+ 'latitude',+ 'device',+ 'brand',+ 'model',+ 'browser',+ 'browser_version',+ 'os',+ 'os_version',+ 'utm_medium',+ 'utm_source',+ 'utm_campaign',+ 'utm_content',+ 'utm_term',+ 'referrer',+ 'referrer_name',+ 'referrer_type',+ 'sign',+ 'version',+ ],+ batch: {+ startDate: firstSessionDate,+ endDate: endDate,+ column: 'toDate(created_at)',+ interval: 'month',+ transform: (date: Date) => {+ const year = date.getFullYear();+ const month = String(date.getMonth() + 1).padStart(2, '0');+ return `${year}-${month}-01`;+ },+ },+ }),+ );+ }++ sqls.push(+ ...renameTable({ from: 'events', to: 'events_20251123', isClustered }),+ );+ sqls.push(+ ...renameTable({ from: 'sessions', to: 'sessions_20251123', isClustered }),+ );++ if (isClustered && sessionTables[1] && eventTables[1]) {+ sqls.push(+ // Drop temporary DISTRIBUTED tables (will be recreated)+ `DROP TABLE IF EXISTS events_new_20251123 ON CLUSTER '{cluster}'`,+ `DROP TABLE IF EXISTS sessions_new_20251123 ON CLUSTER '{cluster}'`,+ // Rename new tables to correct names+ `RENAME TABLE events_new_20251123_replicated TO events_replicated ON CLUSTER '{cluster}'`,+ `RENAME TABLE sessions_new_20251123_replicated TO sessions_replicated ON CLUSTER '{cluster}'`,+ // Create new distributed tables+ eventTables[1].replaceAll('events_new_20251123', 'events'), // creates a new distributed table+ sessionTables[1].replaceAll('sessions_new_20251123', 'sessions'), // creates a new distributed table+ );+ } else {+ sqls.push(+ ...renameTable({+ from: 'events_new_20251123',+ to: 'events',+ isClustered,+ }),+ );+ sqls.push(+ ...renameTable({+ from: 'sessions_new_20251123',+ to: 'sessions',+ isClustered,+ }),+ );+ }++ fs.writeFileSync(+ path.join(__filename.replace('.ts', '.sql')),+ sqls+ .map((sql) =>+ sql+ .trim()+ .replace(/;$/, '')+ .replace(/\n{2,}/g, '\n')+ .concat(';'),+ )+ .join('\n\n---\n\n'),+ );++ if (!process.argv.includes('--dry')) {+ await runClickhouseMigrationCommands(sqls);+ }+}diff --git a/packages/db/code-migrations/helpers.ts b/packages/db/code-migrations/helpers.tsindex 128a3eade..38e060d7e 100644--- a/packages/db/code-migrations/helpers.ts+++ b/packages/db/code-migrations/helpers.ts@@ -30,3 +30,7 @@ export function getIsSelfHosting() {export function getIsDry() {return process.argv.includes('--dry');}++export function getShouldIgnoreRecord() {+ return process.argv.includes('--no-record');+}diff --git a/packages/db/code-migrations/migrate.ts b/packages/db/code-migrations/migrate.tsindex 5f3a0e653..3f4a5eac0 100644--- a/packages/db/code-migrations/migrate.ts+++ b/packages/db/code-migrations/migrate.ts@@ -10,6 +10,7 @@ import {getIsCluster,getIsDry,getIsSelfHosting,+ getShouldIgnoreRecord,printBoxMessage,} from './helpers';@@ -55,8 +56,12 @@ async function migrate() {]);if (!getIsSelfHosting()) {- printBoxMessage('🕒 Migrations starts in 10 seconds', []);- await new Promise((resolve) => setTimeout(resolve, 10000));+ if (!getIsDry()) {+ printBoxMessage('🕒 Migrations starts in 10 seconds', []);+ await new Promise((resolve) => setTimeout(resolve, 10000));+ } else {+ printBoxMessage('🕒 Migrations starts now (dry run)', []);+ }}if (migration) {@@ -81,7 +86,7 @@ async function runMigration(migrationsDir: string, file: string) {try {const migration = await import(path.join(migrationsDir, file));await migration.up();- if (!getIsDry()) {+ if (!getIsDry() && !getShouldIgnoreRecord()) {await db.codeMigration.upsert({where: {name: file,diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.tsindex 4d82eb53f..fad5ee097 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/db/src/buffers/session-buffer.ts@@ -1,6 +1,5 @@import { type Redis, getRedisCache } from '@openpanel/redis';-import { toDots } from '@openpanel/common';import { getSafeJson } from '@openpanel/json';import { assocPath, clone } from 'ramda';import { TABLE_NAMES, ch } from '../clickhouse/client';@@ -91,10 +90,6 @@ export class SessionBuffer extends BaseBuffer {session: newSession,});}- newSession.properties = toDots({- ...(event.properties || {}),- ...(newSession.properties || {}),- });const addedRevenue = event.name === 'revenue' ? (event.revenue ?? 0) : 0;newSession.revenue = (newSession.revenue ?? 0) + addedRevenue;@@ -168,7 +163,6 @@ export class SessionBuffer extends BaseBuffer {: '',sign: 1,version: 1,- properties: toDots(event.properties || {}),},];}diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.tsindex 90c8a33ce..2b2241d4f 100644--- a/packages/db/src/clickhouse/migration.ts+++ b/packages/db/src/clickhouse/migration.ts@@ -217,6 +217,7 @@ export function moveDataBetweenTables({from,to,batch,+ columns,}: {from: string;to: string;@@ -227,11 +228,15 @@ export function moveDataBetweenTables({endDate?: Date;startDate?: Date;};+ columns?: string[];}): string[] {const sqls: string[] = [];+ // Build the SELECT clause+ const selectClause = columns && columns.length > 0 ? columns.join(', ') : '*';+if (!batch) {- return [`INSERT INTO ${to} SELECT * FROM ${from}`];+ return [`INSERT INTO ${to} SELECT ${selectClause} FROM ${from}`];}// Start from today and go back 3 years@@ -247,32 +252,109 @@ export function moveDataBetweenTables({let currentDate = endDate;const interval = batch.interval || 'day';- while (currentDate > startDate) {+ // Helper function to get the start of the week (Monday) for a given date+ const getWeekStart = (date: Date): Date => {+ const d = new Date(date);+ const day = d.getDay();+ const diff = d.getDate() - day + (day === 0 ? -6 : 1); // Adjust to Monday+ d.setDate(diff);+ d.setHours(0, 0, 0, 0); // Normalize to start of day+ return d;+ };++ // Helper function to compare dates based on interval+ const shouldContinue = (+ current: Date,+ start: Date,+ intervalType: string,+ ): boolean => {+ if (intervalType === 'month') {+ // For months, compare by year and month+ // Continue if current month is >= start month+ const currentYear = current.getFullYear();+ const currentMonth = current.getMonth();+ const startYear = start.getFullYear();+ const startMonth = start.getMonth();+ return (+ currentYear > startYear ||+ (currentYear === startYear && currentMonth >= startMonth)+ );+ }+ if (intervalType === 'week') {+ // For weeks, compare by week start dates+ const currentWeekStart = getWeekStart(current);+ const startWeekStart = getWeekStart(start);+ return currentWeekStart >= startWeekStart;+ }+ return current > start;+ };++ while (shouldContinue(currentDate, startDate, interval)) {const previousDate = new Date(currentDate);switch (interval) {case 'month':previousDate.setMonth(previousDate.getMonth() - 1);+ // If we've gone below startDate's month, adjust to start of startDate's month+ // This ensures we generate SQL for the month containing startDate+ if (+ previousDate.getFullYear() < startDate.getFullYear() ||+ (previousDate.getFullYear() === startDate.getFullYear() &&+ previousDate.getMonth() < startDate.getMonth())+ ) {+ previousDate.setFullYear(startDate.getFullYear());+ previousDate.setMonth(startDate.getMonth());+ previousDate.setDate(1);+ }break;- case 'week':+ case 'week': {previousDate.setDate(previousDate.getDate() - 7);- // Ensure we don't go below startDate- if (previousDate < startDate) {- previousDate.setTime(startDate.getTime());+ // If we've gone below startDate's week, adjust to start of startDate's week+ const startWeekStart = getWeekStart(startDate);+ const prevWeekStart = getWeekStart(previousDate);+ if (prevWeekStart < startWeekStart) {+ previousDate.setTime(startWeekStart.getTime());}break;+ }// daydefault:previousDate.setDate(previousDate.getDate() - 1);break;}+ // For monthly/weekly intervals with transform, upperBoundDate should be currentDate+ // because currentDate already represents the start of the period we're processing+ // The WHERE clause uses > previousDate AND <= currentDate to get exactly one period+ let upperBoundDate = currentDate;+ // Don't exceed the endDate+ if (upperBoundDate > endDate) {+ upperBoundDate = endDate;+ }+const sql = `INSERT INTO ${to}- SELECT * FROM ${from}+ SELECT ${selectClause} FROM ${from}WHERE ${batch.column} > '${batch.transform ? batch.transform(previousDate) : formatClickhouseDate(previousDate, true)}'- AND ${batch.column} <= '${batch.transform ? batch.transform(currentDate) : formatClickhouseDate(currentDate, true)}'`;+ AND ${batch.column} <= '${batch.transform ? batch.transform(upperBoundDate) : formatClickhouseDate(upperBoundDate, true)}'`;sqls.push(sql);+ // For monthly/weekly intervals, stop if we've reached the start period+ if (interval === 'month') {+ const prevYear = previousDate.getFullYear();+ const prevMonth = previousDate.getMonth();+ const startYear = startDate.getFullYear();+ const startMonth = startDate.getMonth();+ if (prevYear === startYear && prevMonth === startMonth) {+ break;+ }+ } else if (interval === 'week') {+ const prevWeekStart = getWeekStart(previousDate);+ const startWeekStart = getWeekStart(startDate);+ if (prevWeekStart.getTime() === startWeekStart.getTime()) {+ break;+ }+ }+currentDate = previousDate;}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 832711ff4..02033bddc 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -131,7 +131,6 @@ export function transformSessionToEvent(duration: 0,revenue: session.revenue,properties: {- ...session.properties,is_bounce: session.is_bounce,__query: {utm_medium: session.utm_medium,@@ -631,8 +630,7 @@ export async function getEventList(options: GetEventListOptions) {}}- sb.orderBy.created_at =- 'toDate(created_at) DESC, created_at DESC, profile_id DESC, name DESC';+ sb.orderBy.created_at = 'created_at DESC';if (custom) {custom(sb);diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex aa9fe2fbe..0d39460a5 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -1,6 +1,5 @@import { cacheable } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';-import { uniq } from 'ramda';import sqlstring from 'sqlstring';import {TABLE_NAMES,@@ -53,7 +52,6 @@ export type IClickhouseSession = {revenue: number;sign: 1 | 0;version: number;- properties: Record<string, string>;};export interface IServiceSession {@@ -92,7 +90,6 @@ export interface IServiceSession {utmContent: string;utmTerm: string;revenue: number;- properties: Record<string, string>;profile?: IServiceProfile;}@@ -144,7 +141,6 @@ export function transformSession(session: IClickhouseSession): IServiceSession {utmContent: session.utm_content,utmTerm: session.utm_term,revenue: session.revenue,- properties: session.properties,profile: undefined,};}@@ -200,12 +196,13 @@ export async function getSessionList({if (cursor) {const cAt = sqlstring.escape(cursor.createdAt);+ // TODO: remove id from cursorconst cId = sqlstring.escape(cursor.id);- sb.where.cursor = `(created_at < toDateTime64(${cAt}, 3) OR (created_at = toDateTime64(${cAt}, 3) AND id < ${cId}))`;+ sb.where.cursor = `created_at < toDateTime64(${cAt}, 3)`;sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`;- sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';+ sb.orderBy.created_at = 'created_at DESC';} else {- sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';+ sb.orderBy.created_at = 'created_at DESC';sb.where.created_at = `created_at > now() - INTERVAL ${dateIntervalInDays} DAY`;}diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex e1e64645e..d21bf4e78 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -146,7 +146,7 @@ export const eventsGroupQueues = Array.from({}).map((_, index, list) =>new GroupQueue<EventsQueuePayloadIncomingEvent['payload']>({- logger: queueLogger,+ logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,namespace: getQueueName(list.length === 1 ? 'group_events' : `group_events_${index}`,),diff --git a/self-hosting/docker-compose.template.yml b/self-hosting/docker-compose.template.ymlindex 64c16c2a7..12d9b828a 100644--- a/self-hosting/docker-compose.template.yml+++ b/self-hosting/docker-compose.template.yml@@ -1,12 +1,12 @@-version: '3'+version: "3"services:op-proxy:image: caddy:2-alpinerestart: alwaysports:- - '80:80'- - '443:443'+ - "80:80"+ - "443:443"volumes:- op-proxy-data:/data- op-proxy-config:/config@@ -28,7 +28,7 @@ services:volumes:- op-db-data:/var/lib/postgresql/datahealthcheck:- test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]+ test: ["CMD-SHELL", "pg_isready -U postgres"]interval: 10stimeout: 5sretries: 5@@ -49,9 +49,9 @@ services:restart: alwaysvolumes:- op-kv-data:/data- command: [ 'redis-server', '--maxmemory-policy', 'noeviction' ]+ command: ["redis-server", "--maxmemory-policy", "noeviction"]healthcheck:- test: [ 'CMD-SHELL', 'redis-cli ping' ]+ test: ["CMD-SHELL", "redis-cli ping"]interval: 10stimeout: 5sretries: 5@@ -65,7 +65,7 @@ services:# - 6379:6379op-ch:- image: clickhouse/clickhouse-server:24.3.2-alpine+ image: clickhouse/clickhouse-server:25.10.2.65restart: alwaysvolumes:- op-ch-data:/var/lib/clickhouse@@ -74,7 +74,7 @@ services:- ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro- ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:rohealthcheck:- test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ]+ test: ["CMD-SHELL", 'clickhouse-client --query "SELECT 1"']interval: 10stimeout: 5sretries: 5@@ -99,7 +99,7 @@ services:pnpm start"healthcheck:- test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ]+ test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]interval: 10stimeout: 5sretries: 5@@ -127,7 +127,8 @@ services:env_file:- .envhealthcheck:- test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/api/healthcheck || exit 1' ]+ test:+ ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]interval: 10stimeout: 5sretries: 5@@ -146,7 +147,7 @@ services:env_file:- .envhealthcheck:- test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ]+ test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]interval: 10stimeout: 5sretries: 5@@ -171,4 +172,4 @@ volumes:op-proxy-data:driver: localop-proxy-config:- driver: local\ No newline at end of file+ driver: local