OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore(dashboard,db): prepping for migration time
Carl-Gerhard Lindesvärdab8bce775293
diff --git a/apps/api/src/controllers/healthcheck.controller.ts b/apps/api/src/controllers/healthcheck.controller.tsindex 0e9d80943..a9a18957a 100644--- a/apps/api/src/controllers/healthcheck.controller.ts+++ b/apps/api/src/controllers/healthcheck.controller.ts@@ -21,7 +21,7 @@ export async function healthcheck(request: FastifyRequest,reply: FastifyReply,) {- const redisRes = await withTimings(getRedisCache().keys('keys op:buffer:*'));+ const redisRes = await withTimings(getRedisCache().keys('op:buffer:*'));const dbRes = await withTimings(db.project.findFirst());const queueRes = await withTimings(eventsQueue.getCompleted());const chRes = await withTimings(diff --git a/apps/dashboard/src/app/maintenance/page.tsx b/apps/dashboard/src/app/maintenance/page.tsxnew file mode 100644index 000000000..4532112d7--- /dev/null+++ b/apps/dashboard/src/app/maintenance/page.tsx@@ -0,0 +1,18 @@+import { CalendarCogIcon } from 'lucide-react';++export default function Maintenance() {+ return (+ <div className="h-screen w-full center-center overflow-hidden">+ <div className="relative z-10 col gap-2 center-center">+ <CalendarCogIcon className="size-32 mb-4 animate-wiggle text-def-300" />+ <div className="text-[150px] font-mono font-bold -mb-16 leading-[1] select-none pointer-events-none whitespace-nowrap bg-gradient-to-b from-def-300 to-def-100 bg-clip-text text-transparent">+ Oh no!+ </div>+ <h1 className="text-6xl font-bold">Maintenance</h1>+ <p className="text-xl text-muted-foreground">+ We're doing a planned maintenance. Please check back later.+ </p>+ </div>+ </div>+ );+}diff --git a/apps/dashboard/src/middleware.ts b/apps/dashboard/src/middleware.tsindex c907252f8..1f8e7f61d 100644--- a/apps/dashboard/src/middleware.ts+++ b/apps/dashboard/src/middleware.ts@@ -1,4 +1,5 @@import { clerkMiddleware, createRouteMatcher } from '@clerk/nextjs/server';+import { NextResponse } from 'next/server';// This example protects all routes including api/trpc routes// Please edit this to allow other routes to be public as needed.@@ -13,6 +14,9 @@ const isPublicRoute = createRouteMatcher([export default clerkMiddleware((auth, req) => {+ if (!process.env.MAINTENANCE_MODE && !req.url.includes('/maintenance')) {+ return NextResponse.redirect(new URL('/maintenance', req.url), 307);+ }if (!isPublicRoute(req)) {auth().protect();}diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex da33e596d..a7973069b 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -4,7 +4,12 @@ import { toDots } from '@openpanel/common';import { getRedisCache } from '@openpanel/redis';import { escape } from 'sqlstring';-import { TABLE_NAMES, ch, chQuery } from '../clickhouse-client';+import {+ TABLE_NAMES,+ ch,+ chQuery,+ formatClickhouseDate,+} from '../clickhouse-client';import { transformProfile } from '../services/profile.service';import type {IClickhouseProfile,@@ -23,6 +28,15 @@ export class ProfileBuffer extends RedisBuffer<BufferType> {super(TABLE_NAMES.profiles, BATCH_SIZE);}+ protected transformProfiles(profiles: IClickhouseProfile[]): BufferType[] {+ return profiles.map((profile) => ({+ ...profile,+ created_at: profile.created_at+ ? formatClickhouseDate(profile.created_at)+ : '',+ }));+ }+// this will do a couple of things:// - we slice the queue to maxBufferSize since this queries have a limit on character count// - check redis cache for profiles@@ -40,12 +54,14 @@ export class ProfileBuffer extends RedisBuffer<BufferType> {slicedQueue.filter((_, index) => !redisProfiles[index]),);- const toInsert = this.createProfileValues(+ const profiles = this.createProfileValues(slicedQueue,redisProfiles,dbProfiles,);+ const toInsert = this.transformProfiles(profiles);+if (toInsert.length > 0) {await this.updateRedisCache(toInsert);}diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 2d7413f8c..5b8b67a10 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -2,9 +2,12 @@ import type { ResponseJSON } from '@clickhouse/client';import { createClient } from '@clickhouse/client';import { escape } from 'sqlstring';+import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';import { createLogger } from '@openpanel/logger';import type { IInterval } from '@openpanel/validation';+export { createClient };+const logger = createLogger({ name: 'clickhouse' });export const TABLE_NAMES = {@@ -19,8 +22,7 @@ export const TABLE_NAMES = {cohort_events_mv: 'cohort_events_mv',};-export const originalCh = createClient({- url: process.env.CLICKHOUSE_URL,+export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {max_open_connections: 30,request_timeout: 30000,keep_alive: {@@ -33,14 +35,23 @@ export const originalCh = createClient({clickhouse_settings: {date_time_input_format: 'best_effort',},+};++export const originalCh = createClient({+ // TODO: remove this after migration+ url: process.env.CLICKHOUSE_URL_CLUSTER ?? process.env.CLICKHOUSE_URL,+ ...CLICKHOUSE_OPTIONS,});+const cleanQuery = (query: string) =>+ query.replace(/\n/g, '').replace(/\s+/g, ' ').trim();+export const ch = new Proxy(originalCh, {get(target, property, receiver) {if (property === 'insert' || property === 'query') {return async (...args: any[]) => {const childLogger = logger.child({- query: args[0].query,+ query: cleanQuery(args[0].query),property,});try {@@ -113,7 +124,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(};logger.info('query info', {- query,+ query: cleanQuery(query),rows: json.rows,stats: response.statistics,elapsed: Date.now() - start,