OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
improvements while testing
Carl-Gerhard Lindesvärd41e46570b7c7
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 634fd054e..1123a3e35 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -7,7 +7,9 @@ import { assocPath, pathOr } from 'ramda';import { generateDeviceId } from '@openpanel/common';import {ch,+ chQuery,getProfileById,+ getProfileId,getSalts,TABLE_NAMES,upsertProfile,@@ -31,6 +33,23 @@ export async function handler(const ip = getClientIp(request)!;const ua = request.headers['user-agent']!;const projectId = request.client?.projectId;+ const profileId =+ projectId && request.body.payload.profileId+ ? await getProfileId({+ projectId,+ profileId: request.body.payload.profileId,+ })+ : undefined;++ if (profileId) {+ request.body.payload.profileId = profileId;+ }++ console.log(+ '> Request',+ request.body.type,+ JSON.stringify(request.body.payload, null, 2)+ );if (!projectId) {reply.status(400).send('missing origin');diff --git a/apps/docs/src/pages/docs/script.mdx b/apps/docs/src/pages/docs/script.mdxindex e11f651bc..c706b6ef3 100644--- a/apps/docs/src/pages/docs/script.mdx+++ b/apps/docs/src/pages/docs/script.mdx@@ -176,8 +176,18 @@ declare global {#### Strict typing (from sdk)+<Steps>+##### Step 1: Install the SDK++```bash+npm install @openpanel/web+```++##### Step 2: Create a type definition file+Create a `op.d.ts`file and paste the following code:```ts filename="op.d.ts"/// <reference types="@openpanel/web" />-```\ No newline at end of file+```+</Steps>\ No newline at end of filediff --git a/apps/public/public/op1.js b/apps/public/public/op1.jsnew file mode 100644index 000000000..fc4eee369--- /dev/null+++ b/apps/public/public/op1.js@@ -0,0 +1,2 @@+"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(t<this.maxRetries){let n=this.initialRetryDelay*Math.pow(2,t);return await new Promise(a=>setTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},c=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"1.0.0-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.disable||this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:e.payload.profileId??this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var p=class extends c{constructor(i){super({...i,sdk:"web",sdkVersion:"1.0.0-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-track")?s:n?.getAttribute("data-track")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-track"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let h=a.getAttribute("data-track");h&&super.track(h,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new p(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`OpenPanel: ${t} is not a function`)},e.openpanel=i}})(window);})();+//# sourceMappingURL=tracker.global.js.map\ No newline at end of filediff --git a/apps/public/public/tracker.js b/apps/public/public/tracker.jsdeleted file mode 100644index 070a2779f..000000000--- a/apps/public/public/tracker.js+++ /dev/null@@ -1,2 +0,0 @@-"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(t<this.maxRetries){let n=this.initialRetryDelay*Math.pow(2,t);return await new Promise(a=>setTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},h=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"0.0.11-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var c=class extends h{constructor(i){super({...i,sdk:"web",sdkVersion:"0.0.11-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-event")?s:n?.getAttribute("data-event")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-event"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let p=a.getAttribute("data-event");p&&super.track(p,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new c(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`op.js: ${t} is not a function`)},e.openpanel=i}})(window);})();-//# sourceMappingURL=tracker.global.js.map\ No newline at end of filediff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex 100db59f3..bf2194cfa 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -18,7 +18,7 @@ import type {EventsQueuePayloadCreateSessionEnd,EventsQueuePayloadIncomingEvent,} from '@openpanel/queue';-import { getRedisQueue } from '@openpanel/redis';+import { cacheable, getRedisQueue } from '@openpanel/redis';const GLOBAL_PROPERTIES = ['__path', '__referrer'];const SESSION_TIMEOUT = 1000 * 60 * 30;@@ -47,10 +47,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {};// this will get the profileId from the alias table if it exists- const profileId = await getProfileId({- projectId,- profileId: body.profileId,- });+ const profileId = body.profileId ?? '';const createdAt = new Date(body.timestamp);const url = getProperty('__path');const { path, hash, query, origin } = parsePath(url);@@ -259,29 +256,3 @@ async function getSessionEnd({// Create sessionreturn null;}--async function getProfileId({- profileId,- projectId,-}: {- profileId: string | undefined;- projectId: string;-}) {- if (!profileId) {- return '';- }-- const res = await chQuery<{- alias: string;- profile_id: string;- project_id: string;- }>(- `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')`- );-- if (res[0]) {- return res[0].profile_id;- }-- return profileId;-}diff --git a/packages/db/clickhouse_init.sql b/packages/db/clickhouse_init.sqlindex 80075ee88..69d9ddb4f 100644--- a/packages/db/clickhouse_init.sql+++ b/packages/db/clickhouse_init.sql@@ -60,6 +60,15 @@ CREATE TABLE IF NOT EXISTS openpanel.profiles (ORDER BY(id) SETTINGS index_granularity = 8192;+CREATE TABLE IF NOT EXISTS openpanel.profile_aliases (+ `project_id` String,+ `profile_id` String,+ `alias` String,+ `created_at` DateTime+) ENGINE = MergeTree+ORDER BY+ (project_id, profile_id, alias) SETTINGS index_granularity = 8192;+--- Materialized views (DAU)CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)ORDER BYdiff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 828c2da9d..1184d7ebc 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -111,10 +111,13 @@ export async function chQuery<T extends Record<string, any>>(return (await chQueryWithMeta<T>(query)).data;}-export function formatClickhouseDate(_date: Date | string, skipTime = false) {+export function formatClickhouseDate(+ _date: Date | string,+ skipTime = false+): string {const date = typeof _date === 'string' ? new Date(_date) : _date;if (skipTime) {- return date.toISOString().split('T')[0];+ return date.toISOString().split('T')[0]!;}return date.toISOString().replace('T', ' ').replace(/Z+$/, '');}diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 74f2d6e9c..4f2b43cb0 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -1,6 +1,7 @@import { escape } from 'sqlstring';import { toObject } from '@openpanel/common';+import { cacheable } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';import { profileBuffer } from '../buffers';@@ -191,3 +192,31 @@ export async function upsertProfile({is_external: isExternal,});}++export async function getProfileId({+ profileId,+ projectId,+}: {+ profileId: string | undefined;+ projectId: string;+}) {+ if (!profileId) {+ return '';+ }++ const res = await chQuery<{+ alias: string;+ profile_id: string;+ project_id: string;+ }>(+ `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')`+ );++ if (res[0]) {+ return res[0].profile_id;+ }++ return profileId;+}++export const getProfileIdCached = cacheable(getProfileId, 60 * 30);diff --git a/packages/redis/cachable.ts b/packages/redis/cachable.tsindex 4aa663dd9..f2c03429e 100644--- a/packages/redis/cachable.ts+++ b/packages/redis/cachable.ts@@ -2,7 +2,7 @@ import { getRedisCache } from './redis';export function cacheable<T extends (...args: any) => any>(fn: T,- expire: number+ expireInSec: number) {return async function (...args: Parameters<T>@@ -20,7 +20,7 @@ export function cacheable<T extends (...args: any) => any>(const result = await fn(...(args as any));if (result !== undefined || result !== null) {- getRedisCache().setex(key, expire, JSON.stringify(result));+ getRedisCache().setex(key, expireInSec, JSON.stringify(result));}return result;diff --git a/packages/sdks/sdk/src/index.ts b/packages/sdks/sdk/src/index.tsindex 66df67e0c..12d318ee8 100644--- a/packages/sdks/sdk/src/index.ts+++ b/packages/sdks/sdk/src/index.ts@@ -67,7 +67,7 @@ export type OpenPanelOptions = {sdkVersion?: string;waitForProfile?: boolean;filter?: (payload: TrackHandlerPayload) => boolean;- disable?: boolean;+ disabled?: boolean;};export class OpenPanel {@@ -106,7 +106,7 @@ export class OpenPanel {}async send(payload: TrackHandlerPayload) {- if (this.options.disable) {+ if (this.options.disabled) {return Promise.resolve();}