📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
fix analytics funnel and goal parsing edge cases
Blue Moused76adcc20e84
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.tsindex 8f27d2b34..01f799c6f 100644--- a/backend/apps/cloud/src/analytics/analytics.service.ts+++ b/backend/apps/cloud/src/analytics/analytics.service.ts@@ -1891,14 +1891,19 @@ export class AnalyticsService {}>(),)+ const createDetailsBucket = () =>+ Object.create(null) as Record<number, Record<string, number>>+ const createValueBucket = () =>+ Object.create(null) as Record<string, number>+const details: FunnelStepDetails = {- countries: {},- devices: {},- browsers: {},- sources: {},- campaigns: {},- pages: {},- profileTypes: {},+ countries: createDetailsBucket(),+ devices: createDetailsBucket(),+ browsers: createDetailsBucket(),+ sources: createDetailsBucket(),+ campaigns: createDetailsBucket(),+ pages: createDetailsBucket(),+ profileTypes: createDetailsBucket(),}for (const row of data) {@@ -1906,7 +1911,7 @@ export class AnalyticsService {const type = row.type as keyof FunnelStepDetailsif (details[type]) {- if (!details[type][step]) details[type][step] = {}+ if (!details[type][step]) details[type][step] = createValueBucket()details[type][step][row.val] = row.cnt}}diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.tsindex 98aad063e..765528149 100644--- a/backend/apps/cloud/src/goal/goal.controller.ts+++ b/backend/apps/cloud/src/goal/goal.controller.ts@@ -714,7 +714,7 @@ export class GoalController {UNION ALL- SELECT 'profileTypes' AS kind, if(startsWith(profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 'identified', 'anonymous') AS val, count() AS cnt+ SELECT 'profileTypes' AS kind, if(startsWith(profileId, {profilePrefixUser:String}), 'identified', 'anonymous') AS val, count() AS cntFROM session_infoWHERE profileId IS NOT NULL AND profileId != ''GROUP BY val@@ -726,7 +726,13 @@ export class GoalController {const { data } = await clickhouse.query({query,- query_params: { ...queryParams, pid: projectId, groupFrom, groupTo },+ query_params: {+ ...queryParams,+ pid: projectId,+ groupFrom,+ groupTo,+ profilePrefixUser: AnalyticsService.PROFILE_PREFIX_USER,+ },}).then((resultSet) =>resultSet.json<{diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.tsindex 1541627da..80e461c98 100644--- a/backend/apps/community/src/analytics/analytics.service.ts+++ b/backend/apps/community/src/analytics/analytics.service.ts@@ -1772,6 +1772,7 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND type IN ('pageview', 'custom_event')AND psid != 0+ AND psid IN (SELECT psid FROM funnel_sessions)AND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psid)@@ -2056,9 +2057,7 @@ export class AnalyticsService {median: null,p75: null,}- const pageParams: Record<string, string | number> = {- steps: pages.length,- }+ const pageParams: Record<string, string> = {}const sessionFiltersQuery = filtersQuery? `AND psid IN (SELECT DISTINCT psid@@ -2071,21 +2070,57 @@ export class AnalyticsService {)`: ''- const pagesStr = _join(- _map(pages, (value, index) => {- pageParams[`v${index}`] = value- return `value={v${index}:String}`+ pages.forEach((value, index) => {+ pageParams[`v${index}`] = value+ })++ const funnelStepCtes = _join(+ _map(pages, (_value, index) => {+ const step = index + 1++ if (step === 1) {+ return `+ funnel_step_1 AS (+ SELECT+ psid,+ created AS firstStepAt,+ created AS step1At,+ eventIndex AS step1Index+ FROM funnel_events+ WHERE value = {v0:String}+ )+ `+ }++ const previousStep = step - 1++ return `+ funnel_step_${step} AS (+ SELECT+ prev.psid,+ prev.firstStepAt,+ min(e.created) AS step${step}At,+ min(e.eventIndex) AS step${step}Index+ FROM funnel_step_${previousStep} prev+ INNER JOIN funnel_events e ON e.psid = prev.psid+ AND e.value = {v${index}:String}+ AND e.eventIndex > prev.step${previousStep}Index+ AND e.created <= addSeconds(prev.firstStepAt, 86400)+ GROUP BY prev.psid, prev.firstStepAt+ )+ `}),',',)+ const finalStep = pages.lengthconst query = `- WITH funnel_sessions AS (+ WITH funnel_events AS (SELECTpsid,- minIf(created, value = {v0:String}) AS firstStepAt,- maxIf(created, value = {v${pages.length - 1}:String}) AS conversionAt,- CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level+ value,+ created,+ row_number() OVER (PARTITION BY psid ORDER BY created, value) AS eventIndexFROM (SELECTpsid,@@ -2098,23 +2133,45 @@ export class AnalyticsService {${sessionFiltersQuery}AND created BETWEEN {groupFrom:String} AND {groupTo:String})+ ),+ ${funnelStepCtes},+ funnel_sessions AS (+ SELECT+ psid,+ argMin(firstStepAt, conversionAt) AS firstStepAt,+ min(conversionAt) AS conversionAt+ FROM (+ SELECT+ psid,+ firstStepAt,+ step${finalStep}At AS conversionAt+ FROM funnel_step_${finalStep}+ )GROUP BY psid- HAVING level = {steps:UInt32}),session_starts AS (- SELECT psid, min(firstSeen) AS sessionStart- FROM sessions FINAL- WHERE pid = {pid:FixedString(12)}- GROUP BY psid+ SELECT fs.psid, min(s.firstSeen) AS sessionStart+ FROM funnel_sessions fs+ INNER JOIN (+ SELECT psid, firstSeen, lastSeen+ FROM sessions FINAL+ WHERE pid = {pid:FixedString(12)}+ ) s ON s.psid = fs.psid+ WHERE s.firstSeen <= fs.firstStepAt+ AND s.lastSeen >= fs.conversionAt+ GROUP BY fs.psid),first_pages AS (- SELECT psid, min(created) AS firstPageAt- FROM events- WHERE pid = {pid:FixedString(12)}- AND type = 'pageview'- AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- GROUP BY psid+ SELECT fs.psid, min(e.created) AS firstPageAt+ FROM funnel_sessions fs+ INNER JOIN session_starts ss ON fs.psid = ss.psid+ INNER JOIN events e ON e.psid = fs.psid+ WHERE e.pid = {pid:FixedString(12)}+ AND e.type = 'pageview'+ AND e.psid != 0+ AND e.created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND e.created BETWEEN ss.sessionStart AND fs.conversionAt+ GROUP BY fs.psid)SELECTround(avgOrNull(if(sessionStart > toDateTime(0) AND sessionStart <= conversionAt, dateDiff('second', sessionStart, conversionAt), NULL)), 2) AS averageSession,diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.tsindex 336e55e37..13aab725d 100644--- a/backend/apps/community/src/goal/goal.controller.ts+++ b/backend/apps/community/src/goal/goal.controller.ts@@ -634,7 +634,7 @@ export class GoalController {UNION ALL- SELECT 'profileTypes' AS kind, if(startsWith(profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 'identified', 'anonymous') AS val, count() AS cnt+ SELECT 'profileTypes' AS kind, if(startsWith(profileId, {profilePrefixUser:String}), 'identified', 'anonymous') AS val, count() AS cntFROM session_infoWHERE profileId IS NOT NULL AND profileId != ''GROUP BY val@@ -646,7 +646,13 @@ export class GoalController {const { data } = await clickhouse.query({query,- query_params: { ...queryParams, pid: projectId, groupFrom, groupTo },+ query_params: {+ ...queryParams,+ pid: projectId,+ groupFrom,+ groupTo,+ profilePrefixUser: AnalyticsService.PROFILE_PREFIX_USER,+ },}).then((resultSet) =>resultSet.json<{diff --git a/web/app/pages/Project/View/components/FilterRowsEditor.tsx b/web/app/pages/Project/View/components/FilterRowsEditor.tsxindex 4c675e3b9..6493dff14 100644--- a/web/app/pages/Project/View/components/FilterRowsEditor.tsx+++ b/web/app/pages/Project/View/components/FilterRowsEditor.tsx@@ -67,13 +67,48 @@ const OPERATORS: { value: FilterOperator; labelKey: string }[] = [{ value: 'notContains', labelKey: 'project.contains.not' },]-const createEmptyFilterRow = (): FilterRow => ({+const getMetadataColumnBase = (column: string) => {+ if (column === 'ev:key' || column.startsWith('ev:key:')) return 'ev:key'+ if (column === 'tag:key' || column.startsWith('tag:key:')) return 'tag:key'+ if (column === 'ev:value') return 'ev:value'+ if (column === 'tag:value') return 'tag:value'+ return column+}++const getOperatorsForColumn = (column: string) => {+ if (column === 'ev:key' || column === 'tag:key') {+ return OPERATORS.filter(+ (operator) => operator.value === 'is' || operator.value === 'isNot',+ )+ }++ return OPERATORS+}++const normalizeFilterOperator = (+ column: string,+ operator: FilterOperator,+): FilterOperator => {+ const operatorOptions = getOperatorsForColumn(column)++ return operatorOptions.some((option) => option.value === operator)+ ? operator+ : operatorOptions[0]?.value || 'is'+}++const createFilterRow = ({+ column = '',+ operator = 'is',+ value = '',+}: Partial<Omit<FilterRow, 'id'>> = {}): FilterRow => ({id: crypto.randomUUID(),- column: '',- operator: 'is',- value: '',+ column,+ operator: normalizeFilterOperator(column, operator),+ value,})+const createEmptyFilterRow = (): FilterRow => createFilterRow()+const operatorToFilter = (operator: FilterOperator,): { isExclusive: boolean; isContains: boolean } => {@@ -123,15 +158,14 @@ const filtersToRows = (filters: FilterType[] = []): FilterRow[] => {usedIndexes.add(versionIndex)return [- {- id: crypto.randomUUID(),+ createFilterRow({column: versionColumn,operator,value: createVersionValue(filter.filter,filters[versionIndex].filter,),- },+ }),]}}@@ -139,34 +173,15 @@ const filtersToRows = (filters: FilterType[] = []): FilterRow[] => {usedIndexes.add(index)return [- {- id: crypto.randomUUID(),+ createFilterRow({column: filter.column,operator: filterToOperator(filter),value: filter.filter,- },+ }),]})}-const getMetadataColumnBase = (column: string) => {- if (column === 'ev:key' || column.startsWith('ev:key:')) return 'ev:key'- if (column === 'tag:key' || column.startsWith('tag:key:')) return 'tag:key'- if (column === 'ev:value') return 'ev:value'- if (column === 'tag:value') return 'tag:value'- return column-}--const getOperatorsForColumn = (column: string) => {- if (column === 'ev:key' || column === 'tag:key') {- return OPERATORS.filter(- (operator) => operator.value === 'is' || operator.value === 'isNot',- )- }-- return OPERATORS-}-const InlineButton = ({text,onClick,@@ -413,7 +428,10 @@ const FilterRowsEditor = ({return}- const nextRows = filtersToRows(initialFilters)+ const nextRows = filtersToRows(initialFilters).map((row) => ({+ ...row,+ operator: normalizeFilterOperator(row.column, row.operator),+ }))setFilterRows(nextRows.length ? nextRows : [createEmptyFilterRow()])}, [active, initialFilters, resetKey])@@ -463,12 +481,7 @@ const FilterRowsEditor = ({) {fetchFilterValues(fieldValue)}- const operatorOptions = getOperatorsForColumn(fieldValue)- const operator = operatorOptions.some(- (option) => option.value === row.operator,- )- ? row.operator- : 'is'+ const operator = normalizeFilterOperator(fieldValue, row.operator)return { ...row, column: fieldValue, operator, value: '' }}diff --git a/web/app/routes/projects.$id.tsx b/web/app/routes/projects.$id.tsxindex 0a80bde55..a2e60a905 100644--- a/web/app/routes/projects.$id.tsx+++ b/web/app/routes/projects.$id.tsx@@ -62,6 +62,34 @@ import {hasAuthTokens,} from '~/utils/session.server'+const GOAL_CONDITION_RELATIONS = new Set(['AND', 'OR'])+const GOAL_CONDITION_EVENT_TYPES = new Set(['any', 'pageview', 'custom_event'])+const GOAL_CONDITION_OPERATORS = new Set([+ 'equals',+ 'not_equals',+ 'contains',+ 'not_contains',+ 'exists',+ 'not_exists',+])++type GoalMetadataFilter = {+ key: string+ value: string+}++type JsonFormFieldResult =+ | { ok: true; value: unknown }+ | { ok: false; error: string }++type GoalJsonFieldsResult =+ | {+ ok: true+ metadataFilters: GoalMetadataFilter[]+ conditions: unknown+ }+ | { ok: false; error: string }+function computeDateRangeForPeriod(period: string,from?: string,@@ -107,6 +135,87 @@ function computeDateRangeForPeriod(return compute ? compute() : null}+const parseJsonFormField = (+ formData: FormData,+ field: string,+ fallback: string,+): JsonFormFieldResult => {+ try {+ return {+ ok: true,+ value: JSON.parse(formData.get(field)?.toString() || fallback),+ }+ } catch {+ return { ok: false, error: `Invalid JSON in ${field}` }+ }+}++const isRecord = (value: unknown): value is Record<string, unknown> =>+ typeof value === 'object' && value !== null++const isGoalMetadataFilters = (value: unknown): value is GoalMetadataFilter[] =>+ Array.isArray(value) &&+ value.every(+ (filter) =>+ isRecord(filter) &&+ typeof filter.key === 'string' &&+ typeof filter.value === 'string',+ )++const isGoalConditions = (value: unknown) => {+ if (value === null) {+ return true+ }++ if (+ !isRecord(value) ||+ !GOAL_CONDITION_RELATIONS.has(String(value.relation))+ ) {+ return false+ }++ return (+ Array.isArray(value.conditions) &&+ value.conditions.every(+ (condition) =>+ isRecord(condition) &&+ GOAL_CONDITION_EVENT_TYPES.has(String(condition.eventType)) &&+ typeof condition.field === 'string' &&+ GOAL_CONDITION_OPERATORS.has(String(condition.operator)) &&+ (condition.value === undefined ||+ typeof condition.value === 'string') &&+ (condition.metadataKey === undefined ||+ typeof condition.metadataKey === 'string'),+ )+ )+}++const parseGoalJsonFields = (formData: FormData): GoalJsonFieldsResult => {+ const metadataFilters = parseJsonFormField(formData, 'metadataFilters', '[]')+ if (!metadataFilters.ok) {+ return metadataFilters+ }++ if (!isGoalMetadataFilters(metadataFilters.value)) {+ return { ok: false, error: 'Invalid metadataFilters' }+ }++ const conditions = parseJsonFormField(formData, 'conditions', 'null')+ if (!conditions.ok) {+ return conditions+ }++ if (!isGoalConditions(conditions.value)) {+ return { ok: false, error: 'Invalid conditions' }+ }++ return {+ ok: true,+ metadataFilters: metadataFilters.value,+ conditions: conditions.value,+ }+}+function formatDateForBackend(dateStr: string): string {if (!dateStr) return ''@@ -697,12 +806,14 @@ export async function action({ request, params }: ActionFunctionArgs) {const type = formData.get('type')?.toString() || 'pageview'const matchType = formData.get('matchType')?.toString() || 'exact'const value = formData.get('value')?.toString() || ''- const metadataFilters = JSON.parse(- formData.get('metadataFilters')?.toString() || '[]',- )- const conditions = JSON.parse(- formData.get('conditions')?.toString() || 'null',- )+ const goalJson = parseGoalJsonFields(formData)++ if (!goalJson.ok) {+ return data<ProjectViewActionData>(+ { intent, error: goalJson.error },+ { status: 400 },+ )+ }const result = await serverFetch(request, 'goal', {method: 'POST',@@ -712,8 +823,8 @@ export async function action({ request, params }: ActionFunctionArgs) {type,matchType,value,- metadataFilters,- conditions,+ metadataFilters: goalJson.metadataFilters,+ conditions: goalJson.conditions,},})@@ -736,16 +847,25 @@ export async function action({ request, params }: ActionFunctionArgs) {const type = formData.get('type')?.toString()const matchType = formData.get('matchType')?.toString()const value = formData.get('value')?.toString()- const metadataFilters = JSON.parse(- formData.get('metadataFilters')?.toString() || '[]',- )- const conditions = JSON.parse(- formData.get('conditions')?.toString() || 'null',- )+ const goalJson = parseGoalJsonFields(formData)++ if (!goalJson.ok) {+ return data<ProjectViewActionData>(+ { intent, error: goalJson.error },+ { status: 400 },+ )+ }const result = await serverFetch(request, `goal/${goalId}`, {method: 'PUT',- body: { name, type, matchType, value, metadataFilters, conditions },+ body: {+ name,+ type,+ matchType,+ value,+ metadataFilters: goalJson.metadataFilters,+ conditions: goalJson.conditions,+ },})if (result.error) {