Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix analytics funnel and goal parsing edge cases

Blue Moused76adcc20e84
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 8f27d2b34..01f799c6f 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -1891,14 +1891,19 @@ export class AnalyticsService {
}>(),
)
+ const createDetailsBucket = () =>
+ Object.create(null) as Record<number, Record<string, number>>
+ const createValueBucket = () =>
+ Object.create(null) as Record<string, number>
+
const details: FunnelStepDetails = {
- countries: {},
- devices: {},
- browsers: {},
- sources: {},
- campaigns: {},
- pages: {},
- profileTypes: {},
+ countries: createDetailsBucket(),
+ devices: createDetailsBucket(),
+ browsers: createDetailsBucket(),
+ sources: createDetailsBucket(),
+ campaigns: createDetailsBucket(),
+ pages: createDetailsBucket(),
+ profileTypes: createDetailsBucket(),
}
for (const row of data) {
@@ -1906,7 +1911,7 @@ export class AnalyticsService {
const type = row.type as keyof FunnelStepDetails
if (details[type]) {
- if (!details[type][step]) details[type][step] = {}
+ if (!details[type][step]) details[type][step] = createValueBucket()
details[type][step][row.val] = row.cnt
}
}
diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts
index 98aad063e..765528149 100644
--- a/backend/apps/cloud/src/goal/goal.controller.ts
+++ b/backend/apps/cloud/src/goal/goal.controller.ts
@@ -714,7 +714,7 @@ export class GoalController {
UNION ALL
- SELECT 'profileTypes' AS kind, if(startsWith(profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 'identified', 'anonymous') AS val, count() AS cnt
+ SELECT 'profileTypes' AS kind, if(startsWith(profileId, {profilePrefixUser:String}), 'identified', 'anonymous') AS val, count() AS cnt
FROM session_info
WHERE profileId IS NOT NULL AND profileId != ''
GROUP BY val
@@ -726,7 +726,13 @@ export class GoalController {
const { data } = await clickhouse
.query({
query,
- query_params: { ...queryParams, pid: projectId, groupFrom, groupTo },
+ query_params: {
+ ...queryParams,
+ pid: projectId,
+ groupFrom,
+ groupTo,
+ profilePrefixUser: AnalyticsService.PROFILE_PREFIX_USER,
+ },
})
.then((resultSet) =>
resultSet.json<{
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index 1541627da..80e461c98 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -1772,6 +1772,7 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND type IN ('pageview', 'custom_event')
AND psid != 0
+ AND psid IN (SELECT psid FROM funnel_sessions)
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psid
)
@@ -2056,9 +2057,7 @@ export class AnalyticsService {
median: null,
p75: null,
}
- const pageParams: Record<string, string | number> = {
- steps: pages.length,
- }
+ const pageParams: Record<string, string> = {}
const sessionFiltersQuery = filtersQuery
? `AND psid IN (
SELECT DISTINCT psid
@@ -2071,21 +2070,57 @@ export class AnalyticsService {
)`
: ''
- const pagesStr = _join(
- _map(pages, (value, index) => {
- pageParams[`v${index}`] = value
- return `value={v${index}:String}`
+ pages.forEach((value, index) => {
+ pageParams[`v${index}`] = value
+ })
+
+ const funnelStepCtes = _join(
+ _map(pages, (_value, index) => {
+ const step = index + 1
+
+ if (step === 1) {
+ return `
+ funnel_step_1 AS (
+ SELECT
+ psid,
+ created AS firstStepAt,
+ created AS step1At,
+ eventIndex AS step1Index
+ FROM funnel_events
+ WHERE value = {v0:String}
+ )
+ `
+ }
+
+ const previousStep = step - 1
+
+ return `
+ funnel_step_${step} AS (
+ SELECT
+ prev.psid,
+ prev.firstStepAt,
+ min(e.created) AS step${step}At,
+ min(e.eventIndex) AS step${step}Index
+ FROM funnel_step_${previousStep} prev
+ INNER JOIN funnel_events e ON e.psid = prev.psid
+ AND e.value = {v${index}:String}
+ AND e.eventIndex > prev.step${previousStep}Index
+ AND e.created <= addSeconds(prev.firstStepAt, 86400)
+ GROUP BY prev.psid, prev.firstStepAt
+ )
+ `
}),
',',
)
+ const finalStep = pages.length
const query = `
- WITH funnel_sessions AS (
+ WITH funnel_events AS (
SELECT
psid,
- minIf(created, value = {v0:String}) AS firstStepAt,
- maxIf(created, value = {v${pages.length - 1}:String}) AS conversionAt,
- CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level
+ value,
+ created,
+ row_number() OVER (PARTITION BY psid ORDER BY created, value) AS eventIndex
FROM (
SELECT
psid,
@@ -2098,23 +2133,45 @@ export class AnalyticsService {
${sessionFiltersQuery}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
+ ),
+ ${funnelStepCtes},
+ funnel_sessions AS (
+ SELECT
+ psid,
+ argMin(firstStepAt, conversionAt) AS firstStepAt,
+ min(conversionAt) AS conversionAt
+ FROM (
+ SELECT
+ psid,
+ firstStepAt,
+ step${finalStep}At AS conversionAt
+ FROM funnel_step_${finalStep}
+ )
GROUP BY psid
- HAVING level = {steps:UInt32}
),
session_starts AS (
- SELECT psid, min(firstSeen) AS sessionStart
- FROM sessions FINAL
- WHERE pid = {pid:FixedString(12)}
- GROUP BY psid
+ SELECT fs.psid, min(s.firstSeen) AS sessionStart
+ FROM funnel_sessions fs
+ INNER JOIN (
+ SELECT psid, firstSeen, lastSeen
+ FROM sessions FINAL
+ WHERE pid = {pid:FixedString(12)}
+ ) s ON s.psid = fs.psid
+ WHERE s.firstSeen <= fs.firstStepAt
+ AND s.lastSeen >= fs.conversionAt
+ GROUP BY fs.psid
),
first_pages AS (
- SELECT psid, min(created) AS firstPageAt
- FROM events
- WHERE pid = {pid:FixedString(12)}
- AND type = 'pageview'
- AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- GROUP BY psid
+ SELECT fs.psid, min(e.created) AS firstPageAt
+ FROM funnel_sessions fs
+ INNER JOIN session_starts ss ON fs.psid = ss.psid
+ INNER JOIN events e ON e.psid = fs.psid
+ WHERE e.pid = {pid:FixedString(12)}
+ AND e.type = 'pageview'
+ AND e.psid != 0
+ AND e.created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND e.created BETWEEN ss.sessionStart AND fs.conversionAt
+ GROUP BY fs.psid
)
SELECT
round(avgOrNull(if(sessionStart > toDateTime(0) AND sessionStart <= conversionAt, dateDiff('second', sessionStart, conversionAt), NULL)), 2) AS averageSession,
diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts
index 336e55e37..13aab725d 100644
--- a/backend/apps/community/src/goal/goal.controller.ts
+++ b/backend/apps/community/src/goal/goal.controller.ts
@@ -634,7 +634,7 @@ export class GoalController {
UNION ALL
- SELECT 'profileTypes' AS kind, if(startsWith(profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 'identified', 'anonymous') AS val, count() AS cnt
+ SELECT 'profileTypes' AS kind, if(startsWith(profileId, {profilePrefixUser:String}), 'identified', 'anonymous') AS val, count() AS cnt
FROM session_info
WHERE profileId IS NOT NULL AND profileId != ''
GROUP BY val
@@ -646,7 +646,13 @@ export class GoalController {
const { data } = await clickhouse
.query({
query,
- query_params: { ...queryParams, pid: projectId, groupFrom, groupTo },
+ query_params: {
+ ...queryParams,
+ pid: projectId,
+ groupFrom,
+ groupTo,
+ profilePrefixUser: AnalyticsService.PROFILE_PREFIX_USER,
+ },
})
.then((resultSet) =>
resultSet.json<{
diff --git a/web/app/pages/Project/View/components/FilterRowsEditor.tsx b/web/app/pages/Project/View/components/FilterRowsEditor.tsx
index 4c675e3b9..6493dff14 100644
--- a/web/app/pages/Project/View/components/FilterRowsEditor.tsx
+++ b/web/app/pages/Project/View/components/FilterRowsEditor.tsx
@@ -67,13 +67,48 @@ const OPERATORS: { value: FilterOperator; labelKey: string }[] = [
{ value: 'notContains', labelKey: 'project.contains.not' },
]
-const createEmptyFilterRow = (): FilterRow => ({
+const getMetadataColumnBase = (column: string) => {
+ if (column === 'ev:key' || column.startsWith('ev:key:')) return 'ev:key'
+ if (column === 'tag:key' || column.startsWith('tag:key:')) return 'tag:key'
+ if (column === 'ev:value') return 'ev:value'
+ if (column === 'tag:value') return 'tag:value'
+ return column
+}
+
+const getOperatorsForColumn = (column: string) => {
+ if (column === 'ev:key' || column === 'tag:key') {
+ return OPERATORS.filter(
+ (operator) => operator.value === 'is' || operator.value === 'isNot',
+ )
+ }
+
+ return OPERATORS
+}
+
+const normalizeFilterOperator = (
+ column: string,
+ operator: FilterOperator,
+): FilterOperator => {
+ const operatorOptions = getOperatorsForColumn(column)
+
+ return operatorOptions.some((option) => option.value === operator)
+ ? operator
+ : operatorOptions[0]?.value || 'is'
+}
+
+const createFilterRow = ({
+ column = '',
+ operator = 'is',
+ value = '',
+}: Partial<Omit<FilterRow, 'id'>> = {}): FilterRow => ({
id: crypto.randomUUID(),
- column: '',
- operator: 'is',
- value: '',
+ column,
+ operator: normalizeFilterOperator(column, operator),
+ value,
})
+const createEmptyFilterRow = (): FilterRow => createFilterRow()
+
const operatorToFilter = (
operator: FilterOperator,
): { isExclusive: boolean; isContains: boolean } => {
@@ -123,15 +158,14 @@ const filtersToRows = (filters: FilterType[] = []): FilterRow[] => {
usedIndexes.add(versionIndex)
return [
- {
- id: crypto.randomUUID(),
+ createFilterRow({
column: versionColumn,
operator,
value: createVersionValue(
filter.filter,
filters[versionIndex].filter,
),
- },
+ }),
]
}
}
@@ -139,34 +173,15 @@ const filtersToRows = (filters: FilterType[] = []): FilterRow[] => {
usedIndexes.add(index)
return [
- {
- id: crypto.randomUUID(),
+ createFilterRow({
column: filter.column,
operator: filterToOperator(filter),
value: filter.filter,
- },
+ }),
]
})
}
-const getMetadataColumnBase = (column: string) => {
- if (column === 'ev:key' || column.startsWith('ev:key:')) return 'ev:key'
- if (column === 'tag:key' || column.startsWith('tag:key:')) return 'tag:key'
- if (column === 'ev:value') return 'ev:value'
- if (column === 'tag:value') return 'tag:value'
- return column
-}
-
-const getOperatorsForColumn = (column: string) => {
- if (column === 'ev:key' || column === 'tag:key') {
- return OPERATORS.filter(
- (operator) => operator.value === 'is' || operator.value === 'isNot',
- )
- }
-
- return OPERATORS
-}
-
const InlineButton = ({
text,
onClick,
@@ -413,7 +428,10 @@ const FilterRowsEditor = ({
return
}
- const nextRows = filtersToRows(initialFilters)
+ const nextRows = filtersToRows(initialFilters).map((row) => ({
+ ...row,
+ operator: normalizeFilterOperator(row.column, row.operator),
+ }))
setFilterRows(nextRows.length ? nextRows : [createEmptyFilterRow()])
}, [active, initialFilters, resetKey])
@@ -463,12 +481,7 @@ const FilterRowsEditor = ({
) {
fetchFilterValues(fieldValue)
}
- const operatorOptions = getOperatorsForColumn(fieldValue)
- const operator = operatorOptions.some(
- (option) => option.value === row.operator,
- )
- ? row.operator
- : 'is'
+ const operator = normalizeFilterOperator(fieldValue, row.operator)
return { ...row, column: fieldValue, operator, value: '' }
}
diff --git a/web/app/routes/projects.$id.tsx b/web/app/routes/projects.$id.tsx
index 0a80bde55..a2e60a905 100644
--- a/web/app/routes/projects.$id.tsx
+++ b/web/app/routes/projects.$id.tsx
@@ -62,6 +62,34 @@ import {
hasAuthTokens,
} from '~/utils/session.server'
+const GOAL_CONDITION_RELATIONS = new Set(['AND', 'OR'])
+const GOAL_CONDITION_EVENT_TYPES = new Set(['any', 'pageview', 'custom_event'])
+const GOAL_CONDITION_OPERATORS = new Set([
+ 'equals',
+ 'not_equals',
+ 'contains',
+ 'not_contains',
+ 'exists',
+ 'not_exists',
+])
+
+type GoalMetadataFilter = {
+ key: string
+ value: string
+}
+
+type JsonFormFieldResult =
+ | { ok: true; value: unknown }
+ | { ok: false; error: string }
+
+type GoalJsonFieldsResult =
+ | {
+ ok: true
+ metadataFilters: GoalMetadataFilter[]
+ conditions: unknown
+ }
+ | { ok: false; error: string }
+
function computeDateRangeForPeriod(
period: string,
from?: string,
@@ -107,6 +135,87 @@ function computeDateRangeForPeriod(
return compute ? compute() : null
}
+const parseJsonFormField = (
+ formData: FormData,
+ field: string,
+ fallback: string,
+): JsonFormFieldResult => {
+ try {
+ return {
+ ok: true,
+ value: JSON.parse(formData.get(field)?.toString() || fallback),
+ }
+ } catch {
+ return { ok: false, error: `Invalid JSON in ${field}` }
+ }
+}
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+ typeof value === 'object' && value !== null
+
+const isGoalMetadataFilters = (value: unknown): value is GoalMetadataFilter[] =>
+ Array.isArray(value) &&
+ value.every(
+ (filter) =>
+ isRecord(filter) &&
+ typeof filter.key === 'string' &&
+ typeof filter.value === 'string',
+ )
+
+const isGoalConditions = (value: unknown) => {
+ if (value === null) {
+ return true
+ }
+
+ if (
+ !isRecord(value) ||
+ !GOAL_CONDITION_RELATIONS.has(String(value.relation))
+ ) {
+ return false
+ }
+
+ return (
+ Array.isArray(value.conditions) &&
+ value.conditions.every(
+ (condition) =>
+ isRecord(condition) &&
+ GOAL_CONDITION_EVENT_TYPES.has(String(condition.eventType)) &&
+ typeof condition.field === 'string' &&
+ GOAL_CONDITION_OPERATORS.has(String(condition.operator)) &&
+ (condition.value === undefined ||
+ typeof condition.value === 'string') &&
+ (condition.metadataKey === undefined ||
+ typeof condition.metadataKey === 'string'),
+ )
+ )
+}
+
+const parseGoalJsonFields = (formData: FormData): GoalJsonFieldsResult => {
+ const metadataFilters = parseJsonFormField(formData, 'metadataFilters', '[]')
+ if (!metadataFilters.ok) {
+ return metadataFilters
+ }
+
+ if (!isGoalMetadataFilters(metadataFilters.value)) {
+ return { ok: false, error: 'Invalid metadataFilters' }
+ }
+
+ const conditions = parseJsonFormField(formData, 'conditions', 'null')
+ if (!conditions.ok) {
+ return conditions
+ }
+
+ if (!isGoalConditions(conditions.value)) {
+ return { ok: false, error: 'Invalid conditions' }
+ }
+
+ return {
+ ok: true,
+ metadataFilters: metadataFilters.value,
+ conditions: conditions.value,
+ }
+}
+
function formatDateForBackend(dateStr: string): string {
if (!dateStr) return ''
@@ -697,12 +806,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
const type = formData.get('type')?.toString() || 'pageview'
const matchType = formData.get('matchType')?.toString() || 'exact'
const value = formData.get('value')?.toString() || ''
- const metadataFilters = JSON.parse(
- formData.get('metadataFilters')?.toString() || '[]',
- )
- const conditions = JSON.parse(
- formData.get('conditions')?.toString() || 'null',
- )
+ const goalJson = parseGoalJsonFields(formData)
+
+ if (!goalJson.ok) {
+ return data<ProjectViewActionData>(
+ { intent, error: goalJson.error },
+ { status: 400 },
+ )
+ }
const result = await serverFetch(request, 'goal', {
method: 'POST',
@@ -712,8 +823,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
type,
matchType,
value,
- metadataFilters,
- conditions,
+ metadataFilters: goalJson.metadataFilters,
+ conditions: goalJson.conditions,
},
})
@@ -736,16 +847,25 @@ export async function action({ request, params }: ActionFunctionArgs) {
const type = formData.get('type')?.toString()
const matchType = formData.get('matchType')?.toString()
const value = formData.get('value')?.toString()
- const metadataFilters = JSON.parse(
- formData.get('metadataFilters')?.toString() || '[]',
- )
- const conditions = JSON.parse(
- formData.get('conditions')?.toString() || 'null',
- )
+ const goalJson = parseGoalJsonFields(formData)
+
+ if (!goalJson.ok) {
+ return data<ProjectViewActionData>(
+ { intent, error: goalJson.error },
+ { status: 400 },
+ )
+ }
const result = await serverFetch(request, `goal/${goalId}`, {
method: 'PUT',
- body: { name, type, matchType, value, metadataFilters, conditions },
+ body: {
+ name,
+ type,
+ matchType,
+ value,
+ metadataFilters: goalJson.metadataFilters,
+ conditions: goalJson.conditions,
+ },
})
if (result.error) {