Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

feat: improve experiment planning and results logic

Blue Mouse2d696beb86b8
diff --git a/backend/apps/cloud/src/experiment/dto/experiment.dto.ts b/backend/apps/cloud/src/experiment/dto/experiment.dto.ts
index 0f9d6a9ec..d01f5e502 100644
--- a/backend/apps/cloud/src/experiment/dto/experiment.dto.ts
+++ b/backend/apps/cloud/src/experiment/dto/experiment.dto.ts
@@ -315,6 +315,29 @@ class ExperimentChartDataDto {
winProbability: Record<string, number[]>
}
+class ExperimentResultWindowDto {
+ @ApiProperty()
+ mode: 'selected' | 'active_overlap' | 'final'
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty()
+ selectedFrom: string
+
+ @ApiProperty()
+ selectedTo: string
+
+ @ApiPropertyOptional()
+ activeFrom?: string
+
+ @ApiPropertyOptional()
+ activeTo?: string
+}
+
export class ExperimentResultsDto {
@ApiProperty()
experimentId: string
@@ -348,4 +371,13 @@ export class ExperimentResultsDto {
description: 'Allowed time buckets when period is "all"',
})
timeBucket?: string[]
+
+ @ApiPropertyOptional()
+ resolvedTimeBucket?: string
+
+ @ApiPropertyOptional({ type: ExperimentResultWindowDto })
+ resultWindow?: ExperimentResultWindowDto
+
+ @ApiPropertyOptional()
+ isSegmented?: boolean
}
diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts
index 68c359cf9..c71267d4e 100644
--- a/backend/apps/cloud/src/experiment/experiment.controller.ts
+++ b/backend/apps/cloud/src/experiment/experiment.controller.ts
@@ -30,6 +30,9 @@ import _pick from 'lodash/pick'
import _round from 'lodash/round'
import _sum from 'lodash/sum'
import { DataSource } from 'typeorm'
+import dayjs from 'dayjs'
+import utc from 'dayjs/plugin/utc'
+import dayjsTimezone from 'dayjs/plugin/timezone'
import { UserService } from '../user/user.service'
import { ProjectService } from '../project/project.service'
@@ -37,6 +40,7 @@ import { AppLoggerService } from '../logger/logger.service'
import { PlanCode } from '../user/entities/user.entity'
import {
AnalyticsService,
+ DataType,
getLowestPossibleTimeBucket,
} from '../analytics/analytics.service'
import { TimeBucketType } from '../analytics/dto/getData.dto'
@@ -74,6 +78,9 @@ import { Pagination } from '../common/pagination'
const EXPERIMENTS_MAXIMUM = 20 // Maximum experiments per project
const FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/
+dayjs.extend(utc)
+dayjs.extend(dayjsTimezone)
+
const validateUniqueVariantKeys = (variants: Array<{ key: string }>): void => {
const seen = new Set<string>()
@@ -94,6 +101,16 @@ type GoalEventConditions = {
goalValue: string
}
+type ExperimentResultWindow = {
+ mode: 'selected' | 'active_overlap' | 'final'
+ from: string
+ to: string
+ selectedFrom: string
+ selectedTo: string
+ activeFrom?: string
+ activeTo?: string
+}
+
@ApiTags('Experiment')
@Controller(['experiment', 'v1/experiment'])
export class ExperimentController {
@@ -709,6 +726,8 @@ export class ExperimentController {
)
}
+ this.validateStartConfiguration(experiment)
+
try {
await this.dataSource.transaction(async (transactionalEntityManager) => {
let featureFlag = experiment.featureFlag
@@ -762,7 +781,7 @@ export class ExperimentController {
id,
{
status: ExperimentStatus.RUNNING,
- startedAt: new Date(),
+ startedAt: experiment.startedAt ?? new Date(),
featureFlag,
},
transactionalEntityManager,
@@ -916,9 +935,10 @@ export class ExperimentController {
@Query('from') from?: string,
@Query('to') to?: string,
@Query('timezone') timezone?: string,
+ @Query('filters') filters?: string,
) {
this.logger.log(
- { userId, id, period, timeBucket: timeBucketParam, from, to },
+ { userId, id, period, timeBucket: timeBucketParam, from, to, filters },
'GET /experiment/:id/results',
)
@@ -967,8 +987,34 @@ export class ExperimentController {
diff,
)
- const exposureAttributionSubquery =
- this.getExposureAttributionSubquery(experiment)
+ const resultWindow = this.getEffectiveResultWindow(experiment, {
+ groupFromUTC,
+ groupToUTC,
+ useFinalByDefault: !from && !to,
+ })
+
+ let resolvedTimeBucket = timeBucket
+ if (resultWindow.mode === 'final') {
+ try {
+ resolvedTimeBucket = getLowestPossibleTimeBucket(
+ 'custom',
+ resultWindow.from,
+ resultWindow.to,
+ )
+ } catch {
+ resolvedTimeBucket = timeBucket
+ }
+ }
+
+ const [filtersQuery, filtersParams, appliedFilters] =
+ this.analyticsService.getFiltersQuery(filters || '[]', DataType.ANALYTICS)
+ const isSegmented =
+ Array.isArray(appliedFilters) && appliedFilters.length > 0
+
+ const exposureAttributionSubquery = this.getExposureAttributionSubquery(
+ experiment,
+ filtersQuery,
+ )
const exposuresQuery = `
SELECT
@@ -983,8 +1029,9 @@ export class ExperimentController {
const exposuresParams = {
pid: experiment.project.id,
experimentId: experiment.id,
- groupFrom: groupFromUTC,
- groupTo: groupToUTC,
+ groupFrom: resultWindow.from,
+ groupTo: resultWindow.to,
+ ...filtersParams,
}
let exposuresData: { variantKey: string; exposures: number }[] = []
@@ -1029,10 +1076,11 @@ export class ExperimentController {
const conversionsParams = {
pid: experiment.project.id,
experimentId: experiment.id,
- groupFrom: groupFromUTC,
- groupTo: groupToUTC,
+ groupFrom: resultWindow.from,
+ groupTo: resultWindow.to,
goalValue,
...metaParams,
+ ...filtersParams,
}
try {
@@ -1112,6 +1160,15 @@ export class ExperimentController {
hasWinner = highestProbVariant.probabilityOfBeingBest >= 95
}
+ const chartFrom =
+ resultWindow.mode === 'selected'
+ ? groupFrom
+ : this.formatDateInTimezone(resultWindow.from, safeTimezone)
+ const chartTo =
+ resultWindow.mode === 'selected'
+ ? groupTo
+ : this.formatDateInTimezone(resultWindow.to, safeTimezone)
+
let chart:
| { x: string[]; winProbability: Record<string, number[]> }
| undefined
@@ -1119,12 +1176,14 @@ export class ExperimentController {
try {
chart = await this.generateExperimentChart(
experiment,
- timeBucket,
- groupFrom,
- groupTo,
- groupFromUTC,
- groupToUTC,
+ resolvedTimeBucket,
+ chartFrom,
+ chartTo,
+ resultWindow.from,
+ resultWindow.to,
safeTimezone,
+ filtersQuery,
+ filtersParams,
)
} catch (err) {
this.logger.warn({ err }, 'Failed to generate experiment chart data')
@@ -1142,6 +1201,166 @@ export class ExperimentController {
confidenceLevel: 95,
chart,
timeBucket: allowedTimeBucketForPeriodAll,
+ resolvedTimeBucket,
+ resultWindow,
+ isSegmented,
+ }
+ }
+
+ private getEffectiveResultWindow(
+ experiment: Experiment,
+ selected: {
+ groupFromUTC: string
+ groupToUTC: string
+ useFinalByDefault: boolean
+ },
+ ): ExperimentResultWindow {
+ const selectedWindow = {
+ mode: 'selected' as const,
+ from: selected.groupFromUTC,
+ to: selected.groupToUTC,
+ selectedFrom: selected.groupFromUTC,
+ selectedTo: selected.groupToUTC,
+ }
+
+ if (
+ experiment.status !== ExperimentStatus.COMPLETED ||
+ !experiment.startedAt ||
+ !experiment.endedAt
+ ) {
+ return selectedWindow
+ }
+
+ const activeFrom = this.formatClickHouseDate(experiment.startedAt)
+ const activeTo = this.formatClickHouseDate(experiment.endedAt)
+ const selectedFromMs = this.toDateTimeMs(selected.groupFromUTC)
+ const selectedToMs = this.toDateTimeMs(selected.groupToUTC)
+ const activeFromMs = this.toDateTimeMs(activeFrom)
+ const activeToMs = this.toDateTimeMs(activeTo)
+
+ if (
+ !Number.isFinite(activeFromMs) ||
+ !Number.isFinite(activeToMs) ||
+ activeFromMs >= activeToMs
+ ) {
+ return selectedWindow
+ }
+
+ const overlaps =
+ selectedToMs >= activeFromMs && selectedFromMs <= activeToMs
+
+ if (selected.useFinalByDefault || !overlaps) {
+ return {
+ ...selectedWindow,
+ mode: 'final',
+ from: activeFrom,
+ to: activeTo,
+ activeFrom,
+ activeTo,
+ }
+ }
+
+ const fromMs = Math.max(selectedFromMs, activeFromMs)
+ const toMs = Math.min(selectedToMs, activeToMs)
+ const from = this.formatClickHouseDate(new Date(fromMs))
+ const to = this.formatClickHouseDate(new Date(toMs))
+ const mode =
+ from === selected.groupFromUTC && to === selected.groupToUTC
+ ? 'selected'
+ : 'active_overlap'
+
+ return {
+ ...selectedWindow,
+ mode,
+ from,
+ to,
+ activeFrom,
+ activeTo,
+ }
+ }
+
+ private formatClickHouseDate(value: Date | string): string {
+ if (value instanceof Date) {
+ return value.toISOString().slice(0, 19).replace('T', ' ')
+ }
+
+ const time = this.toDateTimeMs(value)
+ if (!Number.isFinite(time)) {
+ return value
+ }
+
+ return new Date(time).toISOString().slice(0, 19).replace('T', ' ')
+ }
+
+ private toDateTimeMs(value: Date | string): number {
+ if (value instanceof Date) {
+ return value.getTime()
+ }
+
+ const normalised = value.includes('T')
+ ? value
+ : `${value.replace(' ', 'T')}Z`
+
+ return Date.parse(normalised)
+ }
+
+ private formatDateInTimezone(value: string, safeTimezone: string): string {
+ const date = dayjs.utc(value)
+
+ if (!date.isValid()) {
+ return value
+ }
+
+ return date.tz(safeTimezone).format('YYYY-MM-DD HH:mm:ss')
+ }
+
+ private validateStartConfiguration(experiment: Experiment): void {
+ if (!experiment.variants || experiment.variants.length < 2) {
+ throw new BadRequestException(
+ 'An experiment must have at least 2 variants',
+ )
+ }
+
+ const controlVariants = experiment.variants.filter(
+ (variant) => variant.isControl,
+ )
+ if (controlVariants.length !== 1) {
+ throw new BadRequestException(
+ 'An experiment must have exactly one control variant',
+ )
+ }
+
+ const totalPercentage = _sum(
+ experiment.variants.map((variant) => variant.rolloutPercentage),
+ )
+ if (totalPercentage !== 100) {
+ throw new BadRequestException(
+ 'Variant rollout percentages must sum to 100',
+ )
+ }
+
+ if (experiment.variants.some((variant) => variant.rolloutPercentage <= 0)) {
+ throw new BadRequestException(
+ 'Each variant must receive traffic before starting',
+ )
+ }
+
+ if (
+ experiment.exposureTrigger === ExposureTrigger.CUSTOM_EVENT &&
+ !experiment.customEventName?.trim()
+ ) {
+ throw new BadRequestException(
+ 'Custom event name is required when using custom event exposure trigger',
+ )
+ }
+
+ if (
+ experiment.featureFlagMode === FeatureFlagMode.LINK &&
+ !experiment.featureFlag
+ ) {
+ throw new BadRequestException(
+ 'No feature flag linked to this experiment. Please link a feature flag first.',
+ )
}
}
@@ -1189,7 +1408,10 @@ export class ExperimentController {
}
}
- private getExposureAttributionSubquery(experiment: Experiment): string {
+ private getExposureAttributionSubquery(
+ experiment: Experiment,
+ filtersQuery = '',
+ ): string {
const variantSelector =
experiment.multipleVariantHandling ===
MultipleVariantHandling.FIRST_EXPOSURE ||
@@ -1212,11 +1434,29 @@ export class ExperimentController {
ee.pid = {pid:FixedString(12)}
AND ee.experimentId = {experimentId:String}
AND ee.created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${this.getSegmentProfileFilter(filtersQuery)}
GROUP BY ee.pid, ee.profileId
${multiVariantFilter}
`
}
+ private getSegmentProfileFilter(filtersQuery: string): string {
+ if (!filtersQuery) {
+ return ''
+ }
+
+ return `
+ AND ee.profileId IN (
+ SELECT assumeNotNull(profileId)
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ }
+
/**
* Generate time-series chart data for experiment win probabilities
*/
@@ -1228,6 +1468,8 @@ export class ExperimentController {
groupFromUTC: string,
groupToUTC: string,
safeTimezone: string,
+ filtersQuery = '',
+ filtersParams: Record<string, unknown> = {},
): Promise<{ x: string[]; winProbability: Record<string, number[]> }> {
const { xShifted } = this.analyticsService.generateXAxis(
timeBucket,
@@ -1243,8 +1485,10 @@ export class ExperimentController {
// - bucket conversions by the first conversion per attributed variant/profile
// This guarantees that the last chart point uses the same data as the table.
const dateColumnsGroupBy = this.getTimeBucketDateColumnsGroupBy(timeBucket)
- const exposureAttributionSubquery =
- this.getExposureAttributionSubquery(experiment)
+ const exposureAttributionSubquery = this.getExposureAttributionSubquery(
+ experiment,
+ filtersQuery,
+ )
const exposuresDateColumnsSelect = this.getTimeBucketDateColumnsSelect(
timeBucket,
'exposureCreated',
@@ -1268,6 +1512,7 @@ export class ExperimentController {
groupFrom: groupFromUTC,
groupTo: groupToUTC,
timezone: safeTimezone,
+ ...filtersParams,
}
let exposuresData: any[] = []
diff --git a/backend/apps/community/src/experiment/dto/experiment.dto.ts b/backend/apps/community/src/experiment/dto/experiment.dto.ts
index 26765117f..f5d2543fc 100644
--- a/backend/apps/community/src/experiment/dto/experiment.dto.ts
+++ b/backend/apps/community/src/experiment/dto/experiment.dto.ts
@@ -317,6 +317,29 @@ class ExperimentChartDataDto {
winProbability: Record<string, number[]>
}
+class ExperimentResultWindowDto {
+ @ApiProperty()
+ mode: 'selected' | 'active_overlap' | 'final'
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty()
+ selectedFrom: string
+
+ @ApiProperty()
+ selectedTo: string
+
+ @ApiPropertyOptional()
+ activeFrom?: string
+
+ @ApiPropertyOptional()
+ activeTo?: string
+}
+
export class ExperimentResultsDto {
@ApiProperty()
experimentId: string
@@ -347,4 +370,13 @@ export class ExperimentResultsDto {
@ApiPropertyOptional({ type: [String] })
timeBucket?: string[]
+
+ @ApiPropertyOptional()
+ resolvedTimeBucket?: string
+
+ @ApiPropertyOptional({ type: ExperimentResultWindowDto })
+ resultWindow?: ExperimentResultWindowDto
+
+ @ApiPropertyOptional()
+ isSegmented?: boolean
}
diff --git a/backend/apps/community/src/experiment/experiment.controller.ts b/backend/apps/community/src/experiment/experiment.controller.ts
index 362b32c27..91725a85a 100644
--- a/backend/apps/community/src/experiment/experiment.controller.ts
+++ b/backend/apps/community/src/experiment/experiment.controller.ts
@@ -25,11 +25,15 @@ import _map from 'lodash/map'
import _pick from 'lodash/pick'
import _round from 'lodash/round'
import _sum from 'lodash/sum'
+import dayjs from 'dayjs'
+import utc from 'dayjs/plugin/utc'
+import dayjsTimezone from 'dayjs/plugin/timezone'
import { ProjectService } from '../project/project.service'
import { AppLoggerService } from '../logger/logger.service'
import {
AnalyticsService,
+ DataType,
getLowestPossibleTimeBucket,
} from '../analytics/analytics.service'
import { TimeBucketType } from '../analytics/dto/getData.dto'
@@ -64,6 +68,9 @@ const EXPERIMENTS_MAXIMUM = 20
const FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/
const ROLLOUT_PERCENTAGE_EPSILON = 1e-6
+dayjs.extend(utc)
+dayjs.extend(dayjsTimezone)
+
const validateUniqueVariantKeys = (variants: Array<{ key: string }>): void => {
const seen = new Set<string>()
@@ -84,6 +91,16 @@ type GoalEventConditions = {
goalValue: string
}
+type ExperimentResultWindow = {
+ mode: 'selected' | 'active_overlap' | 'final'
+ from: string
+ to: string
+ selectedFrom: string
+ selectedTo: string
+ activeFrom?: string
+ activeTo?: string
+}
+
@ApiTags('Experiment')
@Controller(['experiment', 'v1/experiment'])
export class ExperimentController {
@@ -437,6 +454,8 @@ export class ExperimentController {
)
}
+ this.validateStartConfiguration(experiment)
+
let featureFlagId = experiment.featureFlagId
if (!featureFlagId) {
if (experiment.featureFlagMode !== FeatureFlagMode.CREATE) {
@@ -582,9 +601,10 @@ export class ExperimentController {
@Query('from') from?: string,
@Query('to') to?: string,
@Query('timezone') timezone?: string,
+ @Query('filters') filters?: string,
) {
this.logger.log(
- { userId, id, period, timeBucket: timeBucketParam, from, to },
+ { userId, id, period, timeBucket: timeBucketParam, from, to, filters },
'GET /experiment/:id/results',
)
@@ -634,8 +654,34 @@ export class ExperimentController {
diff,
)
- const exposureAttributionSubquery =
- this.getExposureAttributionSubquery(experiment)
+ const resultWindow = this.getEffectiveResultWindow(experiment, {
+ groupFromUTC,
+ groupToUTC,
+ useFinalByDefault: !from && !to,
+ })
+
+ let resolvedTimeBucket = timeBucket
+ if (resultWindow.mode === 'final') {
+ try {
+ resolvedTimeBucket = getLowestPossibleTimeBucket(
+ 'custom',
+ resultWindow.from,
+ resultWindow.to,
+ )
+ } catch {
+ resolvedTimeBucket = timeBucket
+ }
+ }
+
+ const [filtersQuery, filtersParams, appliedFilters] =
+ this.analyticsService.getFiltersQuery(filters || '[]', DataType.ANALYTICS)
+ const isSegmented =
+ Array.isArray(appliedFilters) && appliedFilters.length > 0
+
+ const exposureAttributionSubquery = this.getExposureAttributionSubquery(
+ experiment,
+ filtersQuery,
+ )
const exposuresQuery = `
SELECT
@@ -650,8 +696,9 @@ export class ExperimentController {
const exposuresParams = {
pid: experiment.projectId,
experimentId: experiment.id,
- groupFrom: groupFromUTC,
- groupTo: groupToUTC,
+ groupFrom: resultWindow.from,
+ groupTo: resultWindow.to,
+ ...filtersParams,
}
let exposuresData: { variantKey: string; exposures: number }[] = []
@@ -699,10 +746,11 @@ export class ExperimentController {
const conversionsParams = {
pid: experiment.projectId,
experimentId: experiment.id,
- groupFrom: groupFromUTC,
- groupTo: groupToUTC,
+ groupFrom: resultWindow.from,
+ groupTo: resultWindow.to,
goalValue,
...metaParams,
+ ...filtersParams,
}
try {
@@ -796,6 +844,15 @@ export class ExperimentController {
hasWinner = highestProbVariant.probabilityOfBeingBest >= 95
}
+ const chartFrom =
+ resultWindow.mode === 'selected'
+ ? groupFrom
+ : this.formatDateInTimezone(resultWindow.from, safeTimezone)
+ const chartTo =
+ resultWindow.mode === 'selected'
+ ? groupTo
+ : this.formatDateInTimezone(resultWindow.to, safeTimezone)
+
let chart:
| { x: string[]; winProbability: Record<string, number[]> }
| undefined
@@ -803,12 +860,14 @@ export class ExperimentController {
try {
chart = await this.generateExperimentChart(
experiment,
- timeBucket,
- groupFrom,
- groupTo,
- groupFromUTC,
- groupToUTC,
+ resolvedTimeBucket,
+ chartFrom,
+ chartTo,
+ resultWindow.from,
+ resultWindow.to,
safeTimezone,
+ filtersQuery,
+ filtersParams,
)
} catch (err) {
this.logger.warn({ err }, 'Failed to generate experiment chart data')
@@ -826,6 +885,166 @@ export class ExperimentController {
confidenceLevel: 95,
chart,
timeBucket: allowedTimeBucketForPeriodAll,
+ resolvedTimeBucket,
+ resultWindow,
+ isSegmented,
+ }
+ }
+
+ private getEffectiveResultWindow(
+ experiment: Experiment,
+ selected: {
+ groupFromUTC: string
+ groupToUTC: string
+ useFinalByDefault: boolean
+ },
+ ): ExperimentResultWindow {
+ const selectedWindow = {
+ mode: 'selected' as const,
+ from: selected.groupFromUTC,
+ to: selected.groupToUTC,
+ selectedFrom: selected.groupFromUTC,
+ selectedTo: selected.groupToUTC,
+ }
+
+ if (
+ experiment.status !== ExperimentStatus.COMPLETED ||
+ !experiment.startedAt ||
+ !experiment.endedAt
+ ) {
+ return selectedWindow
+ }
+
+ const activeFrom = this.formatClickHouseDate(experiment.startedAt)
+ const activeTo = this.formatClickHouseDate(experiment.endedAt)
+ const selectedFromMs = this.toDateTimeMs(selected.groupFromUTC)
+ const selectedToMs = this.toDateTimeMs(selected.groupToUTC)
+ const activeFromMs = this.toDateTimeMs(activeFrom)
+ const activeToMs = this.toDateTimeMs(activeTo)
+
+ if (
+ !Number.isFinite(activeFromMs) ||
+ !Number.isFinite(activeToMs) ||
+ activeFromMs >= activeToMs
+ ) {
+ return selectedWindow
+ }
+
+ const overlaps =
+ selectedToMs >= activeFromMs && selectedFromMs <= activeToMs
+
+ if (selected.useFinalByDefault || !overlaps) {
+ return {
+ ...selectedWindow,
+ mode: 'final',
+ from: activeFrom,
+ to: activeTo,
+ activeFrom,
+ activeTo,
+ }
+ }
+
+ const fromMs = Math.max(selectedFromMs, activeFromMs)
+ const toMs = Math.min(selectedToMs, activeToMs)
+ const from = this.formatClickHouseDate(new Date(fromMs))
+ const to = this.formatClickHouseDate(new Date(toMs))
+ const mode =
+ from === selected.groupFromUTC && to === selected.groupToUTC
+ ? 'selected'
+ : 'active_overlap'
+
+ return {
+ ...selectedWindow,
+ mode,
+ from,
+ to,
+ activeFrom,
+ activeTo,
+ }
+ }
+
+ private formatClickHouseDate(value: Date | string): string {
+ if (value instanceof Date) {
+ return value.toISOString().slice(0, 19).replace('T', ' ')
+ }
+
+ const time = this.toDateTimeMs(value)
+ if (!Number.isFinite(time)) {
+ return value
+ }
+
+ return new Date(time).toISOString().slice(0, 19).replace('T', ' ')
+ }
+
+ private toDateTimeMs(value: Date | string): number {
+ if (value instanceof Date) {
+ return value.getTime()
+ }
+
+ const normalised = value.includes('T')
+ ? value
+ : `${value.replace(' ', 'T')}Z`
+
+ return Date.parse(normalised)
+ }
+
+ private formatDateInTimezone(value: string, safeTimezone: string): string {
+ const date = dayjs.utc(value)
+
+ if (!date.isValid()) {
+ return value
+ }
+
+ return date.tz(safeTimezone).format('YYYY-MM-DD HH:mm:ss')
+ }
+
+ private validateStartConfiguration(experiment: Experiment): void {
+ if (!experiment.variants || experiment.variants.length < 2) {
+ throw new BadRequestException(
+ 'An experiment must have at least 2 variants',
+ )
+ }
+
+ const controlVariants = experiment.variants.filter(
+ (variant) => variant.isControl,
+ )
+ if (controlVariants.length !== 1) {
+ throw new BadRequestException(
+ 'An experiment must have exactly one control variant',
+ )
+ }
+
+ const totalPercentage = _sum(
+ experiment.variants.map((variant) => variant.rolloutPercentage),
+ )
+ if (Math.abs(totalPercentage - 100) > ROLLOUT_PERCENTAGE_EPSILON) {
+ throw new BadRequestException(
+ 'Variant rollout percentages must sum to 100',
+ )
+ }
+
+ if (experiment.variants.some((variant) => variant.rolloutPercentage <= 0)) {
+ throw new BadRequestException(
+ 'Each variant must receive traffic before starting',
+ )
+ }
+
+ if (
+ experiment.exposureTrigger === ExposureTrigger.CUSTOM_EVENT &&
+ !experiment.customEventName?.trim()
+ ) {
+ throw new BadRequestException(
+ 'Custom event name is required when using custom event exposure trigger',
+ )
+ }
+
+ if (
+ experiment.featureFlagMode === FeatureFlagMode.LINK &&
+ !experiment.featureFlagId
+ ) {
+ throw new BadRequestException(
+ 'No feature flag linked to this experiment. Please link a feature flag first.',
+ )
}
}
@@ -1064,7 +1283,10 @@ export class ExperimentController {
}
}
- private getExposureAttributionSubquery(experiment: Experiment): string {
+ private getExposureAttributionSubquery(
+ experiment: Experiment,
+ filtersQuery = '',
+ ): string {
const variantSelector =
'argMin(ee.variantKey, tuple(ee.created, ee.variantKey))'
const multiVariantFilter =
@@ -1083,11 +1305,29 @@ export class ExperimentController {
ee.pid = {pid:FixedString(12)}
AND ee.experimentId = {experimentId:String}
AND ee.created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${this.getSegmentProfileFilter(filtersQuery)}
GROUP BY ee.pid, ee.profileId
${multiVariantFilter}
`
}
+ private getSegmentProfileFilter(filtersQuery: string): string {
+ if (!filtersQuery) {
+ return ''
+ }
+
+ return `
+ AND ee.profileId IN (
+ SELECT assumeNotNull(profileId)
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ }
+
private async generateExperimentChart(
experiment: Experiment,
timeBucket: TimeBucketType,
@@ -1096,6 +1336,8 @@ export class ExperimentController {
groupFromUTC: string,
groupToUTC: string,
safeTimezone: string,
+ filtersQuery = '',
+ filtersParams: Record<string, unknown> = {},
): Promise<{ x: string[]; winProbability: Record<string, number[]> }> {
const { xShifted } = this.analyticsService.generateXAxis(
timeBucket,
@@ -1105,8 +1347,10 @@ export class ExperimentController {
)
const dateColumnsGroupBy = this.getTimeBucketDateColumnsGroupBy(timeBucket)
- const exposureAttributionSubquery =
- this.getExposureAttributionSubquery(experiment)
+ const exposureAttributionSubquery = this.getExposureAttributionSubquery(
+ experiment,
+ filtersQuery,
+ )
const exposuresDateColumnsSelect = this.getTimeBucketDateColumnsSelect(
timeBucket,
'exposureCreated',
@@ -1130,6 +1374,7 @@ export class ExperimentController {
groupFrom: groupFromUTC,
groupTo: groupToUTC,
timezone: safeTimezone,
+ ...filtersParams,
}
let exposuresData: any[] = []
diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts
index 960e6d870..780469c0a 100644
--- a/web/app/api/api.server.ts
+++ b/web/app/api/api.server.ts
@@ -1637,6 +1637,16 @@ export interface ExperimentChartData {
winProbability: Record<string, number[]>
}
+export interface ExperimentResultWindow {
+ mode: 'selected' | 'active_overlap' | 'final'
+ from: string
+ to: string
+ selectedFrom: string
+ selectedTo: string
+ activeFrom?: string
+ activeTo?: string
+}
+
export interface ExperimentResults {
experimentId: string
status: ExperimentStatus
@@ -1647,6 +1657,9 @@ export interface ExperimentResults {
confidenceLevel: number
variants: ExperimentVariantResult[]
chart?: ExperimentChartData
+ resolvedTimeBucket?: string
+ resultWindow?: ExperimentResultWindow
+ isSegmented?: boolean
}
export async function getExperimentResultsServer(
@@ -1657,11 +1670,13 @@ export async function getExperimentResultsServer(
from = '',
to = '',
timezone?: string,
+ filters: AnalyticsFilter[] = [],
): Promise<ServerFetchResult<ExperimentResults>> {
const params = new URLSearchParams({ period, timeBucket })
if (from) params.append('from', from)
if (to) params.append('to', to)
if (timezone) params.append('timezone', timezone)
+ params.append('filters', JSON.stringify(filters))
return serverFetch<ExperimentResults>(
request,
diff --git a/web/app/pages/Project/tabs/Experiments/ExperimentResults.tsx b/web/app/pages/Project/tabs/Experiments/ExperimentResults.tsx
index 2fbdbef64..10b6f0dea 100644
--- a/web/app/pages/Project/tabs/Experiments/ExperimentResults.tsx
+++ b/web/app/pages/Project/tabs/Experiments/ExperimentResults.tsx
@@ -18,9 +18,13 @@ import {
PercentIcon,
CaretDownIcon,
CaretUpIcon,
+ WarningCircleIcon,
+ CheckCircleIcon,
} from '@phosphor-icons/react'
import { useState, useEffect, useRef, useMemo, memo } from 'react'
import { useTranslation } from 'react-i18next'
+import { useFetcher } from 'react-router'
+import { toast } from 'sonner'
import type {
Experiment,
@@ -42,11 +46,15 @@ import {
tbsFormatMapperTooltip24h,
} from '~/lib/constants'
import DashboardHeader from '~/pages/Project/View/components/DashboardHeader'
+import Filters from '~/pages/Project/View/components/Filters'
import { useViewProjectContext } from '~/pages/Project/View/ViewProject'
+import { typeNameMapping } from '~/pages/Project/View/ViewProject.helpers'
+import type { ProjectViewActionData } from '~/routes/projects.$id'
import BillboardChart from '~/ui/BillboardChart'
import Button from '~/ui/Button'
import Loader from '~/ui/Loader'
import { Badge } from '~/ui/Badge'
+import Modal from '~/ui/Modal'
import { Text } from '~/ui/Text'
import Tooltip from '~/ui/Tooltip'
import { nFormatter } from '~/utils/generic'
@@ -456,7 +464,7 @@ const ExposuresTable = memo(
</TableCell>
<TableCell className='text-right'>
<Text weight='semibold' size='sm' className='tabular-nums'>
- 100.0%
+ {totalExposures > 0 ? '100.0%' : '0.0%'}
</Text>
</TableCell>
</tr>
@@ -730,6 +738,32 @@ const CollapsibleSection = memo(
CollapsibleSection.displayName = 'CollapsibleSection'
+type HealthWarning = {
+ severity: 'warning' | 'danger'
+ title: string
+ message: string
+}
+
+const formatWindowDate = (value?: string | null) => {
+ if (!value) return null
+ return dayjs(value).format('MMM D, YYYY HH:mm')
+}
+
+const windowsOverlap = (
+ window: NonNullable<ExperimentResultsType['resultWindow']>,
+) => {
+ if (!window.activeFrom || !window.activeTo) return false
+
+ return (
+ dayjs(window.selectedTo).valueOf() >= dayjs(window.activeFrom).valueOf() &&
+ dayjs(window.selectedFrom).valueOf() <= dayjs(window.activeTo).valueOf()
+ )
+}
+
+const getConfiguredAllocation = (experiment: Experiment | null, key: string) =>
+ experiment?.variants.find((variant) => variant.key === key)
+ ?.rolloutPercentage ?? 0
+
const ExperimentResults = ({
experimentId,
period,
@@ -745,6 +779,9 @@ const ExperimentResults = ({
const experimentProxy = useExperimentProxy()
const resultsProxy = useExperimentResultsProxy()
const goalProxy = useGoalProxy()
+ const completeFetcher = useFetcher<ProjectViewActionData>()
+ const processedCompleteRef = useRef<string | null>(null)
+ const { filters } = useViewProjectContext()
const [experiment, setExperiment] = useState<Experiment | null>(null)
const [results, setResults] = useState<ExperimentResultsType | null>(null)
@@ -752,6 +789,7 @@ const ExperimentResults = ({
const [isLoading, setIsLoading] = useState(true)
const [error, setError] = useState<string | null>(null)
const [isSettingsOpen, setIsSettingsOpen] = useState(false)
+ const [isCompleteModalOpen, setIsCompleteModalOpen] = useState(false)
const [reloadToken, setReloadToken] = useState(0)
const statusBadgeColour = useMemo<
'slate' | 'green' | 'yellow' | 'sky'
@@ -787,6 +825,7 @@ const ExperimentResults = ({
from,
to,
timezone,
+ filters,
}),
])
@@ -824,10 +863,29 @@ const ExperimentResults = ({
from,
to,
timezone,
+ filters,
reloadToken,
refreshTrigger,
])
+ useEffect(() => {
+ if (!completeFetcher.data || completeFetcher.state !== 'idle') return
+
+ const key = `${completeFetcher.data.intent}-${completeFetcher.data.success}-${completeFetcher.data.error || ''}`
+ if (processedCompleteRef.current === key) return
+ processedCompleteRef.current = key
+
+ if (completeFetcher.data.intent !== 'complete-experiment') return
+
+ if (completeFetcher.data.success) {
+ toast.success(t('experiments.completed'))
+ setIsCompleteModalOpen(false)
+ setReloadToken((value) => value + 1)
+ } else if (completeFetcher.data.error) {
+ toast.error(completeFetcher.data.error)
+ }
+ }, [completeFetcher.data, completeFetcher.state, t])
+
const overallConversionRate = useMemo(() => {
if (!results || results.totalExposures === 0) return 0
return ((results.totalConversions / results.totalExposures) * 100).toFixed(
@@ -844,6 +902,170 @@ const ExperimentResults = ({
})
}, [results])
+ const healthWarnings = useMemo<HealthWarning[]>(() => {
+ if (!experiment || !results) return []
+
+ const warnings: HealthWarning[] = []
+ const runningForDays = experiment.startedAt
+ ? dayjs().diff(dayjs(experiment.startedAt), 'day')
+ : 0
+
+ if (results.totalExposures === 0) {
+ warnings.push({
+ severity: 'warning',
+ title: 'No exposures',
+ message:
+ experiment.exposureTrigger === 'custom_event'
+ ? 'No matching custom exposure events were recorded.'
+ : 'No feature flag evaluations were recorded for this experiment.',
+ })
+ }
+
+ if (results.totalExposures >= 100) {
+ const imbalancedVariant = results.variants.find((variant) => {
+ const configured = getConfiguredAllocation(experiment, variant.key)
+ const observed = (variant.exposures / results.totalExposures) * 100
+ return configured > 0 && Math.abs(observed - configured) >= 25
+ })
+
+ if (imbalancedVariant) {
+ warnings.push({
+ severity: 'danger',
+ title: 'Extreme imbalance',
+ message:
+ 'Observed traffic is far from the configured split. Treat winner reads carefully.',
+ })
+ }
+ }
+
+ const missingTrafficVariant = results.variants.find(
+ (variant) => variant.exposures === 0 && results.totalExposures > 0,
+ )
+
+ if (missingTrafficVariant) {
+ warnings.push({
+ severity: 'danger',
+ title: 'Variant not receiving traffic',
+ message: `${missingTrafficVariant.name} has no observed exposures in this window.`,
+ })
+ }
+
+ if (results.totalExposures >= 100 && results.totalConversions < 30) {
+ warnings.push({
+ severity: 'warning',
+ title: 'Low conversion volume',
+ message:
+ 'Conversion counts are still low, probability estimates can swing quickly.',
+ })
+ }
+
+ if (results.status === 'running' && runningForDays >= 45) {
+ warnings.push({
+ severity: 'warning',
+ title: 'Stale running experiment',
+ message: `This experiment has been running for ${runningForDays} days.`,
+ })
+ }
+
+ if (results.status !== 'draft' && (!experiment.goalId || !goal)) {
+ warnings.push({
+ severity: 'danger',
+ title: 'Goal missing after start',
+ message:
+ 'The goal is missing or no longer available, conversions cannot be trusted.',
+ })
+ }
+
+ return warnings
+ }, [experiment, goal, results])
+
+ const stopRecommendation = useMemo(() => {
+ if (!experiment || !results) return null
+ if (results.variants.length === 0) return null
+ if (results.status !== 'running' && results.status !== 'paused') return null
+ if (results.totalExposures < 500 || results.totalConversions < 50) {
+ return null
+ }
+ if (
+ experiment.startedAt &&
+ dayjs().diff(dayjs(experiment.startedAt), 'day') < 7
+ ) {
+ return null
+ }
+ if (
+ healthWarnings.some((warning) =>
+ ['Extreme imbalance', 'Variant not receiving traffic'].includes(
+ warning.title,
+ ),
+ )
+ ) {
+ return null
+ }
+
+ const control = results.variants.find((variant) => variant.isControl)
+ const best = results.variants.reduce((current, variant) =>
+ variant.probabilityOfBeingBest > current.probabilityOfBeingBest
+ ? variant
+ : current,
+ )
+
+ if (
+ best &&
+ !best.isControl &&
+ best.probabilityOfBeingBest >= 98 &&
+ best.improvement > 0
+ ) {
+ return {
+ title: 'Clear winner ready',
+ message: `${best.name} is ahead with conservative safeguards met.`,
+ }
+ }
+
+ const harmedVariant = results.variants.find(
+ (variant) =>
+ !variant.isControl &&
+ variant.improvement <= -10 &&
+ variant.probabilityOfBeingBest <= 5,
+ )
+
+ if (control && harmedVariant && control.probabilityOfBeingBest >= 95) {
+ return {
+ title: 'Clear harm detected',
+ message: `${harmedVariant.name} is materially underperforming control.`,
+ }
+ }
+
+ return null
+ }, [experiment, healthWarnings, results])
+
+ const resultWindowNotice = useMemo(() => {
+ const window = results?.resultWindow
+ if (!window || window.mode === 'selected') return null
+
+ const fromLabel = formatWindowDate(window.from)
+ const toLabel = formatWindowDate(window.to)
+
+ if (window.mode === 'final') {
+ const suffix = windowsOverlap(window)
+ ? 'based on active experiment time.'
+ : 'because the selected period does not overlap the active experiment.'
+ return `Showing final results from ${fromLabel} to ${toLabel}, ${suffix}`
+ }
+
+ return `Showing the selected period clipped to active experiment time, ${fromLabel} to ${toLabel}.`
+ }, [results])
+
+ const handleCompleteExperiment = () => {
+ if (!experiment) return
+ processedCompleteRef.current = null
+ completeFetcher.submit(
+ { intent: 'complete-experiment', experimentId: experiment.id },
+ { method: 'POST' },
+ )
+ }
+
+ const tnMapping = useMemo(() => typeNameMapping(t), [t])
+
if (!experiment || !results) {
if (isLoading) {
return (
@@ -901,7 +1123,26 @@ const ExperimentResults = ({
projectId={experiment.pid}
experimentId={experiment.id}
/>
+ <Modal
+ onClose={() => setIsCompleteModalOpen(false)}
+ onSubmit={handleCompleteExperiment}
+ submitText={t('experiments.complete')}
+ closeText={t('common.cancel')}
+ title={t('experiments.completeConfirmTitle')}
+ message={
+ stopRecommendation
+ ? `${stopRecommendation.message} This will end collection and keep final results available.`
+ : t('experiments.completeConfirmMessage')
+ }
+ submitType='regular'
+ type='info'
+ isOpened={isCompleteModalOpen}
+ isLoading={completeFetcher.state !== 'idle'}
+ />
<div className='space-y-3'>
+ {filters.length > 0 ? (
+ <Filters className='mb-1' tnMapping={tnMapping} />
+ ) : null}
<div className='flex flex-wrap items-center gap-2'>
<div className='flex min-w-0 items-center gap-2'>
<Text as='h2' size='xl' weight='bold' truncate>
@@ -931,6 +1172,173 @@ const ExperimentResults = ({
}
/>
) : null}
+ {experiment.featureFlagKey || experiment.featureFlagId ? (
+ <Badge
+ colour='sky'
+ label={
+ <span className='flex max-w-[260px] items-center gap-1.5'>
+ <FlaskIcon className='size-3.5 shrink-0' />
+ <Text
+ as='span'
+ size='xs'
+ weight='medium'
+ colour='inherit'
+ truncate
+ >
+ {experiment.featureFlagKey || experiment.featureFlagId}
+ </Text>
+ </span>
+ }
+ />
+ ) : null}
+ </div>
+ {resultWindowNotice || results.isSegmented ? (
+ <div className='rounded-lg bg-sky-50 px-4 py-3 ring-1 ring-sky-200 dark:bg-sky-900/20 dark:ring-sky-800/70'>
+ {resultWindowNotice ? (
+ <Text size='sm' className='text-sky-900 dark:text-sky-100'>
+ {resultWindowNotice}
+ </Text>
+ ) : null}
+ {results.isSegmented ? (
+ <Text
+ size='sm'
+ className={cx('text-sky-900 dark:text-sky-100', {
+ 'mt-1': Boolean(resultWindowNotice),
+ })}
+ >
+ Segment filters are applied to exposed profiles before
+ conversions are attributed.
+ </Text>
+ ) : null}
+ </div>
+ ) : null}
+ {healthWarnings.length > 0 ? (
+ <div className='grid gap-2 lg:grid-cols-2'>
+ {healthWarnings.map((warning) => (
+ <div
+ key={`${warning.title}-${warning.message}`}
+ className={cx(
+ 'rounded-lg px-4 py-3 ring-1',
+ warning.severity === 'danger'
+ ? 'bg-red-50 ring-red-200 dark:bg-red-900/20 dark:ring-red-800/70'
+ : 'bg-yellow-50 ring-yellow-200 dark:bg-yellow-900/20 dark:ring-yellow-800/70',
+ )}
+ >
+ <div className='flex items-start gap-2'>
+ <WarningCircleIcon
+ className={cx(
+ 'mt-0.5 size-4 shrink-0',
+ warning.severity === 'danger'
+ ? 'text-red-600 dark:text-red-300'
+ : 'text-yellow-700 dark:text-yellow-300',
+ )}
+ />
+ <div>
+ <Text
+ as='p'
+ size='sm'
+ weight='semibold'
+ className={
+ warning.severity === 'danger'
+ ? 'text-red-900 dark:text-red-100'
+ : 'text-yellow-900 dark:text-yellow-100'
+ }
+ >
+ {warning.title}
+ </Text>
+ <Text
+ as='p'
+ size='xs'
+ className={
+ warning.severity === 'danger'
+ ? 'text-red-800 dark:text-red-200'
+ : 'text-yellow-800 dark:text-yellow-100'
+ }
+ >
+ {warning.message}
+ </Text>
+ </div>
+ </div>
+ </div>
+ ))}
+ </div>
+ ) : null}
+ {stopRecommendation ? (
+ <div className='flex flex-col gap-3 rounded-lg bg-green-50 px-4 py-3 ring-1 ring-green-200 sm:flex-row sm:items-center sm:justify-between dark:bg-green-900/20 dark:ring-green-800/70'>
+ <div className='flex items-start gap-3'>
+ <CheckCircleIcon
+ className='mt-0.5 size-5 shrink-0 text-green-600 dark:text-green-300'
+ weight='duotone'
+ />
+ <div>
+ <Text
+ as='p'
+ size='sm'
+ weight='semibold'
+ className='text-green-900 dark:text-green-100'
+ >
+ {stopRecommendation.title}
+ </Text>
+ <Text
+ as='p'
+ size='xs'
+ className='text-green-800 dark:text-green-100'
+ >
+ {stopRecommendation.message} Confirm before completing.
+ </Text>
+ </div>
+ </div>
+ <Button
+ type='button'
+ size='sm'
+ onClick={() => setIsCompleteModalOpen(true)}
+ >
+ {t('experiments.complete')}
+ </Button>
+ </div>
+ ) : null}
+ <div className='rounded-lg bg-gray-50 px-4 py-3 ring-1 ring-gray-200 dark:bg-slate-900/40 dark:ring-slate-800'>
+ <div className='flex flex-wrap items-center justify-between gap-3'>
+ <div>
+ <Text as='p' size='sm' weight='semibold'>
+ Feature flag relationship
+ </Text>
+ <Text as='p' size='xs' colour='muted'>
+ The flag controls eligibility. Experiment exposures use actual
+ evaluations and variant events.
+ </Text>
+ </div>
+ <Text as='span' size='xs' colour='secondary' code>
+ {experiment.featureFlagKey ||
+ experiment.featureFlagId ||
+ 'created on launch'}
+ </Text>
+ </div>
+ <div className='mt-3 grid gap-2 sm:grid-cols-2 lg:grid-cols-4'>
+ {experiment.variants.map((variant) => (
+ <div
+ key={variant.key}
+ className='rounded-md bg-white px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-950 dark:ring-slate-800'
+ >
+ <Text as='p' size='xs' weight='medium' truncate>
+ {variant.name}
+ </Text>
+ <div className='mt-1 flex items-center justify-between gap-2'>
+ <Text as='span' size='xs' colour='muted' code>
+ {variant.key}
+ </Text>
+ <Text
+ as='span'
+ size='xs'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {variant.rolloutPercentage}%
+ </Text>
+ </div>
+ </div>
+ ))}
+ </div>
</div>
{results.hasWinner && results.winnerKey ? (
<div className='flex items-center gap-4 rounded-lg border border-green-300 bg-green-50 px-4 py-2 dark:border-green-700 dark:bg-green-900/20'>
@@ -999,7 +1407,7 @@ const ExperimentResults = ({
<WinProbabilityChart
chartData={results.chart}
variants={sortedVariants}
- timeBucket={timeBucket}
+ timeBucket={results.resolvedTimeBucket || timeBucket}
/>
</div>
diff --git a/web/app/pages/Project/tabs/Experiments/ExperimentSettingsModal.tsx b/web/app/pages/Project/tabs/Experiments/ExperimentSettingsModal.tsx
index d57e8ae2e..5bada1734 100644
--- a/web/app/pages/Project/tabs/Experiments/ExperimentSettingsModal.tsx
+++ b/web/app/pages/Project/tabs/Experiments/ExperimentSettingsModal.tsx
@@ -14,6 +14,7 @@ import {
TrendUpIcon,
TrendDownIcon,
XIcon,
+ CalculatorIcon,
} from '@phosphor-icons/react'
import { useState, useEffect, useMemo, useRef } from 'react'
import { useTranslation } from 'react-i18next'
@@ -82,10 +83,14 @@ const ExperimentSettingsModal = ({
const [name, setName] = useState('')
const [description, setDescription] = useState('')
+ const [hypothesis, setHypothesis] = useState('')
const [featureFlagKey, setFeatureFlagKey] = useState('')
const [goalId, setGoalId] = useState<string>('')
const [goalDirection, setGoalDirection] = useState<GoalDirection>('increase')
const [variants, setVariants] = useState<ExperimentVariant[]>(defaultVariants)
+ const [baselineConversionRate, setBaselineConversionRate] = useState(5)
+ const [minimumDetectableEffect, setMinimumDetectableEffect] = useState(10)
+ const [dailyExposures, setDailyExposures] = useState(500)
const [showAdvanced, setShowAdvanced] = useState(false)
const [exposureTrigger, setExposureTrigger] =
@@ -111,10 +116,14 @@ const ExperimentSettingsModal = ({
const resetForm = () => {
setName('')
setDescription('')
+ setHypothesis('')
setFeatureFlagKey('')
setGoalId('')
setGoalDirection('increase')
setVariants(defaultVariants)
+ setBaselineConversionRate(5)
+ setMinimumDetectableEffect(10)
+ setDailyExposures(500)
setShowAdvanced(false)
setExposureTrigger('feature_flag')
setCustomEventName('')
@@ -134,6 +143,7 @@ const ExperimentSettingsModal = ({
}
setName(experiment.name)
setDescription(experiment.description || '')
+ setHypothesis(experiment.hypothesis || '')
setFeatureFlagKey(experiment.featureFlagKey || '')
setGoalId(experiment.goalId || '')
setVariants(
@@ -260,6 +270,13 @@ const ExperimentSettingsModal = ({
newErrors.featureFlag = t('experiments.selectFeatureFlag')
}
+ if (
+ selectedFeatureFlag?.experimentId &&
+ selectedFeatureFlag.experimentId !== experimentId
+ ) {
+ newErrors.featureFlag = 'This feature flag is already linked elsewhere.'
+ }
+
if (exposureTrigger === 'custom_event' && !customEventName.trim()) {
newErrors.customEvent = t('experiments.exposureTrigger.eventName')
}
@@ -311,12 +328,14 @@ const ExperimentSettingsModal = ({
)
formData.set('name', name.trim())
formData.set('description', description.trim())
+ formData.set('hypothesis', hypothesis.trim())
formData.set('exposureTrigger', exposureTrigger)
formData.set(
'customEventName',
exposureTrigger === 'custom_event' ? customEventName.trim() : '',
)
formData.set('multipleVariantHandling', multipleVariantHandling)
+ formData.set('filterInternalUsers', 'true')
formData.set('featureFlagMode', featureFlagMode)
formData.set(
'featureFlagKey',
@@ -422,8 +441,165 @@ const ExperimentSettingsModal = ({
...featureFlags.map((flag) => ({ value: flag.id, label: flag.key })),
]
+ const selectedFeatureFlag = useMemo(
+ () => featureFlags.find((flag) => flag.id === existingFeatureFlagId),
+ [featureFlags, existingFeatureFlagId],
+ )
+
const totalPercentage = _sum(variants.map((v) => v.rolloutPercentage))
const isPercentageValid = totalPercentage === 100
+ const minAllocation = Math.min(...variants.map((v) => v.rolloutPercentage))
+ const maxAllocation = Math.max(...variants.map((v) => v.rolloutPercentage))
+
+ const sampleEstimate = useMemo(() => {
+ const baseline = Math.max(0.001, baselineConversionRate / 100)
+ const relativeLift = Math.max(0.001, minimumDetectableEffect / 100)
+ const target = Math.min(0.999, baseline * (1 + relativeLift))
+ const pooled = (baseline + target) / 2
+ const delta = Math.abs(target - baseline)
+ const zAlpha = 1.96
+ const zBeta = 0.84
+ const numerator = Math.pow(
+ zAlpha * Math.sqrt(2 * pooled * (1 - pooled)) +
+ zBeta * Math.sqrt(baseline * (1 - baseline) + target * (1 - target)),
+ 2,
+ )
+ const perVariant = Math.ceil(numerator / Math.pow(delta, 2))
+ const slowestDailyTraffic = Math.max(
+ 0,
+ dailyExposures * (minAllocation / 100),
+ )
+ const estimatedDays =
+ slowestDailyTraffic > 0
+ ? Math.ceil(perVariant / slowestDailyTraffic)
+ : null
+
+ return {
+ perVariant,
+ total: perVariant * variants.length,
+ estimatedDays,
+ targetRate: target * 100,
+ }
+ }, [
+ baselineConversionRate,
+ minimumDetectableEffect,
+ dailyExposures,
+ minAllocation,
+ variants.length,
+ ])
+
+ const launchGuardrails = useMemo(() => {
+ const items: Array<{ severity: 'blocker' | 'warning'; message: string }> =
+ []
+
+ if (!goalId) {
+ items.push({
+ severity: 'blocker',
+ message: 'Add a goal before launch so conversions can be attributed.',
+ })
+ }
+
+ if (
+ exposureTrigger === 'custom_event' &&
+ customEventName.trim().length === 0
+ ) {
+ items.push({
+ severity: 'blocker',
+ message: 'Add the custom exposure event name before launch.',
+ })
+ }
+
+ if (featureFlagMode === 'link' && !existingFeatureFlagId) {
+ items.push({
+ severity: 'blocker',
+ message: 'Select the feature flag that will serve this experiment.',
+ })
+ }
+
+ if (
+ selectedFeatureFlag?.experimentId &&
+ selectedFeatureFlag.experimentId !== experimentId
+ ) {
+ items.push({
+ severity: 'blocker',
+ message: 'The selected feature flag is already linked elsewhere.',
+ })
+ }
+
+ if (!isPercentageValid) {
+ items.push({
+ severity: 'blocker',
+ message: 'Variant rollout percentages must add up to 100%.',
+ })
+ }
+
+ if (variants.some((variant) => variant.rolloutPercentage <= 0)) {
+ items.push({
+ severity: 'blocker',
+ message: 'Every variant needs traffic before launch.',
+ })
+ }
+
+ if (maxAllocation - minAllocation > 10) {
+ items.push({
+ severity: 'warning',
+ message: 'Allocation is uneven, results will take longer to read.',
+ })
+ }
+
+ if (minAllocation > 0 && minAllocation < 10) {
+ items.push({
+ severity: 'warning',
+ message: 'One variant has very low traffic and may miss issues.',
+ })
+ }
+
+ if (sampleEstimate.estimatedDays && sampleEstimate.estimatedDays > 56) {
+ items.push({
+ severity: 'warning',
+ message: 'The current sample estimate is longer than eight weeks.',
+ })
+ }
+
+ if (sampleEstimate.perVariant < 100) {
+ items.push({
+ severity: 'warning',
+ message: 'The estimate is very small, check the baseline and MDE.',
+ })
+ }
+
+ if (!hypothesis.trim()) {
+ items.push({
+ severity: 'warning',
+ message: 'A hypothesis helps interpret mixed or flat results later.',
+ })
+ }
+
+ if (multipleVariantHandling === 'first_exposure') {
+ items.push({
+ severity: 'warning',
+ message: 'First exposure handling can hide variant switching issues.',
+ })
+ }
+
+ return items
+ }, [
+ customEventName,
+ experimentId,
+ exposureTrigger,
+ existingFeatureFlagId,
+ featureFlagMode,
+ goalId,
+ hypothesis,
+ isPercentageValid,
+ maxAllocation,
+ minAllocation,
+ multipleVariantHandling,
+ sampleEstimate.estimatedDays,
+ sampleEstimate.perVariant,
+ selectedFeatureFlag?.experimentId,
+ variants,
+ ])
return (
<Dialog className='relative z-40' open={isOpen} onClose={onClose}>
@@ -436,7 +612,7 @@ const ExperimentSettingsModal = ({
<div className='flex min-h-full items-center justify-center p-4'>
<DialogPanel
transition
- className='w-full max-w-2xl transform rounded-xl bg-white transition-all data-closed:translate-y-4 data-closed:opacity-0 data-enter:ease-out data-leave:duration-200 data-leave:ease-in dark:bg-slate-950'
+ className='w-full max-w-3xl transform rounded-xl bg-white transition-all data-closed:translate-y-4 data-closed:opacity-0 data-enter:ease-out data-leave:duration-200 data-leave:ease-in dark:bg-slate-950'
>
{isLoading ? (
<div className='flex min-h-[300px] items-center justify-center'>
@@ -491,6 +667,20 @@ const ExperimentSettingsModal = ({
onChange={(e) => setDescription(e.target.value)}
placeholder={t('experiments.descriptionPlaceholder')}
/>
+
+ <Input
+ label={
+ <span className='flex items-center gap-1.5'>
+ Hypothesis
+ <Text size='xs' colour='muted'>
+ ({t('common.optional')})
+ </Text>
+ </span>
+ }
+ value={hypothesis}
+ onChange={(e) => setHypothesis(e.target.value)}
+ placeholder='Changing X should improve Y because...'
+ />
</div>
{featureFlagMode === 'create' ? (
@@ -691,6 +881,40 @@ const ExperimentSettingsModal = ({
{totalPercentage}%
</Text>
</div>
+ <div className='mt-3 rounded-md bg-gray-50 px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-900/40 dark:ring-slate-800'>
+ <Text size='xs' weight='medium' colour='secondary'>
+ Variant mapping
+ </Text>
+ <div className='mt-2 grid gap-2 sm:grid-cols-2'>
+ {variants.map((variant) => (
+ <div
+ key={variant.key}
+ className='flex items-center justify-between gap-3 text-xs'
+ >
+ <div className='min-w-0'>
+ <Text as='p' size='xs' weight='medium' truncate>
+ {variant.name}
+ </Text>
+ <Text as='p' size='xs' colour='muted' code>
+ {variant.key}
+ </Text>
+ </div>
+ <Text
+ as='span'
+ size='xs'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {variant.rolloutPercentage}%
+ </Text>
+ </div>
+ ))}
+ </div>
+ <Text size='xs' colour='muted' className='mt-2'>
+ The feature flag decides eligibility. The experiment
+ maps eligible profiles into these variants.
+ </Text>
+ </div>
</div>
<div>
@@ -774,6 +998,172 @@ const ExperimentSettingsModal = ({
) : null}
</div>
+ <div className='rounded-lg bg-gray-50 p-4 ring-1 ring-gray-200 dark:bg-slate-900/40 dark:ring-slate-800'>
+ <div className='flex flex-wrap items-start justify-between gap-3'>
+ <div>
+ <div className='flex items-center gap-2'>
+ <CalculatorIcon className='size-4 text-gray-600 dark:text-gray-300' />
+ <Text size='sm' weight='semibold'>
+ Sample size estimate
+ </Text>
+ </div>
+ <Text size='xs' colour='muted' className='mt-1'>
+ Two-sided 95% confidence, 80% power. Use this as a
+ planning estimate, not a guarantee.
+ </Text>
+ </div>
+ <div className='text-right'>
+ <Text
+ as='p'
+ size='lg'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {sampleEstimate.perVariant.toLocaleString()}
+ </Text>
+ <Text as='p' size='xs' colour='muted'>
+ exposures per variant
+ </Text>
+ </div>
+ </div>
+
+ <div className='mt-4 grid gap-3 sm:grid-cols-3'>
+ <Input
+ label='Baseline conversion'
+ inputMode='decimal'
+ value={baselineConversionRate}
+ onChange={(e) =>
+ setBaselineConversionRate(
+ Math.min(
+ 99,
+ Math.max(0.1, Number(e.target.value) || 0.1),
+ ),
+ )
+ }
+ classes={{ input: 'px-2.5 py-1.5' }}
+ />
+ <Input
+ label='MDE lift'
+ inputMode='decimal'
+ value={minimumDetectableEffect}
+ onChange={(e) =>
+ setMinimumDetectableEffect(
+ Math.min(
+ 500,
+ Math.max(1, Number(e.target.value) || 1),
+ ),
+ )
+ }
+ classes={{ input: 'px-2.5 py-1.5' }}
+ />
+ <Input
+ label='Daily exposures'
+ inputMode='numeric'
+ value={dailyExposures}
+ onChange={(e) =>
+ setDailyExposures(
+ Math.max(0, Number(e.target.value) || 0),
+ )
+ }
+ classes={{ input: 'px-2.5 py-1.5' }}
+ />
+ </div>
+
+ <div className='mt-3 grid gap-2 text-xs sm:grid-cols-3'>
+ <div className='rounded-md bg-white px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-950 dark:ring-slate-800'>
+ <Text as='p' size='xs' colour='muted'>
+ Target rate
+ </Text>
+ <Text
+ as='p'
+ size='sm'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {sampleEstimate.targetRate.toFixed(2)}%
+ </Text>
+ </div>
+ <div className='rounded-md bg-white px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-950 dark:ring-slate-800'>
+ <Text as='p' size='xs' colour='muted'>
+ Total sample
+ </Text>
+ <Text
+ as='p'
+ size='sm'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {sampleEstimate.total.toLocaleString()}
+ </Text>
+ </div>
+ <div className='rounded-md bg-white px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-950 dark:ring-slate-800'>
+ <Text as='p' size='xs' colour='muted'>
+ Runtime estimate
+ </Text>
+ <Text
+ as='p'
+ size='sm'
+ weight='semibold'
+ className='tabular-nums'
+ >
+ {sampleEstimate.estimatedDays
+ ? `${sampleEstimate.estimatedDays} days`
+ : 'Add traffic'}
+ </Text>
+ </div>
+ </div>
+ </div>
+
+ <div className='rounded-lg bg-gray-50 p-4 ring-1 ring-gray-200 dark:bg-slate-900/40 dark:ring-slate-800'>
+ <div className='flex items-center justify-between gap-3'>
+ <Text size='sm' weight='semibold'>
+ Launch guardrails
+ </Text>
+ <Text
+ size='xs'
+ weight='medium'
+ className={
+ launchGuardrails.some(
+ (item) => item.severity === 'blocker',
+ )
+ ? 'text-red-600 dark:text-red-400'
+ : launchGuardrails.length > 0
+ ? 'text-yellow-700 dark:text-yellow-300'
+ : 'text-green-600 dark:text-green-400'
+ }
+ >
+ {launchGuardrails.some(
+ (item) => item.severity === 'blocker',
+ )
+ ? 'Blocked'
+ : launchGuardrails.length > 0
+ ? 'Review'
+ : 'Ready'}
+ </Text>
+ </div>
+ {launchGuardrails.length > 0 ? (
+ <ul className='mt-3 space-y-2'>
+ {launchGuardrails.map((item, index) => (
+ <li
+ key={`${item.severity}-${index}`}
+ className={cx(
+ 'rounded-md px-3 py-2 text-xs ring-1',
+ item.severity === 'blocker'
+ ? 'bg-red-50 text-red-800 ring-red-200 dark:bg-red-900/20 dark:text-red-200 dark:ring-red-800/70'
+ : 'bg-yellow-50 text-yellow-900 ring-yellow-200 dark:bg-yellow-900/20 dark:text-yellow-100 dark:ring-yellow-800/70',
+ )}
+ >
+ {item.message}
+ </li>
+ ))}
+ </ul>
+ ) : (
+ <Text size='xs' colour='muted' className='mt-2'>
+ No launch blockers detected.
+ </Text>
+ )}
+ </div>
+
<div className='border-t border-gray-200 pt-4 dark:border-slate-700'>
<button
type='button'
@@ -865,6 +1255,30 @@ const ExperimentSettingsModal = ({
{errors.featureFlag}
</Text>
) : null}
+ {selectedFeatureFlag ? (
+ <div className='mt-2 rounded-md bg-gray-50 px-3 py-2 ring-1 ring-gray-200 dark:bg-slate-900/40 dark:ring-slate-800'>
+ <div className='flex flex-wrap items-center justify-between gap-2'>
+ <Text size='xs' colour='secondary'>
+ Linked flag
+ </Text>
+ <Text size='xs' colour='secondary' code>
+ {selectedFeatureFlag.key}
+ </Text>
+ </div>
+ <Text
+ size='xs'
+ colour='muted'
+ className='mt-1'
+ >
+ {selectedFeatureFlag.rolloutPercentage}%
+ rollout,{' '}
+ {selectedFeatureFlag.enabled
+ ? 'enabled'
+ : 'disabled'}
+ . Starting the experiment enables the flag.
+ </Text>
+ </div>
+ ) : null}
</div>
) : null}
diff --git a/web/app/pages/Project/tabs/Experiments/ExperimentsView.tsx b/web/app/pages/Project/tabs/Experiments/ExperimentsView.tsx
index aeb5732a2..a8e758c42 100644
--- a/web/app/pages/Project/tabs/Experiments/ExperimentsView.tsx
+++ b/web/app/pages/Project/tabs/Experiments/ExperimentsView.tsx
@@ -97,6 +97,64 @@ interface ExperimentRowProps {
onComplete: (id: string) => void
}
+const getLaunchGuardrails = (experiment: Experiment) => {
+ const blockers: string[] = []
+ const warnings: string[] = []
+ const variants = experiment.variants || []
+ const totalAllocation = variants.reduce(
+ (sum, variant) => sum + variant.rolloutPercentage,
+ 0,
+ )
+ const allocations = variants.map((variant) => variant.rolloutPercentage)
+ const minAllocation = allocations.length ? Math.min(...allocations) : 0
+ const maxAllocation = allocations.length ? Math.max(...allocations) : 0
+
+ if (!experiment.goalId) {
+ blockers.push('Missing goal')
+ }
+
+ if (variants.length < 2) {
+ blockers.push('Needs at least two variants')
+ }
+
+ if (variants.filter((variant) => variant.isControl).length !== 1) {
+ blockers.push('Needs one control')
+ }
+
+ if (totalAllocation !== 100) {
+ blockers.push('Allocation must total 100%')
+ }
+
+ if (variants.some((variant) => variant.rolloutPercentage <= 0)) {
+ blockers.push('Every variant needs traffic')
+ }
+
+ if (
+ experiment.exposureTrigger === 'custom_event' &&
+ !experiment.customEventName?.trim()
+ ) {
+ blockers.push('Missing exposure event')
+ }
+
+ if (experiment.featureFlagMode === 'link' && !experiment.featureFlagId) {
+ blockers.push('Missing linked feature flag')
+ }
+
+ if (maxAllocation - minAllocation > 10) {
+ warnings.push('Uneven allocation')
+ }
+
+ if (minAllocation > 0 && minAllocation < 10) {
+ warnings.push('Low traffic variant')
+ }
+
+ if (!experiment.hypothesis?.trim()) {
+ warnings.push('No hypothesis')
+ }
+
+ return { blockers, warnings }
+}
+
const ExperimentRow = ({
experiment,
onDelete,
@@ -112,6 +170,11 @@ const ExperimentRow = ({
const [actionLoading, setActionLoading] = useState(false)
const variantsCount = experiment.variants?.length || 0
+ const launchGuardrails = useMemo(
+ () => getLaunchGuardrails(experiment),
+ [experiment],
+ )
+ const launchBlockerText = launchGuardrails.blockers.join(', ')
const isEditDisabled =
experiment.status === 'running' || experiment.status === 'completed'
const statusBadgeColour: 'slate' | 'green' | 'yellow' | 'sky' =
@@ -186,6 +249,23 @@ const ExperimentRow = ({
label={`${variantsCount} ${t('experiments.variants')}`}
className='text-[0.625rem] leading-3'
/>
+ {launchGuardrails.blockers.length > 0 &&
+ experiment.status === 'draft' ? (
+ <Badge
+ colour='red'
+ label='Launch blocked'
+ className='text-[0.625rem] leading-3'
+ />
+ ) : null}
+ {launchGuardrails.blockers.length === 0 &&
+ launchGuardrails.warnings.length > 0 &&
+ experiment.status === 'draft' ? (
+ <Badge
+ colour='yellow'
+ label='Review config'
+ className='text-[0.625rem] leading-3'
+ />
+ ) : null}
</div>
{experiment.description ? (
<Text className='mt-1' as='p' size='sm' colour='secondary'>
@@ -197,7 +277,25 @@ const ExperimentRow = ({
{t('experiments.hypothesis')}: {experiment.hypothesis}
</Text>
) : null}
- {/* Timestamps */}
+ <div className='mt-2 flex flex-wrap items-center gap-x-3 gap-y-1'>
+ <Text as='span' size='xs' colour='muted'>
+ Flag:{' '}
+ <Text as='span' size='xs' colour='secondary' code>
+ {experiment.featureFlagKey ||
+ experiment.featureFlagId ||
+ 'created on launch'}
+ </Text>
+ </Text>
+ <Text as='span' size='xs' colour='muted'>
+ Split:{' '}
+ {experiment.variants
+ .map(
+ (variant) =>
+ `${variant.key} ${variant.rolloutPercentage}%`,
+ )
+ .join(' / ')}
+ </Text>
+ </div>
<div className='mt-2 flex flex-wrap items-center gap-x-3 gap-y-1'>
{experiment.startedAt ? (
<Text as='span' size='xs' colour='muted'>
@@ -218,9 +316,9 @@ const ExperimentRow = ({
{/* Action buttons based on status */}
{experiment.status === 'draft' ? (
<>
- {!experiment.goalId ? (
+ {launchGuardrails.blockers.length > 0 ? (
<Tooltip
- text={t('experiments.needGoal')}
+ text={launchBlockerText || t('experiments.needGoal')}
tooltipNode={
<span className='inline-flex'>
<button
diff --git a/web/app/pages/Project/tabs/FeatureFlags/FeatureFlagsView.tsx b/web/app/pages/Project/tabs/FeatureFlags/FeatureFlagsView.tsx
index cc2983982..a415db840 100644
--- a/web/app/pages/Project/tabs/FeatureFlags/FeatureFlagsView.tsx
+++ b/web/app/pages/Project/tabs/FeatureFlags/FeatureFlagsView.tsx
@@ -204,9 +204,17 @@ const FeatureFlagRow = ({
timeFormat,
}: FeatureFlagRowProps) => {
const { t } = useTranslation()
+ const location = useLocation()
const [showDeleteModal, setShowDeleteModal] = useState(false)
const targetingRulesCount = flag.targetingRules?.length || 0
+ const experimentSearch = useMemo(() => {
+ if (!flag.experimentId) return ''
+ const params = new URLSearchParams(location.search)
+ params.set('tab', 'experiments')
+ params.set('experimentId', flag.experimentId)
+ return params.toString()
+ }, [flag.experimentId, location.search])
return (
<>
@@ -263,12 +271,31 @@ const FeatureFlagRow = ({
colour='sky'
className='text-[0.625rem] leading-3'
/>
+ {flag.experimentId ? (
+ <Link
+ to={{ search: experimentSearch }}
+ onClick={(event) => event.stopPropagation()}
+ className='inline-flex'
+ >
+ <Badge
+ label='Experiment linked'
+ colour='indigo'
+ className='text-[0.625rem] leading-3'
+ />
+ </Link>
+ ) : null}
</div>
{flag.description ? (
<Text className='mt-1' as='p' size='sm' colour='secondary'>
{flag.description}
</Text>
) : null}
+ {flag.experimentId ? (
+ <Text className='mt-1' as='p' size='xs' colour='muted'>
+ This flag gates experiment eligibility. Variant split and
+ result decisions live in Experiments.
+ </Text>
+ ) : null}
{targetingRulesCount > 0 ? (
<Text className='mt-1' as='p' size='xs' colour='muted'>
{t('featureFlags.targetingRulesCount', {
diff --git a/web/app/routes/api.analytics.ts b/web/app/routes/api.analytics.ts
index f29f30ef6..c47b17605 100644
--- a/web/app/routes/api.analytics.ts
+++ b/web/app/routes/api.analytics.ts
@@ -382,6 +382,7 @@ export async function action({ request }: ActionFunctionArgs) {
formatDateForBackend(params.from) || '',
formatDateForBackend(params.to) || '',
params.timezone,
+ params.filters || [],
)
return data<ProxyResponse<ExperimentResults>>({
data: result.data,
diff --git a/web/app/routes/projects.$id.tsx b/web/app/routes/projects.$id.tsx
index 4fbd0457e..808d8e7f9 100644
--- a/web/app/routes/projects.$id.tsx
+++ b/web/app/routes/projects.$id.tsx
@@ -1960,11 +1960,13 @@ export async function action({ request, params }: ActionFunctionArgs) {
const fromDate = formData.get('from')?.toString() || ''
const toDate = formData.get('to')?.toString() || ''
const tz = formData.get('timezone')?.toString() || ''
+ const filters = formData.get('filters')?.toString() || '[]'
const params = new URLSearchParams({ period, timeBucket })
if (fromDate) params.append('from', fromDate)
if (toDate) params.append('to', toDate)
if (tz) params.append('timezone', tz)
+ params.append('filters', filters)
const result = await serverFetch(
request,