Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

18.9 KB
Raw Blame
import { describe, expect, it, vi } from "vitest";

vi.mock("../../db/clickhouse/clickhouse.js", () => ({
  clickhouse: { query: vi.fn() },
}));
vi.mock("../../db/postgres/postgres.js", () => ({
  db: {},
}));

import { getTimeStatement } from "../analytics/utils/timeWindow.js";
import { buildExperimentResultQueries, EXPERIMENT_UNIT, resolveExperimentWindow } from "./getExperimentResults.js";
import { buildCumulativeSeries } from "./getExperimentTimeseries.js";
import {
  buildExperimentResults,
  experimentFlagError,
  experimentUpdateError,
  flagUpdateForStatusChange,
  rolloutWinner,
  SERVER_ONLY_FLAG_ERROR,
} from "./utils.js";

const CAMPAIGN_FILTER = JSON.stringify([{ parameter: "utm_campaign", type: "equals", value: ["recipe_book_2026"] }]);

const query = {
  filters: CAMPAIGN_FILTER,
  start_date: "",
  end_date: "",
  time_zone: "UTC",
};

const buildQueries = () =>
  buildExperimentResultQueries({
    query,
    siteId: 1,
    flagKey: "recipe_book_test",
    variantKeys: ["control", "treatment"],
    goalCondition: "type = 'form_submit' AND JSONExtractString(toString(props), 'formId') = 'gform_115'",
  });

describe("experiment result queries", () => {
  it("qualifies the session by its landing campaign before selecting later exposure and goal events", () => {
    const { exposureQuery } = buildQueries();

    expect(exposureQuery).toContain("FilteredSessions AS");
    expect(exposureQuery).toContain("argMinIf(url_parameters['utm_campaign'], timestamp, url_parameters['utm_campaign'] != '') AS utm_campaign");
    expect(exposureQuery).toContain("WHERE 1 = 1 AND utm_campaign = 'recipe_book_2026'");
    expect(exposureQuery.match(/INNER JOIN FilteredSessions USING \(session_id\)/g)).toHaveLength(2);
    expect(exposureQuery).toContain("event_name = 'feature_flag_exposure'");
    expect(exposureQuery).toContain("type = 'form_submit'");
    expect(exposureQuery).toContain("JSONExtractString(toString(props), 'value') IN ('control', 'treatment')");

    // The campaign condition belongs only to FilteredSessions, not separately
    // to the exposure and goal event rows.
    expect(exposureQuery.match(/utm_campaign = 'recipe_book_2026'/g)).toHaveLength(1);
  });

  it("uses the same campaign-qualified sessions for assignment fallback and goal events", () => {
    const { assignmentQuery } = buildQueries();

    expect(assignmentQuery).toContain("FilteredSessions AS");
    expect(assignmentQuery).toContain("WHERE 1 = 1 AND utm_campaign = 'recipe_book_2026'");
    expect(assignmentQuery.match(/INNER JOIN FilteredSessions USING \(session_id\)/g)).toHaveLength(2);
    expect(assignmentQuery).toContain("feature_flags['recipe_book_test'] IN ('control', 'treatment')");
    expect(assignmentQuery).toContain("type = 'form_submit'");
    expect(assignmentQuery.match(/utm_campaign = 'recipe_book_2026'/g)).toHaveLength(1);
  });

  it("attributes a session to its first exposure variant instead of grouping it into every observed variant", () => {
    const { exposureQuery } = buildQueries();

    expect(exposureQuery).toContain("argMin(JSONExtractString(toString(props), 'value'), timestamp_ms) AS variant");
    expect(exposureQuery).not.toContain("GROUP BY session_id, variant");
  });

  it("attributes assignment fallback to the first assigned variant per session", () => {
    const { assignmentQuery } = buildQueries();

    expect(assignmentQuery).toContain("argMin(feature_flags['recipe_book_test'], timestamp_ms) AS variant");
    expect(assignmentQuery).not.toContain("GROUP BY session_id, variant");
  });

  it("groups exposures and goals by visitor, falling back to the session", () => {
    const { exposureQuery, assignmentQuery } = buildQueries();

    expect(EXPERIMENT_UNIT).toBe("ifNull(nullIf(session_visitor, ''), events.session_id)");
    for (const sql of [exposureQuery, assignmentQuery]) {
      expect(sql.split(`${EXPERIMENT_UNIT} AS unit`)).toHaveLength(3);
      expect(sql).toContain("GROUP BY unit");
      expect(sql).toContain("uniqExactArray(u.session_ids) AS sessions");
    }
  });

  it("counts a conversion from a later session against the unit's first exposure", () => {
    const { exposureQuery, assignmentQuery } = buildQueries();

    for (const sql of [exposureQuery, assignmentQuery]) {
      expect(sql).toContain("LEFT JOIN goal_units g ON g.unit = u.unit");
      expect(sql).toContain("uniqExactIf(u.unit, g.last_goal_at >= u.exposed_at) AS conversions");
      expect(sql).toContain("uniqExact(u.unit) AS units");
    }
  });

  it("dates a unit's conversion by its first goal at or after exposure, in the site time zone", () => {
    const { exposureTimeseriesQuery, assignmentTimeseriesQuery } = buildQueries();

    for (const sql of [exposureTimeseriesQuery, assignmentTimeseriesQuery]) {
      expect(sql).toContain("minIf(g.goal_at, g.goal_at >= u.exposed_at) AS converted_at");
      expect(sql).toContain("toDate(exposed_at, 'UTC') AS day");
      expect(sql).toContain("WHERE converted_at > toDateTime(0)");
      expect(sql).toContain("GROUP BY variant, day");
    }
    expect(exposureTimeseriesQuery).toContain("event_name = 'feature_flag_exposure'");
    expect(assignmentTimeseriesQuery).toContain("feature_flags['recipe_book_test'] IN ('control', 'treatment')");
  });
});

describe("experiment unit and timing", () => {
  it("gives every event of a session the visitor id seen in that session", () => {
    const { exposureQuery, assignmentQuery } = buildQueries();

    for (const sql of [exposureQuery, assignmentQuery]) {
      expect(sql).toContain("SELECT session_id AS visitor_session_id, any(visitor_id) AS session_visitor");
      expect(sql.match(/LEFT JOIN session_visitors ON session_visitors.visitor_session_id = events.session_id/g)).toHaveLength(2);
    }
  });

  it("orders exposures and goals by millisecond timestamps", () => {
    const { exposureQuery } = buildQueries();

    expect(exposureQuery).toContain("min(timestamp_ms) AS exposed_at");
    expect(exposureQuery).toContain("timestamp_ms AS goal_at");
    expect(exposureQuery).toContain("uniqExactIf(u.unit, g.last_goal_at >= u.exposed_at) AS conversions");
  });

  it("bounds a run to its exact start and end next to the whole-second window", () => {
    const sql = buildExperimentResultQueries({
      query: { filters: "", start_datetime: "2026-09-25 21:29:05", end_datetime: "2026-09-25 21:37:37" } as never,
      siteId: 1,
      flagKey: "recipe_book_test",
      variantKeys: ["control", "treatment"],
      goalCondition: "type = 'pageview'",
      exact: { start: "2026-09-25 21:29:05.930", end: "2026-09-25 21:37:36.163" },
    }).exposureQuery;

    expect(sql).toContain("timestamp >= toDateTime('2026-09-25 21:29:05', 'UTC')");
    expect(sql).toContain("timestamp_ms >= toDateTime64('2026-09-25 21:29:05.930', 3, 'UTC')");
    expect(sql).toContain("timestamp_ms <= toDateTime64('2026-09-25 21:37:36.163', 3, 'UTC')");
  });
});

describe("resolveExperimentWindow", () => {
  const rangeQuery = { filters: "", start_date: "2026-09-01", end_date: "2026-09-26", time_zone: "Europe/Amsterdam" };
  const now = new Date("2026-09-26T10:00:00.400Z");

  it("measures a running experiment from its start until now", () => {
    const { query, window } = resolveExperimentWindow(rangeQuery, { startedAt: "2026-09-25 21:29:05.93", endedAt: null }, now);

    expect(query).toMatchObject({ start_date: "", end_date: "", start_datetime: "2026-09-25 21:29:05" });
    expect(query.end_datetime).toBe("2026-09-26 10:00:02");
    expect(window).toEqual({ mode: "experiment", start: "2026-09-25 21:29:05", end: null });
    expect(getTimeStatement(query)).toContain("timestamp >= toDateTime('2026-09-25 21:29:05', 'UTC')");
  });

  it("stops a completed experiment at its end", () => {
    const { query, window, exact } = resolveExperimentWindow(
      rangeQuery,
      { startedAt: "2026-09-25 21:29:05.93", endedAt: "2026-09-25 21:37:36.163" },
      now
    );

    expect(query.end_datetime).toBe("2026-09-25 21:37:37");
    expect(window).toEqual({ mode: "experiment", start: "2026-09-25 21:29:05", end: "2026-09-25 21:37:37" });
    expect(exact).toEqual({ start: "2026-09-25 21:29:05.930", end: "2026-09-25 21:37:36.163" });
  });

  it("uses the page date range when asked, or before the experiment starts", () => {
    const started = { startedAt: "2026-09-25 21:29:05.93", endedAt: null };
    expect(resolveExperimentWindow({ ...rangeQuery, window: "range" as const }, started, now).query).toMatchObject(rangeQuery);
    expect(resolveExperimentWindow(rangeQuery, { startedAt: null, endedAt: null }, now).window).toEqual({
      mode: "range",
      start: "2026-09-01",
      end: "2026-09-26",
    });
  });
});

describe("buildCumulativeSeries", () => {
  it("fills every day of the window and accumulates units and conversions", () => {
    const series = buildCumulativeSeries(
      ["control", "test"],
      [
        { variant: "control", day: "2026-09-24", units: "4", conversions: "1" },
        { variant: "test", day: "2026-09-24", units: "5", conversions: "0" },
        { variant: "test", day: "2026-09-26", units: "5", conversions: "3" },
      ],
      { mode: "experiment", start: "2026-09-23 23:30:00", end: null },
      "Europe/Amsterdam",
      "2026-09-26"
    );

    expect(series.map(s => s.points.map(p => p.date))).toEqual([
      ["2026-09-24", "2026-09-25", "2026-09-26"],
      ["2026-09-24", "2026-09-25", "2026-09-26"],
    ]);
    expect(series[0].points.map(p => [p.units, p.conversions])).toEqual([[4, 1], [4, 1], [4, 1]]);
    expect(series[1].points[2]).toEqual({ date: "2026-09-26", units: 10, conversions: 3, conversionRate: 0.3 });
  });

  it("uses the data's own days for an all-time range", () => {
    const series = buildCumulativeSeries(
      ["control"],
      [{ variant: "control", day: "2026-09-20", units: 2, conversions: 1 }, { variant: "control", day: "2026-09-21", units: 2, conversions: 0 }],
      { mode: "range", start: null, end: null },
      "UTC"
    );

    expect(series[0].points.map(p => p.date)).toEqual(["2026-09-20", "2026-09-21"]);
  });

  it("keeps the most recent days of a long window and carries earlier totals", () => {
    const series = buildCumulativeSeries(
      ["control"],
      [
        { variant: "control", day: "2024-01-01", units: 10, conversions: 5 },
        { variant: "control", day: "2026-09-26", units: 10, conversions: 1 },
      ],
      { mode: "range", start: null, end: null },
      "UTC"
    );

    const points = series[0].points;
    expect(points).toHaveLength(366);
    expect(points[points.length - 1]).toEqual({ date: "2026-09-26", units: 20, conversions: 6, conversionRate: 0.3 });
    expect(points[0]).toMatchObject({ date: "2025-09-26", units: 10, conversions: 5 });
  });
});

describe("buildExperimentResults", () => {
  it("uses units, not sessions, as the conversion-rate denominator", () => {
    const results = buildExperimentResults(
      ["control", "test"],
      [
        { variant: "control", units: 10, sessions: 25, exposures: 40, conversions: 5 },
        { variant: "test", units: 10, sessions: 12, exposures: 15, conversions: 6 },
      ]
    );

    expect(results[0]).toMatchObject({ variant: "control", units: 10, sessions: 25, conversionRate: 0.5, isControl: true });
    expect(results[1].conversionRate).toBe(0.6);
    expect(results[1].lift).toBeCloseTo(0.2);
  });

  it("coerces ClickHouse UInt64 strings to numbers", () => {
    const [control] = buildExperimentResults(["control"], [
      { variant: "control", units: "4", sessions: "6", exposures: "8", conversions: "1" } as never,
    ]);

    expect(control).toMatchObject({ units: 4, sessions: 6, exposures: 8, conversions: 1, conversionRate: 0.25 });
  });
});

describe("rolloutWinner", () => {
  const flag = {
    rolloutPercentage: 40,
    variants: [],
    conditionSets: [
      {
        name: "Beta users",
        rules: [{ field: "country", operator: "equals", value: "NL" }],
        rolloutPercentage: 50,
        variants: [
          { key: "control", name: "Control", rolloutPercentage: 50 },
          { key: "variant_a", name: "Variant A", rolloutPercentage: 50, payload: { copy: "Try it" } },
        ],
      },
      { name: "Everyone else", rules: [], variants: [] },
    ],
  } as never;

  it("serves the winner to everyone the flag targets, keeping rules and variant details", () => {
    const rollout = rolloutWinner(flag, "variant_a");

    expect(rollout.rolloutPercentage).toBe(100);
    expect(rollout.conditionSets[0]).toMatchObject({
      name: "Beta users",
      rules: [{ field: "country", operator: "equals", value: "NL" }],
      rolloutPercentage: 100,
      variants: [
        { key: "control", name: "Control", rolloutPercentage: 0 },
        { key: "variant_a", name: "Variant A", rolloutPercentage: 100, payload: { copy: "Try it" } },
      ],
    });
    expect(rollout.conditionSets[1]).toEqual({ name: "Everyone else", rules: [], variants: [], rolloutPercentage: 100 });
    expect(rollout.variants).toEqual([]);
  });

  it("adds the winner to a set that never offered it, so its visitors don't fall through", () => {
    const rollout = rolloutWinner(
      {
        rolloutPercentage: 100,
        variants: [],
        conditionSets: [
          {
            name: "NL",
            rules: [],
            variants: [
              { key: "control", rolloutPercentage: 50 },
              { key: "variant_a", name: "Variant A", rolloutPercentage: 50, payload: { copy: "Try it" } },
            ],
          },
          {
            name: "BE",
            rules: [],
            variants: [
              { key: "control", rolloutPercentage: 50 },
              { key: "variant_b", rolloutPercentage: 50 },
            ],
          },
        ],
      } as never,
      "variant_a"
    );

    expect(rollout.conditionSets[1].variants).toEqual([
      { key: "control", rolloutPercentage: 0 },
      { key: "variant_b", rolloutPercentage: 0 },
      { key: "variant_a", name: "Variant A", rolloutPercentage: 100, payload: { copy: "Try it" } },
    ]);
  });
});

describe("flagUpdateForStatusChange", () => {
  const flag = (enabled: boolean) =>
    ({
      enabled,
      rolloutPercentage: 100,
      variants: [],
      conditionSets: [{ rules: [], variants: [{ key: "control", rolloutPercentage: 50 }, { key: "test", rolloutPercentage: 50 }] }],
    }) as never;

  it("switches the flag off when pausing and back on when resuming", () => {
    expect(flagUpdateForStatusChange("running", "paused", flag(true))).toEqual({ enabled: false });
    expect(flagUpdateForStatusChange("paused", "running", flag(false))).toEqual({ enabled: true });
  });

  it("switches a disabled flag on when a draft starts", () => {
    expect(flagUpdateForStatusChange("draft", "running", flag(false))).toEqual({ enabled: true });
  });

  it("leaves the flag alone when it is already right or nothing changes", () => {
    expect(flagUpdateForStatusChange("draft", "running", flag(true))).toBeNull();
    expect(flagUpdateForStatusChange("running", "running", flag(true))).toBeNull();
    expect(flagUpdateForStatusChange("running", "draft", flag(true))).toBeNull();
  });

  it("ships the winner, enabled, when completing", () => {
    const update = flagUpdateForStatusChange("paused", "completed", flag(false), "test")!;
    expect(update.enabled).toBe(true);
    expect(update.conditionSets?.[0].variants).toEqual([
      { key: "control", rolloutPercentage: 0 },
      { key: "test", rolloutPercentage: 100 },
    ]);
  });
});

describe("experiment arms", () => {
  it('only counts sessions assigned a configured variant, not the out-of-rollout "false"', () => {
    const { exposureQuery, assignmentQuery } = buildQueries();

    expect(exposureQuery).toContain("JSONExtractString(toString(props), 'value') IN ('control', 'treatment')");
    expect(assignmentQuery).toContain("feature_flags['recipe_book_test'] IN ('control', 'treatment')");
  });

  it("only counts exposures assigned as a variant, so a variant keyed false can't absorb other sessions", () => {
    const { exposureQuery } = buildQueries();

    expect(exposureQuery).toContain("JSONExtractString(toString(props), 'reason') IN ('variant', '')");
  });

  it("reports one result per configured variant and drops rows for any other value", () => {
    const results = buildExperimentResults(
      ["control", "treatment"],
      [
        { variant: "control", units: 100, sessions: 100, exposures: 120, conversions: 10 },
        { variant: "false", units: 900, sessions: 900, exposures: 900, conversions: 300 },
        { variant: "treatment", units: 100, sessions: 100, exposures: 110, conversions: 20 },
      ]
    );

    expect(results.map(result => result.variant)).toEqual(["control", "treatment"]);
    expect(results.reduce((sum, result) => sum + result.units, 0)).toBe(200);
    expect(results[1]).toMatchObject({ conversionRate: 0.2, lift: 1, isControl: false });
  });
});

describe("experimentFlagError", () => {
  it("accepts multivariate flags evaluated on the client", () => {
    expect(experimentFlagError({ flagType: "multivariate", runtime: "client" })).toBeNull();
    expect(experimentFlagError({ flagType: "multivariate", runtime: "both" })).toBeNull();
  });

  it("rejects server-only flags, which the tracking script never receives", () => {
    expect(experimentFlagError({ flagType: "multivariate", runtime: "server" })).toBe(SERVER_ONLY_FLAG_ERROR);
  });

  it("rejects flags that aren't multivariate", () => {
    expect(experimentFlagError({ flagType: "boolean", runtime: "client" })).toBe(
      "Experiments require a multivariate feature flag"
    );
  });
});

describe("experimentUpdateError", () => {
  const running = { status: "running" as const, featureFlagId: 7, winningVariant: null };
  const completed = { status: "completed" as const, featureFlagId: 7, winningVariant: "variant_a" };

  it("allows the usual lifecycle and edits that keep the flag", () => {
    expect(experimentUpdateError({ ...running, status: "draft" }, { status: "running", featureFlagId: 9 })).toBeNull();
    expect(experimentUpdateError(running, { status: "paused" })).toBeNull();
    expect(experimentUpdateError(running, { featureFlagId: 7 })).toBeNull();
    expect(experimentUpdateError(running, { status: "completed", winningVariant: "variant_a" })).toBeNull();
    expect(experimentUpdateError(completed, { winningVariant: "variant_a" })).toBeNull();
  });

  it("keeps a completed experiment completed", () => {
    expect(experimentUpdateError(completed, { status: "running" })).toMatch(/can't be reopened/);
    expect(experimentUpdateError(completed, { status: "completed" })).toBeNull();
  });

  it("keeps the flag once the experiment has started", () => {
    expect(experimentUpdateError(running, { featureFlagId: 9 })).toMatch(/flag can't change/);
    expect(experimentUpdateError({ ...running, status: "paused" }, { featureFlagId: 9 })).toMatch(/flag can't change/);
  });

  it("only sets the winner while completing", () => {
    expect(experimentUpdateError(completed, { winningVariant: "control" })).toMatch(/chosen when completing/);
    expect(experimentUpdateError(running, { winningVariant: "variant_a" })).toMatch(/chosen when completing/);
  });
});