Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

42.6 KB
Raw Blame
// @vitest-environment node

import { describe, expect, it } from "vitest";

import {
  EVENTS_FIELD_REGISTRY,
  type FieldRegistry,
  withFieldOptions,
} from "./fields";
import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
import {
  EVALUATOR_FIELD_REGISTRY,
  RULE_SAMPLE_FIELD_REGISTRY,
} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
import {
  SESSIONS_FIELD_REGISTRY,
  sessionsFieldRegistry,
} from "@/src/features/filters/config/sessionsSearchRegistry";
import { getSessionFilterConfig } from "@/src/features/filters/config/sessions-config";
import {
  USERS_FIELD_REGISTRY,
  LEGACY_USERS_FIELD_REGISTRY,
} from "@/src/features/filters/config/usersSearchRegistry";

import { usersEventsFilterConfig } from "@/src/features/filters/config/users-config";

import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";
import {
  SCORES_FIELD_REGISTRY,
  scoresFieldRegistry,
} from "@/src/features/scores/constants/scoresSearchRegistry";
import {
  getScoreFilterConfig,
  type ScoresTableHiddenColumn,
} from "@/src/features/filters/config/scores-config";
import type { FilterState } from "@langfuse/shared";
import { validateQuery } from "./validate";
import { DEFAULT_SEARCH_TYPE, planCommit } from "./commit";
import { filterStateToQueryText } from "./filter-state-to-query";
import {
  applyPick,
  planInputCompletions,
  type CompletionOption,
} from "./completions";
import {
  generateQueryCases,
  runSearchBarInvariants,
  type RegistryUnderTest,
} from "./searchBarInvariants";

describe("search bar invariants — Scores registry", () => {
  it("preserves parent-owned filters as skipped and rejects conflicting scoped fields", () => {
    const hiddenColumns: ScoresTableHiddenColumn[] = [
      "traceId",
      "observationId",
      "traceName",
      "userId",
      "traceTags",
      "jobConfigurationId",
    ];
    const registry = scoresFieldRegistry(getScoreFilterConfig(hiddenColumns));
    const parentFilters: FilterState = [
      { column: "traceId", type: "string", operator: "=", value: "trace" },
      {
        column: "observationId",
        type: "stringOptions",
        operator: "any of",
        value: ["", "observation"],
      },
      {
        column: "traceName",
        type: "stringOptions",
        operator: "any of",
        value: ["parent"],
      },
      {
        column: "userId",
        type: "stringOptions",
        operator: "any of",
        value: ["user"],
      },
      {
        column: "tags",
        type: "arrayOptions",
        operator: "all of",
        value: ["parent"],
      },
    ];
    const visibleFilter: FilterState = [
      {
        column: "name",
        type: "string",
        operator: "contains",
        value: "quality",
      },
    ];
    expect(
      filterStateToQueryText(
        [...parentFilters, ...visibleFilter],
        {},
        registry,
      ),
    ).toMatchObject({ text: "name:quality", skippedFilters: parentFilters });
    for (const query of [
      "trace_id:other",
      "observation_id:other",
      "trace_name:other",
      "user:other",
      "traceTags:other",
      "has:userId",
    ]) {
      expect(planCommit(query, undefined, registry).status, query).toBe(
        "invalid",
      );
    }
    expect(planCommit("quality", undefined, registry)).toMatchObject({
      status: "committed",
      filters: visibleFilter,
    });
    expect(registry.hasExample).toBeNull();
  });

  it("searches within a score name", () => {
    const result = planCommit("Rouge Score", undefined, SCORES_FIELD_REGISTRY);
    expect(result).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "name",
          type: "string",
          operator: "contains",
          value: "Rouge Score",
        },
      ],
    });
  });

  it("round-trips exact names, repeated ranges, values, metadata, and tag filters", () => {
    const sidebarFilters: FilterState[] = [
      [
        {
          column: "name",
          type: "stringOptions",
          operator: "any of",
          value: ["Rouge Score"],
        },
      ],
      [
        {
          column: "name",
          type: "stringOptions",
          operator: "any of",
          value: ["Rouge Score", "confidence"],
        },
      ],
      [
        {
          column: "name",
          type: "stringOptions",
          operator: "none of",
          value: ["Rouge Score"],
        },
      ],
      [
        { column: "value", type: "number", operator: ">=", value: 0.2 },
        { column: "value", type: "number", operator: "<", value: 0.8 },
      ],
      [
        {
          column: "booleanValue",
          type: "stringOptions",
          operator: "any of",
          value: ["true"],
        },
      ],
      [
        {
          column: "stringValue",
          type: "stringOptions",
          operator: "any of",
          value: ["true", "needs review"],
        },
      ],
      [
        {
          column: "metadata",
          type: "stringObject",
          key: "review team",
          operator: "=",
          value: "quality",
        },
      ],
      [
        {
          column: "tags",
          type: "arrayOptions",
          operator: "all of",
          value: ["billing", "urgent"],
        },
      ],
      [
        {
          column: "evaluatorId",
          type: "stringOptions",
          operator: "any of",
          value: ["legacy-evaluator"],
        },
      ],
    ];
    const view: RegistryUnderTest = {
      name: "scores",
      registry: SCORES_FIELD_REGISTRY,
      extraKeys: ["metadata.region", 'metadata."review team"', "has:userId"],
      scoreContexts: [],
      fieldValues: ["x", "true", "false", "0.5", "Rouge Score", "or", "a,b"],
      freeTextValues: [],
      sidebarFilters,
    };
    expect(runSearchBarInvariants(view)).toEqual([]);

    const exactName = filterStateToQueryText(
      sidebarFilters[0],
      {},
      SCORES_FIELD_REGISTRY,
    );
    expect(
      planCommit(exactName.text, undefined, SCORES_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "name",
          type: "stringOptions",
          operator: "any of",
          value: ["Rouge Score"],
        },
      ],
    });

    for (const filters of sidebarFilters.slice(1, -1)) {
      const projection = filterStateToQueryText(
        filters,
        {},
        SCORES_FIELD_REGISTRY,
      );
      expect(projection.skippedFilters).toEqual([]);
      expect(
        planCommit(projection.text, undefined, SCORES_FIELD_REGISTRY),
      ).toMatchObject({
        status: "committed",
        filters,
      });
    }
  });

  it("keeps boolean score values distinct from categorical text", () => {
    expect(
      planCommit(
        "booleanValue:true stringValue:true",
        undefined,
        SCORES_FIELD_REGISTRY,
      ),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "booleanValue",
          type: "stringOptions",
          operator: "any of",
          value: ["true"],
        },
        {
          column: "stringValue",
          type: "stringOptions",
          operator: "any of",
          value: ["true"],
        },
      ],
    });
  });

  it("offers only fields owned by the sidebar and rejects aggregate-score namespaces", () => {
    const facets = new Set(
      getScoreFilterConfig().facets.map((facet) => facet.column),
    );
    for (const field of SCORES_FIELD_REGISTRY.fields) {
      expect(facets.has(field.id), field.id).toBe(true);
    }
    for (const query of [
      "scores.accuracy:>0.5",
      "traceScores.grade:A",
      "latency:>2",
      "evaluatorId:abc",
    ]) {
      expect(
        planCommit(query, undefined, SCORES_FIELD_REGISTRY).status,
        query,
      ).toBe("invalid");
    }
    const completion = planInputCompletions(
      {
        input: "name:",
        caret: 5,
        observed: { name: [{ value: "Rouge Score" }] },
        recents: [],
        currentQueryText: "",
      },
      SCORES_FIELD_REGISTRY,
    );
    const options =
      completion?.sections.flatMap((section) => section.options) ?? [];
    expect(options.some((option) => option.label.includes("Rouge Score"))).toBe(
      true,
    );
  });
});

// Per-view wiring of the property harness. A second filterable view adopts the
// bar by adding its own block here with its registry — the harness is unchanged.
// See README.md "Extending to other views (the universality contract)".
const eventsView: RegistryUnderTest = {
  name: "events v4",
  registry: EVENTS_FIELD_REGISTRY,
  // Grammar overlay: dot-path examples + pseudo-fields (README step 2).
  extraKeys: [
    "metadata.region",
    "scores.accuracy",
    "traceScores.nps",
    // Quoted dot-path segments: score/metadata names with spaces + grammar
    // chars must round-trip through the quoting just like bare keys.
    'scores."Rouge Score"',
    'traceScores."Hallucination Check"',
    'metadata."my key"',
    "has:endTime",
    "has:latency",
  ],
  // Numeric vs categorical routing for scores.accuracy must not change which
  // invariants hold — only how a score lowers.
  scoreContexts: [
    {
      numericScoreNames: new Set(["accuracy"]),
      categoricalScoreNames: new Set(),
      traceNumericScoreNames: new Set(),
      traceCategoricalScoreNames: new Set(["nps"]),
    },
    {
      numericScoreNames: new Set(),
      categoricalScoreNames: new Set(["accuracy"]),
      traceNumericScoreNames: new Set(["nps"]),
      traceCategoricalScoreNames: new Set(),
    },
    // Boolean-observed scores: boolean literals route to booleanObject while
    // numeric shapes keep the legacy scores_avg lowering (old URLs/saved
    // views), so both must hold the invariants under the same context.
    {
      numericScoreNames: new Set(),
      categoricalScoreNames: new Set(),
      booleanScoreNames: new Set(["accuracy"]),
      traceNumericScoreNames: new Set(),
      traceCategoricalScoreNames: new Set(),
      traceBooleanScoreNames: new Set(["nps"]),
    },
  ],
  fieldValues: ["x", "ERROR", "5", "0.8", "2026-06-01", "true", "a b", "gpt-4"],
  // Adversarial free text — the tokens the parser reserves/quotes. The bare
  // boolean keywords and `!`-prefix here are the exact #4 regression class;
  // the bare field words (`type`, `level`) are the LFE-11017 class — a lone
  // field-name free text must serialize QUOTED so it re-parses valid.
  freeTextValues: [
    "hello",
    "refund policy",
    "type",
    "level",
    "or",
    "and",
    "not",
    "OR",
    "AND",
    "NOT",
    "team or kitten",
    "test not really",
    "!important",
    "!critical bug",
    "-foo",
    "a,b",
    "gpt-4-turbo",
    "key:value",
    "(grouped)",
    'has "quote"',
  ],
  sidebarFilters: [
    [
      {
        type: "categoryOptions",
        column: "score_categories",
        key: "verdict",
        operator: "any of",
        value: ["good"],
      },
    ],
    [
      {
        type: "numberObject",
        column: "trace_scores_avg",
        key: "nps",
        operator: ">",
        value: 5,
      },
    ],
  ],
};

const evaluationRulesView: RegistryUnderTest = {
  name: "evaluation rules",
  registry: RULE_FIELD_REGISTRY,
  extraKeys: ["metadata.region", "has:version", "has:parentObservationId"],
  scoreContexts: [],
  fieldValues: ["x", "ERROR", "5", "true", "a b"],
  freeTextValues: [],
};

const withInvariantDatasetOptions = (
  registry: FieldRegistry,
  values: string[],
) =>
  withFieldOptions(
    registry,
    "datasetName",
    values.map((displayValue, index) => ({
      value: `dataset-${index}`,
      displayValue,
    })),
  );

const sampleFilterViews: RegistryUnderTest[] = [
  {
    ...eventsView,
    name: "evaluator sample filters",
    registry: withInvariantDatasetOptions(
      EVALUATOR_FIELD_REGISTRY,
      eventsView.fieldValues,
    ),
  },
  {
    ...evaluationRulesView,
    name: "rule sample filters",
    registry: withInvariantDatasetOptions(
      RULE_SAMPLE_FIELD_REGISTRY,
      evaluationRulesView.fieldValues,
    ),
  },
];

describe.each(sampleFilterViews)(
  "search bar invariants — $name registry",
  (view) => {
    it("holds parity, round-trip, and serialization invariants", () => {
      const failures = runSearchBarInvariants(view);
      expect(
        failures,
        failures.length === 0
          ? "ok"
          : `\n${failures
              .slice(0, 25)
              .map(
                (failure) =>
                  `  [${failure.invariant}] ${failure.case} — ${failure.detail}`,
              )
              .join("\n")}`,
      ).toEqual([]);
    });
  },
);

describe("search bar invariants — events v4 registry", () => {
  it("generates a broad field × operator × value matrix", () => {
    // Sanity: the matrix actually exercises the registry (guards against a
    // future refactor silently emptying the generator).
    expect(generateQueryCases(eventsView).length).toBeGreaterThan(1000);
  });

  it("holds all three invariants (parity, round-trip, serialize symmetry)", () => {
    const failures = runSearchBarInvariants(eventsView);
    // Surface every failing case, not just the first, for a fast diagnosis.
    expect(
      failures,
      failures.length === 0
        ? "ok"
        : `\n${failures
            .slice(0, 25)
            .map((f) => `  [${f.invariant}] ${f.case} — ${f.detail}`)
            .join(
              "\n",
            )}${failures.length > 25 ? `\n  …and ${failures.length - 25} more` : ""}`,
    ).toEqual([]);
  });

  it("supports filtering to experiment item root spans", () => {
    expect(
      planCommit(
        "isExperimentItemRootSpan:true",
        undefined,
        EVENTS_FIELD_REGISTRY,
      ),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "isExperimentItemRootSpan",
          type: "boolean",
          operator: "=",
          value: true,
        },
      ],
    });
  });

  it("supports the dataset alias used by sample observation filters", () => {
    expect(
      planCommit("dataset:dataset-id", undefined, EVENTS_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "experimentDatasetId",
          type: "stringOptions",
          operator: "any of",
          value: ["dataset-id"],
        },
      ],
    });
  });

  it("filters cached tokens and attributed cached cost", () => {
    expect(
      planCommit(
        "cachedTokens:0 cachedCost:<=0",
        undefined,
        EVENTS_FIELD_REGISTRY,
      ),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "cachedInputTokens",
          type: "number",
          operator: "=",
          value: 0,
        },
        {
          column: "cachedInputCost",
          type: "number",
          operator: "<=",
          value: 0,
        },
      ],
    });
  });

  it("supports presence filters for cached metrics", () => {
    const query =
      "has:cachedTokens -has:cachedTokens has:cachedCost -has:cachedCost";
    expect(
      validateQuery(query, undefined, EVENTS_FIELD_REGISTRY).diagnostics,
    ).toEqual([]);
    expect(planCommit(query, undefined, EVENTS_FIELD_REGISTRY)).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "cachedInputTokens",
          type: "null",
          operator: "is not null",
          value: "",
        },
        {
          column: "cachedInputTokens",
          type: "null",
          operator: "is null",
          value: "",
        },
        {
          column: "cachedInputCost",
          type: "null",
          operator: "is not null",
          value: "",
        },
        {
          column: "cachedInputCost",
          type: "null",
          operator: "is null",
          value: "",
        },
      ],
    });
  });
});

describe("search bar invariants — evaluation rules registry", () => {
  it("isolates the rule language from events-only fields and free text", () => {
    expect(
      validateQuery("latency:>2", undefined, RULE_FIELD_REGISTRY).valid,
    ).toBe(false);
    expect(validateQuery("refund", undefined, RULE_FIELD_REGISTRY).valid).toBe(
      false,
    );

    const completion = planInputCompletions(
      {
        input: "",
        caret: 0,
        observed: {},
        recents: ["latency:>2"],
        currentQueryText: "",
      },
      RULE_FIELD_REGISTRY,
    );
    expect(
      completion?.sections
        .flatMap((section) => section.options)
        .some(
          (option) => option.kind === "field" && option.fieldId === "latency",
        ),
    ).toBe(false);
    expect(
      completion?.sections
        .flatMap((section) => section.options)
        .some((option) => option.kind === "recent"),
    ).toBe(false);

    expect(
      planCommit("tags:billing", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [{ column: "tags", type: "arrayOptions" }],
    });

    expect(
      planCommit("model:gpt-4o", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [{ column: "providedModelName", type: "string" }],
    });
    expect(
      planCommit("prompt:support-agent", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [{ column: "promptName", type: "string" }],
    });
    expect(
      planCommit("status:rate-limit", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [{ column: "statusMessage", type: "string" }],
    });
    expect(
      planCommit("experiment:checkout", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [{ column: "experimentName", type: "string" }],
    });

    expect(
      planCommit("dataset:dataset-id", undefined, RULE_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "experimentDatasetId",
          type: "stringOptions",
          operator: "any of",
          value: ["dataset-id"],
        },
      ],
    });

    const datasetCompletion = planInputCompletions(
      {
        input: "dataset:",
        caret: 8,
        observed: {
          experimentDatasetId: [{ value: "dataset-id" }],
        },
        recents: [],
        currentQueryText: "dataset:",
      },
      RULE_FIELD_REGISTRY,
    );
    expect(
      datasetCompletion?.sections
        .flatMap((section) => section.options)
        .some(
          (option) => option.kind === "value" && option.value === "dataset-id",
        ),
    ).toBe(true);

    expect(
      planCommit(
        "isExperimentItemRootSpan:true",
        undefined,
        RULE_FIELD_REGISTRY,
      ),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "isExperimentItemRootSpan",
          type: "boolean",
          operator: "=",
          value: true,
        },
      ],
    });
  });

  it("holds commit parity and FilterState round-trips", () => {
    expect(runSearchBarInvariants(evaluationRulesView)).toEqual([]);
  });
});

const sessionsView: RegistryUnderTest = {
  name: "sessions v4",
  registry: SESSIONS_FIELD_REGISTRY,
  extraKeys: [
    "metadata.region",
    'metadata."my key"',
    "scores.accuracy",
    'scores."Rouge Score"',
    "has:environment",
    "has:userIds",
  ],
  scoreContexts: [
    {
      numericScoreNames: new Set(["accuracy"]),
      categoricalScoreNames: new Set(),
    },
    {
      numericScoreNames: new Set(),
      categoricalScoreNames: new Set(["accuracy"]),
    },
    {
      numericScoreNames: new Set(),
      categoricalScoreNames: new Set(),
      booleanScoreNames: new Set(["accuracy"]),
    },
  ],
  fieldValues: ["x", "default", "5", "0.8", "true", "a b", "user-1"],
  // Free text is rewritten onto `id`, not dropped, so the quoting/reserved-word
  // mirror still has to hold on this registry.
  freeTextValues: ["hello", "refund policy", "or", "and", "!important", "-foo"],
  sidebarFilters: [
    [
      {
        type: "numberObject",
        column: "scores_avg",
        key: "accuracy",
        operator: ">",
        value: 0.5,
      },
    ],
    // Trace scores are not offered here, so this one must stay sidebar-only.
    [
      {
        type: "numberObject",
        column: "trace_scores_avg",
        key: "nps",
        operator: ">",
        value: 5,
      },
    ],
  ],
};

describe("search bar invariants — sessions registry", () => {
  it.each([false, true])(
    "offers only executable examples inside a user detail table (events: %s)",
    (fromEvents) => {
      const registry = sessionsFieldRegistry(
        getSessionFilterConfig(["userIds"], fromEvents),
      );
      const examples = [
        ...registry.searchExamples,
        ...(registry.hasExample ? [`has:${registry.hasExample}`] : []),
      ];
      for (const example of examples) {
        expect(planCommit(example, undefined, registry).status, example).toBe(
          "committed",
        );
      }
    },
  );

  it("holds all three invariants (parity, round-trip, serialize symmetry)", () => {
    const failures = runSearchBarInvariants(sessionsView);
    expect(
      failures,
      failures.length === 0
        ? "ok"
        : `\n${failures
            .slice(0, 25)
            .map((f) => `  [${f.invariant}] ${f.case} — ${f.detail}`)
            .join(
              "\n",
            )}${failures.length > 25 ? `\n  …and ${failures.length - 25} more` : ""}`,
    ).toEqual([]);
  });

  it("exposes the sidebar's facets and nothing else", () => {
    expect(SESSIONS_FIELD_REGISTRY.fields.map((f) => f.id).sort()).toEqual([
      "calledToolNames",
      "commentContent",
      "commentCount",
      "countTraces",
      "environment",
      "id",
      "inputCost",
      "inputTokens",
      "outputCost",
      "outputTokens",
      "sessionDuration",
      "tags",
      "toolCalls",
      "toolNames",
      "totalCost",
      "totalTokens",
      "userIds",
    ]);
    const v3Registry = sessionsFieldRegistry(getSessionFilterConfig([], false));
    for (const column of ["toolNames", "calledToolNames", "toolCalls"]) {
      expect(v3Registry.resolveField(column)).toBeNull();
    }
    // Events-only fields and columns the sidebar never offers stay unresolvable,
    // so a stray token is a diagnostic rather than a filter the sidebar cannot
    // show or remove.
    for (const key of ["latency", "createdAt", "usage", "bookmarked"]) {
      expect(SESSIONS_FIELD_REGISTRY.resolveField(key)).toBeNull();
    }
  });

  it("keeps the trace-score namespace closed but observation scores open", () => {
    expect(SESSIONS_FIELD_REGISTRY.resolveField("scores.accuracy")).toEqual({
      type: "scores",
      key: "accuracy",
      level: "observation",
    });
    expect(
      SESSIONS_FIELD_REGISTRY.resolveField("traceScores.accuracy"),
    ).toBeNull();
  });

  it("round-trips the all-of array filters sessions users actually apply", () => {
    // `tags all of` (1.2k applies/56d) and `userIds all of` are the two shapes
    // the sessions sidebar produces that no other bar surface exercises.
    for (const column of ["tags", "userIds"]) {
      expect(
        planCommit(`${column}:(a AND b)`, undefined, SESSIONS_FIELD_REGISTRY),
      ).toMatchObject({
        status: "committed",
        filters: [
          {
            column,
            type: "arrayOptions",
            operator: "all of",
            value: ["a", "b"],
          },
        ],
      });
    }
  });

  it("lowers metadata and session id the way the sidebar does", () => {
    expect(
      planCommit("metadata.region:eu", undefined, SESSIONS_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "metadata",
          type: "stringObject",
          key: "region",
          value: "eu",
        },
      ],
    });
    // `id contains` is how every one of the 13.6k session-id filters is applied.
    expect(
      planCommit("id:checkout", undefined, SESSIONS_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "id",
          type: "string",
          operator: "contains",
          value: "checkout",
        },
      ],
    });
  });

  it("rewrites a bare word onto the session id and settles there", () => {
    const first = planCommit("refund", undefined, SESSIONS_FIELD_REGISTRY);
    if (first.status !== "committed") throw new Error(first.status);
    expect(first).toMatchObject({
      status: "committed",
      searchQuery: null,
      filters: [
        { column: "id", type: "string", operator: "contains", value: "refund" },
      ],
    });
    // The canonicalization is visible AND terminal: the echo renders `id:refund`
    // and committing that again produces the identical filter, so the bar does
    // not keep rewriting itself.
    expect(
      filterStateToQueryText(first.filters, undefined, SESSIONS_FIELD_REGISTRY)
        .text,
    ).toBe("id:refund");
    expect(
      planCommit("id:refund", undefined, SESSIONS_FIELD_REGISTRY),
    ).toMatchObject({ status: "committed", filters: first.filters });

    // A dangling dot-prefix must stay an error rather than becoming an id search
    // for the literal text "metadata." — and the message has to name the real
    // problem, since this view does support `metadata.<key>`.
    const dangling = planCommit(
      "metadata.",
      undefined,
      SESSIONS_FIELD_REGISTRY,
    );
    expect(dangling.status).toBe("invalid");
    if (dangling.status !== "invalid") throw new Error("expected invalid");
    expect(
      dangling.diagnostics.some((d) =>
        /add a key after the dot/.test(d.message),
      ),
    ).toBe(true);

    // Quoting it makes it an explicit literal, so it searches ids like any word.
    expect(
      planCommit('"metadata."', undefined, SESSIONS_FIELD_REGISTRY),
    ).toMatchObject({
      status: "committed",
      filters: [
        {
          column: "id",
          type: "string",
          operator: "contains",
          value: "metadata.",
        },
      ],
    });
  });

  it("treats a multi-word run as one phrase, not one filter per word", () => {
    // The suggestion offers `id:"test 123"`, so Enter must agree with it. Per
    // word it would AND `id contains test` with `id contains 123` — a query
    // matching neither what was typed nor what was offered.
    const multi = planCommit("test 123", undefined, SESSIONS_FIELD_REGISTRY);
    if (multi.status !== "committed") throw new Error(multi.status);
    expect(multi.filters).toEqual([
      { column: "id", type: "string", operator: "contains", value: "test 123" },
    ]);
    expect(multi.searchQuery).toBeNull();
    // …and it round-trips through the quoting as the same single filter.
    const text = filterStateToQueryText(
      multi.filters,
      undefined,
      SESSIONS_FIELD_REGISTRY,
    ).text;
    expect(text).toBe('id:"test 123"');
    expect(planCommit(text, undefined, SESSIONS_FIELD_REGISTRY)).toMatchObject({
      filters: multi.filters,
    });

    // Words split around a real filter token still coalesce into one phrase.
    const mixed = planCommit(
      "test countTraces:8 123",
      undefined,
      SESSIONS_FIELD_REGISTRY,
    );
    if (mixed.status !== "committed") throw new Error(mixed.status);
    expect(mixed.filters.filter((f) => f.column === "id")).toEqual([
      { column: "id", type: "string", operator: "contains", value: "test 123" },
    ]);
  });

  it("does not offer Ask AI until the view has its own prompt", () => {
    // buildFilterSystemPrompt branches on registry.id and falls back to the
    // EVENTS prompt — events prose, events worked examples. A view without its
    // own branch must not offer AI generation, or the model gets a correct field
    // catalog wrapped in instructions aimed at columns the view lacks.
    expect(SESSIONS_FIELD_REGISTRY.aiFilterPrompt).toBe(false);
    expect(EVENTS_FIELD_REGISTRY.aiFilterPrompt).toBe(true);
    expect(RULE_FIELD_REGISTRY.aiFilterPrompt).toBe(true);
  });

  it("offers only the recent searches that are valid on this view", () => {
    // Recents live in one per-PROJECT store shared with the events bar, so a
    // query typed there can surface here. An events-only one must not be
    // offered: picking it would insert a query that cannot commit.
    const plan = planInputCompletions(
      {
        input: "",
        caret: 0,
        observed: {},
        recents: ["latency:>2", "tags:billing", "level:ERROR", "countTraces:8"],
        currentQueryText: "",
      },
      SESSIONS_FIELD_REGISTRY,
    );
    const offered = (plan?.sections ?? [])
      .flatMap((section) => section.options)
      .filter((option) => option.kind === "recent")
      .map((option) => option.label);
    expect(offered).toEqual(["tags:billing", "countTraces:8"]);

    // Opt-in: a view that has not asked for recents gets none, valid or not.
    expect(RULE_FIELD_REGISTRY.recentSearches).toBe(false);
  });

  it("surfaces has:/-has: while typing a nullable field name", () => {
    const planFor = (term: string) =>
      planInputCompletions(
        {
          input: term,
          caret: term.length,
          observed: {},
          recents: [],
          currentQueryText: term,
        },
        SESSIONS_FIELD_REGISTRY,
      );
    const offered = (term: string) =>
      (planFor(term)?.sections ?? [])
        .flatMap((section) => section.options)
        .filter((option) => option.id.startsWith("presence:"))
        .map((option) => option.label);
    // Typing the column name reveals both presence forms — previously reachable
    // only by knowing the `has:` pseudo-field existed.
    expect(offered("userId")).toEqual(["has:userIds", "-has:userIds"]);
    // Mid-word too, so they appear while typing rather than only on a full match.
    expect(offered("userI")).toEqual(["has:userIds", "-has:userIds"]);
    // A negated term already carries the `-`; the sole option is the missing
    // form (label matches what applyPick commits once the surviving dash joins
    // the inserted `has:`). Offering `-has:` as insert would splice `--has:`.
    expect(offered("-userIds")).toEqual(["-has:userIds"]);
    const negatedPlan = planFor("-userIds");
    const missing = (negatedPlan?.sections ?? [])
      .flatMap((section) => section.options)
      .find((option) => option.id === "presence:-has:userIds");
    expect(missing).toBeDefined();
    expect(
      applyPick(
        missing as Exclude<CompletionOption, { kind: "recent" | "preset" }>,
        "-userIds",
        negatedPlan!,
      ).next,
    ).toBe("-has:userIds ");
    // Non-nullable and too-short terms stay quiet.
    expect(offered("countTraces")).toEqual([]);
    expect(offered("u")).toEqual([]);
  });

  it("pins the has: hint per view instead of deriving it from field order", () => {
    // Derived from "first nullable field", the GA events hint silently became
    // has:name; it is written down so reordering FIELDS cannot rewrite copy.
    expect(EVENTS_FIELD_REGISTRY.hasExample).toBe("endTime");
    expect(SESSIONS_FIELD_REGISTRY.hasExample).toBe("userIds");
  });
});

const usersView: RegistryUnderTest = {
  name: "users v4",
  registry: USERS_FIELD_REGISTRY,
  extraKeys: [
    "metadata.region",
    'metadata."my key"',
    "has:latency",
    "has:sessionId",
  ],
  // The users queries group `events_core` with no score join, so `scores.` is
  // closed at both levels and there is no context to vary.
  scoreContexts: [],
  fieldValues: ["x", "ERROR", "5", "0.8", "true", "a b", "gpt-4"],
  // Free text is its own lane here (it lowers to `user_id ILIKE`), so the same
  // reserved-token quoting the events bar needs has to hold.
  freeTextValues: [
    "hello",
    "alice smith",
    "or",
    "and",
    "NOT",
    "!important",
    "-foo",
    "key:value",
  ],
  sidebarFilters: [
    // Trace-score columns reach this page only from an older saved view. The
    // bar does not own them, so it must skip rather than emit a token that
    // renders as an unknown field.
    [
      {
        type: "numberObject",
        column: "trace_scores_avg",
        key: "nps",
        operator: ">",
        value: 5,
      },
    ],
  ],
};

describe("search bar invariants — users registry", () => {
  it("preserves legacy user metadata while rejecting fields that require joins", () => {
    const registry = LEGACY_USERS_FIELD_REGISTRY;
    const filters: FilterState = [
      {
        type: "stringObject",
        column: "metadata",
        key: "region",
        operator: "=",
        value: "eu",
      },
    ];
    const projection = filterStateToQueryText(filters, {}, registry);
    expect(projection.skippedFilters).toEqual([]);
    expect(planCommit(projection.text, undefined, registry)).toMatchObject({
      status: "committed",
      filters,
    });
    expect(
      runSearchBarInvariants({
        ...usersView,
        name: "users legacy",
        registry,
        sidebarFilters: [filters],
      }),
    ).toEqual([]);
    for (const query of [
      "level:ERROR",
      "latency:>1",
      "scores.quality:>0.8",
      "traceScores.nps:>5",
      "comment:test",
      "input:test",
    ]) {
      expect(planCommit(query, undefined, registry).status).toBe("invalid");
    }
    expect(planCommit("alice smith", undefined, registry)).toMatchObject({
      status: "committed",
      searchQuery: "alice smith",
      filters: [],
    });
  });

  it("holds all three invariants (parity, round-trip, serialize symmetry)", () => {
    const failures = runSearchBarInvariants(usersView);
    expect(
      failures,
      failures.length === 0
        ? "ok"
        : `\n${failures
            .slice(0, 25)
            .map((f) => `  [${f.invariant}] ${f.case} — ${f.detail}`)
            .join(
              "\n",
            )}${failures.length > 25 ? `\n  …and ${failures.length - 25} more` : ""}`,
    ).toEqual([]);
  });

  it("is the events grammar minus what the users query cannot answer", () => {
    // Users groups `events_core` alone: no score join, no comments join. Those
    // facets are excluded from the sidebar, so the bar must not resolve them
    // either — a `scores.` token here would commit a filter that 500s.
    expect(USERS_FIELD_REGISTRY.resolveField("scores.accuracy")).toBeNull();
    expect(USERS_FIELD_REGISTRY.resolveField("traceScores.nps")).toBeNull();
    for (const key of ["comment", "commentCount", "commentContent"]) {
      expect(USERS_FIELD_REGISTRY.resolveField(key)).toBeNull();
    }
    // Everything else the events bar offers survives, and is exactly the
    // sidebar's facet set (metadata excepted — it resolves by dot-path, not as
    // a plain field).
    const facetColumns = new Set(
      usersEventsFilterConfig.facets.map((f) => f.column),
    );
    for (const field of USERS_FIELD_REGISTRY.fields) {
      expect(facetColumns.has(field.filterColumn ?? field.id)).toBe(true);
    }
    expect(USERS_FIELD_REGISTRY.resolveField("metadata.region")).toEqual({
      type: "metadata",
      key: "region",
    });
    // The two aliases that make the events grammar worth reusing rather than
    // re-deriving: `env:` and `tag:` are overlay-only, absent from the columns.
    expect(USERS_FIELD_REGISTRY.resolveField("env")).toMatchObject({
      type: "field",
      field: { id: "environment" },
    });
    expect(USERS_FIELD_REGISTRY.resolveField("user")).toMatchObject({
      type: "field",
      field: { id: "userId" },
    });
  });

  it("keeps a bare word in the free-text lane rather than rewriting it", () => {
    // Unlike sessions (`id contains`), the users backend already has a search
    // lane: `searchQuery` lowers to `user_id ILIKE %q%`. Rewriting a bare word
    // onto the userId column would turn a substring search into an exact
    // any-of match against the facet list.
    const committed = planCommit("alice", undefined, USERS_FIELD_REGISTRY);
    expect(committed).toMatchObject({
      status: "committed",
      searchQuery: "alice",
      filters: [],
    });
    // …and it renders back as the same bare word, so the bar does not churn.
    expect(
      filterStateToQueryText(
        [],
        { searchQuery: "alice", searchType: DEFAULT_SEARCH_TYPE },
        USERS_FIELD_REGISTRY,
      ).text,
    ).toBe("alice");
  });

  it("describes the free-text lane as user ids, and offers no other scope", () => {
    // The generic copy claims a bare word matches "ids, names, input and
    // output". On this page it matches user ids and nothing else, and there is
    // no `input:`/`output:` field to switch into — offering the rewrite would
    // hand the user a token the parser rejects.
    expect(USERS_FIELD_REGISTRY.freeTextScopeLabel).toBe("user IDs");
    expect(USERS_FIELD_REGISTRY.resolveField("input")).toBeNull();
    expect(USERS_FIELD_REGISTRY.resolveField("output")).toBeNull();

    const scopeOptionIds = (registry: FieldRegistry) =>
      (
        planInputCompletions(
          {
            input: "alice",
            caret: 5,
            observed: {},
            recents: [],
            currentQueryText: "alice",
          },
          registry,
        )?.sections ?? []
      )
        .flatMap((section) => section.options)
        .map((option) => option.id)
        .filter((id) => id.startsWith("scope:"));

    expect(scopeOptionIds(USERS_FIELD_REGISTRY)).toEqual([]);
    // The same input on a registry that does have the columns still offers them.
    expect(scopeOptionIds(EVENTS_FIELD_REGISTRY)).toEqual(
      expect.arrayContaining(["scope:input", "scope:output"]),
    );
  });

  it("does not offer Ask AI until the view has its own prompt", () => {
    // buildFilterSystemPrompt falls back to the EVENTS prompt, whose worked
    // examples filter on score columns this page cannot reach.
    expect(USERS_FIELD_REGISTRY.aiFilterPrompt).toBe(false);
  });
});

const experimentsView: RegistryUnderTest = {
  name: "experiments",
  registry: EXPERIMENTS_FIELD_REGISTRY,
  extraKeys: ["metadata.owner", 'metadata."my key"'],
  scoreContexts: [],
  fieldValues: ["x", "sonnet", "5", "0.8", "a b"],
  freeTextValues: ["hello", "run 12", "or", "!important"],
  // Score filters are the bar's to render on this view, so these must derive to
  // valid text rather than being skipped. The `trace_*` columns are not, and
  // stay sidebar-only.
  sidebarFilters: [
    [
      {
        type: "categoryOptions",
        column: "score_categories",
        key: "verified_article_status",
        operator: "any of",
        value: ["none-verified"],
      },
    ],
    [
      {
        type: "numberObject",
        column: "scores_avg",
        key: "groundedness",
        operator: ">",
        value: 0.5,
      },
    ],
    [
      {
        type: "booleanObject",
        column: "score_booleans",
        key: "in_force",
        operator: "=",
        value: true,
      },
    ],
    // A filter the bar DOES own, alongside one it does not: the owned half must
    // still render.
    [
      {
        type: "stringOptions",
        column: "experimentDatasetName",
        operator: "any of",
        value: ["legal-answer-quality"],
      },
      {
        type: "categoryOptions",
        column: "score_categories",
        key: "verified_article_status",
        operator: "any of",
        value: ["none-verified"],
      },
    ],
  ],
};

describe("search bar invariants — experiments registry", () => {
  it("holds all three invariants (parity, round-trip, serialize symmetry)", () => {
    const failures = runSearchBarInvariants(experimentsView);
    expect(
      failures,
      failures.length === 0
        ? "ok"
        : `\n${failures
            .slice(0, 25)
            .map((f) => `  [${f.invariant}] ${f.case} — ${f.detail}`)
            .join(
              "\n",
            )}${failures.length > 25 ? `\n  …and ${failures.length - 25} more` : ""}`,
    ).toEqual([]);
  });

  it("exposes only the sidebar facets whose columns are actually filterable", () => {
    expect(EXPERIMENTS_FIELD_REGISTRY.fields.map((f) => f.id).sort()).toEqual([
      "experimentDatasetName",
      "name",
    ]);
    // `dataset:` filters by the readable, unique name; the id column stays
    // filterable for old saved views but is not offered here.
    expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("dataset")).toMatchObject({
      type: "field",
      field: { id: "experimentDatasetName" },
    });
    expect(
      EXPERIMENTS_FIELD_REGISTRY.resolveField("experimentDatasetId"),
    ).toBeNull();
  });

  it("opens `scores.` and keeps `traceScores.` closed", () => {
    // `scores.<name>` lowers onto the canonical scores_avg / score_categories /
    // score_booleans columns, which is what this view filters on. Those columns
    // already match at trace level, so a separate `traceScores.` namespace would
    // offer a second way to say the same thing — against `trace_*` columns the
    // sidebar no longer offers.
    expect(
      EXPERIMENTS_FIELD_REGISTRY.resolveField("scores.groundedness"),
    ).toEqual({
      type: "scores",
      key: "groundedness",
      level: "observation",
    });
    expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("traceScores.x")).toBeNull();
    // Metadata is unaffected — that column exists under its canonical name.
    expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("metadata.owner")).toEqual({
      type: "metadata",
      key: "owner",
    });
  });

  it("renders a score filter the sidebar set", () => {
    // The sidebar and the bar edit ONE filter state, so a score filter added in
    // the sidebar has to show up as a token — otherwise the bar reads as though
    // no score filter were applied.
    const derived = filterStateToQueryText(
      [
        {
          type: "numberObject",
          column: "scores_avg",
          key: "citation_usable_rate",
          operator: "=",
          value: 4,
        },
      ],
      {},
      EXPERIMENTS_FIELD_REGISTRY,
    );

    expect(derived.text).toBe("scores.citation_usable_rate:4");
    expect(derived.skippedFilters).toEqual([]);
  });

  it("rewrites a bare word onto the experiment name", () => {
    const committed = planCommit(
      "refund eval",
      undefined,
      EXPERIMENTS_FIELD_REGISTRY,
    );
    expect(committed).toMatchObject({
      status: "committed",
      searchQuery: null,
      filters: [
        {
          column: "name",
          type: "string",
          operator: "contains",
          value: "refund eval",
        },
      ],
    });
  });

  it("keeps the run metrics out — the backend silently drops them", () => {
    // itemCount / errorCount / totalCost / latencyAvg have ColumnDefinitions for
    // display and sorting but no UiColumnMapping, and the repository partitions
    // filters by allow-list, discarding anything in neither group. A field here
    // would render a pill and change nothing.
    for (const id of ["itemCount", "errorCount", "totalCost", "latencyAvg"]) {
      expect(EXPERIMENTS_FIELD_REGISTRY.resolveField(id)).toBeNull();
    }
    expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("cost")).toBeNull();
  });
});