πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// @vitest-environment node
import { describe, expect, it } from "vitest";
import { parse, serialize } from "./langQ";
import { semanticDiagnostics, validateQuery } from "./validate";
function warnings(query: string): string[] {
const { ast } = parse(query);
return semanticDiagnostics(ast, query.length)
.filter((d) => d.severity === "warning")
.map((d) => d.message);
}
describe("semanticDiagnostics β has: polarity", () => {
it("positive has: on a non-nullable field matches everything", () => {
expect(warnings("has:level")).toContain(
'"level" always has a value β this filter matches everything',
);
});
// `-has:level` lowers to `level IS NULL`, which is vacuously false on a
// non-nullable column β it matches NOTHING, the opposite of the positive
// form. The warning must flip with polarity.
it("negated has: on a non-nullable field matches nothing (dash form)", () => {
expect(warnings("-has:level")).toContain(
'"level" always has a value β this filter matches nothing',
);
});
it("negated has: on a non-nullable field matches nothing (NOT keyword)", () => {
expect(warnings("NOT has:level")).toContain(
'"level" always has a value β this filter matches nothing',
);
});
it("double negation matches everything again", () => {
expect(warnings("NOT NOT has:level")).toContain(
'"level" always has a value β this filter matches everything',
);
});
});
describe("validateQuery β merged-diagnostic dedup", () => {
// The parser (pushOpIssue) and the adapter (operatorIssue in lowerFilter)
// both flag the same operator/field mismatch at the same span. validateQuery
// merges the two lists, so it must dedupe β otherwise the global tooltip and
// aria-live region announce the same message twice for a single typo.
function errorMessages(query: string): string[] {
return validateQuery(query)
.diagnostics.filter((d) => d.severity === "error")
.map((d) => d.message);
}
it.each(["metadata.region:>5", "level:>5", "traceTags:*tag*", "xyz:1"])(
"emits each error message once for %s",
(query) => {
const messages = errorMessages(query);
expect(messages.length).toBeGreaterThan(0);
expect(new Set(messages).size).toBe(messages.length);
},
);
// A `key:` typo (no value) is flagged by BOTH the parser and the adapter; the
// two messages must be identical so dedup collapses them to ONE (two
// different strings would both survive β the prior bug).
it.each([
["level:", 'Missing value after "level:"'],
["tags:", 'Missing value after "tags:"'],
["metadata.region:", 'Missing value after "metadata.region:"'],
["has:", 'Missing value after "has:"'],
// textSearch column with no value; operator-prefix forms carry the prefix
// in the parser's wording β both previously doubled.
["input:", 'Missing value after "input:"'],
["level:=", 'Missing value after "level:="'],
["latency:>", 'Missing value after "latency:>"'],
])(
"surfaces exactly one empty-value diagnostic for %s",
(query, expected) => {
expect(errorMessages(query)).toEqual([expected]);
},
);
});
describe("validateQuery β compatibility scope diagnostics", () => {
it("anchors an unsupported scope to its token beside other query terms", () => {
const query = "env:prod in:unknown refund";
const result = validateQuery(query);
expect(result.valid).toBe(false);
expect(result.diagnostics).toEqual([
{
from: query.indexOf("in:unknown"),
to: query.indexOf("in:unknown") + "in:unknown".length,
severity: "error",
message: expect.stringContaining("Unsupported search scope"),
},
]);
});
it("anchors a missing phrase to the scope that requires it", () => {
const query = "env:prod in:id";
expect(validateQuery(query).diagnostics).toEqual([
{
from: query.indexOf("in:id"),
to: query.length,
severity: "error",
message: "Add search text after in:",
},
]);
});
it("validates a scope with its phrase elsewhere in the query", () => {
expect(validateQuery("env:prod in:id refund")).toMatchObject({
valid: true,
diagnostics: [],
});
});
it("anchors a duplicate scope to the second scope token", () => {
const query = "assistant in:id in:content";
expect(validateQuery(query).diagnostics).toEqual([
{
from: query.indexOf("in:content"),
to: query.length,
severity: "error",
message: "Only one in: scope selection is supported",
},
]);
});
it("does not add a missing-phrase error to competing scoped searches", () => {
const query = "env:prod content:refund in:id";
expect(validateQuery(query).diagnostics).toEqual([
{
from: 0,
to: query.length,
severity: "error",
message:
"Only one search phrase is supported β use either bare text or one scoped search",
},
]);
});
});
// LFE-11017: a partial filter token that isn't a complete expression must not
// silently apply to the query. A bare field word (`type`, no operator/value)
// used to parse as free text and lower to a full-text searchQuery, wiping the
// results with no signal β while `type:` already errored. It must now be
// treated as an incomplete filter (excluded from the query, rendered red),
// WITHOUT flagging deliberate multi-word free-text phrases.
describe("validateQuery β incomplete bare field token (LFE-11017)", () => {
const errors = (query: string): string[] =>
validateQuery(query)
.diagnostics.filter((d) => d.severity === "error")
.map((d) => d.message);
const isValid = (query: string): boolean => validateQuery(query).valid;
it.each(["type", "level", "env", "tags", "TYPE", "metadata.region"])(
"flags a lone bare field word %s as an incomplete filter",
(query) => {
expect(isValid(query)).toBe(false);
expect(errors(query).some((m) => m.startsWith("Incomplete filter"))).toBe(
true,
);
},
);
it("flags a bare field word appended after an existing filter (the repro)", () => {
expect(isValid("name:foo type")).toBe(false);
expect(
errors("name:foo type").some((m) => m.startsWith("Incomplete filter")),
).toBe(true);
});
it("does not flag a deliberate multi-word phrase containing a field word", () => {
expect(isValid("type error")).toBe(true);
expect(isValid("session timeout")).toBe(true);
expect(isValid("name:foo type error")).toBe(true);
});
// Adjacency (nit 1): the guard is per free-text RUN, not "sole text node".
it("flags a standalone field word isolated by a filter, even with other free text present", () => {
// `type` stands alone (the filter separates it from the free text `error`),
// so it is still an incomplete filter even though a second free-text token
// exists elsewhere in the query β the old "exactly one text node" scoping
// missed this.
expect(isValid("type name:foo error")).toBe(false);
expect(
errors("type name:foo error").some((m) =>
m.startsWith("Incomplete filter"),
),
).toBe(true);
});
it("leaves a field word glued to adjacent free text as a phrase (justified)", () => {
// Adjacent free-text words are ONE contiguous-substring phrase; we cannot
// tell `hello type` (phrase) from `hello` + an incomplete `type`, and
// flagging a glued/trailing word would break real phrases like `error
// type` / `content type`. So a word glued to other free text is never
// flagged β only a word standing on its own is.
expect(isValid("hello type")).toBe(true);
expect(isValid("error type")).toBe(true);
expect(isValid("content type")).toBe(true);
});
it("does not flag a quoted field word (explicit literal text search)", () => {
expect(isValid('"type"')).toBe(true);
expect(errors('"type"')).toEqual([]);
});
it("does not flag free text that is not a field name", () => {
expect(isValid("hello")).toBe(true);
expect(isValid("refund policy")).toBe(true);
});
it("still flags the colon-but-no-value form (unchanged existing error)", () => {
// `type:` keeps its own "Missing value" error β NOT the incomplete-filter one.
const msgs = errors("type:");
expect(msgs).toContain('Missing value after "type:"');
expect(msgs.some((m) => m.startsWith("Incomplete filter"))).toBe(false);
});
it("accepts a complete expression", () => {
expect(isValid("type:chat")).toBe(true);
expect(isValid("name:foo type:chat")).toBe(true);
});
it("round-trips a committed field-word free text as a quoted literal", () => {
// The reverse of the guard: a legacy searchQuery=`type` must serialize as
// `"type"` so it re-derives valid, not as a bare token that lands red.
const text = serialize({ kind: "text", value: "type" });
expect(text).toBe('"type"');
expect(isValid(text)).toBe(true);
});
// Serializer adjacency (nit 2): the field-name force-quote must be per-run,
// not per-word β a phrase word must NOT be quoted in isolation.
it("does not corrupt a multi-word phrase when serializing (no per-word quoting)", () => {
// Would have produced `"type" error` / `"session" timeout` before the fix.
expect(serialize(parse("type error").ast)).toBe("type error");
expect(serialize(parse("session timeout").ast)).toBe("session timeout");
expect(serialize(parse("error type").ast)).toBe("error type");
// β¦and each still re-parses valid (no red chip).
expect(isValid("type error")).toBe(true);
expect(isValid("session timeout")).toBe(true);
});
it("force-quotes a standalone field word beside a filter so the derive stays valid", () => {
// A searchQuery of `type` alongside a filter derives to `name:foo "type"`,
// not the red `name:foo type`. The word is standalone (a filter separates
// it), so it IS quoted β the phrase carve-out above does not apply.
const text = serialize({
kind: "and",
children: [
{ kind: "filter", key: "name", op: "=", values: ["foo"] },
{ kind: "text", value: "type" },
],
});
expect(text).toContain('"type"');
expect(isValid(text)).toBe(true);
});
});