🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// @vitest-environment node
import type { ASTNode } from "@/src/features/search-bar/lib/ast";
import { parse, serialize, termAt } from "@/src/features/search-bar/lib/langQ";
import {
removeToken,
tidyQueryText,
} from "@/src/features/search-bar/lib/edits";
import { validateQuery } from "@/src/features/search-bar/lib/validate";
/** Structural copy without spans, for deep-equal comparisons. */
function strip(node: ASTNode | null): unknown {
if (node === null) return null;
switch (node.kind) {
case "filter":
return {
kind: "filter",
key: node.key,
op: node.op,
values: node.values,
...(node.valueOp === "and" ? { valueOp: "and" } : {}),
};
case "text":
return { kind: "text", value: node.value };
case "not":
return { kind: "not", child: strip(node.child) };
case "and":
case "or":
return { kind: node.kind, children: node.children.map(strip) };
}
}
describe("langQ parser", () => {
it("resolves aliases to canonical field ids", () => {
const r = parse("env:prod apiKey:pk-lf-test");
expect(r.valid).toBe(true);
expect(strip(r.ast)).toEqual({
kind: "and",
children: [
{
kind: "filter",
key: "environment",
op: "=",
values: ["prod"],
},
{
kind: "filter",
key: "ingestionApiKey",
op: "=",
values: ["pk-lf-test"],
},
],
});
});
it("resolves ingestion attribution aliases to canonical field ids", () => {
const r = parse("sdk_name:python sdkVersion:4.7.1 source:otel");
expect(r.valid).toBe(true);
expect(strip(r.ast)).toEqual({
kind: "and",
children: [
{
kind: "filter",
key: "ingestionSdkName",
op: "=",
values: ["python"],
},
{
kind: "filter",
key: "ingestionSdkVersion",
op: "=",
values: ["4.7.1"],
},
{
kind: "filter",
key: "ingestionSource",
op: "=",
values: ["otel"],
},
],
});
// Snake-case and canonical spellings resolve to the same fields.
const alt = parse(
"sdkName:js ingestion_sdk_version:3.2.1 ingestionSource:api",
);
expect(alt.valid).toBe(true);
expect(strip(alt.ast)).toEqual({
kind: "and",
children: [
{ kind: "filter", key: "ingestionSdkName", op: "=", values: ["js"] },
{
kind: "filter",
key: "ingestionSdkVersion",
op: "=",
values: ["3.2.1"],
},
{ kind: "filter", key: "ingestionSource", op: "=", values: ["api"] },
],
});
});
it("parses grouped lists, comma lists, comparisons, and quoted values", () => {
expect(strip(parse("level:(ERROR OR WARNING)").ast)).toEqual({
kind: "filter",
key: "level",
op: "=",
values: ["ERROR", "WARNING"],
});
expect(strip(parse("level:ERROR,WARNING").ast)).toEqual({
kind: "filter",
key: "level",
op: "=",
values: ["ERROR", "WARNING"],
});
expect(strip(parse("latency:>2.5").ast)).toEqual({
kind: "filter",
key: "latency",
op: ">",
values: ["2.5"],
});
expect(strip(parse('name:"hello world"').ast)).toEqual({
kind: "filter",
key: "name",
op: "=",
values: ["hello world"],
});
});
it("parses all-of value groups on array fields", () => {
expect(strip(parse("tags:(a AND b)").ast)).toEqual({
kind: "filter",
key: "traceTags",
op: "=",
values: ["a", "b"],
valueOp: "and",
});
});
it("preserves metadata dot-path key case", () => {
expect(strip(parse("metadata.Region:eu").ast)).toEqual({
kind: "filter",
key: "metadata.Region",
op: "=",
values: ["eu"],
});
});
it("mixes free text with filters via implicit AND", () => {
expect(strip(parse("timeout level:ERROR").ast)).toEqual({
kind: "and",
children: [
{ kind: "text", value: "timeout" },
{ kind: "filter", key: "level", op: "=", values: ["ERROR"] },
],
});
});
it("binds implicit AND tighter than OR and parses NOT/- negation", () => {
expect(strip(parse("a:1 b:2 OR c:3").ast)).toMatchObject({
kind: "or",
});
expect(strip(parse("-env:dev").ast)).toEqual({
kind: "not",
child: {
kind: "filter",
key: "environment",
op: "=",
values: ["dev"],
},
});
expect(strip(parse("NOT level:DEBUG").ast)).toEqual({
kind: "not",
child: { kind: "filter", key: "level", op: "=", values: ["DEBUG"] },
});
});
it("parses positional `*` globs into match operators", () => {
// contains/starts/ends are positional wildcards, not prefix operators.
expect(strip(parse("name:*chat*").ast)).toMatchObject({
op: "~",
values: ["chat"],
});
expect(strip(parse("input:How*").ast)).toMatchObject({
op: "^",
values: ["How"],
});
expect(strip(parse("output:*end").ast)).toMatchObject({
op: "$",
values: ["end"],
});
expect(strip(parse("name:=exact").ast)).toMatchObject({ op: "exact" });
});
it("treats a glob around a quoted value as a literal phrase match", () => {
// The `*` lives outside the quotes; the phrase keeps its spaces.
expect(strip(parse('input:*"refund policy"*').ast)).toMatchObject({
op: "~",
values: ["refund policy"],
});
// A bare value with no leading/trailing `*` is a plain contains-default.
expect(strip(parse("name:gpt-4").ast)).toMatchObject({
op: "=",
values: ["gpt-4"],
});
});
it('accepts a quoted empty exact value (round-trips :="")', () => {
const r = parse('metadata.region:=""');
expect(r.valid).toBe(true);
expect(strip(r.ast)).toMatchObject({ op: "exact", values: [""] });
});
it('accepts quoted-empty glob values (input:*""* round-trips)', () => {
// What the reverse adapter emits for an empty contains/starts/ends filter.
expect(parse('input:*""*').valid).toBe(true);
expect(parse('input:""*').valid).toBe(true);
expect(parse('input:*""').valid).toBe(true);
});
it("accepts a quoted dot-path key (score/metadata names with spaces)", () => {
// Quoting the segment after a dot prefix is how a score/metadata name with
// spaces or grammar chars is addressed; it parses and round-trips.
expect(parse('metadata."foo":bar').valid).toBe(true);
expect(parse('scores."Rouge Score":>=1').valid).toBe(true);
expect(parse('traceScores."Hallucination Check":faithful').valid).toBe(
true,
);
// A quoted UNKNOWN plain field is still rejected (resolves to nothing).
expect(parse('"level":ERROR').valid).toBe(false);
});
it("emits a single diagnostic for a bare AND, not a doubled one", () => {
const errs = parse("AND").diagnostics.filter((d) => d.severity === "error");
expect(errs).toHaveLength(1);
});
it("emits one separator diagnostic for a lone OR/AND inside a group", () => {
// `level:(OR)` must not double the missing-left-hand and dangling messages
// at the same span (mirrors the bare-AND fix in parseAnd).
for (const q of ["level:(OR)", "traceTags:(AND)"]) {
const sep = q.includes("OR") ? "OR" : "AND";
const errs = parse(q).diagnostics.filter(
(d) => d.severity === "error" && d.message.includes(sep),
);
expect(errs, q).toHaveLength(1);
}
});
it("flags unknown fields with an error diagnostic", () => {
const r = parse("nope:1");
expect(r.valid).toBe(false);
expect(r.diagnostics.some((d) => d.message.includes('"nope"'))).toBe(true);
});
it("flags structural issues without throwing", () => {
for (const text of [
"level:",
"level:(ERROR",
'"unclosed',
"(env:dev",
"env:dev)",
"NOT",
"AND",
"level:ERROR OR",
"AND env:dev", // leading AND with no left-hand expression
"tags:(a OR b AND c)",
"-freetext",
]) {
const r = parse(text);
expect(r.valid).toBe(false);
}
});
it("keeps spans pointing at source slices", () => {
const text = "timeout level:ERROR";
const r = parse(text);
expect(r.ast?.kind).toBe("and");
if (r.ast?.kind !== "and") return;
const [t, f] = r.ast.children;
expect(t?.kind === "text" && text.slice(t.span!.from, t.span!.to)).toBe(
"timeout",
);
expect(f?.kind === "filter" && text.slice(f.span!.from, f.span!.to)).toBe(
"level:ERROR",
);
});
it("termAt is quote-aware (one token across quoted spaces)", () => {
const text = 'name:"hello world" x';
expect(termAt(text, 8)?.raw).toBe('name:"hello world"');
});
it("treats unicode whitespace (NBSP) as a token separator", () => {
const r = parse("level:ERRORÂ env:dev");
expect(strip(r.ast)).toEqual({
kind: "and",
children: [
{ kind: "filter", key: "level", op: "=", values: ["ERROR"] },
{ kind: "filter", key: "environment", op: "=", values: ["dev"] },
],
});
});
});
describe("langQ serializer", () => {
it("round-trips canonical text", () => {
for (const text of [
"level:ERROR",
"level:(ERROR OR WARNING)",
"-env:dev",
"latency:>2",
'name:"hello world"',
"tags:(a AND b)",
"timeout level:ERROR",
"has:endTime",
"-has:endTime",
"metadata.region:eu",
"scores.accuracy:>0.8",
]) {
const first = parse(text);
expect(first.valid).toBe(true);
const canonical = serialize(first.ast);
const second = parse(canonical);
expect(strip(second.ast)).toEqual(strip(first.ast));
}
});
it("escaped quotes and backslashes round-trip", () => {
const node: ASTNode = {
kind: "filter",
key: "name",
op: "=",
values: ['say "hi" \\ done'],
};
const text = serialize(node);
const r = parse(text);
expect(strip(r.ast)).toEqual({
kind: "filter",
key: "name",
op: "=",
values: ['say "hi" \\ done'],
});
});
it("collapses same-field OR chains into grouped values", () => {
const r = parse("level:ERROR OR level:WARNING");
expect(serialize(r.ast)).toBe("level:(ERROR OR WARNING)");
});
it("quotes values that would otherwise reparse as operators or keywords", () => {
// Values starting with an operator prefix, equal to a boolean keyword, or
// empty must round-trip through serialize → parse unchanged.
for (const value of [
">5",
"~foo",
"=eq",
"^x",
"$y",
"*z",
"OR",
"AND",
"NOT",
"",
]) {
const node: ASTNode = {
kind: "filter",
key: "name",
op: "=",
values: [value],
};
const r = parse(serialize(node));
expect(r.valid, `value ${JSON.stringify(value)} must round-trip`).toBe(
true,
);
expect(strip(r.ast)).toEqual({
kind: "filter",
key: "name",
op: "=",
values: [value],
});
}
});
it("keeps a keyword value inside an any-of group", () => {
const node: ASTNode = {
kind: "filter",
key: "name",
op: "=",
values: ["x", "OR"],
};
const r = parse(serialize(node));
expect(r.valid).toBe(true);
expect(strip(r.ast)).toEqual({
kind: "filter",
key: "name",
op: "=",
values: ["x", "OR"],
});
});
it("does not over-quote values with mid-string hyphens or operators", () => {
expect(serialize(parse("model:gpt-4-turbo").ast)).toBe("model:gpt-4-turbo");
});
});
describe("edits", () => {
it("removeToken splices when valid", () => {
expect(removeToken("level:ERROR env:dev", { from: 0, to: 11 })).toBe(
"env:dev",
);
expect(removeToken("level:ERROR env:dev", { from: 12, to: 19 })).toBe(
"level:ERROR",
);
});
it("removeToken falls back to AST surgery inside groups", () => {
// Splicing "level:ERROR" out would leave "(OR level:WARNING)" — invalid,
// so the edit reserializes from the surgically-edited AST instead.
const next = removeToken("(level:ERROR OR level:WARNING)", {
from: 1,
to: 12,
});
expect(next).toBe("level:WARNING");
});
it("tidyQueryText strips only provably-redundant parens", () => {
expect(tidyQueryText("(env:dev)")).toBe("env:dev");
expect(tidyQueryText("((env:dev)) level:ERROR")).toBe(
"env:dev level:ERROR",
);
});
});
describe("validateQuery", () => {
it("accepts representable queries", () => {
for (const text of [
"",
"level:ERROR",
"level:(ERROR OR WARNING)",
"-env:dev",
"timeout level:ERROR",
"latency:>2 -has:endTime",
"scores.accuracy:>0.8",
"traceScores.nps:positive",
"metadata.region:*eu*",
"input:refund",
"name:checkout",
"tags:(a AND b)",
"isRootObservation:true",
"-isRootObservation:true",
"level:ERROR OR level:WARNING",
]) {
const r = validateQuery(text);
expect(r.valid, `expected valid: ${text}`).toBe(true);
}
});
it("rejects forms the flat filter contract cannot represent", () => {
for (const text of [
"level:ERROR OR env:dev", // cross-field OR
"NOT (level:ERROR env:dev)", // negated group
"-latency:2", // negated numeric equality
"-input:prefix*", // negated starts-with
"-tags:(a AND b)", // negated all-of
"metadata.region:>5", // metadata comparison
"-metadata.region:eu", // negated metadata equality
"latency:(1 OR 2)", // numeric any-of
"has:(endTime OR userId)", // positive multi-has
"in:nope", // bad scope
"x:1", // unknown field
"-(env:dev)", // standalone "-" before a group (must glue to the filter)
"- env:dev", // stray "-" then a filter
"metadata.", // dot-prefix field with no key
"scores.", // dot-prefix score with no key
"traceScores.", // dot-prefix trace score with no key
"tags:(a,b)", // bare comma inside a group (must use OR/AND)
"level:(ERROR,WARNING)", // same, on an option field
"not level:ERROR", // lowercase `not` reserved (use -field:value)
"!level:ERROR", // `!` reserved (use -field:value)
"level:ERROR or env:dev", // lowercase `or` reserved
"level:ERROR and env:dev", // lowercase `and` reserved
]) {
const r = validateQuery(text);
expect(r.valid, `expected invalid: ${text}`).toBe(false);
}
});
it("nested AND groups behave like the top-level chain", () => {
expect(validateQuery("(env:dev timeout)").valid).toBe(true);
expect(validateQuery("(env:dev level:ERROR)").valid).toBe(true);
});
it("nested same-field OR groups collapse to any-of", () => {
expect(validateQuery("(level:ERROR OR level:WARNING) env:dev").valid).toBe(
true,
);
});
it("warns (without blocking) on has: for non-nullable fields", () => {
const r = validateQuery("has:level");
expect(r.valid).toBe(true);
expect(r.diagnostics.some((d) => d.severity === "warning")).toBe(true);
});
it("caps query length", () => {
const long = `name:${"a".repeat(2100)}`;
expect(validateQuery(long).valid).toBe(false);
});
it("flags a bare comma in a group with a single diagnostic, not a doubled one", () => {
const errors = validateQuery("tags:(a,b)").diagnostics.filter(
(d) => d.severity === "error",
);
expect(errors).toHaveLength(1);
expect(errors[0]!.message).toMatch(/uppercase OR or AND/);
});
it("emits a single Unclosed-paren diagnostic for `level:(`", () => {
const unclosed = validateQuery("level:(").diagnostics.filter((d) =>
d.message.includes("Unclosed"),
);
expect(unclosed).toHaveLength(1);
});
it("reserves operator-looking tokens with a 'not supported yet' diagnostic", () => {
const msgOf = (q: string) =>
validateQuery(q)
.diagnostics.map((d) => d.message)
.join(" | ");
expect(msgOf("not level:ERROR")).toMatch(/not.*not supported yet/i);
expect(msgOf("!level:ERROR")).toMatch(/not supported yet/i);
expect(msgOf("level:ERROR or env:dev")).toMatch(/not supported yet/i);
// quoting escapes the reserved word back into free text
expect(validateQuery('"not" level:ERROR').valid).toBe(true);
});
});