Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

9.9 KB
Raw Blame
import type { FilterCondition } from "../../../types";
import type { OrderByState } from "../../../interfaces/orderBy";
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import { describe, expect, it } from "vitest";
import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
import {
  buildEventsObservationRowSelection,
  groupEventsObservationFilters,
} from "./events-observation-row-selection";
import {
  buildEventsBlobExportStreamQuery,
  buildEventsStreamQuery,
} from "./events-stream-query";

const projectId = "project-characterization";
const cutoffCreatedAt = new Date("2025-01-02T03:04:05.678Z");
const nativeFilter: FilterCondition = {
  column: "type",
  operator: "any of",
  value: ["GENERATION"],
  type: "stringOptions",
};
const scoreFilter: FilterCondition = {
  type: "numberObject",
  column: "scores_avg",
  operator: ">",
  key: "quality",
  value: 0.5,
};

const normalizeSql = (query: string) => query.replace(/\s+/g, " ").trim();

const buildSelection = ({
  filter,
  orderBy,
}: {
  filter: FilterCondition[];
  orderBy?: OrderByState;
}) => {
  const selection = buildEventsObservationRowSelection({
    projectId,
    filter,
    orderBy,
  });

  const built = selection.queryBuilder
    .selectRaw("e.span_id AS id")
    .buildWithParams();

  return { ...selection, ...built };
};

describe("buildEventsStreamQuery", () => {
  it("builds the common event-stream selection", () => {
    const { queryBuilder } = buildEventsStreamQuery({
      projectId,
      cutoffCreatedAt,
      filter: [nativeFilter],
      searchQuery: "needle",
      searchType: ["id", "content"],
      rowLimit: 17,
    });
    const { query: rawQuery, params } = queryBuilder
      .selectFieldSet("eval")
      .buildWithParams();
    const query = normalizeSql(rawQuery);

    expect(query).toContain("e.project_id = {projectId: String}");
    expect(query).toContain("e.is_deleted = 0");
    expect(query).toContain("e.is_app_root as is_app_root");
    expect(query).toContain("e.experiment_name as experiment_name");

    const orderIndex = query.lastIndexOf("ORDER BY ");
    const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");
    const rowLimitIndex = query.lastIndexOf("LIMIT {");
    expect(orderIndex).toBeGreaterThan(-1);
    expect(orderIndex).toBeLessThan(deduplicationIndex);
    expect(deduplicationIndex).toBeLessThan(rowLimitIndex);

    expect(params).toMatchObject({
      projectId,
      searchString: "%needle%",
      limit: 17,
    });
    expect(Object.values(params)).toContainEqual(["GENERATION"]);
    expect(Object.values(params)).toContain(
      convertDateToClickhouseDateTime(cutoffCreatedAt),
    );
  });

  it("omits the optional cutoff when it is absent", () => {
    const { queryBuilder } = buildEventsStreamQuery({
      projectId,
      filter: null,
      rowLimit: 7,
    });
    const { query, params } = queryBuilder
      .selectFieldSet("eval")
      .buildWithParams();

    expect(normalizeSql(query)).not.toMatch(/e\."start_time" < \{/);
    expect(params).toEqual({ projectId, limit: 7 });
  });

  it("reads events_full when a metadata filter is present", () => {
    const { queryBuilder } = buildEventsStreamQuery({
      projectId,
      filter: [
        {
          column: "metadata",
          key: "payload",
          operator: "contains",
          value: "needle",
          type: "stringObject",
        },
      ],
      rowLimit: 7,
    });
    const { query } = queryBuilder.selectFieldSet("eval").buildWithParams();

    expect(normalizeSql(query)).toContain("FROM events_full e");
  });

  it("reads events_core when neither search nor filters need full I/O", () => {
    const { queryBuilder } = buildEventsStreamQuery({
      projectId,
      filter: [nativeFilter],
      rowLimit: 7,
    });
    const { query } = queryBuilder.selectFieldSet("eval").buildWithParams();

    expect(normalizeSql(query)).toContain("FROM events_core e");
  });

  it("shares one bounded score dependency between filtering and projection", () => {
    const { queryBuilder, startTimeFrom } = buildEventsBlobExportStreamQuery({
      projectId,
      filter: [
        {
          type: "datetime",
          column: "startTime",
          operator: ">=",
          value: new Date("2025-01-02T03:04:05.678Z"),
        },
        scoreFilter,
      ],
      rowLimit: 7,
    });
    const { query, params } = queryBuilder.buildWithParams();

    expect(startTimeFrom).toBe(params.startTimeFrom);
    expect(query.match(/\bscores_agg AS \(/g)).toHaveLength(1);
    expect(query).toContain("s.scores_avg as scores_avg");
    expect(query).toContain("s.score_categories as score_categories");
    expect(query).toContain(
      "s.score_categories_tuples as score_categories_tuples",
    );
    expect(query).toContain(
      "ON s.trace_id = e.trace_id AND s.observation_id = e.span_id",
    );
    expect(query).toContain(
      "AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 1 HOUR",
    );
    expect(query).not.toContain("name IN ({");
  });

  it("exports trace-level scores alongside observation-level ones (LFE-10596)", () => {
    // No score filter at all: the export must still join and select the
    // trace-score aggregate so trace-level scores land in the file exactly
    // like the UI's unified Scores column.
    const { queryBuilder } = buildEventsBlobExportStreamQuery({
      projectId,
      filter: [],
      rowLimit: 7,
    });
    const { query } = queryBuilder.buildWithParams();

    expect(query.match(/\btrace_scores_agg AS \(/g)).toHaveLength(1);
    expect(query).toContain("ts.scores_avg as trace_scores_avg");
    expect(query).toContain(
      "ts.score_categories_tuples as trace_score_categories_tuples",
    );
    expect(query).toContain(
      "ON ts.trace_id = e.trace_id AND ts.project_id = e.project_id",
    );
  });

  it("applies score filters without exposing them as native event filters", () => {
    const { queryBuilder } = buildEventsStreamQuery({
      projectId,
      filter: [
        nativeFilter,
        {
          column: "Scores",
          key: "quality",
          operator: ">",
          value: 0.5,
          type: "numberObject",
        },
      ],
      rowLimit: 7,
    });
    const { params } = queryBuilder.selectFieldSet("eval").buildWithParams();

    expect(Object.values(params)).toContain("quality");
  });
});

describe("buildEventsObservationRowSelection", () => {
  it("routes score filters and bounds both score scans from Events time", () => {
    const startTime = new Date("2025-01-02T03:04:05.678Z");
    const { query, params, filterGroups } = buildSelection({
      filter: [
        {
          type: "datetime",
          column: "startTime",
          operator: ">=",
          value: startTime,
        },
        {
          type: "numberObject",
          column: "scores_avg",
          operator: ">",
          key: "observation-quality",
          value: 0.5,
        },
        {
          type: "numberObject",
          column: "trace_scores_avg",
          operator: ">",
          key: "trace-quality",
          value: 0.5,
        },
      ],
    });

    expect(filterGroups.events).toHaveLength(1);
    expect(filterGroups.observationScores).toHaveLength(1);
    expect(filterGroups.traceScores).toHaveLength(1);
    expect(Object.values(params)).toContain("observation-quality");
    expect(Object.values(params)).toContain("trace-quality");
    expect(Object.values(params)).toContainEqual([
      "observation-quality",
      "trace-quality",
    ]);
    expect(query.match(/\bname IN \(\{/g)).toHaveLength(2);
    expect(query).toContain(
      "AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 1 HOUR",
    );
    expect(query).toContain(
      "AND timestamp >= {startTimeFrom: DateTime64(3)} - INTERVAL 2 DAY - INTERVAL 1 HOUR",
    );
  });

  it("does not invent score time bounds without an Events lower bound", () => {
    const { query, startTimeFrom } = buildSelection({
      filter: [
        {
          type: "numberObject",
          column: "scores_avg",
          operator: ">",
          key: "observation-quality",
          value: 0.5,
        },
        {
          type: "numberObject",
          column: "trace_scores_avg",
          operator: ">",
          key: "trace-quality",
          value: 0.5,
        },
      ],
    });

    expect(startTimeFrom).toBeNull();
    expect(query).not.toContain("AND timestamp >=");
  });

  it("keeps complete score data when ordering by a score", () => {
    const { query } = buildSelection({
      filter: [scoreFilter],
      orderBy: { column: "scores_avg", order: "DESC" },
    });

    expect(query).not.toContain("name IN ({");

    for (const [column, expectedJoin] of [
      ["scores_avg", "LEFT JOIN scores_agg AS s"],
      ["trace_scores_avg", "LEFT JOIN trace_scores_agg AS ts"],
    ] as const) {
      const scoreOrderQuery = buildSelection({
        filter: [],
        orderBy: { column, order: "DESC" },
      }).query;
      expect(scoreOrderQuery).toContain(expectedJoin);
    }
  });

  it.each([
    ["scores_avg", "observationScores"],
    ["trace_scores_avg", "traceScores"],
    ["Scores", "observationScores"],
    ["SCORES", "observationScores"],
    ["Scores (numeric)", "observationScores"],
    ["Trace Scores (numeric)", "traceScores"],
  ] as const)(
    "classifies the score column %s as %s",
    (column, expectedGroup) => {
      const filter: FilterCondition = {
        type: "numberObject",
        column,
        operator: "=",
        key: "score-key",
        value: 1,
      };
      const filterGroups = groupEventsObservationFilters([filter]);
      expect(filterGroups[expectedGroup]).toHaveLength(1);
    },
  );

  it("rejects unresolved comment filters", () => {
    expect(() =>
      buildSelection({
        filter: [
          {
            column: "commentContent",
            operator: "contains",
            value: "comment-needle",
            type: "string",
          },
        ],
      }),
    ).toThrow(InvalidRequestError);
  });
});