🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(topics): accept trace transcript rules in readers (#19228)
marliessophie6889e94a4780
diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.tsindex 47d0651b6d..33dba751ec 100644--- a/packages/shared/src/features/evals/types.ts+++ b/packages/shared/src/features/evals/types.ts@@ -120,6 +120,8 @@ export const EvalTargetObject = {EXPERIMENT: "experiment",} as const;+export const TopicsTargetObject = "trace_transcript" as const;+export type EvalTargetObject =(typeof EvalTargetObject)[keyof typeof EvalTargetObject];diff --git a/packages/shared/src/server/topics/processing-gate.ts b/packages/shared/src/server/topics/processing-gate.tsindex 49d478ec75..94d59b1313 100644--- a/packages/shared/src/server/topics/processing-gate.ts+++ b/packages/shared/src/server/topics/processing-gate.ts@@ -1,6 +1,9 @@/* eslint-disable @typescript-eslint/no-use-before-define */import { JobConfigState, prisma, type Prisma } from "../../db";-import { EvalTargetObject } from "../../features/evals/types";+import {+ EvalTargetObject,+ TopicsTargetObject,+} from "../../features/evals/types";import { logger } from "../logger";import { redis } from "../redis/redis";import { isTopicsProjectEnabled } from "./config";@@ -17,13 +20,13 @@ const CACHE_KEY_PREFIX = "langfuse:topics:processing-active";const cacheKey = (projectId: string) => `${CACHE_KEY_PREFIX}:${projectId}`;-/** The project's Topics rule: a trace rule whose evaluators are all facets. */+/** The project's Topics rule: a transcript rule whose evaluators are all facets. */export function topicRuleWhere(projectId: string,): Prisma.EvaluationRuleWhereInput {return {projectId,- targetObject: EvalTargetObject.TRACE,+ targetObject: { in: [EvalTargetObject.TRACE, TopicsTargetObject] },assignments: {some: {},every: { projectId, evaluator: { projectId, type: "FACET" } },diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.tsindex ff7200e096..a8d14e316a 100644--- a/worker/src/features/topics/storage.integration.test.ts+++ b/worker/src/features/topics/storage.integration.test.ts@@ -212,105 +212,133 @@ describe("Topics eval-backed configuration", () => {).toMatchObject({ createdByUserId: userIds[0] });});- it("keeps versions and rule assignments scoped to this project's FACET evaluators", async () => {- const facet = await createTopicFacet({- projectId,- name: "Intent",- description: "Custom facet with a built-in name",- prompt: "Describe the original request.",- });- const next = {- projectId,- facetId: facet.id,- prompt: " Describe the result. ",- };- const versions = await Promise.all([- createTopicFacetVersion(next),- createTopicFacetVersion(next),- ]);- expect(versions.map((version) => version.version)).toEqual([2, 2]);- expect(await getTopicFacetVersion(projectId, facet.id, 1)).toMatchObject({- prompt: "Describe the original request.",- });- const foreign = await createTopicFacet({- projectId: otherProjectId,- name: "Foreign facet",- description: "",- prompt: "Describe another project's request.",- });- const evaluator = await prisma.evaluator.create({- data: {+ it.each(["trace", "trace_transcript"])(+ "reads and saves %s Topics rules while keeping FACET assignments project-scoped",+ async (targetObject) => {+ const { id: projectId } = await prisma.project.create({+ data: { orgId, name: `Topics ${targetObject}` },+ });+ const facet = await createTopicFacet({projectId,- name: "Ordinary evaluator",- type: "LLM_AS_JUDGE",- versions: { create: { version: 1, prompt: "Grade the answer." } },- },- });- for (const id of [foreign.id, evaluator.id]) {- expect(await getTopicFacetVersion(projectId, id, 1)).toBeNull();- await expect(- createTopicFacetVersion({ ...next, facetId: id }),- ).rejects.toThrow(/project/);- await expect(- saveTopicRule({+ name: "Intent",+ description: "Custom facet with a built-in name",+ prompt: "Describe the original request.",+ });+ const next = {+ projectId,+ facetId: facet.id,+ prompt: " Describe the result. ",+ };+ const versions = await Promise.all([+ createTopicFacetVersion(next),+ createTopicFacetVersion(next),+ ]);+ expect(versions.map((version) => version.version)).toEqual([2, 2]);+ expect(await getTopicFacetVersion(projectId, facet.id, 1)).toMatchObject({+ prompt: "Describe the original request.",+ });+ const foreign = await createTopicFacet({+ projectId: otherProjectId,+ name: "Foreign facet",+ description: "",+ prompt: "Describe another project's request.",+ });+ const evaluator = await prisma.evaluator.create({+ data: {projectId,- name: "Invalid",- filter: [],- facetIds: [id],- }),- ).rejects.toThrow(/project/);- }- const visible = await listTopicFacets(projectId);- expect(visible.find((value) => value.id === facet.id)).toMatchObject({- isBuiltIn: false,- });- expect(visible.map((value) => value.id)).not.toContain(evaluator.id);- const rule = await saveTopicRule({- projectId,- name: "Saved selection",- filter: [],- facetIds: [facet.id, facet.id],- });- expect(rule.facetIds).toEqual([facet.id]);- const second = { projectId, filter: [], facetIds: [facet.id] };- await expect(saveTopicRule(second)).rejects.toThrow(- /already has a Topics rule/,- );- expect(- await saveTopicRule({- ...second,- id: rule.id,- name: "Second selection",- }),- ).toMatchObject({ id: rule.id, name: "Second selection" });- expect(- await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),- ).toMatchObject({- targetObject: "trace",- delay: 0,- timeScope: ["NEW"],- status: "ACTIVE",- });- const ordinaryRule = await prisma.evaluationRule.create({- data: {+ name: "Ordinary evaluator",+ type: "LLM_AS_JUDGE",+ versions: { create: { version: 1, prompt: "Grade the answer." } },+ },+ });+ for (const id of [foreign.id, evaluator.id]) {+ expect(await getTopicFacetVersion(projectId, id, 1)).toBeNull();+ await expect(+ createTopicFacetVersion({ ...next, facetId: id }),+ ).rejects.toThrow(/project/);+ await expect(+ saveTopicRule({+ projectId,+ name: "Invalid",+ filter: [],+ facetIds: [id],+ }),+ ).rejects.toThrow(/project/);+ }+ const visible = await listTopicFacets(projectId);+ expect(visible.find((value) => value.id === facet.id)).toMatchObject({+ isBuiltIn: false,+ });+ expect(visible.map((value) => value.id)).not.toContain(evaluator.id);+ const rule = await saveTopicRule({projectId,- name: "Evaluation rule",+ name: "Saved selection",filter: [],- sampling: 1,- delay: 0,+ facetIds: [facet.id, facet.id],+ });+ expect(rule.facetIds).toEqual([facet.id]);+ await prisma.evaluationRule.update({+ where: { id: rule.id, projectId },+ data: { targetObject },+ });+ expect(await getTopicRule(projectId, rule.id)).toMatchObject({+ id: rule.id,+ facetIds: [facet.id],+ });+ expect(+ (await listTopicRules(projectId)).map((value) => value.id),+ ).toEqual([rule.id]);+ const second = { projectId, filter: [], facetIds: [facet.id] };+ await expect(saveTopicRule(second)).rejects.toThrow(+ /already has a Topics rule/,+ );+ expect(+ await saveTopicRule({+ ...second,+ id: rule.id,+ name: "Second selection",+ }),+ ).toMatchObject({ id: rule.id, name: "Second selection" });+ expect(+ await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),+ ).toMatchObject({targetObject: "trace",- assignments: { create: { projectId, evaluatorId: evaluator.id } },- },- });- expect(await getTopicRule(otherProjectId, rule.id)).toBeNull();- expect(await getTopicRule(projectId, ordinaryRule.id)).toBeNull();- expect(- (await listTopicRules(projectId)).map((value) => value.id),- ).not.toContain(ordinaryRule.id);- await expect(- saveTopicRule({ ...rule, id: ordinaryRule.id }),- ).rejects.toThrow(/not found/);- });+ delay: 0,+ timeScope: ["NEW"],+ status: "ACTIVE",+ });+ expect(await getTopicRule(otherProjectId, rule.id)).toBeNull();+ for (const assignments of [+ [{ projectId, evaluatorId: evaluator.id }],+ [+ { projectId, evaluatorId: facet.id },+ { projectId, evaluatorId: evaluator.id },+ ],+ [],+ [{ projectId, evaluatorId: foreign.id }],+ [{ projectId: otherProjectId, evaluatorId: facet.id }],+ ]) {+ const excludedRule = await prisma.evaluationRule.create({+ data: {+ projectId,+ name: "Not a Topics rule",+ filter: [],+ sampling: 1,+ delay: 0,+ targetObject,+ assignments: { create: assignments },+ },+ });+ expect(await getTopicRule(projectId, excludedRule.id)).toBeNull();+ await expect(+ saveTopicRule({ ...rule, id: excludedRule.id }),+ ).rejects.toThrow(/not found/);+ }+ expect(+ (await listTopicRules(projectId)).map((value) => value.id),+ ).toEqual([rule.id]);+ },+ );it("rejects foreign run versions and atomically creates one journal and run on replay", async () => {const facet = await createTopicFacet({