Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(topics): accept trace transcript rules in readers (#19228)

marliessophie6889e94a4780
diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.ts
index 47d0651b6d..33dba751ec 100644
--- a/packages/shared/src/features/evals/types.ts
+++ b/packages/shared/src/features/evals/types.ts
@@ -120,6 +120,8 @@ export const EvalTargetObject = {
EXPERIMENT: "experiment",
} as const;
+export const TopicsTargetObject = "trace_transcript" as const;
+
export type EvalTargetObject =
(typeof EvalTargetObject)[keyof typeof EvalTargetObject];
diff --git a/packages/shared/src/server/topics/processing-gate.ts b/packages/shared/src/server/topics/processing-gate.ts
index 49d478ec75..94d59b1313 100644
--- a/packages/shared/src/server/topics/processing-gate.ts
+++ b/packages/shared/src/server/topics/processing-gate.ts
@@ -1,6 +1,9 @@
/* eslint-disable @typescript-eslint/no-use-before-define */
import { JobConfigState, prisma, type Prisma } from "../../db";
-import { EvalTargetObject } from "../../features/evals/types";
+import {
+ EvalTargetObject,
+ TopicsTargetObject,
+} from "../../features/evals/types";
import { logger } from "../logger";
import { redis } from "../redis/redis";
import { isTopicsProjectEnabled } from "./config";
@@ -17,13 +20,13 @@ const CACHE_KEY_PREFIX = "langfuse:topics:processing-active";
const cacheKey = (projectId: string) => `${CACHE_KEY_PREFIX}:${projectId}`;
-/** The project's Topics rule: a trace rule whose evaluators are all facets. */
+/** The project's Topics rule: a transcript rule whose evaluators are all facets. */
export function topicRuleWhere(
projectId: string,
): Prisma.EvaluationRuleWhereInput {
return {
projectId,
- targetObject: EvalTargetObject.TRACE,
+ targetObject: { in: [EvalTargetObject.TRACE, TopicsTargetObject] },
assignments: {
some: {},
every: { projectId, evaluator: { projectId, type: "FACET" } },
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index ff7200e096..a8d14e316a 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -212,105 +212,133 @@ describe("Topics eval-backed configuration", () => {
).toMatchObject({ createdByUserId: userIds[0] });
});
- it("keeps versions and rule assignments scoped to this project's FACET evaluators", async () => {
- const facet = await createTopicFacet({
- projectId,
- name: "Intent",
- description: "Custom facet with a built-in name",
- prompt: "Describe the original request.",
- });
- const next = {
- projectId,
- facetId: facet.id,
- prompt: " Describe the result. ",
- };
- const versions = await Promise.all([
- createTopicFacetVersion(next),
- createTopicFacetVersion(next),
- ]);
- expect(versions.map((version) => version.version)).toEqual([2, 2]);
- expect(await getTopicFacetVersion(projectId, facet.id, 1)).toMatchObject({
- prompt: "Describe the original request.",
- });
- const foreign = await createTopicFacet({
- projectId: otherProjectId,
- name: "Foreign facet",
- description: "",
- prompt: "Describe another project's request.",
- });
- const evaluator = await prisma.evaluator.create({
- data: {
+ it.each(["trace", "trace_transcript"])(
+ "reads and saves %s Topics rules while keeping FACET assignments project-scoped",
+ async (targetObject) => {
+ const { id: projectId } = await prisma.project.create({
+ data: { orgId, name: `Topics ${targetObject}` },
+ });
+ const facet = await createTopicFacet({
projectId,
- name: "Ordinary evaluator",
- type: "LLM_AS_JUDGE",
- versions: { create: { version: 1, prompt: "Grade the answer." } },
- },
- });
- for (const id of [foreign.id, evaluator.id]) {
- expect(await getTopicFacetVersion(projectId, id, 1)).toBeNull();
- await expect(
- createTopicFacetVersion({ ...next, facetId: id }),
- ).rejects.toThrow(/project/);
- await expect(
- saveTopicRule({
+ name: "Intent",
+ description: "Custom facet with a built-in name",
+ prompt: "Describe the original request.",
+ });
+ const next = {
+ projectId,
+ facetId: facet.id,
+ prompt: " Describe the result. ",
+ };
+ const versions = await Promise.all([
+ createTopicFacetVersion(next),
+ createTopicFacetVersion(next),
+ ]);
+ expect(versions.map((version) => version.version)).toEqual([2, 2]);
+ expect(await getTopicFacetVersion(projectId, facet.id, 1)).toMatchObject({
+ prompt: "Describe the original request.",
+ });
+ const foreign = await createTopicFacet({
+ projectId: otherProjectId,
+ name: "Foreign facet",
+ description: "",
+ prompt: "Describe another project's request.",
+ });
+ const evaluator = await prisma.evaluator.create({
+ data: {
projectId,
- name: "Invalid",
- filter: [],
- facetIds: [id],
- }),
- ).rejects.toThrow(/project/);
- }
- const visible = await listTopicFacets(projectId);
- expect(visible.find((value) => value.id === facet.id)).toMatchObject({
- isBuiltIn: false,
- });
- expect(visible.map((value) => value.id)).not.toContain(evaluator.id);
- const rule = await saveTopicRule({
- projectId,
- name: "Saved selection",
- filter: [],
- facetIds: [facet.id, facet.id],
- });
- expect(rule.facetIds).toEqual([facet.id]);
- const second = { projectId, filter: [], facetIds: [facet.id] };
- await expect(saveTopicRule(second)).rejects.toThrow(
- /already has a Topics rule/,
- );
- expect(
- await saveTopicRule({
- ...second,
- id: rule.id,
- name: "Second selection",
- }),
- ).toMatchObject({ id: rule.id, name: "Second selection" });
- expect(
- await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),
- ).toMatchObject({
- targetObject: "trace",
- delay: 0,
- timeScope: ["NEW"],
- status: "ACTIVE",
- });
- const ordinaryRule = await prisma.evaluationRule.create({
- data: {
+ name: "Ordinary evaluator",
+ type: "LLM_AS_JUDGE",
+ versions: { create: { version: 1, prompt: "Grade the answer." } },
+ },
+ });
+ for (const id of [foreign.id, evaluator.id]) {
+ expect(await getTopicFacetVersion(projectId, id, 1)).toBeNull();
+ await expect(
+ createTopicFacetVersion({ ...next, facetId: id }),
+ ).rejects.toThrow(/project/);
+ await expect(
+ saveTopicRule({
+ projectId,
+ name: "Invalid",
+ filter: [],
+ facetIds: [id],
+ }),
+ ).rejects.toThrow(/project/);
+ }
+ const visible = await listTopicFacets(projectId);
+ expect(visible.find((value) => value.id === facet.id)).toMatchObject({
+ isBuiltIn: false,
+ });
+ expect(visible.map((value) => value.id)).not.toContain(evaluator.id);
+ const rule = await saveTopicRule({
projectId,
- name: "Evaluation rule",
+ name: "Saved selection",
filter: [],
- sampling: 1,
- delay: 0,
+ facetIds: [facet.id, facet.id],
+ });
+ expect(rule.facetIds).toEqual([facet.id]);
+ await prisma.evaluationRule.update({
+ where: { id: rule.id, projectId },
+ data: { targetObject },
+ });
+ expect(await getTopicRule(projectId, rule.id)).toMatchObject({
+ id: rule.id,
+ facetIds: [facet.id],
+ });
+ expect(
+ (await listTopicRules(projectId)).map((value) => value.id),
+ ).toEqual([rule.id]);
+ const second = { projectId, filter: [], facetIds: [facet.id] };
+ await expect(saveTopicRule(second)).rejects.toThrow(
+ /already has a Topics rule/,
+ );
+ expect(
+ await saveTopicRule({
+ ...second,
+ id: rule.id,
+ name: "Second selection",
+ }),
+ ).toMatchObject({ id: rule.id, name: "Second selection" });
+ expect(
+ await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),
+ ).toMatchObject({
targetObject: "trace",
- assignments: { create: { projectId, evaluatorId: evaluator.id } },
- },
- });
- expect(await getTopicRule(otherProjectId, rule.id)).toBeNull();
- expect(await getTopicRule(projectId, ordinaryRule.id)).toBeNull();
- expect(
- (await listTopicRules(projectId)).map((value) => value.id),
- ).not.toContain(ordinaryRule.id);
- await expect(
- saveTopicRule({ ...rule, id: ordinaryRule.id }),
- ).rejects.toThrow(/not found/);
- });
+ delay: 0,
+ timeScope: ["NEW"],
+ status: "ACTIVE",
+ });
+ expect(await getTopicRule(otherProjectId, rule.id)).toBeNull();
+ for (const assignments of [
+ [{ projectId, evaluatorId: evaluator.id }],
+ [
+ { projectId, evaluatorId: facet.id },
+ { projectId, evaluatorId: evaluator.id },
+ ],
+ [],
+ [{ projectId, evaluatorId: foreign.id }],
+ [{ projectId: otherProjectId, evaluatorId: facet.id }],
+ ]) {
+ const excludedRule = await prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name: "Not a Topics rule",
+ filter: [],
+ sampling: 1,
+ delay: 0,
+ targetObject,
+ assignments: { create: assignments },
+ },
+ });
+ expect(await getTopicRule(projectId, excludedRule.id)).toBeNull();
+ await expect(
+ saveTopicRule({ ...rule, id: excludedRule.id }),
+ ).rejects.toThrow(/not found/);
+ }
+ expect(
+ (await listTopicRules(projectId)).map((value) => value.id),
+ ).toEqual([rule.id]);
+ },
+ );
it("rejects foreign run versions and atomically creates one journal and run on replay", async () => {
const facet = await createTopicFacet({