🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(topics): keep traces running when a not-applicable result carries text (#18038)
Lysander Kiesel85e1bd21447a
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.tsindex 320e1f2d54..bde3e03bb5 100644--- a/worker/src/features/topics/processTopicsExecution.ts+++ b/worker/src/features/topics/processTopicsExecution.ts@@ -205,7 +205,10 @@ async function summarizeTrace(result.output.status === "applicable"? "summarized": result.output.status,- summary: result.output.summary.trim(),+ summary:+ result.output.status === "applicable"+ ? result.output.summary.trim()+ : "",providedUsageDetails: result.providedUsageDetails,usageDetails: result.usageDetails,providedCostDetails: result.providedCostDetails,diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 88e5e7c493..956f47ece6 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -254,6 +254,27 @@ describe("summarizeAssembledTrace", () => {});});+ it("stores a not-applicable result that carries stray text instead of failing the trace", async () => {+ state.summarize.mockResolvedValue({+ output: {+ intent_1: { summary: "No problems found.", status: "not_applicable" },+ },+ ...usage,+ });+ await summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ environment: "default",+ traceName: "agent-turn",+ transcript,+ });+ const [written] = state.write.mock.calls[0][0];+ expect(written).toMatchObject({ state: "not_applicable" });+ expect(written.summary ?? "").toBe("");+ expect(state.embed).not.toHaveBeenCalled();+ });+it("keeps the other facets when one facet's output is invalid", async () => {const issues: TopicFacet = {...facet,diff --git a/worker/src/features/topics/summaryResult.ts b/worker/src/features/topics/summaryResult.tsindex 7addaaf182..78ba0d4d07 100644--- a/worker/src/features/topics/summaryResult.ts+++ b/worker/src/features/topics/summaryResult.ts@@ -13,12 +13,9 @@ export function topicSummaryOutputError(output: {status: "applicable" | "not_applicable" | "insufficient_input";}): string | undefined {const summary = output.summary.trim();- if (output.status === "applicable") {- if (!summary || summary.length > 2000)- return "Applicable facet summary must contain a concise summary.";- } else if (summary) {- return "Non-applicable facet result contains a summary.";- }+ // Models sometimes add text to a non-applicable result; callers keep the status and drop the text.+ if (output.status === "applicable" && (!summary || summary.length > 2000))+ return "Applicable facet summary must contain a concise summary.";}function mergeDetails(