Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(topics): keep traces running when a not-applicable result carries text (#18038)

Lysander Kiesel85e1bd21447a
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index 320e1f2d54..bde3e03bb5 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -205,7 +205,10 @@ async function summarizeTrace(
result.output.status === "applicable"
? "summarized"
: result.output.status,
- summary: result.output.summary.trim(),
+ summary:
+ result.output.status === "applicable"
+ ? result.output.summary.trim()
+ : "",
providedUsageDetails: result.providedUsageDetails,
usageDetails: result.usageDetails,
providedCostDetails: result.providedCostDetails,
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index 88e5e7c493..956f47ece6 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -254,6 +254,27 @@ describe("summarizeAssembledTrace", () => {
});
});
+ it("stores a not-applicable result that carries stray text instead of failing the trace", async () => {
+ state.summarize.mockResolvedValue({
+ output: {
+ intent_1: { summary: "No problems found.", status: "not_applicable" },
+ },
+ ...usage,
+ });
+ await summarizeAssembledTrace({
+ projectId: "project-a",
+ traceId: "trace-1",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ environment: "default",
+ traceName: "agent-turn",
+ transcript,
+ });
+ const [written] = state.write.mock.calls[0][0];
+ expect(written).toMatchObject({ state: "not_applicable" });
+ expect(written.summary ?? "").toBe("");
+ expect(state.embed).not.toHaveBeenCalled();
+ });
+
it("keeps the other facets when one facet's output is invalid", async () => {
const issues: TopicFacet = {
...facet,
diff --git a/worker/src/features/topics/summaryResult.ts b/worker/src/features/topics/summaryResult.ts
index 7addaaf182..78ba0d4d07 100644
--- a/worker/src/features/topics/summaryResult.ts
+++ b/worker/src/features/topics/summaryResult.ts
@@ -13,12 +13,9 @@ export function topicSummaryOutputError(output: {
status: "applicable" | "not_applicable" | "insufficient_input";
}): string | undefined {
const summary = output.summary.trim();
- if (output.status === "applicable") {
- if (!summary || summary.length > 2000)
- return "Applicable facet summary must contain a concise summary.";
- } else if (summary) {
- return "Non-applicable facet result contains a summary.";
- }
+ // Models sometimes add text to a non-applicable result; callers keep the status and drop the text.
+ if (output.status === "applicable" && (!summary || summary.length > 2000))
+ return "Applicable facet summary must contain a concise summary.";
}
function mergeDetails(