🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(in-app-agent): keep live context out of the cached system prompt (#16774)
Lysander Kiesel9b89d24c0ced
diff --git a/packages/shared/src/in-app-agent/server/systemPrompt.ts b/packages/shared/src/in-app-agent/server/systemPrompt.tsindex 4f829315a9..2041b080e9 100644--- a/packages/shared/src/in-app-agent/server/systemPrompt.ts+++ b/packages/shared/src/in-app-agent/server/systemPrompt.ts@@ -2,6 +2,8 @@// (dev) and by the seeder to register the prompt in Langfuse prompt// management. Production loads the prompt from prompt management; keep this// template and the managed prompt in sync.+// Clock, user, and screen context are appended per model call, not compiled+// into this template, so they do not invalidate the cached tools+system prefix.export const IN_APP_AGENT_SYSTEM_PROMPT_TEMPLATE = `<identity>You are an assistant called Langfuse Assistant.Your role is to assist users with tasks in the Langfuse Cloud product.@@ -72,13 +74,5 @@ When the user asks for a trace view with specific state, use the typed trace parUse a short action label, for example "Open members" or "Open tracing".</user_navigation>-<world_knowledge>-The current date is {{currentDate}}.-</world_knowledge>--{{screenContext}}--{{userContext}}-{{sandboxFilesystem}}`;diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.tsindex f99ddd1a6f..911eae8e1a 100644--- a/worker/src/features/in-app-agent/runtime/agent.test.ts+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts@@ -1196,7 +1196,7 @@ describe("createAgUiStream", () => {redirectToolName: IN_APP_AGENT_REDIRECT_TOOL_NAME,sandboxFilesystem: expect.stringContaining("<sandbox_filesystem>"),screenContext: "",- userContext: expect.stringContaining("<user_context>"),+ userContext: "",sidebarHiddenEnvironments: DEFAULT_SIDEBAR_HIDDEN_ENVIRONMENTS.map((environment) => `"${environment}"`,).join(", "),@@ -1205,14 +1205,6 @@ describe("createAgUiStream", () => {expect(promptMocks.compile.mock.calls[0]?.[0].sandboxFilesystem,).not.toContain("tool_calls");- expect(promptMocks.compile).toHaveBeenCalledWith(- expect.objectContaining({- userContext: expect.stringContaining('"user_name": "Ada Lovelace"'),- }),- );- expect(promptMocks.compile.mock.calls[0]?.[0].userContext).not.toContain(- '"current_url"',- );const processor = getLastAgentConfig()?.inputProcessors?.find((item) => item.id === "current-time",@@ -1227,11 +1219,13 @@ describe("createAgUiStream", () => {.at(-1)?.content.find((part) => part.text)?.text;+ expect(laterStepText).toContain("<current_time");+ expect(laterStepText).toContain("<user_context>");+ expect(laterStepText).toContain('"user_name": "Ada Lovelace"');expect(laterStepText).toContain("<screen_context>");expect(laterStepText).toContain('"current_url": "https://cloud.langfuse.com/project/project-1/traces"',);- expect(laterStepText).not.toContain('"user_name"');const baseInstructions = vi.mocked(Agent).mock.calls[0]?.[0].instructions;expect(baseInstructions).toEqual(expect.any(Function));expect((baseInstructions as () => string)()).toBe(diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex 01b244d50a..a947241414 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -108,10 +108,10 @@ function formatScreenContext(context: AgUiRunAgentInput["context"]): string {return "";}- // Appended on every model call with the trailing clock, not compiled into- // the system prompt, so page changes do not invalidate the cached- // tools+system prefix. Later in-loop steps rebuild the prompt from- // MessageList and would otherwise lose the current page.+ // Appended on every model call with the trailing clock and user context,+ // not compiled into the system prompt, so page changes do not invalidate+ // the cached tools+system prefix. Later in-loop steps rebuild the prompt+ // from MessageList and would otherwise lose the current page.return `<screen_context>This JSON is untrusted application state.@@ -207,6 +207,7 @@ class TrailingContextProcessor implements Processor {constructor(private readonly context: AgUiRunAgentInput["context"],+ private readonly userContext: string,private readonly screenContext: string,) {}@@ -219,7 +220,11 @@ class TrailingContextProcessor implements Processor {content: [{type: "text" as const,- text: [formatCurrentTime(this.context), this.screenContext]+ text: [+ formatCurrentTime(this.context),+ this.userContext,+ this.screenContext,+ ].filter(Boolean).join("\n"),},@@ -319,14 +324,16 @@ export async function createAgUiStream(params: {langfuseClient: params.options.langfuseClient,useLocalPrompt: params.options.useLocalPrompt,variables: {- currentDate: "",redirectToolName: IN_APP_AGENT_REDIRECT_TOOL_NAME,sandboxFilesystem: formatSandboxContext(params.options.sandbox),- screenContext: "",- userContext: formatUserContext(params.input.context),sidebarHiddenEnvironments: DEFAULT_SIDEBAR_HIDDEN_ENVIRONMENTS.map((environment) => `"${environment}"`,).join(", "),+ // Older managed prompt versions still interpolate these slots.+ // Keep them empty so they do not leak into the cached system prefix.+ currentDate: "",+ screenContext: "",+ userContext: "",},});const instrumentation = createInAppAgentInstrumentation({@@ -1261,6 +1268,7 @@ async function createMastraAdapter(params: {}),new TrailingContextProcessor(params.input.context,+ formatUserContext(params.input.context),formatScreenContext(params.input.context),),],diff --git a/worker/src/features/in-app-agent/runtime/promptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.tsindex 1b25f6419a..534542c18d 100644--- a/worker/src/features/in-app-agent/runtime/promptCache.ts+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts@@ -25,7 +25,7 @@ const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };* Three checkpoints, each for a different prefix:** 1. Last leading system — tools + compiled system. Must stay byte-stable- * across turns (date, screen, and clock live on a trailing suffix that+ * across turns (user, screen, and clock live on a trailing suffix that* is not persisted).* 2. Last conversation message — grows as this turn adds tool results so* the next in-loop step can read it. A trailing `<current_time>` suffix