Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

feat(agent-chat): log measured prompt-prefix sizes per turn

Khuram Niaz6901a7483a65
diff --git a/__tests__/agentTurnMetrics.spec.ts b/__tests__/agentTurnMetrics.spec.ts
new file mode 100644
index 0000000..8e3f0a5
--- /dev/null
+++ b/__tests__/agentTurnMetrics.spec.ts
@@ -0,0 +1,103 @@
+import { tool, type ModelMessage, type ToolSet } from "ai";
+import { z } from "zod";
+import { measureTurnPrefix } from "@/lib/agent/turnMetrics";
+
+const messages: ModelMessage[] = [{ role: "user", content: "add this job" }];
+
+function toolsWith(description: string): ToolSet {
+ return {
+ add_job: tool({
+ description,
+ inputSchema: z.object({
+ company: z.string().describe("Who is hiring."),
+ }),
+ execute: async () => ({ ok: true }),
+ }),
+ };
+}
+
+describe("measureTurnPrefix", () => {
+ it("counts the JSON schema, not just the description", () => {
+ const metrics = measureTurnPrefix({
+ userId: "u1",
+ system: "system prompt",
+ tools: toolsWith("Add a job."),
+ modelMessages: messages,
+ });
+
+ expect(metrics.systemChars).toBe("system prompt".length);
+ expect(metrics.messageChars).toBe(JSON.stringify(messages).length);
+ // Name plus description is 18 characters; the rest is the schema, which
+ // is the half the review says grows fastest.
+ expect(metrics.toolChars).toBeGreaterThan(100);
+ });
+
+ it("reports no change on the first turn and on an identical repeat", () => {
+ const args = {
+ userId: "u2",
+ system: "system prompt",
+ tools: toolsWith("Add a job."),
+ modelMessages: messages,
+ };
+
+ expect(measureTurnPrefix(args).prefixChanged).toBe(false);
+ expect(measureTurnPrefix(args).prefixChanged).toBe(false);
+ });
+
+ it("reports a change when the tool material differs", () => {
+ measureTurnPrefix({
+ userId: "u3",
+ system: "system prompt",
+ tools: toolsWith("Add a job."),
+ modelMessages: messages,
+ });
+
+ const changed = measureTurnPrefix({
+ userId: "u3",
+ system: "system prompt",
+ tools: toolsWith("Add a job application."),
+ modelMessages: messages,
+ });
+
+ expect(changed.prefixChanged).toBe(true);
+ });
+
+ it("reports a change when the system prompt differs", () => {
+ const tools = toolsWith("Add a job.");
+ measureTurnPrefix({
+ userId: "u4",
+ system: "system prompt",
+ tools,
+ modelMessages: messages,
+ });
+
+ const changed = measureTurnPrefix({
+ userId: "u4",
+ system: "a different system prompt",
+ tools,
+ modelMessages: messages,
+ });
+
+ expect(changed.prefixChanged).toBe(true);
+ });
+
+ it("tracks each user's prefix separately", () => {
+ measureTurnPrefix({
+ userId: "u5",
+ system: "system prompt",
+ tools: toolsWith("Add a job."),
+ modelMessages: messages,
+ });
+
+ // A different user's first turn has no previous prefix of its own, so it
+ // must not read u5's.
+ const other = measureTurnPrefix({
+ userId: "u6",
+ system: "a different system prompt",
+ tools: toolsWith("Add a job application."),
+ modelMessages: messages,
+ });
+
+ expect(other.prefixChanged).toBe(false);
+ });
+});
diff --git a/src/app/api/ai/chat/route.ts b/src/app/api/ai/chat/route.ts
index d3a210b..39b6477 100644
--- a/src/app/api/ai/chat/route.ts
+++ b/src/app/api/ai/chat/route.ts
@@ -30,6 +30,7 @@ import {
} from "@/lib/agent/paste";
import { truncateForModel } from "@/lib/agent/paste.server";
import { buildAgentTools } from "@/lib/agent/tools";
+import { measureTurnPrefix, type TurnPrefixMetrics } from "@/lib/agent/turnMetrics";
import { mapAgentError } from "@/lib/agent/errors";
import { getUserSettings } from "@/actions/userSettings.actions";
import { saveChatConversation } from "@/actions/agentChat.actions";
@@ -178,22 +179,32 @@ export const POST = async (req: NextRequest) => {
AbortSignal.timeout(APP_CONSTANTS.AGENT_CHAT_TIMEOUT_MS),
]);
+ // Written inside execute, read in onFinish, which runs after it.
+ let prefixMetrics: TurnPrefixMetrics | undefined;
+
const stream = createUIMessageStream({
originalMessages: messages,
execute: ({ writer }) => {
+ const tools = buildAgentTools({
+ userId,
+ pastedText,
+ pageContext,
+ model,
+ provider,
+ modelName,
+ writer,
+ });
+ prefixMetrics = measureTurnPrefix({
+ userId,
+ system: AGENT_CHAT_SYSTEM_PROMPT,
+ tools,
+ modelMessages,
+ });
const result = streamText({
model,
system: AGENT_CHAT_SYSTEM_PROMPT,
messages: modelMessages,
- tools: buildAgentTools({
- userId,
- pastedText,
- pageContext,
- model,
- provider,
- modelName,
- writer,
- }),
+ tools,
// Which tools end the turn — and why — lives beside the tool metadata
// in agent.model.ts, where a new tool is registered.
stopWhen: [
@@ -231,6 +242,13 @@ export const POST = async (req: NextRequest) => {
// Windowing is invisible to the user, so this is the only signal that
// the model was given less history than the transcript holds.
historyDropped: messages.length - windowed.length,
+ // What was actually sent. prefixChanged is the cache signal: the
+ // system prompt and tool material must stay byte-identical within a
+ // conversation or Ollama recomputes the whole prefix.
+ systemChars: prefixMetrics?.systemChars ?? 0,
+ toolChars: prefixMetrics?.toolChars ?? 0,
+ messageChars: prefixMetrics?.messageChars ?? 0,
+ prefixChanged: prefixMetrics?.prefixChanged ?? false,
});
// A cancelled turn never writes back. Clear deletes the conversation and
// this fires afterwards on the stream's cancel path, so saving here would
diff --git a/src/lib/agent/turnMetrics.ts b/src/lib/agent/turnMetrics.ts
new file mode 100644
index 0000000..666eea3
--- /dev/null
+++ b/src/lib/agent/turnMetrics.ts
@@ -0,0 +1,55 @@
+import { createHash } from "node:crypto";
+import { asSchema, type ModelMessage, type ToolSet } from "ai";
+
+export type TurnPrefixMetrics = {
+ systemChars: number;
+ toolChars: number;
+ messageChars: number;
+ prefixChanged: boolean;
+};
+
+// Ollama reuses the KV cache only while the leading bytes match the previous
+// request, so a prefix that changes mid-conversation costs a full recompute.
+// Keyed by user because that IS the conversation — ChatConversation.userId is
+// unique. Process-local on purpose: a restart mislabels one turn, which is
+// cheaper than persisting a diagnostic.
+const lastPrefixHash = new Map<string, string>();
+
+// The tool material as the provider will send it: every offered tool's name,
+// description and JSON schema. Assembled once so the size and the hash read
+// the same bytes.
+function toolMaterial(tools: ToolSet): string {
+ return Object.entries(tools)
+ .map(([name, tool]) => {
+ const schema = JSON.stringify(asSchema(tool.inputSchema).jsonSchema);
+ return `${name}\n${tool.description ?? ""}\n${schema}`;
+ })
+ .join("\n");
+}
+
+// Sizes of what was actually sent, not of what the transcript holds. The
+// estimates in docs/reviews/review-chat-agent-optimizations.md become facts
+// here, and prefixChanged is the only direct evidence that the prefix stays
+// cacheable as tools are scoped or deferred.
+export function measureTurnPrefix(args: {
+ userId: string;
+ system: string;
+ tools: ToolSet;
+ modelMessages: ModelMessage[];
+}): TurnPrefixMetrics {
+ const material = toolMaterial(args.tools);
+ const hash = createHash("sha1")
+ .update(args.system)
+ .update("\n")
+ .update(material)
+ .digest("hex");
+ const previous = lastPrefixHash.get(args.userId);
+ lastPrefixHash.set(args.userId, hash);
+
+ return {
+ systemChars: args.system.length,
+ toolChars: material.length,
+ messageChars: JSON.stringify(args.modelMessages).length,
+ prefixChanged: previous !== undefined && previous !== hash,
+ };
+}