Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
feat(agent-chat): log measured prompt-prefix sizes per turn
Khuram Niaz6901a7483a65
diff --git a/__tests__/agentTurnMetrics.spec.ts b/__tests__/agentTurnMetrics.spec.tsnew file mode 100644index 0000000..8e3f0a5--- /dev/null+++ b/__tests__/agentTurnMetrics.spec.ts@@ -0,0 +1,103 @@+import { tool, type ModelMessage, type ToolSet } from "ai";+import { z } from "zod";+import { measureTurnPrefix } from "@/lib/agent/turnMetrics";++const messages: ModelMessage[] = [{ role: "user", content: "add this job" }];++function toolsWith(description: string): ToolSet {+ return {+ add_job: tool({+ description,+ inputSchema: z.object({+ company: z.string().describe("Who is hiring."),+ }),+ execute: async () => ({ ok: true }),+ }),+ };+}++describe("measureTurnPrefix", () => {+ it("counts the JSON schema, not just the description", () => {+ const metrics = measureTurnPrefix({+ userId: "u1",+ system: "system prompt",+ tools: toolsWith("Add a job."),+ modelMessages: messages,+ });++ expect(metrics.systemChars).toBe("system prompt".length);+ expect(metrics.messageChars).toBe(JSON.stringify(messages).length);+ // Name plus description is 18 characters; the rest is the schema, which+ // is the half the review says grows fastest.+ expect(metrics.toolChars).toBeGreaterThan(100);+ });++ it("reports no change on the first turn and on an identical repeat", () => {+ const args = {+ userId: "u2",+ system: "system prompt",+ tools: toolsWith("Add a job."),+ modelMessages: messages,+ };++ expect(measureTurnPrefix(args).prefixChanged).toBe(false);+ expect(measureTurnPrefix(args).prefixChanged).toBe(false);+ });++ it("reports a change when the tool material differs", () => {+ measureTurnPrefix({+ userId: "u3",+ system: "system prompt",+ tools: toolsWith("Add a job."),+ modelMessages: messages,+ });++ const changed = measureTurnPrefix({+ userId: "u3",+ system: "system prompt",+ tools: toolsWith("Add a job application."),+ modelMessages: messages,+ });++ expect(changed.prefixChanged).toBe(true);+ });++ it("reports a change when the system prompt differs", () => {+ const tools = toolsWith("Add a job.");+ measureTurnPrefix({+ userId: "u4",+ system: "system prompt",+ tools,+ modelMessages: messages,+ });++ const changed = measureTurnPrefix({+ userId: "u4",+ system: "a different system prompt",+ tools,+ modelMessages: messages,+ });++ expect(changed.prefixChanged).toBe(true);+ });++ it("tracks each user's prefix separately", () => {+ measureTurnPrefix({+ userId: "u5",+ system: "system prompt",+ tools: toolsWith("Add a job."),+ modelMessages: messages,+ });++ // A different user's first turn has no previous prefix of its own, so it+ // must not read u5's.+ const other = measureTurnPrefix({+ userId: "u6",+ system: "a different system prompt",+ tools: toolsWith("Add a job application."),+ modelMessages: messages,+ });++ expect(other.prefixChanged).toBe(false);+ });+});diff --git a/src/app/api/ai/chat/route.ts b/src/app/api/ai/chat/route.tsindex d3a210b..39b6477 100644--- a/src/app/api/ai/chat/route.ts+++ b/src/app/api/ai/chat/route.ts@@ -30,6 +30,7 @@ import {} from "@/lib/agent/paste";import { truncateForModel } from "@/lib/agent/paste.server";import { buildAgentTools } from "@/lib/agent/tools";+import { measureTurnPrefix, type TurnPrefixMetrics } from "@/lib/agent/turnMetrics";import { mapAgentError } from "@/lib/agent/errors";import { getUserSettings } from "@/actions/userSettings.actions";import { saveChatConversation } from "@/actions/agentChat.actions";@@ -178,22 +179,32 @@ export const POST = async (req: NextRequest) => {AbortSignal.timeout(APP_CONSTANTS.AGENT_CHAT_TIMEOUT_MS),]);+ // Written inside execute, read in onFinish, which runs after it.+ let prefixMetrics: TurnPrefixMetrics | undefined;+const stream = createUIMessageStream({originalMessages: messages,execute: ({ writer }) => {+ const tools = buildAgentTools({+ userId,+ pastedText,+ pageContext,+ model,+ provider,+ modelName,+ writer,+ });+ prefixMetrics = measureTurnPrefix({+ userId,+ system: AGENT_CHAT_SYSTEM_PROMPT,+ tools,+ modelMessages,+ });const result = streamText({model,system: AGENT_CHAT_SYSTEM_PROMPT,messages: modelMessages,- tools: buildAgentTools({- userId,- pastedText,- pageContext,- model,- provider,- modelName,- writer,- }),+ tools,// Which tools end the turn — and why — lives beside the tool metadata// in agent.model.ts, where a new tool is registered.stopWhen: [@@ -231,6 +242,13 @@ export const POST = async (req: NextRequest) => {// Windowing is invisible to the user, so this is the only signal that// the model was given less history than the transcript holds.historyDropped: messages.length - windowed.length,+ // What was actually sent. prefixChanged is the cache signal: the+ // system prompt and tool material must stay byte-identical within a+ // conversation or Ollama recomputes the whole prefix.+ systemChars: prefixMetrics?.systemChars ?? 0,+ toolChars: prefixMetrics?.toolChars ?? 0,+ messageChars: prefixMetrics?.messageChars ?? 0,+ prefixChanged: prefixMetrics?.prefixChanged ?? false,});// A cancelled turn never writes back. Clear deletes the conversation and// this fires afterwards on the stream's cancel path, so saving here woulddiff --git a/src/lib/agent/turnMetrics.ts b/src/lib/agent/turnMetrics.tsnew file mode 100644index 0000000..666eea3--- /dev/null+++ b/src/lib/agent/turnMetrics.ts@@ -0,0 +1,55 @@+import { createHash } from "node:crypto";+import { asSchema, type ModelMessage, type ToolSet } from "ai";++export type TurnPrefixMetrics = {+ systemChars: number;+ toolChars: number;+ messageChars: number;+ prefixChanged: boolean;+};++// Ollama reuses the KV cache only while the leading bytes match the previous+// request, so a prefix that changes mid-conversation costs a full recompute.+// Keyed by user because that IS the conversation — ChatConversation.userId is+// unique. Process-local on purpose: a restart mislabels one turn, which is+// cheaper than persisting a diagnostic.+const lastPrefixHash = new Map<string, string>();++// The tool material as the provider will send it: every offered tool's name,+// description and JSON schema. Assembled once so the size and the hash read+// the same bytes.+function toolMaterial(tools: ToolSet): string {+ return Object.entries(tools)+ .map(([name, tool]) => {+ const schema = JSON.stringify(asSchema(tool.inputSchema).jsonSchema);+ return `${name}\n${tool.description ?? ""}\n${schema}`;+ })+ .join("\n");+}++// Sizes of what was actually sent, not of what the transcript holds. The+// estimates in docs/reviews/review-chat-agent-optimizations.md become facts+// here, and prefixChanged is the only direct evidence that the prefix stays+// cacheable as tools are scoped or deferred.+export function measureTurnPrefix(args: {+ userId: string;+ system: string;+ tools: ToolSet;+ modelMessages: ModelMessage[];+}): TurnPrefixMetrics {+ const material = toolMaterial(args.tools);+ const hash = createHash("sha1")+ .update(args.system)+ .update("\n")+ .update(material)+ .digest("hex");+ const previous = lastPrefixHash.get(args.userId);+ lastPrefixHash.set(args.userId, hash);++ return {+ systemChars: args.system.length,+ toolChars: material.length,+ messageChars: JSON.stringify(args.modelMessages).length,+ prefixChanged: previous !== undefined && previous !== hash,+ };+}