Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

feat(ai-usage): record every AI call — chat turns (excluding nested time), nested review/match/cover letter, resume import, automation match and the Analyze button — with first visible token and timeout-vs-stop classification

Khuram Niaz54109269d360
diff --git a/__tests__/agentAddJobTool.spec.ts b/__tests__/agentAddJobTool.spec.ts
index 78d8e8b..d4f3ddb 100644
--- a/__tests__/agentAddJobTool.spec.ts
+++ b/__tests__/agentAddJobTool.spec.ts
@@ -25,6 +25,7 @@ const toolCtx = {
provider: "ollama",
modelName: "qwen3.5:9b",
writer: { write: () => {}, merge: () => {}, onError: undefined } as any,
+ nestedGuard: { running: false, elapsedMs: 0 },
};
describe("add_job agent tool", () => {
diff --git a/__tests__/agentChatRoute.spec.ts b/__tests__/agentChatRoute.spec.ts
index 90c9e58..d4fc295 100644
--- a/__tests__/agentChatRoute.spec.ts
+++ b/__tests__/agentChatRoute.spec.ts
@@ -16,6 +16,14 @@ vi.mock("@/lib/agent/tools", async (importOriginal) => {
return { ...actual, buildAgentTools: vi.fn(actual.buildAgentTools) };
});
+const markFirstToken = vi.fn();
+const finish = vi.fn();
+const startAiCall = vi.fn((..._args: unknown[]) => ({ markFirstToken, finish }));
+vi.mock("@/lib/aiUsage/tracker", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("@/lib/aiUsage/tracker")>();
+ return { ...actual, startAiCall: (...args: unknown[]) => startAiCall(...args) };
+});
+
// The route merges this into createUIMessageStream now. An immediately
// closed stream is enough — these tests assert on the streamText ARGS.
const toUIMessageStream = vi.fn(
@@ -404,4 +412,65 @@ describe("POST /api/ai/chat", () => {
const sent = JSON.stringify(streamArgs().messages);
expect(sent).not.toContain("<page-context>");
});
+
+ describe("usage", () => {
+ const finishTurn = (isAborted = false) =>
+ capturedOnFinish!({ messages: [pasteMessage("posting")], responseMessage: undefined, isAborted });
+
+ it("starts one agent_chat call with the chat num_ctx", async () => {
+ await POST(req({ messages: [pasteMessage("posting")] }));
+ expect(startAiCall).toHaveBeenCalledWith({
+ userId: "user-1",
+ feature: "agent_chat",
+ provider: "ollama",
+ model: "qwen3.5:9b",
+ numCtx: APP_CONSTANTS.AGENT_CHAT_NUM_CTX,
+ });
+ });
+
+ it("marks the first token on text or a tool call, not on reasoning", async () => {
+ await POST(req({ messages: [pasteMessage("posting")] }));
+ const { onChunk } = streamArgs();
+ onChunk({ chunk: { type: "reasoning-delta", text: "hmm" } });
+ // Ollama with think:true sends content "" beside every thinking chunk.
+ onChunk({ chunk: { type: "text-delta", text: "" } });
+ expect(markFirstToken).not.toHaveBeenCalled();
+ onChunk({ chunk: { type: "tool-input-start" } });
+ expect(markFirstToken).toHaveBeenCalledTimes(1);
+ onChunk({ chunk: { type: "text-delta", text: "Hi" } });
+ expect(markFirstToken).toHaveBeenCalledTimes(2);
+ });
+
+ it("records a finished turn with its total usage", async () => {
+ await POST(req({ messages: [pasteMessage("posting")] }));
+ streamArgs().onFinish({ totalUsage: { inputTokens: 5000, outputTokens: 200 }, finishReason: "stop" });
+ await finishTurn();
+ expect(finish).toHaveBeenCalledWith({
+ usage: { inputTokens: 5000, outputTokens: 200 },
+ finishReason: "stop",
+ excludeMs: 0,
+ });
+ });
+
+ it("records a stopped turn with finished-step usage", async () => {
+ const controller = new AbortController();
+ await POST({ json: async () => ({ messages: [pasteMessage("posting")] }), signal: controller.signal } as any);
+ streamArgs().onAbort({ steps: [{ usage: { inputTokens: 4000, outputTokens: 50 } }] });
+ controller.abort();
+ await finishTurn(true);
+ const arg = finish.mock.calls[0][0];
+ expect(arg.usage).toEqual({ inputTokens: 4000, outputTokens: 50 });
+ expect(arg.abortedBy.aborted).toBe(true);
+ expect(arg.abortedBy.reason.name).not.toBe("TimeoutError");
+ });
+
+ it("subtracts nested time from the turn", async () => {
+ await POST(req({ messages: [pasteMessage("posting")] }));
+ const { nestedGuard } = (buildAgentTools as any).mock.calls[0][0];
+ nestedGuard.elapsedMs = 90_000;
+ streamArgs().onFinish({ totalUsage: {}, finishReason: "stop" });
+ await finishTurn();
+ expect(finish.mock.calls[0][0].excludeMs).toBe(90_000);
+ });
+ });
});
diff --git a/__tests__/agentCoverLetterTool.spec.ts b/__tests__/agentCoverLetterTool.spec.ts
index 71efb8c..e71a983 100644
--- a/__tests__/agentCoverLetterTool.spec.ts
+++ b/__tests__/agentCoverLetterTool.spec.ts
@@ -58,7 +58,7 @@ const ctx = (overrides: Record<string, unknown> = {}) => ({
provider: "ollama",
modelName: "qwen3.5:9b",
writer: writer as any,
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
...overrides,
});
@@ -317,7 +317,7 @@ describe("generate_cover_letter agent tool", () => {
// The guard is shared with the other two nested tools, per request.
it("declines to start while another nested generation is running", async () => {
const result = await execute(
- buildGenerateCoverLetterTool(ctx({ guard: { running: true } })),
+ buildGenerateCoverLetterTool(ctx({ guard: { running: true, elapsedMs: 0 } })),
{},
);
expect(result.status).toBe("generation_failed");
diff --git a/__tests__/agentGetResumeTool.spec.ts b/__tests__/agentGetResumeTool.spec.ts
index ecb1882..bf574e8 100644
--- a/__tests__/agentGetResumeTool.spec.ts
+++ b/__tests__/agentGetResumeTool.spec.ts
@@ -22,6 +22,7 @@ const toolCtx = {
provider: "ollama",
modelName: "qwen3.5:9b",
writer: { write: () => {}, merge: () => {}, onError: undefined } as any,
+ nestedGuard: { running: false, elapsedMs: 0 },
};
describe("get_resume agent tool", () => {
diff --git a/__tests__/agentMatchJobTool.spec.ts b/__tests__/agentMatchJobTool.spec.ts
index aebadae..ae883b4 100644
--- a/__tests__/agentMatchJobTool.spec.ts
+++ b/__tests__/agentMatchJobTool.spec.ts
@@ -53,7 +53,7 @@ const ctx = () => ({
provider: "ollama",
modelName: "qwen3.5:9b",
writer: writer as any,
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
});
const execute = (agentTool: any, input: any) =>
diff --git a/__tests__/agentNestedGeneration.spec.ts b/__tests__/agentNestedGeneration.spec.ts
index b495faf..d652fa6 100644
--- a/__tests__/agentNestedGeneration.spec.ts
+++ b/__tests__/agentNestedGeneration.spec.ts
@@ -7,12 +7,24 @@ vi.mock("ai", async (importOriginal) => {
return { ...actual, streamText: (...args: unknown[]) => streamText(...(args as [])) };
});
-function textStreamOf(chunks: string[], finishReason: unknown = "stop") {
+const markFirstToken = vi.fn();
+const finish = vi.fn();
+const startAiCall = vi.fn((..._args: unknown[]) => ({ markFirstToken, finish }));
+vi.mock("@/lib/aiUsage/tracker", () => ({
+ startAiCall: (...args: unknown[]) => startAiCall(...args),
+}));
+
+function textStreamOf(
+ chunks: string[],
+ finishReason: unknown = "stop",
+ totalUsage: unknown = { inputTokens: 700, outputTokens: 300 },
+) {
return {
textStream: (async function* () {
for (const chunk of chunks) yield chunk;
})(),
finishReason: Promise.resolve(finishReason),
+ totalUsage: Promise.resolve(totalUsage),
};
}
@@ -27,10 +39,11 @@ const args = () => ({
timeoutMs: 1000,
writer: writer as any,
toolCallId: "call-1",
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
label: "test_tool",
provider: "ollama",
modelName: "test-model",
+ userId: "user-1",
});
describe("runNestedGeneration", () => {
@@ -110,14 +123,14 @@ describe("runNestedGeneration", () => {
// Two nested generations in one turn serialize on Ollama to ~360s against a
// 300s deadline, so the second must be refused rather than started.
it("refuses a second generation while one is already running", async () => {
- const guard = { running: true };
+ const guard = { running: true, elapsedMs: 0 };
const result = await runNestedGeneration({ ...args(), guard });
expect(result).toEqual({ status: "busy" });
expect(streamText).not.toHaveBeenCalled();
});
it("releases the guard so a later generation can run", async () => {
- const guard = { running: false };
+ const guard = { running: false, elapsedMs: 0 };
await runNestedGeneration({ ...args(), guard });
expect(guard.running).toBe(false);
const second = await runNestedGeneration({ ...args(), guard });
@@ -125,7 +138,7 @@ describe("runNestedGeneration", () => {
});
it("releases the guard even when the generation throws", async () => {
- const guard = { running: false };
+ const guard = { running: false, elapsedMs: 0 };
streamText.mockReturnValue({
textStream: (async function* () {
throw new Error("boom");
@@ -135,3 +148,75 @@ describe("runNestedGeneration", () => {
expect(guard.running).toBe(false);
});
});
+
+describe("runNestedGeneration usage", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ streamText.mockReturnValue(textStreamOf(["hello ", "world"]));
+ });
+
+ it("records the call under the feature its label maps to", async () => {
+ await runNestedGeneration({ ...args(), label: "review_resume" });
+ expect(startAiCall).toHaveBeenCalledWith({
+ userId: "user-1",
+ feature: "resume_review",
+ provider: "ollama",
+ model: "test-model",
+ numCtx: APP_CONSTANTS.AI_OLLAMA_NUM_CTX,
+ });
+ });
+
+ it("marks the first token and finishes with usage on a clean run", async () => {
+ await runNestedGeneration(args());
+ expect(markFirstToken).toHaveBeenCalled();
+ expect(finish).toHaveBeenCalledWith({
+ usage: { inputTokens: 700, outputTokens: 300 },
+ finishReason: "stop",
+ });
+ });
+
+ it("finishes with the finish reason when incomplete", async () => {
+ streamText.mockReturnValue(textStreamOf(["partial"], "length"));
+ await runNestedGeneration(args());
+ expect(finish).toHaveBeenCalledWith(
+ expect.objectContaining({ finishReason: "length" }),
+ );
+ });
+
+ it("records timed_out when the sub-call deadline fires", async () => {
+ // Getters, so the rejection is created only when awaited — an eager
+ // Promise.reject is reported by Vitest as an unhandled rejection.
+ streamText.mockImplementation(() => ({
+ textStream: (async function* () {
+ await new Promise((r) => setTimeout(r, 30));
+ })(),
+ get finishReason() {
+ return Promise.reject(new Error("aborted"));
+ },
+ get totalUsage() {
+ return Promise.reject(new Error("aborted"));
+ },
+ }));
+ await runNestedGeneration({ ...args(), timeoutMs: 5 });
+ const { abortedBy } = finish.mock.calls[0][0];
+ expect(abortedBy.reason.name).toBe("TimeoutError");
+ });
+
+ it("records nothing for a busy rejection", async () => {
+ await runNestedGeneration({ ...args(), guard: { running: true, elapsedMs: 0 } });
+ expect(startAiCall).not.toHaveBeenCalled();
+ });
+
+ it("adds its elapsed time to the guard", async () => {
+ const guard = { running: false, elapsedMs: 0 };
+ streamText.mockReturnValue({
+ ...textStreamOf([]),
+ textStream: (async function* () {
+ await new Promise((r) => setTimeout(r, 20));
+ yield "done";
+ })(),
+ });
+ await runNestedGeneration({ ...args(), guard });
+ expect(guard.elapsedMs).toBeGreaterThanOrEqual(15);
+ });
+});
diff --git a/__tests__/agentReviewResumeTool.spec.ts b/__tests__/agentReviewResumeTool.spec.ts
index c2e3c29..10b0ca0 100644
--- a/__tests__/agentReviewResumeTool.spec.ts
+++ b/__tests__/agentReviewResumeTool.spec.ts
@@ -49,7 +49,7 @@ const ctx = () => ({
provider: "ollama",
modelName: "qwen3.5:9b",
writer: writer as any,
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
});
const execute = (agentTool: any, input: any) =>
@@ -80,7 +80,7 @@ describe("review_resume agent tool", () => {
it("is registered in the tool registry", async () => {
const { buildAgentTools } = await import("@/lib/agent/tools");
- const tools = buildAgentTools(ctx());
+ const tools = buildAgentTools({ ...ctx(), nestedGuard: ctx().guard });
expect(Object.keys(tools)).toContain("review_resume");
});
diff --git a/__tests__/automationMatchUsage.spec.ts b/__tests__/automationMatchUsage.spec.ts
new file mode 100644
index 0000000..e579cc8
--- /dev/null
+++ b/__tests__/automationMatchUsage.spec.ts
@@ -0,0 +1,53 @@
+const generateText = vi.fn();
+vi.mock("ai", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return { ...actual, generateText: (...a: unknown[]) => generateText(...a) };
+});
+vi.mock("@/lib/ai", () => ({
+ getModel: vi.fn(async () => ({ id: "m" })),
+ parseJobMatch: () => ({ scores: { matchScore: 80, recommendation: "good" }, body: "" }),
+ AUTOMATION_JOB_MATCH_SYSTEM_PROMPT: "SYS",
+ buildAutomationJobMatchPrompt: () => "PROMPT",
+ removeHtmlTags: (s: string) => s,
+}));
+vi.mock("@/lib/scraper/automation-run/resumeText", () => ({ convertResumeForMatch: async () => "resume" }));
+vi.mock("@/lib/scraper/automation-run/aiSettings", () => ({ getDefaultModelForProvider: () => "llama3.2" }));
+vi.mock("@/lib/telemetry", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("@/lib/telemetry")>();
+ return { ...actual, log: { error: vi.fn(), info: vi.fn(), warn: vi.fn() } };
+});
+const finish = vi.fn();
+const startAiCall = vi.fn((..._a: unknown[]) => ({ markFirstToken: vi.fn(), finish }));
+vi.mock("@/lib/aiUsage/tracker", () => ({ startAiCall: (...a: unknown[]) => startAiCall(...a) }));
+
+import { matchJobToResume } from "@/lib/scraper/automation-run/match";
+
+const job = { title: "Dev", company: "Acme", location: "Remote", description: "d" } as any;
+const run = (signal?: AbortSignal) =>
+ matchJobToResume(job, {} as any, "greenhouse" as any, { provider: "ollama", model: "qwen3.5:9b" } as any, "user-1", signal);
+
+describe("automation match usage", () => {
+ beforeEach(() => vi.clearAllMocks());
+
+ it("records an automations call with no num_ctx", async () => {
+ generateText.mockResolvedValue({ text: "x", totalUsage: { inputTokens: 2000, outputTokens: 150 }, finishReason: "stop" });
+ await run();
+ expect(startAiCall).toHaveBeenCalledWith({ userId: "user-1", feature: "automations", provider: "ollama", model: "qwen3.5:9b" });
+ expect(finish).toHaveBeenCalledWith({ usage: { inputTokens: 2000, outputTokens: 150 }, finishReason: "stop" });
+ });
+
+ it("records a provider error", async () => {
+ const err = new Error("fetch failed");
+ generateText.mockRejectedValue(err);
+ await run();
+ expect(finish).toHaveBeenCalledWith({ error: err, abortedBy: undefined });
+ });
+
+ it("passes the run's signal so a cancelled run records stopped", async () => {
+ const controller = new AbortController();
+ controller.abort();
+ generateText.mockRejectedValue(Object.assign(new Error("aborted"), { name: "AbortError" }));
+ await run(controller.signal);
+ expect(finish.mock.calls[0][0].abortedBy).toBe(controller.signal);
+ });
+});
diff --git a/__tests__/resumeImportUsage.spec.ts b/__tests__/resumeImportUsage.spec.ts
new file mode 100644
index 0000000..44857a8
--- /dev/null
+++ b/__tests__/resumeImportUsage.spec.ts
@@ -0,0 +1,83 @@
+// @vitest-environment node
+vi.mock("@/auth", () => ({ auth: vi.fn(async () => ({ user: { id: "user-1" } })) }));
+vi.mock("next/server", () => ({
+ NextResponse: { json: (data: unknown, init?: { status?: number }) => ({ status: init?.status ?? 200, json: async () => data }) },
+}));
+vi.mock("@/lib/ai/rate-limiter", () => ({ checkRateLimit: () => ({ allowed: true }) }));
+vi.mock("@/lib/ai/providers", () => ({ getModel: vi.fn(async () => ({ id: "m" })) }));
+vi.mock("@/lib/ai", () => ({ preprocessText: async (t: string) => ({ success: true, data: { normalizedText: t } }) }));
+vi.mock("@/lib/ai/import/extract-text", () => ({
+ extractText: async () => ({ success: true, data: { text: "resume text", truncated: false } }),
+}));
+vi.mock("@/lib/resumeFiles", () => ({ isResumeFilePath: () => true }));
+vi.mock("@/lib/db", () => ({
+ default: { resume: { findUnique: async () => ({ File: { filePath: "/uploads/files/resumes/a.pdf" } }) } },
+}));
+vi.mock("fs", () => ({ default: { existsSync: () => true, readFileSync: () => Buffer.from("x") } }));
+
+const markFirstToken = vi.fn();
+const finish = vi.fn();
+const startAiCall = vi.fn((..._a: unknown[]) => ({ markFirstToken, finish }));
+vi.mock("@/lib/aiUsage/tracker", () => ({ startAiCall: (...a: unknown[]) => startAiCall(...a) }));
+
+let options: any;
+vi.mock("ai", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return {
+ ...actual,
+ streamText: (o: any) => {
+ options = o;
+ return {
+ partialOutputStream: (async function* () {
+ yield { summary: "a" };
+ yield { summary: "ab" };
+ })(),
+ };
+ },
+ };
+});
+
+import { POST } from "@/app/api/ai/resume/import/route";
+
+const req = () =>
+ ({
+ json: async () => ({ resumeId: "r1", selectedModel: { provider: "ollama", model: "qwen3.5:9b" } }),
+ signal: new AbortController().signal,
+ }) as any;
+
+async function drain(res: Response) {
+ const reader = res.body!.getReader();
+ while (!(await reader.read()).done);
+}
+
+describe("resume import usage", () => {
+ beforeEach(() => vi.clearAllMocks());
+
+ it("records one resume_import call and marks the first snapshot", async () => {
+ const res = await POST(req());
+ await drain(res as Response);
+ expect(startAiCall).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: "user-1", feature: "resume_import", provider: "ollama", model: "qwen3.5:9b" }),
+ );
+ expect(markFirstToken).toHaveBeenCalled();
+ });
+
+ it("finishes with usage on completion, and error on a stream error", async () => {
+ await POST(req());
+ options.onFinish({ totalUsage: { inputTokens: 3000, outputTokens: 900 }, finishReason: "stop" });
+ expect(finish).toHaveBeenCalledWith({ usage: { inputTokens: 3000, outputTokens: 900 }, finishReason: "stop" });
+ const err = new Error("boom");
+ options.onError({ error: err });
+ expect(finish).toHaveBeenLastCalledWith({ error: err, abortedBy: expect.any(AbortSignal) });
+ });
+
+ it("aborts with a TimeoutError reason when the deadline fires", async () => {
+ vi.useFakeTimers();
+ await POST(req());
+ vi.advanceTimersByTime(240_000);
+ options.onAbort({ steps: [] });
+ const { abortedBy } = finish.mock.calls[0][0];
+ expect(abortedBy.reason.name).toBe("TimeoutError");
+ vi.useRealTimers();
+ });
+});
diff --git a/__tests__/telemetry/nestedGenerationSpan.spec.ts b/__tests__/telemetry/nestedGenerationSpan.spec.ts
index 3a8b9e6..86385af 100644
--- a/__tests__/telemetry/nestedGenerationSpan.spec.ts
+++ b/__tests__/telemetry/nestedGenerationSpan.spec.ts
@@ -48,10 +48,11 @@ describe("agent.nested span", () => {
timeoutMs: 1000,
writer,
toolCallId: "call-1",
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
label: "review_resume",
provider: "ollama",
modelName: "qwen3.5:9b",
+ userId: "user-1",
attrs: { "jobsync.input.resume_chars": 14500 },
});
@@ -75,10 +76,11 @@ describe("agent.nested span", () => {
timeoutMs: 1000,
writer,
toolCallId: "call-1",
- guard: { running: false },
+ guard: { running: false, elapsedMs: 0 },
label: "review_resume",
provider: "ollama",
modelName: "qwen3.5:9b",
+ userId: "user-1",
});
expect(result).toEqual({ status: "incomplete" });
@@ -99,10 +101,11 @@ describe("agent.nested span", () => {
timeoutMs: 1000,
writer,
toolCallId: "call-1",
- guard: { running: true },
+ guard: { running: true, elapsedMs: 0 },
label: "match_job",
provider: "ollama",
modelName: "qwen3.5:9b",
+ userId: "user-1",
});
expect(result).toEqual({ status: "busy" });
diff --git a/src/actions/automation/analyze.ts b/src/actions/automation/analyze.ts
index 51e9d2f..9f55317 100644
--- a/src/actions/automation/analyze.ts
+++ b/src/actions/automation/analyze.ts
@@ -3,6 +3,7 @@
import db from "@/lib/db";
import { requireUser } from "../shared";
import { generateText } from "ai";
+import { startAiCall } from "@/lib/aiUsage/tracker";
import {
getModel,
parseJobMatch,
@@ -109,11 +110,28 @@ export async function analyzeDiscoveredJob(jobId: string): Promise<{
"jobsync.job_id": jobId,
},
async (span) => {
- const generated = await generateText({
- model,
- system: JOB_MATCH_SYSTEM_PROMPT,
- prompt: promptText,
- temperature: 0.3,
+ // The Analyze button on a discovered job counts as Automations.
+ const tracker = startAiCall({
+ userId: user.id,
+ feature: "automations",
+ provider: ai.provider,
+ model: ai.model || "llama3.2",
+ });
+ let generated;
+ try {
+ generated = await generateText({
+ model,
+ system: JOB_MATCH_SYSTEM_PROMPT,
+ prompt: promptText,
+ temperature: 0.3,
+ });
+ } catch (error) {
+ void tracker.finish({ error });
+ throw error;
+ }
+ void tracker.finish({
+ usage: generated.totalUsage,
+ finishReason: generated.finishReason,
});
span.setAttrs(
genAiResponseAttrs({
diff --git a/src/app/api/ai/chat/route.ts b/src/app/api/ai/chat/route.ts
index 20bd51d..b29f2ed 100644
--- a/src/app/api/ai/chat/route.ts
+++ b/src/app/api/ai/chat/route.ts
@@ -32,6 +32,7 @@ import {
} from "@/lib/agent/paste";
import { truncateForModel } from "@/lib/agent/paste.server";
import { buildAgentTools } from "@/lib/agent/tools";
+import type { NestedGenerationGuard } from "@/lib/agent/nestedGeneration";
import { measureTurnPrefix, type TurnPrefixMetrics } from "@/lib/agent/turnMetrics";
import { mapAgentError } from "@/lib/agent/errors";
import { getUserSettings } from "@/actions/userSettings.actions";
@@ -46,6 +47,7 @@ import {
SURFACES,
} from "@/lib/telemetry";
import { addJobSettled, AGENT_CHAT_TERMINAL_TOOLS } from "@/models/agent.model";
+import { startAiCall, usageOfSteps, type TokenUsage } from "@/lib/aiUsage/tracker";
// The one terminal tool whose stop condition is not "was it called" — see
// addJobSettled.
@@ -207,6 +209,21 @@ export const POST = async (req: NextRequest) => {
// totalUsage, not one step's usage: stopWhen permits AGENT_CHAT_MAX_STEPS.
let turnUsage: LanguageModelUsage | undefined;
let turnFinishReason: string | undefined;
+ // streamText's onFinish never fires on abort; onAbort reports the steps that
+ // did finish. The in-flight step's tokens are lost with the connection.
+ let turnAbortUsage: TokenUsage | undefined;
+
+ // Created here, not in buildAgentTools, so the turn can subtract the nested
+ // generations' time — they are recorded as their own calls.
+ const nestedGuard: NestedGenerationGuard = { running: false, elapsedMs: 0 };
+
+ const turnUsageTracker = startAiCall({
+ userId,
+ feature: "agent_chat",
+ provider,
+ model: modelName,
+ numCtx: APP_CONSTANTS.AGENT_CHAT_NUM_CTX,
+ });
const turnSpan = startSpan("agent.chat.turn", {
...genAiRequestAttrs({
@@ -247,6 +264,7 @@ export const POST = async (req: NextRequest) => {
provider,
modelName,
writer,
+ nestedGuard,
});
prefixMetrics = measureTurnPrefix({
userId,
@@ -277,6 +295,19 @@ export const POST = async (req: NextRequest) => {
// exclusive — add_job measured 1/7 with it off, 7/7 with it on.
ollama: { think: true, options: { num_ctx: APP_CONSTANTS.AGENT_CHAT_NUM_CTX } },
},
+ // First thing the user sees: reasoning is never rendered, and Ollama
+ // sends an empty text delta beside every thinking chunk.
+ onChunk: ({ chunk }) => {
+ if (
+ (chunk.type === "text-delta" && chunk.text.length > 0) ||
+ chunk.type === "tool-input-start"
+ ) {
+ turnUsageTracker.markFirstToken();
+ }
+ },
+ onAbort: ({ steps }) => {
+ turnAbortUsage = usageOfSteps(steps);
+ },
// streamText's onFinish, not createUIMessageStream's: result is
// scoped inside execute while the span ends in the outer onFinish.
onFinish: ({ totalUsage, finishReason }) => {
@@ -340,6 +371,11 @@ export const POST = async (req: NextRequest) => {
"jobsync.message_chars": fields.messageChars,
"jobsync.prefix_changed": fields.prefixChanged,
});
+ void turnUsageTracker.finish(
+ isAborted
+ ? { usage: turnAbortUsage, abortedBy: turnSignal, excludeMs: nestedGuard.elapsedMs }
+ : { usage: turnUsage, finishReason: turnFinishReason, excludeMs: nestedGuard.elapsedMs },
+ );
// A cancelled turn never writes back. Clear deletes the conversation and
// this fires afterwards on the stream's cancel path, so saving here would
// restore exactly what the user just deleted. The write-on-receipt above
diff --git a/src/app/api/ai/resume/import/route.ts b/src/app/api/ai/resume/import/route.ts
index 7c5eb08..2fc96c9 100644
--- a/src/app/api/ai/resume/import/route.ts
+++ b/src/app/api/ai/resume/import/route.ts
@@ -28,6 +28,7 @@ import {
startSpan,
SURFACES,
} from "@/lib/telemetry";
+import { startAiCall } from "@/lib/aiUsage/tracker";
export const POST = async (req: NextRequest) => {
const session = await auth();
@@ -122,8 +123,9 @@ export const POST = async (req: NextRequest) => {
);
const controller = new AbortController();
+ // A named reason is what tells "timed out" from "client went away".
const timer = setTimeout(
- () => controller.abort(),
+ () => controller.abort(new DOMException("Resume import timed out", "TimeoutError")),
APP_CONSTANTS.AI_RESUME_IMPORT_TIMEOUT_MS,
);
@@ -156,6 +158,14 @@ export const POST = async (req: NextRequest) => {
"jobsync.resume_id": resumeId,
});
+ const importUsage = startAiCall({
+ userId,
+ feature: "resume_import",
+ provider: selectedModel.provider,
+ model: selectedModel.model,
+ numCtx: APP_CONSTANTS.AI_OLLAMA_NUM_CTX,
+ });
+
const result = runInSpan(importSpan, () =>
streamText({
model,
@@ -174,17 +184,23 @@ export const POST = async (req: NextRequest) => {
},
onFinish: ({ totalUsage, finishReason }) => {
clearTimeout(timer);
+ void importUsage.finish({ usage: totalUsage, finishReason });
importSpan.end(
genAiResponseAttrs({ usage: totalUsage, finishReason }),
);
},
onError: ({ error }) => {
clearTimeout(timer);
+ void importUsage.finish({ error, abortedBy: controller.signal });
log.error("Resume import stream error", { error: String(error) });
streamErrorMessage = error instanceof Error ? error.message : undefined;
importSpan.setError(error);
importSpan.end();
},
+ onAbort: () => {
+ clearTimeout(timer);
+ void importUsage.finish({ abortedBy: controller.signal });
+ },
}),
);
@@ -198,6 +214,7 @@ export const POST = async (req: NextRequest) => {
async start(controller) {
try {
for await (const partial of result.partialOutputStream) {
+ importUsage.markFirstToken();
controller.enqueue(encoder.encode(JSON.stringify(partial) + "\n"));
}
} catch (err) {
diff --git a/src/lib/agent/nestedGeneration.ts b/src/lib/agent/nestedGeneration.ts
index 933bf28..870a60b 100644
--- a/src/lib/agent/nestedGeneration.ts
+++ b/src/lib/agent/nestedGeneration.ts
@@ -11,6 +11,8 @@ import {
SURFACE_BY_NESTED_LABEL,
SURFACES,
} from "@/lib/telemetry";
+import { startAiCall } from "@/lib/aiUsage/tracker";
+import { FEATURE_BY_NESTED_LABEL } from "@/models/aiUsage.model";
export type NestedGenerationResult =
| { status: "ok"; text: string }
@@ -18,9 +20,10 @@ export type NestedGenerationResult =
| { status: "busy" }
| { status: "failed" };
-// One per request, created in buildAgentTools. Process-wide state would block
-// one user behind another's generation.
-export type NestedGenerationGuard = { running: boolean };
+// One per request, created by the chat route. Process-wide state would block
+// one user behind another's generation. elapsedMs is what the turn subtracts
+// so its own duration excludes the nested work recorded separately.
+export type NestedGenerationGuard = { running: boolean; elapsedMs: number };
type NestedGenerationArgs = {
model: LanguageModel;
@@ -36,6 +39,7 @@ type NestedGenerationArgs = {
label: string;
provider: string;
modelName: string;
+ userId: string;
// Call-site facts the invariant half cannot know: input sizes and whether
// the caller clipped what it sent.
attrs?: Record<string, unknown>;
@@ -61,6 +65,7 @@ export async function runNestedGeneration({
label,
provider,
modelName,
+ userId,
attrs,
}: NestedGenerationArgs): Promise<NestedGenerationResult> {
// Started before the guard check so a rejected call is visible too — "busy"
@@ -92,6 +97,16 @@ export async function runNestedGeneration({
// this too.
const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
if (abortSignal) signals.push(abortSignal);
+ const signal = AbortSignal.any(signals);
+
+ const startedAt = Date.now();
+ const tracker = startAiCall({
+ userId,
+ feature: FEATURE_BY_NESTED_LABEL[label] ?? "agent_chat",
+ provider,
+ model: modelName,
+ numCtx,
+ });
let text = "";
try {
@@ -100,13 +115,14 @@ export async function runNestedGeneration({
system,
prompt,
temperature,
- abortSignal: AbortSignal.any(signals),
+ abortSignal: signal,
// Deliberately no think:true. These sub-calls have no tools, so
// reasoning buys nothing and costs 30s. The chat loop enables it.
providerOptions: { ollama: { options: { num_ctx: numCtx } } },
});
for await (const delta of sub.textStream) {
+ tracker.markFirstToken();
text += delta;
writer.write({
type: AGENT_NESTED_STREAM_PART_TYPE,
@@ -121,6 +137,10 @@ export async function runNestedGeneration({
// promise, and a body that stops without Ollama's done chunk resolves
// to "other".
const finishReason = await sub.finishReason;
+ // Defensively: an aborted stream rejects this, the same trap the
+ // finishReason guard exists for.
+ const tokens = await Promise.resolve(sub.totalUsage).catch(() => undefined);
+ void tracker.finish({ usage: tokens, finishReason });
if (finishReason !== "stop") {
span.setError(new Error(`finishReason=${finishReason}`));
span.end({
@@ -130,20 +150,19 @@ export async function runNestedGeneration({
return { status: "incomplete" };
}
- // Only on the ok path, and defensively: an aborted stream rejects this,
- // the same trap the finishReason guard above exists for.
- const usage = await Promise.resolve(sub.totalUsage).catch(() => undefined);
span.end({
"jobsync.nested.status": "ok",
- ...genAiResponseAttrs({ usage, finishReason, text }),
+ ...genAiResponseAttrs({ usage: tokens, finishReason, text }),
});
return { status: "ok", text };
} catch (error) {
+ void tracker.finish({ error, abortedBy: signal });
log.error(`[agent-chat] ${label} generation failed`, { error: String(error) });
span.setError(error);
span.end({ "jobsync.nested.status": "failed" });
return { status: "failed" };
} finally {
+ guard.elapsedMs += Date.now() - startedAt;
guard.running = false;
}
});
diff --git a/src/lib/agent/tools/generateCoverLetter.ts b/src/lib/agent/tools/generateCoverLetter.ts
index 3d812b7..4355140 100644
--- a/src/lib/agent/tools/generateCoverLetter.ts
+++ b/src/lib/agent/tools/generateCoverLetter.ts
@@ -97,6 +97,7 @@ export function buildGenerateCoverLetterTool(ctx: CoverLetterContext) {
const generation = await runNestedGeneration({
model: ctx.model,
+ userId: ctx.userId,
system: COVER_LETTER_SYSTEM_PROMPT,
prompt: buildCoverLetterPrompt(
resumePre.data.normalizedText,
diff --git a/src/lib/agent/tools/index.ts b/src/lib/agent/tools/index.ts
index 45a9a26..87fc7cc 100644
--- a/src/lib/agent/tools/index.ts
+++ b/src/lib/agent/tools/index.ts
@@ -19,11 +19,8 @@ export function buildAgentTools(ctx: {
provider: string;
modelName: string;
writer: UIMessageStreamWriter;
+ nestedGuard: NestedGenerationGuard;
}): ToolSet {
- // One per request, so no two nested tools run at once but two users never
- // block each other.
- const nestedGuard: NestedGenerationGuard = { running: false };
-
return {
add_job: buildAddJobTool(ctx.userId, ctx.pastedText),
get_resume: buildGetResumeTool(ctx.userId, ctx.pageContext?.resumeId),
@@ -34,7 +31,7 @@ export function buildAgentTools(ctx: {
provider: ctx.provider,
modelName: ctx.modelName,
writer: ctx.writer,
- guard: nestedGuard,
+ guard: ctx.nestedGuard,
}),
match_job: buildMatchJobTool({
userId: ctx.userId,
@@ -43,7 +40,7 @@ export function buildAgentTools(ctx: {
provider: ctx.provider,
modelName: ctx.modelName,
writer: ctx.writer,
- guard: nestedGuard,
+ guard: ctx.nestedGuard,
}),
generate_cover_letter: buildGenerateCoverLetterTool({
userId: ctx.userId,
@@ -52,7 +49,7 @@ export function buildAgentTools(ctx: {
provider: ctx.provider,
modelName: ctx.modelName,
writer: ctx.writer,
- guard: nestedGuard,
+ guard: ctx.nestedGuard,
}),
};
}
diff --git a/src/lib/agent/tools/matchJob.ts b/src/lib/agent/tools/matchJob.ts
index ee4699c..5d877c1 100644
--- a/src/lib/agent/tools/matchJob.ts
+++ b/src/lib/agent/tools/matchJob.ts
@@ -87,6 +87,7 @@ export function buildMatchJobTool(ctx: MatchJobContext) {
const generation = await runNestedGeneration({
model: ctx.model,
+ userId: ctx.userId,
system: JOB_MATCH_SYSTEM_PROMPT,
prompt: buildJobMatchPrompt(
resumePre.data.normalizedText,
diff --git a/src/lib/agent/tools/reviewResume.ts b/src/lib/agent/tools/reviewResume.ts
index 83c841b..4d85994 100644
--- a/src/lib/agent/tools/reviewResume.ts
+++ b/src/lib/agent/tools/reviewResume.ts
@@ -66,6 +66,7 @@ export function buildReviewResumeTool(ctx: ReviewResumeContext) {
const generation = await runNestedGeneration({
model: ctx.model,
+ userId: ctx.userId,
system: RESUME_REVIEW_SYSTEM_PROMPT,
prompt: buildResumeReviewPrompt(pre.data.normalizedText),
temperature: TEMPERATURES.FEEDBACK,
diff --git a/src/lib/scraper/automation-run/match.ts b/src/lib/scraper/automation-run/match.ts
index e14ad2b..8a3c84e 100644
--- a/src/lib/scraper/automation-run/match.ts
+++ b/src/lib/scraper/automation-run/match.ts
@@ -21,6 +21,7 @@ import type { JobDetails } from "../types";
import type { ResumeWithSections } from "./types";
import { getDefaultModelForProvider } from "./aiSettings";
import { convertResumeForMatch } from "./resumeText";
+import { startAiCall } from "@/lib/aiUsage/tracker";
export interface MatchResult {
success: boolean;
@@ -75,12 +76,29 @@ ${removeHtmlTags(job.description)}
"jobsync.user_id": userId,
},
async (span) => {
- const generated = await generateText({
- model,
- system: AUTOMATION_JOB_MATCH_SYSTEM_PROMPT,
- prompt: promptText,
- temperature: 0.3,
- abortSignal: signal,
+ // No numCtx: this call passes none, so Ollama uses its daemon default.
+ const tracker = startAiCall({
+ userId,
+ feature: "automations",
+ provider,
+ model: modelName,
+ });
+ let generated;
+ try {
+ generated = await generateText({
+ model,
+ system: AUTOMATION_JOB_MATCH_SYSTEM_PROMPT,
+ prompt: promptText,
+ temperature: 0.3,
+ abortSignal: signal,
+ });
+ } catch (error) {
+ void tracker.finish({ error, abortedBy: signal });
+ throw error;
+ }
+ void tracker.finish({
+ usage: generated.totalUsage,
+ finishReason: generated.finishReason,
});
span.setAttrs(
genAiResponseAttrs({