Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

feat(ai-usage): add the AiCall table, a never-throwing per-call usage tracker with ok/error/truncated/interrupted/timed-out/stopped classification, and 90-day hourly pruning

Khuram Niazae7860e321d2
diff --git a/__tests__/aiUsageTracker.spec.ts b/__tests__/aiUsageTracker.spec.ts
new file mode 100644
index 0000000..ce2fbfc
--- /dev/null
+++ b/__tests__/aiUsageTracker.spec.ts
@@ -0,0 +1,134 @@
+import { startAiCall, statusOf, usageOfSteps } from "@/lib/aiUsage/tracker";
+import prisma from "@/lib/db";
+
+vi.mock("@/lib/db", () => ({ default: { aiCall: { create: vi.fn() } } }));
+vi.mock("@/lib/telemetry", () => ({ log: { warn: vi.fn() } }));
+
+const db = prisma as any;
+
+const timedOut = () => {
+ const c = new AbortController();
+ c.abort(new DOMException("deadline", "TimeoutError"));
+ return c.signal;
+};
+const stoppedByUser = () => {
+ const c = new AbortController();
+ c.abort();
+ return c.signal;
+};
+
+describe("statusOf", () => {
+ it.each([
+ ["stop", "ok"],
+ ["tool-calls", "ok"],
+ ["length", "truncated"],
+ ["error", "error"],
+ ["content-filter", "error"],
+ ["other", "interrupted"],
+ [undefined, "interrupted"],
+ ])("maps finishReason %s to %s", (finishReason, expected) => {
+ expect(statusOf({ finishReason })).toBe(expected);
+ });
+
+ it("treats a thrown error as error", () => {
+ expect(statusOf({ error: new Error("ECONNREFUSED") })).toBe("error");
+ });
+
+ it("tells a timeout from a user stop by the abort reason", () => {
+ expect(statusOf({ abortedBy: timedOut() })).toBe("timed_out");
+ expect(statusOf({ abortedBy: stoppedByUser() })).toBe("stopped");
+ });
+
+ it("reads a timeout through AbortSignal.any", () => {
+ const combined = AbortSignal.any([new AbortController().signal, timedOut()]);
+ expect(statusOf({ error: new Error("aborted"), abortedBy: combined })).toBe("timed_out");
+ });
+
+ it("ignores a signal that never fired", () => {
+ const idle = new AbortController().signal;
+ expect(statusOf({ error: new Error("boom"), abortedBy: idle })).toBe("error");
+ });
+});
+
+describe("usageOfSteps", () => {
+ it("sums finished steps", () => {
+ expect(
+ usageOfSteps([
+ { usage: { inputTokens: 100, outputTokens: 10 } },
+ { usage: { inputTokens: 50, outputTokens: undefined } },
+ ]),
+ ).toEqual({ inputTokens: 150, outputTokens: 10 });
+ });
+
+ it("is undefined when no step finished", () => {
+ expect(usageOfSteps([])).toBeUndefined();
+ });
+});
+
+describe("startAiCall", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date("2026-09-28T10:00:00Z"));
+ db.aiCall.create.mockResolvedValue({});
+ });
+ afterEach(() => vi.useRealTimers());
+
+ const start = (provider = "ollama") =>
+ startAiCall({ userId: "u1", feature: "agent_chat", provider, model: "qwen3.5:9b", numCtx: 16384 });
+
+ it("writes one row with duration, first token and usage", async () => {
+ const t = start();
+ vi.advanceTimersByTime(1500);
+ t.markFirstToken();
+ vi.advanceTimersByTime(1500);
+ t.markFirstToken(); // only the first one counts
+ vi.advanceTimersByTime(2000);
+ await t.finish({ usage: { inputTokens: 900, outputTokens: 80 }, finishReason: "stop" });
+
+ expect(db.aiCall.create).toHaveBeenCalledWith({
+ data: {
+ userId: "u1",
+ feature: "agent_chat",
+ provider: "ollama",
+ model: "qwen3.5:9b",
+ status: "ok",
+ inputTokens: 900,
+ outputTokens: 80,
+ durationMs: 5000,
+ firstTokenMs: 1500,
+ numCtx: 16384,
+ startedAt: new Date("2026-09-28T10:00:00Z"),
+ },
+ });
+ });
+
+ it("subtracts excluded time and never goes negative", async () => {
+ const t = start();
+ vi.advanceTimersByTime(3000);
+ await t.finish({ finishReason: "stop", excludeMs: 5000 });
+ expect(db.aiCall.create.mock.calls[0][0].data.durationMs).toBe(0);
+ });
+
+ it("stores numCtx for Ollama only", async () => {
+ await start("deepseek").finish({ finishReason: "stop" });
+ expect(db.aiCall.create.mock.calls[0][0].data.numCtx).toBeNull();
+ });
+
+ it("records nulls when the provider reported no usage", async () => {
+ await start().finish({ error: new Error("fetch failed") });
+ const { data } = db.aiCall.create.mock.calls[0][0];
+ expect(data).toMatchObject({ status: "error", inputTokens: null, outputTokens: null, firstTokenMs: null });
+ });
+
+ it("writes only once when finish is called twice", async () => {
+ const t = start();
+ await t.finish({ finishReason: "stop" });
+ await t.finish({ error: new Error("late") });
+ expect(db.aiCall.create).toHaveBeenCalledTimes(1);
+ });
+
+ it("never throws when the insert fails", async () => {
+ db.aiCall.create.mockRejectedValue(new Error("database is locked"));
+ await expect(start().finish({ finishReason: "stop" })).resolves.toBeUndefined();
+ });
+});
diff --git a/__tests__/pruneAiCalls.spec.ts b/__tests__/pruneAiCalls.spec.ts
new file mode 100644
index 0000000..0ece124
--- /dev/null
+++ b/__tests__/pruneAiCalls.spec.ts
@@ -0,0 +1,27 @@
+import { pruneAiCalls, aiUsageCutoff } from "@/lib/aiUsage/prune";
+import prisma from "@/lib/db";
+
+vi.mock("@/lib/db", () => ({
+ default: { aiCall: { deleteMany: vi.fn() } },
+}));
+vi.mock("@/lib/telemetry", () => ({ log: { info: vi.fn(), error: vi.fn() } }));
+
+const db = prisma as any;
+const now = new Date("2026-09-28T12:00:00Z");
+
+it("cuts off exactly 90 days back", () => {
+ expect(aiUsageCutoff(now).toISOString()).toBe("2026-06-30T12:00:00.000Z");
+});
+
+it("deletes rows older than the cutoff", async () => {
+ db.aiCall.deleteMany.mockResolvedValue({ count: 4 });
+ await expect(pruneAiCalls(now)).resolves.toBe(4);
+ expect(db.aiCall.deleteMany).toHaveBeenCalledWith({
+ where: { startedAt: { lt: aiUsageCutoff(now) } },
+ });
+});
+
+it("never throws", async () => {
+ db.aiCall.deleteMany.mockRejectedValue(new Error("locked"));
+ await expect(pruneAiCalls(now)).resolves.toBe(0);
+});
diff --git a/prisma/migrations/20260929012120_ai_usage/migration.sql b/prisma/migrations/20260929012120_ai_usage/migration.sql
new file mode 100644
index 0000000..892c6aa
--- /dev/null
+++ b/prisma/migrations/20260929012120_ai_usage/migration.sql
@@ -0,0 +1,19 @@
+-- CreateTable
+CREATE TABLE "AiCall" (
+ "id" TEXT NOT NULL PRIMARY KEY,
+ "userId" TEXT NOT NULL,
+ "feature" TEXT NOT NULL,
+ "provider" TEXT NOT NULL,
+ "model" TEXT NOT NULL,
+ "status" TEXT NOT NULL,
+ "inputTokens" INTEGER,
+ "outputTokens" INTEGER,
+ "durationMs" INTEGER NOT NULL,
+ "firstTokenMs" INTEGER,
+ "numCtx" INTEGER,
+ "startedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ CONSTRAINT "AiCall_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE INDEX "AiCall_userId_startedAt_idx" ON "AiCall"("userId", "startedAt");
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index 59f0f6a..00aa7f5 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -38,6 +38,7 @@ model User {
Question Question[]
McpAccessToken McpAccessToken[]
Notification Notification[]
+ aiCalls AiCall[]
ChatConversation ChatConversation?
defaultResumeId String?
defaultResume Resume? @relation("UserDefaultResume", fields: [defaultResumeId], references: [id], onDelete: SetNull)
@@ -631,6 +632,24 @@ model Notification {
@@index([automationId, kind, boardToken])
}
+model AiCall {
+ id String @id @default(uuid())
+ userId String
+ user User @relation(fields: [userId], references: [id])
+ feature String // AiUsageFeature in aiUsage.model.ts
+ provider String
+ model String
+ status String // AiCallStatus in aiUsage.model.ts
+ inputTokens Int?
+ outputTokens Int?
+ durationMs Int
+ firstTokenMs Int?
+ numCtx Int? // Ollama only, and only where the app passes num_ctx
+ startedAt DateTime @default(now())
+
+ @@index([userId, startedAt])
+}
+
model Note {
id String @id @default(uuid())
jobId String
diff --git a/src/lib/aiUsage/prune.ts b/src/lib/aiUsage/prune.ts
new file mode 100644
index 0000000..e57a30d
--- /dev/null
+++ b/src/lib/aiUsage/prune.ts
@@ -0,0 +1,22 @@
+import db from "@/lib/db";
+import { log } from "@/lib/telemetry";
+import { AI_USAGE_CONSTANTS } from "@/lib/constants";
+
+export function aiUsageCutoff(now: Date = new Date()): Date {
+ return new Date(
+ now.getTime() - AI_USAGE_CONSTANTS.RETENTION_DAYS * 24 * 60 * 60 * 1000,
+ );
+}
+
+export async function pruneAiCalls(now: Date = new Date()): Promise<number> {
+ try {
+ const { count } = await db.aiCall.deleteMany({
+ where: { startedAt: { lt: aiUsageCutoff(now) } },
+ });
+ if (count > 0) log.info("[AiUsage] Pruned", { "ai_usage.pruned": count });
+ return count;
+ } catch (error) {
+ log.error("[AiUsage] Prune failed", { error: String(error) });
+ return 0;
+ }
+}
diff --git a/src/lib/aiUsage/tracker.ts b/src/lib/aiUsage/tracker.ts
new file mode 100644
index 0000000..0acf20d
--- /dev/null
+++ b/src/lib/aiUsage/tracker.ts
@@ -0,0 +1,104 @@
+import "server-only";
+
+import db from "@/lib/db";
+import { log } from "@/lib/telemetry";
+import type { AiCallStatus, AiUsageFeature } from "@/models/aiUsage.model";
+
+export type TokenUsage = { inputTokens?: number; outputTokens?: number };
+
+export type AiCallFinish = {
+ usage?: TokenUsage;
+ finishReason?: string;
+ error?: unknown;
+ // Pass the call's signal on the abort/error path; it only counts if fired.
+ abortedBy?: AbortSignal;
+ // Time spent inside nested generations, which carry their own rows.
+ excludeMs?: number;
+};
+
+export type AiCallTracker = {
+ markFirstToken(): void;
+ finish(input: AiCallFinish): Promise<void>;
+};
+
+export function statusOf(input: AiCallFinish): AiCallStatus {
+ if (input.abortedBy?.aborted) {
+ // AbortSignal.timeout and AbortSignal.any both surface the firing
+ // signal's reason, so a deadline reads the same at every call site.
+ const reason = input.abortedBy.reason as { name?: string } | undefined;
+ return reason?.name === "TimeoutError" ? "timed_out" : "stopped";
+ }
+ if (input.error !== undefined) return "error";
+ switch (input.finishReason) {
+ case "stop":
+ case "tool-calls":
+ return "ok";
+ case "length":
+ return "truncated";
+ case "error":
+ case "content-filter":
+ return "error";
+ default:
+ // "other" / missing: Ollama's body ended without its done chunk.
+ return "interrupted";
+ }
+}
+
+export function usageOfSteps(
+ steps: readonly { usage: TokenUsage }[],
+): TokenUsage | undefined {
+ if (steps.length === 0) return undefined;
+ return steps.reduce<Required<TokenUsage>>(
+ (sum, step) => ({
+ inputTokens: sum.inputTokens + (step.usage.inputTokens ?? 0),
+ outputTokens: sum.outputTokens + (step.usage.outputTokens ?? 0),
+ }),
+ { inputTokens: 0, outputTokens: 0 },
+ );
+}
+
+export function startAiCall(start: {
+ userId: string;
+ feature: AiUsageFeature;
+ provider: string;
+ model: string;
+ numCtx?: number;
+}): AiCallTracker {
+ const startedAt = new Date();
+ let firstTokenMs: number | undefined;
+ let finished = false;
+
+ return {
+ markFirstToken() {
+ if (firstTokenMs === undefined) {
+ firstTokenMs = Date.now() - startedAt.getTime();
+ }
+ },
+ async finish(input) {
+ // Several callbacks can report one call (onError then onFinish).
+ if (finished) return;
+ finished = true;
+ try {
+ const elapsed = Date.now() - startedAt.getTime();
+ await db.aiCall.create({
+ data: {
+ userId: start.userId,
+ feature: start.feature,
+ provider: start.provider,
+ model: start.model,
+ status: statusOf(input),
+ inputTokens: input.usage?.inputTokens ?? null,
+ outputTokens: input.usage?.outputTokens ?? null,
+ durationMs: Math.max(0, elapsed - (input.excludeMs ?? 0)),
+ firstTokenMs: firstTokenMs ?? null,
+ numCtx: start.provider === "ollama" ? (start.numCtx ?? null) : null,
+ startedAt,
+ },
+ });
+ } catch (error) {
+ // Usage is bookkeeping; it must never fail the call it describes.
+ log.warn("[AiUsage] Failed to record call", { error: String(error) });
+ }
+ },
+ };
+}
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index abf17b5..2a34950 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -428,3 +428,11 @@ export const NOTIFICATION_CONSTANTS = {
POPOVER_LIMIT: 20,
PAGE_SIZE: 50,
} as const;
+
+export const AI_USAGE_CONSTANTS = {
+ RETENTION_DAYS: 90,
+ RECENT_LIMIT: 50,
+ // Ollama drops the oldest context silently on overflow, so "near" is the
+ // only warning available.
+ CONTEXT_WARN_RATIO: 0.9,
+} as const;
diff --git a/src/lib/scheduler/index.ts b/src/lib/scheduler/index.ts
index c1a9f89..b39e797 100644
--- a/src/lib/scheduler/index.ts
+++ b/src/lib/scheduler/index.ts
@@ -5,6 +5,7 @@ import { runAutomation, AutomationAlreadyRunningError } from "@/lib/scraper";
import { ATS_BOARDS, type JobBoard } from "@/models/automation.model";
import { log } from "@/lib/telemetry";
import { pruneNotifications } from "@/lib/notifications/prune";
+import { pruneAiCalls } from "@/lib/aiUsage/prune";
let scheduledTask: ScheduledTask | null = null;
@@ -16,6 +17,7 @@ async function runDueAutomations() {
try {
await pruneNotifications(now);
+ await pruneAiCalls(now);
const dueAutomations = await db.automation.findMany({
where: {
status: "active",
diff --git a/src/models/aiUsage.model.ts b/src/models/aiUsage.model.ts
new file mode 100644
index 0000000..8fa84f0
--- /dev/null
+++ b/src/models/aiUsage.model.ts
@@ -0,0 +1,103 @@
+// Own vocabulary, not telemetry's SURFACES: the two disagree on the Analyze
+// button (Automations here, job.match there) and SURFACES must not grow.
+export const AI_USAGE_FEATURES = {
+ agent_chat: "Agent chat",
+ job_match: "Job match",
+ automations: "Automations",
+ resume_review: "Resume review",
+ cover_letter: "Cover letter",
+ resume_import: "Resume import",
+} as const;
+
+export type AiUsageFeature = keyof typeof AI_USAGE_FEATURES;
+
+export const FEATURE_BY_NESTED_LABEL: Record<string, AiUsageFeature> = {
+ review_resume: "resume_review",
+ match_job: "job_match",
+ generate_cover_letter: "cover_letter",
+};
+
+export const AI_CALL_STATUSES = [
+ "ok",
+ "error",
+ "truncated",
+ "interrupted",
+ "timed_out",
+ "stopped",
+] as const;
+
+export type AiCallStatus = (typeof AI_CALL_STATUSES)[number];
+
+// Stopped is a call but never a failure — the user chose it.
+export const AI_FAILURE_STATUSES = [
+ "error",
+ "truncated",
+ "interrupted",
+ "timed_out",
+] as const;
+
+export type AiFailureStatus = (typeof AI_FAILURE_STATUSES)[number];
+
+export const AI_PROVIDER_LABELS: Record<string, string> = {
+ ollama: "Ollama",
+ openai: "OpenAI",
+ deepseek: "DeepSeek",
+ gemini: "Gemini",
+ openrouter: "OpenRouter",
+};
+
+export type AiUsageRange = 7 | 30;
+
+export type AiCallRow = {
+ id: string;
+ startedAt: Date;
+ feature: AiUsageFeature;
+ provider: string;
+ model: string;
+ status: AiCallStatus;
+ inputTokens: number | null;
+ outputTokens: number | null;
+ durationMs: number;
+ firstTokenMs: number | null;
+};
+
+export type AiModelUsage = {
+ model: string;
+ tokens: number;
+ calls: number;
+ medianMs: number | null;
+ p95Ms: number | null;
+ medianFirstTokenMs: number | null;
+ nearContextLimit: number;
+};
+
+export type AiUsageSummary = {
+ rangeDays: AiUsageRange;
+ totals: {
+ inputTokens: number;
+ outputTokens: number;
+ calls: number;
+ previousCalls: number;
+ failures: number;
+ stopped: number;
+ medianMs: number | null;
+ p95Ms: number | null;
+ medianFirstTokenMs: number | null;
+ };
+ failuresByStatus: Record<AiFailureStatus, number>;
+ // Oldest first; one entry per calendar day in the viewer's timezone.
+ days: { date: string; tokensByProvider: Record<string, number> }[];
+ providers: {
+ provider: string;
+ tokens: number;
+ calls: number;
+ models: AiModelUsage[];
+ }[];
+ features: {
+ feature: AiUsageFeature;
+ tokens: number;
+ calls: number;
+ medianMs: number | null;
+ }[];
+ recent: AiCallRow[];
+};