Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
feat(ai-usage): add the AiCall table, a never-throwing per-call usage tracker with ok/error/truncated/interrupted/timed-out/stopped classification, and 90-day hourly pruning
Khuram Niazae7860e321d2
diff --git a/__tests__/aiUsageTracker.spec.ts b/__tests__/aiUsageTracker.spec.tsnew file mode 100644index 0000000..ce2fbfc--- /dev/null+++ b/__tests__/aiUsageTracker.spec.ts@@ -0,0 +1,134 @@+import { startAiCall, statusOf, usageOfSteps } from "@/lib/aiUsage/tracker";+import prisma from "@/lib/db";++vi.mock("@/lib/db", () => ({ default: { aiCall: { create: vi.fn() } } }));+vi.mock("@/lib/telemetry", () => ({ log: { warn: vi.fn() } }));++const db = prisma as any;++const timedOut = () => {+ const c = new AbortController();+ c.abort(new DOMException("deadline", "TimeoutError"));+ return c.signal;+};+const stoppedByUser = () => {+ const c = new AbortController();+ c.abort();+ return c.signal;+};++describe("statusOf", () => {+ it.each([+ ["stop", "ok"],+ ["tool-calls", "ok"],+ ["length", "truncated"],+ ["error", "error"],+ ["content-filter", "error"],+ ["other", "interrupted"],+ [undefined, "interrupted"],+ ])("maps finishReason %s to %s", (finishReason, expected) => {+ expect(statusOf({ finishReason })).toBe(expected);+ });++ it("treats a thrown error as error", () => {+ expect(statusOf({ error: new Error("ECONNREFUSED") })).toBe("error");+ });++ it("tells a timeout from a user stop by the abort reason", () => {+ expect(statusOf({ abortedBy: timedOut() })).toBe("timed_out");+ expect(statusOf({ abortedBy: stoppedByUser() })).toBe("stopped");+ });++ it("reads a timeout through AbortSignal.any", () => {+ const combined = AbortSignal.any([new AbortController().signal, timedOut()]);+ expect(statusOf({ error: new Error("aborted"), abortedBy: combined })).toBe("timed_out");+ });++ it("ignores a signal that never fired", () => {+ const idle = new AbortController().signal;+ expect(statusOf({ error: new Error("boom"), abortedBy: idle })).toBe("error");+ });+});++describe("usageOfSteps", () => {+ it("sums finished steps", () => {+ expect(+ usageOfSteps([+ { usage: { inputTokens: 100, outputTokens: 10 } },+ { usage: { inputTokens: 50, outputTokens: undefined } },+ ]),+ ).toEqual({ inputTokens: 150, outputTokens: 10 });+ });++ it("is undefined when no step finished", () => {+ expect(usageOfSteps([])).toBeUndefined();+ });+});++describe("startAiCall", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(new Date("2026-09-28T10:00:00Z"));+ db.aiCall.create.mockResolvedValue({});+ });+ afterEach(() => vi.useRealTimers());++ const start = (provider = "ollama") =>+ startAiCall({ userId: "u1", feature: "agent_chat", provider, model: "qwen3.5:9b", numCtx: 16384 });++ it("writes one row with duration, first token and usage", async () => {+ const t = start();+ vi.advanceTimersByTime(1500);+ t.markFirstToken();+ vi.advanceTimersByTime(1500);+ t.markFirstToken(); // only the first one counts+ vi.advanceTimersByTime(2000);+ await t.finish({ usage: { inputTokens: 900, outputTokens: 80 }, finishReason: "stop" });++ expect(db.aiCall.create).toHaveBeenCalledWith({+ data: {+ userId: "u1",+ feature: "agent_chat",+ provider: "ollama",+ model: "qwen3.5:9b",+ status: "ok",+ inputTokens: 900,+ outputTokens: 80,+ durationMs: 5000,+ firstTokenMs: 1500,+ numCtx: 16384,+ startedAt: new Date("2026-09-28T10:00:00Z"),+ },+ });+ });++ it("subtracts excluded time and never goes negative", async () => {+ const t = start();+ vi.advanceTimersByTime(3000);+ await t.finish({ finishReason: "stop", excludeMs: 5000 });+ expect(db.aiCall.create.mock.calls[0][0].data.durationMs).toBe(0);+ });++ it("stores numCtx for Ollama only", async () => {+ await start("deepseek").finish({ finishReason: "stop" });+ expect(db.aiCall.create.mock.calls[0][0].data.numCtx).toBeNull();+ });++ it("records nulls when the provider reported no usage", async () => {+ await start().finish({ error: new Error("fetch failed") });+ const { data } = db.aiCall.create.mock.calls[0][0];+ expect(data).toMatchObject({ status: "error", inputTokens: null, outputTokens: null, firstTokenMs: null });+ });++ it("writes only once when finish is called twice", async () => {+ const t = start();+ await t.finish({ finishReason: "stop" });+ await t.finish({ error: new Error("late") });+ expect(db.aiCall.create).toHaveBeenCalledTimes(1);+ });++ it("never throws when the insert fails", async () => {+ db.aiCall.create.mockRejectedValue(new Error("database is locked"));+ await expect(start().finish({ finishReason: "stop" })).resolves.toBeUndefined();+ });+});diff --git a/__tests__/pruneAiCalls.spec.ts b/__tests__/pruneAiCalls.spec.tsnew file mode 100644index 0000000..0ece124--- /dev/null+++ b/__tests__/pruneAiCalls.spec.ts@@ -0,0 +1,27 @@+import { pruneAiCalls, aiUsageCutoff } from "@/lib/aiUsage/prune";+import prisma from "@/lib/db";++vi.mock("@/lib/db", () => ({+ default: { aiCall: { deleteMany: vi.fn() } },+}));+vi.mock("@/lib/telemetry", () => ({ log: { info: vi.fn(), error: vi.fn() } }));++const db = prisma as any;+const now = new Date("2026-09-28T12:00:00Z");++it("cuts off exactly 90 days back", () => {+ expect(aiUsageCutoff(now).toISOString()).toBe("2026-06-30T12:00:00.000Z");+});++it("deletes rows older than the cutoff", async () => {+ db.aiCall.deleteMany.mockResolvedValue({ count: 4 });+ await expect(pruneAiCalls(now)).resolves.toBe(4);+ expect(db.aiCall.deleteMany).toHaveBeenCalledWith({+ where: { startedAt: { lt: aiUsageCutoff(now) } },+ });+});++it("never throws", async () => {+ db.aiCall.deleteMany.mockRejectedValue(new Error("locked"));+ await expect(pruneAiCalls(now)).resolves.toBe(0);+});diff --git a/prisma/migrations/20260929012120_ai_usage/migration.sql b/prisma/migrations/20260929012120_ai_usage/migration.sqlnew file mode 100644index 0000000..892c6aa--- /dev/null+++ b/prisma/migrations/20260929012120_ai_usage/migration.sql@@ -0,0 +1,19 @@+-- CreateTable+CREATE TABLE "AiCall" (+ "id" TEXT NOT NULL PRIMARY KEY,+ "userId" TEXT NOT NULL,+ "feature" TEXT NOT NULL,+ "provider" TEXT NOT NULL,+ "model" TEXT NOT NULL,+ "status" TEXT NOT NULL,+ "inputTokens" INTEGER,+ "outputTokens" INTEGER,+ "durationMs" INTEGER NOT NULL,+ "firstTokenMs" INTEGER,+ "numCtx" INTEGER,+ "startedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ CONSTRAINT "AiCall_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE+);++-- CreateIndex+CREATE INDEX "AiCall_userId_startedAt_idx" ON "AiCall"("userId", "startedAt");diff --git a/prisma/schema.prisma b/prisma/schema.prismaindex 59f0f6a..00aa7f5 100644--- a/prisma/schema.prisma+++ b/prisma/schema.prisma@@ -38,6 +38,7 @@ model User {Question Question[]McpAccessToken McpAccessToken[]Notification Notification[]+ aiCalls AiCall[]ChatConversation ChatConversation?defaultResumeId String?defaultResume Resume? @relation("UserDefaultResume", fields: [defaultResumeId], references: [id], onDelete: SetNull)@@ -631,6 +632,24 @@ model Notification {@@index([automationId, kind, boardToken])}+model AiCall {+ id String @id @default(uuid())+ userId String+ user User @relation(fields: [userId], references: [id])+ feature String // AiUsageFeature in aiUsage.model.ts+ provider String+ model String+ status String // AiCallStatus in aiUsage.model.ts+ inputTokens Int?+ outputTokens Int?+ durationMs Int+ firstTokenMs Int?+ numCtx Int? // Ollama only, and only where the app passes num_ctx+ startedAt DateTime @default(now())++ @@index([userId, startedAt])+}+model Note {id String @id @default(uuid())jobId Stringdiff --git a/src/lib/aiUsage/prune.ts b/src/lib/aiUsage/prune.tsnew file mode 100644index 0000000..e57a30d--- /dev/null+++ b/src/lib/aiUsage/prune.ts@@ -0,0 +1,22 @@+import db from "@/lib/db";+import { log } from "@/lib/telemetry";+import { AI_USAGE_CONSTANTS } from "@/lib/constants";++export function aiUsageCutoff(now: Date = new Date()): Date {+ return new Date(+ now.getTime() - AI_USAGE_CONSTANTS.RETENTION_DAYS * 24 * 60 * 60 * 1000,+ );+}++export async function pruneAiCalls(now: Date = new Date()): Promise<number> {+ try {+ const { count } = await db.aiCall.deleteMany({+ where: { startedAt: { lt: aiUsageCutoff(now) } },+ });+ if (count > 0) log.info("[AiUsage] Pruned", { "ai_usage.pruned": count });+ return count;+ } catch (error) {+ log.error("[AiUsage] Prune failed", { error: String(error) });+ return 0;+ }+}diff --git a/src/lib/aiUsage/tracker.ts b/src/lib/aiUsage/tracker.tsnew file mode 100644index 0000000..0acf20d--- /dev/null+++ b/src/lib/aiUsage/tracker.ts@@ -0,0 +1,104 @@+import "server-only";++import db from "@/lib/db";+import { log } from "@/lib/telemetry";+import type { AiCallStatus, AiUsageFeature } from "@/models/aiUsage.model";++export type TokenUsage = { inputTokens?: number; outputTokens?: number };++export type AiCallFinish = {+ usage?: TokenUsage;+ finishReason?: string;+ error?: unknown;+ // Pass the call's signal on the abort/error path; it only counts if fired.+ abortedBy?: AbortSignal;+ // Time spent inside nested generations, which carry their own rows.+ excludeMs?: number;+};++export type AiCallTracker = {+ markFirstToken(): void;+ finish(input: AiCallFinish): Promise<void>;+};++export function statusOf(input: AiCallFinish): AiCallStatus {+ if (input.abortedBy?.aborted) {+ // AbortSignal.timeout and AbortSignal.any both surface the firing+ // signal's reason, so a deadline reads the same at every call site.+ const reason = input.abortedBy.reason as { name?: string } | undefined;+ return reason?.name === "TimeoutError" ? "timed_out" : "stopped";+ }+ if (input.error !== undefined) return "error";+ switch (input.finishReason) {+ case "stop":+ case "tool-calls":+ return "ok";+ case "length":+ return "truncated";+ case "error":+ case "content-filter":+ return "error";+ default:+ // "other" / missing: Ollama's body ended without its done chunk.+ return "interrupted";+ }+}++export function usageOfSteps(+ steps: readonly { usage: TokenUsage }[],+): TokenUsage | undefined {+ if (steps.length === 0) return undefined;+ return steps.reduce<Required<TokenUsage>>(+ (sum, step) => ({+ inputTokens: sum.inputTokens + (step.usage.inputTokens ?? 0),+ outputTokens: sum.outputTokens + (step.usage.outputTokens ?? 0),+ }),+ { inputTokens: 0, outputTokens: 0 },+ );+}++export function startAiCall(start: {+ userId: string;+ feature: AiUsageFeature;+ provider: string;+ model: string;+ numCtx?: number;+}): AiCallTracker {+ const startedAt = new Date();+ let firstTokenMs: number | undefined;+ let finished = false;++ return {+ markFirstToken() {+ if (firstTokenMs === undefined) {+ firstTokenMs = Date.now() - startedAt.getTime();+ }+ },+ async finish(input) {+ // Several callbacks can report one call (onError then onFinish).+ if (finished) return;+ finished = true;+ try {+ const elapsed = Date.now() - startedAt.getTime();+ await db.aiCall.create({+ data: {+ userId: start.userId,+ feature: start.feature,+ provider: start.provider,+ model: start.model,+ status: statusOf(input),+ inputTokens: input.usage?.inputTokens ?? null,+ outputTokens: input.usage?.outputTokens ?? null,+ durationMs: Math.max(0, elapsed - (input.excludeMs ?? 0)),+ firstTokenMs: firstTokenMs ?? null,+ numCtx: start.provider === "ollama" ? (start.numCtx ?? null) : null,+ startedAt,+ },+ });+ } catch (error) {+ // Usage is bookkeeping; it must never fail the call it describes.+ log.warn("[AiUsage] Failed to record call", { error: String(error) });+ }+ },+ };+}diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex abf17b5..2a34950 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -428,3 +428,11 @@ export const NOTIFICATION_CONSTANTS = {POPOVER_LIMIT: 20,PAGE_SIZE: 50,} as const;++export const AI_USAGE_CONSTANTS = {+ RETENTION_DAYS: 90,+ RECENT_LIMIT: 50,+ // Ollama drops the oldest context silently on overflow, so "near" is the+ // only warning available.+ CONTEXT_WARN_RATIO: 0.9,+} as const;diff --git a/src/lib/scheduler/index.ts b/src/lib/scheduler/index.tsindex c1a9f89..b39e797 100644--- a/src/lib/scheduler/index.ts+++ b/src/lib/scheduler/index.ts@@ -5,6 +5,7 @@ import { runAutomation, AutomationAlreadyRunningError } from "@/lib/scraper";import { ATS_BOARDS, type JobBoard } from "@/models/automation.model";import { log } from "@/lib/telemetry";import { pruneNotifications } from "@/lib/notifications/prune";+import { pruneAiCalls } from "@/lib/aiUsage/prune";let scheduledTask: ScheduledTask | null = null;@@ -16,6 +17,7 @@ async function runDueAutomations() {try {await pruneNotifications(now);+ await pruneAiCalls(now);const dueAutomations = await db.automation.findMany({where: {status: "active",diff --git a/src/models/aiUsage.model.ts b/src/models/aiUsage.model.tsnew file mode 100644index 0000000..8fa84f0--- /dev/null+++ b/src/models/aiUsage.model.ts@@ -0,0 +1,103 @@+// Own vocabulary, not telemetry's SURFACES: the two disagree on the Analyze+// button (Automations here, job.match there) and SURFACES must not grow.+export const AI_USAGE_FEATURES = {+ agent_chat: "Agent chat",+ job_match: "Job match",+ automations: "Automations",+ resume_review: "Resume review",+ cover_letter: "Cover letter",+ resume_import: "Resume import",+} as const;++export type AiUsageFeature = keyof typeof AI_USAGE_FEATURES;++export const FEATURE_BY_NESTED_LABEL: Record<string, AiUsageFeature> = {+ review_resume: "resume_review",+ match_job: "job_match",+ generate_cover_letter: "cover_letter",+};++export const AI_CALL_STATUSES = [+ "ok",+ "error",+ "truncated",+ "interrupted",+ "timed_out",+ "stopped",+] as const;++export type AiCallStatus = (typeof AI_CALL_STATUSES)[number];++// Stopped is a call but never a failure — the user chose it.+export const AI_FAILURE_STATUSES = [+ "error",+ "truncated",+ "interrupted",+ "timed_out",+] as const;++export type AiFailureStatus = (typeof AI_FAILURE_STATUSES)[number];++export const AI_PROVIDER_LABELS: Record<string, string> = {+ ollama: "Ollama",+ openai: "OpenAI",+ deepseek: "DeepSeek",+ gemini: "Gemini",+ openrouter: "OpenRouter",+};++export type AiUsageRange = 7 | 30;++export type AiCallRow = {+ id: string;+ startedAt: Date;+ feature: AiUsageFeature;+ provider: string;+ model: string;+ status: AiCallStatus;+ inputTokens: number | null;+ outputTokens: number | null;+ durationMs: number;+ firstTokenMs: number | null;+};++export type AiModelUsage = {+ model: string;+ tokens: number;+ calls: number;+ medianMs: number | null;+ p95Ms: number | null;+ medianFirstTokenMs: number | null;+ nearContextLimit: number;+};++export type AiUsageSummary = {+ rangeDays: AiUsageRange;+ totals: {+ inputTokens: number;+ outputTokens: number;+ calls: number;+ previousCalls: number;+ failures: number;+ stopped: number;+ medianMs: number | null;+ p95Ms: number | null;+ medianFirstTokenMs: number | null;+ };+ failuresByStatus: Record<AiFailureStatus, number>;+ // Oldest first; one entry per calendar day in the viewer's timezone.+ days: { date: string; tokensByProvider: Record<string, number> }[];+ providers: {+ provider: string;+ tokens: number;+ calls: number;+ models: AiModelUsage[];+ }[];+ features: {+ feature: AiUsageFeature;+ tokens: number;+ calls: number;+ medianMs: number | null;+ }[];+ recent: AiCallRow[];+};