Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(billing): hold an AI call's share of the budget before it runs (#1387)

Cihane211d4d55470
diff --git a/server/src/lib/action-center/task-writer/index.js b/server/src/lib/action-center/task-writer/index.js
index 82c3b3c15..74a6a58b8 100644
--- a/server/src/lib/action-center/task-writer/index.js
+++ b/server/src/lib/action-center/task-writer/index.js
@@ -21,7 +21,7 @@ import { logger } from '../../logger.js';
import { resolveModel } from '../../ai-provider.js';
import { withRetry } from '../../retry.js';
import { generationCostUsd } from '../../improve-cost.js';
-import { PlanLimitError, enforceImproveQuota } from '../../plan-guard.js';
+import { PlanLimitError, reserveAiBudget } from '../../plan-guard.js';
import { loadDefinitions } from '../definitions/index.js';
import { getTask } from '../tasks/registry.js';
import { IMPROVE_ACTION_KIND } from '../from-prompt-improve.js';
@@ -96,19 +96,6 @@ export class TaskWriterError extends Error {
}
}
-async function recordUsage({ organizationId, actionId, modelId, usage }) {
- const { error } = await supabaseAdmin.from('prompt_improve_usage').insert({
- organization_id: organizationId,
- action_id: actionId,
- feature: 'action_center',
- model: modelId,
- input_tokens: usage?.inputTokens ?? null,
- output_tokens: usage?.outputTokens ?? null,
- cost_usd: generationCostUsd(modelId, usage),
- });
- if (error) logger.error({ err: error, actionId }, '[task-writer] usage insert failed');
-}
-
/** What the writer needs: the action, its brand, its signals. */
async function loadAction(actionId) {
const { data: action, error } = await supabaseAdmin
@@ -282,51 +269,61 @@ async function compose({ action, brand, domains, signals, name, trigger, userId,
}
const modelId = taskWriterModelId();
- if (!dryRun) await enforceImproveQuota(brand.organization_id, modelId);
-
- let costUsd = 0;
- const charge = async (usage) => {
- costUsd += generationCostUsd(modelId, usage);
- if (!dryRun) {
- await recordUsage({ organizationId: brand.organization_id, actionId, modelId, usage });
+ // Held on the budget before the first call, so actions written at once
+ // (a sweep, a regenerate) can't pass it. A dry run charges nothing.
+ const budget = dryRun
+ ? null
+ : await reserveAiBudget(brand.organization_id, {
+ feature: 'action_center',
+ modelId,
+ actionId,
+ });
+ try {
+ let costUsd = 0;
+ const charge = async (usage) => {
+ costUsd += generationCostUsd(modelId, usage);
+ await budget?.charge(usage);
+ };
+
+ const shown = evidenceForPrompt(evidence);
+ const base = buildTaskWriterPrompt({
+ action: { name, category: action.category },
+ brand: { name: brand.name, description: brand.description, domains },
+ evidence: shown,
+ });
+ const first = sift(await callModel({ modelId, ...base, charge }), evidence, []);
+ let tasks = first.accepted;
+ let refused = first.refused;
+
+ // Refused tasks come back once, with their reasons.
+ const missing = Math.min(refused.length, MAX_TASKS - tasks.length);
+ if (missing > 0) {
+ const retry = buildRetryPrompt({ base, kept: tasks, refused, count: missing });
+ const second = sift(await callModel({ modelId, ...retry, charge }), evidence, tasks);
+ tasks = [...tasks, ...second.accepted];
+ // Both rounds' refusals, so a review sees how often the first answer misses.
+ refused = [...refused, ...second.refused];
}
- };
- const shown = evidenceForPrompt(evidence);
- const base = buildTaskWriterPrompt({
- action: { name, category: action.category },
- brand: { name: brand.name, description: brand.description, domains },
- evidence: shown,
- });
- const first = sift(await callModel({ modelId, ...base, charge }), evidence, []);
- let tasks = first.accepted;
- let refused = first.refused;
-
- // Refused tasks come back once, with their reasons.
- const missing = Math.min(refused.length, MAX_TASKS - tasks.length);
- if (missing > 0) {
- const retry = buildRetryPrompt({ base, kept: tasks, refused, count: missing });
- const second = sift(await callModel({ modelId, ...retry, charge }), evidence, tasks);
- tasks = [...tasks, ...second.accepted];
- // Both rounds' refusals, so a review sees how often the first answer misses.
- refused = [...refused, ...second.refused];
- }
+ if (dryRun) return { status: 'preview', tasks, refused, costUsd, evidence: shown };
+ if (tasks.length === 0) {
+ await markWritten(actionId, now);
+ return { status: 'skipped', reason: 'no_valid_tasks', refused, costUsd };
+ }
- if (dryRun) return { status: 'preview', tasks, refused, costUsd, evidence: shown };
- if (tasks.length === 0) {
- await markWritten(actionId, now);
- return { status: 'skipped', reason: 'no_valid_tasks', refused, costUsd };
+ const kept = await replaceTasks(actionId, tasks, { now });
+ await markWritten(actionId, now, { changed: true });
+ await supabaseAdmin.from('action_events').insert({
+ action_id: actionId,
+ event: 'tasks_generated',
+ data: { count: tasks.length, kept, trigger },
+ actor_id: userId,
+ });
+ return { status: 'written', tasks, refused, kept, costUsd };
+ } finally {
+ // Nothing charged (the first call failed): the hold goes.
+ await budget?.release();
}
-
- const kept = await replaceTasks(actionId, tasks, { now });
- await markWritten(actionId, now, { changed: true });
- await supabaseAdmin.from('action_events').insert({
- action_id: actionId,
- event: 'tasks_generated',
- data: { count: tasks.length, kept, trigger },
- actor_id: userId,
- });
- return { status: 'written', tasks, refused, kept, costUsd };
}
/** Record the writing; the action counts as updated only when its tasks changed. */
diff --git a/server/src/lib/action-center/task-writer/index.test.js b/server/src/lib/action-center/task-writer/index.test.js
index 0834924aa..e828e3579 100644
--- a/server/src/lib/action-center/task-writer/index.test.js
+++ b/server/src/lib/action-center/task-writer/index.test.js
@@ -85,10 +85,13 @@ vi.mock('ai', () => ({
vi.mock('../../ai-provider.js', () => ({ resolveModel: () => ({}) }));
vi.mock('../../logger.js', () => ({ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() } }));
-const enforceImproveQuota = vi.fn();
+// The budget hold (reserveAiBudget): what each call charged, and its release.
+const charges = [];
+const release = vi.fn();
+const reserveAiBudget = vi.fn();
class PlanLimitError extends Error {}
vi.mock('../../plan-guard.js', () => ({
- enforceImproveQuota: (...args) => enforceImproveQuota(...args),
+ reserveAiBudget: (...args) => reserveAiBudget(...args),
PlanLimitError,
}));
vi.mock('../definitions/index.js', () => ({
@@ -116,7 +119,12 @@ const answer = (tasks) => ({ object: { tasks }, usage: { inputTokens: 1000, outp
beforeEach(() => {
ops = [];
generateObject.mockReset();
- enforceImproveQuota.mockReset();
+ charges.length = 0;
+ release.mockReset();
+ reserveAiBudget.mockReset().mockResolvedValue({
+ charge: async (usage) => charges.push(usage),
+ release,
+ });
process.env.ACTION_TASKS_AI = 'true';
db = {
actions: [
@@ -253,13 +261,17 @@ describe('writeTasksForAction', () => {
]);
expect(tasks[2].details).toMatchObject({ source: 'ai', type: 'content', refs: ['page:1'] });
- const usage = ops.filter((op) => op.table === 'prompt_improve_usage');
- expect(usage).toHaveLength(2);
- expect(usage[0].payload).toMatchObject({
- organization_id: 'o1',
- action_id: 'a1',
+ // Held on the budget once, then both calls charged to it.
+ expect(reserveAiBudget).toHaveBeenCalledWith('o1', {
feature: 'action_center',
+ modelId: expect.any(String),
+ actionId: 'a1',
});
+ expect(charges).toEqual([
+ { inputTokens: 1000, outputTokens: 500 },
+ { inputTokens: 1000, outputTokens: 500 },
+ ]);
+ expect(release).toHaveBeenCalled();
expect(ops.find((op) => op.table === 'action_events').payload).toMatchObject({
event: 'tasks_generated',
data: { count: 3, kept: 2, trigger: 'regenerate' },
@@ -274,7 +286,7 @@ describe('writeTasksForAction', () => {
expect(result.status).toBe('preview');
expect(result.costUsd).toBeGreaterThan(0);
expect(ops).toEqual([]);
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
});
it('calls no model when the signals name nothing to work on', async () => {
@@ -286,7 +298,7 @@ describe('writeTasksForAction', () => {
});
it('leaves the tasks as they are once the budget is spent', async () => {
- enforceImproveQuota.mockRejectedValueOnce(new PlanLimitError('Budget used up'));
+ reserveAiBudget.mockRejectedValueOnce(new PlanLimitError('Budget used up'));
await expect(writeTasksForAction('a1', { trigger: 'regenerate' })).rejects.toThrow(
'Budget used up',
);
@@ -295,6 +307,13 @@ describe('writeTasksForAction', () => {
expect(db.actions[0].tasks_writing_since).toBeNull();
});
+ it('lets the hold on the budget go when the model fails', async () => {
+ generateObject.mockRejectedValue(new Error('timeout'));
+ await expect(writeTasksForAction('a1', { trigger: 'regenerate' })).rejects.toThrow();
+ expect(charges).toEqual([]);
+ expect(release).toHaveBeenCalled();
+ });
+
it('leaves a plan someone is working through to the night, and releases its claim', async () => {
const result = await writeTasksForAction('a1', { trigger: 'auto' });
expect(result).toMatchObject({ status: 'skipped', reason: 'in_use' });
@@ -326,9 +345,9 @@ describe('sweepActionTasks', () => {
it('skips the rest of a brand once its budget is spent', async () => {
db.actions.push({ ...db.actions[0], id: 'a2' });
db.action_tasks = [];
- enforceImproveQuota.mockRejectedValue(new PlanLimitError('Budget used up'));
+ reserveAiBudget.mockRejectedValue(new PlanLimitError('Budget used up'));
const outcome = await sweepActionTasks();
expect(outcome).toEqual({ written: 0, skipped: 0, failed: 1 });
- expect(enforceImproveQuota).toHaveBeenCalledTimes(1);
+ expect(reserveAiBudget).toHaveBeenCalledTimes(1);
});
});
diff --git a/server/src/lib/content-agent.js b/server/src/lib/content-agent.js
index f6cc276ac..0582e5d6a 100644
--- a/server/src/lib/content-agent.js
+++ b/server/src/lib/content-agent.js
@@ -1,14 +1,14 @@
import { randomUUID } from 'crypto';
-import { generateObject } from 'ai';
+import { NoObjectGeneratedError, generateObject } from 'ai';
import * as cheerio from 'cheerio';
import { z } from 'zod';
import supabaseAdmin from '../config/supabase.js';
import { resolveModel } from './ai-provider.js';
-import { contentBriefModelId, recordUsage } from './content-brief.js';
+import { contentBriefModelId } from './content-brief.js';
import { linkAllowList, linkKey, linkTargetsPrompt, readLinkTargets } from './content-links.js';
import { pageTypeFormat } from './content-page-types.js';
import { getLanguageName } from './languages.js';
-import { enforceImproveQuota } from './plan-guard.js';
+import { reserveAiBudget } from './plan-guard.js';
/**
* Fix open rules with agent (Content Editor CE-018, spec §8): the Content
@@ -493,8 +493,6 @@ export async function fixDraftRules({ draftId, brandId, input }) {
if (brandError) throw brandError;
const modelId = contentBriefModelId();
- await enforceImproveQuota(brand.organization_id, modelId);
-
const lockedKeys = new Set(input.locked.map(passageKey));
const ruleKeys = new Set(input.rules.map((r) => r.key));
// Where it may link: the brand's pages and the draft's sources (A).
@@ -504,22 +502,40 @@ export async function fixDraftRules({ draftId, brandId, input }) {
sourceUrl: draft.source_url,
ruleKeys,
});
- const { object, usage } = await generateObject({
- model: resolveModel(modelId),
- schema: suggestionsSchema,
- system: SYSTEM_PROMPT,
- prompt: buildFixPrompt({
- brand,
- draft,
- passages,
- more,
- lockedKeys,
- input,
- linkTargets: linkTargetsPrompt(targets, ruleKeys),
- }),
- abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),
+ // Held on the budget before the call, so calls at once can't pass it.
+ const budget = await reserveAiBudget(brand.organization_id, {
+ feature: 'content',
+ modelId,
+ draftId,
});
- await recordUsage({ organizationId: brand.organization_id, draftId, modelId, usage });
+ let object;
+ try {
+ const generated = await generateObject({
+ model: resolveModel(modelId),
+ schema: suggestionsSchema,
+ system: SYSTEM_PROMPT,
+ prompt: buildFixPrompt({
+ brand,
+ draft,
+ passages,
+ more,
+ lockedKeys,
+ input,
+ linkTargets: linkTargetsPrompt(targets, ruleKeys),
+ }),
+ abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),
+ });
+ await budget.charge(generated.usage);
+ object = generated.object;
+ } catch (err) {
+ // An answer the provider billed but that missed the schema is charged,
+ // as Improve's is: else it, and each retry after it, would run free.
+ if (NoObjectGeneratedError.isInstance(err) && err.usage) await budget.charge(err.usage);
+ throw err;
+ } finally {
+ // Nothing charged (a timeout, a refusal): the hold goes.
+ await budget.release();
+ }
const rows = keepSuggestions(
object.suggestions,
diff --git a/server/src/lib/content-agent.test.js b/server/src/lib/content-agent.test.js
index 4210532cc..0ee750e31 100644
--- a/server/src/lib/content-agent.test.js
+++ b/server/src/lib/content-agent.test.js
@@ -12,7 +12,7 @@ const DRAFT = '77777777-7777-7777-7777-777777777777';
let rows;
let writes;
const generateObject = vi.fn();
-const enforceImproveQuota = vi.fn();
+const reserveAiBudget = vi.fn();
vi.mock('../config/supabase.js', () => ({
default: {
@@ -31,11 +31,19 @@ vi.mock('../config/supabase.js', () => ({
},
},
}));
-vi.mock('ai', () => ({ generateObject: (...args) => generateObject(...args) }));
+class NoObjectGeneratedError extends Error {
+ static isInstance(err) {
+ return err instanceof NoObjectGeneratedError;
+ }
+}
+vi.mock('ai', () => ({
+ generateObject: (...args) => generateObject(...args),
+ NoObjectGeneratedError,
+}));
vi.mock('./ai-provider.js', () => ({ resolveModel: vi.fn((id) => `model:${id}`) }));
vi.mock('./plan-guard.js', async (importOriginal) => ({
...(await importOriginal()),
- enforceImproveQuota: (...args) => enforceImproveQuota(...args),
+ reserveAiBudget: (...args) => reserveAiBudget(...args),
}));
vi.mock('./logger.js', () => ({ logger: { error: vi.fn(), warn: vi.fn() } }));
const readLinkTargets = vi.fn();
@@ -60,6 +68,8 @@ const {
} = await import('./content-agent.js');
const { linkKey, NO_LINK_TARGETS } = await import('./content-links.js');
const { PlanLimitError } = await import('./plan-guard.js');
+// The real hold: self-hosted here, so it reserves nothing and writes each charge.
+const realGuard = await vi.importActual('./plan-guard.js');
const BODY =
'<h1>How to brew pour-over coffee</h1><p>Pour-over coffee tastes better than other methods.</p>' +
@@ -100,7 +110,7 @@ beforeEach(() => {
brands: { name: 'Acme Coffee', organization_id: 'org-1', language: 'en' },
};
generateObject.mockReset();
- enforceImproveQuota.mockReset().mockResolvedValue(undefined);
+ reserveAiBudget.mockReset().mockImplementation(realGuard.reserveAiBudget);
readLinkTargets.mockReset().mockResolvedValue(NO_LINK_TARGETS);
});
@@ -569,7 +579,10 @@ describe('fixDraftRules', () => {
const result = await fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() });
- expect(enforceImproveQuota).toHaveBeenCalledWith('org-1', expect.any(String));
+ expect(reserveAiBudget).toHaveBeenCalledWith(
+ 'org-1',
+ expect.objectContaining({ feature: 'content', modelId: expect.any(String) }),
+ );
expect(generateObject).toHaveBeenCalledWith(
expect.objectContaining({
model: expect.stringMatching(/^model:/),
@@ -618,7 +631,7 @@ describe('fixDraftRules', () => {
await expect(
fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toMatchObject({ status: 422, code: 'empty' });
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
expect(generateObject).not.toHaveBeenCalled();
});
@@ -638,7 +651,7 @@ describe('fixDraftRules', () => {
await expect(
fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toMatchObject({ status: 409, code: 'open_suggestions' });
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
expect(generateObject).not.toHaveBeenCalled();
// An older batch left open doesn't count: the editor shows the latest.
@@ -653,10 +666,36 @@ describe('fixDraftRules', () => {
});
it('stops at the budget', async () => {
- enforceImproveQuota.mockRejectedValue(new PlanLimitError('AI usage is spent', 'starter'));
+ reserveAiBudget.mockRejectedValue(new PlanLimitError('AI usage is spent', 'starter'));
await expect(
fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toBeInstanceOf(PlanLimitError);
expect(generateObject).not.toHaveBeenCalled();
});
+
+ it('lets the hold on the budget go when the model fails', async () => {
+ const held = { charge: vi.fn(), release: vi.fn() };
+ reserveAiBudget.mockResolvedValueOnce(held);
+ generateObject.mockRejectedValueOnce(new Error('timeout'));
+ await expect(fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() })).rejects.toThrow(
+ 'timeout',
+ );
+ expect(held.charge).not.toHaveBeenCalled();
+ expect(held.release).toHaveBeenCalled();
+ });
+
+ it('charges an answer that missed the schema, as the provider billed it', async () => {
+ const held = { charge: vi.fn(), release: vi.fn() };
+ reserveAiBudget.mockResolvedValueOnce(held);
+ const billed = { inputTokens: 900, outputTokens: 4000 };
+ generateObject.mockRejectedValueOnce(
+ Object.assign(new NoObjectGeneratedError('no object'), { usage: billed }),
+ );
+ await expect(
+ fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() }),
+ ).rejects.toBeInstanceOf(NoObjectGeneratedError);
+ expect(held.charge).toHaveBeenCalledTimes(1);
+ expect(held.charge).toHaveBeenCalledWith(billed);
+ expect(held.release).toHaveBeenCalled();
+ });
});
diff --git a/server/src/lib/content-brief.js b/server/src/lib/content-brief.js
index 047a40a7d..09a4ddbf7 100644
--- a/server/src/lib/content-brief.js
+++ b/server/src/lib/content-brief.js
@@ -3,10 +3,8 @@ import { z } from 'zod';
import supabaseAdmin from '../config/supabase.js';
import { resolveModel } from './ai-provider.js';
import { pageTypeFormat } from './content-page-types.js';
-import { generationCostUsd } from './improve-cost.js';
import { getLanguageName } from './languages.js';
-import { logger } from './logger.js';
-import { enforceImproveQuota } from './plan-guard.js';
+import { reserveAiBudget } from './plan-guard.js';
/**
* The Content Editor's brief (CE-007, spec §10): Rebuild outline writes a
@@ -167,7 +165,12 @@ export async function rebuildDraftBrief({ draftId, brandId, sources }) {
}
const modelId = contentBriefModelId();
- await enforceImproveQuota(brand.organization_id, modelId);
+ // Held on the budget before the call, so calls at once can't pass it.
+ const budget = await reserveAiBudget(brand.organization_id, {
+ feature: 'content',
+ modelId,
+ draftId,
+ });
let generated;
try {
@@ -178,21 +181,17 @@ export async function rebuildDraftBrief({ draftId, brandId, sources }) {
prompt: buildBriefPrompt({ brand, draft, prompts, sources }),
abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),
});
+ await budget.charge(generated.usage);
} catch (err) {
// An answer that didn't fit the schema was still paid for: charged as
// Improve charges it, so retries don't run free.
- if (NoObjectGeneratedError.isInstance(err) && err.usage) {
- await recordUsage({
- organizationId: brand.organization_id,
- draftId,
- modelId,
- usage: err.usage,
- });
- }
+ if (NoObjectGeneratedError.isInstance(err) && err.usage) await budget.charge(err.usage);
throw err;
+ } finally {
+ // Nothing charged (a timeout, a refusal): the hold goes.
+ await budget.release();
}
- const { object, usage } = generated;
- await recordUsage({ organizationId: brand.organization_id, draftId, modelId, usage });
+ const { object } = generated;
const brief = {
goal: object.goal.trim().slice(0, BRIEF_LIMITS.goal),
@@ -275,17 +274,3 @@ ${list(sources.pages.map((p) => `${p.url} (${p.kind})`))}
${sources.headings.length ? `The page's headings now:\n${list(sources.headings)}\n\n` : ''}Write the brief in ${getLanguageName(brand?.language)}.`;
}
-
-/** Records an editor AI call on the AI usage budget, with the draft it was for. */
-export async function recordUsage({ organizationId, draftId, modelId, usage }) {
- const { error } = await supabaseAdmin.from('prompt_improve_usage').insert({
- organization_id: organizationId,
- content_draft_id: draftId,
- feature: 'content',
- model: modelId,
- input_tokens: usage?.inputTokens ?? null,
- output_tokens: usage?.outputTokens ?? null,
- cost_usd: generationCostUsd(modelId, usage),
- });
- if (error) logger.error({ err: error, draftId }, '[content-brief] usage insert failed');
-}
diff --git a/server/src/lib/content-brief.test.js b/server/src/lib/content-brief.test.js
index 14c8fe6d4..83a80868e 100644
--- a/server/src/lib/content-brief.test.js
+++ b/server/src/lib/content-brief.test.js
@@ -13,7 +13,7 @@ let rows;
let writes;
let filters;
const generateObject = vi.fn();
-const enforceImproveQuota = vi.fn();
+const reserveAiBudget = vi.fn();
vi.mock('../config/supabase.js', () => ({
default: {
@@ -47,13 +47,15 @@ vi.mock('ai', () => ({
vi.mock('./ai-provider.js', () => ({ resolveModel: vi.fn((id) => `model:${id}`) }));
vi.mock('./plan-guard.js', async (importOriginal) => ({
...(await importOriginal()),
- enforceImproveQuota: (...args) => enforceImproveQuota(...args),
+ reserveAiBudget: (...args) => reserveAiBudget(...args),
}));
vi.mock('./logger.js', () => ({ logger: { error: vi.fn() } }));
const { buildBriefPrompt, mapOutline, rebuildDraftBrief, rebuildSources } =
await import('./content-brief.js');
const { PlanLimitError } = await import('./plan-guard.js');
+// The real hold: self-hosted here, so it reserves nothing and writes each charge.
+const realGuard = await vi.importActual('./plan-guard.js');
const sources = (over = {}) => ({
promptIds: [PROMPT],
@@ -114,14 +116,18 @@ beforeEach(() => {
},
usage: { inputTokens: 1000, outputTokens: 500 },
});
- enforceImproveQuota.mockReset().mockResolvedValue({ unlimited: true });
+ reserveAiBudget.mockReset().mockImplementation(realGuard.reserveAiBudget);
});
describe('rebuildDraftBrief', () => {
it('writes the brief from the kept sources, charges AI usage, and saves it on the draft', async () => {
const brief = await rebuildDraftBrief({ draftId: DRAFT, brandId: BRAND, sources: sources() });
- expect(enforceImproveQuota).toHaveBeenCalledWith('o1', 'google/gemini-3-flash-preview');
+ expect(reserveAiBudget).toHaveBeenCalledWith('o1', {
+ feature: 'content',
+ modelId: 'google/gemini-3-flash-preview',
+ draftId: DRAFT,
+ });
const call = generateObject.mock.calls[0][0];
expect(call.model).toBe('model:google/gemini-3-flash-preview');
expect(call.prompt).toContain('- Best espresso machines under $500?');
@@ -205,7 +211,7 @@ describe('rebuildDraftBrief', () => {
sources: sources({ fanouts: [], queries: [] }),
}),
).rejects.toMatchObject({ status: 422, code: 'no_sources' });
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
expect(generateObject).not.toHaveBeenCalled();
});
@@ -271,8 +277,19 @@ describe('rebuildDraftBrief', () => {
]);
});
+ it('lets the hold on the budget go when the model times out', async () => {
+ const held = { charge: vi.fn(), release: vi.fn() };
+ reserveAiBudget.mockResolvedValueOnce(held);
+ generateObject.mockRejectedValue(new Error('timeout'));
+ await expect(
+ rebuildDraftBrief({ draftId: DRAFT, brandId: BRAND, sources: sources() }),
+ ).rejects.toThrow('timeout');
+ expect(held.charge).not.toHaveBeenCalled();
+ expect(held.release).toHaveBeenCalled();
+ });
+
it('stops when the AI usage budget is spent', async () => {
- enforceImproveQuota.mockRejectedValue(new PlanLimitError('used up', 429));
+ reserveAiBudget.mockRejectedValue(new PlanLimitError('used up', 429));
await expect(
rebuildDraftBrief({ draftId: DRAFT, brandId: BRAND, sources: sources() }),
).rejects.toBeInstanceOf(PlanLimitError);
diff --git a/server/src/lib/content-writer.js b/server/src/lib/content-writer.js
index ef4779309..e06558aed 100644
--- a/server/src/lib/content-writer.js
+++ b/server/src/lib/content-writer.js
@@ -1,5 +1,5 @@
import { randomUUID } from 'crypto';
-import { generateObject } from 'ai';
+import { NoObjectGeneratedError, generateObject } from 'ai';
import { z } from 'zod';
import supabaseAdmin from '../config/supabase.js';
import { resolveModel } from './ai-provider.js';
@@ -14,12 +14,12 @@ import {
promptPassages,
suggestionsSchema,
} from './content-agent.js';
-import { contentBriefModelId, rebuildSources, recordUsage } from './content-brief.js';
+import { contentBriefModelId, rebuildSources } from './content-brief.js';
import { linkAllowList, linkTargetsPrompt, readLinkTargets } from './content-links.js';
import { pageTypeFormat } from './content-page-types.js';
import { getLanguageName } from './languages.js';
import { logger } from './logger.js';
-import { enforceImproveQuota } from './plan-guard.js';
+import { reserveAiBudget } from './plan-guard.js';
/**
* "Agent" in the Content Editor's Start (CE-015, spec §5): the Content Agent
@@ -299,36 +299,47 @@ export async function startDraftWriting({ draftId, brandId, input }) {
.single();
if (brandError) throw brandError;
const modelId = contentBriefModelId();
- await enforceImproveQuota(brand.organization_id, modelId);
- if (await organizationWriting(brand.organization_id, draftId, staleBefore)) {
- throw Object.assign(new Error('The agent is writing another draft'), {
- status: 409,
- code: 'org_writing',
- });
- }
- // A provider that isn't set up fails here, before anything is claimed.
- const model = resolveModel(modelId);
+ // Held on the budget now, for the run below: starts at once can't pass it.
+ const budget = await reserveAiBudget(brand.organization_id, {
+ feature: 'content',
+ modelId,
+ draftId,
+ });
+ try {
+ if (await organizationWriting(brand.organization_id, draftId, staleBefore)) {
+ throw Object.assign(new Error('The agent is writing another draft'), {
+ status: 409,
+ code: 'org_writing',
+ });
+ }
+ // A provider that isn't set up fails here, before anything is claimed.
+ const model = resolveModel(modelId);
+
+ // Claimed once: a second start meanwhile finds it writing.
+ const { data: claimed, error: claimError } = await supabaseAdmin
+ .from('content_drafts')
+ .update({ agent_state: 'writing', agent_state_at: new Date().toISOString() })
+ .eq('id', draftId)
+ .in('status', OPEN_STATUSES)
+ .or(`agent_state.is.null,agent_state.eq.failed,agent_state_at.lt."${staleBefore}"`)
+ .select('id')
+ .maybeSingle();
+ if (claimError) throw claimError;
+ if (!claimed) {
+ throw Object.assign(new Error('The agent is already writing it'), {
+ status: 409,
+ code: 'writing',
+ });
+ }
- // Claimed once: a second start meanwhile finds it writing.
- const { data: claimed, error: claimError } = await supabaseAdmin
- .from('content_drafts')
- .update({ agent_state: 'writing', agent_state_at: new Date().toISOString() })
- .eq('id', draftId)
- .in('status', OPEN_STATUSES)
- .or(`agent_state.is.null,agent_state.eq.failed,agent_state_at.lt."${staleBefore}"`)
- .select('id')
- .maybeSingle();
- if (claimError) throw claimError;
- if (!claimed) {
- throw Object.assign(new Error('The agent is already writing it'), {
- status: 409,
- code: 'writing',
- });
+ runDraftWriting({ draft, brand, model, input, budget }).catch((err) =>
+ logger.error({ err, draftId }, '[content-writer] run failed'),
+ );
+ } catch (err) {
+ // Not started: the hold goes.
+ await budget.release();
+ throw err;
}
-
- runDraftWriting({ draft, brand, model, modelId, input }).catch((err) =>
- logger.error({ err, draftId }, '[content-writer] run failed'),
- );
return { started: true };
}
@@ -358,9 +369,11 @@ async function letGo(draftId) {
/**
* The run itself: the page written, or the suggestions left, and the draft
- * In review; 'failed' when the model fails or leaves nothing to review.
+ * In review; 'failed' when the model fails or leaves nothing to review. The
+ * model's answer is charged to the hold Start took on the budget
+ * (reserveAiBudget); a run that ends before one lets the hold go.
*/
-export async function runDraftWriting({ draft, brand, model, modelId, input }) {
+export async function runDraftWriting({ draft, brand, model, input, budget }) {
try {
const [promptsRes, opportunityRes] = await Promise.all([
input.sources.promptIds.length
@@ -393,12 +406,7 @@ export async function runDraftWriting({ draft, brand, model, modelId, input }) {
prompt: buildWritePrompt({ brand, draft, prompts, brief, input }),
abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),
});
- await recordUsage({
- organizationId: brand.organization_id,
- draftId: draft.id,
- modelId,
- usage,
- });
+ await budget.charge(usage);
const title = clean(object.title).slice(0, 300) || draft.title;
// Try again over a page the person has written in: History keeps it
// first. Their saves wait while the agent writes (saveContentDraft).
@@ -452,7 +460,7 @@ export async function runDraftWriting({ draft, brand, model, modelId, input }) {
}),
abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),
});
- await recordUsage({ organizationId: brand.organization_id, draftId: draft.id, modelId, usage });
+ await budget.charge(usage);
const rows = keepSuggestions(
object.suggestions,
passages,
@@ -492,10 +500,16 @@ export async function runDraftWriting({ draft, brand, model, modelId, input }) {
}
return { suggestions: rows.length };
} catch (err) {
+ // An answer the provider billed but that missed the schema is charged,
+ // as Improve's is: else the costliest call, a whole page, would run free.
+ if (NoObjectGeneratedError.isInstance(err) && err.usage) await budget.charge(err.usage);
logger.error({ err, draftId: draft.id }, '[content-writer] the agent could not finish');
await finish(draft.id, { agent_state: 'failed' }).catch((finishErr) =>
logger.error({ err: finishErr, draftId: draft.id }, '[content-writer] state not saved'),
);
return { failed: true };
+ } finally {
+ // Nothing charged (it failed before an answer): the hold goes.
+ await budget.release();
}
}
diff --git a/server/src/lib/content-writer.test.js b/server/src/lib/content-writer.test.js
index 7c33da8bb..7cae2fca5 100644
--- a/server/src/lib/content-writer.test.js
+++ b/server/src/lib/content-writer.test.js
@@ -16,7 +16,7 @@ let finished;
let rpcCalls;
let reads;
const generateObject = vi.fn();
-const enforceImproveQuota = vi.fn();
+const reserveAiBudget = vi.fn();
vi.mock('../config/supabase.js', () => ({
default: {
@@ -56,12 +56,20 @@ vi.mock('../config/supabase.js', () => ({
},
},
}));
-vi.mock('ai', () => ({ generateObject: (...args) => generateObject(...args) }));
+class NoObjectGeneratedError extends Error {
+ static isInstance(err) {
+ return err instanceof NoObjectGeneratedError;
+ }
+}
+vi.mock('ai', () => ({
+ generateObject: (...args) => generateObject(...args),
+ NoObjectGeneratedError,
+}));
const resolveModel = vi.hoisted(() => vi.fn((id) => `model:${id}`));
vi.mock('./ai-provider.js', () => ({ resolveModel }));
vi.mock('./plan-guard.js', async (importOriginal) => ({
...(await importOriginal()),
- enforceImproveQuota: (...args) => enforceImproveQuota(...args),
+ reserveAiBudget: (...args) => reserveAiBudget(...args),
}));
vi.mock('./logger.js', () => ({ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() } }));
const readLinkTargets = vi.hoisted(() => vi.fn());
@@ -79,6 +87,8 @@ const { briefText, renderPage, runDraftWriting, startDraftWriting } =
await import('./content-writer.js');
const { passageKey } = await import('./content-agent.js');
const { PlanLimitError } = await import('./plan-guard.js');
+// The real hold: self-hosted here, so it reserves nothing and writes each charge.
+const realGuard = await vi.importActual('./plan-guard.js');
const input = (over = {}) => ({
sources: {
@@ -142,7 +152,7 @@ beforeEach(() => {
object: page,
usage: { inputTokens: 10, outputTokens: 20 },
});
- enforceImproveQuota.mockReset().mockResolvedValue(undefined);
+ reserveAiBudget.mockReset().mockImplementation(realGuard.reserveAiBudget);
hasOpenSuggestions.mockReset().mockResolvedValue(false);
});
@@ -193,7 +203,10 @@ describe('startDraftWriting', () => {
await expect(
startDraftWriting({ draftId: DRAFT, brandId: BRAND, input: input() }),
).resolves.toEqual({ started: true });
- expect(enforceImproveQuota).toHaveBeenCalledWith('org-1', expect.any(String));
+ expect(reserveAiBudget).toHaveBeenCalledWith(
+ 'org-1',
+ expect.objectContaining({ feature: 'content', modelId: expect.any(String) }),
+ );
expect(draftUpdates()[0]).toMatchObject({ agent_state: 'writing' });
await vi.waitFor(() => expect(draftUpdates()).toHaveLength(2));
@@ -255,7 +268,7 @@ describe('startDraftWriting', () => {
agent_state_at: new Date().toISOString(),
});
await expect(start()).rejects.toMatchObject({ status: 409, code: 'writing' });
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
expect(writes).toEqual([]);
// A 'writing' left long ago can start again; a start that lost the claim can't.
@@ -275,7 +288,7 @@ describe('startDraftWriting', () => {
startDraftWriting({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toMatchObject({ status: 409, code: 'open_suggestions' });
expect(hasOpenSuggestions).toHaveBeenCalledWith(DRAFT);
- expect(enforceImproveQuota).not.toHaveBeenCalled();
+ expect(reserveAiBudget).not.toHaveBeenCalled();
expect(writes).toEqual([]);
// A new page is written whole: its run adds no batch.
@@ -289,11 +302,16 @@ describe('startDraftWriting', () => {
it('writes one of the organization’s drafts at a time', async () => {
rows.writingInOrg = [{ id: 'other-draft' }];
+ const held = { charge: vi.fn(), release: vi.fn() };
+ reserveAiBudget.mockResolvedValueOnce(held);
await expect(
startDraftWriting({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toMatchObject({ status: 409, code: 'org_writing' });
expect(writes).toEqual([]);
expect(generateObject).not.toHaveBeenCalled();
+ // Not started: the hold on the budget goes.
+ expect(held.release).toHaveBeenCalled();
+ expect(held.charge).not.toHaveBeenCalled();
});
it('fails at once when the model isn’t set up, claiming nothing', async () => {
@@ -307,7 +325,7 @@ describe('startDraftWriting', () => {
});
it('stops at the budget', async () => {
- enforceImproveQuota.mockRejectedValue(new PlanLimitError('used up', 429));
+ reserveAiBudget.mockRejectedValue(new PlanLimitError('used up', 429));
await expect(
startDraftWriting({ draftId: DRAFT, brandId: BRAND, input: input() }),
).rejects.toBeInstanceOf(PlanLimitError);
@@ -315,9 +333,16 @@ describe('startDraftWriting', () => {
});
});
+// The hold Start took on the AI budget (reserveAiBudget), handed to the run.
+const budget = { charge: vi.fn(), release: vi.fn() };
+beforeEach(() => {
+ budget.charge.mockReset();
+ budget.release.mockReset();
+});
+
describe('runDraftWriting on a new page', () => {
const run = (draft) =>
- runDraftWriting({ draft, brand, model: 'model:m', modelId: 'm', input: input() });
+ runDraftWriting({ draft, brand, model: 'model:m', input: input(), budget });
it('keeps the person’s text in History before writing over it, and reads prompts by their set', async () => {
const tried = draftRow({
@@ -325,6 +350,9 @@ describe('runDraftWriting on a new page', () => {
body_html: '<h1>How to brew pour-over coffee</h1><p>My own notes.</p>',
});
await expect(run(tried)).resolves.toEqual({ written: true });
+ // The answer is charged to the hold; nothing is left to release after.
+ expect(budget.charge).toHaveBeenCalledTimes(1);
+ expect(budget.release).toHaveBeenCalled();
expect(rpcCalls).toEqual([
['content_draft_snapshot', { p_draft_id: DRAFT, p_origin: 'agent_before' }],
]);
@@ -359,8 +387,8 @@ describe('runDraftWriting on an existing page', () => {
draft: existing,
brand,
model: 'model:m',
- modelId: 'm',
input: input({ locked: [passageKey(QUOTED)], ...over }),
+ budget,
});
it('leaves its suggestions for review, never on a locked passage', async () => {
@@ -455,5 +483,20 @@ describe('runDraftWriting on an existing page', () => {
{ agent_state: 'failed', agent_state_at: expect.any(String) },
{ agent_state: 'failed', agent_state_at: expect.any(String) },
]);
+ // The first answer was paid for; the second never came, and its hold goes.
+ expect(budget.charge).toHaveBeenCalledTimes(1);
+ expect(budget.release).toHaveBeenCalledTimes(2);
+ });
+
+ it('charges an answer that missed the schema, as the provider billed it', async () => {
+ const billed = { inputTokens: 5000, outputTokens: 6000 };
+ generateObject.mockRejectedValueOnce(
+ Object.assign(new NoObjectGeneratedError('no object'), { usage: billed }),
+ );
+ await expect(run()).resolves.toEqual({ failed: true });
+ expect(budget.charge).toHaveBeenCalledTimes(1);
+ expect(budget.charge).toHaveBeenCalledWith(billed);
+ // The run still ends failed.
+ expect(draftUpdates()).toEqual([{ agent_state: 'failed', agent_state_at: expect.any(String) }]);
});
});
diff --git a/server/src/lib/improve-cost.js b/server/src/lib/improve-cost.js
index 1980778dc..a51cf20a4 100644
--- a/server/src/lib/improve-cost.js
+++ b/server/src/lib/improve-cost.js
@@ -23,7 +23,8 @@ export const IMPROVE_BUDGET_SHARE = 0.01;
/**
* A typical generation, used to estimate "generations left" before an org
- * has any history: ~7k input and ~2k output tokens on the configured model.
+ * has any history, and the least a call holds on the budget before it runs
+ * (reserveAiBudget): ~7k input and ~2k output tokens on the configured model.
*/
const TYPICAL_TOKENS = { input: 7000, output: 2000 };
diff --git a/server/src/lib/plan-guard-ai-budget.test.js b/server/src/lib/plan-guard-ai-budget.test.js
new file mode 100644
index 000000000..8499d9fef
--- /dev/null
+++ b/server/src/lib/plan-guard-ai-budget.test.js
@@ -0,0 +1,181 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * A call's share of the workspace's AI budget is held before it runs
+ * (reserve_ai_usage, 00243), so calls at once can't pass a nearly spent
+ * budget; the hold becomes the call's real cost, or goes when nothing was
+ * charged.
+ */
+
+let org;
+let reserved;
+let rpcCalls;
+let ops;
+// The org's recent calls this period, newest first (what a hold is sized on).
+let recent;
+
+function query(table) {
+ const op = { table, kind: 'select', filters: [] };
+ const builder = {
+ select: () => builder,
+ eq: (col, val) => (op.filters.push([col, val]), builder),
+ gte: (col, val) => (op.filters.push([col, val]), builder),
+ order: () => builder,
+ limit: () => builder,
+ maybeSingle: () => builder,
+ single: () => builder,
+ insert: (value) => (Object.assign(op, { kind: 'insert', value }), ops.push(op), builder),
+ update: (value) => (Object.assign(op, { kind: 'update', value }), ops.push(op), builder),
+ delete: () => (Object.assign(op, { kind: 'delete' }), ops.push(op), builder),
+ then: (resolve) =>
+ resolve(
+ table === 'prompt_improve_usage' && op.kind === 'select'
+ ? recent
+ : { data: table === 'organizations' ? org : null, error: null },
+ ),
+ };
+ return builder;
+}
+
+let guard;
+
+beforeEach(async () => {
+ vi.resetModules();
+ process.env.IS_CLOUD = 'true';
+ org = { plan: 'starter', subscription_status: 'active', subscription_ends_at: null };
+ reserved = 'u1';
+ recent = { data: [], error: null };
+ rpcCalls = [];
+ ops = [];
+ vi.doMock('../config/supabase.js', () => ({
+ default: {
+ from: (t) => query(t),
+ rpc: async (fn, args) => {
+ rpcCalls.push({ fn, args });
+ return { data: reserved, error: null };
+ },
+ },
+ }));
+ vi.doMock('./logger.js', () => ({ logger: { error: vi.fn() } }));
+ guard = await import('./plan-guard.js');
+});
+
+afterEach(() => {
+ delete process.env.IS_CLOUD;
+ vi.doUnmock('../config/supabase.js');
+ vi.doUnmock('./logger.js');
+});
+
+const usage = { inputTokens: 7000, outputTokens: 2000 };
+
+describe('reserveAiBudget', () => {
+ it('holds the org’s recent median cost when calls run above the model’s estimate', async () => {
+ recent = {
+ data: [0.05, 0.021, 0.02, 0.019, 0.004].map((cost_usd) => ({ cost_usd })),
+ error: null,
+ };
+ await guard.reserveAiBudget('o1', {
+ feature: 'content',
+ modelId: 'google/gemini-3-flash-preview',
+ });
+ expect(rpcCalls[0].args.p_cost).toBe(0.02);
+ // Never less than the model's typical cost, nor when the read fails.
+ rpcCalls = [];
+ recent = { data: [{ cost_usd: 0.001 }], error: null };
+ await guard.reserveAiBudget('o1', {
+ feature: 'content',
+ modelId: 'google/gemini-3-flash-preview',
+ });
+ recent = { data: null, error: { message: 'timeout' } };
+ await guard.reserveAiBudget('o1', {
+ feature: 'content',
+ modelId: 'google/gemini-3-flash-preview',
+ });
+ expect(rpcCalls.map((c) => c.args.p_cost)).toEqual([0.0095, 0.0095]);
+ });
+
+ it('holds the estimate on the period’s budget, then turns it into the real cost', async () => {
+ const budget = await guard.reserveAiBudget('o1', {
+ feature: 'content',
+ modelId: 'google/gemini-3-flash-preview',
+ draftId: 'd1',
+ });
+ expect(rpcCalls).toEqual([
+ {
+ fn: 'reserve_ai_usage',
+ args: expect.objectContaining({
+ p_organization_id: 'o1',
+ // Starter's $1, from the period's start.
+ p_budget: 1,
+ p_since: expect.any(String),
+ p_cost: 0.0095,
+ p_feature: 'content',
+ p_model: 'google/gemini-3-flash-preview',
+ p_prompt_id: null,
+ p_action_id: null,
+ p_content_draft_id: 'd1',
+ }),
+ },
+ ]);
+
+ await budget.charge({ inputTokens: 1000, outputTokens: 500 });
+ // The first charge is the hold, at the call's real cost…
+ expect(ops).toEqual([
+ expect.objectContaining({
+ table: 'prompt_improve_usage',
+ kind: 'update',
+ value: expect.objectContaining({ input_tokens: 1000, cost_usd: 0.002 }),
+ filters: [['id', 'u1']],
+ }),
+ ]);
+ // …a retry adds its own row, and nothing is left to release.
+ await budget.charge(usage);
+ await budget.release();
+ expect(ops.map((op) => op.kind)).toEqual(['update', 'insert']);
+ expect(ops[1].value).toMatchObject({
+ organization_id: 'o1',
+ feature: 'content',
+ content_draft_id: 'd1',
+ input_tokens: 7000,
+ });
+ });
+
+ it('lets the hold go when nothing was charged', async () => {
+ const budget = await guard.reserveAiBudget('o1', {
+ feature: 'improve',
+ modelId: 'm',
+ promptId: 'p1',
+ });
+ await budget.release();
+ await budget.release();
+ expect(ops).toEqual([expect.objectContaining({ kind: 'delete', filters: [['id', 'u1']] })]);
+ });
+
+ it('refuses once the budget is spent, as the quota did', async () => {
+ reserved = null;
+ await expect(
+ guard.reserveAiBudget('o1', { feature: 'improve', modelId: 'm', promptId: 'p1' }),
+ ).rejects.toMatchObject({ statusCode: 429 });
+ expect(ops).toEqual([]);
+ });
+
+ it('holds nothing on an unlimited budget, and still writes each charge', async () => {
+ delete process.env.IS_CLOUD;
+ vi.resetModules();
+ guard = await import('./plan-guard.js');
+ const budget = await guard.reserveAiBudget('o1', {
+ feature: 'action_center',
+ modelId: 'm',
+ actionId: 'a1',
+ });
+ expect(rpcCalls).toEqual([]);
+ await budget.charge(usage);
+ await budget.release();
+ expect(ops).toEqual([
+ expect.objectContaining({
+ kind: 'insert',
+ value: expect.objectContaining({ feature: 'action_center', action_id: 'a1' }),
+ }),
+ ]);
+ });
+});
diff --git a/server/src/lib/plan-guard.js b/server/src/lib/plan-guard.js
index c7d69ad68..065c8db60 100644
--- a/server/src/lib/plan-guard.js
+++ b/server/src/lib/plan-guard.js
@@ -10,9 +10,11 @@ import { anchoredPeriodStart, shiftMonthClamped } from './quota-period.js';
import {
IMPROVE_BUDGET_SHARE,
estimateGenerationsLeft,
+ generationCostUsd,
medianCost,
typicalCostUsd,
} from './improve-cost.js';
+import { logger } from './logger.js';
import { RELEVANCE_BUDGET_SHARE } from './suggestion-relevance.js';
import { AUDIT_RUN_TIMEOUT_MS } from './audit/history.js';
import {
@@ -304,6 +306,9 @@ export async function getBriefQuotaStatus(orgId) {
return { used, limit: max, remaining: Math.max(0, max - used) };
}
+/** How many of the org's latest calls its typical cost is the median of. */
+const RECENT_CALLS = 20;
+
async function improveSpendThisPeriod(orgId) {
const periodStart = await quotaPeriodStart(orgId);
const { data } = await supabaseAdmin
@@ -315,7 +320,7 @@ async function improveSpendThisPeriod(orgId) {
.limit(5000);
const costs = (data ?? []).map((r) => Number(r.cost_usd) || 0);
const spent = costs.reduce((a, b) => a + b, 0);
- return { spent, typical: medianCost(costs.slice(0, 20)) };
+ return { spent, typical: medianCost(costs.slice(0, RECENT_CALLS)) };
}
/**
@@ -342,16 +347,114 @@ export async function getImproveQuotaStatus(orgId, modelId) {
};
}
-/** Throws PlanLimitError (429) once the period's Improve budget is spent. */
-export async function enforceImproveQuota(orgId, modelId) {
- const quota = await getImproveQuotaStatus(orgId, modelId);
- if (!quota.unlimited && quota.remainingUsd <= 0) {
- throw new PlanLimitError(
- `This month's suggestion budget on the ${quota.planName} plan is used up. It resets when your subscription renews.`,
- 429,
- );
+/**
+ * Holds a call's share of the workspace's AI budget before it runs (00243),
+ * so calls running at once can't all pass a nearly spent budget: the
+ * period's spend, reservations included, is read and the call's estimated
+ * cost written under a per-organization lock. A call runs while the spend
+ * is under the budget, as it did when the spend was only read; once it is
+ * spent, PlanLimitError (429).
+ *
+ * Returns the call's meter:
+ * - charge(usage): one model attempt at its real token cost. The first
+ * turns the reservation into it; any later one (a retry, a second call)
+ * adds its own row. Errors are logged, not thrown: what the user paid
+ * for still goes out.
+ * - release(): when nothing was charged (the call failed before an
+ * answer), the reservation goes. Safe to call more than once; call it
+ * in a finally.
+ * An unlimited budget (self-hosted) reserves nothing; charge() still writes
+ * the rows, as before.
+ */
+export async function reserveAiBudget(
+ orgId,
+ { feature, modelId, promptId = null, actionId = null, draftId = null },
+) {
+ const plan = await resolveOrgPlanByOrgId(orgId);
+ const budget = plan.limits.improveMonthlyBudgetUsd ?? -1;
+ let reservedId = null;
+ if (budget !== -1) {
+ const since = (await quotaPeriodStart(orgId)).toISOString();
+ const { data, error } = await supabaseAdmin.rpc('reserve_ai_usage', {
+ p_organization_id: orgId,
+ p_since: since,
+ p_budget: budget,
+ p_cost: await holdCostUsd(orgId, modelId, since),
+ p_feature: feature,
+ p_model: modelId,
+ p_prompt_id: promptId,
+ p_action_id: actionId,
+ p_content_draft_id: draftId,
+ });
+ if (error) throw new Error(error.message);
+ if (!data) {
+ throw new PlanLimitError(
+ `This month's suggestion budget on the ${plan.name} plan is used up. It resets when your subscription renews.`,
+ 429,
+ );
+ }
+ reservedId = data;
+ }
+
+ let charged = false;
+ return {
+ async charge(usage) {
+ const cost = {
+ model: modelId,
+ input_tokens: usage?.inputTokens ?? null,
+ output_tokens: usage?.outputTokens ?? null,
+ cost_usd: generationCostUsd(modelId, usage),
+ };
+ const first = reservedId && !charged;
+ charged = true;
+ const { error } = first
+ ? await supabaseAdmin.from('prompt_improve_usage').update(cost).eq('id', reservedId)
+ : await supabaseAdmin.from('prompt_improve_usage').insert({
+ organization_id: orgId,
+ feature,
+ prompt_id: promptId,
+ action_id: actionId,
+ content_draft_id: draftId,
+ ...cost,
+ });
+ if (error) logger.error({ err: error, orgId, feature }, '[ai-budget] usage write failed');
+ },
+ async release() {
+ if (!reservedId || charged) return;
+ charged = true;
+ const { error } = await supabaseAdmin
+ .from('prompt_improve_usage')
+ .delete()
+ .eq('id', reservedId);
+ if (error) logger.error({ err: error, orgId, feature }, '[ai-budget] release failed');
+ },
+ };
+}
+
+/**
+ * What a call holds before it runs: the org's median cost over its last
+ * RECENT_CALLS calls this period, never less than the model's typical one.
+ * Calls average about twice the typical estimate and reach five times it (a
+ * whole page, a long draft), so holding only the estimate let each call in
+ * flight pass the budget by the difference. Holds in flight count at their
+ * own size, so they don't raise the median. A read that fails holds the
+ * typical cost.
+ */
+async function holdCostUsd(orgId, modelId, since) {
+ const typical = typicalCostUsd(modelId);
+ const { data, error } = await supabaseAdmin
+ .from('prompt_improve_usage')
+ .select('cost_usd')
+ .eq('organization_id', orgId)
+ .gte('used_at', since)
+ .order('used_at', { ascending: false })
+ .limit(RECENT_CALLS);
+ if (error) {
+ logger.error({ err: error, orgId }, '[ai-budget] recent costs read failed');
+ return typical;
}
- return quota;
+ const recent = medianCost((data ?? []).map((r) => Number(r.cost_usd) || 0));
+ return Math.max(typical, recent ?? 0);
}
/** What the workspace's AI budget is spent on, as prompt_improve_usage.feature says (00242). */
@@ -395,8 +498,8 @@ export async function getAiUsage(orgId) {
return {
...base,
unlimited: false,
- // 100 only once it's spent, as enforceImproveQuota stops there (a budget
- // of 0 included); just under reads as 99.9, never rounded up to "used up".
+ // 100 only once it's spent, as reserveAiBudget stops there (a budget of
+ // 0 included); just under reads as 99.9, never rounded up to "used up".
percentUsed:
budget <= 0 || spent >= budget ? 100 : Math.min(99.9, tenths((spent * 100) / budget)),
features: [
diff --git a/server/src/routes/prompts-improve-send.test.js b/server/src/routes/prompts-improve-send.test.js
index 9bf20ab0a..47bd18b6a 100644
--- a/server/src/routes/prompts-improve-send.test.js
+++ b/server/src/routes/prompts-improve-send.test.js
@@ -49,7 +49,7 @@ vi.mock('../config/supabase.js', () => ({ default: { from: (table) => builder(ta
vi.mock('../lib/plan-guard.js', () => ({
PlanLimitError,
enforceFeature,
- enforceImproveQuota: vi.fn(),
+ reserveAiBudget: vi.fn(),
getImproveQuotaStatus: vi.fn(),
requireFeature: () => (req, res, next) => next(),
}));
diff --git a/server/src/routes/prompts-page-suggestions.test.js b/server/src/routes/prompts-page-suggestions.test.js
index 8cf5f2890..ec724c294 100644
--- a/server/src/routes/prompts-page-suggestions.test.js
+++ b/server/src/routes/prompts-page-suggestions.test.js
@@ -52,7 +52,7 @@ vi.mock('../config/supabase.js', () => ({ default: { from: (name) => table(name)
vi.mock('../lib/plan-guard.js', () => ({
PlanLimitError: class extends Error {},
enforceFeature: vi.fn(),
- enforceImproveQuota: vi.fn(),
+ reserveAiBudget: vi.fn(),
getImproveQuotaStatus: vi.fn(),
requireFeature: () => (req, res, next) => next(),
reserveRelevanceAllowance: vi.fn(),
diff --git a/server/src/routes/prompts-suggestions-refresh.test.js b/server/src/routes/prompts-suggestions-refresh.test.js
index 81505fde2..576cc78e1 100644
--- a/server/src/routes/prompts-suggestions-refresh.test.js
+++ b/server/src/routes/prompts-suggestions-refresh.test.js
@@ -64,7 +64,7 @@ vi.mock('../config/supabase.js', () => ({ default: { from: (name) => table(name)
vi.mock('../lib/plan-guard.js', () => ({
PlanLimitError: class extends Error {},
enforceFeature: vi.fn(),
- enforceImproveQuota: vi.fn(),
+ reserveAiBudget: vi.fn(),
getImproveQuotaStatus: vi.fn(),
requireFeature: () => (req, res, next) => next(),
reserveRelevanceAllowance,
diff --git a/server/src/routes/prompts-untracked-queries.test.js b/server/src/routes/prompts-untracked-queries.test.js
index 776e8e978..9c41014fd 100644
--- a/server/src/routes/prompts-untracked-queries.test.js
+++ b/server/src/routes/prompts-untracked-queries.test.js
@@ -35,7 +35,7 @@ vi.mock('../config/supabase.js', () => ({ default: { from: (name) => table(name)
vi.mock('../lib/plan-guard.js', () => ({
PlanLimitError: class extends Error {},
enforceFeature: vi.fn(),
- enforceImproveQuota: vi.fn(),
+ reserveAiBudget: vi.fn(),
getImproveQuotaStatus: vi.fn(),
requireFeature: () => (req, res, next) => next(),
reserveRelevanceAllowance: vi.fn(),
diff --git a/server/src/routes/prompts.js b/server/src/routes/prompts.js
index 1a2972a73..c739116be 100644
--- a/server/src/routes/prompts.js
+++ b/server/src/routes/prompts.js
@@ -8,9 +8,9 @@ import { getLanguageName } from '../lib/languages.js';
import {
PlanLimitError,
enforceFeature,
- enforceImproveQuota,
getImproveQuotaStatus,
requireFeature,
+ reserveAiBudget,
reserveRelevanceAllowance,
settleRelevanceUsage,
} from '../lib/plan-guard.js';
@@ -31,7 +31,6 @@ import {
improveSchema,
sanitizeSuggestions,
} from '../lib/prompt-improve.js';
-import { generationCostUsd } from '../lib/improve-cost.js';
import {
liveImproveSends,
sendImproveToActionCenter,
@@ -1004,23 +1003,6 @@ async function loadImproveStateRows({ promptId, brandId, range }) {
return rows.map((r) => ({ ...r, citations: citationsById.get(r.id) ?? [] }));
}
-/**
- * Charges one model attempt to the org's Improve budget. A failed insert is
- * logged, not thrown: the suggestions the user paid for still go out.
- */
-async function recordImproveUsage(log, { organizationId, promptId, modelId, usage }) {
- const { error } = await supabaseAdmin.from('prompt_improve_usage').insert({
- organization_id: organizationId,
- prompt_id: promptId,
- feature: 'improve',
- model: modelId,
- input_tokens: usage?.inputTokens ?? null,
- output_tokens: usage?.outputTokens ?? null,
- cost_usd: generationCostUsd(modelId, usage),
- });
- if (error) log.error({ err: error, promptId }, 'prompt improve usage insert error');
-}
-
function improveResultsQuery(columns, { promptId, brandId, range }) {
let q = supabaseAdmin
.from('prompt_results')
@@ -1166,41 +1148,45 @@ router.post('/:promptId/improve', requireFeature('prompt_improve'), async (req,
const modelId = improveModelId();
let suggestions = [];
// Only a model call is metered; a range with nothing to improve is free.
- if (withEvidence.length > 0) await enforceImproveQuota(brand.organization_id, modelId);
if (withEvidence.length > 0) {
- const { system, prompt: userPrompt } = buildImprovePrompt({
- promptText: prompt.text,
- brand: { name: brand.name, description: brand.description, domains: brandDomains },
- platforms: withEvidence,
+ // Held on the budget before the call, so calls at once can't pass it.
+ const budget = await reserveAiBudget(brand.organization_id, {
+ feature: 'improve',
+ modelId,
+ promptId,
});
- // Every attempt is charged at its real token cost, so the budget holds
- // for any model, including attempts withRetry repeats after a schema miss.
- const charge = (usage) =>
- recordImproveUsage(req.log, {
- organizationId: brand.organization_id,
- promptId,
- modelId,
- usage,
+ try {
+ const { system, prompt: userPrompt } = buildImprovePrompt({
+ promptText: prompt.text,
+ brand: { name: brand.name, description: brand.description, domains: brandDomains },
+ platforms: withEvidence,
});
- const { object } = await withRetry(
- async () => {
- try {
- const result = await generateObject({
- model: resolveModel(modelId),
- schema: improveSchema,
- system,
- prompt: userPrompt,
- });
- await charge(result.usage);
- return result;
- } catch (err) {
- if (NoObjectGeneratedError.isInstance(err) && err.usage) await charge(err.usage);
- throw err;
- }
- },
- { attempts: 3, baseDelayMs: 500, label: 'prompt-improve' },
- );
- suggestions = sanitizeSuggestions(object.suggestions, withEvidence);
+ // Every attempt is charged at its real token cost, so the budget holds
+ // for any model, including attempts withRetry repeats after a schema miss.
+ const { object } = await withRetry(
+ async () => {
+ try {
+ const result = await generateObject({
+ model: resolveModel(modelId),
+ schema: improveSchema,
+ system,
+ prompt: userPrompt,
+ });
+ await budget.charge(result.usage);
+ return result;
+ } catch (err) {
+ if (NoObjectGeneratedError.isInstance(err) && err.usage) {
+ await budget.charge(err.usage);
+ }
+ throw err;
+ }
+ },
+ { attempts: 3, baseDelayMs: 500, label: 'prompt-improve' },
+ );
+ suggestions = sanitizeSuggestions(object.suggestions, withEvidence);
+ } finally {
+ await budget.release();
+ }
}
// What each platform's group header shows: why it's there and which
diff --git a/supabase/migrations/00243_ai_usage_reservation.sql b/supabase/migrations/00243_ai_usage_reservation.sql
new file mode 100644
index 000000000..2c5bb68fb
--- /dev/null
+++ b/supabase/migrations/00243_ai_usage_reservation.sql
@@ -0,0 +1,65 @@
+-- A call's share of the workspace's AI budget, held before the call runs.
+--
+-- The budget (plan.limits.improveMonthlyBudgetUsd: Improve, the Content
+-- Editor's AI, the Action Center's task writer) was checked by reading the
+-- period's spend, and each call's cost written once the model answered. Calls
+-- running at once for one workspace (two tabs, a teammate, an Action Center
+-- sweep over several actions) all read the same spend and all ran, so the
+-- budget could be passed by as many calls as were in flight.
+--
+-- reserve_ai_usage takes a per-organization lock, reads the period's spend
+-- (reservations included), and, while it is under the budget, writes a row
+-- for the call at its estimated cost and returns its id. The server turns
+-- that row into the call's real cost once the model answers, or deletes it
+-- when nothing was charged. The rule stays the one before: a call runs while
+-- the spend is under the budget; only now the calls in flight count too.
+
+create or replace function public.reserve_ai_usage(
+ p_organization_id uuid,
+ p_since timestamptz,
+ p_budget numeric,
+ p_cost numeric,
+ p_feature text,
+ p_model text,
+ p_prompt_id uuid default null,
+ p_action_id uuid default null,
+ p_content_draft_id uuid default null
+)
+returns uuid
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+ v_spent numeric;
+ v_id uuid;
+begin
+ -- One reservation at a time per organization: the next waits and sees this one.
+ perform pg_advisory_xact_lock(hashtext('ai_usage:' || p_organization_id::text));
+
+ select coalesce(sum(cost_usd), 0) into v_spent
+ from public.prompt_improve_usage
+ where organization_id = p_organization_id
+ and used_at >= p_since;
+
+ if v_spent >= p_budget then
+ return null;
+ end if;
+
+ insert into public.prompt_improve_usage (
+ organization_id, feature, model, prompt_id, action_id, content_draft_id, cost_usd)
+ values (
+ p_organization_id, p_feature, p_model, p_prompt_id, p_action_id, p_content_draft_id,
+ greatest(p_cost, 0))
+ returning id into v_id;
+ return v_id;
+end;
+$$;
+
+comment on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid) is
+ 'Holds an AI call''s estimated cost on its organization''s budget under a per-organization lock, while the period''s spend since p_since is under p_budget; returns the usage row to settle (reserveAiBudget), or null when the budget is spent.';
+
+revoke all on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid)
+ from public, anon, authenticated;
+grant execute on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid)
+ to service_role;
diff --git a/supabase/schema.sql b/supabase/schema.sql
index 86c9651ad..13d9ec64a 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -36427,3 +36427,72 @@ comment on function public.ai_usage_by_feature(uuid, timestamptz) is
revoke all on function public.ai_usage_by_feature(uuid, timestamptz) from public, anon, authenticated;
grant execute on function public.ai_usage_by_feature(uuid, timestamptz) to service_role;
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00243_ai_usage_reservation.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- A call's share of the workspace's AI budget, held before the call runs.
+--
+-- The budget (plan.limits.improveMonthlyBudgetUsd: Improve, the Content
+-- Editor's AI, the Action Center's task writer) was checked by reading the
+-- period's spend, and each call's cost written once the model answered. Calls
+-- running at once for one workspace (two tabs, a teammate, an Action Center
+-- sweep over several actions) all read the same spend and all ran, so the
+-- budget could be passed by as many calls as were in flight.
+--
+-- reserve_ai_usage takes a per-organization lock, reads the period's spend
+-- (reservations included), and, while it is under the budget, writes a row
+-- for the call at its estimated cost and returns its id. The server turns
+-- that row into the call's real cost once the model answers, or deletes it
+-- when nothing was charged. The rule stays the one before: a call runs while
+-- the spend is under the budget; only now the calls in flight count too.
+
+create or replace function public.reserve_ai_usage(
+ p_organization_id uuid,
+ p_since timestamptz,
+ p_budget numeric,
+ p_cost numeric,
+ p_feature text,
+ p_model text,
+ p_prompt_id uuid default null,
+ p_action_id uuid default null,
+ p_content_draft_id uuid default null
+)
+returns uuid
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+ v_spent numeric;
+ v_id uuid;
+begin
+ -- One reservation at a time per organization: the next waits and sees this one.
+ perform pg_advisory_xact_lock(hashtext('ai_usage:' || p_organization_id::text));
+
+ select coalesce(sum(cost_usd), 0) into v_spent
+ from public.prompt_improve_usage
+ where organization_id = p_organization_id
+ and used_at >= p_since;
+
+ if v_spent >= p_budget then
+ return null;
+ end if;
+
+ insert into public.prompt_improve_usage (
+ organization_id, feature, model, prompt_id, action_id, content_draft_id, cost_usd)
+ values (
+ p_organization_id, p_feature, p_model, p_prompt_id, p_action_id, p_content_draft_id,
+ greatest(p_cost, 0))
+ returning id into v_id;
+ return v_id;
+end;
+$$;
+
+comment on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid) is
+ 'Holds an AI call''s estimated cost on its organization''s budget under a per-organization lock, while the period''s spend since p_since is under p_budget; returns the usage row to settle (reserveAiBudget), or null when the budget is spent.';
+
+revoke all on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid)
+ from public, anon, authenticated;
+grant execute on function public.reserve_ai_usage(uuid, timestamptz, numeric, numeric, text, text, uuid, uuid, uuid)
+ to service_role;
+