Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: price Grok usage at published xAI rates (#13426)

[._.]/ce7db9953468
diff --git a/client/src/pages/UsagePage.jsx b/client/src/pages/UsagePage.jsx
index 4fdbabcfe4..2c3155f9c6 100644
--- a/client/src/pages/UsagePage.jsx
+++ b/client/src/pages/UsagePage.jsx
@@ -24,6 +24,23 @@ import { USAGE_PERIOD_OPTIONS, DEFAULT_USAGE_PERIOD } from '../lib/usagePeriods'
const BACKFILL_EVENTS = ['usage-backfill:updated'];
+function joinAnd(parts) {
+ if (parts.length <= 1) return parts[0] || '';
+ if (parts.length === 2) return `${parts[0]} and ${parts[1]}`;
+ return `${parts.slice(0, -1).join(', ')}, and ${parts.at(-1)}`;
+}
+
+/** What Reconcile now added to the Grok ledger. Absent until that job finishes. */
+function grokRepairLine(grok) {
+ if (!grok) return '';
+ const parts = [];
+ if (grok.turns) parts.push(`${formatCount(grok.turns)} long-context turn${grok.turns === 1 ? '' : 's'}`);
+ if (grok.images) parts.push(`${formatCount(grok.images)} image${grok.images === 1 ? '' : 's'}`);
+ if (grok.videos) parts.push(`${formatCount(grok.videos)} video${grok.videos === 1 ? '' : 's'}`);
+ const added = parts.length ? `Added ${joinAnd(parts)}.` : 'Grok usage was already recorded.';
+ return grok.note ? `${added} ${grok.note}` : added;
+}
+
// A subscription is one account across every federated instance, but each
// instance can only read its own CLI's panel. When peers have contributed a
// reading, say so on the card and name them — the meters are the freshest
@@ -304,6 +321,9 @@ function CostReportTable({ report }) {
<span>{formatNumber(m.tokensIn)} in · {formatNumber(m.tokensOut)} out</span>
<span>Cache: {formatNumber((m.cacheReadTokens ?? 0) + (m.cacheWriteTokens ?? 0))}</span>
</div>
+ {usageExtras(m) && (
+ <div className="text-[10px] text-gray-500">{usageExtras(m)}</div>
+ )}
</div>
))}
</div>
@@ -358,6 +378,18 @@ function CostReportTable({ report }) {
const cacheTitle = (row) =>
`Cache read ${formatCount(row.cacheReadTokens, { fallback: '0' })} · cache write ${formatCount(row.cacheWriteTokens, { fallback: '0' })} tokens`;
+// Imagine units and the 200k subset are not visible in the token columns.
+function usageExtras(row) {
+ if (!row) return '';
+ const parts = [];
+ if (row.images > 0) parts.push(`${formatNumber(row.images)} images`);
+ if (row.inputImages > 0) parts.push(`${formatNumber(row.inputImages)} input images`);
+ if (row.videoSeconds > 0) parts.push(`${formatNumber(row.videoSeconds)}s video`);
+ const longTokens = (row.longTokensIn || 0) + (row.longCacheReadTokens || 0) + (row.longCacheWriteTokens || 0);
+ if (longTokens > 0) parts.push(`${formatNumber(longTokens)} long-context`);
+ return parts.join(' · ');
+}
+
function ProviderCostRows({ provider }) {
return (
<>
@@ -390,6 +422,9 @@ function ProviderCostRows({ provider }) {
: undefined}
>
{m.model}
+ {usageExtras(m) && (
+ <div className="font-sans text-[10px] text-gray-500 truncate">{usageExtras(m)}</div>
+ )}
</td>
<td className="py-1.5 px-2 text-right text-xs">{formatNumber(m.sessions)}</td>
<td className="py-1.5 px-2 text-right text-xs" title={cacheTitle(m)}>{formatNumber(m.tokensIn)}</td>
@@ -675,7 +710,7 @@ function InternalUsageMetrics() {
Reconcile historical usage
</div>
<p className="text-xs text-gray-500 mt-1">
- Re-read PortOS run transcripts to replace older token estimates. This changes recorded history but makes no provider calls.
+ Re-read saved transcripts and Grok sessions to replace older estimates and record Imagine and long-context charges. This changes recorded history but makes no provider calls.
</p>
{backfill?.status === 'running' && (
<p className="text-xs text-port-accent mt-1" role="status">
@@ -687,6 +722,9 @@ function InternalUsageMetrics() {
Corrected {backfill.corrected || 0} run{backfill.corrected === 1 ? '' : 's'}.
</p>
)}
+ {backfill?.grok && (backfill.status === 'complete' || backfill.status === 'error') && (
+ <p className="text-xs text-port-success mt-1" role="status">{grokRepairLine(backfill.grok)}</p>
+ )}
{backfill?.status === 'error' && (
<p className="text-xs text-port-error mt-1" role="alert">{backfill.error}</p>
)}
diff --git a/client/src/pages/UsagePage.test.jsx b/client/src/pages/UsagePage.test.jsx
index 79843a389f..0e3c673fe2 100644
--- a/client/src/pages/UsagePage.test.jsx
+++ b/client/src/pages/UsagePage.test.jsx
@@ -138,6 +138,36 @@ describe('UsagePage historical reconciliation', () => {
expect(api.startUsageBackfill).toHaveBeenCalledTimes(1);
});
+ it('shows the Grok rows reconcile added', async () => {
+ api.startUsageBackfill.mockResolvedValue({
+ status: 'complete',
+ corrected: 2,
+ grok: {
+ added: 3,
+ turns: 1,
+ images: 2,
+ videos: 0,
+ backedUp: true,
+ note: 'Long-context premiums were not added because no Grok usage scan has run yet.',
+ },
+ });
+ render(<MemoryRouter><UsagePage /></MemoryRouter>);
+ fireEvent.click(await screen.findByRole('button', { name: 'Reconcile now' }));
+ expect(await screen.findByText('Corrected 2 runs.')).toBeInTheDocument();
+ expect(await screen.findByText('Added 1 long-context turn and 2 images. Long-context premiums were not added because no Grok usage scan has run yet.')).toBeInTheDocument();
+ });
+
+ it('says when reconcile found the Grok ledger already repaired', async () => {
+ api.startUsageBackfill.mockResolvedValue({
+ status: 'complete',
+ corrected: 0,
+ grok: { added: 0, turns: 0, images: 0, videos: 0, backedUp: false, note: null },
+ });
+ render(<MemoryRouter><UsagePage /></MemoryRouter>);
+ fireEvent.click(await screen.findByRole('button', { name: 'Reconcile now' }));
+ expect(await screen.findByText('Grok usage was already recorded.')).toBeInTheDocument();
+ });
+
it('starts only from the explicit user action and reports completion', async () => {
render(<MemoryRouter><UsagePage /></MemoryRouter>);
@@ -181,6 +211,8 @@ describe('UsagePage breakdown visualization and responsiveness', () => {
tokensOut: 3000,
cacheReadTokens: 1000,
cacheWriteTokens: 200,
+ images: 2,
+ videoSeconds: 6,
estimatedCost: 1.5
}
]
@@ -202,6 +234,8 @@ describe('UsagePage breakdown visualization and responsiveness', () => {
const costDisplays = screen.getAllByText('$1.50');
expect(costDisplays.length).toBeGreaterThan(0);
+ expect(screen.getAllByText(/2 images/).length).toBeGreaterThan(0);
+ expect(screen.getAllByText(/6s video/).length).toBeGreaterThan(0);
});
});
diff --git a/scripts/backfill-grok-usage.js b/scripts/backfill-grok-usage.js
new file mode 100644
index 0000000000..28c1b3c5f0
--- /dev/null
+++ b/scripts/backfill-grok-usage.js
@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+/**
+ * Dry-run, or stopped-server, wrapper around the Grok usage repair.
+ *
+ * While PortOS is running, use Reconcile now on the Usage page. That path
+ * writes the server's in-memory ledger, so the next save cannot drop the rows.
+ * This script prints the same plan and, with --apply, calls that same repair
+ * only when this process is the one that owns data/usage.json. Stop PortOS
+ * first. A running server keeps the ledger in memory and will overwrite a
+ * file this process saved.
+ *
+ * node scripts/backfill-grok-usage.js
+ * node scripts/backfill-grok-usage.js --home /path/to/home --data /path/to/data
+ * node scripts/backfill-grok-usage.js --apply
+ *
+ * Cache rates re-price when the Usage page is built. This script does not
+ * rewrite those tokens. It also cannot recover past HTTP calls that never
+ * stored prompt size, cache, or reasoning; the 200k tier for chars/4
+ * estimates; the subscription allowance percent; or Imagine resolution above
+ * the default tier.
+ */
+
+import { homedir } from 'os';
+import { dirname, join, resolve } from 'path';
+import { fileURLToPath } from 'url';
+import { PATHS } from '../server/lib/fileUtils.js';
+import { applyGrokUsageRepair, planGrokUsageRepairFromDisk } from '../server/services/grokUsageBackfill.js';
+import { summarizeGrokRepairAdditions } from '../server/services/grokUsageRepair.js';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+
+function argValue(argv, name) {
+ const index = argv.indexOf(name);
+ return index >= 0 ? argv[index + 1] : null;
+}
+
+function printPlan(plan) {
+ const counts = summarizeGrokRepairAdditions(plan.additions);
+ console.log(`Grok usage repair: ${counts.added} addition(s).`);
+ for (const item of plan.additions) {
+ console.log(` ${item.day} ${item.repairId} model=${item.record.model || '(default)'} images=${item.record.images || 0} videoSeconds=${item.record.videoSeconds || 0} longIn=${item.record.longTokensIn || 0}`);
+ }
+ console.log(`Summary: ${counts.turns} long-context turn(s), ${counts.images} image row(s), ${counts.videos} video row(s).`);
+ for (const note of plan.notes) console.log(`Note: ${note}`);
+ console.log('Not recoverable: past HTTP calls that never stored prompt size, cache, or reasoning; chars/4 estimates; the subscription allowance percent; Imagine resolution above the default tier.');
+ console.log('Cache rates re-price themselves when you reload the Usage page. This script does not rewrite those tokens.');
+}
+
+async function main() {
+ const argv = process.argv.slice(2);
+ const apply = argv.includes('--apply');
+ const home = argValue(argv, '--home') || homedir();
+ const dataDir = argValue(argv, '--data') || join(ROOT, 'data');
+ if (!apply) {
+ printPlan(await planGrokUsageRepairFromDisk({ home, dataDir }));
+ console.log('Dry run. Reconcile now on the Usage page applies this while PortOS is running. --apply writes the ledger only after PortOS is stopped.');
+ return;
+ }
+ if (resolve(dataDir) !== resolve(PATHS.data)) {
+ console.error(`❌ --apply writes ${PATHS.data}/usage.json. Re-run without --data, or pass --data ${PATHS.data}.`);
+ process.exitCode = 1;
+ return;
+ }
+ const result = await applyGrokUsageRepair({ home, dataDir });
+ console.log(`Applied ${result.added} addition(s): ${result.turns} long-context turn(s), ${result.images} image row(s), ${result.videos} video row(s).`);
+ if (result.note) console.log(`Note: ${result.note}`);
+ if (result.backedUp) console.log(`Backed up the ledger to ${result.backupDir}`);
+ if (result.added === 0) console.log('Nothing to apply.');
+}
+
+main().catch((err) => {
+ console.error(`❌ Grok usage backfill failed: ${err.message}`);
+ process.exitCode = 1;
+});
diff --git a/server/lib/README.md b/server/lib/README.md
index c1a8342deb..c77f60af02 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -300,7 +300,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `modelPinMembership.js` | THE model-pin membership rule, and the local-daemon classification it rests on — `modelPinIsOffered(provider, model)` (may this provider be handed this model? a configured-default sentinel, an empty catalog and a local daemon's cached snapshot are all pass-throughs, so a freshly pulled Ollama model is never rejected as "not offered"; beyond those the match is exact, with two same-model-spelled-differently tolerances: agy compares on BASE ids, since `--effort` carries the tier and the spawner splits a suffixed pin into exactly that, and OpenCode's `namespace/model` form matches a bare pin, since pins are stored bare and namespaced at spawn), plus `localRuntimeKind(provider)` (the `*Backed` markers first, then the endpoint/name classifier; `orcarouter` excluded as a remote API) and `localBackendForProvider`. Split out of `localProviderRuntime.js` — which re-exports all three, so existing importers are untouched — because the browser needs this rule for a picker and must not pull that module's endpoint graph (`opencodeConfig.js` → `zod`, not a client dependency) to get it. Carries `LOCAL_RUNTIME_MANAGE_URLS` for the same reason (#7414): the route that manages each local runtime — Models → Runtimes for the servers, Models → LLMs for the weights catalog, `null` for the operator-owned compose projects — read by `LOCAL_RUNTIMES[*].manageUrl` and by the client surfaces that must name the same page. Consumed by `cliProviderRun.js`, `routes/brainSettings.js`, `services/agentProviderResolution.js`, `modelPinReconcile.js` and `client/src/utils/providerModels.js` — one answer for all five (#7327). |
| `modelPinReconcile.js` | Retired-model-pin AUDIT over the shared membership rule (#7315) — `providerCatalogListsModel(provider, modelId)` (is a stored pin still served? `modelPinIsOffered` from `modelPinMembership.js` verbatim, plus the one case a retirement check answers differently from a spawn-time one: nothing is pinned at all. The agy base-id and OpenCode namespace tolerances used to be layered HERE above a stricter rule, which made the audit and the three spawn-time callers disagree about the same pin; they moved down into the rule in #7327), `catalogOfferings` (the listed ids with sentinels dropped, agy collapsed to base ids for a picker), `reconcileModelPins(pins, providers)` — which pin descriptors name a model their provider no longer lists — and, since #7339, the two helpers that make a pin's provider field singular-or-plural-agnostic: `pinProviderIds(pin)` (THE coercion from `providerId` / `providerIds` to a trimmed id list, called by the audit's descriptor builder too so the two layers cannot drift) and `pinProviderNames(pin, providers)` (how a surface names those records — shared by the warnings panel and the notification card so they cannot describe one pin differently). A reviewer pin names a BINARY several records front, so it is stale only when EVERY named record stops listing the model. Every UNKNOWN case (no pin, empty catalog, unresolved provider) answers "still listed": a false "your pin is gone" tells the user to change a setting that works. Derived on read, never persisted — see `services/modelPinAudit.js`. |
| `reviewerProviderMatchers.js` | Which PROVIDER RECORDS front each model-taking reviewer's binary — `REVIEWER_PROVIDER_MATCHERS` (slug → record predicates, in preference order), `providersForReviewer(reviewer, providers)` (every match, de-duped, preference-ordered so `[0]` is the record whose default a picker shows) and `reviewerProviderIds`. A reviewer slug names a BINARY and PortOS ships more than one record per binary (`claude` spans `claude-code` + `claude-code-tui`, `grok` spans `grok-cli` + its TUI), so both the picker's option list and the retired-pin audit union every match: sourcing from one record makes the answer hostage to that record's staleness, and judging a pin against one produces a false retirement for a model the other still lists. Deliberately excludes Bedrock/Vertex and the `opencode-<backend>` presets (ids that resolve only under another record's environment). Imported directly by `client/src/hooks/useReviewerModelOptions.js` — one table, no mirror to drift (#7339). |
-| `modelPricing.js` | Per-model API billing rates for informational cost estimates — `resolveModelRates(providerId, model)` (exact → family regex → provider default → blended fallback, with a `matched` tier; also derives cache tiers), `pricingAsOfForModel(model)`, `isFreeProvider`, `estimateCostUsd(tokensIn, tokensOut, rates, cache?)` (uncached input plus separately priced cache tokens), and baseline `PRICING_AS_OF`. Subscription allowance and local inference bills remain unknown; batch/long-context tiers are not modeled. |
+| `modelPricing.js` | Per-model API billing rates for informational cost estimates — `resolveModelRates(providerId, model)` (exact → family regex → provider default → blended fallback, with a `matched` tier; also derives cache tiers), `pricingAsOfForModel(model)`, `isFreeProvider`, `estimateCostUsd(tokensIn, tokensOut, rates, cache?)` (uncached input plus separately priced cache tokens), `estimateGrokImagineCost(model, units)` (published Imagine image / second / input-image rates; longest model id wins), `GROK_LONG_CONTEXT_PROMPT_TOKENS` (200k — the usage report adds one extra base-rate charge for that per-request subset), and baseline `PRICING_AS_OF`. Subscription allowance and local inference bills remain unknown; batch discounts and non-Grok long-context tiers are not modeled. |
+| `chatCompletionUsage.js` | `normalizeChatCompletionUsage(usage, { model, endpoint })` splits an OpenAI-compatible chat-completions usage body. Cached tokens are removed from fresh input. xAI (`api.x.ai`, `us.api.x.ai`, or a `grok*` model when the endpoint has no host) adds `reasoning_tokens` to output; every other host keeps output as `completion_tokens` only, even for a grok-named model. A prompt at `GROK_LONG_CONTEXT_PROMPT_TOKENS` sets `longContext` for xAI only. |
| `usageRange.js` | `resolveUsageRange({ period, from, to })` — pure period→inclusive-YYYY-MM-DD range resolution for the usage cost report (explicit dates win; `all` unbounded; default 7d). |
| `subscriptionPlanTiers.js` | Which TIER of a subscription plan the user is on ("Max 20x", "Pro"), as a short free-text label per provider family — `MAX_PLAN_TIER_LENGTH` (the cap the Zod schema enforces), `normalizePlanTier` / `normalizePlanTiers` (trim, cap, and collapse empty/non-string to `null` = not recorded). Deliberately free text: tiers are vendor marketing names that change off the PortOS release cycle. Pure; persistence lives in `services/subscriptions.js`. |
| `subscriptionSavings.js` | Subscription-vs-API savings math for the usage page — `resolveSavingsWindow` (clamps an open-ended report range to today / first activity day), `prorateMonthlyCost` (monthly plan price → this window's share, `DAYS_PER_MONTH`, capped by `MAX_MONTHLY_COST`), `savingsPercent` / `costMultiplier` (null, never 0, when the comparison is undefined), `attributeReportCostToFamilies` (groups report rows by their stamped `family`), `roundCents` (the one money rounder), and `buildSubscriptionSavings({ entries, range, unmatchedApiCost })` → per-family rows + totals. Pure. |
diff --git a/server/lib/chatCompletionUsage.js b/server/lib/chatCompletionUsage.js
new file mode 100644
index 0000000000..ddbf9d3faa
--- /dev/null
+++ b/server/lib/chatCompletionUsage.js
@@ -0,0 +1,70 @@
+/**
+ * OpenAI-compatible `usage` objects do not all include tokens the same way.
+ *
+ * xAI's chat-completions example is prompt_tokens 32 + completion_tokens 9 +
+ * reasoning_tokens 94 = total_tokens 135, so reasoning is OUTSIDE completion
+ * and has to be added. OpenAI includes reasoning inside completion_tokens;
+ * adding it there would bill it twice. Cached tokens are a subset of
+ * prompt_tokens on both (xAI's prompt-caching guide: a later turn can report
+ * prompt 120 of which 50 are cached).
+ *
+ * A parsed endpoint host wins over the model id. `api.x.ai` and `us.api.x.ai`
+ * are xAI. Any other host is not, even when the model id starts with `grok`
+ * (a proxy in front of a different vendor). With no parseable endpoint, a
+ * `grok*` model is xAI. A provider id alone is not a signal.
+ */
+
+import { GROK_LONG_CONTEXT_PROMPT_TOKENS } from './modelPricing.js';
+
+const XAI_HOSTS = new Set(['api.x.ai', 'us.api.x.ai']);
+
+const num = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : 0);
+
+function endpointHost(endpoint) {
+ if (typeof endpoint !== 'string' || !endpoint) return '';
+ try {
+ return new URL(endpoint).hostname.toLowerCase();
+ } catch {
+ return '';
+ }
+}
+
+function isXaiChatUsage({ endpoint, model } = {}) {
+ const host = endpointHost(endpoint);
+ if (host) return XAI_HOSTS.has(host);
+ return /^grok/i.test(String(model || '').trim());
+}
+
+/**
+ * Split one chat-completions `usage` body into the ledger's buckets.
+ * @param {object|null|undefined} usage
+ * @param {{ model?: string, endpoint?: string }} [context]
+ * @returns {{ inputTokens: number, outputTokens: number, cacheReadTokens: number,
+ * cacheWriteTokens: number, promptTokens: number, longContext: boolean,
+ * source: 'measured' }|null} null when the body carries no finite counts.
+ */
+export function normalizeChatCompletionUsage(usage, { model, endpoint } = {}) {
+ if (!usage || typeof usage !== 'object') return null;
+ const promptField = usage.prompt_tokens;
+ const completionField = usage.completion_tokens;
+ const cachedField = usage.prompt_tokens_details?.cached_tokens ?? usage.cached_tokens;
+ const reasoningField = usage.completion_tokens_details?.reasoning_tokens ?? usage.reasoning_tokens;
+ const present = [promptField, completionField, cachedField, reasoningField]
+ .some((value) => typeof value === 'number' && Number.isFinite(value));
+ if (!present) return null;
+
+ const prompt = num(promptField);
+ const completion = num(completionField);
+ const cached = Math.min(prompt, Math.max(0, num(cachedField)));
+ const reasoning = Math.max(0, num(reasoningField));
+ const xai = isXaiChatUsage({ endpoint, model });
+ return {
+ inputTokens: Math.max(0, prompt - cached),
+ outputTokens: xai ? completion + reasoning : completion,
+ cacheReadTokens: cached,
+ cacheWriteTokens: 0,
+ promptTokens: prompt,
+ longContext: xai && prompt >= GROK_LONG_CONTEXT_PROMPT_TOKENS,
+ source: 'measured',
+ };
+}
diff --git a/server/lib/chatCompletionUsage.test.js b/server/lib/chatCompletionUsage.test.js
new file mode 100644
index 0000000000..990568d2c4
--- /dev/null
+++ b/server/lib/chatCompletionUsage.test.js
@@ -0,0 +1,57 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeChatCompletionUsage } from './chatCompletionUsage.js';
+
+const xaiExample = {
+ prompt_tokens: 32,
+ completion_tokens: 9,
+ total_tokens: 135,
+ prompt_tokens_details: { cached_tokens: 6 },
+ completion_tokens_details: { reasoning_tokens: 94 },
+};
+
+describe('normalizeChatCompletionUsage', () => {
+ it('splits xAI cached input out of the prompt and adds reasoning on top of completion', () => {
+ const usage = normalizeChatCompletionUsage(xaiExample, {
+ model: 'grok-4.7', endpoint: 'https://api.x.ai/v1',
+ });
+ expect(usage).toMatchObject({
+ inputTokens: 26,
+ cacheReadTokens: 6,
+ outputTokens: 103,
+ longContext: false,
+ source: 'measured',
+ });
+ });
+
+ it('keeps the published 32 + 9 + 94 = 135 total when nothing was cached', () => {
+ const usage = normalizeChatCompletionUsage({
+ prompt_tokens: 32, completion_tokens: 9, completion_tokens_details: { reasoning_tokens: 94 },
+ }, { endpoint: 'https://us.api.x.ai/v1' });
+ expect(usage.inputTokens + usage.cacheReadTokens + usage.outputTokens).toBe(135);
+ expect(usage.outputTokens).toBe(103);
+ });
+
+ it('does not add reasoning for a non-xAI host, even when the model id says grok', () => {
+ const usage = normalizeChatCompletionUsage(xaiExample, {
+ model: 'grok-4.7', endpoint: 'https://api.example.com/v1',
+ });
+ expect(usage.outputTokens).toBe(9);
+ expect(usage.inputTokens).toBe(26);
+ expect(usage.longContext).toBe(false);
+ });
+
+ it('sets the long-context subset only for an xAI prompt of at least 200k tokens', () => {
+ const body = { prompt_tokens: 200_000, completion_tokens: 10, completion_tokens_details: { reasoning_tokens: 5 } };
+ expect(normalizeChatCompletionUsage(body, { model: 'grok-4.7' }).longContext).toBe(true);
+ expect(normalizeChatCompletionUsage(body, { endpoint: 'https://api.openai.com/v1', model: 'gpt-5.4' }).longContext).toBe(false);
+ expect(normalizeChatCompletionUsage(
+ { prompt_tokens: 199_999, completion_tokens: 1 },
+ { endpoint: 'https://api.x.ai/v1' },
+ ).longContext).toBe(false);
+ });
+
+ it('returns null when the body has no finite counts', () => {
+ expect(normalizeChatCompletionUsage(null, {})).toBeNull();
+ expect(normalizeChatCompletionUsage({}, { model: 'grok-4.7' })).toBeNull();
+ });
+});
diff --git a/server/lib/grokVideoClip.js b/server/lib/grokVideoClip.js
index 1984299b24..5f6e806ee4 100644
--- a/server/lib/grokVideoClip.js
+++ b/server/lib/grokVideoClip.js
@@ -81,3 +81,25 @@ export const nearestGrokDuration = (requested) => {
if (!Number.isFinite(n) || n <= 0) return GROK_VIDEO_DEFAULT_DURATION;
return GROK_VIDEO_DURATIONS.find((d) => d >= n) ?? GROK_VIDEO_DURATIONS[GROK_VIDEO_DURATIONS.length - 1];
};
+
+/**
+ * Billable units for one accepted Grok video.
+ *
+ * Text-to-video has no source image: the agent generates a base still
+ * (`images: 1`) and then feeds that still to `image_to_video` (`inputImages: 1`).
+ * Image-to-video already has a still, so it bills the delivered seconds plus
+ * that one input image and does not bill a second generated image.
+ * Delivered length wins; otherwise the length grok was asked for.
+ * @param {{ requestedSeconds?: number|string, deliveredSeconds?: number|null, hasSourceImage?: boolean }} [opts]
+ * @returns {{ videoSeconds: number, images: number, inputImages: number }}
+ */
+export function grokVideoBillableUnits({ requestedSeconds, deliveredSeconds, hasSourceImage = false } = {}) {
+ const delivered = typeof deliveredSeconds === 'number' && Number.isFinite(deliveredSeconds) && deliveredSeconds > 0
+ ? deliveredSeconds
+ : null;
+ return {
+ videoSeconds: delivered ?? resolveGrokDuration(requestedSeconds),
+ images: hasSourceImage ? 0 : 1,
+ inputImages: 1,
+ };
+}
diff --git a/server/lib/grokVideoClip.test.js b/server/lib/grokVideoClip.test.js
index e70a542096..748f917299 100644
--- a/server/lib/grokVideoClip.test.js
+++ b/server/lib/grokVideoClip.test.js
@@ -1,6 +1,7 @@
import { describe, it, expect } from 'vitest';
import {
GROK_VIDEO_DURATIONS, GROK_VIDEO_DEFAULT_DURATION, resolveGrokDuration, nearestGrokDuration,
+ grokVideoBillableUnits,
} from './grokVideoClip.js';
describe('grokVideoClip', () => {
@@ -81,6 +82,18 @@ describe('nearestGrokDuration — translating another backend\'s continuous leng
});
});
+describe('grokVideoBillableUnits', () => {
+ it('bills a text-to-video base image plus the delivered seconds and one input image', () => {
+ expect(grokVideoBillableUnits({ requestedSeconds: 6, deliveredSeconds: 6.04, hasSourceImage: false }))
+ .toEqual({ videoSeconds: 6.04, images: 1, inputImages: 1 });
+ });
+
+ it('does not bill a second generated image when the caller supplied the still', () => {
+ expect(grokVideoBillableUnits({ requestedSeconds: 10, deliveredSeconds: null, hasSourceImage: true }))
+ .toEqual({ videoSeconds: 10, images: 0, inputImages: 1 });
+ });
+});
+
describe('grokVideoClip client mirror', () => {
it('matches client/src/lib/grokVideoClip.js', async () => {
// The walk workflow's Clip picker builds its options from the client mirror,
diff --git a/server/lib/index.js b/server/lib/index.js
index b81474db66..3e9b206ac1 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -602,6 +602,7 @@ export * from './migrationMarker.js';
export * from './modelPinMembership.js';
export * from './modelPinReconcile.js';
export * from './modelPricing.js';
+export * from './chatCompletionUsage.js';
export * from './navManifest.js';
export * from './noChangeCompletion.js';
export * from './noReplaceMove.js';
diff --git a/server/lib/modelPricing.js b/server/lib/modelPricing.js
index daa17a836f..3de84ee926 100644
--- a/server/lib/modelPricing.js
+++ b/server/lib/modelPricing.js
@@ -8,8 +8,11 @@
* the prompt-cache tiers are derived from the input rate by the multipliers in
* `CACHE_MULTIPLIERS` (every vendor publishes caching as a fixed ratio of the
* base input rate, so one multiplier per family beats 4 hand-maintained numbers
- * per model). Batch and long-context tier discounts are still ignored, and the
- * UI discloses the approximation.
+ * per model, except Grok rows whose cached-input price is not that ratio).
+ * Batch discounts are still ignored. Grok's ≥200k doubling is NOT a second
+ * rate in this table: it is a per-request subset the usage report prices once
+ * more at these base rates (see GROK_LONG_CONTEXT_PROMPT_TOKENS). The UI
+ * discloses the approximation.
*
* Cache tiers are NOT a rounding detail: for agentic CLI use, cache reads are
* >90% of input volume, so pricing them at the standard input rate (or, as
@@ -45,8 +48,70 @@ const MODEL_RATE_AS_OF = Object.freeze({
'gpt-6-luna': '2026-09-22',
'gpt-5.6-sol': '2026-09-22',
'gpt-5.6-luna': '2026-09-22',
+ // https://docs.x.ai/docs/models — text cache rates and Imagine headline tiers.
+ 'grok-4.7': '2026-10-24',
+ 'grok-4.6': '2026-10-24',
+ 'grok-4.5': '2026-10-24',
+ 'grok-4.3': '2026-10-24',
+ 'grok-4.20': '2026-10-24',
+ 'grok-build-0.1': '2026-10-24',
+ 'grok-imagine-image': '2026-10-24',
+ 'grok-imagine-image-2.0': '2026-10-24',
+ 'grok-imagine-image-quality': '2026-10-24',
+ 'grok-imagine-video': '2026-10-24',
+ 'grok-imagine-video-1.5': '2026-10-24',
+ 'grok-imagine-video-1.5-lite': '2026-10-24',
});
+/**
+ * xAI doubles every token price, including cached input, on a request whose
+ * prompt reaches this size. The doubling applies to that request only — a
+ * day's summed tokens are not a prompt — so the rate table stays at the short
+ * tier and the usage report adds one extra base-rate charge for the subset.
+ */
+export const GROK_LONG_CONTEXT_PROMPT_TOKENS = 200_000;
+
+/** Docs' recommended Imagine models, and what the Grok CLI bills when it names none. */
+export const GROK_IMAGINE_IMAGE_MODEL = 'grok-imagine-image-2.0';
+export const GROK_IMAGINE_VIDEO_MODEL = 'grok-imagine-video-1.5';
+
+/**
+ * Headline / lowest-resolution Imagine rates (USD). Higher resolutions cost
+ * more; PortOS does not request a resolution, so the estimate is this tier.
+ * Longest id first: `grok-imagine-image` is a prefix of the 2.0 and quality ids.
+ */
+const GROK_IMAGINE_RATES = Object.freeze([
+ { id: 'grok-imagine-video-1.5-lite', kind: 'video', perSecond: 0.02, perInputImage: 0.01 },
+ { id: 'grok-imagine-image-quality', kind: 'image', perImage: 0.05, perInputImage: 0.01 },
+ { id: 'grok-imagine-image-2.0', kind: 'image', perImage: 0.04, perInputImage: 0.01 },
+ { id: 'grok-imagine-video-1.5', kind: 'video', perSecond: 0.08, perInputImage: 0.01 },
+ { id: 'grok-imagine-image', kind: 'image', perImage: 0.02, perInputImage: 0.002 },
+ { id: 'grok-imagine-video', kind: 'video', perSecond: 0.05, perInputImage: 0.002 },
+].sort((a, b) => b.id.length - a.id.length));
+
+/** The Imagine rate row a model id names, or null. Longest published id wins. */
+function grokImagineRate(model) {
+ const id = String(model || '').trim().toLowerCase();
+ if (!id.includes('grok-imagine-')) return null;
+ return GROK_IMAGINE_RATES.find((row) => id === row.id || id.includes(row.id)) || null;
+}
+
+/**
+ * USD for one Imagine bucket. Token rates do not apply; input-image fees use
+ * the same model's published input price. Unknown ids cost nothing here so a
+ * text row cannot pick up an image price by substring accident.
+ * @param {string|null|undefined} model
+ * @param {{ images?: number, videoSeconds?: number, inputImages?: number }} [units]
+ * @returns {number}
+ */
+export function estimateGrokImagineCost(model, units = null) {
+ const rate = grokImagineRate(model);
+ if (!rate) return 0;
+ return (units?.images || 0) * (rate.perImage || 0)
+ + (units?.videoSeconds || 0) * (rate.perSecond || 0)
+ + (units?.inputImages || 0) * (rate.perInputImage || 0);
+}
+
export const pricingAsOfForModel = (model) =>
MODEL_RATE_AS_OF[String(model || '').trim().toLowerCase()] || PRICING_AS_OF;
@@ -116,9 +181,9 @@ const EXACT_RATES = {
'gpt-5.4-mini': [0.75, 4.5],
'gpt-5.4-nano': [0.2, 1.25],
'gpt-5.3-codex': [1.75, 14.0],
- // xAI
- // Verified 2026-09-21: https://docs.x.ai/developers/models/grok-4.7
- // Standard <=200k input tier; >200k uses $4/$12 and is not modeled here.
+ // xAI. Short-context (prompt <200k) rates. The ≥200k doubling is
+ // GROK_LONG_CONTEXT_PROMPT_TOKENS, applied per request by the usage report.
+ // Verified 2026-10-24: https://docs.x.ai/docs/models
'grok-4.7': [2.0, 6.0],
'grok-4.6': [2.0, 6.0],
'grok-4.5': [2.0, 6.0],
@@ -210,9 +275,13 @@ const FALLBACK_RATES = { rateModel: null, inputPer1M: 3.0, outputPer1M: 15.0 };
* Claude Code does not use — `cache_creation_input_tokens` is 5-minute).
* - OpenAI: 0.1x cached input; only gpt-5.6 lists a separate cache-write rate
* (~1.25x), and Codex reports no cache-write tokens at all.
- * - xAI: Grok 4.7's cached input is $0.50/M (0.25x), verified 2026-09-21;
- * Grok 4.5 retains its 0.15x estimate. No published cache-write surcharge
- * for Grok 4.7: uncached input uses its standard input rate.
+ * - xAI (2026-10-24, https://docs.x.ai/docs/models): cached input is a
+ * published price, not one ratio. grok-4.7 and grok-4.6 are $0.50 (0.25x
+ * of $2), grok-4.5 is $0.30 (0.15x), grok-4.3 / grok-4.20 and
+ * grok-build-0.1 are $0.20 (absolute — 0.20 is not a binary fraction of
+ * $1.25). xAI publishes no cache-write product; creation is ordinary
+ * input, so every Grok write multiplier is 1. An unknown grok* id uses
+ * the grok-4.7 pair.
* - Google: 0.1x cached input, plus an hourly storage fee we do not model
* (PortOS records no cache duration to price it against).
*
@@ -233,7 +302,12 @@ const CACHE_MULTIPLIER_RULES = [
// GPT-6.1 Sol cached input is $0.10/MTok against a $2 input rate.
{ test: /^gpt-6\.1-sol$/, read: 0.05, write: 1.25 },
{ test: /^grok-4\.7$/, read: 0.25, write: 1 },
- { test: /^grok/, read: 0.15, write: 1.25 },
+ { test: /^grok-4\.6$/, read: 0.25, write: 1 },
+ { test: /^grok-4\.5$/, read: 0.15, write: 1 },
+ // $0.20 cached input against $1.25 (grok-4.3) and $1 (grok-build-0.1).
+ { test: /^grok-4\.3$/, cacheReadPer1M: 0.20, write: 1 },
+ { test: /^grok-build-0\.1$/, cacheReadPer1M: 0.20, write: 1 },
+ { test: /^grok/, read: 0.25, write: 1 },
{ test: /^claude-fable-5-1/, read: 0.025, write: 1.25 },
// Opus 5.5 cache reads are $0.20/MTok against a $4 input rate.
{ test: /^claude-opus-5-5/, read: 0.05, write: 1.25 },
@@ -254,11 +328,15 @@ const cacheMultipliers = (rateModel) => {
* tier answered.
*/
const withCacheRates = (rates) => {
- const { read, write } = cacheMultipliers(rates.rateModel);
+ const rule = cacheMultipliers(rates.rateModel);
+ const read = rule.cacheReadPer1M != null
+ ? rule.cacheReadPer1M
+ : rates.inputPer1M * (rule.read ?? DEFAULT_CACHE_MULTIPLIERS.read);
+ const write = rates.inputPer1M * (rule.write ?? DEFAULT_CACHE_MULTIPLIERS.write);
return {
...rates,
- cacheReadPer1M: rates.inputPer1M * read,
- cacheWritePer1M: rates.inputPer1M * write,
+ cacheReadPer1M: read,
+ cacheWritePer1M: write,
};
};
@@ -279,6 +357,20 @@ const toRates = (rateModel) => withCacheRates({
*/
export function resolveModelRates(providerId, model) {
const id = typeof model === 'string' ? model.trim() : '';
+ // Before the `/grok/i` family rule, which would otherwise price an Imagine
+ // id as grok-4.7 text. Imagine dollars are estimateGrokImagineCost; these
+ // token rates stay zero so a media row is not also billed as chat.
+ const imagine = grokImagineRate(id);
+ if (imagine) {
+ return {
+ rateModel: imagine.id,
+ inputPer1M: 0,
+ outputPer1M: 0,
+ cacheReadPer1M: 0,
+ cacheWritePer1M: 0,
+ matched: id.toLowerCase() === imagine.id ? 'exact' : 'family',
+ };
+ }
if (id && EXACT_RATES[id]) {
return { ...toRates(id), matched: 'exact' };
}
diff --git a/server/lib/modelPricing.test.js b/server/lib/modelPricing.test.js
index 34f30df5fd..a7a67677da 100644
--- a/server/lib/modelPricing.test.js
+++ b/server/lib/modelPricing.test.js
@@ -1,5 +1,8 @@
import { describe, it, expect } from 'vitest';
-import { resolveModelRates, isFreeProvider, isFreeModelId, estimateCostUsd, PRICING_AS_OF } from './modelPricing.js';
+import {
+ resolveModelRates, isFreeProvider, isFreeModelId, estimateCostUsd, estimateGrokImagineCost,
+ pricingAsOfForModel, PRICING_AS_OF, GROK_IMAGINE_IMAGE_MODEL, GROK_IMAGINE_VIDEO_MODEL,
+} from './modelPricing.js';
describe('resolveModelRates', () => {
it('matches exact model ids', () => {
@@ -310,7 +313,7 @@ describe('cache-tier rates on resolveModelRates', () => {
const rates = resolveModelRates('grok', 'grok-4.7');
expect(rates).toMatchObject({ inputPer1M: 2, outputPer1M: 6, cacheReadPer1M: 0.5, cacheWritePer1M: 2, matched: 'exact' });
expect(estimateCostUsd(100_000, 10_000, rates, { cacheReadTokens: 100_000 })).toBeCloseTo(0.31);
- expect(resolveModelRates('grok-cli', 'grok-4.6')).toMatchObject({ inputPer1M: 2, outputPer1M: 6, cacheReadPer1M: 0.3 });
+ expect(resolveModelRates('grok-cli', 'grok-4.6')).toMatchObject({ inputPer1M: 2, outputPer1M: 6, cacheReadPer1M: 0.5, cacheWritePer1M: 2 });
});
it('derives both cache tiers from the input rate on every match tier', () => {
for (const [providerId, model] of [
@@ -325,10 +328,15 @@ describe('cache-tier rates on resolveModelRates', () => {
}
});
- it('uses xAI\'s higher published cached-input ratio for grok', () => {
- const rates = resolveModelRates('grok', 'grok-4.5');
- expect(rates.inputPer1M).toBe(2);
- expect(rates.cacheReadPer1M).toBeCloseTo(0.3); // 0.15x per docs.x.ai
+ it('uses xAI\'s published cached-input prices and does not surcharge cache writes', () => {
+ const grok45 = resolveModelRates('grok', 'grok-4.5');
+ expect(grok45.cacheReadPer1M).toBeCloseTo(0.3);
+ expect(grok45.cacheWritePer1M).toBe(grok45.inputPer1M);
+ expect(resolveModelRates('grok', 'grok-4.3')).toMatchObject({ cacheReadPer1M: 0.2, cacheWritePer1M: 1.25 });
+ expect(resolveModelRates('grok', 'grok-4.20-reasoning')).toMatchObject({ inputPer1M: 1.25, cacheReadPer1M: 0.2, cacheWritePer1M: 1.25 });
+ expect(resolveModelRates('grok', 'grok-build-0.1')).toMatchObject({ cacheReadPer1M: 0.2, cacheWritePer1M: 1 });
+ // Unknown grok ids follow grok-4.7's cache pair, not the old 0.15× / 1.25× guess.
+ expect(resolveModelRates('grok', 'grok-5')).toMatchObject({ cacheReadPer1M: 0.5, cacheWritePer1M: 2, matched: 'family' });
});
it('keeps the OpenAI/Codex 10% cached-input ratio', () => {
@@ -418,3 +426,22 @@ describe('isFreeModelId', () => {
expect(isFreeModelId('unsloth/Qwen3-30B-free-quant')).toBe(false);
});
});
+
+describe('Grok Imagine rates', () => {
+ it('prices the recommended image and video models at the headline tier', () => {
+ expect(estimateGrokImagineCost(GROK_IMAGINE_IMAGE_MODEL, { images: 1 })).toBeCloseTo(0.04);
+ expect(estimateGrokImagineCost(GROK_IMAGINE_IMAGE_MODEL, { images: 1, inputImages: 2 })).toBeCloseTo(0.06);
+ expect(estimateGrokImagineCost(GROK_IMAGINE_VIDEO_MODEL, { videoSeconds: 6, inputImages: 1 })).toBeCloseTo(0.49);
+ expect(estimateGrokImagineCost('grok-imagine-image', { images: 1 })).toBeCloseTo(0.02);
+ expect(pricingAsOfForModel('grok-4.7')).toBe('2026-10-24');
+ });
+
+ it('does not let the shorter Imagine id swallow a longer one, or bill Imagine as grok-4.7 text', () => {
+ expect(resolveModelRates('grok', 'grok-imagine-image-2.0').rateModel).toBe('grok-imagine-image-2.0');
+ expect(resolveModelRates('grok', 'grok-imagine-image-quality').rateModel).toBe('grok-imagine-image-quality');
+ expect(resolveModelRates('grok', 'grok-imagine-video-1.5').rateModel).toBe('grok-imagine-video-1.5');
+ expect(resolveModelRates('grok', 'grok-imagine-video-1.5-lite').inputPer1M).toBe(0);
+ expect(isFreeModelId('grok-imagine-image-2.0')).toBe(false);
+ expect(estimateGrokImagineCost('grok-4.7', { images: 1 })).toBe(0);
+ });
+});
diff --git a/server/lib/providerTranscriptUsage.js b/server/lib/providerTranscriptUsage.js
index 968060df4e..af323c4692 100644
--- a/server/lib/providerTranscriptUsage.js
+++ b/server/lib/providerTranscriptUsage.js
@@ -28,6 +28,8 @@
* written), and ignore unknown fields so a CLI update can't break them.
*/
+import { GROK_LONG_CONTEXT_PROMPT_TOKENS } from './modelPricing.js';
+
/**
* Slugify a working directory the way Claude Code names its project folder:
* every `/` and `.` becomes `-`. An absolute POSIX path therefore keeps its
@@ -474,18 +476,44 @@ function grokEventMs(line) {
return ts < 1e11 ? Math.round(ts * 1000) : ts;
}
-/** The four billable buckets of one grok `usage` block, cache tiers split out. */
-const grokBuckets = (usage) => ({
- messages: 1,
- // `inputTokens` includes the cached read — split so each tier prices at its
- // own rate instead of billing cache reads as fresh input.
- tokensIn: Math.max(0, num(usage?.inputTokens) - num(usage?.cachedReadTokens)),
- // `reasoningTokens` is a SUBSET of `outputTokens`, not an addition.
- tokensOut: num(usage?.outputTokens),
- cacheReadTokens: num(usage?.cachedReadTokens),
- cacheWriteTokens: num(usage?.cacheCreationTokens)
+const grokTotals = () => ({
+ ...emptyTotals(),
+ longTokensIn: 0,
+ longTokensOut: 0,
+ longCacheReadTokens: 0,
+ longCacheWriteTokens: 0,
});
+/**
+ * The billable buckets of one grok `usage` block, cache tiers split out.
+ * `long*` is a subset of those buckets, set only when this request's own
+ * prompt (`inputTokens`, which already includes the cache read) reaches
+ * xAI's 200k doubling threshold. It is not a second copy of the tokens.
+ */
+const grokBuckets = (usage) => {
+ const buckets = {
+ messages: 1,
+ // `inputTokens` includes the cached read — split so each tier prices at its
+ // own rate instead of billing cache reads as fresh input.
+ tokensIn: Math.max(0, num(usage?.inputTokens) - num(usage?.cachedReadTokens)),
+ // `reasoningTokens` is a SUBSET of `outputTokens`, not an addition.
+ tokensOut: num(usage?.outputTokens),
+ cacheReadTokens: num(usage?.cachedReadTokens),
+ cacheWriteTokens: num(usage?.cacheCreationTokens),
+ longTokensIn: 0,
+ longTokensOut: 0,
+ longCacheReadTokens: 0,
+ longCacheWriteTokens: 0,
+ };
+ if (num(usage?.inputTokens) >= GROK_LONG_CONTEXT_PROMPT_TOKENS) {
+ buckets.longTokensIn = buckets.tokensIn;
+ buckets.longTokensOut = buckets.tokensOut;
+ buckets.longCacheReadTokens = buckets.cacheReadTokens;
+ buckets.longCacheWriteTokens = buckets.cacheWriteTokens;
+ }
+ return buckets;
+};
+
/**
* True when every consecutive pair of turns is non-decreasing in all three
* independently-moving fields — the signature of a cumulative stream.
@@ -519,9 +547,10 @@ function looksCumulative(usages) {
* share of it is legitimately zero".
*/
export function parseGrokTurns(jsonlText, { from = null, to = null, exclude = null } = {}) {
- const totals = emptyTotals();
+ const totals = grokTotals();
const byModel = new Map();
const counted = [];
+ const longTurns = [];
let sessionId = null;
// Every turn_completed in file order — the whole sequence is needed before
@@ -567,7 +596,6 @@ export function parseGrokTurns(jsonlText, { from = null, to = null, exclude = nu
const buckets = grokBuckets(usage);
if (totalTranscriptTokens(buckets) === 0) continue;
counted.push(turn.key);
- for (const field of Object.keys(totals)) totals[field] += buckets[field];
// `modelUsage` names the model(s) the turn actually called. In cumulative
// mode its per-model figures are cumulative too, so bucket the DELTA'd
@@ -576,9 +604,23 @@ export function parseGrokTurns(jsonlText, { from = null, to = null, exclude = nu
// one turn as a unit, and splitting the delta across models would need
// per-model deltas the cumulative shape doesn't reliably provide.
const bucketKey = Object.keys(turn.usage.modelUsage || {})[0] ?? UNKNOWN_MODEL;
- if (!byModel.has(bucketKey)) byModel.set(bucketKey, emptyTotals());
+ if (!byModel.has(bucketKey)) byModel.set(bucketKey, grokTotals());
const bucket = byModel.get(bucketKey);
- for (const field of Object.keys(bucket)) bucket[field] += buckets[field];
+ for (const field of Object.keys(bucket)) bucket[field] += buckets[field] || 0;
+ for (const field of Object.keys(totals)) totals[field] += buckets[field] || 0;
+ if (buckets.longTokensIn || buckets.longTokensOut || buckets.longCacheReadTokens || buckets.longCacheWriteTokens) {
+ longTurns.push({
+ key: turn.key,
+ ms: turn.ms,
+ sessionId,
+ model: bucketKey,
+ promptTokens: num(usage.inputTokens),
+ tokensIn: buckets.tokensIn,
+ tokensOut: buckets.tokensOut,
+ cacheReadTokens: buckets.cacheReadTokens,
+ cacheWriteTokens: buckets.cacheWriteTokens,
+ });
+ }
}
const models = [...byModel.keys()].filter((model) => model !== UNKNOWN_MODEL);
@@ -589,7 +631,8 @@ export function parseGrokTurns(jsonlText, { from = null, to = null, exclude = nu
byModel: Object.fromEntries(byModel),
...totals,
countedKeys: counted,
- turns: turns.length
+ turns: turns.length,
+ longTurns,
};
}
diff --git a/server/lib/providerTranscriptUsage.test.js b/server/lib/providerTranscriptUsage.test.js
index a633a12054..ed8f7070e2 100644
--- a/server/lib/providerTranscriptUsage.test.js
+++ b/server/lib/providerTranscriptUsage.test.js
@@ -520,6 +520,8 @@ describe('parseGrokTurns', () => {
// `reasoningTokens` is a SUBSET of `outputTokens`, never an addition.
expect(parsed.tokensOut).toBe(1_800);
expect(parsed.cacheWriteTokens).toBe(0);
+ expect(parsed.longTokensIn).toBe(0);
+ expect(parsed.longTurns).toEqual([]);
expect(parsed.byModel[GROK_MODEL].tokensOut).toBe(1_800);
expect(parsed.model).toBe(GROK_MODEL);
expect(parsed.sessionId).toBe('session-aaaa');
@@ -533,6 +535,24 @@ describe('parseGrokTurns', () => {
expect(parsed.turns).toBe(0);
});
+ it('marks only a turn whose own prompt reaches 200k as the long-context subset', () => {
+ const text = [
+ grokTurn({ promptId: 'short', input: 15_000, cachedRead: 11_000, output: 100 }),
+ grokTurn({ promptId: 'long', input: 250_000, cachedRead: 50_000, output: 2_000, reasoning: 500 }),
+ ].join('\n');
+ const parsed = parseGrokTurns(text);
+ expect(parsed.tokensIn).toBe(4_000 + 200_000);
+ expect(parsed.tokensOut).toBe(2_100);
+ expect(parsed.longTokensIn).toBe(200_000);
+ expect(parsed.longTokensOut).toBe(2_000);
+ expect(parsed.longCacheReadTokens).toBe(50_000);
+ expect(parsed.longTurns).toEqual([expect.objectContaining({
+ key: 'long', promptTokens: 250_000, tokensIn: 200_000, cacheReadTokens: 50_000,
+ })]);
+ // Reasoning stays inside output. 500 of the 2,000 are not added again.
+ expect(parsed.tokensOut).toBe(2_100);
+ });
+
it('sums per-prompt turns without treating them as cumulative', () => {
const text = [
grokTurn({ promptId: 'p1', input: 10_000, cachedRead: 6_000, output: 500 }),
diff --git a/server/routes/usage.test.js b/server/routes/usage.test.js
index 1c7bf2ac7e..e2bd24745b 100644
--- a/server/routes/usage.test.js
+++ b/server/routes/usage.test.js
@@ -414,6 +414,14 @@ describe('usage routes', () => {
expect(startHistoricalUsageBackfill).toHaveBeenCalledTimes(1);
});
+ it('POST /api/usage/backfill returns the Grok repair summary on the job', async () => {
+ const grok = { added: 2, turns: 1, images: 1, videos: 0, backedUp: true, note: null };
+ startHistoricalUsageBackfill.mockResolvedValue({ status: 'complete', corrected: 1, grok });
+ const started = await request(buildApp()).post('/api/usage/backfill');
+ expect(started.status).toBe(202);
+ expect(started.body.grok).toEqual(grok);
+ });
+
it('POST /api/usage/session records a session and returns its number', async () => {
usage.recordSession.mockResolvedValue(42);
const res = await request(buildApp())
diff --git a/server/services/aiProvider.js b/server/services/aiProvider.js
index 31cc9075e4..870ec3bf73 100644
--- a/server/services/aiProvider.js
+++ b/server/services/aiProvider.js
@@ -5,6 +5,7 @@ import { awaitWithAbort } from '../lib/abortTimeout.js';
*/
import { normalizeReviewFinishReason } from '../lib/reviewerHealth.js';
+import { normalizeChatCompletionUsage } from '../lib/chatCompletionUsage.js';
import { getAllProviders } from './providers.js';
import { startAIOp } from './aiStatusEvents.js';
// localModelHealing is lazy-imported at its (rare, error-recovery) call site
@@ -259,16 +260,38 @@ async function postChatCompletion(provider, model, prompt, { temperature, max_to
// OpenAI-compatible bodies report token counts under `usage`; surface the completion
// token count so the AI Core landmark can size its activity beam by output volume
// (tokens/sec). Absent on some providers — callers treat a missing count as "unknown".
- const completionTokens = Number.isFinite(data.usage?.completion_tokens) ? data.usage.completion_tokens : NaN;
- const promptTokens = Number.isFinite(data.usage?.prompt_tokens) ? data.usage.prompt_tokens : NaN;
+ const normalized = normalizeChatCompletionUsage(data.usage, { model, endpoint: provider.endpoint });
+ if (normalized && (normalized.inputTokens || normalized.outputTokens || normalized.cacheReadTokens)) {
+ // Ledger failure must not fail the completion. Codex subscription turns
+ // never reach this HTTP path; their transcript reconciler owns that bill.
+ void import('./usage.js').then(({ recordRunUsage }) => recordRunUsage({
+ providerId: provider.id,
+ model,
+ messages: 1,
+ tokensIn: normalized.inputTokens,
+ tokensOut: normalized.outputTokens,
+ cacheReadTokens: normalized.cacheReadTokens,
+ cacheWriteTokens: normalized.cacheWriteTokens,
+ ...(normalized.longContext ? {
+ longTokensIn: normalized.inputTokens,
+ longTokensOut: normalized.outputTokens,
+ longCacheReadTokens: normalized.cacheReadTokens,
+ longCacheWriteTokens: normalized.cacheWriteTokens,
+ } : {}),
+ source: 'measured',
+ })).catch((err) => console.error(`❌ Failed to record chat usage: ${err.message}`));
+ }
+ const completionTokens = normalized ? normalized.outputTokens : (Number.isFinite(data.usage?.completion_tokens) ? data.usage.completion_tokens : NaN);
return {
text: content,
finishReason: normalizeReviewFinishReason(data?.choices?.[0]?.finish_reason),
tokens: Number.isFinite(completionTokens) ? completionTokens : undefined,
- ...(Number.isFinite(promptTokens) || Number.isFinite(completionTokens)
+ ...(normalized
? { usage: {
- ...(Number.isFinite(promptTokens) ? { inputTokens: promptTokens } : {}),
- ...(Number.isFinite(completionTokens) ? { outputTokens: completionTokens } : {}),
+ inputTokens: normalized.inputTokens,
+ outputTokens: normalized.outputTokens,
+ cacheReadTokens: normalized.cacheReadTokens,
+ source: normalized.source,
} }
: {}),
};
diff --git a/server/services/aiProvider.test.js b/server/services/aiProvider.test.js
index 9d20141428..3fbfd8b70b 100644
--- a/server/services/aiProvider.test.js
+++ b/server/services/aiProvider.test.js
@@ -9,6 +9,11 @@ const { statusOp } = vi.hoisted(() => ({
}));
vi.mock('../services/providers.js', () => ({ getAllProviders: vi.fn() }));
+const { recordRunUsage } = vi.hoisted(() => ({ recordRunUsage: vi.fn(async () => {}) }));
+vi.mock('./usage.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, recordRunUsage };
+});
vi.mock('../services/aiStatusEvents.js', () => ({ startAIOp: vi.fn(() => statusOp) }));
vi.mock('../services/ollamaManager.js', () => ({
ensureProviderReady: vi.fn(),
@@ -238,6 +243,7 @@ describe('callProviderAISimple through the ChatGPT subscription', () => {
statusOp.update.mockClear();
statusOp.complete.mockClear();
statusOp.error.mockClear();
+ recordRunUsage.mockClear();
runTurn.mockReset();
bench.mockReset();
peekBench.mockReset().mockReturnValue(null);
@@ -257,6 +263,7 @@ describe('callProviderAISimple through the ChatGPT subscription', () => {
expect(result).toMatchObject({ text: 'a subscription answer' });
expect(result.usage.source).toBe('chatgpt-subscription');
+ expect(recordRunUsage).not.toHaveBeenCalled();
expect(runTurn).toHaveBeenCalledWith(expect.objectContaining({
prompt: 'Summarize this.', model: 'model-alpha',
}));
@@ -621,3 +628,38 @@ describe('provider review preparation deadline', () => {
vi.mocked(isOllamaProvider).mockReturnValue(false);
});
});
+
+describe('callProviderAISimple xAI usage ledger', () => {
+ afterEach(() => vi.unstubAllGlobals());
+
+ it('records cached and reasoning tokens from a successful xAI completion', async () => {
+ recordRunUsage.mockClear();
+ vi.stubGlobal('fetch', vi.fn(async () => ({
+ ok: true,
+ status: 200,
+ text: async () => JSON.stringify({
+ choices: [{ message: { content: 'done' } }],
+ usage: {
+ prompt_tokens: 32,
+ completion_tokens: 9,
+ prompt_tokens_details: { cached_tokens: 6 },
+ completion_tokens_details: { reasoning_tokens: 94 },
+ },
+ }),
+ })));
+ const result = await callProviderAISimple(
+ { id: 'grok', type: 'api', endpoint: 'https://api.x.ai/v1' },
+ 'grok-4.7',
+ 'prompt',
+ );
+ expect(result.usage).toMatchObject({ inputTokens: 26, outputTokens: 103, cacheReadTokens: 6 });
+ await vi.waitFor(() => expect(recordRunUsage).toHaveBeenCalledWith(expect.objectContaining({
+ providerId: 'grok',
+ model: 'grok-4.7',
+ tokensIn: 26,
+ tokensOut: 103,
+ cacheReadTokens: 6,
+ source: 'measured',
+ })));
+ });
+});
diff --git a/server/services/grokMediaUsage.js b/server/services/grokMediaUsage.js
new file mode 100644
index 0000000000..e670828d70
--- /dev/null
+++ b/server/services/grokMediaUsage.js
@@ -0,0 +1,96 @@
+/**
+ * Ledger writes for an accepted Grok Imagine image or video.
+ *
+ * Counts go in usage.json and are priced at read time. A repair id written in
+ * the same save keeps the historical backfill from adding the job again.
+ * Provider lookup prefers the configured Grok CLI so the row sits with the
+ * token usage the interactive scan already attributes there, and falls back
+ * to the id `grok` when this install has no Grok provider.
+ */
+
+import { familyForProvider } from '../lib/providerFamilies.js';
+import {
+ GROK_IMAGINE_IMAGE_MODEL,
+ GROK_IMAGINE_VIDEO_MODEL,
+} from '../lib/modelPricing.js';
+
+function dayKey(iso) {
+ const ms = Date.parse(iso || '');
+ const when = Number.isFinite(ms) ? new Date(ms) : new Date();
+ return when.toISOString().slice(0, 10);
+}
+
+async function grokUsageProviderId() {
+ try {
+ const { listProviders } = await import('./providers.js');
+ const list = await listProviders();
+ return grokUsageProviderIdFromList(list);
+ } catch (err) {
+ console.error(`❌ Grok usage provider lookup failed: ${err.message}`);
+ return 'grok';
+ }
+}
+
+/** CLI, then TUI, then any Grok family record, else the id `grok`. */
+export function grokUsageProviderIdFromList(providers) {
+ const grok = (providers || []).filter((provider) => (
+ provider && provider.enabled !== false && familyForProvider(provider) === 'grok'
+ ));
+ const preferred = grok.find((provider) => provider.type === 'cli')
+ || grok.find((provider) => provider.type === 'tui')
+ || grok[0];
+ return preferred?.id || 'grok';
+}
+
+async function record(day, record, repairId) {
+ const { addDatedUsage } = await import('./usage.js');
+ await addDatedUsage(day, record, { repairId, flush: true });
+}
+
+/**
+ * One accepted gallery image. `inputImages` is the init image plus references
+ * that were actually passed to the tool.
+ */
+export async function recordGrokImageUsage({
+ jobId, createdAt, inputImages = 0, providerId = null, repairId = null,
+} = {}) {
+ if (!jobId) return;
+ const id = repairId || `gallery:${jobId}`;
+ await record(dayKey(createdAt), {
+ providerId: providerId || await grokUsageProviderId(),
+ model: GROK_IMAGINE_IMAGE_MODEL,
+ messages: 0,
+ images: 1,
+ inputImages: Math.max(0, inputImages || 0),
+ source: 'measured',
+ }, id);
+}
+
+/**
+ * One accepted video. `images: 1` is the hidden text-to-video base still,
+ * recorded as its own image-model row (`video-base:<jobId>`).
+ */
+export async function recordGrokVideoUsage({
+ jobId, createdAt, videoSeconds = 0, images = 0, inputImages = 0, providerId = null,
+} = {}) {
+ if (!jobId) return;
+ const resolved = providerId || await grokUsageProviderId();
+ const day = dayKey(createdAt);
+ await record(day, {
+ providerId: resolved,
+ model: GROK_IMAGINE_VIDEO_MODEL,
+ messages: 0,
+ videoSeconds: Math.max(0, videoSeconds || 0),
+ inputImages: Math.max(0, inputImages || 0),
+ source: 'measured',
+ }, `video:${jobId}`);
+ if (images > 0) {
+ await record(day, {
+ providerId: resolved,
+ model: GROK_IMAGINE_IMAGE_MODEL,
+ messages: 0,
+ images: 1,
+ source: 'measured',
+ }, `video-base:${jobId}`);
+ }
+}
diff --git a/server/services/grokUsageBackfill.js b/server/services/grokUsageBackfill.js
new file mode 100644
index 0000000000..479a67ded2
--- /dev/null
+++ b/server/services/grokUsageBackfill.js
@@ -0,0 +1,194 @@
+/**
+ * Apply the Grok usage repair to the server's in-memory ledger.
+ *
+ * Reconcile on the Usage page calls this from the historical-backfill job, on
+ * the same thread that owns usage.json. A separate process writing that file
+ * would lose to the server's next save. Repair ids already on the ledger make
+ * a repeat a no-op. The on-disk ledger is copied before the first write of a
+ * batch that has something to add.
+ *
+ * The CLI script is a dry-run and stopped-server wrapper around these two
+ * functions. Cache-rate corrections are not part of either path: the report
+ * prices tokens when it is built.
+ */
+
+import { copyFile, mkdir, readdir, readFile, stat } from 'fs/promises';
+import { homedir } from 'os';
+import { join, resolve } from 'path';
+import { PATHS } from '../lib/fileUtils.js';
+import { familyForProvider } from '../lib/providerFamilies.js';
+import { parseGrokTurns } from '../lib/providerTranscriptUsage.js';
+import { grokUsageProviderIdFromList } from './grokMediaUsage.js';
+import {
+ galleryImageUsageEvent,
+ imagineEventsFromGrokSession,
+ planGrokUsageRepair,
+ summarizeGrokRepairAdditions,
+ videoHistoryUsageEvents,
+} from './grokUsageRepair.js';
+import { addDatedUsage, flushUsageLedger, getUsage, loadUsage } from './usage.js';
+
+const yieldToEventLoop = () => new Promise((done) => setImmediate(done));
+
+async function readJson(path, fallback) {
+ try {
+ return JSON.parse(await readFile(path, 'utf8'));
+ } catch (err) {
+ if (err.code === 'ENOENT') return fallback;
+ throw err;
+ }
+}
+
+async function listDir(path) {
+ try {
+ return await readdir(path);
+ } catch (err) {
+ if (err.code === 'ENOENT') return [];
+ throw err;
+ }
+}
+
+async function fileMtime(path) {
+ try {
+ return (await stat(path)).mtimeMs;
+ } catch {
+ return null;
+ }
+}
+
+async function collectSessions(home) {
+ const root = join(home, '.grok', 'sessions');
+ const turns = [];
+ const media = [];
+ for (const dirName of await listDir(root)) {
+ let cwd = dirName;
+ try { cwd = decodeURIComponent(dirName); } catch { /* keep the raw directory name */ }
+ const cwdDir = join(root, dirName);
+ for (const sessionId of await listDir(cwdDir)) {
+ const sessionDir = join(cwdDir, sessionId);
+ const updatesPath = join(sessionDir, 'updates.jsonl');
+ const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');
+ if (updatesText) {
+ const parsed = parseGrokTurns(updatesText);
+ for (const turn of parsed.longTurns || []) turns.push(turn);
+ }
+ const summary = await readJson(join(sessionDir, 'summary.json'), null);
+ const chatPath = join(sessionDir, 'chat_history.jsonl');
+ const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')
+ || await fileMtime(chatPath)
+ || await fileMtime(updatesPath);
+ const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;
+ const chatText = await readFile(chatPath, 'utf8').catch(() => '');
+ media.push(...imagineEventsFromGrokSession({
+ chatText, updatesText, sessionId, cwd, day,
+ }));
+ await yieldToEventLoop();
+ }
+ }
+ return { turns, media };
+}
+
+async function collectGallery(dataDir) {
+ const dir = join(dataDir, 'images');
+ const media = [];
+ for (const name of await listDir(dir)) {
+ if (!name.endsWith('.metadata.json')) continue;
+ const meta = await readJson(join(dir, name), null);
+ const event = galleryImageUsageEvent(meta);
+ if (event) media.push(event);
+ }
+ return media;
+}
+
+async function collectVideos(dataDir) {
+ const history = await readJson(join(dataDir, 'video-history.json'), []);
+ return (Array.isArray(history) ? history : []).flatMap(videoHistoryUsageEvents);
+}
+
+async function providerIdFromFile(dataDir) {
+ const stored = await readJson(join(dataDir, 'providers.json'), null);
+ const list = Array.isArray(stored) ? stored : (stored?.providers || []);
+ const grok = list.filter((provider) => provider && familyForProvider(provider) === 'grok');
+ return grokUsageProviderIdFromList(grok.length ? list : []);
+}
+
+/**
+ * Read sessions, the gallery, and video history without touching the ledger.
+ * `repairIds` default to the on-disk list so a dry run does not load the
+ * server's in-memory copy.
+ */
+export async function planGrokUsageRepairFromDisk({
+ home = homedir(),
+ dataDir = PATHS.data,
+ repairIds = null,
+ providerId = null,
+} = {}) {
+ const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} });
+ const watermarkMs = typeof scan?.watermarks?.grok === 'number' ? scan.watermarks.grok : null;
+ const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}))?.grokRepairIds;
+ const sessions = await collectSessions(home);
+ return planGrokUsageRepair({
+ turns: sessions.turns,
+ media: [
+ ...sessions.media,
+ ...await collectGallery(dataDir),
+ ...await collectVideos(dataDir),
+ ],
+ repairIds: Array.isArray(storedIds) ? storedIds : [],
+ watermarkMs,
+ providerId: providerId || await providerIdFromFile(dataDir),
+ });
+}
+
+async function backupLedger(dataDir) {
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const dest = join(dataDir, 'backups', `grok-usage-${stamp}`);
+ await mkdir(dest, { recursive: true });
+ for (const name of ['usage.json', 'usage-interactive-scan.json']) {
+ const source = join(dataDir, name);
+ try {
+ await copyFile(source, join(dest, name));
+ } catch (err) {
+ if (err.code !== 'ENOENT') throw err;
+ }
+ }
+ return dest;
+}
+
+/**
+ * Plan against the in-memory repair ids and add any missing rows there.
+ * Writes only PATHS.data, which is the file the running server will save.
+ * @returns {Promise<{ added: number, turns: number, images: number, videos: number,
+ * backedUp: boolean, note: string|null, backupDir: string|null }>}
+ */
+export async function applyGrokUsageRepair({
+ home = homedir(),
+ dataDir = PATHS.data,
+ providers = null,
+ providerId = null,
+} = {}) {
+ if (resolve(dataDir) !== resolve(PATHS.data)) {
+ throw new Error(`Grok usage repair writes ${PATHS.data}/usage.json`);
+ }
+ await loadUsage();
+ const resolvedProviderId = providerId
+ || (providers ? grokUsageProviderIdFromList(providers) : await providerIdFromFile(dataDir));
+ const plan = await planGrokUsageRepairFromDisk({
+ home,
+ dataDir,
+ repairIds: getUsage().grokRepairIds || [],
+ providerId: resolvedProviderId,
+ });
+ const note = plan.notes[0] || null;
+ if (plan.additions.length === 0) {
+ return { ...summarizeGrokRepairAdditions([]), backedUp: false, note, backupDir: null };
+ }
+ const backupDir = await backupLedger(dataDir);
+ const written = [];
+ for (const item of plan.additions) {
+ const wrote = await addDatedUsage(item.day, item.record, { repairId: item.repairId, flush: false });
+ if (wrote) written.push(item);
+ }
+ if (written.length > 0) await flushUsageLedger();
+ return { ...summarizeGrokRepairAdditions(written), backedUp: true, note, backupDir };
+}
diff --git a/server/services/grokUsageBackfill.test.js b/server/services/grokUsageBackfill.test.js
new file mode 100644
index 0000000000..9599ce72c7
--- /dev/null
+++ b/server/services/grokUsageBackfill.test.js
@@ -0,0 +1,81 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const ledger = vi.hoisted(() => ({ grokRepairIds: [] }));
+const written = vi.hoisted(() => []);
+const paths = vi.hoisted(() => ({ data: '' }));
+
+vi.mock('./usage.js', () => ({
+ loadUsage: vi.fn(async () => ledger),
+ getUsage: vi.fn(() => ledger),
+ addDatedUsage: vi.fn(async (day, record, { repairId } = {}) => {
+ if (ledger.grokRepairIds.includes(repairId)) return false;
+ ledger.grokRepairIds.push(repairId);
+ written.push({ day, record, repairId });
+ return true;
+ }),
+ flushUsageLedger: vi.fn(async () => {}),
+}));
+
+vi.mock('../lib/fileUtils.js', () => ({
+ PATHS: paths,
+}));
+
+const { applyGrokUsageRepair, planGrokUsageRepairFromDisk } = await import('./grokUsageBackfill.js');
+const { flushUsageLedger } = await import('./usage.js');
+
+let home;
+let dataDir;
+
+beforeEach(async () => {
+ home = await mkdtemp(join(tmpdir(), 'grok-repair-home-'));
+ dataDir = await mkdtemp(join(tmpdir(), 'grok-repair-data-'));
+ paths.data = dataDir;
+ ledger.grokRepairIds = [];
+ written.length = 0;
+ flushUsageLedger.mockClear();
+ await mkdir(join(dataDir, 'images'), { recursive: true });
+ await writeFile(join(dataDir, 'usage.json'), JSON.stringify({ grokRepairIds: [] }));
+ await writeFile(join(dataDir, 'images', 'job-1.metadata.json'), JSON.stringify({
+ id: 'job-1', mode: 'grok', createdAt: '2026-10-02T00:00:00.000Z',
+ }));
+});
+
+afterEach(async () => {
+ await rm(home, { recursive: true, force: true });
+ await rm(dataDir, { recursive: true, force: true });
+});
+
+describe('planGrokUsageRepairFromDisk', () => {
+ it('plans gallery images without writing the ledger', async () => {
+ const plan = await planGrokUsageRepairFromDisk({ home, dataDir });
+ expect(plan.additions.map((item) => item.repairId)).toEqual(['gallery:job-1']);
+ expect(plan.notes[0]).toMatch(/no Grok usage scan has run yet/);
+ await expect(readFile(join(dataDir, 'usage.json'), 'utf8')).resolves.toBe(JSON.stringify({ grokRepairIds: [] }));
+ });
+});
+
+describe('applyGrokUsageRepair', () => {
+ it('backs up the ledger once, writes the in-memory repair, and skips a repeat', async () => {
+ const first = await applyGrokUsageRepair({ home, dataDir });
+ expect(first).toMatchObject({ added: 1, images: 1, turns: 0, videos: 0, backedUp: true });
+ expect(first.backupDir).toMatch(/grok-usage-/);
+ expect(JSON.parse(await readFile(join(first.backupDir, 'usage.json'), 'utf8')).grokRepairIds).toEqual([]);
+ expect(written).toHaveLength(1);
+ expect(written[0].repairId).toBe('gallery:job-1');
+ expect(flushUsageLedger).toHaveBeenCalledTimes(1);
+
+ const second = await applyGrokUsageRepair({ home, dataDir });
+ expect(second).toMatchObject({ added: 0, backedUp: false });
+ expect(written).toHaveLength(1);
+ expect(flushUsageLedger).toHaveBeenCalledTimes(1);
+ });
+
+ it('refuses to apply a plan read from a directory the server will not save', async () => {
+ await expect(applyGrokUsageRepair({ home, dataDir: join(dataDir, 'elsewhere') }))
+ .rejects.toThrow(/usage\.json/);
+ expect(written).toHaveLength(0);
+ });
+});
diff --git a/server/services/grokUsageRepair.js b/server/services/grokUsageRepair.js
new file mode 100644
index 0000000000..366f860e51
--- /dev/null
+++ b/server/services/grokUsageRepair.js
@@ -0,0 +1,259 @@
+/**
+ * Pure plan for repairing Grok usage that was recorded without Imagine units
+ * or the 200k price doubling.
+ *
+ * The plan never rewrites base token counts. Cache-rate corrections need no
+ * plan at all: cost is computed at read time. A long-context turn already in
+ * the ledger contributes only its extra 1× (`long*` is a subset). Turns with
+ * no interactive-scan watermark, or a timestamp after that watermark, are
+ * skipped — the scan has not billed them yet, or never ran, and adding the
+ * premium alone would invent a charge whose base may be absent.
+ *
+ * Media recovered from the gallery and video history is the whole charge.
+ * Session tool calls inside a PortOS `portos-grok-` scratch directory are
+ * omitted so a gallery job is not billed twice.
+ */
+
+import { parseJsonLines } from '../lib/providerTranscriptUsage.js';
+import { grokVideoBillableUnits } from '../lib/grokVideoClip.js';
+import {
+ GROK_IMAGINE_IMAGE_MODEL,
+ GROK_IMAGINE_VIDEO_MODEL,
+ GROK_LONG_CONTEXT_PROMPT_TOKENS,
+} from '../lib/modelPricing.js';
+
+const IMAGINE_IMAGE_TOOLS = new Set(['image_gen', 'image_edit']);
+const IMAGINE_VIDEO_TOOLS = new Set(['image_to_video']);
+
+export function grokTurnRepairId(sessionId, key) {
+ return `turn:${sessionId || 'unknown'}:${key}`;
+}
+
+function isPortosGrokScratch(cwd) {
+ return String(cwd || '').includes('portos-grok-');
+}
+
+function dayFromIso(iso) {
+ const ms = Date.parse(iso || '');
+ if (!Number.isFinite(ms)) return null;
+ return new Date(ms).toISOString().slice(0, 10);
+}
+
+function dayFromMs(ms) {
+ if (typeof ms !== 'number' || !Number.isFinite(ms)) return null;
+ return new Date(ms).toISOString().slice(0, 10);
+}
+
+function callName(call) {
+ return String(call?.name || call?.toolName || call?.function?.name || call?.tool_name || '').toLowerCase();
+}
+
+function callArgs(call) {
+ const raw = call?.arguments ?? call?.args ?? call?.function?.arguments ?? call?.input;
+ if (raw && typeof raw === 'object') return raw;
+ if (typeof raw !== 'string' || !raw) return {};
+ try {
+ const parsed = JSON.parse(raw);
+ return parsed && typeof parsed === 'object' ? parsed : {};
+ } catch {
+ return {};
+ }
+}
+
+function toolCallsFrom(entry) {
+ const calls = [];
+ for (const list of [entry?.tool_calls, entry?.toolCalls, entry?.message?.tool_calls]) {
+ if (Array.isArray(list)) calls.push(...list);
+ }
+ if (entry?.type === 'tool_call' || entry?.type === 'function_call') calls.push(entry);
+ const update = entry?.params?.update;
+ if (update && (update.sessionUpdate === 'tool_call' || update.sessionUpdate === 'tool_call_update')) calls.push(update);
+ return calls;
+}
+
+function inputImageCount(args) {
+ const images = args.image || args.images || args.input_images || args.reference_images;
+ if (Array.isArray(images)) return images.length;
+ if (typeof images === 'string' && images) return 1;
+ if (args.image_path || args.init_image) return 1;
+ return 0;
+}
+
+function eventsFromToolText(text, { sessionId, day }) {
+ const events = [];
+ let index = 0;
+ for (const entry of parseJsonLines(text)) {
+ for (const call of toolCallsFrom(entry)) {
+ const name = callName(call);
+ if (!IMAGINE_IMAGE_TOOLS.has(name) && !IMAGINE_VIDEO_TOOLS.has(name)) continue;
+ const args = callArgs(call);
+ const idBase = `${sessionId || 'unknown'}:${index}`;
+ index += 1;
+ if (IMAGINE_VIDEO_TOOLS.has(name)) {
+ const requested = args.duration ?? args.seconds ?? args.length;
+ const hasDuration = requested != null && Number.isFinite(Number(requested));
+ const units = grokVideoBillableUnits({
+ requestedSeconds: requested,
+ deliveredSeconds: null,
+ // The still this call animates is either a prior image_gen in the
+ // same session or a file the user already had. It is an input, not
+ // a second generated image.
+ hasSourceImage: true,
+ });
+ events.push({
+ id: `session-video:${idBase}`,
+ day,
+ model: GROK_IMAGINE_VIDEO_MODEL,
+ images: 0,
+ videoSeconds: units.videoSeconds,
+ inputImages: units.inputImages,
+ source: hasDuration ? 'measured' : 'estimate',
+ });
+ } else {
+ events.push({
+ id: `session-image:${idBase}`,
+ day,
+ model: GROK_IMAGINE_IMAGE_MODEL,
+ images: 1,
+ inputImages: inputImageCount(args),
+ videoSeconds: 0,
+ source: 'measured',
+ });
+ }
+ }
+ }
+ return events;
+}
+
+/**
+ * Imagine tool calls from one Grok CLI session. `chat_history.jsonl` has no
+ * per-call timestamps, so every call shares `day` (summary time, else file
+ * mtime). Updates are read only when the chat history names no Imagine call,
+ * so the same invocation is not billed from both files. A PortOS scratch
+ * cwd is skipped entirely.
+ */
+export function imagineEventsFromGrokSession({
+ chatText = '', updatesText = '', sessionId, cwd, day,
+} = {}) {
+ if (!day || isPortosGrokScratch(cwd)) return [];
+ const fromChat = eventsFromToolText(chatText, { sessionId, day });
+ if (fromChat.length) return fromChat;
+ return eventsFromToolText(updatesText, { sessionId, day });
+}
+
+export function galleryImageUsageEvent(meta) {
+ if (!meta || meta.mode !== 'grok' || !meta.id) return null;
+ const day = dayFromIso(meta.createdAt);
+ if (!day) return null;
+ const refs = Array.isArray(meta.referenceImageFilenames) ? meta.referenceImageFilenames.length : 0;
+ return {
+ id: `gallery:${meta.id}`,
+ day,
+ model: GROK_IMAGINE_IMAGE_MODEL,
+ images: 1,
+ inputImages: (meta.initImageFilename ? 1 : 0) + refs,
+ videoSeconds: 0,
+ source: 'measured',
+ };
+}
+
+export function videoHistoryUsageEvents(entry) {
+ if (!entry || entry.modelId !== 'grok' || !entry.id) return [];
+ const day = dayFromIso(entry.createdAt);
+ if (!day) return [];
+ const units = grokVideoBillableUnits({
+ requestedSeconds: entry.duration,
+ deliveredSeconds: Number.isFinite(Number(entry.deliveredSec)) ? Number(entry.deliveredSec) : null,
+ hasSourceImage: entry.mode === 'image',
+ });
+ const events = [{
+ id: `video:${entry.id}`,
+ day,
+ model: GROK_IMAGINE_VIDEO_MODEL,
+ images: 0,
+ videoSeconds: units.videoSeconds,
+ inputImages: units.inputImages,
+ source: Number.isFinite(Number(entry.deliveredSec)) ? 'measured' : 'estimate',
+ }];
+ if (units.images > 0) {
+ events.push({
+ id: `video-base:${entry.id}`,
+ day,
+ model: GROK_IMAGINE_IMAGE_MODEL,
+ images: 1,
+ inputImages: 0,
+ videoSeconds: 0,
+ source: 'measured',
+ });
+ }
+ return events;
+}
+
+/**
+ * @param {{ turns?: object[], media?: object[], repairIds?: string[], watermarkMs?: number|null, providerId?: string }} input
+ * @returns {{ additions: object[], notes: string[] }}
+ */
+export function planGrokUsageRepair({
+ turns = [], media = [], repairIds = [], watermarkMs = null, providerId = 'grok',
+} = {}) {
+ const known = new Set(repairIds.filter((id) => typeof id === 'string' && id));
+ const additions = [];
+ const notes = [];
+ if (watermarkMs == null) {
+ notes.push('Long-context premiums were not added because no Grok usage scan has run yet. The next scan records those turns with the premium included.');
+ }
+ for (const turn of turns) {
+ const promptTokens = turn?.promptTokens ?? ((turn?.tokensIn || 0) + (turn?.cacheReadTokens || 0));
+ if (promptTokens < GROK_LONG_CONTEXT_PROMPT_TOKENS) continue;
+ const id = grokTurnRepairId(turn.sessionId, turn.key);
+ if (known.has(id)) continue;
+ if (watermarkMs == null || turn.ms == null || turn.ms > watermarkMs) continue;
+ const day = dayFromMs(turn.ms);
+ if (!day) continue;
+ known.add(id);
+ additions.push({
+ repairId: id,
+ day,
+ record: {
+ providerId,
+ model: turn.model || null,
+ messages: 0,
+ longTokensIn: turn.tokensIn || 0,
+ longTokensOut: turn.tokensOut || 0,
+ longCacheReadTokens: turn.cacheReadTokens || 0,
+ longCacheWriteTokens: turn.cacheWriteTokens || 0,
+ source: 'measured',
+ },
+ });
+ }
+ for (const item of media) {
+ if (!item?.id || !item.day || known.has(item.id)) continue;
+ known.add(item.id);
+ additions.push({
+ repairId: item.id,
+ day: item.day,
+ record: {
+ providerId,
+ model: item.model,
+ messages: 0,
+ images: item.images || 0,
+ videoSeconds: item.videoSeconds || 0,
+ inputImages: item.inputImages || 0,
+ source: item.source || 'measured',
+ },
+ });
+ }
+ return { additions, notes };
+}
+
+/** Counts for the reconcile status line. A text-to-video base still counts as an image. */
+export function summarizeGrokRepairAdditions(additions = []) {
+ const summary = { added: 0, turns: 0, images: 0, videos: 0 };
+ for (const item of additions) {
+ summary.added += 1;
+ if (String(item?.repairId || '').startsWith('turn:')) summary.turns += 1;
+ else if (item?.record?.videoSeconds) summary.videos += 1;
+ else summary.images += 1;
+ }
+ return summary;
+}
diff --git a/server/services/grokUsageRepair.test.js b/server/services/grokUsageRepair.test.js
new file mode 100644
index 0000000000..ab8332c4bb
--- /dev/null
+++ b/server/services/grokUsageRepair.test.js
@@ -0,0 +1,91 @@
+import { describe, expect, it } from 'vitest';
+import {
+ galleryImageUsageEvent,
+ grokTurnRepairId,
+ imagineEventsFromGrokSession,
+ planGrokUsageRepair,
+ summarizeGrokRepairAdditions,
+ videoHistoryUsageEvents,
+} from './grokUsageRepair.js';
+
+const LONG_TURN = {
+ sessionId: 'sess-1',
+ key: 'prompt-long',
+ ms: 1_700_000_000_000,
+ model: 'grok-4.7',
+ promptTokens: 250_000,
+ tokensIn: 200_000,
+ tokensOut: 1_000,
+ cacheReadTokens: 50_000,
+ cacheWriteTokens: 0,
+};
+
+describe('planGrokUsageRepair', () => {
+ it('adds the long-context premium once, and only for turns the scan already passed', () => {
+ const first = planGrokUsageRepair({
+ turns: [LONG_TURN, { ...LONG_TURN, key: 'later', ms: 1_800_000_000_000 }],
+ watermarkMs: 1_750_000_000_000,
+ providerId: 'grok-cli',
+ });
+ expect(first.additions).toHaveLength(1);
+ expect(first.additions[0].repairId).toBe(grokTurnRepairId('sess-1', 'prompt-long'));
+ expect(first.additions[0].record.tokensIn).toBeUndefined();
+ expect(first.additions[0].record.longTokensIn).toBe(200_000);
+ const again = planGrokUsageRepair({
+ turns: [LONG_TURN],
+ repairIds: first.additions.map((item) => item.repairId),
+ watermarkMs: 1_750_000_000_000,
+ });
+ expect(again.additions).toEqual([]);
+ });
+
+ it('skips long-context premiums when the scan has never written a watermark, and still adds media', () => {
+ const plan = planGrokUsageRepair({
+ turns: [LONG_TURN],
+ media: [{ id: 'gallery:job-1', day: '2026-10-01', model: 'grok-imagine-image-2.0', images: 1, source: 'measured' }],
+ watermarkMs: null,
+ });
+ expect(plan.additions.map((item) => item.repairId)).toEqual(['gallery:job-1']);
+ expect(plan.notes[0]).toMatch(/no Grok usage scan has run yet/);
+ expect(summarizeGrokRepairAdditions(plan.additions)).toEqual({ added: 1, turns: 0, images: 1, videos: 0 });
+ });
+
+ it('skips a turn with no timestamp', () => {
+ const plan = planGrokUsageRepair({
+ turns: [{ ...LONG_TURN, ms: null }],
+ watermarkMs: 1_800_000_000_000,
+ });
+ expect(plan.additions).toEqual([]);
+ });
+});
+
+describe('Grok media recovery', () => {
+ it('reads a gallery sidecar and a text-to-video history row', () => {
+ expect(galleryImageUsageEvent({
+ id: 'job-9', mode: 'grok', createdAt: '2026-10-02T03:04:05.000Z',
+ initImageFilename: 'source.png', referenceImageFilenames: ['a.png'],
+ })).toMatchObject({ id: 'gallery:job-9', day: '2026-10-02', images: 1, inputImages: 2 });
+ expect(galleryImageUsageEvent({ id: 'local', mode: 'local', createdAt: '2026-10-02T00:00:00.000Z' })).toBeNull();
+ const video = videoHistoryUsageEvents({
+ id: 'clip-1', modelId: 'grok', mode: 'text', duration: 6, createdAt: '2026-10-03T00:00:00.000Z',
+ });
+ expect(video.map((event) => event.id)).toEqual(['video:clip-1', 'video-base:clip-1']);
+ expect(video[0].videoSeconds).toBe(6);
+ });
+
+ it('ignores Imagine tool calls inside a PortOS scratch directory and defaults a missing duration', () => {
+ const chat = JSON.stringify({
+ type: 'assistant',
+ tool_calls: [{ name: 'image_to_video', arguments: '{"prompt":"a fox"}' }],
+ });
+ expect(imagineEventsFromGrokSession({
+ chatText: chat, sessionId: 's', cwd: '/tmp/portos-grok-abc', day: '2026-10-01',
+ })).toEqual([]);
+ const events = imagineEventsFromGrokSession({
+ chatText: chat, sessionId: 's', cwd: '/home/example/notes', day: '2026-10-01',
+ });
+ expect(events).toEqual([expect.objectContaining({
+ id: 'session-video:s:0', videoSeconds: 6, inputImages: 1, images: 0, source: 'estimate',
+ })]);
+ });
+});
diff --git a/server/services/imageGen/grok.js b/server/services/imageGen/grok.js
index 98e8813748..096bfd8fab 100644
--- a/server/services/imageGen/grok.js
+++ b/server/services/imageGen/grok.js
@@ -324,6 +324,7 @@ async function generateImagePrepared({
runGrok(job, jobId, bin, baseArgs, {
fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA, denoise,
toolName: grokImageTool(inputImages.paths.length > 0),
+ inputImageCount: (inputImages.initPath ? 1 : 0) + inputImages.referencePaths.length,
});
return {
@@ -342,7 +343,7 @@ async function generateImagePrepared({
// leaves the close handler to release resources once it has physically exited.
function runGrok(job, jobId, bin, baseArgs, {
fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA = false, denoise = false,
- toolName = grokImageTool(false),
+ toolName = grokImageTool(false), inputImageCount = 0,
}) {
let cleanupPromptFile = () => {};
// Every terminal path releases both job-owned temp resources. A failed
@@ -465,6 +466,12 @@ function runGrok(job, jobId, bin, baseArgs, {
// Cleaners run BEFORE the SSE complete + completed events so
// subscribers see the cleaned bytes.
await autoCleanGeneratedImage({ cleanC2PA, denoise, pngPath: outputPath, sidecarPath: sidecar, mode: IMAGE_GEN_MODE.GROK });
+ try {
+ const { recordGrokImageUsage } = await import('../grokMediaUsage.js');
+ await recordGrokImageUsage({ jobId, createdAt: meta.createdAt, inputImages: inputImageCount });
+ } catch (usageErr) {
+ console.error(`❌ Failed to record Grok image usage [${jobId.slice(0, 8)}]: ${usageErr.message}`);
+ }
job.status = 'complete';
if (activeProcs.get(jobId) === proc) activeProcs.delete(jobId);
activeJobs.delete(jobId);
diff --git a/server/services/imageGen/grok.test.js b/server/services/imageGen/grok.test.js
index 4497b58ec5..bf7099bc41 100644
--- a/server/services/imageGen/grok.test.js
+++ b/server/services/imageGen/grok.test.js
@@ -83,6 +83,9 @@ vi.mock('../../lib/maintenanceAdmission.js', async importOriginal => ({
maintenance: new Proxy({}, { get: (_target, property) => fixture.admission[property] }),
}));
+const { recordGrokImageUsage } = vi.hoisted(() => ({ recordGrokImageUsage: vi.fn(async () => {}) }));
+vi.mock('../grokMediaUsage.js', () => ({ recordGrokImageUsage }));
+
const grok = await import('./grok.js');
const { imageGenEvents } = await import('../imageGenEvents.js');
const { spawn } = await import('../../lib/childProcess.js');
@@ -343,6 +346,9 @@ describe('grok provider — directed-path harvest', () => {
expect(meta.referenceImageFilenames).toEqual(['ref-a.png', 'ref-b.png']);
expect(Date.parse(meta.renderStartedAt)).toBeLessThanOrEqual(Date.parse(meta.renderCompletedAt));
expect(Date.parse(meta.renderCompletedAt) - Date.parse(meta.renderStartedAt)).toBe(meta.renderMs);
+ expect(recordGrokImageUsage).toHaveBeenCalledWith(expect.objectContaining({
+ jobId: job.jobId, inputImages: 3,
+ }));
});
it('transcodes a staged JPEG to PNG instead of shipping mislabeled bytes', async () => {
diff --git a/server/services/usage.js b/server/services/usage.js
index eb543dea65..0db680d010 100644
--- a/server/services/usage.js
+++ b/server/services/usage.js
@@ -1,6 +1,6 @@
import { join } from 'path';
import { atomicWrite, ensureDir, PATHS, readJSONFile } from '../lib/fileUtils.js';
-import { resolveModelRates, isFreeProvider, isFreeModelId, estimateCostUsd, PRICING_AS_OF } from '../lib/modelPricing.js';
+import { resolveModelRates, isFreeProvider, isFreeModelId, estimateCostUsd, estimateGrokImagineCost, PRICING_AS_OF } from '../lib/modelPricing.js';
import { familyForProvider } from '../lib/providerFamilies.js';
import { roundCents } from '../lib/subscriptionSavings.js';
import { POLLUTING_KEYS } from '../lib/objects.js';
@@ -80,6 +80,9 @@ function getEmptyUsage() {
// Usage page's free-tier section reads as an estimated-quota signal.
// Additive: absent on installs that predate it, seeded on load.
blockEvents: [],
+ // Ids of Grok repairs and live media/long-context records already added.
+ // Local bookkeeping, like reconciledRuns: not part of the peer digest.
+ grokRepairIds: [],
lastUpdated: null
};
}
@@ -203,6 +206,9 @@ export async function loadUsage() {
usageData.blockEvents = [];
normalizedBlocks = true;
}
+ // In memory only until the next real save. Forcing a write here would
+ // rewrite every install's usage.json just to add an empty id list.
+ if (!Array.isArray(usageData.grokRepairIds)) usageData.grokRepairIds = [];
if (rolledUp || normalizedProviders || earliestChanged || normalizedBlocks) {
if (normalizedProviders) console.log('📊 Normalized undefined usage providers to unknown');
if (rolledUp) console.log(`📊 Rolled up old daily usage into ${Object.keys(usageData.monthlyActivity).length} monthly buckets`);
@@ -290,21 +296,42 @@ function modelDayBucket(providerDay, model) {
return providerDay.byModel[model];
}
-const countFields = ['messages', 'tokensIn', 'tokensOut', 'cacheReadTokens', 'cacheWriteTokens'];
+// `long*` is a subset of the base token fields (already included there), priced
+// a second time by the report. `images` / `videoSeconds` / `inputImages` are
+// Imagine units, priced by estimateGrokImagineCost rather than as tokens.
+const countFields = [
+ 'messages', 'tokensIn', 'tokensOut', 'cacheReadTokens', 'cacheWriteTokens',
+ 'longTokensIn', 'longTokensOut', 'longCacheReadTokens', 'longCacheWriteTokens',
+ 'images', 'videoSeconds', 'inputImages',
+];
+
+const zeroCounts = () => Object.fromEntries(countFields.map((field) => [field, 0]));
const recordTotals = (records) => (Array.isArray(records) ? records : [records]).reduce((total, record) => {
for (const field of countFields) total[field] += Math.max(0, record?.[field] || 0);
return total;
-}, { messages: 0, tokensIn: 0, tokensOut: 0, cacheReadTokens: 0, cacheWriteTokens: 0 });
+}, zeroCounts());
const adjustCounts = (bucket, counts, direction) => {
if (!bucket) return;
for (const field of countFields) {
- const next = (bucket[field] || 0) + direction * Math.max(0, counts?.[field] || 0);
- bucket[field] = Math.max(0, next);
+ const delta = direction * Math.max(0, counts?.[field] || 0);
+ // A zero delta must not stamp a new key onto a pre-existing bucket.
+ if (!delta && bucket[field] == null) continue;
+ bucket[field] = Math.max(0, (bucket[field] || 0) + delta);
}
};
+function rememberRepairIds(ids) {
+ if (!Array.isArray(ids) || !usageData) return;
+ usageData.grokRepairIds ??= [];
+ for (const id of ids) {
+ if (typeof id !== 'string' || id.length === 0 || id.length > 400) continue;
+ if (usageData.grokRepairIds.includes(id)) continue;
+ usageData.grokRepairIds.push(id);
+ }
+}
+
const hasUsageCounts = (bucket) => countFields.some((field) => (bucket?.[field] || 0) > 0);
const BACKFILL_YIELD_INTERVAL = 25;
const yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve));
@@ -392,6 +419,43 @@ export async function markUsageRunReconciled(runId, { siblings = false } = {}) {
* Flat day totals are rebuilt from the provider split so report residual
* reconciliation cannot resurrect the removed estimate as a synthetic legacy row.
*/
+
+/**
+ * Add one usage record to its own provider's day bucket and the all-time
+ * rollups. `long*` and Imagine units ride the same count fields as tokens.
+ * Long-context tokens are NOT added to totalTokens: they are a subset of
+ * tokens that were, or will be, counted in the base fields.
+ */
+function addUsageRecord(day, record) {
+ if (!record?.providerId || !isUsageKey(record.providerId)) return;
+ const bucket = providerDayBucket(day, record.providerId, record.providerName);
+ const hadProviderCounts = hasUsageCounts(bucket);
+ adjustCounts(bucket, record, 1);
+ bucket.source = hadProviderCounts
+ ? mergeSource(bucket.source, record.source || 'measured')
+ : (record.source || 'measured');
+ if (record.model && usageModelKey(record.model)) {
+ const target = modelDayBucket(bucket, record.model);
+ const hadCounts = hasUsageCounts(target);
+ adjustCounts(target, record, 1);
+ target.source = hadCounts
+ ? mergeSource(target.source, record.source || 'measured')
+ : (record.source || 'measured');
+ }
+ usageData.totalMessages = Math.max(0, (usageData.totalMessages || 0) + (record.messages || 0));
+ usageData.totalTokens.input = Math.max(0, (usageData.totalTokens.input || 0)
+ + (record.tokensIn || 0) + (record.cacheReadTokens || 0) + (record.cacheWriteTokens || 0));
+ usageData.totalTokens.output = Math.max(0, (usageData.totalTokens.output || 0) + (record.tokensOut || 0));
+ usageData.byProvider[record.providerId] ??= { name: bucket.name, sessions: 0, messages: 0, tokens: 0 };
+ usageData.byProvider[record.providerId].messages += record.messages || 0;
+ usageData.byProvider[record.providerId].tokens += record.tokensOut || 0;
+ if (record.model && usageModelKey(record.model)) {
+ usageData.byModel[record.model] ??= { sessions: 0, messages: 0, tokens: 0 };
+ usageData.byModel[record.model].messages += record.messages || 0;
+ usageData.byModel[record.model].tokens += record.tokensOut || 0;
+ }
+}
+
export async function applyHistoricalUsageCorrections(corrections = []) {
if (!usageData) await loadUsage();
usageData.reconciledRuns ??= {};
@@ -438,37 +502,7 @@ export async function applyHistoricalUsageCorrections(corrections = []) {
}
}
- // Add one usage record to its OWN provider's day bucket and to the flat
- // all-time rollups. Shared by the parent swap and the sibling add — the only
- // difference between them is which bucket `record.providerId` names.
- const addRecord = (day, record) => {
- const bucket = providerDayBucket(day, record.providerId);
- const hadProviderCounts = hasUsageCounts(bucket);
- adjustCounts(bucket, record, 1);
- bucket.source = hadProviderCounts
- ? mergeSource(bucket.source, record.source || 'measured')
- : (record.source || 'measured');
- if (record?.model) {
- const target = modelDayBucket(bucket, record.model);
- const hadCounts = hasUsageCounts(target);
- adjustCounts(target, record, 1);
- target.source = hadCounts
- ? mergeSource(target.source, record.source || 'measured')
- : (record.source || 'measured');
- }
- usageData.totalMessages = Math.max(0, (usageData.totalMessages || 0) + (record.messages || 0));
- usageData.totalTokens.input = Math.max(0, (usageData.totalTokens.input || 0)
- + (record.tokensIn || 0) + (record.cacheReadTokens || 0) + (record.cacheWriteTokens || 0));
- usageData.totalTokens.output = Math.max(0, (usageData.totalTokens.output || 0) + (record.tokensOut || 0));
- usageData.byProvider[record.providerId] ??= { name: bucket.name, sessions: 0, messages: 0, tokens: 0 };
- usageData.byProvider[record.providerId].messages += record.messages || 0;
- usageData.byProvider[record.providerId].tokens += record.tokensOut || 0;
- if (record?.model) {
- usageData.byModel[record.model] ??= { sessions: 0, messages: 0, tokens: 0 };
- usageData.byModel[record.model].messages += record.messages || 0;
- usageData.byModel[record.model].tokens += record.tokensOut || 0;
- }
- };
+ const addRecord = (day, record) => addUsageRecord(day, record);
for (const [index, correction] of eligible.entries()) {
if (index > 0 && index % BACKFILL_YIELD_INTERVAL === 0) await yieldToEventLoop();
@@ -540,8 +574,12 @@ export async function applyHistoricalUsageCorrections(corrections = []) {
for (const record of correction.siblings) {
if (!record?.providerId) continue;
addRecord(day, record);
+ rememberRepairIds(record.repairIds);
}
}
+ // Same save as the counts. A later Grok repair skips any id recorded here,
+ // so a turn billed with its long-context subset is not billed again.
+ rememberRepairIds(correction.repairIds);
// Marked whenever this run's sibling pass actually ran — including a run
// whose only correction was the parent swap — so a second backfill can't
// add the same nested session twice.
@@ -649,6 +687,10 @@ function applyMessageUsage(providerId, rawModel, messageCount, outputTokens = 0,
const model = usageModelKey(rawModel);
const cacheReadTokens = Math.max(0, extra?.cacheReadTokens || 0);
const cacheWriteTokens = Math.max(0, extra?.cacheWriteTokens || 0);
+ const longTokensIn = Math.max(0, extra?.longTokensIn || 0);
+ const longTokensOut = Math.max(0, extra?.longTokensOut || 0);
+ const longCacheReadTokens = Math.max(0, extra?.longCacheReadTokens || 0);
+ const longCacheWriteTokens = Math.max(0, extra?.longCacheWriteTokens || 0);
const source = extra?.source === 'measured' ? 'measured' : 'estimate';
usageData.totalMessages += messageCount;
@@ -688,6 +730,12 @@ function applyMessageUsage(providerId, rawModel, messageCount, outputTokens = 0,
// pass over its whole history.
bucket.cacheReadTokens = (bucket.cacheReadTokens ?? 0) + cacheReadTokens;
bucket.cacheWriteTokens = (bucket.cacheWriteTokens ?? 0) + cacheWriteTokens;
+ // Long-context counts are a subset. Write them only when this call has
+ // some, so a short turn does not stamp zeros onto an older bucket.
+ if (longTokensIn || bucket.longTokensIn != null) bucket.longTokensIn = (bucket.longTokensIn ?? 0) + longTokensIn;
+ if (longTokensOut || bucket.longTokensOut != null) bucket.longTokensOut = (bucket.longTokensOut ?? 0) + longTokensOut;
+ if (longCacheReadTokens || bucket.longCacheReadTokens != null) bucket.longCacheReadTokens = (bucket.longCacheReadTokens ?? 0) + longCacheReadTokens;
+ if (longCacheWriteTokens || bucket.longCacheWriteTokens != null) bucket.longCacheWriteTokens = (bucket.longCacheWriteTokens ?? 0) + longCacheWriteTokens;
bucket.source = mergeSource(bucket.source, source);
};
const day = todayBucket();
@@ -743,11 +791,21 @@ export async function recordRunUsage(record) {
tokensOut = 0,
cacheReadTokens = 0,
cacheWriteTokens = 0,
- source = 'estimate'
+ longTokensIn = 0,
+ longTokensOut = 0,
+ longCacheReadTokens = 0,
+ longCacheWriteTokens = 0,
+ source = 'estimate',
+ repairIds = null,
} = entry || {};
+ if (Array.isArray(repairIds)) rememberRepairIds(repairIds);
applyMessageUsage(providerId, model, messages, tokensOut, tokensIn, {
cacheReadTokens,
cacheWriteTokens,
+ longTokensIn,
+ longTokensOut,
+ longCacheReadTokens,
+ longCacheWriteTokens,
source
});
}
@@ -755,6 +813,34 @@ export async function recordRunUsage(record) {
await saveUsage();
}
+/**
+ * Add counts on a specific UTC day, optionally guarded by a repair id so the
+ * same image, video, or long-context turn cannot be added twice. `flush: false`
+ * leaves the write to {@link flushUsageLedger} so a backfill is one save.
+ * @returns {Promise<boolean>} false when the id was already recorded or the day is unusable.
+ */
+export async function addDatedUsage(dayKey, record, { repairId = null, flush = true } = {}) {
+ if (!DAY_KEY_RE.test(dayKey || '') || !record) return false;
+ if (!usageData) await loadUsage();
+ usageData.grokRepairIds ??= [];
+ if (repairId && usageData.grokRepairIds.includes(repairId)) return false;
+ if (!usageData.dailyActivity[dayKey]) {
+ usageData.dailyActivity[dayKey] = { sessions: 0, messages: 0, tokens: 0, byProvider: {} };
+ cacheActivityDay(dayKey);
+ }
+ addUsageRecord(usageData.dailyActivity[dayKey], record);
+ rebuildDayTotals(usageData.dailyActivity[dayKey]);
+ if (repairId) rememberRepairIds([repairId]);
+ if (flush) await saveUsage();
+ return true;
+}
+
+/** Persist the in-memory ledger. One call after a batch of `addDatedUsage(..., { flush: false })`. */
+export async function flushUsageLedger() {
+ if (!usageData) return;
+ await saveUsage();
+}
+
/**
* Record token usage
*/
@@ -883,6 +969,21 @@ export function providerVolumeSinceBlock(dailyActivity, monthlyActivity, provide
* per-1M tiers) and a `source` of `measured` | `estimate` | `mixed` — buckets
* written before #3124 report `estimate`, since that is what their counts are.
*/
+/**
+ * Base token cost, plus one extra base-rate charge for the long-context
+ * subset, plus Imagine unit cost. The subset is already inside the base
+ * counts, so this is 2× for those tokens and 1× for everything else.
+ */
+function usageRowCost(modelId, counts, rates, free) {
+ if (free) return 0;
+ const tokens = rates ? estimateCostUsd(counts?.tokensIn, counts?.tokensOut, rates, counts) : 0;
+ const premium = rates ? estimateCostUsd(counts?.longTokensIn, counts?.longTokensOut, rates, {
+ cacheReadTokens: counts?.longCacheReadTokens,
+ cacheWriteTokens: counts?.longCacheWriteTokens,
+ }) : 0;
+ return tokens + premium + estimateGrokImagineCost(modelId, counts);
+}
+
export function buildUsageReport(dailyActivity, { from = null, to = null, providers = [], monthlyActivity = null, totalTokens = null } = {}) {
const configById = new Map((providers || []).map((p) => [p.id, p]));
const agg = new Map(); // providerId -> { name, sessions, messages, tokensIn, tokensOut, byModel: Map }
@@ -898,6 +999,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
tokensOut: 0,
cacheReadTokens: 0,
cacheWriteTokens: 0,
+ longTokensIn: 0,
+ longTokensOut: 0,
+ longCacheReadTokens: 0,
+ longCacheWriteTokens: 0,
+ images: 0,
+ videoSeconds: 0,
+ inputImages: 0,
source: null,
byModel: new Map()
});
@@ -922,10 +1030,17 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
p.tokensOut += pDay.tokensOut || 0;
p.cacheReadTokens += pDay.cacheReadTokens || 0;
p.cacheWriteTokens += pDay.cacheWriteTokens || 0;
+ p.longTokensIn += pDay.longTokensIn || 0;
+ p.longTokensOut += pDay.longTokensOut || 0;
+ p.longCacheReadTokens += pDay.longCacheReadTokens || 0;
+ p.longCacheWriteTokens += pDay.longCacheWriteTokens || 0;
+ p.images += pDay.images || 0;
+ p.videoSeconds += pDay.videoSeconds || 0;
+ p.inputImages += pDay.inputImages || 0;
// A bucket with counts but no `source` predates the field — those counts
// are estimates, so fold it in as such rather than leaving the row
// unlabeled and letting the UI imply it was measured.
- if ((pDay.tokensIn || pDay.tokensOut || pDay.cacheReadTokens || pDay.cacheWriteTokens || pDay.messages) && !pDay.source) {
+ if (hasUsageCounts(pDay) && !pDay.source) {
p.source = mergeSource(p.source, 'estimate');
} else {
p.source = mergeSource(p.source, pDay.source);
@@ -943,6 +1058,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
tokensOut: 0,
cacheReadTokens: 0,
cacheWriteTokens: 0,
+ longTokensIn: 0,
+ longTokensOut: 0,
+ longCacheReadTokens: 0,
+ longCacheWriteTokens: 0,
+ images: 0,
+ videoSeconds: 0,
+ inputImages: 0,
source: null
});
}
@@ -953,7 +1075,14 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
m.tokensOut += mDay.tokensOut || 0;
m.cacheReadTokens += mDay.cacheReadTokens || 0;
m.cacheWriteTokens += mDay.cacheWriteTokens || 0;
- m.source = mergeSource(m.source, mDay.source || (mDay.tokensIn || mDay.tokensOut || mDay.messages ? 'estimate' : null));
+ m.longTokensIn += mDay.longTokensIn || 0;
+ m.longTokensOut += mDay.longTokensOut || 0;
+ m.longCacheReadTokens += mDay.longCacheReadTokens || 0;
+ m.longCacheWriteTokens += mDay.longCacheWriteTokens || 0;
+ m.images += mDay.images || 0;
+ m.videoSeconds += mDay.videoSeconds || 0;
+ m.inputImages += mDay.inputImages || 0;
+ m.source = mergeSource(m.source, mDay.source || (hasUsageCounts(mDay) ? 'estimate' : null));
}
}
const residualSessions = Math.max(0, (bucket.sessions || 0) - representedSessions);
@@ -1023,6 +1152,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
tokensOut: 0,
cacheReadTokens: 0,
cacheWriteTokens: 0,
+ longTokensIn: 0,
+ longTokensOut: 0,
+ longCacheReadTokens: 0,
+ longCacheWriteTokens: 0,
+ images: 0,
+ videoSeconds: 0,
+ inputImages: 0,
estimatedCost: 0
};
const providerRows = [];
@@ -1043,7 +1179,10 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
// measured at $131 for one day of it.
const freeModel = free || isFreeModelId(model);
const rates = freeModel ? null : resolveModelRates(pid, model);
- const cost = freeModel ? 0 : estimateCostUsd(m.tokensIn, m.tokensOut, rates, m);
+ // The long-context subset is already inside the base counts. One extra
+ // pass at the same short rates is the missing half of xAI's 2× bill.
+ // Imagine units are not tokens; they price off the model id.
+ const cost = usageRowCost(model, m, rates, freeModel);
providerCost += cost;
models.push({
model,
@@ -1053,6 +1192,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
tokensOut: m.tokensOut,
cacheReadTokens: m.cacheReadTokens,
cacheWriteTokens: m.cacheWriteTokens,
+ longTokensIn: m.longTokensIn,
+ longTokensOut: m.longTokensOut,
+ longCacheReadTokens: m.longCacheReadTokens,
+ longCacheWriteTokens: m.longCacheWriteTokens,
+ images: m.images,
+ videoSeconds: m.videoSeconds,
+ inputImages: m.inputImages,
source: m.source || 'estimate',
estimatedCost: roundCents(cost),
rateModel: rates?.rateModel ?? null,
@@ -1075,11 +1221,23 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
const unattributedOut = Math.max(0, p.tokensOut - modelTokensOut);
const unattributedCacheRead = Math.max(0, p.cacheReadTokens - modelCacheRead);
const unattributedCacheWrite = Math.max(0, p.cacheWriteTokens - modelCacheWrite);
- if (!free && (unattributedIn > 0 || unattributedOut > 0 || unattributedCacheRead > 0 || unattributedCacheWrite > 0)) {
- providerCost += estimateCostUsd(unattributedIn, unattributedOut, providerRates, {
+ const modelLongIn = models.reduce((s, row) => s + row.longTokensIn, 0);
+ const modelLongOut = models.reduce((s, row) => s + row.longTokensOut, 0);
+ const modelLongCacheRead = models.reduce((s, row) => s + row.longCacheReadTokens, 0);
+ const modelLongCacheWrite = models.reduce((s, row) => s + row.longCacheWriteTokens, 0);
+ if (!free && (unattributedIn > 0 || unattributedOut > 0 || unattributedCacheRead > 0 || unattributedCacheWrite > 0
+ || p.longTokensIn > modelLongIn || p.longTokensOut > modelLongOut
+ || p.longCacheReadTokens > modelLongCacheRead || p.longCacheWriteTokens > modelLongCacheWrite)) {
+ providerCost += usageRowCost(null, {
+ tokensIn: unattributedIn,
+ tokensOut: unattributedOut,
cacheReadTokens: unattributedCacheRead,
- cacheWriteTokens: unattributedCacheWrite
- });
+ cacheWriteTokens: unattributedCacheWrite,
+ longTokensIn: Math.max(0, p.longTokensIn - modelLongIn),
+ longTokensOut: Math.max(0, p.longTokensOut - modelLongOut),
+ longCacheReadTokens: Math.max(0, p.longCacheReadTokens - modelLongCacheRead),
+ longCacheWriteTokens: Math.max(0, p.longCacheWriteTokens - modelLongCacheWrite),
+ }, providerRates, false);
}
totals.sessions += p.sessions;
@@ -1088,6 +1246,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
totals.tokensOut += p.tokensOut;
totals.cacheReadTokens += p.cacheReadTokens;
totals.cacheWriteTokens += p.cacheWriteTokens;
+ totals.longTokensIn += p.longTokensIn;
+ totals.longTokensOut += p.longTokensOut;
+ totals.longCacheReadTokens += p.longCacheReadTokens;
+ totals.longCacheWriteTokens += p.longCacheWriteTokens;
+ totals.images += p.images;
+ totals.videoSeconds += p.videoSeconds;
+ totals.inputImages += p.inputImages;
totals.estimatedCost += providerCost;
providerRows.push({
@@ -1105,6 +1270,13 @@ export function buildUsageReport(dailyActivity, { from = null, to = null, provid
tokensOut: p.tokensOut,
cacheReadTokens: p.cacheReadTokens,
cacheWriteTokens: p.cacheWriteTokens,
+ longTokensIn: p.longTokensIn,
+ longTokensOut: p.longTokensOut,
+ longCacheReadTokens: p.longCacheReadTokens,
+ longCacheWriteTokens: p.longCacheWriteTokens,
+ images: p.images,
+ videoSeconds: p.videoSeconds,
+ inputImages: p.inputImages,
source: p.source || 'estimate',
estimatedCost: roundCents(providerCost),
rateMatch: pid === LEGACY_PROVIDER_ID
diff --git a/server/services/usage.test.js b/server/services/usage.test.js
index 5051f5fec6..130e84ce24 100644
--- a/server/services/usage.test.js
+++ b/server/services/usage.test.js
@@ -1479,3 +1479,39 @@ describe('usage.js — limit-block ledger (#7408)', () => {
expect(summary.freeTier.blocks[0].volumeSince).toMatchObject({ sessions: 1, messages: 1 });
});
});
+
+describe('Grok long-context and Imagine pricing', () => {
+ it('doubles only the long-context subset and prices an image row on its own', () => {
+ const report = buildUsageReport({
+ '2026-10-01': {
+ byProvider: {
+ 'grok-cli': {
+ name: 'Grok',
+ tokensIn: 300_000,
+ tokensOut: 10_000,
+ longTokensIn: 200_000,
+ longTokensOut: 1_000,
+ byModel: {
+ 'grok-4.7': {
+ tokensIn: 300_000,
+ tokensOut: 10_000,
+ longTokensIn: 200_000,
+ longTokensOut: 1_000,
+ source: 'measured',
+ },
+ 'grok-imagine-image-2.0': { images: 1, source: 'measured' },
+ },
+ },
+ },
+ },
+ });
+ const models = Object.fromEntries(report.providers[0].models.map((row) => [row.model, row]));
+ // 300k in + 10k out at $2/$6 = $0.66, plus 200k in + 1k out again = $0.406.
+ expect(models['grok-4.7'].estimatedCost).toBeCloseTo(1.07);
+ expect(models['grok-4.7'].tokensIn).toBe(300_000);
+ expect(models['grok-imagine-image-2.0'].estimatedCost).toBeCloseTo(0.04);
+ expect(models['grok-imagine-image-2.0'].tokensIn).toBe(0);
+ expect(report.totals.tokensIn).toBe(300_000);
+ expect(report.totals.estimatedCost).toBeCloseTo(1.11);
+ });
+});
diff --git a/server/services/usageBackfill.js b/server/services/usageBackfill.js
index 5fb7380cdc..a65e878843 100644
--- a/server/services/usageBackfill.js
+++ b/server/services/usageBackfill.js
@@ -2,6 +2,7 @@ import { notifyUsageBackfillUpdated } from './usageBackfillEvents.js';
import { Worker } from 'worker_threads';
import { homedir } from 'os';
import { atomicWrite, PATHS, readJSONFile } from '../lib/fileUtils.js';
+import { applyGrokUsageRepair } from './grokUsageBackfill.js';
import {
applyHistoricalUsageCorrections,
getReconciledUsageRunIds,
@@ -10,16 +11,19 @@ import {
import { listProviders } from './providers.js';
import { mergeUsageClaims, snapshotUsageClaims } from './usageReconciler.js';
-let job = {
+const idleJob = () => ({
status: 'idle',
processed: 0,
total: 0,
found: 0,
corrected: 0,
+ grok: null,
error: null,
startedAt: null,
completedAt: null
-};
+});
+
+let job = idleJob();
const publicJob = () => ({ ...job });
@@ -62,14 +66,9 @@ export async function startHistoricalUsageBackfill({
if (job.status === 'running') return publicJob();
job = {
+ ...idleJob(),
status: 'running',
- processed: 0,
- total: 0,
- found: 0,
- corrected: 0,
- error: null,
- startedAt: new Date().toISOString(),
- completedAt: null
+ startedAt: new Date().toISOString()
};
notifyUsageBackfillUpdated();
@@ -96,16 +95,45 @@ export async function startHistoricalUsageBackfill({
}
});
+ // One repair per click, including when the transcript worker fails. It
+ // writes the in-memory ledger, so it stays on this thread rather than in
+ // the worker (that thread has its own copy of usage.js).
+ let grokOnce = null;
+ const repairGrok = () => {
+ grokOnce ??= applyGrokUsageRepair({ home, providers: providerList })
+ .then((result) => ({
+ grok: {
+ added: result.added || 0,
+ turns: result.turns || 0,
+ images: result.images || 0,
+ videos: result.videos || 0,
+ backedUp: Boolean(result.backedUp),
+ note: result.note || null,
+ },
+ grokError: null,
+ }))
+ .catch((error) => ({ grok: null, grokError: error.message }));
+ return grokOnce;
+ };
+
+ const fail = (error) => {
+ job = { ...job, status: 'error', error, completedAt: new Date().toISOString() };
+ notifyUsageBackfillUpdated();
+ };
+
let messageTail = Promise.resolve();
worker.on('message', (message) => {
messageTail = messageTail.then(async () => {
if (message?.type === 'progress') {
- job = { ...job, ...message.progress };
+ job = { ...job, ...message.progress, grok: job.grok ?? null };
notifyUsageBackfillUpdated();
return;
}
if (message?.type === 'error') {
- job = { ...job, status: 'error', error: message.error || 'Backfill failed', completedAt: new Date().toISOString() };
+ const repaired = await repairGrok();
+ const error = [message.error || 'Backfill failed', repaired.grokError && `Grok usage repair failed: ${repaired.grokError}`]
+ .filter(Boolean).join('. ');
+ job = { ...job, status: 'error', error, grok: repaired.grok, completedAt: new Date().toISOString() };
notifyUsageBackfillUpdated();
return;
}
@@ -116,6 +144,19 @@ export async function startHistoricalUsageBackfill({
const appliedCorrections = (result.corrections || [])
.filter((correction) => applied.correctedRunIds.includes(correction.runId));
await markRunMetadata(appliedCorrections);
+ const repaired = await repairGrok();
+ if (repaired.grokError) {
+ job = {
+ ...job,
+ status: 'error',
+ error: `Grok usage repair failed: ${repaired.grokError}`,
+ corrected: applied.corrected,
+ grok: null,
+ completedAt: new Date().toISOString()
+ };
+ notifyUsageBackfillUpdated();
+ return;
+ }
job = {
...job,
status: 'complete',
@@ -123,17 +164,22 @@ export async function startHistoricalUsageBackfill({
total: result.total || 0,
found: result.corrections?.length || 0,
corrected: applied.corrected,
+ grok: repaired.grok,
completedAt: new Date().toISOString()
};
notifyUsageBackfillUpdated();
}).catch((error) => {
- job = { ...job, status: 'error', error: error.message, completedAt: new Date().toISOString() };
- notifyUsageBackfillUpdated();
+ fail(error.message);
});
});
worker.on('error', (error) => {
- job = { ...job, status: 'error', error: error.message, completedAt: new Date().toISOString() };
- notifyUsageBackfillUpdated();
+ messageTail = messageTail.then(async () => {
+ const repaired = await repairGrok();
+ const message = [error.message, repaired.grokError && `Grok usage repair failed: ${repaired.grokError}`]
+ .filter(Boolean).join('. ');
+ job = { ...job, status: 'error', error: message, grok: repaired.grok, completedAt: new Date().toISOString() };
+ notifyUsageBackfillUpdated();
+ }).catch((err) => fail(err.message));
});
worker.unref();
return publicJob();
@@ -141,14 +187,5 @@ export async function startHistoricalUsageBackfill({
export function __resetHistoricalUsageBackfillForTests() {
- job = {
- status: 'idle',
- processed: 0,
- total: 0,
- found: 0,
- corrected: 0,
- error: null,
- startedAt: null,
- completedAt: null
- };
+ job = idleJob();
}
diff --git a/server/services/usageBackfill.test.js b/server/services/usageBackfill.test.js
index dbe9839d4f..37792aa9e7 100644
--- a/server/services/usageBackfill.test.js
+++ b/server/services/usageBackfill.test.js
@@ -23,11 +23,18 @@ vi.mock('./providers.js', () => ({
])
}));
+vi.mock('./grokUsageBackfill.js', () => ({
+ applyGrokUsageRepair: vi.fn(async () => ({
+ added: 0, turns: 0, images: 0, videos: 0, backedUp: false, note: null, backupDir: null,
+ })),
+}));
+
const {
__resetHistoricalUsageBackfillForTests,
getHistoricalUsageBackfillStatus,
startHistoricalUsageBackfill
} = await import('./usageBackfill.js');
+const { applyGrokUsageRepair } = await import('./grokUsageBackfill.js');
class FakeWorker extends EventEmitter {
static instances = [];
@@ -46,6 +53,10 @@ beforeEach(() => {
usageBackfillEvents.removeAllListeners();
FakeWorker.instances = [];
__resetHistoricalUsageBackfillForTests();
+ applyGrokUsageRepair.mockReset();
+ applyGrokUsageRepair.mockResolvedValue({
+ added: 0, turns: 0, images: 0, videos: 0, backedUp: false, note: null, backupDir: null,
+ });
});
describe('historical usage backfill job', () => {
@@ -113,6 +124,55 @@ it('notifies clients when the worker fails', async () => {
await startHistoricalUsageBackfill({ WorkerClass: FakeWorker, providers: [] });
updated.mockClear();
FakeWorker.instances[0].emit('error', new Error('Example worker failure'));
+ await vi.waitFor(() => expect(getHistoricalUsageBackfillStatus()).toMatchObject({
+ status: 'error',
+ error: 'Example worker failure',
+ grok: { added: 0 },
+ }));
expect(updated).toHaveBeenCalledWith({});
- expect(getHistoricalUsageBackfillStatus()).toMatchObject({ status: 'error', error: 'Example worker failure' });
+ expect(applyGrokUsageRepair).toHaveBeenCalledTimes(1);
+});
+
+it('puts the Grok repair on the reconcile result and runs it once', async () => {
+ applyGrokUsageRepair.mockResolvedValue({
+ added: 3,
+ turns: 1,
+ images: 2,
+ videos: 0,
+ backedUp: true,
+ note: 'Long-context premiums were not added because no Grok usage scan has run yet.',
+ backupDir: '/tmp/grok-usage-backup',
+ });
+ await startHistoricalUsageBackfill({ WorkerClass: FakeWorker, providers: [] });
+ FakeWorker.instances[0].emit('message', {
+ type: 'complete',
+ result: { processed: 1, total: 1, corrections: [] },
+ });
+ await vi.waitFor(() => expect(getHistoricalUsageBackfillStatus()).toMatchObject({
+ status: 'complete',
+ corrected: 1,
+ grok: {
+ added: 3,
+ turns: 1,
+ images: 2,
+ videos: 0,
+ backedUp: true,
+ note: 'Long-context premiums were not added because no Grok usage scan has run yet.',
+ },
+ }));
+ expect(getHistoricalUsageBackfillStatus().grok.backupDir).toBeUndefined();
+ expect(applyGrokUsageRepair).toHaveBeenCalledTimes(1);
+});
+
+it('keeps a transcript failure from dropping a Grok repair that already landed', async () => {
+ applyGrokUsageRepair.mockResolvedValue({
+ added: 1, turns: 0, images: 1, videos: 0, backedUp: true, note: null, backupDir: '/tmp/backup',
+ });
+ await startHistoricalUsageBackfill({ WorkerClass: FakeWorker, providers: [] });
+ FakeWorker.instances[0].emit('error', new Error('Example worker failure'));
+ await vi.waitFor(() => expect(getHistoricalUsageBackfillStatus()).toMatchObject({
+ status: 'error',
+ error: 'Example worker failure',
+ grok: { added: 1, images: 1, backedUp: true },
+ }));
});
diff --git a/server/services/usageInteractiveScan.js b/server/services/usageInteractiveScan.js
index 008999cc8c..ba271af3b8 100644
--- a/server/services/usageInteractiveScan.js
+++ b/server/services/usageInteractiveScan.js
@@ -23,6 +23,7 @@ import {
resolveFamilyProvider
} from './usageReconciler.js';
import { applyHistoricalUsageCorrections, forgetSiblingReconciledUsageRuns } from './usage.js';
+import { grokTurnRepairId } from './grokUsageRepair.js';
/**
* Attribute Claude Code, Codex, Grok and Antigravity sessions that PortOS never
@@ -170,7 +171,10 @@ async function listSessions(family, home, from) {
return sessions;
}
-const BUCKET_FIELDS = ['messages', 'tokensIn', 'tokensOut', 'cacheReadTokens', 'cacheWriteTokens'];
+const BUCKET_FIELDS = [
+ 'messages', 'tokensIn', 'tokensOut', 'cacheReadTokens', 'cacheWriteTokens',
+ 'longTokensIn', 'longTokensOut', 'longCacheReadTokens', 'longCacheWriteTokens',
+];
const addBuckets = (target, buckets) => {
for (const field of BUCKET_FIELDS) target[field] = (target[field] || 0) + (buckets[field] || 0);
};
@@ -193,12 +197,21 @@ function parseSlice(family, entries, slice, seen) {
if (family === 'claude') {
const parsed = parseClaudeTranscript(entries, { from: slice.from, to: slice.to, exclude: seen });
for (const key of parsed.countedKeys) seen.add(key);
- return parsed.byModel;
+ return { byModel: parsed.byModel, repairIds: [] };
}
// A rollout's totals are cumulative; a windowed read is the slice's delta.
- if (family === 'codex') return parseCodexRollout(entries, { from: slice.from, to: slice.to }).byModel;
- if (family === 'grok') return parseGrokTurns(entries, { from: slice.from, to: slice.to }).byModel;
- return { [UNKNOWN_MODEL]: agyEstimatedBuckets(parseAgyTranscript(entries, { from: slice.from, to: slice.to })) };
+ if (family === 'codex') return { byModel: parseCodexRollout(entries, { from: slice.from, to: slice.to }).byModel, repairIds: [] };
+ if (family === 'grok') {
+ const parsed = parseGrokTurns(entries, { from: slice.from, to: slice.to });
+ return {
+ byModel: parsed.byModel,
+ // Registered in the same correction save as the long-context subset, so
+ // a later repair of turns already at or behind the watermark does not
+ // add that premium a second time.
+ repairIds: (parsed.longTurns || []).map((turn) => grokTurnRepairId(turn.sessionId, turn.key)),
+ };
+ }
+ return { byModel: { [UNKNOWN_MODEL]: agyEstimatedBuckets(parseAgyTranscript(entries, { from: slice.from, to: slice.to })) }, repairIds: [] };
}
/** One session file's billed buckets per `{ day, model }` inside `[from, to]`, minus run windows. */
@@ -217,8 +230,13 @@ async function parseSession(family, session, from, to, byCwd, seen) {
const results = [];
for (const [a, b] of subtractWindows(lo, hi, windowsFor(byCwd, cwd))) {
for (const slice of splitByUtcDay(a, b)) {
- for (const [model, buckets] of Object.entries(parseSlice(family, entries, slice, seen))) {
- if (totalTranscriptTokens(buckets) > 0) results.push({ day: slice.day, model, buckets });
+ const parsed = parseSlice(family, entries, slice, seen);
+ let attachedIds = false;
+ for (const [model, buckets] of Object.entries(parsed.byModel)) {
+ if (totalTranscriptTokens(buckets) <= 0) continue;
+ const repairIds = attachedIds ? [] : (parsed.repairIds || []);
+ attachedIds = true;
+ results.push({ day: slice.day, model, buckets, repairIds });
}
}
}
@@ -254,12 +272,13 @@ export const refreshInteractiveUsage = ({ home = homedir(), runsDir = PATHS.runs
const perDay = new Map();
const seen = new Set();
for (const session of await listSessions(family, home, from)) {
- for (const { day, model, buckets } of await parseSession(family, session, from, to, byCwd, seen)) {
- if (!perDay.has(day)) perDay.set(day, { byModel: {} });
+ for (const { day, model, buckets, repairIds } of await parseSession(family, session, from, to, byCwd, seen)) {
+ if (!perDay.has(day)) perDay.set(day, { byModel: {}, repairIds: [] });
const dayTotals = perDay.get(day);
addBuckets(dayTotals, buckets);
dayTotals.byModel[model] ??= {};
addBuckets(dayTotals.byModel[model], buckets);
+ if (repairIds?.length) dayTotals.repairIds.push(...repairIds);
}
// A first scan parses many multi-MB files back to back; let the server breathe.
await yieldToEventLoop();
@@ -280,7 +299,8 @@ export const refreshInteractiveUsage = ({ home = homedir(), runsDir = PATHS.runs
day,
providerId: provider.id,
siblings: recordsFromMeasured(provider.id, provider.defaultModel ?? null, measured, 'sibling'),
- siblingScanned: true
+ siblingScanned: true,
+ repairIds: totals.repairIds || [],
});
}
if (corrections.length) {
diff --git a/server/services/usageReconciler.js b/server/services/usageReconciler.js
index d60803fab1..8637a5b583 100644
--- a/server/services/usageReconciler.js
+++ b/server/services/usageReconciler.js
@@ -62,6 +62,7 @@ import { familyForProvider } from '../lib/providerFamilies.js';
import { commandBasename } from '../lib/providerModels.js';
import { summarizeOpenCodeEvents } from '../lib/opencodeStream.js';
import { markUsageRunReconciled, recordRunUsage } from './usage.js';
+import { grokTurnRepairId } from './grokUsageRepair.js';
// Widen the correlation window past the recorded run bounds: the CLI writes its
// first line slightly before PortOS stamps startTime, and flushes its last
@@ -394,11 +395,26 @@ export const agyEstimatedBuckets = (parsed) => ({
const TOKEN_FIELDS = ['messages', 'tokensIn', 'tokensOut', 'cacheReadTokens', 'cacheWriteTokens'];
+const LONG_USAGE_FIELDS = ['longTokensIn', 'longTokensOut', 'longCacheReadTokens', 'longCacheWriteTokens'];
+
function addBuckets(target, source) {
for (const field of TOKEN_FIELDS) target[field] = (target[field] || 0) + (source[field] || 0);
+ // Copied only when the source has some, so a Codex high-water object that
+ // spreads addBuckets does not grow long-context keys it never had.
+ for (const field of LONG_USAGE_FIELDS) {
+ if (source?.[field]) target[field] = (target[field] || 0) + source[field];
+ }
return target;
}
+function longUsageFrom(bucket) {
+ const extra = {};
+ for (const field of LONG_USAGE_FIELDS) {
+ if (bucket?.[field]) extra[field] = bucket[field];
+ }
+ return extra;
+}
+
function subtractBuckets(absolute, billed) {
return Object.fromEntries(TOKEN_FIELDS.map(field => [field, Math.max(0, (absolute[field] || 0) - (billed?.[field] || 0))]));
}
@@ -663,6 +679,10 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam
sourcesSeen.add(source);
totals.sessions += 1;
addBuckets(totals, parsed);
+ if (parsed.longTurns?.length) {
+ totals.longTurns ??= [];
+ totals.longTurns.push(...parsed.longTurns);
+ }
for (const model of parsed.models?.length ? parsed.models : [parsed.model]) {
if (model) modelCounts.set(model, (modelCounts.get(model) || 0) + 1);
}
@@ -745,9 +765,10 @@ export function recordsFromMeasured(providerId, recordedModel, measured, role) {
// `mixed` on every record it produced: the two shapes fold into the same
// per-model buckets, so no record can honestly claim to be purely measured.
const source = measured.source || 'measured';
+ const repairIds = (measured.longTurns || []).map((turn) => grokTurnRepairId(turn.sessionId, turn.key));
const perModel = Object.entries(measured.byModel || {});
- if (perModel.length > 0) {
- return perModel.map(([model, bucket]) => ({
+ const records = perModel.length > 0
+ ? perModel.map(([model, bucket]) => ({
providerId,
role,
model: attributedModel(recordedModel ?? null, model, perModel.length === 1),
@@ -756,23 +777,29 @@ export function recordsFromMeasured(providerId, recordedModel, measured, role) {
tokensOut: bucket.tokensOut,
cacheReadTokens: bucket.cacheReadTokens,
cacheWriteTokens: bucket.cacheWriteTokens,
+ ...longUsageFrom(bucket),
source
- }));
- }
- return [{
- providerId,
- role,
- // Prefer the model PortOS recorded (it carries the provider's own id shape,
- // e.g. a Bedrock-prefixed id the pricing table resolves); fall back to the
- // transcript's when PortOS captured none.
- model: recordedModel ?? measured.model ?? null,
- messages: measured.messages || 1,
- tokensIn: measured.tokensIn,
- tokensOut: measured.tokensOut,
- cacheReadTokens: measured.cacheReadTokens,
- cacheWriteTokens: measured.cacheWriteTokens,
- source
- }];
+ }))
+ : [{
+ providerId,
+ role,
+ // Prefer the model PortOS recorded (it carries the provider's own id shape,
+ // e.g. a Bedrock-prefixed id the pricing table resolves); fall back to the
+ // transcript's when PortOS captured none.
+ model: recordedModel ?? measured.model ?? null,
+ messages: measured.messages || 1,
+ tokensIn: measured.tokensIn,
+ tokensOut: measured.tokensOut,
+ cacheReadTokens: measured.cacheReadTokens,
+ cacheWriteTokens: measured.cacheWriteTokens,
+ ...longUsageFrom(measured),
+ source
+ }];
+ // One id list on the first record. recordRunUsage persists it in the same
+ // save as the counts, and the array shape the tests lock stays unchanged
+ // when this transcript had no long-context turn.
+ if (repairIds.length) records[0].repairIds = repairIds;
+ return records;
}
// `lib/providerFamilies.js` deliberately excludes `kimi` from PROVIDER_FAMILIES
diff --git a/server/services/videoGen/grok.js b/server/services/videoGen/grok.js
index 684aa69e16..17db9d895b 100644
--- a/server/services/videoGen/grok.js
+++ b/server/services/videoGen/grok.js
@@ -37,7 +37,7 @@ import { videoGenEvents } from './events.js';
import { discardUnpublishedVideo, finalizeGeneratedVideo, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';
import { mutateVideoHistory } from './history.js';
import { noImageReason, deriveAspectRatio, GROK_ASPECT_RATIOS } from '../imageGen/grok.js';
-import { resolveGrokDuration } from '../../lib/grokVideoClip.js';
+import { grokVideoBillableUnits, resolveGrokDuration } from '../../lib/grokVideoClip.js';
import { withSpawnCwdEnv } from '../../lib/spawnCwd.js';
import { probeVideoDuration } from '../../lib/ffmpeg.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
@@ -337,6 +337,17 @@ async function runGrokVideo(job, jobId, bin, args, {
meta.deliveredSec = Math.round(deliveredSec * 100) / 100;
if (deliveredSec < meta.duration - 1) console.warn(`⚠️ grok delivered ${meta.deliveredSec}s for a ${meta.duration}s request [${jobId.slice(0, 8)}]`);
}
+ try {
+ const units = grokVideoBillableUnits({
+ requestedSeconds: meta.duration,
+ deliveredSeconds: Number.isFinite(deliveredSec) ? deliveredSec : null,
+ hasSourceImage: meta.mode === 'image',
+ });
+ const { recordGrokVideoUsage } = await import('../grokMediaUsage.js');
+ await recordGrokVideoUsage({ jobId, createdAt: meta.createdAt, ...units });
+ } catch (usageErr) {
+ console.error(`❌ Failed to record Grok video usage [${jobId.slice(0, 8)}]: ${usageErr.message}`);
+ }
// Removing the scratch directory also removes the harvested staging file.
await cleanup();
if (activeProcs.get(jobId) === proc) activeProcs.delete(jobId);
diff --git a/server/services/videoGen/grok.test.js b/server/services/videoGen/grok.test.js
index 6f12e1adfd..29a6977518 100644
--- a/server/services/videoGen/grok.test.js
+++ b/server/services/videoGen/grok.test.js
@@ -87,6 +87,9 @@ vi.mock('../../lib/ffmpeg.js', async () => {
};
});
+const { recordGrokVideoUsage } = vi.hoisted(() => ({ recordGrokVideoUsage: vi.fn(async () => {}) }));
+vi.mock('../grokMediaUsage.js', () => ({ recordGrokVideoUsage }));
+
const grok = await import('./grok.js');
const { videoGenEvents } = await import('./events.js');
const { loadHistory } = await import('./history.js');
@@ -228,6 +231,9 @@ describe('videoGen/grok — harvest and finalize', () => {
expect(existsSync(scratchDirFor(job.jobId))).toBe(false);
const history = await loadHistory();
expect(history[0]).toEqual(expect.objectContaining({ id: job.jobId, modelId: 'grok', mode: 'text', duration: 6 }));
+ expect(recordGrokVideoUsage).toHaveBeenCalledWith(expect.objectContaining({
+ jobId: job.jobId, videoSeconds: 6, images: 1, inputImages: 1,
+ }));
});
it('records the delivered length and warns when a 15 s request comes back short', async () => {