The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { expect, it } from "vitest";
import { estimateProviderCost, normalizeProviderUsage, ProviderUsageAccumulator, type EvalModelPrice } from "./provider-usage.js";
const price: EvalModelPrice = { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.test/prices", inputUsdPerMillion: "2.5", cachedReadUsdPerMillion: "0.25", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" };
it("excludes cached reads from OpenAI input and treats reasoning as a subset of output", () => {
const usage = normalizeProviderUsage("openai", { id: "response", model: "fixture", usage: { input_tokens: 100, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 40 }, output_tokens: 20, output_tokens_details: { reasoning_tokens: 15 } } });
expect(usage).toMatchObject({ complete: true, tokens: { input: 60, cachedRead: 40, cachedWrite: 0, output: 20, reasoning: 15 } });
expect(estimateProviderCost(usage, price)).toBe("0.000360000");
});
it("never turns missing usage or incompatible model/pricing into a free charge", () => {
const absent = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, output_tokens: 0 } });
expect(absent.complete).toBe(false);
expect(estimateProviderCost(absent, price)).toBeNull();
const zero = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 0 } });
expect(estimateProviderCost(zero, price)).toBe("0.000000000");
expect(estimateProviderCost(zero, { ...price, model: "other" })).toBeNull();
});
it("does not add Anthropic cumulative output counters and prices cache writes by TTL", () => {
const stream = new ProviderUsageAccumulator("anthropic");
stream.observe({ type: "message_start", message: { id: "message", model: "fixture", usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 20, cache_creation_input_tokens: 30, cache_creation: { ephemeral_5m_input_tokens: 20, ephemeral_1h_input_tokens: 10 } } } });
stream.observe({ type: "message_delta", usage: { output_tokens: 5 } });
stream.observe({ type: "message_delta", usage: { output_tokens: 8 } });
expect(stream.receipt().final).toBe(false);
stream.observe({ type: "message_stop" });
expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 10, cachedRead: 20, cachedWrite: 30, output: 8 } });
expect(estimateProviderCost(stream.receipt(), { ...price, provider: "anthropic", cacheWrite5mUsdPerMillion: "3.125", cacheWrite1hUsdPerMillion: "5" })).toBe("0.000222500");
});
it("retains missing cache TTL and malformed token subsets as unknown", () => {
const write = normalizeProviderUsage("anthropic", { model: "fixture", usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 4, output_tokens: 1 } });
expect(estimateProviderCost(write, { ...price, provider: "anthropic" })).toBeNull();
expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 1 } }).complete).toBe(false);
expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1, output_tokens_details: { reasoning_tokens: 2 } } }).complete).toBe(false);
});
it("uses one terminal Responses usage receipt regardless of streamed text or repeat notifications", () => {
const stream = new ProviderUsageAccumulator("openai");
stream.observe({ type: "response.output_text.delta", delta: "not a usage receipt" });
expect(stream.receipt().complete).toBe(false);
const terminal = { type: "response.completed", response: { id: "response", model: "fixture", usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1 } } };
stream.observe(terminal); stream.observe(terminal);
expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 1, output: 1 } });
expect(estimateProviderCost(stream.receipt(), price)).toBe("0.000012500");
});
const modernPrice: EvalModelPrice = { ...price, inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", outputUsdPerMillion: "20",
cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10",
longContext: { aboveInputTokens: 272_000, rates: { inputUsdPerMillion: "8", cachedReadUsdPerMillion: "0.8", outputUsdPerMillion: "30", cacheWrite30mUsdPerMillion: "10", cacheWrite12hUsdPerMillion: "20" } } };
const modernUsage = (total = 12_000, read = 4000, shortWrite = 3000, longWrite = 4000) => ({ model: "fixture", usage: {
input_tokens: total, input_tokens_details: { cached_tokens: read, cache_write_tokens: shortWrite + longWrite,
cache_write_tokens_details: { ttl_30m_tokens: shortWrite, ttl_12h_tokens: longWrite } }, output_tokens: 100,
} });
it("separates modern OpenAI ordinary input and mixed-TTL writes without adding the base rate twice", () => {
const receipt = normalizeProviderUsage("openai", modernUsage());
expect(receipt).toMatchObject({ complete: true, tokens: { input: 1000, cachedRead: 4000, cachedWrite: 7000 }, cacheWrite30m: 3000, cacheWrite12h: 4000 });
expect(estimateProviderCost(receipt, modernPrice)).toBe("0.062600000");
});
it("applies long-context pricing to the entire request using total input including cache", () => {
expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_000, 270_000, 0, 1000)), modernPrice)).toBe("0.124000000");
expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_001, 270_000, 0, 1000)), modernPrice)).toBe("0.247008000");
});
it("keeps missing modern write counters or missing/inconsistent TTL splits unknown", () => {
const missing = modernUsage(); delete (missing.usage.input_tokens_details as any).cache_write_tokens;
expect(normalizeProviderUsage("openai", missing).complete).toBe(false);
expect(estimateProviderCost(normalizeProviderUsage("openai", missing), modernPrice)).toBeNull();
const noTtl = modernUsage(); delete (noTtl.usage.input_tokens_details as any).cache_write_tokens_details;
expect(estimateProviderCost(normalizeProviderUsage("openai", noTtl), modernPrice)).toBeNull();
const mismatch = modernUsage(); mismatch.usage.input_tokens_details.cache_write_tokens_details.ttl_12h_tokens++;
expect(normalizeProviderUsage("openai", mismatch).complete).toBe(false);
});
const policyPrice = { ...modernPrice, model: "gpt-5.6-sol" };
function requestPolicyReceipt(request: Record<string, unknown> = {}, details: Record<string, unknown> = {}) {
const stream = new ProviderUsageAccumulator("openai", { model: policyPrice.model, input: "hello", ...request });
stream.observe({ type: "response.completed", response: { model: policyPrice.model, usage: {
input_tokens: 12_000, input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, ...details }, output_tokens: 100,
} } });
return stream.receipt();
}
it.each([
[{}, "30m", "0.042600000"],
[{ prompt_cache_retention: "24h" }, "30m", "0.042600000"],
[{ prompt_cache_options: { mode: "implicit", ttl: "12h" } }, "12h", "0.077600000"],
] as const)("prices omitted write TTLs from a documented visible request policy: %j", (request, ttl, cost) => {
const receipt = requestPolicyReceipt(request);
expect(receipt).toMatchObject({ final: true, complete: true, cacheWriteAttribution: {
source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl,
} });
expect(estimateProviderCost(receipt, policyPrice)).toBe(cost);
});
it.each([
{ model: "unqualified-model" }, { previous_response_id: "stored-response" }, { conversation: "stored-conversation" },
{ prompt: { id: "stored-prompt" } }, { context_management: [] },
{ input: [{ type: "item_reference", id: "stored-item" }] },
{ input: [{ type: "compaction", encrypted_content: "opaque" }] },
{ input: [{ role: "user", content: [{ type: "input_text", text: "hello", prompt_cache_breakpoint: { mode: "explicit", ttl: "12h" } }] }] },
{ prompt_cache_options: { mode: "explicit" } }, { prompt_cache_options: { ttl: "unknown" } },
{ prompt_cache_options: { ttl: null } }, { prompt_cache_options: null },
{ prompt_cache_options: { future_option: true } },
])("keeps hidden, explicit, or unqualified cache policies unknown: %j", request => {
const receipt = requestPolicyReceipt(request);
expect(receipt.cacheWriteAttribution).toBeUndefined();
expect(estimateProviderCost(receipt, policyPrice)).toBeNull();
});
it("prefers provider TTL splits and never repairs malformed or partial provider counters", () => {
const reported = requestPolicyReceipt({}, { cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } });
expect(reported.cacheWriteAttribution).toBeUndefined();
expect(estimateProviderCost(reported, policyPrice)).toBe("0.062600000");
for (const details of [{ cache_write_tokens: undefined }, { cache_write_tokens_details: null },
{ cache_write_tokens_details: { ttl_30m_tokens: 7000 } }, { cache_write_tokens_details: { ttl_30m_tokens: 7000, ttl_12h_tokens: 1 } }]) {
const receipt = requestPolicyReceipt({}, details);
expect(receipt.cacheWriteAttribution).toBeUndefined();
expect(estimateProviderCost(receipt, policyPrice)).toBeNull();
}
});
it.each([
["openai", { input_tokens: 1001, output_tokens: 1 }, 1001],
["openai", { input_tokens_details: { cached_tokens: 1001 } }, 1001],
["anthropic", { input_tokens: 1, cache_read_input_tokens: 2, cache_creation_input_tokens: 1000,
cache_creation: { ephemeral_5m_input_tokens: 999, ephemeral_1h_input_tokens: 0 } }, 1003],
["anthropic", { input_tokens: 1, cache_creation: { ephemeral_5m_input_tokens: 900, ephemeral_1h_input_tokens: 100 } }, 1001],
] as const)("retains %s input bounds independently of incomplete or contradictory price counters", (provider, usage, input) => {
const receipt = normalizeProviderUsage(provider, { usage });
expect(receipt).toMatchObject({ complete: false, tokenBounds: { input, valid: true } });
expect(estimateProviderCost(receipt, { ...price, provider })).toBeNull();
});
it.each([null, -1, 1.5, "1001", Number.MAX_SAFE_INTEGER + 1])("marks malformed input evidence %j ineligible for a bound", input_tokens => {
expect(normalizeProviderUsage("openai", { usage: { input_tokens } }).tokenBounds?.valid).toBe(false);
});