Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Merge remote-tracking branch 'origin/main' into codex/lifetime-basic-cost-fix

Louis Beaumont6805ddcb893b
diff --cc packages/ai-gateway/src/index.ts
index 075f1ee1d,f20e34b9c..4f4cc589f
--- a/packages/ai-gateway/src/index.ts
+++ b/packages/ai-gateway/src/index.ts
@@@ -15,16 -15,12 +15,16 @@@ import { handleRealtimeTranscriptionUpg
import { handleVoiceTranscription, handleVoiceQuery, handleTextToSpeech, handleVoiceChat } from './handlers/voice';
import { handleVertexProxy, handleVertexModels } from './handlers/vertex-proxy';
import { handleWebSearch } from './handlers/web-search';
-import { handleTinfoilAttestation, handleTinfoilProxy } from './handlers/tinfoil-proxy';
-import { logCost, getModelCost, inferProvider, getSpendSummary, getDailyUserCost, getMaxDailyCostPerUser, getTierDailyCostCap, resolveServedModel } from './services/cost-tracker';
+import { handleTinfoilAttestation, handleTinfoilProxy, parseTinfoilUsageMetrics } from './handlers/tinfoil-proxy';
+import { logCost, getModelCost, getStreamModelCost, inferProvider, getSpendSummary, getDailyUserCost, getMaxDailyCostPerUser, getTierDailyCostCap, resolveServedModel } from './services/cost-tracker';
import { trackResponseUsage } from './utils/stream-usage-tracker';
- import { pruneModelHealth } from './services/model-health';
+ import { pruneRuntimeState } from './services/runtime-state-maintenance';
import { resolveLatencyClass, isBackgroundRequest } from './utils/latency';
-import { enforceDailyCostCap } from './services/cost-cap';
+import {
+ releaseDailyCostLease,
+ reserveDailyCostCap,
+ withDailyCostSettlement,
+} from './services/cost-cap';
import {
FREE_CHAT_MAX_PROVIDER_CALLS_PER_MESSAGE,
FREE_CHAT_MAX_OUTPUT_TOKENS,
@@@ -645,14 -543,8 +645,12 @@@ export async function handleRequest(req
}
if (path === '/v1/models' && request.method === 'GET') {
- // Prune old health records opportunistically (fire-and-forget)
- ctx.waitUntil(pruneModelHealth(env));
// Return tier-filtered models with live health status
- return await handleModelListing(env, authResult.tier);
+ return await handleModelListing(
+ env,
+ authResult.tier,
+ isHostedAiUpgradeEligible(authResult),
+ );
}
// ─── Tinfoil E2EE proxy ────────────────────────────────────────
diff --cc packages/ai-gateway/src/services/cost-tracker.ts
index aa5d0c2f1,0f9adf6cf..7ed94ed61
--- a/packages/ai-gateway/src/services/cost-tracker.ts
+++ b/packages/ai-gateway/src/services/cost-tracker.ts
@@@ -246,11 -234,9 +260,11 @@@ function utcToday(): string
* the (device_id, timestamp) index that would have made the SUM cheap
* can't even build at that size (SQLITE_NOMEM).
*
- * The write still falls back independently from cost_log insertion. Callers
- * that hold a priced-request lease use the boolean result to remain closed;
- * read-only status callers retain the legacy fail-open behavior.
- * Best-effort: a telemetry failure must never block an inference request.
++ * The per-device accumulator is the quota source of truth. Callers that hold
++ * a priced-request lease use the boolean result to remain closed when this
++ * write fails; the separate bounded aggregate remains best-effort telemetry.
*/
-async function bumpDailyCostAccumulator(env: Env, deviceId: string, cost: number): Promise<void> {
+async function bumpDailyCostAccumulator(env: Env, deviceId: string, cost: number): Promise<boolean> {
const today = utcToday();
try {
await env.DB.prepare(
@@@ -258,97 -244,77 +272,81 @@@
VALUES (?1, ?2, ?2, ?3)
ON CONFLICT(device_id) DO UPDATE SET
daily_cost_usd = CASE WHEN usage.cost_day = ?2 THEN usage.daily_cost_usd + ?3 ELSE ?3 END,
- cost_day = ?2`
+ cost_day = ?2,
+ updated_at = CURRENT_TIMESTAMP`
).bind(deviceId, today, cost).run();
+ return true;
} catch (error) {
console.warn('daily cost accumulator update failed:', error);
+ return false;
}
}
+ function nonNegativeNumber(value: number | null | undefined): number {
+ return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, value) : 0;
+ }
+
+ function boundedDimension(value: string | null | undefined, fallback: string): string {
+ if (typeof value !== 'string' || value.length === 0 || value.length > 128) return fallback;
+ return value;
+ }
+
/**
- * Insert a cost record into the cost_log table.
+ * Roll a request into the bounded daily cost table.
*
- * Falls back to the legacy column set if the cache columns from migration
- * 0004 haven't been applied yet, so a deploy/migration ordering mismatch
- * never drops cost rows.
+ * No request identifier, user identifier, or device identifier is stored in
+ * this table. The per-device daily accumulator in `usage` remains the O(1)
+ * quota source of truth; this aggregate is only for operational summaries.
*/
-export async function logCost(env: Env, entry: CostLogEntry): Promise<void> {
+export async function logCost(env: Env, entry: CostLogEntry): Promise<boolean> {
+ let accumulatorRecorded = true;
if (entry.device_id && entry.estimated_cost_usd > 0) {
- await bumpDailyCostAccumulator(env, entry.device_id, entry.estimated_cost_usd);
+ accumulatorRecorded = await bumpDailyCostAccumulator(env, entry.device_id, entry.estimated_cost_usd);
}
+
+ const latencyMs = nonNegativeNumber(entry.latency_ms);
+ const model = normalizeTelemetryModel(entry.model);
try {
- // Newest column set (migration 0007: + latency_ms, router_tier).
- await env.DB.prepare(
- `INSERT INTO cost_log (device_id, user_id, tier, provider, model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, estimated_cost_usd, endpoint, stream, latency_ms, router_tier)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
- )
- .bind(
- entry.device_id ?? null,
- entry.user_id ?? null,
- entry.tier,
- entry.provider,
- entry.model,
- entry.input_tokens,
- entry.output_tokens,
- entry.cache_read_tokens ?? null,
- entry.cache_creation_tokens ?? null,
- entry.estimated_cost_usd,
- entry.endpoint,
- entry.stream ? 1 : 0,
- entry.latency_ms ?? null,
- entry.router_tier ?? null,
- )
- .run();
- } catch (routerColsError) {
- try {
- // Migration 0004 applied (cache cols) but not 0007 yet.
await env.DB.prepare(
- `INSERT INTO cost_log (device_id, user_id, tier, provider, model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, estimated_cost_usd, endpoint, stream)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+ `INSERT INTO cost_daily (
+ date, tier, provider, model, endpoint, stream, router_tier,
+ requests, input_tokens, output_tokens, cache_read_tokens,
+ cache_creation_tokens, estimated_cost_usd, latency_ms_sum,
+ latency_samples
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(date, tier, provider, model, endpoint, stream, router_tier)
+ DO UPDATE SET
+ requests = requests + 1,
+ input_tokens = input_tokens + excluded.input_tokens,
+ output_tokens = output_tokens + excluded.output_tokens,
+ cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
+ cache_creation_tokens = cache_creation_tokens + excluded.cache_creation_tokens,
+ estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd,
+ latency_ms_sum = latency_ms_sum + excluded.latency_ms_sum,
+ latency_samples = latency_samples + excluded.latency_samples,
+ updated_at = datetime('now')`
)
.bind(
- entry.device_id ?? null,
- entry.user_id ?? null,
- entry.tier,
- entry.provider,
- entry.model,
- entry.input_tokens,
- entry.output_tokens,
- entry.cache_read_tokens ?? null,
- entry.cache_creation_tokens ?? null,
- entry.estimated_cost_usd,
- entry.endpoint,
+ utcToday(),
+ boundedDimension(entry.tier, 'unknown'),
+ boundedDimension(entry.provider, 'unknown'),
+ model,
+ boundedDimension(entry.endpoint, 'unknown'),
entry.stream ? 1 : 0,
+ boundedDimension(entry.router_tier, 'none'),
+ nonNegativeNumber(entry.input_tokens),
+ nonNegativeNumber(entry.output_tokens),
+ nonNegativeNumber(entry.cache_read_tokens),
+ nonNegativeNumber(entry.cache_creation_tokens),
+ nonNegativeNumber(entry.estimated_cost_usd),
+ latencyMs,
+ latencyMs > 0 ? 1 : 0,
)
.run();
- } catch (error) {
- try {
- await env.DB.prepare(
- `INSERT INTO cost_log (device_id, user_id, tier, provider, model, input_tokens, output_tokens, estimated_cost_usd, endpoint, stream)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
- )
- .bind(
- entry.device_id ?? null,
- entry.user_id ?? null,
- entry.tier,
- entry.provider,
- entry.model,
- entry.input_tokens,
- entry.output_tokens,
- entry.estimated_cost_usd,
- entry.endpoint,
- entry.stream ? 1 : 0,
- )
- .run();
- } catch (fallbackError) {
- console.error('cost logging failed:', fallbackError);
- }
- }
+ } catch (error) {
+ console.error('cost aggregation failed:', error);
}
+ return accumulatorRecorded;
}
/**
@@@ -423,22 -389,11 +421,21 @@@ export function getTierDailyCostCap(tie
/**
* Get a user's estimated cost for today. Used to enforce per-user daily cost caps.
*
- * Fast path: single-row read of the usage-table accumulator (migration
- * 0006), maintained by logCost. Falls back to the legacy SUM over cost_log
- * only while the migration hasn't been applied — that scan is what hit
- * D1's CPU limit at scale (SCREENPIPE-AI-PROXY-1T/-1X/-1E).
+ * Single-row read of the usage-table accumulator (migration 0006), maintained
+ * by logCost. There is intentionally no request-log fallback: that unbounded
+ * scan was one of the failure modes this accumulator replaced.
*/
export async function getDailyUserCost(env: Env, deviceId: string): Promise<number> {
+ try {
+ return await getDailyUserCostOrThrow(env, deviceId);
+ } catch (error) {
+ console.error('getDailyUserCost failed:', error);
+ return 0;
+ }
+}
+
+/** Same lookup as getDailyUserCost, but lets spend enforcement fail closed. */
+export async function getDailyUserCostOrThrow(env: Env, deviceId: string): Promise<number> {
const today = utcToday();
try {
const row = await env.DB.prepare(
@@@ -448,17 -403,8 +445,8 @@@
// No usage row yet = no recorded spend today.
return row?.daily_cost ?? 0;
} catch (error) {
- console.warn('daily cost accumulator read failed, falling back to cost_log scan:', error);
- }
- try {
- const result = await env.DB.prepare(
- `SELECT COALESCE(SUM(estimated_cost_usd), 0) as daily_cost
- FROM cost_log WHERE device_id = ? AND timestamp >= ?`
- ).bind(deviceId, today + ' 00:00:00').first<{ daily_cost: number }>();
- return result?.daily_cost ?? 0;
- } catch (error) {
- console.error('daily cost fallback read failed:', error);
+ console.error('daily cost accumulator read failed:', error);
- return 0; // Preserve the existing fail-open behavior on telemetry failure.
+ throw new Error('daily cost accounting unavailable');
}
}
diff --cc packages/ai-gateway/src/test/daily-cost.unit.test.ts
index 3ef023ea3,5e80c4d2e..81625ac0e
--- a/packages/ai-gateway/src/test/daily-cost.unit.test.ts
+++ b/packages/ai-gateway/src/test/daily-cost.unit.test.ts
@@@ -81,10 -82,10 +82,10 @@@ describe('logCost — usage-table dail
expect(captured.some((c) => c.sql.includes('ON CONFLICT(device_id)'))).toBe(false);
});
- it('still writes the cost row when the accumulator column is missing (pre-migration)', async () => {
+ it('still writes the aggregate when the quota accumulator update fails', async () => {
const { env, captured } = stubEnv({ failWhen: (sql) => sql.includes('ON CONFLICT(device_id)') });
- await logCost(env, baseEntry);
+ expect(await logCost(env, baseEntry)).toBe(false);
- expect(captured.some((c) => c.sql.includes('INSERT INTO cost_log'))).toBe(true);
+ expect(captured.some((c) => c.sql.includes('INSERT INTO cost_daily'))).toBe(true);
});
});