Personal AI, On Personal Devices
Python
fix: correct O(N²) energy/FLOPs savings calculation (#95) (#97)
Jon Saad-Falcon8468f08c41b5
diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.jsindex 60d64c90..742b398a 100644--- a/docs/javascripts/leaderboard.js+++ b/docs/javascripts/leaderboard.js@@ -10,6 +10,8 @@var currentPage = 0;// No-KV-cache formula: FLOPs = params_b * 1e9 * N * (N+1)+ // Reference values only (GPT-5.3 constants) — recompute functions below+ // are defined but not called; the leaderboard displays database values directly.var DEFAULT_PARAMS_B = 137;var ENERGY_WH_PER_FLOP = 0.4 / (1000 * 3e12);diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.pyindex 3f6144a2..993af063 100644--- a/src/openjarvis/server/savings.py+++ b/src/openjarvis/server/savings.py@@ -101,30 +101,26 @@ def compute_savings(# No-KV-cache FLOPs: P * N * (N+1)params_b = pricing.get("params_b", 200.0)params = params_b * 1e9- flops = (- params * total_tokens * (total_tokens + 1)- if total_tokens > 0 else 0.0+ flops = params * total_tokens * (total_tokens + 1) if total_tokens > 0 else 0.0+ # Derive Wh-per-FLOP from the provider's per-token constants:+ # energy_wh_per_1k_tokens / (1000 * flops_per_token) = Wh per FLOP+ wh_per_flop = pricing["energy_wh_per_1k_tokens"] / (+ 1000 * pricing.get("flops_per_token", 3e12))- # Scale energy by same ratio as FLOPs (energy ∝ compute)- flops_with_cache = (- total_tokens * pricing.get("flops_per_token", 3e12)- if total_tokens > 0 else 0.0+ energy_wh = flops * wh_per_flop++ providers.append(+ ProviderSavings(+ provider=key,+ label=pricing["label"],+ input_cost=input_cost,+ output_cost=output_cost,+ total_cost=total_cost,+ energy_wh=energy_wh,+ energy_joules=energy_wh * 3600, # 1 Wh = 3600 J+ flops=flops,+ ))- scale = (flops / flops_with_cache) if flops_with_cache > 0 else 1.0- energy_wh = (- (total_tokens / 1000) * pricing["energy_wh_per_1k_tokens"] * scale- )-- providers.append(ProviderSavings(- provider=key,- label=pricing["label"],- input_cost=input_cost,- output_cost=output_cost,- total_cost=total_cost,- energy_wh=energy_wh,- energy_joules=energy_wh * 3600, # 1 Wh = 3600 J- flops=flops,- ))# Monthly projection: extrapolate current spend to 720 hours/monthif session_duration_hours > 0:diff --git a/tests/evals/test_use_case_benchmarks.py b/tests/evals/test_use_case_benchmarks.pyindex ec3ebbe3..5bddb4d1 100644--- a/tests/evals/test_use_case_benchmarks.py+++ b/tests/evals/test_use_case_benchmarks.py@@ -422,3 +422,40 @@ class TestSavings:assert isinstance(d, dict)assert "per_provider" in dassert "total_calls" in d++ def test_energy_scales_linearly(self) -> None:+ """Energy should scale with FLOPs (quadratic in N), not N^3."""+ from openjarvis.server.savings import compute_savings++ s1 = compute_savings(1000, 0)+ s10 = compute_savings(10000, 0)+ # FLOPs ~ N^2, so 10x tokens => ~100x FLOPs => ~100x energy+ for p1, p10 in zip(s1.per_provider, s10.per_provider):+ ratio = p10.energy_wh / p1.energy_wh+ # Allow some tolerance for the (N+1) factor+ assert 90 < ratio < 110, (+ f"{p1.provider}: energy ratio {ratio:.1f}, expected ~100"+ )++ def test_energy_wh_matches_direct_formula(self) -> None:+ """Energy must equal flops * wh_per_flop for known constants."""+ from openjarvis.server.savings import CLOUD_PRICING, compute_savings++ summary = compute_savings(10000, 0)+ for p in summary.per_provider:+ pricing = CLOUD_PRICING[p.provider]+ wh_per_flop = pricing["energy_wh_per_1k_tokens"] / (+ 1000 * pricing.get("flops_per_token", 3e12)+ )+ expected = p.flops * wh_per_flop+ assert abs(p.energy_wh - expected) < 1e-6, (+ f"{p.provider}: energy_wh={p.energy_wh}, expected={expected}"+ )++ def test_energy_not_zero(self) -> None:+ """Energy must be positive for non-zero token counts."""+ from openjarvis.server.savings import compute_savings++ summary = compute_savings(500, 500)+ for p in summary.per_provider:+ assert p.energy_wh > 0, f"{p.provider}: energy_wh should be > 0"