Skip to content

Personal AI, On Personal Devices

Python

fix: correct O(N²) energy/FLOPs savings calculation (#95) (#97)

Jon Saad-Falcon8468f08c41b5
diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.js
index 60d64c90..742b398a 100644
--- a/docs/javascripts/leaderboard.js
+++ b/docs/javascripts/leaderboard.js
@@ -10,6 +10,8 @@
var currentPage = 0;
// No-KV-cache formula: FLOPs = params_b * 1e9 * N * (N+1)
+ // Reference values only (GPT-5.3 constants) — recompute functions below
+ // are defined but not called; the leaderboard displays database values directly.
var DEFAULT_PARAMS_B = 137;
var ENERGY_WH_PER_FLOP = 0.4 / (1000 * 3e12);
diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.py
index 3f6144a2..993af063 100644
--- a/src/openjarvis/server/savings.py
+++ b/src/openjarvis/server/savings.py
@@ -101,30 +101,26 @@ def compute_savings(
# No-KV-cache FLOPs: P * N * (N+1)
params_b = pricing.get("params_b", 200.0)
params = params_b * 1e9
- flops = (
- params * total_tokens * (total_tokens + 1)
- if total_tokens > 0 else 0.0
+ flops = params * total_tokens * (total_tokens + 1) if total_tokens > 0 else 0.0
+ # Derive Wh-per-FLOP from the provider's per-token constants:
+ # energy_wh_per_1k_tokens / (1000 * flops_per_token) = Wh per FLOP
+ wh_per_flop = pricing["energy_wh_per_1k_tokens"] / (
+ 1000 * pricing.get("flops_per_token", 3e12)
)
- # Scale energy by same ratio as FLOPs (energy ∝ compute)
- flops_with_cache = (
- total_tokens * pricing.get("flops_per_token", 3e12)
- if total_tokens > 0 else 0.0
+ energy_wh = flops * wh_per_flop
+
+ providers.append(
+ ProviderSavings(
+ provider=key,
+ label=pricing["label"],
+ input_cost=input_cost,
+ output_cost=output_cost,
+ total_cost=total_cost,
+ energy_wh=energy_wh,
+ energy_joules=energy_wh * 3600, # 1 Wh = 3600 J
+ flops=flops,
+ )
)
- scale = (flops / flops_with_cache) if flops_with_cache > 0 else 1.0
- energy_wh = (
- (total_tokens / 1000) * pricing["energy_wh_per_1k_tokens"] * scale
- )
-
- providers.append(ProviderSavings(
- provider=key,
- label=pricing["label"],
- input_cost=input_cost,
- output_cost=output_cost,
- total_cost=total_cost,
- energy_wh=energy_wh,
- energy_joules=energy_wh * 3600, # 1 Wh = 3600 J
- flops=flops,
- ))
# Monthly projection: extrapolate current spend to 720 hours/month
if session_duration_hours > 0:
diff --git a/tests/evals/test_use_case_benchmarks.py b/tests/evals/test_use_case_benchmarks.py
index ec3ebbe3..5bddb4d1 100644
--- a/tests/evals/test_use_case_benchmarks.py
+++ b/tests/evals/test_use_case_benchmarks.py
@@ -422,3 +422,40 @@ class TestSavings:
assert isinstance(d, dict)
assert "per_provider" in d
assert "total_calls" in d
+
+ def test_energy_scales_linearly(self) -> None:
+ """Energy should scale with FLOPs (quadratic in N), not N^3."""
+ from openjarvis.server.savings import compute_savings
+
+ s1 = compute_savings(1000, 0)
+ s10 = compute_savings(10000, 0)
+ # FLOPs ~ N^2, so 10x tokens => ~100x FLOPs => ~100x energy
+ for p1, p10 in zip(s1.per_provider, s10.per_provider):
+ ratio = p10.energy_wh / p1.energy_wh
+ # Allow some tolerance for the (N+1) factor
+ assert 90 < ratio < 110, (
+ f"{p1.provider}: energy ratio {ratio:.1f}, expected ~100"
+ )
+
+ def test_energy_wh_matches_direct_formula(self) -> None:
+ """Energy must equal flops * wh_per_flop for known constants."""
+ from openjarvis.server.savings import CLOUD_PRICING, compute_savings
+
+ summary = compute_savings(10000, 0)
+ for p in summary.per_provider:
+ pricing = CLOUD_PRICING[p.provider]
+ wh_per_flop = pricing["energy_wh_per_1k_tokens"] / (
+ 1000 * pricing.get("flops_per_token", 3e12)
+ )
+ expected = p.flops * wh_per_flop
+ assert abs(p.energy_wh - expected) < 1e-6, (
+ f"{p.provider}: energy_wh={p.energy_wh}, expected={expected}"
+ )
+
+ def test_energy_not_zero(self) -> None:
+ """Energy must be positive for non-zero token counts."""
+ from openjarvis.server.savings import compute_savings
+
+ summary = compute_savings(500, 500)
+ for p in summary.per_provider:
+ assert p.energy_wh > 0, f"{p.provider}: energy_wh should be > 0"