Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

test(oracle): assert the published score means win, not moon

nirholas23ed24ddaca2
diff --git a/tests/oracle/score-semantics.test.js b/tests/oracle/score-semantics.test.js
index 9c014a850..658549ee9 100644
--- a/tests/oracle/score-semantics.test.js
+++ b/tests/oracle/score-semantics.test.js
@@ -57,10 +57,14 @@ describe('score to probability', () => {
expect(probabilityFromScore(-40)).toBe(0);
});
- it('names the event it predicts, and that a later collapse does not undo a hit', () => {
- expect(PREDICTED_EVENT.id).toBe('spike_or_graduate');
+ it('names the event it predicts, and that a collapse now DOES undo a hit', () => {
+ // v2 ranked the bare spike and said so: a later collapse did not undo a
+ // hit. v3 ranks the survivable win, which is the reverse claim, and the
+ // caveat has to change with it or every surface quoting it starts lying.
+ expect(PREDICTED_EVENT.id).toBe('runs_and_holds');
expect(PREDICTED_EVENT.label).toMatch(/3x/);
- expect(PREDICTED_EVENT.caveat).toMatch(/not the odds of a safe hold/);
+ expect(PREDICTED_EVENT.label).toMatch(/at first sight/);
+ expect(PREDICTED_EVENT.caveat).toMatch(/NOT a hit/);
});
});
@@ -109,9 +113,11 @@ describe('calibration bands', () => {
expect(top.n).toBe(792);
expect(top.realized).toBe(26); // graduated or 2x without ever rugging
expect(top.realized_spike).toBe(68); // the trained event
- // The retired code would have printed 95 here (the band midpoint).
+ // The retired code would have printed 95 here (the band midpoint). What it
+ // prints instead is the probability the 90-100 band actually claims, which
+ // moved when v3 re-anchored the ladder for a harder target.
expect(top.predicted).toBeLessThan(95);
- expect(top.predicted).toBe(90);
+ expect(top.predicted).toBe(Math.round(100 * probabilityFromScore(90)));
});
it('drops empty bands instead of inventing zero rows', () => {
@@ -127,7 +133,12 @@ describe('calibration bands', () => {
});
it('scores Brier per exact score, not per band', () => {
- expect(brierScore(rows)).toBeCloseTo(0.1303, 4);
+ // Pinned loosely on purpose: the exact value tracks the tier anchors, and
+ // re-pinning a magic constant on every re-anchor teaches nobody anything.
+ // What must hold is that it is scored per exact score, which the bounds and
+ // the degenerate cases below establish.
+ expect(brierScore(rows)).toBeGreaterThan(0.12);
+ expect(brierScore(rows)).toBeLessThan(0.14);
expect(brierScore([])).toBeNull();
// A claim of 1.0 that never happens is the worst possible score.
expect(brierScore([{ score: 100, n: 1, spikes: 0 }])).toBe(1);