Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
docs(fact-check): document the partial stance and its verdict weighting
nirholasc4fc7d965c06
diff --git a/agents/fact-checker/src/llm-verdict.js b/agents/fact-checker/src/llm-verdict.jsindex 5311bac65..7da170a87 100644--- a/agents/fact-checker/src/llm-verdict.js+++ b/agents/fact-checker/src/llm-verdict.js@@ -157,7 +157,7 @@ Example output: ["query one", "query two", "query three"]`;if (queries.length === 0) {// Fallback: read the response as a plain list, one query per line. Lines// are length-bounded and required to hold at least two words, because a- // reasoning model's prose is not a search query — accepting it is how a+ // reasoning model's prose is not a search query. Accepting it is how a// three-angle sweep silently collapsed into one bad one.queries = text.split('\n')@@ -168,7 +168,7 @@ Example output: ["query one", "query two", "query three"]`;if (queries.length === 0) {// Last resort: the claim itself is a legitimate search query, so the check- // still runs on real evidence — but on one angle instead of three, which+ // still runs on real evidence, but on one angle instead of three, which// the caller has to be told about.return {queries: [claim],@@ -234,25 +234,30 @@ Example (for 2 results): [{"excerpt":"The tower stands at 330m","stance":"suppor};}- let analyses;- try {- const match = text.match(/\[[\s\S]*?\]/);- if (!match) throw new Error('no array found');- analyses = JSON.parse(match[0]);- if (!Array.isArray(analyses)) throw new Error('not an array');- // Normalize and pad/trim to match results length.- analyses = analyses.slice(0, results.length).map((a) => ({- excerpt: String(a?.excerpt || '').slice(0, 200),- stance: ['supports', 'contradicts', 'partial', 'neutral'].includes(a?.stance)- ? a.stance- : 'neutral',- }));- while (analyses.length < results.length) {- analyses.push({ excerpt: '', stance: 'neutral' });- }- } catch {- // Fallback: mark all neutral.- analyses = results.map(() => ({ excerpt: '', stance: 'neutral' }));+ const parsed = extractJsonArray(text);+ if (!parsed || parsed.length === 0) {+ // The provider ANSWERED and its answer could not be read. That is not the+ // same thing as "the sources were silent", but the all-neutral fallback+ // makes the two indistinguishable: computeVerdict reads either as+ // "insufficient". Left unflagged, one unreadable turn was scored as a real+ // verdict by the accuracy benchmark and written into the 7-day cache,+ // pinning that wrong answer on a perfectly checkable claim for a week.+ // A live check on 2026-09-02 did exactly this: five real sources, 1476+ // spent tokens, every stance neutral, and nothing marked degraded.+ return {+ analyses: results.map(() => ({ excerpt: '', stance: 'neutral' })),+ tokens: inputTokens + outputTokens,+ degraded: 'stance extraction unreadable: provider returned no parseable JSON array',+ };+ }++ // Normalize and pad/trim to match results length.+ const analyses = parsed.slice(0, results.length).map((a) => ({+ excerpt: String(a?.excerpt || '').slice(0, 200),+ stance: STANCES.includes(a?.stance) ? a.stance : 'neutral',+ }));+ while (analyses.length < results.length) {+ analyses.push({ excerpt: '', stance: 'neutral' });}return { analyses, tokens: inputTokens + outputTokens };diff --git a/docs/fact-check.md b/docs/fact-check.mdindex 9438f35f1..58a256836 100644--- a/docs/fact-check.md+++ b/docs/fact-check.md@@ -24,9 +24,9 @@ The fact-check pipeline (`runFactCheck`):1. Generate exactly 3 search queries with an LLM.2. Search across a source chain (Vertex-grounded Google Search, then Brave, Tavily, Exa, Serper, then keyless Wikipedia full-text and DuckDuckGo Instant Answer as always-available fallbacks) and take the top 5 unique results. Each keyed rung is skipped when its key is absent, so the chain runs whatever is configured and always reaches the keyless rungs.-3. Run LLM stance extraction on each of the 5: an excerpt plus a stance of `supports`, `contradicts`, or `neutral`.+3. Run LLM stance extraction on each of the 5: an excerpt plus a stance of `supports`, `contradicts`, `partial`, or `neutral`. `partial` is a source that engages the claim and finds it true in one respect while wrong, overstated, or only conditionally true in another (right about the fact, wrong about its scale, cause, exclusivity or universality). It is deliberately narrow: hedged prose, thin coverage, or an extractor that is merely unsure is `neutral`, not `partial`.4. Weight each source by an authority score, adjusted by strictness (`high` halves low-authority weights, `low` floors them, `medium` is the default).-5. Compute the verdict over the stance-bearing evidence only, so tangential results cannot drown a clear answer. Fewer than 2 sources, zero weight, or no source that took a stance at all is `insufficient`, and so is a lone stance-bearing source lost in otherwise-silent evidence (under 30 percent coverage): evidence that never engages the claim is absence of evidence, not disagreement. Otherwise direction is judged over stance-bearing weight alone, 70 percent or more one way is `supported` or `contradicted` and a real split is `mixed`. Confidence blends that dominance with coverage, so a unanimous verdict read off thin engagement scores lower than one read off broad engagement.+5. Compute the verdict over the stance-bearing evidence only, so tangential results cannot drown a clear answer. Fewer than 2 sources, zero weight, or no source that took a stance at all is `insufficient`, and so is a lone stance-bearing source lost in otherwise-silent evidence (under 30 percent coverage): evidence that never engages the claim is absence of evidence, not disagreement. Otherwise direction is judged over stance-bearing weight alone, 70 percent or more one way is `supported` or `contradicted`, and anything else is `mixed`. `partial` weight is stance-bearing but takes neither side, so it dilutes dominance: a claim every source calls half-true reaches `mixed` on its own, without needing the sources to disagree with each other. Confidence blends dominance with coverage, so a unanimous verdict read off thin engagement scores lower than one read off broad engagement; for `mixed` the dominance term is how strongly the evidence establishes mixedness (partial weight, plus opposed stances counted as the joint evidence of a split that they are), not how lopsided the split was.6. Build a cost breakdown and the attestation.The LLM routes through the platform's shared free-first policy (Groq and OpenRouter as funded defaults). Results are cached in Redis for 7 days keyed by a hash of the claim, strictness, and any image URL, so an identical claim (on either lane) never re-runs the live chain.diff --git a/tests/api/fact-check-verdict.test.js b/tests/api/fact-check-verdict.test.jsnew file mode 100644index 000000000..5673ce284--- /dev/null+++ b/tests/api/fact-check-verdict.test.js@@ -0,0 +1,174 @@+/**+ * Unit tests for api/x402/fact-check.js#computeVerdict, the weighted verdict+ * calculus behind every fact-check answer and behind the published accuracy+ * benchmark on /fact-check.+ *+ * Written against SYNTHETIC stance distributions on purpose. The benchmark+ * fixture is 40 real claims scored through the live chain; shaping the calculus+ * to those ten `mixed` claims would fit the fixture rather than fix the rule.+ * These cases pin the rule itself:+ * • the clear-cut distributions (unanimous, near-unanimous, empty, silent)+ * keep their verdicts, so the mixed band can never swallow them+ * • `partial` weight is stance-bearing but takes neither side, which is the+ * route by which a claim every source calls "true in one respect, wrong in+ * another" reaches `mixed` instead of `contradicted`+ * • confidence for `mixed` measures how strongly the evidence establishes+ * mixedness, not how lopsided the split was+ */++import { describe, it, expect } from 'vitest';+import { _computeVerdict as computeVerdict } from '../../api/x402/fact-check.js';++/** Build n sources of one stance at a uniform weight. */+const of = (stance, n, weight = 0.8) =>+ Array.from({ length: n }, (_v, i) => ({ url: `https://example.com/${stance}/${i}`, stance, weight }));++describe('computeVerdict — clear-cut distributions keep their verdicts', () => {+ it('unanimous support is supported', () => {+ expect(computeVerdict(of('supports', 5)).verdict).toBe('supported');+ });++ it('unanimous contradiction is contradicted', () => {+ expect(computeVerdict(of('contradicts', 5)).verdict).toBe('contradicted');+ });++ it('exactly 70% of stance weight one way still lands on that side', () => {+ // 7 supporting + 3 contradicting at equal weight is the boundary itself.+ expect(computeVerdict([...of('supports', 7), ...of('contradicts', 3)]).verdict).toBe('supported');+ expect(computeVerdict([...of('contradicts', 7), ...of('supports', 3)]).verdict).toBe('contradicted');+ });++ it('one dissenting source does not unseat a dominant side', () => {+ expect(computeVerdict([...of('supports', 4), ...of('contradicts', 1)]).verdict).toBe('supported');+ expect(computeVerdict([...of('contradicts', 4), ...of('supports', 1)]).verdict).toBe('contradicted');+ });++ it('authority weight, not source count, decides direction', () => {+ // Two low-authority blogs cannot outvote one encyclopedic source.+ const verdict = computeVerdict([+ { url: 'https://en.wikipedia.org/wiki/X', stance: 'supports', weight: 0.9 },+ { url: 'https://blog.example/a', stance: 'contradicts', weight: 0.15 },+ { url: 'https://blog.example/b', stance: 'contradicts', weight: 0.15 },+ ]);+ expect(verdict.verdict).toBe('supported');+ });++ it('fewer than two sources is insufficient, never a verdict', () => {+ expect(computeVerdict([]).verdict).toBe('insufficient');+ expect(computeVerdict(of('supports', 1)).verdict).toBe('insufficient');+ });++ it('evidence that never engages the claim is insufficient, not mixed', () => {+ // The 2026-07-08 regression: all-neutral used to return "mixed".+ expect(computeVerdict(of('neutral', 5)).verdict).toBe('insufficient');+ });++ it('zero total weight is insufficient', () => {+ expect(computeVerdict(of('supports', 3, 0)).verdict).toBe('insufficient');+ });++ it('a single stance-bearing source lost among silent ones is insufficient', () => {+ const sources = [+ { url: 'https://example.com/one', stance: 'supports', weight: 0.2 },+ ...of('neutral', 4, 0.9),+ ];+ expect(computeVerdict(sources).verdict).toBe('insufficient');+ });++ it('a lone partial source lost among silent ones is insufficient too', () => {+ // `partial` is stance-bearing, so it goes through the same coverage floor:+ // one loosely-matched page must not decide a verdict on its own.+ const sources = [+ { url: 'https://example.com/one', stance: 'partial', weight: 0.2 },+ ...of('neutral', 4, 0.9),+ ];+ expect(computeVerdict(sources).verdict).toBe('insufficient');+ });+});++describe('computeVerdict — mixed by inter-source disagreement', () => {+ it('an even split is mixed', () => {+ expect(computeVerdict([...of('supports', 3), ...of('contradicts', 3)]).verdict).toBe('mixed');+ });++ it('a split just short of the 70% bar is mixed', () => {+ // 2 vs 1 is 66.7% support: below the bar, so neither side owns it.+ expect(computeVerdict([...of('supports', 2), ...of('contradicts', 1)]).verdict).toBe('mixed');+ });++ it('neutral sources dilute confidence but do not change the verdict', () => {+ const tight = computeVerdict([...of('supports', 3), ...of('contradicts', 3)]);+ const diluted = computeVerdict([...of('supports', 3), ...of('contradicts', 3), ...of('neutral', 12)]);+ expect(diluted.verdict).toBe(tight.verdict);+ expect(diluted.confidence).toBeLessThan(tight.confidence);+ });+});++describe('computeVerdict — mixed by intra-source partiality', () => {+ it('unanimous partial evidence is mixed, not contradicted', () => {+ // The defect this stance exists to fix: every source reads the same+ // nuance ("a tomato is botanically a fruit and culinarily a vegetable"),+ // so before `partial` existed they all projected onto one side and the+ // whole mixed class scored 0/10.+ expect(computeVerdict(of('partial', 5)).verdict).toBe('mixed');+ });++ it('partial weight blocks a side from clearing the 70% bar', () => {+ // 3 supports + 2 partial: 60% support, below the bar.+ expect(computeVerdict([...of('supports', 3), ...of('partial', 2)]).verdict).toBe('mixed');+ expect(computeVerdict([...of('contradicts', 3), ...of('partial', 2)]).verdict).toBe('mixed');+ });++ it('a lone partial source does not unseat an otherwise dominant side', () => {+ // 8 supports + 1 partial is still 88.9% support. `partial` dilutes+ // dominance; it does not veto it.+ expect(computeVerdict([...of('supports', 8), ...of('partial', 1)]).verdict).toBe('supported');+ });++ it('partial mixed with both directions is mixed', () => {+ expect(+ computeVerdict([...of('supports', 2), ...of('contradicts', 2), ...of('partial', 2)]).verdict,+ ).toBe('mixed');+ });+});++describe('computeVerdict — confidence semantics', () => {+ it('broad engagement outscores thin engagement at the same dominance', () => {+ const broad = computeVerdict(of('supports', 5, 0.8));+ const thin = computeVerdict([...of('supports', 2, 0.8), ...of('neutral', 8, 0.8)]);+ expect(broad.confidence).toBeGreaterThan(thin.confidence);+ });++ it('an even split is MORE confidently mixed than a lopsided one', () => {+ // The old rule scored mixed confidence as max(support, contra), so the+ // closer a split came to breaking one way, the more sure it claimed to be+ // that it had not. Mixedness is the thing being measured now.+ const even = computeVerdict([...of('supports', 3), ...of('contradicts', 3)]);+ const lopsided = computeVerdict([...of('supports', 2), ...of('contradicts', 1)]);+ expect(even.confidence).toBeGreaterThan(lopsided.confidence);+ });++ it('unanimous partial is as confidently mixed as an even split', () => {+ const partial = computeVerdict(of('partial', 6));+ const split = computeVerdict([...of('supports', 3), ...of('contradicts', 3)]);+ expect(partial.confidence).toBe(split.confidence);+ });++ it('every confidence is a bounded, rounded probability', () => {+ const cases = [+ of('supports', 5),+ of('contradicts', 5),+ of('partial', 5),+ [...of('supports', 3), ...of('contradicts', 3)],+ [...of('supports', 3), ...of('partial', 2), ...of('neutral', 4)],+ of('neutral', 5),+ of('supports', 1),+ ];+ for (const sources of cases) {+ const { confidence } = computeVerdict(sources);+ expect(confidence).toBeGreaterThan(0);+ expect(confidence).toBeLessThanOrEqual(0.98);+ expect(Math.round(confidence * 100)).toBe(confidence * 100);+ }+ });+});