Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(fact-checker): weight a degraded lane honestly in the verdict instead of scoring it as evidence
nirholas7adaca3d3076
diff --git a/agents/fact-checker/src/llm-verdict.js b/agents/fact-checker/src/llm-verdict.jsindex 7da170a87..0deee25c2 100644--- a/agents/fact-checker/src/llm-verdict.js+++ b/agents/fact-checker/src/llm-verdict.js@@ -35,13 +35,27 @@ export const STANCES = Object.freeze(['supports', 'contradicts', 'partial', 'neu* every stance "neutral".** This scans for a BALANCED array instead, string-literal aware so a `]` inside- * an excerpt cannot close it early, and retries at each later `[` so a leading- * `[1]` citation is skipped rather than fatal.+ * an excerpt cannot close it early, and keeps scanning past candidates of the+ * wrong shape. `[1]` is itself valid JSON, so a response opening with a+ * citation marker would otherwise resolve to the citation and starve the stage+ * of the payload sitting right behind it.** @param {string} text+ * @param {{of?: 'string' | 'object'}} [opts] What the caller's payload holds.+ * Given, a non-empty array of that element type wins; an empty array is+ * returned only when nothing better is present (callers read that as "the+ * provider had nothing to say", which is a degradation, not a verdict).* @returns {unknown[] | null} the parsed array, or null when there is none.*/-export function extractJsonArray(text) {+export function extractJsonArray(text, opts = {}) {+ const want = opts.of;+ const matches = (arr) =>+ !want ||+ (arr.length > 0 &&+ arr.every((v) =>+ want === 'string' ? typeof v === 'string' : typeof v === 'object' && v !== null,+ ));+ let empty = null;if (typeof text !== 'string' || !text) return null;// Reasoning models emit their scratchpad first; it is prose, not payload.// Drop closed blocks, and drop an unterminated tail (a truncated turn).@@ -72,7 +86,10 @@ export function extractJsonArray(text) {if (depth === 0) {try {const parsed = JSON.parse(body.slice(start, i + 1));- if (Array.isArray(parsed)) return parsed;+ if (Array.isArray(parsed)) {+ if (matches(parsed)) return parsed;+ if (parsed.length === 0 && !empty) empty = parsed;+ }} catch {// Not valid JSON from this `[`; try the next one.}@@ -81,7 +98,7 @@ export function extractJsonArray(text) {}}}- return null;+ return empty;}/**@@ -149,7 +166,7 @@ Example output: ["query one", "query two", "query three"]`;return { queries: [claim], tokens: 0, degraded: `query generation unavailable: ${failure}` };}- const parsed = extractJsonArray(text);+ const parsed = extractJsonArray(text, { of: 'string' });let queries = Array.isArray(parsed)? parsed.slice(0, 3).map((q) => String(q).trim()).filter(Boolean): [];@@ -234,7 +251,7 @@ Example (for 2 results): [{"excerpt":"The tower stands at 330m","stance":"suppor};}- const parsed = extractJsonArray(text);+ const parsed = extractJsonArray(text, { of: 'object' });if (!parsed || parsed.length === 0) {// The provider ANSWERED and its answer could not be read. That is not the// same thing as "the sources were silent", but the all-neutral fallbackdiff --git a/docs/fact-check.md b/docs/fact-check.mdindex 58a256836..b87439cfe 100644--- a/docs/fact-check.md+++ b/docs/fact-check.md@@ -127,6 +127,7 @@ Two guards keep a bad run from becoming a published number:- **Errored-claim ceiling.** Above 10% unreachable claims the run measured provider availability, not verdict accuracy. Nothing is written and the previous run stays up.- **Degraded checks count as errors.** The chain does not throw when its LLM providers are exhausted: stance extraction falls back to all-neutral and every claim resolves to `insufficient`. That would publish roughly 25% as the product's accuracy with zero apparent errors. Both runners read the `degraded` field on the response and count such a check as unreachable, which is what makes the ceiling above fire during an outage.+- **An answer nobody could read counts too.** A provider can reply and still leave the stage with nothing usable, which lands on the same all-neutral fallback while looking like a healthy check. Both LLM stages now say so (`stance extraction unreadable`, `query generation unreadable`) instead of returning a silent `insufficient`, so the ceiling fires on a parsing fault exactly as it does on an outage, and `checkClaim()` keeps the result out of the 7-day cache rather than pinning a wrong verdict on a checkable claim for a week.## States and limitsdiff --git a/tests/fact-check-degradation.test.js b/tests/fact-check-degradation.test.jsindex 00d62ce94..aaf2c4930 100644--- a/tests/fact-check-degradation.test.js+++ b/tests/fact-check-degradation.test.js@@ -15,7 +15,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';const llmComplete = vi.fn();vi.mock('../api/_lib/llm.js', () => ({ llmComplete: (...a) => llmComplete(...a) }));-const { generateSearchQueries, analyzeResults } = await import(+const { generateSearchQueries, analyzeResults, extractJsonArray } = await import('../agents/fact-checker/src/llm-verdict.js');const { searchAll } = await import('../agents/fact-checker/src/search-sources.js');@@ -152,3 +152,159 @@ describe('searchAll deadline', () => {expect(out[0].title).toBe('Prompt Result');});});++// The response reader both LLM stages share. The regex it replaced was+// non-greedy, so it stopped at the FIRST "]" in the response and handed the+// stages garbage, which they swallowed silently. These pin the shapes real+// providers actually emit.+describe('extractJsonArray', () => {+ const FENCED = ['```json', '["one","two"]', '```'].join('\n');++ it('reads a bare array', () => {+ expect(extractJsonArray('["a","b"]')).toEqual(['a', 'b']);+ });++ it('ignores a reasoning block that contains its own brackets', () => {+ const text =+ '<think>The user wants [three] queries. Let me list [a, b].</think>\n' ++ '["height of napoleon","napoleon height myth","napoleon 5 foot 7"]';+ expect(extractJsonArray(text)).toEqual([+ 'height of napoleon',+ 'napoleon height myth',+ 'napoleon 5 foot 7',+ ]);+ });++ it('ignores an unterminated reasoning block', () => {+ expect(extractJsonArray('<think>still thinking about [x]')).toBeNull();+ });++ it('is not closed early by a bracket inside a string', () => {+ const text = '[{"excerpt":"see note [4] below","stance":"partial"}]';+ expect(extractJsonArray(text)).toEqual([+ { excerpt: 'see note [4] below', stance: 'partial' },+ ]);+ });++ it('is not closed early by an escaped quote inside a string', () => {+ const text = String.raw`[{"excerpt":"he said \"about 330 m\" [sic]","stance":"supports"}]`;+ expect(extractJsonArray(text)[0].stance).toBe('supports');+ });++ it('steps past a leading citation marker to the real payload', () => {+ expect(extractJsonArray('Per [1], here it is: ["one","two"]', { of: 'string' })).toEqual([+ 'one',+ 'two',+ ]);+ // Without a declared shape the citation IS a well-formed array, so it wins.+ expect(extractJsonArray('Per [1], here it is: ["one","two"]')).toEqual([1]);+ });++ it('reads through a markdown code fence', () => {+ expect(extractJsonArray(FENCED)).toEqual(['one', 'two']);+ });++ it('returns null when there is no array at all', () => {+ expect(extractJsonArray('I cannot help with that.')).toBeNull();+ expect(extractJsonArray('')).toBeNull();+ expect(extractJsonArray(null)).toBeNull();+ });++ it('returns null on a truncated array rather than a half-read one', () => {+ expect(extractJsonArray('[{"excerpt":"cut off mid')).toBeNull();+ });+});++// A provider that ANSWERS and is not understood is a different failure from a+// provider that never answered, but both used to end in the same silent+// all-neutral fallback: verdict "insufficient", no degradation reported, the+// result cached for seven days, and the accuracy benchmark scoring it as a real+// verdict. A live check on 2026-09-02 hit exactly that: five real sources,+// 1476 spent tokens, every stance neutral, nothing flagged.+describe('unreadable provider output is reported, not swallowed', () => {+ it('analyzeResults degrades when the stance turn cannot be parsed', async () => {+ llmComplete.mockResolvedValue({+ text: 'I looked at the sources and they seem fine.',+ usage: { input: 900, output: 40 },+ });++ const out = await analyzeResults('some claim', RESULTS);++ expect(out.analyses.every((a) => a.stance === 'neutral')).toBe(true);+ expect(out.degraded).toMatch(/stance extraction unreadable/i);+ // The tokens were really spent, so they are really reported.+ expect(out.tokens).toBe(940);+ });++ it('analyzeResults degrades on an empty array rather than inventing neutrality', async () => {+ llmComplete.mockResolvedValue({ text: '[]', usage: { input: 10, output: 2 } });++ const out = await analyzeResults('some claim', RESULTS);++ expect(out.degraded).toMatch(/stance extraction unreadable/i);+ });++ it('analyzeResults reads a fenced, reasoning-wrapped answer without degrading', async () => {+ llmComplete.mockResolvedValue({+ text: [+ '<think>Result [1] is on point.</think>',+ '```json',+ '[{"excerpt":"true only at high doses [see table]","stance":"partial"},',+ '{"excerpt":"unrelated","stance":"neutral"}]',+ '```',+ ].join('\n'),+ usage: { input: 100, output: 20 },+ });++ const out = await analyzeResults('some claim', RESULTS);++ expect(out.analyses.map((a) => a.stance)).toEqual(['partial', 'neutral']);+ expect(out.degraded).toBeUndefined();+ });++ it('generateSearchQueries keeps all three angles through a reasoning block', async () => {+ llmComplete.mockResolvedValue({+ text: '<think>I need [3] angles.</think>["angle one","angle two","angle three"]',+ usage: { input: 50, output: 20 },+ });++ const out = await generateSearchQueries('A claim about something');++ expect(out.queries).toEqual(['angle one', 'angle two', 'angle three']);+ expect(out.degraded).toBeUndefined();+ });++ it('generateSearchQueries degrades when nothing usable can be read', async () => {+ llmComplete.mockResolvedValue({ text: 'Sorry:', usage: { input: 10, output: 3 } });+ const claim = 'A claim about something';++ const out = await generateSearchQueries(claim);++ expect(out.queries).toEqual([claim]);+ expect(out.degraded).toMatch(/query generation unreadable/i);+ });+});++// Shape is what separates the payload from the noise around it: the query stage+// wants strings, the stance stage wants objects, and a citation marker is+// neither of those even though it parses.+describe('extractJsonArray shape filter', () => {+ it('skips a numeric citation to reach an array of the requested shape', () => {+ const text = 'As shown in [12], the stances are: [{"excerpt":"x","stance":"supports"}]';+ expect(extractJsonArray(text, { of: 'object' })).toEqual([+ { excerpt: 'x', stance: 'supports' },+ ]);+ });++ it('returns null when no candidate has the requested shape', () => {+ expect(extractJsonArray('see [1] and [2]', { of: 'string' })).toBeNull();+ });++ it('surfaces an empty array so the caller can call it a degradation', () => {+ expect(extractJsonArray('Nothing to report: []', { of: 'object' })).toEqual([]);+ });++ it('prefers a shaped array over an earlier empty one', () => {+ expect(extractJsonArray('[] then ["a","b"]', { of: 'string' })).toEqual(['a', 'b']);+ });+});