Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix(studio): explicit discriminant for comparison failures; honest loading state; wire the scripted chat surface

FABIOTESS9f384571cc93
diff --git a/studio/src/AppShell.tsx b/studio/src/AppShell.tsx
index 145358e4..50993aea 100644
--- a/studio/src/AppShell.tsx
+++ b/studio/src/AppShell.tsx
@@ -4,9 +4,10 @@ import { CorpusPanel } from './components/CorpusPanel'
import { FindingsPanel } from './components/FindingsPanel'
import { ResolutionPanel } from './components/ResolutionPanel'
import { ComparePanel } from './components/ComparePanel'
+import { ScriptedChatPanel } from './components/ScriptedChatPanel'
import { Graph3D } from './components/Graph3D'
import { chooseSourceKind } from './lib/source-factory'
-import { getCompareSource, type CompareResult } from './lib/compare-source'
+import { getCompareSource, compareError, type CompareResult } from './lib/compare-source'
import './App.css'
// Desktop-only chrome (chat over the agent sidecar, engine status, save/open,
@@ -21,7 +22,7 @@ const LiveChrome = lazy(() => import('./components/LiveChrome').then((m) => ({ d
const isLive =
chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'live'
-type Tab = 'findings' | 'corpus' | 'compare'
+type Tab = 'findings' | 'corpus' | 'compare' | 'chat'
/**
* The tabbed sidebar: findings + resolution, corpus, and the grounded-vs-
@@ -38,6 +39,9 @@ function DemoPanels() {
const ledger = useDemoStore((s) => s.ledger)
const select = useDemoStore((s) => s.select)
const resolve = useDemoStore((s) => s.resolve)
+ const chat = useDemoStore((s) => s.chat)
+ const chatPending = useDemoStore((s) => s.chatPending)
+ const ask = useDemoStore((s) => s.ask)
// Surfaced here too, not only in ReplayBody's header: DemoPanels is also
// embedded inside LiveChrome, which has no header banner of its own, and
// a rejected resolve() (see live-source.ts: no engine tool resolves a
@@ -48,23 +52,30 @@ function DemoPanels() {
const [compareResult, setCompareResult] = useState<CompareResult | null>(null)
const [comparePending, setComparePending] = useState(false)
const [compareQuestions, setCompareQuestions] = useState<string[]>([])
-
- // The scripted question list for the compare picker. CompareSource has no
- // "list questions" method by design (see compare-source.ts: it answers
- // one question at a time), so this reads the same precomputed bundle
- // replay mode's CompareSource reads internally, purely to populate the
- // picker. Live mode has no scripted set (getCompareSource() throws
- // there), so the picker stays empty and the panel explains why on ask.
+ const [chatQuestions, setChatQuestions] = useState<string[]>([])
+
+ // The scripted question lists for the compare and chat pickers. Neither
+ // CompareSource nor DemoSource has a "list questions" method by design
+ // (compare-source.ts answers one question at a time; ask() in
+ // demo-source.ts streams chunks for one question), so this reads the same
+ // precomputed bundle replay mode's sources read internally, purely to
+ // populate the pickers. Live mode has no scripted set for either (a live
+ // compare throws, and live ask() throws -- see compare-source.ts and
+ // live-source.ts), so both pickers stay empty there and each panel
+ // explains why on ask.
useEffect(() => {
let cancelled = false
if (!isLive) {
fetch('./precomputed/bundle.json')
.then((r) => r.json())
- .then((bundle: { compare?: Record<string, unknown> }) => {
+ .then((bundle: { compare?: Record<string, unknown>; chat?: Record<string, unknown> }) => {
if (cancelled) return
if (bundle.compare && typeof bundle.compare === 'object') {
setCompareQuestions(Object.keys(bundle.compare))
}
+ if (bundle.chat && typeof bundle.chat === 'object') {
+ setChatQuestions(Object.keys(bundle.chat))
+ }
})
.catch(() => {})
}
@@ -80,12 +91,11 @@ function DemoPanels() {
const result = await src.compare(q)
setCompareResult(result)
} catch (e) {
- setCompareResult({
- question: q,
- grounded: { answer: e instanceof Error ? e.message : String(e), citations: [] },
- baseline: { answer: '', citations: [] },
- divergence: null,
- })
+ // status: 'error' is an explicit discriminant (compare-source.ts's
+ // compareError): ComparePanel renders it as a status line, never under
+ // the "Grounded" column heading, so a failed comparison cannot read as
+ // a grounded win with no baseline answer.
+ setCompareResult(compareError(q, e))
} finally {
setComparePending(false)
}
@@ -96,7 +106,7 @@ function DemoPanels() {
return (
<div className="flex flex-col h-full overflow-hidden">
<div className="flex border-b text-xs shrink-0" style={{ borderColor: 'var(--border)' }}>
- {(['findings', 'corpus', 'compare'] as Tab[]).map((t) => (
+ {(['findings', 'corpus', 'compare', 'chat'] as Tab[]).map((t) => (
<button
key={t}
onClick={() => setTab(t)}
@@ -139,6 +149,9 @@ function DemoPanels() {
pending={comparePending}
/>
)}
+ {tab === 'chat' && (
+ <ScriptedChatPanel chat={chat} pending={chatPending} onAsk={ask} questions={chatQuestions} />
+ )}
</div>
</div>
)
@@ -188,6 +201,24 @@ function ReplayBody() {
)
}
+// While LiveChrome's lazy chunk is still loading, the desktop app has an
+// engine, a model and a network -- it just has not finished loading its own
+// chrome yet. ReplayBody's header claims the opposite ("Offline replay: no
+// engine, no model, no network"), which briefly lies to a live session on
+// every load. This fallback is genuinely neutral instead.
+function LoadingShell() {
+ return (
+ <div
+ className="h-screen flex items-center justify-center"
+ style={{ background: 'var(--bg-primary)' }}
+ >
+ <span className="text-sm" style={{ color: 'var(--text-secondary)' }}>
+ Loading&hellip;
+ </span>
+ </div>
+ )
+}
+
/**
* The single entry point for the whole studio interface. Both build targets
* mount this and nothing else: it decides which chrome to add around the
@@ -206,7 +237,7 @@ export function AppShell() {
if (!isLive) return <ReplayBody />
return (
- <Suspense fallback={<ReplayBody />}>
+ <Suspense fallback={<LoadingShell />}>
<LiveChrome>
<DemoPanels />
</LiveChrome>
diff --git a/studio/src/components/ComparePanel.tsx b/studio/src/components/ComparePanel.tsx
index 453e5930..8a250dfd 100644
--- a/studio/src/components/ComparePanel.tsx
+++ b/studio/src/components/ComparePanel.tsx
@@ -60,14 +60,32 @@ export function ComparePanel({ result, onAsk, questions = [], pending }: Compare
{result.question}
</div>
- <div className="grid grid-cols-2 gap-3">
- <AnswerColumn label="Grounded (ontology)" answer={result.grounded} />
- <AnswerColumn label="Baseline (plain retrieval)" answer={result.baseline} />
- </div>
+ {/* status is an explicit discriminant (compare-source.ts): a
+ failed or unscripted comparison renders as a status line, not
+ under either column's heading, so it can never read as a
+ grounded win with an empty baseline. */}
+ {result.status !== 'ok' ? (
+ <div
+ className="text-sm p-3 rounded"
+ style={{
+ background: 'var(--bg-panel)',
+ color: result.status === 'error' ? 'var(--error)' : 'var(--text-secondary)',
+ }}
+ >
+ {result.status === 'error' ? `Comparison failed: ${result.grounded.answer}` : result.grounded.answer}
+ </div>
+ ) : (
+ <>
+ <div className="grid grid-cols-2 gap-3">
+ <AnswerColumn label="Grounded (ontology)" answer={result.grounded} />
+ <AnswerColumn label="Baseline (plain retrieval)" answer={result.baseline} />
+ </div>
- <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>
- {result.divergence ?? 'No divergence recorded: this question was not in the scripted set.'}
- </div>
+ <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>
+ {result.divergence ?? 'No divergence recorded: this question was not in the scripted set.'}
+ </div>
+ </>
+ )}
</div>
)}
</div>
diff --git a/studio/src/components/ScriptedChatPanel.tsx b/studio/src/components/ScriptedChatPanel.tsx
new file mode 100644
index 00000000..4cd1f8be
--- /dev/null
+++ b/studio/src/components/ScriptedChatPanel.tsx
@@ -0,0 +1,127 @@
+import { useState } from 'react'
+import type { Chunk } from '../lib/demo-source'
+
+export interface ScriptedChatPanelProps {
+ chat: Chunk[]
+ pending?: boolean
+ onAsk: (question: string) => void
+ // The scripted question set, read from the same chat.json keys
+ // replay-source.ts's ask() looks up. Empty in live mode: there is no
+ // fixed set there, and DemoSource.ask() throws with an honest message
+ // instead (see live-source.ts) that this panel just displays like any
+ // other 'unscripted' chunk.
+ questions?: string[]
+}
+
+/**
+ * The scripted-Q&A chat surface, backed by demo/precomputed/chat.json (built
+ * mechanically from compare.json's grounded half by build_precomputed.py --
+ * see its module docstring). Distinct from ChatPanel.tsx, which drives a
+ * live model conversation over the agent sidecar and is desktop-only: this
+ * reads through the same DemoSource.ask() every other panel in this file
+ * uses, so it works in the static web replay build with no engine and no
+ * model.
+ *
+ * Renders on Chunk.type, never guesses from string content: 'tool_call'
+ * chunks render as a small mono trace line, 'unscripted' chunks (the
+ * fixture's own honest "not scripted" fallback, or live mode's thrown-and-
+ * caught error) render as a status line, and only 'text' renders as an
+ * answer.
+ */
+export function ScriptedChatPanel({ chat, pending, onAsk, questions = [] }: ScriptedChatPanelProps) {
+ const [input, setInput] = useState('')
+
+ const send = (q: string) => {
+ const trimmed = q.trim()
+ if (!trimmed) return
+ onAsk(trimmed)
+ setInput('')
+ }
+
+ return (
+ <div className="flex flex-col h-full overflow-hidden">
+ {questions.length > 0 && (
+ <div className="p-3 border-b flex flex-col gap-1 shrink-0" style={{ borderColor: 'var(--border)' }}>
+ <span className="text-xs font-semibold uppercase tracking-wide" style={{ color: 'var(--text-secondary)' }}>
+ Scripted questions
+ </span>
+ {questions.map((q) => (
+ <button
+ key={q}
+ onClick={() => send(q)}
+ className="text-left text-xs px-2 py-1.5 rounded"
+ style={{ background: 'transparent', color: 'var(--text-primary)' }}
+ >
+ {q}
+ </button>
+ ))}
+ </div>
+ )}
+
+ <div className="flex-1 overflow-y-auto p-3 space-y-2">
+ {chat.length === 0 && !pending && (
+ <p className="text-sm" style={{ color: 'var(--text-secondary)' }}>
+ Ask a question, or pick one of the scripted questions above.
+ </p>
+ )}
+ {chat.map((chunk, i) => (
+ <ChunkView key={i} chunk={chunk} />
+ ))}
+ {pending && (
+ <p className="text-xs" style={{ color: 'var(--text-secondary)' }}>
+ Running&hellip;
+ </p>
+ )}
+ </div>
+
+ <div className="p-3 border-t flex gap-2 shrink-0" style={{ borderColor: 'var(--border)' }}>
+ <input
+ value={input}
+ onChange={(e) => setInput(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === 'Enter') send(input)
+ }}
+ placeholder="Ask a scripted question..."
+ className="flex-1 px-2 py-1.5 text-xs rounded outline-none"
+ style={{ background: 'var(--bg-panel)', color: 'var(--text-primary)', border: '1px solid var(--border)' }}
+ />
+ <button
+ onClick={() => send(input)}
+ disabled={!input.trim() || pending}
+ className="px-2 py-1.5 text-xs rounded font-medium"
+ style={{
+ background: input.trim() && !pending ? 'var(--accent)' : 'var(--bg-panel)',
+ color: input.trim() && !pending ? 'var(--bg-primary)' : 'var(--text-secondary)',
+ }}
+ >
+ Ask
+ </button>
+ </div>
+ </div>
+ )
+}
+
+function ChunkView({ chunk }: { chunk: Chunk }) {
+ if (chunk.type === 'tool_call') {
+ return (
+ <div
+ className="text-xs px-2 py-1 rounded font-mono"
+ style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}
+ >
+ {chunk.value}
+ </div>
+ )
+ }
+ if (chunk.type === 'unscripted') {
+ return (
+ <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>
+ {chunk.value}
+ </div>
+ )
+ }
+ return (
+ <div className="text-sm whitespace-pre-wrap" style={{ color: 'var(--text-primary)' }}>
+ {chunk.value}
+ </div>
+ )
+}
diff --git a/studio/src/lib/__tests__/compare-source.test.ts b/studio/src/lib/__tests__/compare-source.test.ts
index b83d94ce..810540b6 100644
--- a/studio/src/lib/__tests__/compare-source.test.ts
+++ b/studio/src/lib/__tests__/compare-source.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'
-import { createReplayCompareSource, compareFixturesFromBundle } from '../compare-source'
+import { createReplayCompareSource, compareFixturesFromBundle, compareError } from '../compare-source'
const fixture = {
'does this profile implement W3C DCAT?': {
@@ -20,6 +20,7 @@ describe('ReplayCompareSource', () => {
it('returns both answers with their citations', async () => {
const src = createReplayCompareSource(fixture)
const r = await src.compare('does this profile implement W3C DCAT?')
+ expect(r.status).toBe('ok')
expect(r.grounded.citations).toContain('examples.json')
expect(r.baseline.citations).toEqual(['profile-readme.md'])
expect(r.divergence).not.toBeNull()
@@ -28,11 +29,40 @@ describe('ReplayCompareSource', () => {
it('reports an unscripted question rather than fabricating a comparison', async () => {
const src = createReplayCompareSource(fixture)
const r = await src.compare('something nobody scripted')
+ // status is the explicit discriminant ComparePanel renders on. Before it
+ // existed, this result was structurally identical to a real answer
+ // (grounded.answer set, baseline empty), so a caller had to infer
+ // "unscripted" from an empty baseline string -- the same shape a failed
+ // comparison produces, which is exactly what let a failure read as a
+ // grounded win.
+ expect(r.status).toBe('unscripted')
expect(r.divergence).toBeNull()
expect(r.grounded.answer).toMatch(/not scripted|offline replay/i)
})
})
+describe('compareError', () => {
+ // The regression this guards: AppShell.tsx's handleAsk catch block used to
+ // build this result inline, putting the exception message into
+ // grounded.answer with an empty baseline -- the same shape as a real
+ // answer and as the unscripted fallback above. ComparePanel then rendered
+ // it under the "Grounded (ontology)" heading with the baseline column
+ // showing "(no answer)", which reads as the grounded side winning.
+ it('marks a failed comparison with status: error, not an answer shape', () => {
+ const r = compareError('does this profile conform?', new Error('engine not listening'))
+ expect(r.status).toBe('error')
+ expect(r.grounded.answer).toBe('engine not listening')
+ expect(r.baseline).toEqual({ answer: '', citations: [] })
+ expect(r.divergence).toBeNull()
+ })
+
+ it('stringifies a thrown value that is not an Error', () => {
+ const r = compareError('q', 'plain string failure')
+ expect(r.status).toBe('error')
+ expect(r.grounded.answer).toBe('plain string failure')
+ })
+})
+
describe('compareFixturesFromBundle', () => {
// bundle.json carries a fifth top-level key, "compare", that
// ReplayFixtures (demo-source.ts / replay-source.ts) only tolerates
diff --git a/studio/src/lib/compare-source.ts b/studio/src/lib/compare-source.ts
index 88abf60c..6b7fd371 100644
--- a/studio/src/lib/compare-source.ts
+++ b/studio/src/lib/compare-source.ts
@@ -3,8 +3,20 @@ import { chooseSourceKind } from './source-factory'
// Deliberately a separate interface from DemoSource (see demo-source.ts).
// Tasks 9 through 11 stay untouched, and the comparison is an optional demo
// surface layered on top rather than a fifth core method.
+//
+// `status` is an explicit discriminant, the same fix Chunk got for its
+// 'unscripted' variant: 'ok' is a real grounded-vs-baseline answer pair from
+// compare.json, 'unscripted' is a question outside the scripted set (the
+// replay fallback below), and 'error' is a comparison that failed to run at
+// all (AppShell.tsx's handleAsk catch). Before this field existed, a failed
+// comparison built a result with the exception message in grounded.answer
+// and an empty baseline -- structurally identical to a real answer, so
+// ComparePanel rendered it under the "Grounded" heading with the baseline
+// showing no answer, which reads as a grounded win. Callers must render on
+// `status`, not by inspecting whether the strings happen to be empty.
export interface CompareResult {
question: string
+ status: 'ok' | 'unscripted' | 'error'
grounded: { answer: string; citations: string[] }
baseline: { answer: string; citations: string[] }
divergence: string | null
@@ -14,15 +26,38 @@ export interface CompareSource {
compare(question: string): Promise<CompareResult>
}
-export type CompareFixtures = Record<string, CompareResult>
+// The single place a failed comparison becomes a CompareResult, used by
+// AppShell.tsx's handleAsk catch block. Before this existed, that catch
+// block built the failure result inline: the exception message went into
+// grounded.answer with baseline left empty, a shape structurally identical
+// to a real answer. ComparePanel then rendered it under the "Grounded"
+// heading with the baseline column showing "(no answer)" -- a comparison
+// failure reading as a grounded win. Extracted as a pure function so the
+// discriminant is unit-testable without a DOM/component test harness (this
+// repo has neither jsdom nor React Testing Library configured).
+export function compareError(question: string, error: unknown): CompareResult {
+ return {
+ question,
+ status: 'error',
+ grounded: { answer: error instanceof Error ? error.message : String(error), citations: [] },
+ baseline: { answer: '', citations: [] },
+ divergence: null,
+ }
+}
+
+// Fixtures on disk (compare.json, via bundle.json) predate the `status`
+// field and carry no such key; every fixture is a real answer pair, so it
+// is always 'ok'.
+export type CompareFixtures = Record<string, Omit<CompareResult, 'status'>>
export function createReplayCompareSource(fixtures: CompareFixtures): CompareSource {
return {
async compare(question: string): Promise<CompareResult> {
const hit = fixtures[question] ?? fixtures[question.trim().toLowerCase()]
- if (hit) return hit
+ if (hit) return { ...hit, status: 'ok' }
return {
question,
+ status: 'unscripted',
grounded: {
answer: 'This question is not scripted in the offline replay.',
citations: [],