Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
fix(studio): explicit discriminant for comparison failures; honest loading state; wire the scripted chat surface
FABIOTESS9f384571cc93
diff --git a/studio/src/AppShell.tsx b/studio/src/AppShell.tsxindex 145358e4..50993aea 100644--- a/studio/src/AppShell.tsx+++ b/studio/src/AppShell.tsx@@ -4,9 +4,10 @@ import { CorpusPanel } from './components/CorpusPanel'import { FindingsPanel } from './components/FindingsPanel'import { ResolutionPanel } from './components/ResolutionPanel'import { ComparePanel } from './components/ComparePanel'+import { ScriptedChatPanel } from './components/ScriptedChatPanel'import { Graph3D } from './components/Graph3D'import { chooseSourceKind } from './lib/source-factory'-import { getCompareSource, type CompareResult } from './lib/compare-source'+import { getCompareSource, compareError, type CompareResult } from './lib/compare-source'import './App.css'// Desktop-only chrome (chat over the agent sidecar, engine status, save/open,@@ -21,7 +22,7 @@ const LiveChrome = lazy(() => import('./components/LiveChrome').then((m) => ({ dconst isLive =chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'live'-type Tab = 'findings' | 'corpus' | 'compare'+type Tab = 'findings' | 'corpus' | 'compare' | 'chat'/*** The tabbed sidebar: findings + resolution, corpus, and the grounded-vs-@@ -38,6 +39,9 @@ function DemoPanels() {const ledger = useDemoStore((s) => s.ledger)const select = useDemoStore((s) => s.select)const resolve = useDemoStore((s) => s.resolve)+ const chat = useDemoStore((s) => s.chat)+ const chatPending = useDemoStore((s) => s.chatPending)+ const ask = useDemoStore((s) => s.ask)// Surfaced here too, not only in ReplayBody's header: DemoPanels is also// embedded inside LiveChrome, which has no header banner of its own, and// a rejected resolve() (see live-source.ts: no engine tool resolves a@@ -48,23 +52,30 @@ function DemoPanels() {const [compareResult, setCompareResult] = useState<CompareResult | null>(null)const [comparePending, setComparePending] = useState(false)const [compareQuestions, setCompareQuestions] = useState<string[]>([])-- // The scripted question list for the compare picker. CompareSource has no- // "list questions" method by design (see compare-source.ts: it answers- // one question at a time), so this reads the same precomputed bundle- // replay mode's CompareSource reads internally, purely to populate the- // picker. Live mode has no scripted set (getCompareSource() throws- // there), so the picker stays empty and the panel explains why on ask.+ const [chatQuestions, setChatQuestions] = useState<string[]>([])++ // The scripted question lists for the compare and chat pickers. Neither+ // CompareSource nor DemoSource has a "list questions" method by design+ // (compare-source.ts answers one question at a time; ask() in+ // demo-source.ts streams chunks for one question), so this reads the same+ // precomputed bundle replay mode's sources read internally, purely to+ // populate the pickers. Live mode has no scripted set for either (a live+ // compare throws, and live ask() throws -- see compare-source.ts and+ // live-source.ts), so both pickers stay empty there and each panel+ // explains why on ask.useEffect(() => {let cancelled = falseif (!isLive) {fetch('./precomputed/bundle.json').then((r) => r.json())- .then((bundle: { compare?: Record<string, unknown> }) => {+ .then((bundle: { compare?: Record<string, unknown>; chat?: Record<string, unknown> }) => {if (cancelled) returnif (bundle.compare && typeof bundle.compare === 'object') {setCompareQuestions(Object.keys(bundle.compare))}+ if (bundle.chat && typeof bundle.chat === 'object') {+ setChatQuestions(Object.keys(bundle.chat))+ }}).catch(() => {})}@@ -80,12 +91,11 @@ function DemoPanels() {const result = await src.compare(q)setCompareResult(result)} catch (e) {- setCompareResult({- question: q,- grounded: { answer: e instanceof Error ? e.message : String(e), citations: [] },- baseline: { answer: '', citations: [] },- divergence: null,- })+ // status: 'error' is an explicit discriminant (compare-source.ts's+ // compareError): ComparePanel renders it as a status line, never under+ // the "Grounded" column heading, so a failed comparison cannot read as+ // a grounded win with no baseline answer.+ setCompareResult(compareError(q, e))} finally {setComparePending(false)}@@ -96,7 +106,7 @@ function DemoPanels() {return (<div className="flex flex-col h-full overflow-hidden"><div className="flex border-b text-xs shrink-0" style={{ borderColor: 'var(--border)' }}>- {(['findings', 'corpus', 'compare'] as Tab[]).map((t) => (+ {(['findings', 'corpus', 'compare', 'chat'] as Tab[]).map((t) => (<buttonkey={t}onClick={() => setTab(t)}@@ -139,6 +149,9 @@ function DemoPanels() {pending={comparePending}/>)}+ {tab === 'chat' && (+ <ScriptedChatPanel chat={chat} pending={chatPending} onAsk={ask} questions={chatQuestions} />+ )}</div></div>)@@ -188,6 +201,24 @@ function ReplayBody() {)}+// While LiveChrome's lazy chunk is still loading, the desktop app has an+// engine, a model and a network -- it just has not finished loading its own+// chrome yet. ReplayBody's header claims the opposite ("Offline replay: no+// engine, no model, no network"), which briefly lies to a live session on+// every load. This fallback is genuinely neutral instead.+function LoadingShell() {+ return (+ <div+ className="h-screen flex items-center justify-center"+ style={{ background: 'var(--bg-primary)' }}+ >+ <span className="text-sm" style={{ color: 'var(--text-secondary)' }}>+ Loading…+ </span>+ </div>+ )+}+/*** The single entry point for the whole studio interface. Both build targets* mount this and nothing else: it decides which chrome to add around the@@ -206,7 +237,7 @@ export function AppShell() {if (!isLive) return <ReplayBody />return (- <Suspense fallback={<ReplayBody />}>+ <Suspense fallback={<LoadingShell />}><LiveChrome><DemoPanels /></LiveChrome>diff --git a/studio/src/components/ComparePanel.tsx b/studio/src/components/ComparePanel.tsxindex 453e5930..8a250dfd 100644--- a/studio/src/components/ComparePanel.tsx+++ b/studio/src/components/ComparePanel.tsx@@ -60,14 +60,32 @@ export function ComparePanel({ result, onAsk, questions = [], pending }: Compare{result.question}</div>- <div className="grid grid-cols-2 gap-3">- <AnswerColumn label="Grounded (ontology)" answer={result.grounded} />- <AnswerColumn label="Baseline (plain retrieval)" answer={result.baseline} />- </div>+ {/* status is an explicit discriminant (compare-source.ts): a+ failed or unscripted comparison renders as a status line, not+ under either column's heading, so it can never read as a+ grounded win with an empty baseline. */}+ {result.status !== 'ok' ? (+ <div+ className="text-sm p-3 rounded"+ style={{+ background: 'var(--bg-panel)',+ color: result.status === 'error' ? 'var(--error)' : 'var(--text-secondary)',+ }}+ >+ {result.status === 'error' ? `Comparison failed: ${result.grounded.answer}` : result.grounded.answer}+ </div>+ ) : (+ <>+ <div className="grid grid-cols-2 gap-3">+ <AnswerColumn label="Grounded (ontology)" answer={result.grounded} />+ <AnswerColumn label="Baseline (plain retrieval)" answer={result.baseline} />+ </div>- <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>- {result.divergence ?? 'No divergence recorded: this question was not in the scripted set.'}- </div>+ <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>+ {result.divergence ?? 'No divergence recorded: this question was not in the scripted set.'}+ </div>+ </>+ )}</div>)}</div>diff --git a/studio/src/components/ScriptedChatPanel.tsx b/studio/src/components/ScriptedChatPanel.tsxnew file mode 100644index 00000000..4cd1f8be--- /dev/null+++ b/studio/src/components/ScriptedChatPanel.tsx@@ -0,0 +1,127 @@+import { useState } from 'react'+import type { Chunk } from '../lib/demo-source'++export interface ScriptedChatPanelProps {+ chat: Chunk[]+ pending?: boolean+ onAsk: (question: string) => void+ // The scripted question set, read from the same chat.json keys+ // replay-source.ts's ask() looks up. Empty in live mode: there is no+ // fixed set there, and DemoSource.ask() throws with an honest message+ // instead (see live-source.ts) that this panel just displays like any+ // other 'unscripted' chunk.+ questions?: string[]+}++/**+ * The scripted-Q&A chat surface, backed by demo/precomputed/chat.json (built+ * mechanically from compare.json's grounded half by build_precomputed.py --+ * see its module docstring). Distinct from ChatPanel.tsx, which drives a+ * live model conversation over the agent sidecar and is desktop-only: this+ * reads through the same DemoSource.ask() every other panel in this file+ * uses, so it works in the static web replay build with no engine and no+ * model.+ *+ * Renders on Chunk.type, never guesses from string content: 'tool_call'+ * chunks render as a small mono trace line, 'unscripted' chunks (the+ * fixture's own honest "not scripted" fallback, or live mode's thrown-and-+ * caught error) render as a status line, and only 'text' renders as an+ * answer.+ */+export function ScriptedChatPanel({ chat, pending, onAsk, questions = [] }: ScriptedChatPanelProps) {+ const [input, setInput] = useState('')++ const send = (q: string) => {+ const trimmed = q.trim()+ if (!trimmed) return+ onAsk(trimmed)+ setInput('')+ }++ return (+ <div className="flex flex-col h-full overflow-hidden">+ {questions.length > 0 && (+ <div className="p-3 border-b flex flex-col gap-1 shrink-0" style={{ borderColor: 'var(--border)' }}>+ <span className="text-xs font-semibold uppercase tracking-wide" style={{ color: 'var(--text-secondary)' }}>+ Scripted questions+ </span>+ {questions.map((q) => (+ <button+ key={q}+ onClick={() => send(q)}+ className="text-left text-xs px-2 py-1.5 rounded"+ style={{ background: 'transparent', color: 'var(--text-primary)' }}+ >+ {q}+ </button>+ ))}+ </div>+ )}++ <div className="flex-1 overflow-y-auto p-3 space-y-2">+ {chat.length === 0 && !pending && (+ <p className="text-sm" style={{ color: 'var(--text-secondary)' }}>+ Ask a question, or pick one of the scripted questions above.+ </p>+ )}+ {chat.map((chunk, i) => (+ <ChunkView key={i} chunk={chunk} />+ ))}+ {pending && (+ <p className="text-xs" style={{ color: 'var(--text-secondary)' }}>+ Running…+ </p>+ )}+ </div>++ <div className="p-3 border-t flex gap-2 shrink-0" style={{ borderColor: 'var(--border)' }}>+ <input+ value={input}+ onChange={(e) => setInput(e.target.value)}+ onKeyDown={(e) => {+ if (e.key === 'Enter') send(input)+ }}+ placeholder="Ask a scripted question..."+ className="flex-1 px-2 py-1.5 text-xs rounded outline-none"+ style={{ background: 'var(--bg-panel)', color: 'var(--text-primary)', border: '1px solid var(--border)' }}+ />+ <button+ onClick={() => send(input)}+ disabled={!input.trim() || pending}+ className="px-2 py-1.5 text-xs rounded font-medium"+ style={{+ background: input.trim() && !pending ? 'var(--accent)' : 'var(--bg-panel)',+ color: input.trim() && !pending ? 'var(--bg-primary)' : 'var(--text-secondary)',+ }}+ >+ Ask+ </button>+ </div>+ </div>+ )+}++function ChunkView({ chunk }: { chunk: Chunk }) {+ if (chunk.type === 'tool_call') {+ return (+ <div+ className="text-xs px-2 py-1 rounded font-mono"+ style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}+ >+ {chunk.value}+ </div>+ )+ }+ if (chunk.type === 'unscripted') {+ return (+ <div className="text-xs p-2 rounded" style={{ background: 'var(--bg-panel)', color: 'var(--text-secondary)' }}>+ {chunk.value}+ </div>+ )+ }+ return (+ <div className="text-sm whitespace-pre-wrap" style={{ color: 'var(--text-primary)' }}>+ {chunk.value}+ </div>+ )+}diff --git a/studio/src/lib/__tests__/compare-source.test.ts b/studio/src/lib/__tests__/compare-source.test.tsindex b83d94ce..810540b6 100644--- a/studio/src/lib/__tests__/compare-source.test.ts+++ b/studio/src/lib/__tests__/compare-source.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest'-import { createReplayCompareSource, compareFixturesFromBundle } from '../compare-source'+import { createReplayCompareSource, compareFixturesFromBundle, compareError } from '../compare-source'const fixture = {'does this profile implement W3C DCAT?': {@@ -20,6 +20,7 @@ describe('ReplayCompareSource', () => {it('returns both answers with their citations', async () => {const src = createReplayCompareSource(fixture)const r = await src.compare('does this profile implement W3C DCAT?')+ expect(r.status).toBe('ok')expect(r.grounded.citations).toContain('examples.json')expect(r.baseline.citations).toEqual(['profile-readme.md'])expect(r.divergence).not.toBeNull()@@ -28,11 +29,40 @@ describe('ReplayCompareSource', () => {it('reports an unscripted question rather than fabricating a comparison', async () => {const src = createReplayCompareSource(fixture)const r = await src.compare('something nobody scripted')+ // status is the explicit discriminant ComparePanel renders on. Before it+ // existed, this result was structurally identical to a real answer+ // (grounded.answer set, baseline empty), so a caller had to infer+ // "unscripted" from an empty baseline string -- the same shape a failed+ // comparison produces, which is exactly what let a failure read as a+ // grounded win.+ expect(r.status).toBe('unscripted')expect(r.divergence).toBeNull()expect(r.grounded.answer).toMatch(/not scripted|offline replay/i)})})+describe('compareError', () => {+ // The regression this guards: AppShell.tsx's handleAsk catch block used to+ // build this result inline, putting the exception message into+ // grounded.answer with an empty baseline -- the same shape as a real+ // answer and as the unscripted fallback above. ComparePanel then rendered+ // it under the "Grounded (ontology)" heading with the baseline column+ // showing "(no answer)", which reads as the grounded side winning.+ it('marks a failed comparison with status: error, not an answer shape', () => {+ const r = compareError('does this profile conform?', new Error('engine not listening'))+ expect(r.status).toBe('error')+ expect(r.grounded.answer).toBe('engine not listening')+ expect(r.baseline).toEqual({ answer: '', citations: [] })+ expect(r.divergence).toBeNull()+ })++ it('stringifies a thrown value that is not an Error', () => {+ const r = compareError('q', 'plain string failure')+ expect(r.status).toBe('error')+ expect(r.grounded.answer).toBe('plain string failure')+ })+})+describe('compareFixturesFromBundle', () => {// bundle.json carries a fifth top-level key, "compare", that// ReplayFixtures (demo-source.ts / replay-source.ts) only toleratesdiff --git a/studio/src/lib/compare-source.ts b/studio/src/lib/compare-source.tsindex 88abf60c..6b7fd371 100644--- a/studio/src/lib/compare-source.ts+++ b/studio/src/lib/compare-source.ts@@ -3,8 +3,20 @@ import { chooseSourceKind } from './source-factory'// Deliberately a separate interface from DemoSource (see demo-source.ts).// Tasks 9 through 11 stay untouched, and the comparison is an optional demo// surface layered on top rather than a fifth core method.+//+// `status` is an explicit discriminant, the same fix Chunk got for its+// 'unscripted' variant: 'ok' is a real grounded-vs-baseline answer pair from+// compare.json, 'unscripted' is a question outside the scripted set (the+// replay fallback below), and 'error' is a comparison that failed to run at+// all (AppShell.tsx's handleAsk catch). Before this field existed, a failed+// comparison built a result with the exception message in grounded.answer+// and an empty baseline -- structurally identical to a real answer, so+// ComparePanel rendered it under the "Grounded" heading with the baseline+// showing no answer, which reads as a grounded win. Callers must render on+// `status`, not by inspecting whether the strings happen to be empty.export interface CompareResult {question: string+ status: 'ok' | 'unscripted' | 'error'grounded: { answer: string; citations: string[] }baseline: { answer: string; citations: string[] }divergence: string | null@@ -14,15 +26,38 @@ export interface CompareSource {compare(question: string): Promise<CompareResult>}-export type CompareFixtures = Record<string, CompareResult>+// The single place a failed comparison becomes a CompareResult, used by+// AppShell.tsx's handleAsk catch block. Before this existed, that catch+// block built the failure result inline: the exception message went into+// grounded.answer with baseline left empty, a shape structurally identical+// to a real answer. ComparePanel then rendered it under the "Grounded"+// heading with the baseline column showing "(no answer)" -- a comparison+// failure reading as a grounded win. Extracted as a pure function so the+// discriminant is unit-testable without a DOM/component test harness (this+// repo has neither jsdom nor React Testing Library configured).+export function compareError(question: string, error: unknown): CompareResult {+ return {+ question,+ status: 'error',+ grounded: { answer: error instanceof Error ? error.message : String(error), citations: [] },+ baseline: { answer: '', citations: [] },+ divergence: null,+ }+}++// Fixtures on disk (compare.json, via bundle.json) predate the `status`+// field and carry no such key; every fixture is a real answer pair, so it+// is always 'ok'.+export type CompareFixtures = Record<string, Omit<CompareResult, 'status'>>export function createReplayCompareSource(fixtures: CompareFixtures): CompareSource {return {async compare(question: string): Promise<CompareResult> {const hit = fixtures[question] ?? fixtures[question.trim().toLowerCase()]- if (hit) return hit+ if (hit) return { ...hit, status: 'ok' }return {question,+ status: 'unscripted',grounded: {answer: 'This question is not scripted in the offline replay.',citations: [],