Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
feat(studio): render the vacuous-pass SHACL finding in the web replay
FABIOTESSf49442003f3a
diff --git a/.github/workflows/demo-artifacts.yml b/.github/workflows/demo-artifacts.ymlindex 9e5c7fb3..69fdaf06 100644--- a/.github/workflows/demo-artifacts.yml+++ b/.github/workflows/demo-artifacts.yml@@ -9,7 +9,7 @@ jobs:run: cd demo/precomputed && shasum -a 256 -c MANIFEST.sha256# MANIFEST.sha256 only proves each committed file matches its OWN# recorded checksum; it says nothing about whether bundle.json still- # reflects a real merge of the other four parts. A part and the bundle+ # reflects a real merge of the other five parts. A part and the bundle# could drift apart together (edit findings.json, forget to# regenerate bundle.json, re-run shasum so both individually still# check out) and this job would still pass. Regenerating bundle.jsondiff --git a/Makefile b/Makefileindex 62bd4fdd..a1b031fb 100644--- a/Makefile+++ b/Makefile@@ -87,6 +87,7 @@ demo:python3 demo/ontology_from_docs.py --corpus demo/corpus/dcat-us --workers 2python3 demo/corpus_text.pypython3 demo/build_precomputed.py --out demo/precomputed+ python3 demo/build_validation.py --out demo/precomputedpython3 demo/bundle_fixtures.py --in demo/precomputed --out demo/precomputed/bundle.jsoncd demo/precomputed && shasum -a 256 *.json | sort > MANIFEST.sha256diff --git a/demo/build_validation.py b/demo/build_validation.pynew file mode 100644index 00000000..f7c7fb3c--- /dev/null+++ b/demo/build_validation.py@@ -0,0 +1,136 @@+#!/usr/bin/env python3+"""+Build validation.json, the sixth artifact demo/bundle_fixtures.py merges+into bundle.json (PARTS gains "validation" alongside corpus, graph,+findings, chat, compare).++This is the surface the pivot recorded in .superpowers/sdd/progress.md could+not put in the web replay, because showing it required the live engine: a+SHACL validation that reports conforms=true over zero focus nodes, meaning+nothing was actually checked. src/shacl.rs (around lines 552-567) refuses to+call that a pass -- it reports conforms: null with a warning that the shapes+selected nothing to check -- and this script carries the same two runs a+viewer would need to see that refusal for themselves, read straight out of+the measurements demo/dcat_conformance.py already produced and+demo/tests/test_dcat_conformance.py already pins (demo/dcat_conformance_+measurements.json):++ - "as published": the corpus exactly as GSA/dcat-us publishes it, checked+ against the profile's own recovered SHACL shapes (recovered-shapes.ttl,+ unchanged). shacl.legacyShapesOverPublishedCorpus: conforms=true,+ 0 focus nodes, 0 of 34 target classes matched.+ - "schema-derived binding applied": the same corpus, the same unchanged+ shapes file, with the schema-derived RDF binding demo/dcat_conformance.py+ builds from the JSON Schema's own _oldDocs residue (the "observed"+ variant, which additionally relaxes terms the corpus itself publishes as+ prose rather than IRIs).+ shacl.legacyShapesOverObservedBoundCorpus: conforms=false,+ 228 focus nodes, 24 of 34 target classes matched.++Deliberately excludes every violation count. Three legitimate measurement+methods over identical inputs give 178 / 272 / 147 violations, and a fourth+figure (287) is already public in case-studies/dcat-us-binding/README.md;+demo/README.md's own stated position is that none of the four is defensible+as "the" figure. Leaving violation counts out of this artifact means the+client cannot render one even by accident -- see+studio/src/lib/validation-source.ts and studio/src/components/+ValidationPanel.tsx, neither of which is given a number to show.++Usage:+ python3 demo/build_validation.py --out demo/precomputed+"""+import argparse+import json+import pathlib++ROOT = pathlib.Path(__file__).resolve().parent.parent+MEASUREMENTS = ROOT / "demo" / "dcat_conformance_measurements.json"+SHAPES_FILE = "recovered-shapes.ttl"+++def build_validation() -> dict:+ m = json.loads(MEASUREMENTS.read_text(encoding="utf-8"))+ shacl = m["shacl"]+ published = shacl["legacyShapesOverPublishedCorpus"]+ bound = shacl["legacyShapesOverObservedBoundCorpus"]++ # The whole point of this panel depends on this premise. Fail loudly at+ # build time rather than silently shipping a validation.json that no+ # longer demonstrates a vacuous pass.+ if published["focusNodes"] != 0 or published["conforms"] is not True:+ raise SystemExit(+ "legacyShapesOverPublishedCorpus no longer measures a vacuous pass "+ f"(focusNodes={published['focusNodes']}, conforms={published['conforms']}); "+ "the premise this panel demonstrates no longer holds against "+ "demo/dcat_conformance_measurements.json. Re-check before regenerating."+ )+ if bound["focusNodes"] == 0 or bound["conforms"] is not False:+ raise SystemExit(+ "legacyShapesOverObservedBoundCorpus no longer measures a real failure "+ f"(focusNodes={bound['focusNodes']}, conforms={bound['conforms']}); "+ "re-check before regenerating."+ )++ return {+ "shapesFile": SHAPES_FILE,+ "commit": m["commit"],+ "measured": m["measured"],+ "runs": [+ {+ "id": "as-published",+ "label": "Corpus as published",+ "corpusDescription": (+ "115 GSA/dcat-us good examples, exactly as published, checked "+ "against the profile's own recovered SHACL shapes"+ ),+ "dataTriples": published["dataTriples"],+ "focusNodes": published["focusNodes"],+ "matchedClassCount": published["matchedClassCount"],+ "targetClassCount": published["targetClassCount"],+ "conformsRaw": published["conforms"],+ "verdict": "undetermined",+ "reason": (+ "0 of {target} target classes in the shapes file matched anything "+ "in the data, so nothing was checked. Reporting conforms=true here "+ "would be the same claim the engine's own SHACL tool refuses to make "+ "(src/shacl.rs): a validator that selects nothing to check has not "+ "determined conformance."+ ).format(target=published["targetClassCount"]),+ },+ {+ "id": "schema-derived-binding",+ "label": "Corpus with schema-derived binding applied",+ "corpusDescription": (+ "the same 115 examples, the same unchanged shapes file, with the "+ "schema-derived RDF binding applied (demo/dcat_conformance.py)"+ ),+ "dataTriples": bound["dataTriples"],+ "focusNodes": bound["focusNodes"],+ "matchedClassCount": bound["matchedClassCount"],+ "targetClassCount": bound["targetClassCount"],+ "conformsRaw": bound["conforms"],+ "verdict": "fails",+ "reason": None,+ },+ ],+ }+++def main() -> None:+ ap = argparse.ArgumentParser()+ ap.add_argument("--out", required=True, type=pathlib.Path)+ args = ap.parse_args()+ payload = build_validation()+ args.out.mkdir(parents=True, exist_ok=True)+ (args.out / "validation.json").write_text(+ json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n",+ encoding="utf-8",+ )+ print(+ "validation.json: as-published focusNodes="+ f"{payload['runs'][0]['focusNodes']}, bound focusNodes={payload['runs'][1]['focusNodes']}"+ )+++if __name__ == "__main__":+ main()diff --git a/demo/bundle_fixtures.py b/demo/bundle_fixtures.pyindex f1686028..2ee6149d 100644--- a/demo/bundle_fixtures.py+++ b/demo/bundle_fixtures.py@@ -1,10 +1,11 @@-"""Combine the pipeline's five artifacts (PARTS below) into the single object-ReplaySource and CompareSource load: corpus, graph, findings, chat, compare."""+"""Combine the pipeline's six artifacts (PARTS below) into the single object+ReplaySource, CompareSource and ValidationSource load: corpus, graph,+findings, chat, compare, validation."""import argparseimport jsonfrom pathlib import Path-PARTS = ("corpus", "graph", "findings", "chat", "compare")+PARTS = ("corpus", "graph", "findings", "chat", "compare", "validation")def bundle(indir: Path) -> dict:diff --git a/demo/precomputed/MANIFEST.sha256 b/demo/precomputed/MANIFEST.sha256index 290006af..87ecee2a 100644--- a/demo/precomputed/MANIFEST.sha256+++ b/demo/precomputed/MANIFEST.sha256@@ -1,6 +1,7 @@198cbb5a2734e83c7347ad739f2bf8d3d038526bb89b3383ee277929bd307c07 compare.json3a8356eda11e1319c44bcaf8dc156408ac98eae3dada243fbdb56492fa75052e graph.json+55f70cdd599b41becf57ffae5516fd98289f5addeec99ca1d23c78cb91b06be9 bundle.json5c08beaae292b2393262132a4ccb6e582b85e085db88edbefc6c0b760b6dce4b findings.json+a10470d36e5acf6db4d925dbaf434f1a0bf4918dead58ad6b93b57547ea59af4 validation.jsona25163cc058251af89316932c0ebbb76f1b206d529bd65277ffca8c633879b49 corpus.jsonb91d367cc4722fe65a18a49d57b79cea8194673c780579968f644a9f4ef0cada chat.json-d39f7c9797e7eb67874bc60bac15aa1ac3ff126f50ec1969d0993a1df2a07159 bundle.jsondiff --git a/demo/precomputed/bundle.json b/demo/precomputed/bundle.jsonindex 49d99222..3592b408 100644--- a/demo/precomputed/bundle.json+++ b/demo/precomputed/bundle.json@@ -3514,5 +3514,36 @@"label": "version notes"}]+ },+ "validation": {+ "commit": "7a6e803fb94ee9903e7e7405ec4afcc8da13383f",+ "measured": "2026-08-25, this repository, no network, no model call",+ "runs": [+ {+ "conformsRaw": true,+ "corpusDescription": "115 GSA/dcat-us good examples, exactly as published, checked against the profile's own recovered SHACL shapes",+ "dataTriples": 73,+ "focusNodes": 0,+ "id": "as-published",+ "label": "Corpus as published",+ "matchedClassCount": 0,+ "reason": "0 of 34 target classes in the shapes file matched anything in the data, so nothing was checked. Reporting conforms=true here would be the same claim the engine's own SHACL tool refuses to make (src/shacl.rs): a validator that selects nothing to check has not determined conformance.",+ "targetClassCount": 34,+ "verdict": "undetermined"+ },+ {+ "conformsRaw": false,+ "corpusDescription": "the same 115 examples, the same unchanged shapes file, with the schema-derived RDF binding applied (demo/dcat_conformance.py)",+ "dataTriples": 1383,+ "focusNodes": 228,+ "id": "schema-derived-binding",+ "label": "Corpus with schema-derived binding applied",+ "matchedClassCount": 24,+ "reason": null,+ "targetClassCount": 34,+ "verdict": "fails"+ }+ ],+ "shapesFile": "recovered-shapes.ttl"}}diff --git a/demo/precomputed/validation.json b/demo/precomputed/validation.jsonnew file mode 100644index 00000000..2f197c36--- /dev/null+++ b/demo/precomputed/validation.json@@ -0,0 +1,31 @@+{+ "commit": "7a6e803fb94ee9903e7e7405ec4afcc8da13383f",+ "measured": "2026-08-25, this repository, no network, no model call",+ "runs": [+ {+ "conformsRaw": true,+ "corpusDescription": "115 GSA/dcat-us good examples, exactly as published, checked against the profile's own recovered SHACL shapes",+ "dataTriples": 73,+ "focusNodes": 0,+ "id": "as-published",+ "label": "Corpus as published",+ "matchedClassCount": 0,+ "reason": "0 of 34 target classes in the shapes file matched anything in the data, so nothing was checked. Reporting conforms=true here would be the same claim the engine's own SHACL tool refuses to make (src/shacl.rs): a validator that selects nothing to check has not determined conformance.",+ "targetClassCount": 34,+ "verdict": "undetermined"+ },+ {+ "conformsRaw": false,+ "corpusDescription": "the same 115 examples, the same unchanged shapes file, with the schema-derived RDF binding applied (demo/dcat_conformance.py)",+ "dataTriples": 1383,+ "focusNodes": 228,+ "id": "schema-derived-binding",+ "label": "Corpus with schema-derived binding applied",+ "matchedClassCount": 24,+ "reason": null,+ "targetClassCount": 34,+ "verdict": "fails"+ }+ ],+ "shapesFile": "recovered-shapes.ttl"+}diff --git a/demo/tests/test_build_validation.py b/demo/tests/test_build_validation.pynew file mode 100644index 00000000..6b02e094--- /dev/null+++ b/demo/tests/test_build_validation.py@@ -0,0 +1,85 @@+"""Guard the validation.json artifact: the panel this feeds exists to show a+SHACL run with zero focus nodes as NOT a pass. These tests pin that both the+data shape carries no violation count and that the zero-focus-node run is+never marked 'fails' being reinterpreted as a pass, or otherwise indistinct+from a genuine clean result.++demo/dcat_conformance_measurements.json is the single source of truth for+these figures, already pinned by demo/tests/test_dcat_conformance.py; this+file tests build_validation.py's own transform of it, not the measurements+themselves.+"""++import json++import demo.build_validation as bv+++def test_as_published_run_is_undetermined_not_a_pass():+ payload = bv.build_validation()+ as_published = next(r for r in payload["runs"] if r["id"] == "as-published")+ assert as_published["focusNodes"] == 0+ assert as_published["verdict"] == "undetermined"+ assert as_published["verdict"] != "fails"+ # conformsRaw is allowed to be True (the honest raw reading of the SHACL+ # run), but nothing downstream is allowed to call that reading a pass:+ # there is no 'passes' verdict value anywhere in this artifact.+ assert as_published["conformsRaw"] is True+ assert "nothing" in as_published["reason"] or "matched" in as_published["reason"]+++def test_bound_run_is_a_real_failure_distinct_from_the_vacuous_run():+ payload = bv.build_validation()+ bound = next(r for r in payload["runs"] if r["id"] == "schema-derived-binding")+ assert bound["focusNodes"] == 228+ assert bound["verdict"] == "fails"+ assert bound["conformsRaw"] is False+++def test_no_run_carries_a_violation_count():+ # The repository's own position, stated in demo/README.md: no single+ # SHACL violation count is defensible (three legitimate methods over+ # identical inputs give 178 / 272 / 147, and a fourth, 287, is already+ # public elsewhere). Keeping the field out of this artifact means the+ # client cannot render one even by accident.+ payload = bv.build_validation()+ for run in payload["runs"]:+ keys = set(run.keys())+ assert not (keys & {"violations", "violationCount", "violation_count", "bySeverity"})+++def test_verdict_type_has_no_pass_value():+ payload = bv.build_validation()+ verdicts = {r["verdict"] for r in payload["runs"]}+ assert verdicts <= {"undetermined", "fails"}+ assert "passes" not in verdicts+ assert "pass" not in verdicts+++def test_guards_against_a_premise_that_no_longer_holds(tmp_path, monkeypatch):+ """If the as-published corpus ever measures a non-zero focus node count+ (e.g. GSA/dcat-us republishes the deleted context), the premise this+ panel demonstrates no longer holds, and build_validation() must refuse+ to produce a silently misleading artifact rather than ship stale data."""+ broken = tmp_path / "measurements.json"+ real = json.loads(bv.MEASUREMENTS.read_text(encoding="utf-8"))+ real["shacl"]["legacyShapesOverPublishedCorpus"]["focusNodes"] = 1+ broken.write_text(json.dumps(real), encoding="utf-8")++ monkeypatch.setattr(bv, "MEASUREMENTS", broken)+ try:+ bv.build_validation()+ assert False, "expected build_validation() to refuse a broken premise"+ except SystemExit as e:+ assert "vacuous pass" in str(e)+++def test_output_matches_committed_artifact():+ """demo/precomputed/validation.json is committed, like the other+ precomputed parts; this proves it is not stale relative to what+ build_validation.py currently produces from the committed measurements+ file (the CI job in .github/workflows/demo-artifacts.yml catches drift+ at the bundle.json level; this catches it one layer lower)."""+ committed = json.loads((bv.ROOT / "demo" / "precomputed" / "validation.json").read_text(encoding="utf-8"))+ fresh = bv.build_validation()+ assert committed == freshdiff --git a/studio/src/AppShell.tsx b/studio/src/AppShell.tsxindex 50993aea..1fdec8a0 100644--- a/studio/src/AppShell.tsx+++ b/studio/src/AppShell.tsx@@ -5,9 +5,11 @@ import { FindingsPanel } from './components/FindingsPanel'import { ResolutionPanel } from './components/ResolutionPanel'import { ComparePanel } from './components/ComparePanel'import { ScriptedChatPanel } from './components/ScriptedChatPanel'+import { ValidationPanel } from './components/ValidationPanel'import { Graph3D } from './components/Graph3D'import { chooseSourceKind } from './lib/source-factory'import { getCompareSource, compareError, type CompareResult } from './lib/compare-source'+import { getValidationSource, type ValidationFixtures } from './lib/validation-source'import './App.css'// Desktop-only chrome (chat over the agent sidecar, engine status, save/open,@@ -22,7 +24,7 @@ const LiveChrome = lazy(() => import('./components/LiveChrome').then((m) => ({ dconst isLive =chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'live'-type Tab = 'findings' | 'corpus' | 'compare' | 'chat'+type Tab = 'findings' | 'corpus' | 'compare' | 'validation' | 'chat'/*** The tabbed sidebar: findings + resolution, corpus, and the grounded-vs-@@ -53,6 +55,9 @@ function DemoPanels() {const [comparePending, setComparePending] = useState(false)const [compareQuestions, setCompareQuestions] = useState<string[]>([])const [chatQuestions, setChatQuestions] = useState<string[]>([])+ const [validationFixtures, setValidationFixtures] = useState<ValidationFixtures | null>(null)+ const [validationError, setValidationError] = useState<string | null>(null)+ const [validationPending, setValidationPending] = useState(false)// The scripted question lists for the compare and chat pickers. Neither// CompareSource nor DemoSource has a "list questions" method by design@@ -84,6 +89,31 @@ function DemoPanels() {}}, [])+ // The validation panel's two runs are static: there is no question to+ // ask, so they load once on mount rather than on a user action, the way+ // graph and findings load in demo-store.ts. In live mode getValidationSource()+ // throws immediately (see validation-source.ts), and that message is+ // shown as-is rather than swallowed, the same honesty compare-source.ts's+ // live branch already applies.+ useEffect(() => {+ let cancelled = false+ setValidationPending(true)+ getValidationSource()+ .then((src) => src.runs())+ .then((fixtures) => {+ if (!cancelled) setValidationFixtures(fixtures)+ })+ .catch((e) => {+ if (!cancelled) setValidationError(e instanceof Error ? e.message : String(e))+ })+ .finally(() => {+ if (!cancelled) setValidationPending(false)+ })+ return () => {+ cancelled = true+ }+ }, [])+const handleAsk = useCallback(async (q: string) => {setComparePending(true)try {@@ -106,7 +136,7 @@ function DemoPanels() {return (<div className="flex flex-col h-full overflow-hidden"><div className="flex border-b text-xs shrink-0" style={{ borderColor: 'var(--border)' }}>- {(['findings', 'corpus', 'compare', 'chat'] as Tab[]).map((t) => (+ {(['findings', 'corpus', 'compare', 'validation', 'chat'] as Tab[]).map((t) => (<buttonkey={t}onClick={() => setTab(t)}@@ -149,6 +179,9 @@ function DemoPanels() {pending={comparePending}/>)}+ {tab === 'validation' && (+ <ValidationPanel fixtures={validationFixtures} error={validationError} pending={validationPending} />+ )}{tab === 'chat' && (<ScriptedChatPanel chat={chat} pending={chatPending} onAsk={ask} questions={chatQuestions} />)}diff --git a/studio/src/components/ValidationPanel.tsx b/studio/src/components/ValidationPanel.tsxnew file mode 100644index 00000000..92f5baf1--- /dev/null+++ b/studio/src/components/ValidationPanel.tsx@@ -0,0 +1,126 @@+import type { ValidationFixtures, ValidationRun } from '../lib/validation-source'++export interface ValidationPanelProps {+ fixtures: ValidationFixtures | null+ error: string | null+ pending?: boolean+}++/**+ * The moment the rest of the replay cannot show without an engine: the same+ * unchanged SHACL shapes file, run against the corpus twice. Once as+ * GSA/dcat-us actually publishes it -- 0 focus nodes, meaning the shapes+ * selected nothing to check -- and once with the schema-derived binding+ * applied, where 228 focus nodes are actually reached and the run fails.+ *+ * The zero-focus-node run is the whole point of this panel and is rendered+ * as the loudest thing on it, never as a pass with a footnote: `verdict` is+ * either 'undetermined' or 'fails', there is no 'passes' case in the type+ * (validation-source.ts), and this component does not invent one. No+ * violation count appears anywhere here -- ValidationRun does not carry+ * one, so there is nothing for this component to render even by accident.+ */+export function ValidationPanel({ fixtures, error, pending }: ValidationPanelProps) {+ if (pending) {+ return (+ <p className="p-4 text-sm" style={{ color: 'var(--text-secondary)' }}>+ Loading the validation runs…+ </p>+ )+ }+ if (error) {+ return (+ <p className="p-4 text-sm" style={{ color: 'var(--error)' }}>+ {error}+ </p>+ )+ }+ if (!fixtures) {+ return (+ <p className="p-4 text-sm" style={{ color: 'var(--text-secondary)' }}>+ No validation runs available.+ </p>+ )+ }++ return (+ <div className="flex flex-col h-full overflow-y-auto">+ <div className="p-3 border-b" style={{ borderColor: 'var(--border)' }}>+ <div className="text-xs font-semibold uppercase tracking-wide" style={{ color: 'var(--text-secondary)' }}>+ Same shapes file, two corpus states+ </div>+ <div className="text-xs mt-1 font-mono break-all" style={{ color: 'var(--text-secondary)' }}>+ {fixtures.shapesFile}, unchanged in both runs · measured {fixtures.measured}+ </div>+ </div>+ <div className="p-3 flex flex-col gap-3">+ {fixtures.runs.map((run) => (+ <ValidationRunCard key={run.id} run={run} />+ ))}+ </div>+ </div>+ )+}++function ValidationRunCard({ run }: { run: ValidationRun }) {+ const vacuous = run.verdict === 'undetermined'+ return (+ <div+ className="rounded p-3"+ style={{+ background: 'var(--bg-panel)',+ border: `2px solid ${vacuous ? 'var(--error)' : 'var(--border)'}`,+ }}+ >+ <div className="text-sm font-semibold" style={{ color: 'var(--text-primary)' }}>+ {run.label}+ </div>+ <p className="text-xs mt-1" style={{ color: 'var(--text-secondary)' }}>+ {run.corpusDescription}+ </p>++ {/* Focus nodes is the loudest thing on the card: a large numeral, not+ a line item next to a green tick. For the vacuous run this is 0,+ and 0 is rendered exactly as prominently as 228 -- neither number+ is hidden or shrunk to make the panel look tidier. */}+ <div className="mt-3 flex items-baseline gap-2">+ <span+ className="text-3xl font-bold tabular-nums"+ style={{ color: vacuous ? 'var(--error)' : 'var(--text-primary)' }}+ >+ {run.focusNodes}+ </span>+ <span className="text-xs" style={{ color: 'var(--text-secondary)' }}>+ focus node{run.focusNodes === 1 ? '' : 's'} · {run.matchedClassCount} of{' '}+ {run.targetClassCount} target classes matched · {run.dataTriples} data triples+ </span>+ </div>++ {vacuous ? (+ <div+ className="mt-3 p-2 rounded text-xs font-semibold"+ style={{ background: 'var(--error)', color: 'var(--bg-primary)' }}+ >+ NOT A PASS. The validator checked nothing.+ </div>+ ) : (+ <div+ className="mt-3 p-2 rounded text-xs font-semibold"+ style={{ background: 'var(--error)', color: 'var(--bg-primary)' }}+ >+ FAILS. Conforms = false.+ </div>+ )}++ {run.reason && (+ <p className="mt-2 text-xs" style={{ color: 'var(--text-primary)' }}>+ {run.reason}+ </p>+ )}++ <p className="mt-2 text-xs" style={{ color: 'var(--text-secondary)' }}>+ conforms (as reported by the run, before this panel's reading of it): {String(run.conformsRaw)}+ </p>+ </div>+ )+}diff --git a/studio/src/lib/__tests__/validation-source.test.ts b/studio/src/lib/__tests__/validation-source.test.tsnew file mode 100644index 00000000..5390c985--- /dev/null+++ b/studio/src/lib/__tests__/validation-source.test.ts@@ -0,0 +1,113 @@+import { describe, it, expect } from 'vitest'+import {+ createReplayValidationSource,+ validationFixturesFromBundle,+ type ValidationFixtures,+} from '../validation-source'++const fixtures: ValidationFixtures = {+ shapesFile: 'recovered-shapes.ttl',+ commit: '7a6e803fb94ee9903e7e7405ec4afcc8da13383f',+ measured: '2026-08-25, this repository, no network, no model call',+ runs: [+ {+ id: 'as-published',+ label: 'Corpus as published',+ corpusDescription: '115 good examples, exactly as published',+ dataTriples: 73,+ focusNodes: 0,+ matchedClassCount: 0,+ targetClassCount: 34,+ conformsRaw: true,+ verdict: 'undetermined',+ reason: '0 of 34 target classes in the shapes file matched anything in the data.',+ },+ {+ id: 'schema-derived-binding',+ label: 'Corpus with schema-derived binding applied',+ corpusDescription: 'the same 115 examples, with the schema-derived binding applied',+ dataTriples: 1383,+ focusNodes: 228,+ matchedClassCount: 24,+ targetClassCount: 34,+ conformsRaw: false,+ verdict: 'fails',+ reason: null,+ },+ ],+}++describe('createReplayValidationSource', () => {+ it('returns the committed validation runs', async () => {+ const src = createReplayValidationSource(fixtures)+ const out = await src.runs()+ expect(out.runs).toHaveLength(2)+ expect(out.shapesFile).toBe('recovered-shapes.ttl')+ })++ // This is the regression the whole panel exists to guard against: a SHACL+ // run with zero focus nodes checked nothing, and must never be+ // indistinguishable from -- or worse, presented as -- a genuine pass.+ // ValidationRun's `verdict` field has exactly two values, 'undetermined'+ // and 'fails' (validation-source.ts); there is no 'passes' member of the+ // type at all, so a run cannot be typed as a pass even by mistake. This+ // test pins the runtime data alongside that static guarantee: the run+ // with focusNodes === 0 must carry verdict 'undetermined', never 'fails'+ // read as a clean pass and never a bare `conformsRaw: true` left to speak+ // for itself.+ it('marks the zero-focus-node run as undetermined, never as a pass', async () => {+ const src = createReplayValidationSource(fixtures)+ const out = await src.runs()+ const asPublished = out.runs.find((r) => r.focusNodes === 0)+ expect(asPublished).toBeDefined()+ expect(asPublished!.verdict).toBe('undetermined')+ expect(asPublished!.verdict).not.toBe('fails')+ // conformsRaw is allowed to be true (that is the honest raw reading),+ // but nothing in this type or fixture calls that reading a pass.+ expect(asPublished!.conformsRaw).toBe(true)+ expect(asPublished!.reason).toMatch(/nothing|matched/i)+ })++ it('marks the real failure as fails, distinct from the vacuous run', async () => {+ const src = createReplayValidationSource(fixtures)+ const out = await src.runs()+ const bound = out.runs.find((r) => r.focusNodes > 0)+ expect(bound).toBeDefined()+ expect(bound!.verdict).toBe('fails')+ expect(bound!.conformsRaw).toBe(false)+ expect(bound!.focusNodes).toBe(228)+ })++ it('never exposes a violation count field on any run', async () => {+ const src = createReplayValidationSource(fixtures)+ const out = await src.runs()+ for (const run of out.runs) {+ expect(Object.keys(run)).not.toContain('violations')+ expect(Object.keys(run)).not.toContain('violationCount')+ expect(Object.keys(run)).not.toContain('violation_count')+ }+ })+})++describe('validationFixturesFromBundle', () => {+ // bundle.json carries "validation" as a sixth top-level key alongside the+ // four ReplayFixtures declares plus "compare". This is the one place that+ // reaches into it, mirroring compareFixturesFromBundle in+ // compare-source.ts.+ it('extracts the validation key from a bundle carrying the other keys too', () => {+ const bundle = {+ corpus: [],+ graph: { classes: [], properties: [], edges: [] },+ findings: [],+ chat: {},+ compare: {},+ validation: fixtures,+ }+ expect(validationFixturesFromBundle(bundle)).toBe(fixtures)+ })++ it('throws rather than silently returning an empty validation set when the key is missing', () => {+ const bundle = { corpus: [], graph: { classes: [], properties: [], edges: [] }, findings: [], chat: {} }+ expect(() => validationFixturesFromBundle(bundle)).toThrow(/validation/)+ })+})diff --git a/studio/src/lib/validation-source.ts b/studio/src/lib/validation-source.tsnew file mode 100644index 00000000..a20f7f8b--- /dev/null+++ b/studio/src/lib/validation-source.ts@@ -0,0 +1,109 @@+import { chooseSourceKind } from './source-factory'++// Deliberately a separate interface from DemoSource, the same choice+// compare-source.ts made and explains in its own header comment: this is an+// optional demo surface layered on top of the four core methods, not a+// fifth one bolted onto DemoSource itself.+//+// This is the moment the PIVOT in .superpowers/sdd/progress.md could not put+// in the web replay, because it needed the live engine: a SHACL validation+// that returns conforms=true over zero focus nodes, meaning nothing was+// actually checked. src/shacl.rs (around lines 552-567) already refuses to+// call that a pass -- it reports `conforms: null` with a warning that the+// shapes selected nothing -- and `verdict` here is the client-side mirror of+// that refusal: 'undetermined' for a run with zero focus nodes, never+// 'passes'. There is no 'passes' verdict in this type on purpose. A run+// either failed outright ('fails') or checked nothing ('undetermined');+// nothing here can render as a green tick.+//+// No violation count anywhere in this shape, deliberately. Three legitimate+// measurement methods over identical inputs give 178, 272 and 147+// violations, and a fourth figure (287) is already public+// (case-studies/dcat-us-binding/README.md); demo/README.md's own stated+// position is that none of the four is defensible as "the" figure. Focus+// node counts are stable across all methods and are the only counts this+// type carries.+export interface ValidationRun {+ id: string+ label: string+ corpusDescription: string+ dataTriples: number+ focusNodes: number+ matchedClassCount: number+ targetClassCount: number+ // The raw boolean the SHACL run itself returned (true for the+ // as-published run, since `violations.is_empty()` is vacuously true over+ // zero focus nodes). Kept alongside `verdict` rather than replaced by it,+ // so a caller can see exactly what a naive reading of `conforms` would+ // have said, next to why that reading is not the verdict.+ conformsRaw: boolean+ verdict: 'undetermined' | 'fails'+ reason: string | null+}++export interface ValidationFixtures {+ shapesFile: string+ commit: string+ measured: string+ runs: ValidationRun[]+}++export interface ValidationSource {+ runs(): Promise<ValidationFixtures>+}++// The single place a bundle's "validation" key becomes ValidationFixtures,+// mirroring compareFixturesFromBundle in compare-source.ts. bundle.json+// carries this as a sixth top-level key alongside corpus, graph, findings,+// chat and compare; ReplayFixtures does not name it, the same tolerance+// compare-source.ts relies on for "compare".+export function validationFixturesFromBundle(bundle: Record<string, unknown>): ValidationFixtures {+ const raw = bundle.validation+ if (!raw || typeof raw !== 'object') {+ throw new Error(+ 'The precomputed bundle has no "validation" key (or it is not an object); the ' ++ 'SHACL validation panel cannot be shown.',+ )+ }+ return raw as ValidationFixtures+}++export function createReplayValidationSource(fixtures: ValidationFixtures): ValidationSource {+ return {+ async runs(): Promise<ValidationFixtures> {+ return fixtures+ },+ }+}++// The single construction site for ValidationSource, mirroring+// getCompareSource() and getDemoSource(). In replay mode it reads the same+// bundle.json DemoSource does and pulls the "validation" key out of it via+// validationFixturesFromBundle. In live mode there is no honest+// implementation: showing this panel means running the SAME unchanged+// shapes file against TWO different corpus states (as published, and with+// the schema-derived binding applied), which is a build-time measurement+// pipeline (demo/dcat_conformance.py), not a single onto_shacl call a live+// session can make on demand against whatever happens to be loaded.+export async function getValidationSource(): Promise<ValidationSource> {+ if (chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'replay') {+ const response = await fetch('./precomputed/bundle.json')+ if (!response.ok) {+ throw new Error(`Could not load the precomputed demonstration: ${response.status}`)+ }+ const bundle = (await response.json()) as Record<string, unknown>+ return createReplayValidationSource(validationFixturesFromBundle(bundle))+ }+ return {+ async runs(): Promise<ValidationFixtures> {+ throw new Error(+ 'No live validation comparison is implemented. Showing the as-published run next to ' ++ 'the schema-derived-binding run means validating the same unchanged shapes file ' ++ 'against two different corpus states, a measurement pipeline (see ' ++ 'demo/dcat_conformance.py) rather than a single onto_shacl call against whatever ' ++ 'is currently loaded. Run the offline replay (npm run build:web) to see the ' ++ 'precomputed comparison.',+ )+ },+ }+}