Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

feat(studio): render the vacuous-pass SHACL finding in the web replay

FABIOTESSf49442003f3a
diff --git a/.github/workflows/demo-artifacts.yml b/.github/workflows/demo-artifacts.yml
index 9e5c7fb3..69fdaf06 100644
--- a/.github/workflows/demo-artifacts.yml
+++ b/.github/workflows/demo-artifacts.yml
@@ -9,7 +9,7 @@ jobs:
run: cd demo/precomputed && shasum -a 256 -c MANIFEST.sha256
# MANIFEST.sha256 only proves each committed file matches its OWN
# recorded checksum; it says nothing about whether bundle.json still
- # reflects a real merge of the other four parts. A part and the bundle
+ # reflects a real merge of the other five parts. A part and the bundle
# could drift apart together (edit findings.json, forget to
# regenerate bundle.json, re-run shasum so both individually still
# check out) and this job would still pass. Regenerating bundle.json
diff --git a/Makefile b/Makefile
index 62bd4fdd..a1b031fb 100644
--- a/Makefile
+++ b/Makefile
@@ -87,6 +87,7 @@ demo:
python3 demo/ontology_from_docs.py --corpus demo/corpus/dcat-us --workers 2
python3 demo/corpus_text.py
python3 demo/build_precomputed.py --out demo/precomputed
+ python3 demo/build_validation.py --out demo/precomputed
python3 demo/bundle_fixtures.py --in demo/precomputed --out demo/precomputed/bundle.json
cd demo/precomputed && shasum -a 256 *.json | sort > MANIFEST.sha256
diff --git a/demo/build_validation.py b/demo/build_validation.py
new file mode 100644
index 00000000..f7c7fb3c
--- /dev/null
+++ b/demo/build_validation.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""
+Build validation.json, the sixth artifact demo/bundle_fixtures.py merges
+into bundle.json (PARTS gains "validation" alongside corpus, graph,
+findings, chat, compare).
+
+This is the surface the pivot recorded in .superpowers/sdd/progress.md could
+not put in the web replay, because showing it required the live engine: a
+SHACL validation that reports conforms=true over zero focus nodes, meaning
+nothing was actually checked. src/shacl.rs (around lines 552-567) refuses to
+call that a pass -- it reports conforms: null with a warning that the shapes
+selected nothing to check -- and this script carries the same two runs a
+viewer would need to see that refusal for themselves, read straight out of
+the measurements demo/dcat_conformance.py already produced and
+demo/tests/test_dcat_conformance.py already pins (demo/dcat_conformance_
+measurements.json):
+
+ - "as published": the corpus exactly as GSA/dcat-us publishes it, checked
+ against the profile's own recovered SHACL shapes (recovered-shapes.ttl,
+ unchanged). shacl.legacyShapesOverPublishedCorpus: conforms=true,
+ 0 focus nodes, 0 of 34 target classes matched.
+ - "schema-derived binding applied": the same corpus, the same unchanged
+ shapes file, with the schema-derived RDF binding demo/dcat_conformance.py
+ builds from the JSON Schema's own _oldDocs residue (the "observed"
+ variant, which additionally relaxes terms the corpus itself publishes as
+ prose rather than IRIs).
+ shacl.legacyShapesOverObservedBoundCorpus: conforms=false,
+ 228 focus nodes, 24 of 34 target classes matched.
+
+Deliberately excludes every violation count. Three legitimate measurement
+methods over identical inputs give 178 / 272 / 147 violations, and a fourth
+figure (287) is already public in case-studies/dcat-us-binding/README.md;
+demo/README.md's own stated position is that none of the four is defensible
+as "the" figure. Leaving violation counts out of this artifact means the
+client cannot render one even by accident -- see
+studio/src/lib/validation-source.ts and studio/src/components/
+ValidationPanel.tsx, neither of which is given a number to show.
+
+Usage:
+ python3 demo/build_validation.py --out demo/precomputed
+"""
+import argparse
+import json
+import pathlib
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+MEASUREMENTS = ROOT / "demo" / "dcat_conformance_measurements.json"
+SHAPES_FILE = "recovered-shapes.ttl"
+
+
+def build_validation() -> dict:
+ m = json.loads(MEASUREMENTS.read_text(encoding="utf-8"))
+ shacl = m["shacl"]
+ published = shacl["legacyShapesOverPublishedCorpus"]
+ bound = shacl["legacyShapesOverObservedBoundCorpus"]
+
+ # The whole point of this panel depends on this premise. Fail loudly at
+ # build time rather than silently shipping a validation.json that no
+ # longer demonstrates a vacuous pass.
+ if published["focusNodes"] != 0 or published["conforms"] is not True:
+ raise SystemExit(
+ "legacyShapesOverPublishedCorpus no longer measures a vacuous pass "
+ f"(focusNodes={published['focusNodes']}, conforms={published['conforms']}); "
+ "the premise this panel demonstrates no longer holds against "
+ "demo/dcat_conformance_measurements.json. Re-check before regenerating."
+ )
+ if bound["focusNodes"] == 0 or bound["conforms"] is not False:
+ raise SystemExit(
+ "legacyShapesOverObservedBoundCorpus no longer measures a real failure "
+ f"(focusNodes={bound['focusNodes']}, conforms={bound['conforms']}); "
+ "re-check before regenerating."
+ )
+
+ return {
+ "shapesFile": SHAPES_FILE,
+ "commit": m["commit"],
+ "measured": m["measured"],
+ "runs": [
+ {
+ "id": "as-published",
+ "label": "Corpus as published",
+ "corpusDescription": (
+ "115 GSA/dcat-us good examples, exactly as published, checked "
+ "against the profile's own recovered SHACL shapes"
+ ),
+ "dataTriples": published["dataTriples"],
+ "focusNodes": published["focusNodes"],
+ "matchedClassCount": published["matchedClassCount"],
+ "targetClassCount": published["targetClassCount"],
+ "conformsRaw": published["conforms"],
+ "verdict": "undetermined",
+ "reason": (
+ "0 of {target} target classes in the shapes file matched anything "
+ "in the data, so nothing was checked. Reporting conforms=true here "
+ "would be the same claim the engine's own SHACL tool refuses to make "
+ "(src/shacl.rs): a validator that selects nothing to check has not "
+ "determined conformance."
+ ).format(target=published["targetClassCount"]),
+ },
+ {
+ "id": "schema-derived-binding",
+ "label": "Corpus with schema-derived binding applied",
+ "corpusDescription": (
+ "the same 115 examples, the same unchanged shapes file, with the "
+ "schema-derived RDF binding applied (demo/dcat_conformance.py)"
+ ),
+ "dataTriples": bound["dataTriples"],
+ "focusNodes": bound["focusNodes"],
+ "matchedClassCount": bound["matchedClassCount"],
+ "targetClassCount": bound["targetClassCount"],
+ "conformsRaw": bound["conforms"],
+ "verdict": "fails",
+ "reason": None,
+ },
+ ],
+ }
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--out", required=True, type=pathlib.Path)
+ args = ap.parse_args()
+ payload = build_validation()
+ args.out.mkdir(parents=True, exist_ok=True)
+ (args.out / "validation.json").write_text(
+ json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
+ encoding="utf-8",
+ )
+ print(
+ "validation.json: as-published focusNodes="
+ f"{payload['runs'][0]['focusNodes']}, bound focusNodes={payload['runs'][1]['focusNodes']}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/demo/bundle_fixtures.py b/demo/bundle_fixtures.py
index f1686028..2ee6149d 100644
--- a/demo/bundle_fixtures.py
+++ b/demo/bundle_fixtures.py
@@ -1,10 +1,11 @@
-"""Combine the pipeline's five artifacts (PARTS below) into the single object
-ReplaySource and CompareSource load: corpus, graph, findings, chat, compare."""
+"""Combine the pipeline's six artifacts (PARTS below) into the single object
+ReplaySource, CompareSource and ValidationSource load: corpus, graph,
+findings, chat, compare, validation."""
import argparse
import json
from pathlib import Path
-PARTS = ("corpus", "graph", "findings", "chat", "compare")
+PARTS = ("corpus", "graph", "findings", "chat", "compare", "validation")
def bundle(indir: Path) -> dict:
diff --git a/demo/precomputed/MANIFEST.sha256 b/demo/precomputed/MANIFEST.sha256
index 290006af..87ecee2a 100644
--- a/demo/precomputed/MANIFEST.sha256
+++ b/demo/precomputed/MANIFEST.sha256
@@ -1,6 +1,7 @@
198cbb5a2734e83c7347ad739f2bf8d3d038526bb89b3383ee277929bd307c07 compare.json
3a8356eda11e1319c44bcaf8dc156408ac98eae3dada243fbdb56492fa75052e graph.json
+55f70cdd599b41becf57ffae5516fd98289f5addeec99ca1d23c78cb91b06be9 bundle.json
5c08beaae292b2393262132a4ccb6e582b85e085db88edbefc6c0b760b6dce4b findings.json
+a10470d36e5acf6db4d925dbaf434f1a0bf4918dead58ad6b93b57547ea59af4 validation.json
a25163cc058251af89316932c0ebbb76f1b206d529bd65277ffca8c633879b49 corpus.json
b91d367cc4722fe65a18a49d57b79cea8194673c780579968f644a9f4ef0cada chat.json
-d39f7c9797e7eb67874bc60bac15aa1ac3ff126f50ec1969d0993a1df2a07159 bundle.json
diff --git a/demo/precomputed/bundle.json b/demo/precomputed/bundle.json
index 49d99222..3592b408 100644
--- a/demo/precomputed/bundle.json
+++ b/demo/precomputed/bundle.json
@@ -3514,5 +3514,36 @@
"label": "version notes"
}
]
+ },
+ "validation": {
+ "commit": "7a6e803fb94ee9903e7e7405ec4afcc8da13383f",
+ "measured": "2026-08-25, this repository, no network, no model call",
+ "runs": [
+ {
+ "conformsRaw": true,
+ "corpusDescription": "115 GSA/dcat-us good examples, exactly as published, checked against the profile's own recovered SHACL shapes",
+ "dataTriples": 73,
+ "focusNodes": 0,
+ "id": "as-published",
+ "label": "Corpus as published",
+ "matchedClassCount": 0,
+ "reason": "0 of 34 target classes in the shapes file matched anything in the data, so nothing was checked. Reporting conforms=true here would be the same claim the engine's own SHACL tool refuses to make (src/shacl.rs): a validator that selects nothing to check has not determined conformance.",
+ "targetClassCount": 34,
+ "verdict": "undetermined"
+ },
+ {
+ "conformsRaw": false,
+ "corpusDescription": "the same 115 examples, the same unchanged shapes file, with the schema-derived RDF binding applied (demo/dcat_conformance.py)",
+ "dataTriples": 1383,
+ "focusNodes": 228,
+ "id": "schema-derived-binding",
+ "label": "Corpus with schema-derived binding applied",
+ "matchedClassCount": 24,
+ "reason": null,
+ "targetClassCount": 34,
+ "verdict": "fails"
+ }
+ ],
+ "shapesFile": "recovered-shapes.ttl"
}
}
diff --git a/demo/precomputed/validation.json b/demo/precomputed/validation.json
new file mode 100644
index 00000000..2f197c36
--- /dev/null
+++ b/demo/precomputed/validation.json
@@ -0,0 +1,31 @@
+{
+ "commit": "7a6e803fb94ee9903e7e7405ec4afcc8da13383f",
+ "measured": "2026-08-25, this repository, no network, no model call",
+ "runs": [
+ {
+ "conformsRaw": true,
+ "corpusDescription": "115 GSA/dcat-us good examples, exactly as published, checked against the profile's own recovered SHACL shapes",
+ "dataTriples": 73,
+ "focusNodes": 0,
+ "id": "as-published",
+ "label": "Corpus as published",
+ "matchedClassCount": 0,
+ "reason": "0 of 34 target classes in the shapes file matched anything in the data, so nothing was checked. Reporting conforms=true here would be the same claim the engine's own SHACL tool refuses to make (src/shacl.rs): a validator that selects nothing to check has not determined conformance.",
+ "targetClassCount": 34,
+ "verdict": "undetermined"
+ },
+ {
+ "conformsRaw": false,
+ "corpusDescription": "the same 115 examples, the same unchanged shapes file, with the schema-derived RDF binding applied (demo/dcat_conformance.py)",
+ "dataTriples": 1383,
+ "focusNodes": 228,
+ "id": "schema-derived-binding",
+ "label": "Corpus with schema-derived binding applied",
+ "matchedClassCount": 24,
+ "reason": null,
+ "targetClassCount": 34,
+ "verdict": "fails"
+ }
+ ],
+ "shapesFile": "recovered-shapes.ttl"
+}
diff --git a/demo/tests/test_build_validation.py b/demo/tests/test_build_validation.py
new file mode 100644
index 00000000..6b02e094
--- /dev/null
+++ b/demo/tests/test_build_validation.py
@@ -0,0 +1,85 @@
+"""Guard the validation.json artifact: the panel this feeds exists to show a
+SHACL run with zero focus nodes as NOT a pass. These tests pin that both the
+data shape carries no violation count and that the zero-focus-node run is
+never marked 'fails' being reinterpreted as a pass, or otherwise indistinct
+from a genuine clean result.
+
+demo/dcat_conformance_measurements.json is the single source of truth for
+these figures, already pinned by demo/tests/test_dcat_conformance.py; this
+file tests build_validation.py's own transform of it, not the measurements
+themselves.
+"""
+
+import json
+
+import demo.build_validation as bv
+
+
+def test_as_published_run_is_undetermined_not_a_pass():
+ payload = bv.build_validation()
+ as_published = next(r for r in payload["runs"] if r["id"] == "as-published")
+ assert as_published["focusNodes"] == 0
+ assert as_published["verdict"] == "undetermined"
+ assert as_published["verdict"] != "fails"
+ # conformsRaw is allowed to be True (the honest raw reading of the SHACL
+ # run), but nothing downstream is allowed to call that reading a pass:
+ # there is no 'passes' verdict value anywhere in this artifact.
+ assert as_published["conformsRaw"] is True
+ assert "nothing" in as_published["reason"] or "matched" in as_published["reason"]
+
+
+def test_bound_run_is_a_real_failure_distinct_from_the_vacuous_run():
+ payload = bv.build_validation()
+ bound = next(r for r in payload["runs"] if r["id"] == "schema-derived-binding")
+ assert bound["focusNodes"] == 228
+ assert bound["verdict"] == "fails"
+ assert bound["conformsRaw"] is False
+
+
+def test_no_run_carries_a_violation_count():
+ # The repository's own position, stated in demo/README.md: no single
+ # SHACL violation count is defensible (three legitimate methods over
+ # identical inputs give 178 / 272 / 147, and a fourth, 287, is already
+ # public elsewhere). Keeping the field out of this artifact means the
+ # client cannot render one even by accident.
+ payload = bv.build_validation()
+ for run in payload["runs"]:
+ keys = set(run.keys())
+ assert not (keys & {"violations", "violationCount", "violation_count", "bySeverity"})
+
+
+def test_verdict_type_has_no_pass_value():
+ payload = bv.build_validation()
+ verdicts = {r["verdict"] for r in payload["runs"]}
+ assert verdicts <= {"undetermined", "fails"}
+ assert "passes" not in verdicts
+ assert "pass" not in verdicts
+
+
+def test_guards_against_a_premise_that_no_longer_holds(tmp_path, monkeypatch):
+ """If the as-published corpus ever measures a non-zero focus node count
+ (e.g. GSA/dcat-us republishes the deleted context), the premise this
+ panel demonstrates no longer holds, and build_validation() must refuse
+ to produce a silently misleading artifact rather than ship stale data."""
+ broken = tmp_path / "measurements.json"
+ real = json.loads(bv.MEASUREMENTS.read_text(encoding="utf-8"))
+ real["shacl"]["legacyShapesOverPublishedCorpus"]["focusNodes"] = 1
+ broken.write_text(json.dumps(real), encoding="utf-8")
+
+ monkeypatch.setattr(bv, "MEASUREMENTS", broken)
+ try:
+ bv.build_validation()
+ assert False, "expected build_validation() to refuse a broken premise"
+ except SystemExit as e:
+ assert "vacuous pass" in str(e)
+
+
+def test_output_matches_committed_artifact():
+ """demo/precomputed/validation.json is committed, like the other
+ precomputed parts; this proves it is not stale relative to what
+ build_validation.py currently produces from the committed measurements
+ file (the CI job in .github/workflows/demo-artifacts.yml catches drift
+ at the bundle.json level; this catches it one layer lower)."""
+ committed = json.loads((bv.ROOT / "demo" / "precomputed" / "validation.json").read_text(encoding="utf-8"))
+ fresh = bv.build_validation()
+ assert committed == fresh
diff --git a/studio/src/AppShell.tsx b/studio/src/AppShell.tsx
index 50993aea..1fdec8a0 100644
--- a/studio/src/AppShell.tsx
+++ b/studio/src/AppShell.tsx
@@ -5,9 +5,11 @@ import { FindingsPanel } from './components/FindingsPanel'
import { ResolutionPanel } from './components/ResolutionPanel'
import { ComparePanel } from './components/ComparePanel'
import { ScriptedChatPanel } from './components/ScriptedChatPanel'
+import { ValidationPanel } from './components/ValidationPanel'
import { Graph3D } from './components/Graph3D'
import { chooseSourceKind } from './lib/source-factory'
import { getCompareSource, compareError, type CompareResult } from './lib/compare-source'
+import { getValidationSource, type ValidationFixtures } from './lib/validation-source'
import './App.css'
// Desktop-only chrome (chat over the agent sidecar, engine status, save/open,
@@ -22,7 +24,7 @@ const LiveChrome = lazy(() => import('./components/LiveChrome').then((m) => ({ d
const isLive =
chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'live'
-type Tab = 'findings' | 'corpus' | 'compare' | 'chat'
+type Tab = 'findings' | 'corpus' | 'compare' | 'validation' | 'chat'
/**
* The tabbed sidebar: findings + resolution, corpus, and the grounded-vs-
@@ -53,6 +55,9 @@ function DemoPanels() {
const [comparePending, setComparePending] = useState(false)
const [compareQuestions, setCompareQuestions] = useState<string[]>([])
const [chatQuestions, setChatQuestions] = useState<string[]>([])
+ const [validationFixtures, setValidationFixtures] = useState<ValidationFixtures | null>(null)
+ const [validationError, setValidationError] = useState<string | null>(null)
+ const [validationPending, setValidationPending] = useState(false)
// The scripted question lists for the compare and chat pickers. Neither
// CompareSource nor DemoSource has a "list questions" method by design
@@ -84,6 +89,31 @@ function DemoPanels() {
}
}, [])
+ // The validation panel's two runs are static: there is no question to
+ // ask, so they load once on mount rather than on a user action, the way
+ // graph and findings load in demo-store.ts. In live mode getValidationSource()
+ // throws immediately (see validation-source.ts), and that message is
+ // shown as-is rather than swallowed, the same honesty compare-source.ts's
+ // live branch already applies.
+ useEffect(() => {
+ let cancelled = false
+ setValidationPending(true)
+ getValidationSource()
+ .then((src) => src.runs())
+ .then((fixtures) => {
+ if (!cancelled) setValidationFixtures(fixtures)
+ })
+ .catch((e) => {
+ if (!cancelled) setValidationError(e instanceof Error ? e.message : String(e))
+ })
+ .finally(() => {
+ if (!cancelled) setValidationPending(false)
+ })
+ return () => {
+ cancelled = true
+ }
+ }, [])
+
const handleAsk = useCallback(async (q: string) => {
setComparePending(true)
try {
@@ -106,7 +136,7 @@ function DemoPanels() {
return (
<div className="flex flex-col h-full overflow-hidden">
<div className="flex border-b text-xs shrink-0" style={{ borderColor: 'var(--border)' }}>
- {(['findings', 'corpus', 'compare', 'chat'] as Tab[]).map((t) => (
+ {(['findings', 'corpus', 'compare', 'validation', 'chat'] as Tab[]).map((t) => (
<button
key={t}
onClick={() => setTab(t)}
@@ -149,6 +179,9 @@ function DemoPanels() {
pending={comparePending}
/>
)}
+ {tab === 'validation' && (
+ <ValidationPanel fixtures={validationFixtures} error={validationError} pending={validationPending} />
+ )}
{tab === 'chat' && (
<ScriptedChatPanel chat={chat} pending={chatPending} onAsk={ask} questions={chatQuestions} />
)}
diff --git a/studio/src/components/ValidationPanel.tsx b/studio/src/components/ValidationPanel.tsx
new file mode 100644
index 00000000..92f5baf1
--- /dev/null
+++ b/studio/src/components/ValidationPanel.tsx
@@ -0,0 +1,126 @@
+import type { ValidationFixtures, ValidationRun } from '../lib/validation-source'
+
+export interface ValidationPanelProps {
+ fixtures: ValidationFixtures | null
+ error: string | null
+ pending?: boolean
+}
+
+/**
+ * The moment the rest of the replay cannot show without an engine: the same
+ * unchanged SHACL shapes file, run against the corpus twice. Once as
+ * GSA/dcat-us actually publishes it -- 0 focus nodes, meaning the shapes
+ * selected nothing to check -- and once with the schema-derived binding
+ * applied, where 228 focus nodes are actually reached and the run fails.
+ *
+ * The zero-focus-node run is the whole point of this panel and is rendered
+ * as the loudest thing on it, never as a pass with a footnote: `verdict` is
+ * either 'undetermined' or 'fails', there is no 'passes' case in the type
+ * (validation-source.ts), and this component does not invent one. No
+ * violation count appears anywhere here -- ValidationRun does not carry
+ * one, so there is nothing for this component to render even by accident.
+ */
+export function ValidationPanel({ fixtures, error, pending }: ValidationPanelProps) {
+ if (pending) {
+ return (
+ <p className="p-4 text-sm" style={{ color: 'var(--text-secondary)' }}>
+ Loading the validation runs&hellip;
+ </p>
+ )
+ }
+ if (error) {
+ return (
+ <p className="p-4 text-sm" style={{ color: 'var(--error)' }}>
+ {error}
+ </p>
+ )
+ }
+ if (!fixtures) {
+ return (
+ <p className="p-4 text-sm" style={{ color: 'var(--text-secondary)' }}>
+ No validation runs available.
+ </p>
+ )
+ }
+
+ return (
+ <div className="flex flex-col h-full overflow-y-auto">
+ <div className="p-3 border-b" style={{ borderColor: 'var(--border)' }}>
+ <div className="text-xs font-semibold uppercase tracking-wide" style={{ color: 'var(--text-secondary)' }}>
+ Same shapes file, two corpus states
+ </div>
+ <div className="text-xs mt-1 font-mono break-all" style={{ color: 'var(--text-secondary)' }}>
+ {fixtures.shapesFile}, unchanged in both runs &middot; measured {fixtures.measured}
+ </div>
+ </div>
+ <div className="p-3 flex flex-col gap-3">
+ {fixtures.runs.map((run) => (
+ <ValidationRunCard key={run.id} run={run} />
+ ))}
+ </div>
+ </div>
+ )
+}
+
+function ValidationRunCard({ run }: { run: ValidationRun }) {
+ const vacuous = run.verdict === 'undetermined'
+ return (
+ <div
+ className="rounded p-3"
+ style={{
+ background: 'var(--bg-panel)',
+ border: `2px solid ${vacuous ? 'var(--error)' : 'var(--border)'}`,
+ }}
+ >
+ <div className="text-sm font-semibold" style={{ color: 'var(--text-primary)' }}>
+ {run.label}
+ </div>
+ <p className="text-xs mt-1" style={{ color: 'var(--text-secondary)' }}>
+ {run.corpusDescription}
+ </p>
+
+ {/* Focus nodes is the loudest thing on the card: a large numeral, not
+ a line item next to a green tick. For the vacuous run this is 0,
+ and 0 is rendered exactly as prominently as 228 -- neither number
+ is hidden or shrunk to make the panel look tidier. */}
+ <div className="mt-3 flex items-baseline gap-2">
+ <span
+ className="text-3xl font-bold tabular-nums"
+ style={{ color: vacuous ? 'var(--error)' : 'var(--text-primary)' }}
+ >
+ {run.focusNodes}
+ </span>
+ <span className="text-xs" style={{ color: 'var(--text-secondary)' }}>
+ focus node{run.focusNodes === 1 ? '' : 's'} &middot; {run.matchedClassCount} of{' '}
+ {run.targetClassCount} target classes matched &middot; {run.dataTriples} data triples
+ </span>
+ </div>
+
+ {vacuous ? (
+ <div
+ className="mt-3 p-2 rounded text-xs font-semibold"
+ style={{ background: 'var(--error)', color: 'var(--bg-primary)' }}
+ >
+ NOT A PASS. The validator checked nothing.
+ </div>
+ ) : (
+ <div
+ className="mt-3 p-2 rounded text-xs font-semibold"
+ style={{ background: 'var(--error)', color: 'var(--bg-primary)' }}
+ >
+ FAILS. Conforms = false.
+ </div>
+ )}
+
+ {run.reason && (
+ <p className="mt-2 text-xs" style={{ color: 'var(--text-primary)' }}>
+ {run.reason}
+ </p>
+ )}
+
+ <p className="mt-2 text-xs" style={{ color: 'var(--text-secondary)' }}>
+ conforms (as reported by the run, before this panel&apos;s reading of it): {String(run.conformsRaw)}
+ </p>
+ </div>
+ )
+}
diff --git a/studio/src/lib/__tests__/validation-source.test.ts b/studio/src/lib/__tests__/validation-source.test.ts
new file mode 100644
index 00000000..5390c985
--- /dev/null
+++ b/studio/src/lib/__tests__/validation-source.test.ts
@@ -0,0 +1,113 @@
+import { describe, it, expect } from 'vitest'
+import {
+ createReplayValidationSource,
+ validationFixturesFromBundle,
+ type ValidationFixtures,
+} from '../validation-source'
+
+const fixtures: ValidationFixtures = {
+ shapesFile: 'recovered-shapes.ttl',
+ commit: '7a6e803fb94ee9903e7e7405ec4afcc8da13383f',
+ measured: '2026-08-25, this repository, no network, no model call',
+ runs: [
+ {
+ id: 'as-published',
+ label: 'Corpus as published',
+ corpusDescription: '115 good examples, exactly as published',
+ dataTriples: 73,
+ focusNodes: 0,
+ matchedClassCount: 0,
+ targetClassCount: 34,
+ conformsRaw: true,
+ verdict: 'undetermined',
+ reason: '0 of 34 target classes in the shapes file matched anything in the data.',
+ },
+ {
+ id: 'schema-derived-binding',
+ label: 'Corpus with schema-derived binding applied',
+ corpusDescription: 'the same 115 examples, with the schema-derived binding applied',
+ dataTriples: 1383,
+ focusNodes: 228,
+ matchedClassCount: 24,
+ targetClassCount: 34,
+ conformsRaw: false,
+ verdict: 'fails',
+ reason: null,
+ },
+ ],
+}
+
+describe('createReplayValidationSource', () => {
+ it('returns the committed validation runs', async () => {
+ const src = createReplayValidationSource(fixtures)
+ const out = await src.runs()
+ expect(out.runs).toHaveLength(2)
+ expect(out.shapesFile).toBe('recovered-shapes.ttl')
+ })
+
+ // This is the regression the whole panel exists to guard against: a SHACL
+ // run with zero focus nodes checked nothing, and must never be
+ // indistinguishable from -- or worse, presented as -- a genuine pass.
+ // ValidationRun's `verdict` field has exactly two values, 'undetermined'
+ // and 'fails' (validation-source.ts); there is no 'passes' member of the
+ // type at all, so a run cannot be typed as a pass even by mistake. This
+ // test pins the runtime data alongside that static guarantee: the run
+ // with focusNodes === 0 must carry verdict 'undetermined', never 'fails'
+ // read as a clean pass and never a bare `conformsRaw: true` left to speak
+ // for itself.
+ it('marks the zero-focus-node run as undetermined, never as a pass', async () => {
+ const src = createReplayValidationSource(fixtures)
+ const out = await src.runs()
+ const asPublished = out.runs.find((r) => r.focusNodes === 0)
+ expect(asPublished).toBeDefined()
+ expect(asPublished!.verdict).toBe('undetermined')
+ expect(asPublished!.verdict).not.toBe('fails')
+ // conformsRaw is allowed to be true (that is the honest raw reading),
+ // but nothing in this type or fixture calls that reading a pass.
+ expect(asPublished!.conformsRaw).toBe(true)
+ expect(asPublished!.reason).toMatch(/nothing|matched/i)
+ })
+
+ it('marks the real failure as fails, distinct from the vacuous run', async () => {
+ const src = createReplayValidationSource(fixtures)
+ const out = await src.runs()
+ const bound = out.runs.find((r) => r.focusNodes > 0)
+ expect(bound).toBeDefined()
+ expect(bound!.verdict).toBe('fails')
+ expect(bound!.conformsRaw).toBe(false)
+ expect(bound!.focusNodes).toBe(228)
+ })
+
+ it('never exposes a violation count field on any run', async () => {
+ const src = createReplayValidationSource(fixtures)
+ const out = await src.runs()
+ for (const run of out.runs) {
+ expect(Object.keys(run)).not.toContain('violations')
+ expect(Object.keys(run)).not.toContain('violationCount')
+ expect(Object.keys(run)).not.toContain('violation_count')
+ }
+ })
+})
+
+describe('validationFixturesFromBundle', () => {
+ // bundle.json carries "validation" as a sixth top-level key alongside the
+ // four ReplayFixtures declares plus "compare". This is the one place that
+ // reaches into it, mirroring compareFixturesFromBundle in
+ // compare-source.ts.
+ it('extracts the validation key from a bundle carrying the other keys too', () => {
+ const bundle = {
+ corpus: [],
+ graph: { classes: [], properties: [], edges: [] },
+ findings: [],
+ chat: {},
+ compare: {},
+ validation: fixtures,
+ }
+ expect(validationFixturesFromBundle(bundle)).toBe(fixtures)
+ })
+
+ it('throws rather than silently returning an empty validation set when the key is missing', () => {
+ const bundle = { corpus: [], graph: { classes: [], properties: [], edges: [] }, findings: [], chat: {} }
+ expect(() => validationFixturesFromBundle(bundle)).toThrow(/validation/)
+ })
+})
diff --git a/studio/src/lib/validation-source.ts b/studio/src/lib/validation-source.ts
new file mode 100644
index 00000000..a20f7f8b
--- /dev/null
+++ b/studio/src/lib/validation-source.ts
@@ -0,0 +1,109 @@
+import { chooseSourceKind } from './source-factory'
+
+// Deliberately a separate interface from DemoSource, the same choice
+// compare-source.ts made and explains in its own header comment: this is an
+// optional demo surface layered on top of the four core methods, not a
+// fifth one bolted onto DemoSource itself.
+//
+// This is the moment the PIVOT in .superpowers/sdd/progress.md could not put
+// in the web replay, because it needed the live engine: a SHACL validation
+// that returns conforms=true over zero focus nodes, meaning nothing was
+// actually checked. src/shacl.rs (around lines 552-567) already refuses to
+// call that a pass -- it reports `conforms: null` with a warning that the
+// shapes selected nothing -- and `verdict` here is the client-side mirror of
+// that refusal: 'undetermined' for a run with zero focus nodes, never
+// 'passes'. There is no 'passes' verdict in this type on purpose. A run
+// either failed outright ('fails') or checked nothing ('undetermined');
+// nothing here can render as a green tick.
+//
+// No violation count anywhere in this shape, deliberately. Three legitimate
+// measurement methods over identical inputs give 178, 272 and 147
+// violations, and a fourth figure (287) is already public
+// (case-studies/dcat-us-binding/README.md); demo/README.md's own stated
+// position is that none of the four is defensible as "the" figure. Focus
+// node counts are stable across all methods and are the only counts this
+// type carries.
+export interface ValidationRun {
+ id: string
+ label: string
+ corpusDescription: string
+ dataTriples: number
+ focusNodes: number
+ matchedClassCount: number
+ targetClassCount: number
+ // The raw boolean the SHACL run itself returned (true for the
+ // as-published run, since `violations.is_empty()` is vacuously true over
+ // zero focus nodes). Kept alongside `verdict` rather than replaced by it,
+ // so a caller can see exactly what a naive reading of `conforms` would
+ // have said, next to why that reading is not the verdict.
+ conformsRaw: boolean
+ verdict: 'undetermined' | 'fails'
+ reason: string | null
+}
+
+export interface ValidationFixtures {
+ shapesFile: string
+ commit: string
+ measured: string
+ runs: ValidationRun[]
+}
+
+export interface ValidationSource {
+ runs(): Promise<ValidationFixtures>
+}
+
+// The single place a bundle's "validation" key becomes ValidationFixtures,
+// mirroring compareFixturesFromBundle in compare-source.ts. bundle.json
+// carries this as a sixth top-level key alongside corpus, graph, findings,
+// chat and compare; ReplayFixtures does not name it, the same tolerance
+// compare-source.ts relies on for "compare".
+export function validationFixturesFromBundle(bundle: Record<string, unknown>): ValidationFixtures {
+ const raw = bundle.validation
+ if (!raw || typeof raw !== 'object') {
+ throw new Error(
+ 'The precomputed bundle has no "validation" key (or it is not an object); the ' +
+ 'SHACL validation panel cannot be shown.',
+ )
+ }
+ return raw as ValidationFixtures
+}
+
+export function createReplayValidationSource(fixtures: ValidationFixtures): ValidationSource {
+ return {
+ async runs(): Promise<ValidationFixtures> {
+ return fixtures
+ },
+ }
+}
+
+// The single construction site for ValidationSource, mirroring
+// getCompareSource() and getDemoSource(). In replay mode it reads the same
+// bundle.json DemoSource does and pulls the "validation" key out of it via
+// validationFixturesFromBundle. In live mode there is no honest
+// implementation: showing this panel means running the SAME unchanged
+// shapes file against TWO different corpus states (as published, and with
+// the schema-derived binding applied), which is a build-time measurement
+// pipeline (demo/dcat_conformance.py), not a single onto_shacl call a live
+// session can make on demand against whatever happens to be loaded.
+export async function getValidationSource(): Promise<ValidationSource> {
+ if (chooseSourceKind(import.meta.env as unknown as Record<string, string | undefined>) === 'replay') {
+ const response = await fetch('./precomputed/bundle.json')
+ if (!response.ok) {
+ throw new Error(`Could not load the precomputed demonstration: ${response.status}`)
+ }
+ const bundle = (await response.json()) as Record<string, unknown>
+ return createReplayValidationSource(validationFixturesFromBundle(bundle))
+ }
+ return {
+ async runs(): Promise<ValidationFixtures> {
+ throw new Error(
+ 'No live validation comparison is implemented. Showing the as-published run next to ' +
+ 'the schema-derived-binding run means validating the same unchanged shapes file ' +
+ 'against two different corpus states, a measurement pipeline (see ' +
+ 'demo/dcat_conformance.py) rather than a single onto_shacl call against whatever ' +
+ 'is currently loaded. Run the offline replay (npm run build:web) to see the ' +
+ 'precomputed comparison.',
+ )
+ },
+ }
+}