Skip to content

Open-source Revit MCP server: let Claude and other AI assistants read and edit BIM models, with human approval on every change.

test(e2e): add canonical model golden harness (P17.3)

A. Sam Mohammad5c9342de6648
diff --git a/fixtures/canonical-model/README.md b/fixtures/canonical-model/README.md
index 2b3c2b5..27d5483 100644
--- a/fixtures/canonical-model/README.md
+++ b/fixtures/canonical-model/README.md
@@ -13,3 +13,10 @@ The script discovers the active Revit bridge from `%LOCALAPPDATA%\AECModelBridge
If the bridge-created document is not activated by Revit, open a clean project template yourself and rerun with `--use-active-document`.
Seeded QA/QC expectations live in `seeded-defects.json`. It is count-level on purpose: Revit element IDs and UniqueIds are generated per model build.
+
+Local/nightly e2e goldens live in `goldens/`. To run them, open the generated canonical model in Revit, then run:
+
+```powershell
+$env:AEC_MODEL_BRIDGE_E2E = "1"
+python -m pytest packages\mcp-server-revit\tests\e2e -m e2e
+```
diff --git a/fixtures/canonical-model/goldens/qaqc-findings-summary.json b/fixtures/canonical-model/goldens/qaqc-findings-summary.json
new file mode 100644
index 0000000..885c52e
--- /dev/null
+++ b/fixtures/canonical-model/goldens/qaqc-findings-summary.json
@@ -0,0 +1,9 @@
+{
+ "schema": "amb.fixture.qaqc_findings_summary/1",
+ "fixture": "canonical_test_model",
+ "rule_pack": "core",
+ "seeded_rule_counts": {
+ "door_missing_mark": 12,
+ "room_missing_number": 3
+ }
+}
diff --git a/fixtures/canonical-model/goldens/snapshot-summary.json b/fixtures/canonical-model/goldens/snapshot-summary.json
new file mode 100644
index 0000000..51151a3
--- /dev/null
+++ b/fixtures/canonical-model/goldens/snapshot-summary.json
@@ -0,0 +1,13 @@
+{
+ "schema": "amb.fixture.snapshot_summary/1",
+ "fixture": "canonical_test_model",
+ "required_category_counts": {
+ "OST_Levels": 5,
+ "OST_Walls": 200,
+ "OST_Doors": 60,
+ "OST_Windows": 40,
+ "OST_Rooms": 25,
+ "OST_Sheets": 10
+ },
+ "minimum_element_count": 340
+}
diff --git a/fixtures/canonical-model/manifest.json b/fixtures/canonical-model/manifest.json
index 4772349..72b843c 100644
--- a/fixtures/canonical-model/manifest.json
+++ b/fixtures/canonical-model/manifest.json
@@ -4,6 +4,10 @@
"generator": "scripts/revit/generate_canonical_test_model.py",
"output_path": "fixtures/canonical-model/generated/canonical_test_model.rvt",
"seeded_defects": "fixtures/canonical-model/seeded-defects.json",
+ "goldens": {
+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",
+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json"
+ },
"expected_counts": {
"levels": 5,
"walls": 200,
diff --git a/packages/mcp-server-revit/pyproject.toml b/packages/mcp-server-revit/pyproject.toml
index ba787a5..3402163 100644
--- a/packages/mcp-server-revit/pyproject.toml
+++ b/packages/mcp-server-revit/pyproject.toml
@@ -43,5 +43,6 @@ build-backend = "setuptools.build_meta"
[tool.pytest.ini_options]
markers = [
+ "e2e: marks Revit bridge end-to-end tests (opt in with AEC_MODEL_BRIDGE_E2E=1)",
"perf: marks tests as performance benchmarks (deselect with '-m \"not perf\"')"
]
diff --git a/packages/mcp-server-revit/tests/e2e/test_e2e.py b/packages/mcp-server-revit/tests/e2e/test_e2e.py
new file mode 100644
index 0000000..6fcc7b8
--- /dev/null
+++ b/packages/mcp-server-revit/tests/e2e/test_e2e.py
@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import os
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from revit_mcp_server.bridge.client import BridgeClient as HubBridgeClient
+from revit_mcp_server.config import BridgeMode, Config
+from revit_mcp_server.errors import BridgeError
+from revit_mcp_server.providers import ApprovalProvider, ProviderRegistry, RevitProvider
+from revit_mcp_server.security.workspace import WorkspaceMonitor
+
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+TESTS_DIR = Path(__file__).resolve().parents[1]
+GENERATOR_PATH = REPO_ROOT / "scripts" / "revit" / "generate_canonical_test_model.py"
+QAQC_MODULE_PATH = (
+ REPO_ROOT
+ / "packages"
+ / "mcp-server-revit"
+ / "src"
+ / "revit_mcp_server"
+ / "modules"
+ / "qaqc_checker"
+ / "module.py"
+)
+SNAPSHOT_GOLDEN = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "snapshot-summary.json"
+QAQC_GOLDEN = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "qaqc-findings-summary.json"
+
+pytestmark = pytest.mark.e2e
+
+
+def load_json(path: Path) -> dict[str, Any]:
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+def load_generator():
+ spec = importlib.util.spec_from_file_location("generate_canonical_test_model", GENERATOR_PATH)
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ spec.loader.exec_module(module)
+ return module
+
+
+def load_mcp_server_class():
+ spec = importlib.util.spec_from_file_location("test_helpers", TESTS_DIR / "helpers.py")
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ spec.loader.exec_module(module)
+ return module.MCPServer
+
+
+def load_qaqc_module():
+ spec = importlib.util.spec_from_file_location("qaqc_checker_module", QAQC_MODULE_PATH)
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ spec.loader.exec_module(module)
+ return module.QaqcCheckerModule()
+
+
[email protected](scope="session")
+def bridge_switch():
+ if os.getenv("AEC_MODEL_BRIDGE_E2E") != "1":
+ pytest.skip("Set AEC_MODEL_BRIDGE_E2E=1 and open the canonical model in Revit.")
+
+ generator = load_generator()
+ url = os.getenv("AEC_MODEL_BRIDGE_REVIT_URL") or os.getenv("MCP_REVIT_BRIDGE_URL")
+ token = os.getenv("AEC_MODEL_BRIDGE_REVIT_TOKEN") or os.getenv("MCP_REVIT_BRIDGE_TOKEN")
+ switch = generator.discover_switch(url, token)
+ try:
+ generator.BridgeClient(switch, timeout=5).get("/health")
+ except generator.BridgeCallError as exc:
+ pytest.skip(f"Revit bridge is not reachable: {exc}")
+ return switch
+
+
+def test_mutating_tools_require_approved_plan(tmp_path):
+ workspace = WorkspaceMonitor([tmp_path])
+ registry = ProviderRegistry()
+ approval = ApprovalProvider(workspace=workspace, registry=registry, approval_mode="required")
+ registry.register(approval)
+ registry.register(RevitProvider(workspace=workspace, mode=BridgeMode.mock))
+
+ tool_def = registry.lookup_tool("revit_create_wall")
+ assert tool_def is not None
+ assert tool_def.is_mutating
+
+ with pytest.raises(BridgeError, match="requires a valid 'plan_id'"):
+ approval.gate.check_tool_execution(tool_def.name, {})
+
+ plan = approval.gate.create_plan(
+ [{"tool": tool_def.name, "arguments": {"start_x": 0, "end_x": 10}}],
+ [{"status": "before-state-captured"}],
+ )
+ with pytest.raises(BridgeError, match="not 'approved'"):
+ approval.gate.check_tool_execution(tool_def.name, {"plan_id": plan["plan_id"]})
+
+ approval.gate.update_plan_state(plan["plan_id"], "approved")
+ approval.gate.check_tool_execution(tool_def.name, {"plan_id": plan["plan_id"]})
+
+
+def test_hub_reaches_revit_addin(bridge_switch, tmp_path):
+ server_cls = load_mcp_server_class()
+ cfg = Config(
+ workspace_dir=tmp_path,
+ allowed_directories=[tmp_path],
+ audit_log=tmp_path / "audit.log",
+ bridge_url=bridge_switch.endpoint,
+ mode=BridgeMode.bridge,
+ )
+ server = server_cls(
+ config=cfg,
+ bridge_factory=lambda url, token=None: HubBridgeClient(url, token=token or bridge_switch.token),
+ )
+ try:
+ response = server.handle_tool("revit_health", {"request_id": "e2e-health"})
+ finally:
+ server.shutdown()
+
+ assert response
+ assert response.get("status", "healthy") not in {"error", "unhealthy"}
+
+
+def test_canonical_snapshot_and_qaqc_goldens(bridge_switch, tmp_path):
+ workspace = WorkspaceMonitor([tmp_path])
+ provider = RevitProvider(
+ workspace=workspace,
+ mode=BridgeMode.bridge,
+ bridge_url=bridge_switch.endpoint,
+ bridge_factory=lambda url, token=None: HubBridgeClient(url, token=token or bridge_switch.token),
+ )
+
+ result = asyncio.run(provider.execute_tool("revit_extract_snapshot", {"dirty_only": False}))
+ snapshot_path = Path(result.get("path", ""))
+ if not snapshot_path.exists():
+ pytest.skip(f"Bridge returned inaccessible snapshot path: {snapshot_path}")
+
+ snapshot = load_json(snapshot_path)
+ snapshot_id = snapshot["snapshot_id"]
+ snapshots_dir = tmp_path / "snapshots"
+ snapshots_dir.mkdir(exist_ok=True)
+ (snapshots_dir / f"{snapshot_id}.json").write_text(json.dumps(snapshot), encoding="utf-8")
+
+ snapshot_golden = load_json(SNAPSHOT_GOLDEN)
+ counts = Counter(element.get("category") for element in snapshot.get("elements", []))
+ for category, expected in snapshot_golden["required_category_counts"].items():
+ assert counts[category] == expected
+ assert len(snapshot.get("elements", [])) >= snapshot_golden["minimum_element_count"]
+
+ qaqc = load_qaqc_module().run_check(
+ snapshot_id=snapshot_id,
+ rule_pack=load_json(QAQC_GOLDEN)["rule_pack"],
+ workspace=workspace,
+ )
+ actual = Counter(finding["rule_id"] for finding in qaqc["findings"])
+ expected = load_json(QAQC_GOLDEN)["seeded_rule_counts"]
+ assert {rule_id: actual[rule_id] for rule_id in expected} == expected
diff --git a/packages/mcp-server-revit/tests/test_canonical_model_generator.py b/packages/mcp-server-revit/tests/test_canonical_model_generator.py
index 2c5c789..cbee08f 100644
--- a/packages/mcp-server-revit/tests/test_canonical_model_generator.py
+++ b/packages/mcp-server-revit/tests/test_canonical_model_generator.py
@@ -12,6 +12,8 @@ REPO_ROOT = Path(__file__).resolve().parents[3]
SCRIPT_PATH = REPO_ROOT / "scripts" / "revit" / "generate_canonical_test_model.py"
MANIFEST_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "manifest.json"
SEEDED_DEFECTS_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "seeded-defects.json"
+SNAPSHOT_GOLDEN_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "snapshot-summary.json"
+QAQC_GOLDEN_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "qaqc-findings-summary.json"
RULES_PATH = (
REPO_ROOT
/ "packages"
@@ -87,6 +89,31 @@ def test_seeded_defect_register_tracks_known_manifest_gaps():
assert {entry["status"] for entry in register["known_gaps"]} == {"bridge_gap"}
+def test_canonical_goldens_match_manifest_and_seed_register():
+ manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
+ register = json.loads(SEEDED_DEFECTS_PATH.read_text(encoding="utf-8"))
+ snapshot_golden = json.loads(SNAPSHOT_GOLDEN_PATH.read_text(encoding="utf-8"))
+ qaqc_golden = json.loads(QAQC_GOLDEN_PATH.read_text(encoding="utf-8"))
+
+ assert manifest["goldens"] == {
+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",
+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json",
+ }
+ assert snapshot_golden["fixture"] == manifest["name"]
+ assert qaqc_golden["fixture"] == manifest["name"]
+ assert snapshot_golden["required_category_counts"] == {
+ "OST_Levels": manifest["expected_counts"]["levels"],
+ "OST_Walls": manifest["expected_counts"]["walls"],
+ "OST_Doors": manifest["expected_counts"]["doors"],
+ "OST_Windows": manifest["expected_counts"]["windows"],
+ "OST_Rooms": manifest["expected_counts"]["rooms"],
+ "OST_Sheets": manifest["expected_counts"]["sheets"],
+ }
+ assert qaqc_golden["seeded_rule_counts"] == {
+ entry["rule_id"]: entry["expected_count"] for entry in register["expected_findings"]
+ }
+
+
def test_registry_discovery_uses_newest_switch(tmp_path, monkeypatch):
generator = load_generator()
registry = tmp_path / "AECModelBridge" / "registry"
diff --git a/scripts/revit/generate_canonical_test_model.py b/scripts/revit/generate_canonical_test_model.py
index 3e62a19..fefb2e7 100644
--- a/scripts/revit/generate_canonical_test_model.py
+++ b/scripts/revit/generate_canonical_test_model.py
@@ -214,6 +214,10 @@ def manifest(output_path: Path = DEFAULT_OUTPUT) -> dict[str, Any]:
"generator": "scripts/revit/generate_canonical_test_model.py",
"output_path": output_label(output_path),
"seeded_defects": "fixtures/canonical-model/seeded-defects.json",
+ "goldens": {
+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",
+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json",
+ },
"expected_counts": EXPECTED_COUNTS,
"supported_seeded_rules": SUPPORTED_SEEDED_RULES,
"known_gaps": KNOWN_GAPS,