Open-source Revit MCP server: let Claude and other AI assistants read and edit BIM models, with human approval on every change.
Python aecai-agentsautodeskautomationbimclaudeifcmcpmcp-servermodel-context-protocolnavisworksrevitrevit-addinrevit-apirhino
test(e2e): add canonical model golden harness (P17.3)
A. Sam Mohammad5c9342de6648
diff --git a/fixtures/canonical-model/README.md b/fixtures/canonical-model/README.mdindex 2b3c2b5..27d5483 100644--- a/fixtures/canonical-model/README.md+++ b/fixtures/canonical-model/README.md@@ -13,3 +13,10 @@ The script discovers the active Revit bridge from `%LOCALAPPDATA%\AECModelBridgeIf the bridge-created document is not activated by Revit, open a clean project template yourself and rerun with `--use-active-document`.Seeded QA/QC expectations live in `seeded-defects.json`. It is count-level on purpose: Revit element IDs and UniqueIds are generated per model build.++Local/nightly e2e goldens live in `goldens/`. To run them, open the generated canonical model in Revit, then run:++```powershell+$env:AEC_MODEL_BRIDGE_E2E = "1"+python -m pytest packages\mcp-server-revit\tests\e2e -m e2e+```diff --git a/fixtures/canonical-model/goldens/qaqc-findings-summary.json b/fixtures/canonical-model/goldens/qaqc-findings-summary.jsonnew file mode 100644index 0000000..885c52e--- /dev/null+++ b/fixtures/canonical-model/goldens/qaqc-findings-summary.json@@ -0,0 +1,9 @@+{+ "schema": "amb.fixture.qaqc_findings_summary/1",+ "fixture": "canonical_test_model",+ "rule_pack": "core",+ "seeded_rule_counts": {+ "door_missing_mark": 12,+ "room_missing_number": 3+ }+}diff --git a/fixtures/canonical-model/goldens/snapshot-summary.json b/fixtures/canonical-model/goldens/snapshot-summary.jsonnew file mode 100644index 0000000..51151a3--- /dev/null+++ b/fixtures/canonical-model/goldens/snapshot-summary.json@@ -0,0 +1,13 @@+{+ "schema": "amb.fixture.snapshot_summary/1",+ "fixture": "canonical_test_model",+ "required_category_counts": {+ "OST_Levels": 5,+ "OST_Walls": 200,+ "OST_Doors": 60,+ "OST_Windows": 40,+ "OST_Rooms": 25,+ "OST_Sheets": 10+ },+ "minimum_element_count": 340+}diff --git a/fixtures/canonical-model/manifest.json b/fixtures/canonical-model/manifest.jsonindex 4772349..72b843c 100644--- a/fixtures/canonical-model/manifest.json+++ b/fixtures/canonical-model/manifest.json@@ -4,6 +4,10 @@"generator": "scripts/revit/generate_canonical_test_model.py","output_path": "fixtures/canonical-model/generated/canonical_test_model.rvt","seeded_defects": "fixtures/canonical-model/seeded-defects.json",+ "goldens": {+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json"+ },"expected_counts": {"levels": 5,"walls": 200,diff --git a/packages/mcp-server-revit/pyproject.toml b/packages/mcp-server-revit/pyproject.tomlindex ba787a5..3402163 100644--- a/packages/mcp-server-revit/pyproject.toml+++ b/packages/mcp-server-revit/pyproject.toml@@ -43,5 +43,6 @@ build-backend = "setuptools.build_meta"[tool.pytest.ini_options]markers = [+ "e2e: marks Revit bridge end-to-end tests (opt in with AEC_MODEL_BRIDGE_E2E=1)","perf: marks tests as performance benchmarks (deselect with '-m \"not perf\"')"]diff --git a/packages/mcp-server-revit/tests/e2e/test_e2e.py b/packages/mcp-server-revit/tests/e2e/test_e2e.pynew file mode 100644index 0000000..6fcc7b8--- /dev/null+++ b/packages/mcp-server-revit/tests/e2e/test_e2e.py@@ -0,0 +1,163 @@+from __future__ import annotations++import asyncio+import importlib.util+import json+import os+from collections import Counter+from pathlib import Path+from typing import Any++import pytest++from revit_mcp_server.bridge.client import BridgeClient as HubBridgeClient+from revit_mcp_server.config import BridgeMode, Config+from revit_mcp_server.errors import BridgeError+from revit_mcp_server.providers import ApprovalProvider, ProviderRegistry, RevitProvider+from revit_mcp_server.security.workspace import WorkspaceMonitor+++REPO_ROOT = Path(__file__).resolve().parents[4]+TESTS_DIR = Path(__file__).resolve().parents[1]+GENERATOR_PATH = REPO_ROOT / "scripts" / "revit" / "generate_canonical_test_model.py"+QAQC_MODULE_PATH = (+ REPO_ROOT+ / "packages"+ / "mcp-server-revit"+ / "src"+ / "revit_mcp_server"+ / "modules"+ / "qaqc_checker"+ / "module.py"+)+SNAPSHOT_GOLDEN = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "snapshot-summary.json"+QAQC_GOLDEN = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "qaqc-findings-summary.json"++pytestmark = pytest.mark.e2e+++def load_json(path: Path) -> dict[str, Any]:+ return json.loads(path.read_text(encoding="utf-8"))+++def load_generator():+ spec = importlib.util.spec_from_file_location("generate_canonical_test_model", GENERATOR_PATH)+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module+++def load_mcp_server_class():+ spec = importlib.util.spec_from_file_location("test_helpers", TESTS_DIR / "helpers.py")+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module.MCPServer+++def load_qaqc_module():+ spec = importlib.util.spec_from_file_location("qaqc_checker_module", QAQC_MODULE_PATH)+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module.QaqcCheckerModule()++[email protected](scope="session")+def bridge_switch():+ if os.getenv("AEC_MODEL_BRIDGE_E2E") != "1":+ pytest.skip("Set AEC_MODEL_BRIDGE_E2E=1 and open the canonical model in Revit.")++ generator = load_generator()+ url = os.getenv("AEC_MODEL_BRIDGE_REVIT_URL") or os.getenv("MCP_REVIT_BRIDGE_URL")+ token = os.getenv("AEC_MODEL_BRIDGE_REVIT_TOKEN") or os.getenv("MCP_REVIT_BRIDGE_TOKEN")+ switch = generator.discover_switch(url, token)+ try:+ generator.BridgeClient(switch, timeout=5).get("/health")+ except generator.BridgeCallError as exc:+ pytest.skip(f"Revit bridge is not reachable: {exc}")+ return switch+++def test_mutating_tools_require_approved_plan(tmp_path):+ workspace = WorkspaceMonitor([tmp_path])+ registry = ProviderRegistry()+ approval = ApprovalProvider(workspace=workspace, registry=registry, approval_mode="required")+ registry.register(approval)+ registry.register(RevitProvider(workspace=workspace, mode=BridgeMode.mock))++ tool_def = registry.lookup_tool("revit_create_wall")+ assert tool_def is not None+ assert tool_def.is_mutating++ with pytest.raises(BridgeError, match="requires a valid 'plan_id'"):+ approval.gate.check_tool_execution(tool_def.name, {})++ plan = approval.gate.create_plan(+ [{"tool": tool_def.name, "arguments": {"start_x": 0, "end_x": 10}}],+ [{"status": "before-state-captured"}],+ )+ with pytest.raises(BridgeError, match="not 'approved'"):+ approval.gate.check_tool_execution(tool_def.name, {"plan_id": plan["plan_id"]})++ approval.gate.update_plan_state(plan["plan_id"], "approved")+ approval.gate.check_tool_execution(tool_def.name, {"plan_id": plan["plan_id"]})+++def test_hub_reaches_revit_addin(bridge_switch, tmp_path):+ server_cls = load_mcp_server_class()+ cfg = Config(+ workspace_dir=tmp_path,+ allowed_directories=[tmp_path],+ audit_log=tmp_path / "audit.log",+ bridge_url=bridge_switch.endpoint,+ mode=BridgeMode.bridge,+ )+ server = server_cls(+ config=cfg,+ bridge_factory=lambda url, token=None: HubBridgeClient(url, token=token or bridge_switch.token),+ )+ try:+ response = server.handle_tool("revit_health", {"request_id": "e2e-health"})+ finally:+ server.shutdown()++ assert response+ assert response.get("status", "healthy") not in {"error", "unhealthy"}+++def test_canonical_snapshot_and_qaqc_goldens(bridge_switch, tmp_path):+ workspace = WorkspaceMonitor([tmp_path])+ provider = RevitProvider(+ workspace=workspace,+ mode=BridgeMode.bridge,+ bridge_url=bridge_switch.endpoint,+ bridge_factory=lambda url, token=None: HubBridgeClient(url, token=token or bridge_switch.token),+ )++ result = asyncio.run(provider.execute_tool("revit_extract_snapshot", {"dirty_only": False}))+ snapshot_path = Path(result.get("path", ""))+ if not snapshot_path.exists():+ pytest.skip(f"Bridge returned inaccessible snapshot path: {snapshot_path}")++ snapshot = load_json(snapshot_path)+ snapshot_id = snapshot["snapshot_id"]+ snapshots_dir = tmp_path / "snapshots"+ snapshots_dir.mkdir(exist_ok=True)+ (snapshots_dir / f"{snapshot_id}.json").write_text(json.dumps(snapshot), encoding="utf-8")++ snapshot_golden = load_json(SNAPSHOT_GOLDEN)+ counts = Counter(element.get("category") for element in snapshot.get("elements", []))+ for category, expected in snapshot_golden["required_category_counts"].items():+ assert counts[category] == expected+ assert len(snapshot.get("elements", [])) >= snapshot_golden["minimum_element_count"]++ qaqc = load_qaqc_module().run_check(+ snapshot_id=snapshot_id,+ rule_pack=load_json(QAQC_GOLDEN)["rule_pack"],+ workspace=workspace,+ )+ actual = Counter(finding["rule_id"] for finding in qaqc["findings"])+ expected = load_json(QAQC_GOLDEN)["seeded_rule_counts"]+ assert {rule_id: actual[rule_id] for rule_id in expected} == expecteddiff --git a/packages/mcp-server-revit/tests/test_canonical_model_generator.py b/packages/mcp-server-revit/tests/test_canonical_model_generator.pyindex 2c5c789..cbee08f 100644--- a/packages/mcp-server-revit/tests/test_canonical_model_generator.py+++ b/packages/mcp-server-revit/tests/test_canonical_model_generator.py@@ -12,6 +12,8 @@ REPO_ROOT = Path(__file__).resolve().parents[3]SCRIPT_PATH = REPO_ROOT / "scripts" / "revit" / "generate_canonical_test_model.py"MANIFEST_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "manifest.json"SEEDED_DEFECTS_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "seeded-defects.json"+SNAPSHOT_GOLDEN_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "snapshot-summary.json"+QAQC_GOLDEN_PATH = REPO_ROOT / "fixtures" / "canonical-model" / "goldens" / "qaqc-findings-summary.json"RULES_PATH = (REPO_ROOT/ "packages"@@ -87,6 +89,31 @@ def test_seeded_defect_register_tracks_known_manifest_gaps():assert {entry["status"] for entry in register["known_gaps"]} == {"bridge_gap"}+def test_canonical_goldens_match_manifest_and_seed_register():+ manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))+ register = json.loads(SEEDED_DEFECTS_PATH.read_text(encoding="utf-8"))+ snapshot_golden = json.loads(SNAPSHOT_GOLDEN_PATH.read_text(encoding="utf-8"))+ qaqc_golden = json.loads(QAQC_GOLDEN_PATH.read_text(encoding="utf-8"))++ assert manifest["goldens"] == {+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json",+ }+ assert snapshot_golden["fixture"] == manifest["name"]+ assert qaqc_golden["fixture"] == manifest["name"]+ assert snapshot_golden["required_category_counts"] == {+ "OST_Levels": manifest["expected_counts"]["levels"],+ "OST_Walls": manifest["expected_counts"]["walls"],+ "OST_Doors": manifest["expected_counts"]["doors"],+ "OST_Windows": manifest["expected_counts"]["windows"],+ "OST_Rooms": manifest["expected_counts"]["rooms"],+ "OST_Sheets": manifest["expected_counts"]["sheets"],+ }+ assert qaqc_golden["seeded_rule_counts"] == {+ entry["rule_id"]: entry["expected_count"] for entry in register["expected_findings"]+ }++def test_registry_discovery_uses_newest_switch(tmp_path, monkeypatch):generator = load_generator()registry = tmp_path / "AECModelBridge" / "registry"diff --git a/scripts/revit/generate_canonical_test_model.py b/scripts/revit/generate_canonical_test_model.pyindex 3e62a19..fefb2e7 100644--- a/scripts/revit/generate_canonical_test_model.py+++ b/scripts/revit/generate_canonical_test_model.py@@ -214,6 +214,10 @@ def manifest(output_path: Path = DEFAULT_OUTPUT) -> dict[str, Any]:"generator": "scripts/revit/generate_canonical_test_model.py","output_path": output_label(output_path),"seeded_defects": "fixtures/canonical-model/seeded-defects.json",+ "goldens": {+ "snapshot_summary": "fixtures/canonical-model/goldens/snapshot-summary.json",+ "qaqc_findings_summary": "fixtures/canonical-model/goldens/qaqc-findings-summary.json",+ },"expected_counts": EXPECTED_COUNTS,"supported_seeded_rules": SUPPORTED_SEEDED_RULES,"known_gaps": KNOWN_GAPS,