Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Capability evidence store (``model-hub-capability-model.md`` §3; ``vibey-hosted-tools.md`` §3.2).
Evidence is passive: a refusal is routing state with no time limit, a success is
display history. These tests pin how each is kept, read and cleared at the
store, the owning boundary.
"""
from __future__ import annotations
import json
import logging
from dataclasses import replace
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
from jsonschema import Draft7Validator
from referencing import Registry, Resource
from config.v2_config import ModelHubModelConfig, ModelHubSourceConfig, ModelHubSourceStateConfig
from core.handlers.model_hub.capability_evidence import (
CapabilityEvidenceStore,
EvidenceFeature,
EvidenceRecord,
identity_digest,
wire_stamp,
)
CONTRACTS = Path(__file__).parents[1] / "docs/plans/model-hub-contracts"
NOW = datetime(2026, 10, 15, 12, 0, tzinfo=timezone.utc)
ENGINE = "model-hub-engine-v8.0.23-3"
AVIBE = "3.2.1"
def _source(source_id: str = "src_xai00001", **changes) -> ModelHubSourceConfig:
source = ModelHubSourceConfig(
id=source_id,
kind="api_key",
vendor="xai",
display_name="xAI 密钥",
protocol="openai_responses",
supply_channel="hub",
billing="metered",
state=ModelHubSourceStateConfig(status="standby"),
models=[ModelHubModelConfig(id="grok-4.6", provenance="discovered", reasoning_efforts=["low", "high"])],
credential_ref=f"cred_{source_id}",
)
return replace(source, **changes)
def _wire(source: ModelHubSourceConfig, *, engine: str = ENGINE, avibe: str = AVIBE) -> str:
return wire_stamp(engine=engine, avibe=avibe, source=source, model_id="grok-4.6")
def WIRE_FOR(source: ModelHubSourceConfig, model_id: str) -> str:
"""The wire this process renders now, as the service's ``HostedEvidence.wire_for`` states it."""
return wire_stamp(engine=ENGINE, avibe=AVIBE, source=source, model_id=model_id)
def _refusal(source: ModelHubSourceConfig, feature: EvidenceFeature, **changes) -> EvidenceRecord:
fields = dict(
source_id=source.id,
model_id="grok-4.6",
feature=feature,
status="unsupported",
observed_at=NOW,
wire=_wire(source),
source_identity=identity_digest(source),
detail="400 invalid_request_error",
)
fields.update(changes)
return EvidenceRecord(**fields)
def _success(source: ModelHubSourceConfig, feature: EvidenceFeature, **changes) -> EvidenceRecord:
fields = dict(
source_id=source.id,
model_id="grok-4.6",
feature=feature,
status="verified",
observed_at=NOW,
wire=_wire(source),
source_identity=identity_digest(source),
mode="native",
)
fields.update(changes)
return EvidenceRecord(**fields)
CALL = EvidenceFeature.hosted("x_search", "call")
REPLAY = EvidenceFeature.hosted("web_search", "replay")
ULTRA = EvidenceFeature.effort("ultra")
def _store(tmp_path) -> CapabilityEvidenceStore:
return CapabilityEvidenceStore(tmp_path / "model_hub_capability_evidence.json")
@pytest.mark.parametrize(
"case,current,applies",
[
("same-wire-a-year-later", lambda source: (source, _wire(source)), True),
("engine-upgrade", lambda source: (source, _wire(source, engine="model-hub-engine-v8.0.28-1")), False),
("avibe-release", lambda source: (source, _wire(source, avibe="3.2.2")), False),
(
"model-tiers-edited",
lambda source: (
(edited := replace(source, models=[replace(source.models[0], reasoning_efforts=["low", "high", "max"])])),
_wire(edited),
),
False,
),
(
"credential-rotated",
lambda source: ((rotated := replace(source, credential_ref="cred_rotated")), _wire(rotated)),
False,
),
(
"base-url-edited",
lambda source: ((moved := replace(source, base_url="https://relay.example/v1")), _wire(moved)),
False,
),
],
)
def test_a_refusal_holds_until_the_wire_or_the_source_changes(tmp_path, case, current, applies):
"""MH-HOSTED-003: a refusal has no time limit; it stops applying when the Source's config, the engine release, the
Avibe release or the model's rendered tiers change (owner rule: a Source config change resets it)."""
source = _source()
store = _store(tmp_path)
for feature in (CALL, ULTRA):
store.write_refusal(_refusal(source, feature), live=lambda: [source], wire_for=WIRE_FOR)
now_source, wire = current(source)
for feature in (CALL, ULTRA):
assert (store.refusal(now_source, "grok-4.6", feature, wire=wire) is not None) is applies
def test_a_success_never_replaces_a_refusal_and_never_routes(tmp_path):
"""MH-HOSTED-003: a success that settles after a concurrent refusal must not erase it; both are kept."""
source = _source()
store = _store(tmp_path)
store.write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
store.note_success(_success(source, CALL, observed_at=NOW + timedelta(seconds=1)), live=lambda: [source], now=NOW)
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None
assert {record.status for record in store.records()} == {"verified", "unsupported"}
def test_successes_reach_memory_at_once_and_the_file_at_most_hourly(tmp_path, monkeypatch):
"""MH-HOSTED-003: every success is read at once while file writes are throttled per key, and a restart keeps what
reached the file."""
from core.handlers.model_hub import capability_evidence
path = tmp_path / "model_hub_capability_evidence.json"
source, other = _source(), _source("src_xai00002")
live = [source, other]
store = CapabilityEvidenceStore(path)
writes = []
persist = capability_evidence.write_state_document
monkeypatch.setattr(capability_evidence, "write_state_document", lambda *args: (writes.append(1), persist(*args)))
def seen(store, source):
return store.get(source.id, "grok-4.6", CALL, "verified").observed_at
store.note_success(_success(source, CALL), live=lambda: live, now=NOW)
store.note_success(_success(other, CALL, observed_at=NOW + timedelta(minutes=5)), live=lambda: live, now=NOW + timedelta(minutes=5))
store.note_success(_success(source, CALL, observed_at=NOW + timedelta(minutes=10)), live=lambda: live, now=NOW + timedelta(minutes=10))
assert len(writes) == 2
assert (seen(store, source), seen(store, other)) == (NOW + timedelta(minutes=10), NOW + timedelta(minutes=5))
assert seen(CapabilityEvidenceStore(path), source) == NOW
store.note_success(_success(source, CALL, observed_at=NOW + timedelta(hours=2)), live=lambda: live, now=NOW + timedelta(hours=2))
assert len(writes) == 3
assert seen(CapabilityEvidenceStore(path), source) == NOW + timedelta(hours=2)
# A new identity's first success is written at once: the file's copy belonged to the old one.
rotated = replace(source, credential_ref="cred_rotated")
at = NOW + timedelta(hours=2, minutes=1)
store.note_success(
_success(rotated, CALL, observed_at=at, source_identity=identity_digest(rotated), wire=_wire(rotated)),
live=lambda: [rotated, other], now=at,
)
assert len(writes) == 4
assert seen(CapabilityEvidenceStore(path), rotated) == at
def test_the_newest_success_is_kept_whatever_order_settlements_land_in(tmp_path):
"""MH-HOSTED-003: two successes of one key race to the lock; the older landing second does not regress it."""
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
store = CapabilityEvidenceStore(path)
later = NOW + timedelta(seconds=2)
store.note_success(_success(source, CALL, observed_at=later), live=lambda: [source], now=later)
store.note_success(_success(source, CALL), live=lambda: [source], now=later)
assert store.get(source.id, "grok-4.6", CALL, "verified").observed_at == later
assert CapabilityEvidenceStore(path).get(source.id, "grok-4.6", CALL, "verified").observed_at == later
def test_the_newest_refusal_is_kept_whatever_order_settlements_land_in(tmp_path):
"""MH-HOSTED-003: one ordering rule for every write; an older refusal landing second keeps the newer one."""
source = _source()
store = _store(tmp_path)
later = NOW + timedelta(seconds=2)
store.write_refusal(_refusal(source, CALL, observed_at=later, detail="400 newer"), live=lambda: [source], wire_for=WIRE_FOR)
store.write_refusal(_refusal(source, CALL, detail="400 older"), live=lambda: [source], wire_for=WIRE_FOR)
kept = store.refusal(source, "grok-4.6", CALL, wire=_wire(source))
assert (kept.observed_at, kept.detail) == (later, "400 newer")
def test_a_refusal_learned_under_a_superseded_wire_never_replaces_the_current_one(tmp_path):
"""MH-HOSTED-003: a request admitted before the model's tiers changed settles last; its refusal is dropped, so the
refusal learned under the wire rendered now keeps routing."""
source = _source()
retiered = replace(source, models=[replace(source.models[0], reasoning_efforts=["low"])])
store = _store(tmp_path)
current = _refusal(retiered, CALL, observed_at=NOW + timedelta(seconds=2), wire=_wire(retiered))
store.write_refusal(current, live=lambda: [retiered], wire_for=WIRE_FOR)
store.write_refusal(_refusal(source, CALL), live=lambda: [retiered], wire_for=WIRE_FOR)
assert store.refusal(retiered, "grok-4.6", CALL, wire=_wire(retiered)) == current
def test_a_current_refusal_replaces_one_from_an_older_engine_whatever_its_time(tmp_path):
"""MH-HOSTED-003: a record the file kept from another engine no longer applies, so a current one replaces it."""
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
old_engine = "model-hub-engine-v8.0.20-1"
stale = _refusal(source, CALL, observed_at=NOW + timedelta(days=1), wire=_wire(source, engine=old_engine))
CapabilityEvidenceStore(path).write_refusal(
stale, live=lambda: [source],
wire_for=lambda source, model_id: wire_stamp(engine=old_engine, avibe=AVIBE, source=source, model_id=model_id),
)
store = CapabilityEvidenceStore(path)
store.write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)).observed_at == NOW
def test_a_refusal_holds_in_memory_when_its_write_fails(tmp_path, monkeypatch, caplog):
"""MH-HOSTED-003: a refusal is routing state; the re-render that follows must not declare the feature again."""
from core.handlers.model_hub import capability_evidence
source = _source()
store = _store(tmp_path)
def full_disk(*_args):
raise OSError("disk full")
monkeypatch.setattr(capability_evidence, "write_state_document", full_disk)
with caplog.at_level(logging.WARNING):
store.write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None
assert "holds until restart" in caplog.text
def test_a_clear_is_persisted_before_it_takes_effect(tmp_path, monkeypatch):
"""MH-HOSTED-003: a clear that cannot be written changes nothing, so it never comes back after a restart; one that
can removes only the refusals it names, never a success."""
from core.handlers.model_hub import capability_evidence
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
store = CapabilityEvidenceStore(path)
store.write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
store.write_refusal(_refusal(source, ULTRA), live=lambda: [source], wire_for=WIRE_FOR)
store.note_success(_success(source, CALL), live=lambda: [source], now=NOW)
persist = capability_evidence.write_state_document
def full_disk(*_args):
raise OSError("disk full")
monkeypatch.setattr(capability_evidence, "write_state_document", full_disk)
with pytest.raises(OSError):
store.clear(lambda record: record.feature == CALL, live=lambda: [source])
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None
monkeypatch.setattr(capability_evidence, "write_state_document", persist)
assert store.clear(lambda record: record.feature == CALL, live=lambda: [source]) == 1
assert store.clear(lambda record: record.feature == CALL, live=lambda: [source]) == 0
reloaded = CapabilityEvidenceStore(path)
assert reloaded.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is None
assert reloaded.refusal(source, "grok-4.6", ULTRA, wire=_wire(source)) is not None
assert reloaded.get(source.id, "grok-4.6", CALL, "verified") is not None
def test_the_bound_evicts_display_history_before_any_refusal(tmp_path):
"""MH-HOSTED-003: a refusal holds until a clear or a change; the record bound never re-enables a feature."""
source = _source()
store = CapabilityEvidenceStore(tmp_path / "model_hub_capability_evidence.json", max_records=3)
store.write_refusal(_refusal(source, CALL, observed_at=NOW - timedelta(days=400)), live=lambda: [source], wire_for=WIRE_FOR)
for index, tool in enumerate(("web_search", "x_search", "image_generate")):
feature = EvidenceFeature.hosted(tool, "call")
store.note_success(_success(source, feature, observed_at=NOW + timedelta(hours=index)), live=lambda: [source], now=NOW)
store.write_refusal(_refusal(source, ULTRA), live=lambda: [source], wire_for=WIRE_FOR)
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None
assert store.refusal(source, "grok-4.6", ULTRA, wire=_wire(source)) is not None
assert [record.feature.name for record in store.records() if record.status == "verified"] == ["image_generate"]
@pytest.mark.parametrize(
"marker,digest",
[
(None, "sha256:b682a96ef48e2f2834ed22ba2d42cdae568e5768f3fc3a1334a24728ffdb7166"),
("vp_" + "7" * 32, "sha256:d0b054e8827133cbeb6b4427af5ea696e2d29ab8f08de21d079336155bed8333"),
],
ids=["verified", "pending"],
)
def test_a_released_sources_identity_digest_is_unchanged(marker, digest):
"""MH-HOSTED-003: evidence follows the Source a released build saved. The digests are the ones released builds
computed, whose identity held the verification marker; a pending Source's marker seeds its generation."""
released = replace(_source(), models=[ModelHubModelConfig(id="grok-4.6", provenance="discovered")]).to_payload()
if marker is not None:
released["verification_pending"] = marker
assert "credential_generation" not in released
assert identity_digest(ModelHubSourceConfig.from_payload(released)) == digest
def test_a_changed_source_identity_is_pruned_at_the_next_write(tmp_path):
store = _store(tmp_path)
keep, rekeyed = _source("src_keep0001"), _source("src_swap0001")
store.write_refusal(_refusal(keep, CALL), live=lambda: [keep, rekeyed], wire_for=WIRE_FOR)
store.write_refusal(_refusal(rekeyed, CALL), live=lambda: [keep, rekeyed], wire_for=WIRE_FOR)
replaced = replace(rekeyed, credential_ref="cred_rotated")
store.write_refusal(_refusal(keep, REPLAY), live=lambda: [keep, replaced], wire_for=WIRE_FOR)
assert {record.source_id for record in store.records()} == {"src_keep0001"}
# A refusal learned before the rotation settles after it: it describes nothing that exists now, and is dropped.
store.write_refusal(_refusal(rekeyed, CALL), live=lambda: [keep, replaced], wire_for=WIRE_FOR)
assert {record.source_id for record in store.records()} == {"src_keep0001"}
def test_a_write_reads_the_sources_as_they_are_when_it_runs(tmp_path):
"""MH-HOSTED-003: a write that waited for the lock prunes against the Sources then, not an older snapshot, so it
never removes what a newer Source made valid."""
import threading
store = _store(tmp_path)
source, added = _source(), _source("src_added001")
sources = [source]
with store._lock:
# Another write holds the lock: this one waits, then must see the Source that write added.
waiting = threading.Thread(
target=store.write_refusal, args=(_refusal(source, ULTRA),),
kwargs={"live": lambda: list(sources), "wire_for": WIRE_FOR},
)
waiting.start()
waiting.join(0.1)
sources.append(added)
store.write_refusal(_refusal(added, CALL), live=lambda: list(sources), wire_for=WIRE_FOR)
waiting.join(5)
assert {(record.source_id, record.feature) for record in store.records()} == {
("src_xai00001", ULTRA), ("src_added001", CALL),
}
@pytest.mark.parametrize("mode", ["native", "executor", "direct", "service"])
def test_persisted_records_match_the_schema_and_rc_era_ones_are_pruned(tmp_path, mode):
"""MH-HOSTED-003: every execution mode survives a restart; rc probe records still do not load.
S2a adds service-mode reader acceptance without writing it from any live path.
"""
path = tmp_path / "model_hub_capability_evidence.json"
store = CapabilityEvidenceStore(path)
source = _source()
store.write_refusal(_refusal(source, ULTRA), live=lambda: [source], wire_for=WIRE_FOR)
store.note_success(_success(source, CALL, mode=mode), live=lambda: [source], now=NOW)
assert CapabilityEvidenceStore(path).get(source.id, "grok-4.6", CALL, "verified").mode == mode
schema = json.loads((CONTRACTS / "capability-evidence.schema.json").read_text())
validator = Draft7Validator(schema, registry=Registry().with_resources([(schema["$id"], Resource.from_contents(schema))]))
written = json.loads(path.read_text())
assert len(written) == 2
for payload in written:
validator.validate(payload)
rc_era = {
"version": 1, "source_id": source.id, "model_id": "grok-4.6",
"feature": {"kind": "hosted_tool", "name": "x_search", "aspect": "call"}, "status": "verified",
"basis": "probe", "observed_at": "2026-10-14T00:00:00Z", "engine": ENGINE, "shape": 1,
"source_identity": identity_digest(source),
}
# A damaged file's record is unknown and pruned, never echoed by a read: the shared identifier rules apply on load.
damaged = [
{**written[0], "model_id": "grok-\ud800"},
{**written[0], "model_id": " grok-4.6"},
{**written[0], "source_id": "xai"},
{**written[0], "wire": "engine-x"},
{**next(item for item in written if item["status"] == "unsupported"), "detail": "400 \ud800"},
]
path.write_text(json.dumps([*written, rc_era, {**written[0], "status": "maybe"}, *damaged]))
assert len(CapabilityEvidenceStore(path).records()) == 2
assert not validator.is_valid(rc_era)
@pytest.mark.parametrize(
"build",
[
lambda: _refusal(_source(), CALL, detail=None),
lambda: _refusal(_source(), CALL, mode="native"),
lambda: _success(_source(), CALL, mode=None),
lambda: _success(_source(), CALL, detail="400 x"),
lambda: _refusal(_source(), CALL, detail="x" * 65),
lambda: _refusal(_source(), CALL, observed_at=datetime(2026, 10, 15)),
lambda: _refusal(_source(), CALL, source_identity=""),
],
ids=["refusal-without-detail", "refusal-with-mode", "success-without-mode", "success-with-detail", "long-detail",
"naive-time", "no-identity"],
)
def test_a_record_refuses_what_the_contract_forbids(build):
with pytest.raises(ValueError):
build()
def test_image_generate_has_no_replay_evidence():
with pytest.raises(ValueError):
EvidenceFeature.hosted("image_generate", "replay")
with pytest.raises(ValueError):
EvidenceFeature.hosted("image_generation", "call")
def test_lookups_after_construction_never_touch_the_disk(tmp_path, monkeypatch):
"""MH-HOSTED-003: the store reads its file when built at startup; no lookup or write reads it again."""
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
CapabilityEvidenceStore(path).write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
store = CapabilityEvidenceStore(path)
assert len(store.records()) == 1
touched = []
original_stat, original_read = Path.stat, Path.read_text
def stat(self, *args, **kwargs):
if self == path:
touched.append("stat")
return original_stat(self, *args, **kwargs)
def read_text(self, *args, **kwargs):
if self == path:
touched.append("read")
return original_read(self, *args, **kwargs)
monkeypatch.setattr(Path, "stat", stat)
monkeypatch.setattr(Path, "read_text", read_text)
for _ in range(50):
assert store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None
store.write_refusal(_refusal(source, REPLAY), live=lambda: [source], wire_for=WIRE_FOR)
assert store.refusal(source, "grok-4.6", REPLAY, wire=_wire(source)) is not None
assert touched == []
def test_a_read_never_waits_for_a_write_on_a_slow_disk(tmp_path, monkeypatch):
"""MH-HOSTED-003: the request path reads a published snapshot, so a write stuck on the disk cannot stall it."""
import threading
from core.handlers.model_hub import capability_evidence
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
store = CapabilityEvidenceStore(path)
store.write_refusal(_refusal(source, CALL), live=lambda: [source], wire_for=WIRE_FOR)
on_disk = threading.Event()
release = threading.Event()
persist = capability_evidence.write_state_document
def slow_disk(target, payload):
on_disk.set()
assert release.wait(5)
persist(target, payload)
monkeypatch.setattr(capability_evidence, "write_state_document", slow_disk)
writer = threading.Thread(target=store.write_refusal, args=(_refusal(source, REPLAY),), kwargs={"live": lambda: [source], "wire_for": WIRE_FOR})
writer.start()
try:
assert on_disk.wait(5)
read = []
reader = threading.Thread(
target=lambda: read.append(store.refusal(source, "grok-4.6", CALL, wire=_wire(source)) is not None)
)
reader.start()
reader.join(1)
assert read == [True]
finally:
release.set()
writer.join(5)
assert store.refusal(source, "grok-4.6", REPLAY, wire=_wire(source)) is not None