Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Model Hub usage and resolution-event E2E scenarios."""
from __future__ import annotations
import asyncio
import json
import subprocess
import sys
from datetime import datetime, timedelta, timezone
import pytest
from core.handlers.model_hub.stream_wire import ProtocolUsageReport
from core.handlers.model_hub.usage import BoundedUsageLedger, UsageWriter
from tests.e2e.test_model_hub_sources import (
_configure_protocol,
_create_source,
)
pytestmark = pytest.mark.e2e_model_hub
# The counters these scenarios seed and read back; a report's cost and cache-write
# totals (#2189) are additive and owned by the pricing tests.
_SCENARIO_COUNTERS = ("requests", "token_reports", "input_tokens", "cached_input_tokens", "output_tokens")
def _counts(totals: dict) -> dict:
return {key: totals[key] for key in _SCENARIO_COUNTERS}
@pytest.fixture
def usage_analytics_app(model_hub_app_factory):
"""Seed configured identities without depending on live credential discovery."""
def seed_sources(app) -> None:
result = subprocess.run(
[
sys.executable,
"-c",
"""
from config.v2_config import (
ModelHubModelConfig, ModelHubSourceConfig,
ModelHubSourceStateConfig, V2Config,
)
config = V2Config.default()
config.model_hub.enabled = False
for backend in ("claude", "codex", "opencode"):
config.model_hub.agents[backend].mode = "direct"
config.model_hub.sources = [
ModelHubSourceConfig(
id=source_id, kind="api_key", vendor="custom",
display_name="供应商 · 同名", protocol="openai_chat",
supply_channel="hub", billing="metered",
state=ModelHubSourceStateConfig(),
models=[
ModelHubModelConfig(id=model_id, provenance="manual")
for model_id in ("shared-model", "模型-β")
],
credential_ref="synthetic-unavailable-" + source_id,
)
for source_id in ("src_usageaaa1", "src_usagebbb2")
]
config.update.auto_update = False
config.update.check_interval_minutes = 0
config.save()
""",
],
cwd=app.repo_root,
env=app.env,
capture_output=True,
text=True,
timeout=20,
)
assert result.returncode == 0, result.stdout + result.stderr
with model_hub_app_factory(before_start=seed_sources) as app:
yield app
def test_e2_usage_windows_are_bounded(
model_hub_app,
) -> None:
"""E2: valid, garbage, and pathological windows stay bounded."""
for query, expected in (
("1", 1),
("62", 62),
("garbage", 30),
("100000", 62),
):
response = model_hub_app.client.get(
f"/api/models/usage?days={query}"
)
body = response.json()
assert response.status == 200, body
assert body["usage"]["window_days"] == expected
assert _counts(body["usage"]["totals"]) == {
"requests": 0,
"token_reports": 0,
"input_tokens": 0,
"cached_input_tokens": 0,
"output_tokens": 0,
}
def test_e2_usage_reports_reconcile_across_http_ipc_and_restart(
usage_analytics_app,
) -> None:
"""MH-USAGE-E2E-001: the matrix retains exact pair identities through real IPC."""
model_hub_app = usage_analytics_app
common_model = "shared-model"
unicode_model = "模型-β"
sources = model_hub_app.client.get("/api/models/sources").json()["sources"]
first = next(source for source in sources if source["id"] == "src_usageaaa1")
second = next(source for source in sources if source["id"] == "src_usagebbb2")
assert first["id"] != second["id"]
now = datetime.now(timezone.utc)
ledger = BoundedUsageLedger(
model_hub_app.avibe_home / "state" / "model_hub_usage.json"
)
calls = (
(first["id"], common_model, timedelta(minutes=70), (150, 100, 25)),
(first["id"], common_model, timedelta(minutes=130), (40, 20, 5)),
(second["id"], common_model, timedelta(minutes=130), (80, 0, 10)),
(second["id"], unicode_model, timedelta(minutes=190), None),
(first["id"], common_model, timedelta(hours=48), (200, 40, 30)),
)
async def meter_calls() -> None:
# Queue together so the production writer must preserve different hours
# for the same pair before flushing one atomic ledger transaction.
writer = UsageWriter(ledger)
for source_id, model_id, ago, counts in calls:
writer.record(
source_id=source_id,
model_id=model_id,
usage=(
ProtocolUsageReport.of(
input_tokens=counts[0],
cached_input_tokens=counts[1],
output_tokens=counts[2],
)
if counts is not None
else None
),
at=now - ago,
)
assert await writer.drain(timeout=5) == 0
# The test runtime has no active turns, so the controller cannot race this
# fixture with another metering write.
asyncio.run(meter_calls())
expected_hourly = {
"requests": 4,
"token_reports": 3,
"input_tokens": 270,
"cached_input_tokens": 120,
"output_tokens": 40,
}
expected_daily = {
"requests": 5,
"token_reports": 4,
"input_tokens": 470,
"cached_input_tokens": 160,
"output_tokens": 70,
}
for window, count in (("24h", 24), ("7d", 7), ("30d", 30), ("60d", 60)):
response = model_hub_app.client.get(f"/api/models/usage?window={window}")
body = response.json()
assert response.status == 200, body
report = body["usage"]
expected = expected_hourly if window == "24h" else expected_daily
assert report["window_key"] == window
assert report["granularity"] == ("hour" if window == "24h" else "day")
assert _counts(report["totals"]) == expected
buckets = report["buckets"]
assert len(buckets) == count
assert len({bucket["key"] for bucket in buckets}) == count
assert buckets[0]["start_at"] == report["from_at"]
assert buckets[-1]["end_at"] == report["to_at"]
for before, after in zip(buckets, buckets[1:]):
assert before["end_at"] == after["start_at"]
for bucket in buckets:
assert isinstance(bucket["history_complete"], bool)
start = datetime.fromisoformat(bucket["start_at"])
end = datetime.fromisoformat(bucket["end_at"])
assert start.utcoffset() is not None
assert end.utcoffset() is not None
assert start <= end
pairs = [
(row["source_id"], row["model_id"]) for row in bucket["rows"]
]
assert len(pairs) == len(set(pairs))
rows = [row for bucket in buckets for row in bucket["rows"]]
if window == "24h":
assert len([
row for row in rows
if row["source_id"] == first["id"] and row["model_id"] == common_model
]) == 2
assert {(row["source_id"], row["model_id"]) for row in rows} == {
(first["id"], common_model),
(second["id"], common_model),
(second["id"], unicode_model),
}
for key, value in expected.items():
assert sum(row[key] for row in rows) == value
assert sum(source[key] for source in report["sources"]) == value
assert sum(day[key] for day in report["days"]) == value
assert {source["label"] for source in report["sources"]} == {
"供应商 · 同名"
}
for source in report["sources"]:
for key in expected:
assert sum(model[key] for model in source["models"]) == source[key]
for query in (
"window=1d", "window=", "window=24h&days=7",
"window=24h&window=7d", "window=7d&window=24h",
"window=invalid&window=24h", "window=24h&window=invalid",
):
response = model_hub_app.client.get(f"/api/models/usage?{query}")
assert response.status == 400, response.json()
repeated = model_hub_app.client.get("/api/models/usage?window=24h&window=24h")
assert repeated.status == 200, repeated.json()
assert _counts(repeated.json()["usage"]["totals"]) == expected_hourly
legacy = model_hub_app.client.get("/api/models/usage?days=7").json()["usage"]
assert _counts(legacy["totals"]) == expected_daily
assert "buckets" not in legacy
model_hub_app.restart_controller()
for window, expected in (("24h", expected_hourly), ("7d", expected_daily)):
response = model_hub_app.client.get(f"/api/models/usage?window={window}")
assert response.status == 200, response.json()
assert _counts(response.json()["usage"]["totals"]) == expected
def test_e2_daily_only_history_is_not_invented_as_hourly_usage(
model_hub_app_factory,
) -> None:
"""MH-USAGE-E2E-002: released daily files never fabricate hourly history."""
now = datetime.now(timezone.utc)
counts = {
"requests": 3,
"token_reports": 2,
"input_tokens": 150,
"cached_input_tokens": 100,
"output_tokens": 25,
}
def seed_legacy(app) -> None:
path = app.avibe_home / "state" / "model_hub_usage.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps([{
"day": now.astimezone().date().isoformat(),
"source_id": "historical-source",
"model_id": "历史模型",
"last_metered_at": now.isoformat(),
**counts,
}]),
encoding="utf-8",
)
with model_hub_app_factory(before_start=seed_legacy) as app:
daily = app.client.get("/api/models/usage?window=7d")
assert daily.status == 200, daily.json()
assert _counts(daily.json()["usage"]["totals"]) == counts
hourly = app.client.get("/api/models/usage?window=24h")
assert hourly.status == 200, hourly.json()
report = hourly.json()["usage"]
assert _counts(report["totals"]) == dict.fromkeys(counts, 0)
assert any(not bucket["history_complete"] for bucket in report["buckets"])
assert not any(bucket["rows"] for bucket in report["buckets"])
def test_e3_event_feed_paginates_and_never_persists_credentials(
mock_llm_upstream,
model_hub_app,
) -> None:
"""E3: event pages are cursor-stable and credential-free on disk/wire."""
_configure_protocol(mock_llm_upstream)
source, _ = _create_source(model_hub_app, mock_llm_upstream)
secret = "sk-model-hub-e2e-not-real"
mock_llm_upstream.configure(models_endpoint="http_500")
for _ in range(3):
response = model_hub_app.client.post(
f"/api/models/sources/{source['id']}/refresh", {}
)
assert response.status == 502, response.json()
assert response.json()["error"] == "discovery_failed"
first = model_hub_app.client.get("/api/models/events?limit=2")
first_body = first.json()
assert first.status == 200, first_body
assert len(first_body["events"]) == 2
assert all(event["kind"] == "needs_action" for event in first_body["events"])
assert all(
event["reason"] == "unclassified_error"
for event in first_body["events"]
)
cursor = first_body["events"][-1]["id"]
second = model_hub_app.client.get(
f"/api/models/events?limit=2&before={cursor}"
)
second_body = second.json()
assert second.status == 200, second_body
assert len(second_body["events"]) == 1
assert second_body["events"][0]["id"] not in {
event["id"] for event in first_body["events"]
}
wire = json.dumps(
[*first_body["events"], *second_body["events"]],
sort_keys=True,
)
assert secret not in wire
assert "authorization" not in wire.lower()
event_file = (
model_hub_app.avibe_home
/ "state"
/ "model_hub_resolution_events.json"
)
assert event_file.is_file()
assert secret not in event_file.read_text(encoding="utf-8")