Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
from __future__ import annotations
import json
import time
import urllib.error
import pytest
from vibe import models_dev_catalog
class _Response:
def __init__(self, payload, *, headers=None):
self.payload = payload
self.headers = headers or {}
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def read(self, _limit):
return json.dumps(self.payload).encode()
def _catalog():
return {
"deepseek": {
"id": "deepseek",
"name": "DeepSeek",
"models": {
"deepseek-v4": {
"id": "deepseek-v4",
"name": "DeepSeek V4",
"reasoning": True,
"reasoning_options": [
{
"type": "effort",
"values": ["low", "x" * 65, "high", "low"],
}
],
"tool_call": True,
"modalities": {
"input": ["text", "image", "image", "pdf", "unknown"],
"output": ["text", "text", "pdf"],
},
"limit": {"context": 1_048_576, "output": 131_072},
}
},
}
}
@pytest.mark.real_catalog_refresh
def test_models_dev_search_normalizes_editable_metadata(monkeypatch, tmp_path):
cache_path = tmp_path / "models-dev.json"
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: cache_path)
monkeypatch.setattr(
models_dev_catalog.urllib.request,
"urlopen",
lambda request, timeout: _Response(
_catalog(),
headers={"ETag": '"catalog-v1"'},
),
)
matches = models_dev_catalog.search_models_dev("deepseek-v4")
assert matches == [
{
"provider_id": "deepseek",
"provider_name": "DeepSeek",
"first_party": True,
"model_id": "deepseek-v4",
"models_dev_id": "deepseek/deepseek-v4",
"display_name": "DeepSeek V4",
"context_window": 1_048_576,
"max_output_tokens": 131_072,
"input_modalities": ["text", "image", "pdf"],
"output_modalities": ["text"],
"supports_tools": True,
"supports_reasoning": True,
"reasoning_efforts": ["low", "high"],
"native_protocol": "openai_responses",
}
]
cached = json.loads(cache_path.read_text(encoding="utf-8"))
assert cached["etag"] == '"catalog-v1"'
assert cached["catalog"] == _catalog()
def test_models_dev_search_preserves_validator_accepted_long_names(monkeypatch):
model_id = "m" * 80
display_name = "Model " + "x" * 80
monkeypatch.setattr(
models_dev_catalog,
"load_models_dev_catalog",
lambda: {
"provider": {
"name": "Provider",
"models": {model_id: {"name": display_name}},
}
},
)
match = models_dev_catalog.search_models_dev(model_id)[0]
assert match["model_id"] == model_id
assert match["display_name"] == display_name
@pytest.mark.real_catalog_refresh
def test_models_dev_search_uses_fresh_cache_without_network(monkeypatch, tmp_path):
cache_path = tmp_path / "models-dev.json"
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: cache_path)
cache_path.write_text(
json.dumps(
{
"fetched_at": time.time(),
"url": models_dev_catalog.DEFAULT_MODELS_DEV_URL,
"catalog": _catalog(),
}
),
encoding="utf-8",
)
def unexpected(*_args, **_kwargs):
raise AssertionError("fresh cache must avoid network")
monkeypatch.setattr(models_dev_catalog.urllib.request, "urlopen", unexpected)
assert models_dev_catalog.search_models_dev("deepseek/deepseek-v4")[0]["models_dev_id"] == "deepseek/deepseek-v4"
@pytest.mark.real_catalog_refresh
def test_models_dev_search_falls_back_to_stale_cache(monkeypatch, tmp_path):
cache_path = tmp_path / "models-dev.json"
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: cache_path)
cache_path.write_text(
json.dumps(
{
"fetched_at": 0,
"url": models_dev_catalog.DEFAULT_MODELS_DEV_URL,
"catalog": _catalog(),
}
),
encoding="utf-8",
)
def unavailable(*_args, **_kwargs):
raise urllib.error.URLError("offline")
monkeypatch.setattr(models_dev_catalog.urllib.request, "urlopen", unavailable)
assert models_dev_catalog.search_models_dev("DeepSeek V4")[0]["context_window"] == 1_048_576
@pytest.mark.real_catalog_refresh
def test_models_dev_stale_cache_is_scoped_to_its_url(monkeypatch, tmp_path):
cache_path = tmp_path / "models-dev.json"
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: cache_path)
cache_path.write_text(
json.dumps(
{
"fetched_at": 0,
"url": models_dev_catalog.DEFAULT_MODELS_DEV_URL,
"catalog": _catalog(),
}
),
encoding="utf-8",
)
monkeypatch.setenv(
models_dev_catalog.MODELS_DEV_URL_ENV,
"https://catalog.example.invalid/api.json",
)
def unavailable(*_args, **_kwargs):
raise urllib.error.URLError("offline")
monkeypatch.setattr(models_dev_catalog.urllib.request, "urlopen", unavailable)
with pytest.raises(RuntimeError, match="unavailable"):
models_dev_catalog.search_models_dev("deepseek-v4")
def test_model_vendor_map_is_versioned_unambiguous_and_ordered():
vendor_map = models_dev_catalog.load_model_vendor_map()
families = vendor_map["families"]
aggregators = vendor_map["aggregators"]
assert vendor_map["schema_version"] == 1
assert families
assert all(set(item) == {"prefix", "vendor_id"} and item["prefix"] and item["vendor_id"] for item in families)
assert len({item["prefix"] for item in families}) == len(families)
assert aggregators
assert len(set(aggregators)) == len(aggregators)
assert all(isinstance(provider_id, str) and provider_id for provider_id in aggregators)
def test_native_protocol_derivation_uses_every_vendor_family_on_the_last_segment():
vendor_map = models_dev_catalog.load_model_vendor_map()
for family in vendor_map["families"]:
model_id = f"aggregator/nested/{family['prefix']}fixture"
expected = (
"anthropic"
if family["vendor_id"] == "anthropic"
else "openai_responses"
)
assert models_dev_catalog.native_protocol_for_model_id(
model_id,
vendor_map=vendor_map,
) == expected
def test_models_dev_deduplicates_by_model_id_and_ranks_first_party_then_match(
monkeypatch,
):
catalog = {
"openrouter": {
"name": "OpenRouter",
"models": {
"gpt-target": {"name": "Target proxy"},
"target": {"name": "Target"},
},
},
"openai": {
"name": "OpenAI",
"models": {"gpt-target": {"name": "GPT target"}},
},
"zeta": {
"name": "Zeta",
"models": {"target": {"name": "Target"}},
},
}
monkeypatch.setattr(
models_dev_catalog,
"load_models_dev_catalog",
lambda: catalog,
)
matches = models_dev_catalog.search_models_dev("target")
assert [item["model_id"] for item in matches] == ["gpt-target", "target"]
assert matches[0]["provider_id"] == "openai"
assert matches[0]["first_party"] is True
assert matches[1]["provider_id"] == "openrouter"
assert matches[1]["first_party"] is False
def test_models_dev_openai_o_series_does_not_claim_other_o_families(monkeypatch):
catalog = {
"openrouter": {
"name": "OpenRouter",
"models": {
"o3-target": {"name": "O3 proxy"},
"olmo-target": {"name": "OLMo proxy"},
},
},
"openai": {
"name": "OpenAI",
"models": {
"o3-target": {"name": "O3 target"},
"olmo-target": {"name": "OLMo OpenAI copy"},
},
},
}
monkeypatch.setattr(
models_dev_catalog,
"load_models_dev_catalog",
lambda: catalog,
)
matches = {
item["model_id"]: item for item in models_dev_catalog.search_models_dev("target")
}
assert matches["o3-target"]["provider_id"] == "openai"
assert matches["o3-target"]["first_party"] is True
assert matches["olmo-target"]["provider_id"] == "openrouter"
assert matches["olmo-target"]["first_party"] is False
def test_models_dev_normalizes_each_copy_before_ranking_and_deduplication(
monkeypatch,
):
catalog = {
"openai": {
"name": "OpenAI",
"models": {
"gpt-target": {"name": " "},
"gpt-unusable-target": {
"name": "Unusable target",
"reasoning_options": [
{"type": "effort", "values": [" "]}
],
},
},
},
"openrouter": {
"name": "OpenRouter",
"models": {"gpt-target": {"name": "Valid aggregator copy"}},
},
}
monkeypatch.setattr(
models_dev_catalog,
"load_models_dev_catalog",
lambda: catalog,
)
matches = models_dev_catalog.search_models_dev("target")
assert [item["model_id"] for item in matches] == [
"gpt-unusable-target",
"gpt-target",
]
assert matches[0]["reasoning_efforts"] == []
assert matches[1]["provider_id"] == "openrouter"
assert matches[1]["first_party"] is False
def test_models_dev_caps_matches_at_eight(monkeypatch):
catalog = {
"unknown": {
"name": "Unknown",
"models": {f"test-{index:02d}": {"name": f"Test {index:02d}"} for index in range(12)},
}
}
monkeypatch.setattr(
models_dev_catalog,
"load_models_dev_catalog",
lambda: catalog,
)
matches = models_dev_catalog.search_models_dev("test")
assert len(matches) == models_dev_catalog.MODELS_DEV_MAX_MATCHES == 8
assert [item["model_id"] for item in matches] == sorted(item["model_id"] for item in matches)
def test_exact_models_dev_matches_never_borrow_a_neighbour():
catalog = {
"openrouter": {
"name": "OpenRouter",
"models": {
"gpt-target": {"name": "Proxy"},
"claude-3-5-target": {"name": "Claude 3.5 proxy"},
},
},
"openai": {
"name": "OpenAI",
"models": {
"gpt-target": {"name": "GPT target"},
"gpt-target-mini": {"name": "GPT target mini"},
"foo": {"name": "Foo"},
"Case-Target": {"name": "Case target"},
},
},
}
matches = models_dev_catalog.exact_models_dev_matches(
[
"gpt-target", "openrouter/gpt-target", "relay/gpt-target-mini",
"open_router/gpt.target", "claude-3.5-target", "gpt", "gpt-target-max",
"avibe-foo", "case-target",
],
catalog,
)
# A bare id prefers the first-party copy; a full identity names its own.
assert matches["gpt-target"]["models_dev_id"] == "openai/gpt-target"
assert matches["openrouter/gpt-target"]["models_dev_id"] == "openrouter/gpt-target"
# A relay prefix falls back to the last segment.
assert matches["relay/gpt-target-mini"]["models_dev_id"] == "openai/gpt-target-mini"
# Only identical spellings match: no punctuation, case, or alias folding and
# no substrings.
assert set(matches) == {"gpt-target", "openrouter/gpt-target", "relay/gpt-target-mini"}
assert models_dev_catalog.exact_models_dev_matches(["gpt-target"], {}) == {}
def _entry(declared: object, **fields: object) -> dict:
return {**fields, "modalities": {"input": declared}}
@pytest.mark.parametrize(
("entries", "requested", "text_only"),
[
({"target": _entry(["text"])}, "target", True),
({"target": _entry([" TEXT ", "pdf"])}, "target", True),
({"target": _entry(["text", "image"])}, "target", False),
({"target": _entry(["text", " Image "])}, "target", False),
# Undeclared or malformed input leaves the model undeclared.
({"target": {"name": "No modalities"}}, "target", False),
({"target": _entry([])}, "target", False),
({"target": _entry(["text", 1])}, "target", False),
({"target": _entry("text")}, "target", False),
({"target": "not-an-object"}, "target", False),
({"target": _entry(["image"])}, "target", False),
# Model identity is Avibe's: trimmed, case preserved.
({"target": _entry(["text"])}, " target ", True),
({"target": _entry(["text"])}, "TARGET", False),
({"TARGET": _entry(["text", "image"]), "target": _entry(["text"])}, "TARGET", False),
# An entry is found by its key or its id; one that disagrees vetoes.
({"hosted-7": _entry(["text"], id="target")}, "target", True),
({"target": _entry(["text"]), "alias": _entry(["text", "image"], id="target")}, "target", False),
({"other": _entry(["text"])}, "target", False),
],
)
def test_text_only_reads_the_upstream_providers_own_entry(entries, requested, text_only):
"""MH-MODALITIES-004: a model is text-only when its upstream provider's own entry declares text and no image."""
catalog = {"deepseek": {"models": entries}}
assert models_dev_catalog.text_only_model_ids("deepseek", [requested], catalog) == (
{requested} if text_only else set()
)
def test_text_only_never_reads_another_providers_copy():
"""MH-MODALITIES-004: a relay's declaration describes the relay, so it neither marks nor vetoes the vendor."""
catalog = {
"zhipuai": {"models": {"glm-5.2": _entry(["text"])}},
"baseten": {"models": {"zai-org/GLM-5.2": _entry(["text", "image"])}},
"relay": {"models": {"deepseek-model": _entry(["text"])}},
"deepseek": {"models": {}},
}
assert models_dev_catalog.text_only_model_ids("zhipuai", ["glm-5.2"], catalog) == {"glm-5.2"}
assert models_dev_catalog.text_only_model_ids("deepseek", ["deepseek-model"], catalog) == set()
def test_text_only_warns_about_an_unreadable_provider_without_touching_others(caplog):
"""MH-MODALITIES-004: an unreadable or missing provider declares nothing and says so; other providers are unaffected."""
catalog = {
"deepseek": {"models": [_entry(["text"])]},
"zhipuai": {"models": {"glm-5": _entry(["text"])}},
}
with caplog.at_level("WARNING", logger=models_dev_catalog.__name__):
assert models_dev_catalog.text_only_model_ids("deepseek", ["deepseek-model"], catalog) == set()
assert models_dev_catalog.text_only_model_ids("zhipuai", ["glm-5"], catalog) == {"glm-5"}
assert models_dev_catalog.text_only_model_ids("groq", ["model"], catalog) == set()
# No cached copy yet is not a fault.
assert models_dev_catalog.text_only_model_ids("groq", ["model"], {}) == set()
messages = [record.getMessage() for record in caplog.records if record.name == models_dev_catalog.__name__]
assert len(messages) == 2
assert "deepseek" in messages[0] and "zhipuai" not in messages[0]
assert "groq" in messages[1]
def test_cached_catalog_read_does_not_wait_on_a_foreground_fetch(monkeypatch, tmp_path):
"""MH-MODALITIES-002: an engine sync reads the cached copy while a picker fetch holds the cache lock."""
import threading
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: tmp_path / "models_dev_catalog.json")
models_dev_catalog._write_cache({"url": models_dev_catalog._models_dev_url(), "fetched_at": 0, "catalog": _catalog()})
read: list[dict] = []
with models_dev_catalog._CACHE_LOCK:
reader = threading.Thread(target=lambda: read.append(models_dev_catalog.cached_models_dev_catalog()))
reader.start()
reader.join(timeout=5)
waited = reader.is_alive()
reader.join(timeout=5)
assert not waited
assert read == [_catalog()]
@pytest.mark.real_catalog_refresh
def test_first_catalog_read_reports_one_fetch_in_flight_until_it_fails(monkeypatch, tmp_path):
"""MH-PRICE-014: With no cached copy, readers start one fetch and say one is coming; after a failure they do not."""
import threading
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: tmp_path / "models_dev_catalog.json")
monkeypatch.setattr(models_dev_catalog, "_refresh_in_flight", False, raising=False)
monkeypatch.setattr(models_dev_catalog, "_last_refresh_failed", False, raising=False)
release = threading.Event()
fetches = []
def fetch(_previous):
fetches.append(threading.current_thread().name)
release.wait(timeout=5)
raise OSError("offline")
monkeypatch.setattr(models_dev_catalog, "_fetch_catalog", fetch)
readers = [threading.Thread(target=models_dev_catalog.load_models_dev_catalog_with_date) for _ in range(8)]
for reader in readers:
reader.start()
for reader in readers:
reader.join(timeout=5)
assert models_dev_catalog.load_models_dev_catalog_with_date() == ({}, None, True)
release.set()
def join_refreshes():
for thread in threading.enumerate():
if thread.name == "models-dev-refresh":
thread.join(timeout=5)
join_refreshes()
assert fetches == ["models-dev-refresh"]
# The fetch failed: the next read retries it, but no longer promises prices.
assert models_dev_catalog.load_models_dev_catalog_with_date() == ({}, None, False)
# That retry runs on its own thread: it must finish while this test's fetch
# and cache path are still patched, not after teardown restores the real ones.
join_refreshes()
@pytest.mark.real_catalog_refresh
def test_mh_limits_001_every_saved_copy_signals_its_arrival(monkeypatch, tmp_path):
"""MH-LIMITS-001: a copy saved by a foreground or a background fetch signals; no other fetch does.
Rows written while no copy was cached are filled when one arrives, and the
copy that most often arrives is the background refresh a picker read starts.
"""
import threading
monkeypatch.setattr(models_dev_catalog, "_cache_path", lambda: tmp_path / "models_dev_catalog.json")
monkeypatch.setattr(models_dev_catalog, "_refresh_in_flight", False)
monkeypatch.setattr(models_dev_catalog, "_last_refresh_failed", False)
monkeypatch.setattr(models_dev_catalog, "_COPY_ARRIVED", None)
arrivals: list[str] = []
models_dev_catalog.set_models_dev_copy_arrived(lambda: arrivals.append(threading.current_thread().name))
outcomes: list[object] = []
def urlopen(_request, timeout):
outcome = outcomes.pop(0)
if isinstance(outcome, Exception):
raise outcome
return outcome
monkeypatch.setattr(models_dev_catalog.urllib.request, "urlopen", urlopen)
def refresh_in_background(outcome):
models_dev_catalog._write_cache({**models_dev_catalog._read_cache(), "fetched_at": 0})
outcomes.append(outcome)
assert models_dev_catalog.load_models_dev_catalog_with_date()[0] == _catalog()
for thread in threading.enumerate():
if thread.name == "models-dev-refresh":
thread.join(timeout=5)
assert not outcomes
outcomes.append(urllib.error.URLError("offline"))
with pytest.raises(RuntimeError, match="unavailable"):
models_dev_catalog.load_models_dev_catalog()
assert arrivals == []
outcomes.append(_Response(_catalog(), headers={"ETag": "v1"}))
assert models_dev_catalog.load_models_dev_catalog() == _catalog()
assert arrivals == [threading.current_thread().name]
# Unchanged upstream: the copy stays as it was, and nothing arrived.
refresh_in_background(urllib.error.HTTPError(models_dev_catalog._models_dev_url(), 304, "Not Modified", {}, None))
assert arrivals == [threading.current_thread().name]
refresh_in_background(_Response(_catalog(), headers={"ETag": "v2"}))
assert arrivals == [threading.current_thread().name, "models-dev-refresh"]