Skip to content

The agent that grows with you

test(config): fold MoA preset key cases into the runtime-keys suite

kshitijk4poord7f296079af4
diff --git a/tests/agent/test_moa_fanout_cadence.py b/tests/agent/test_moa_fanout_cadence.py
index 35a0b2d85f8..b1a455c63cb 100644
--- a/tests/agent/test_moa_fanout_cadence.py
+++ b/tests/agent/test_moa_fanout_cadence.py
@@ -5,7 +5,8 @@ Redesigned from PR #63448's intent (issue #63393 — advisor fan-out multiplies
turn latency/cost by the tool-iteration count). Unlike the submitted shape
(which dropped references entirely on off-cadence iterations), off-cadence
iterations here still feed the aggregator the LAST advisor guidance via the
-same cache-reuse mechanism the user_turn fanout uses.
+same cache-reuse mechanism the user_turn fanout uses. Also: a preset's
+reference_timeout reaches each advisor request.
"""
from types import SimpleNamespace
@@ -17,15 +18,16 @@ def _response(content="done", *, tool_calls=None):
return SimpleNamespace(choices=[choice], usage=None, model="fake-model")
-def _cadence_config(home, fanout="every_n:3"):
+def _cadence_config(home, fanout="every_n:3", *, reference_timeout=None):
home.mkdir()
+ timeout_line = f"\n reference_timeout: {reference_timeout}" if reference_timeout is not None else ""
(home / "config.yaml").write_text(
f"""
moa:
default_preset: review
presets:
review:
- fanout: "{fanout}"
+ fanout: "{fanout}"{timeout_line}
reference_models:
- provider: openai-codex
model: gpt-5.5
@@ -132,6 +134,27 @@ def test_every_n_off_cadence_iterations_reuse_cached_guidance(monkeypatch, tmp_p
+def test_preset_reference_timeout_reaches_reference_calls(monkeypatch, tmp_path):
+ """A preset's reference_timeout is the timeout each advisor call_llm request carries."""
+ home = tmp_path / ".hermes"
+ _cadence_config(home, "user_turn", reference_timeout=900)
+ monkeypatch.setenv("HERMES_HOME", str(home))
+
+ timeouts = []
+
+ def fake_call_llm(**kwargs):
+ if kwargs["task"] == "moa_reference":
+ timeouts.append(kwargs.get("timeout"))
+ return _response("advice")
+
+ monkeypatch.setattr("agent.moa_loop.call_llm", fake_call_llm)
+ monkeypatch.setattr("agent.moa_loop._reference_context_length", lambda *a, **k: None) # no metadata fetch
+ from agent.moa_loop import MoAChatCompletions
+
+ MoAChatCompletions("review").create(messages=[{"role": "user", "content": "task"}], tools=[])
+ assert timeouts == [900]
+
+
def test_per_iteration_default_unchanged_by_cadence_state(monkeypatch, tmp_path):
"""Default fanout still re-runs references on every state change."""
home = tmp_path / ".hermes"
diff --git a/tests/hermes_cli/test_config_runtime_keys.py b/tests/hermes_cli/test_config_runtime_keys.py
index 985038bf02c..1693c8b4ec0 100644
--- a/tests/hermes_cli/test_config_runtime_keys.py
+++ b/tests/hermes_cli/test_config_runtime_keys.py
@@ -1,6 +1,7 @@
"""`hermes config get/set` must recognize keys the runtime reads but DEFAULT_CONFIG leaves unseeded:
-core provider selections (``browser.cloud_provider``, ``<category>.use_gateway``), core-owned keys with
-a known reader (``computer_use.backend``), and keys an ACTIVE plugin declares through ``config_keys``.
+core provider selections (``browser.cloud_provider``, ``<category>.use_gateway``), MoA preset fields
+(``moa.presets.<name>.fanout``), core-owned keys with a known reader (``computer_use.backend``), and keys
+an ACTIVE plugin declares through ``config_keys``.
Declarations never weaken the schema's typo notices."""
import argparse
@@ -64,6 +65,40 @@ def test_config_set_runtime_read_key_is_silent(key, value, hermes_home, capsys):
assert not _set_notice(key, value, capsys)
+def _moa_key(name, field):
+ return "moa.presets." + name.replace(".", r"\.") + "." + field
+
+
[email protected]("name", ["default", "review", "review.v2", "code review"])
+def test_moa_preset_fields_set_and_get_without_notice(name, hermes_home, capsys):
+ from hermes_cli.config import get_config_value, load_config
+ from hermes_cli.moa_config import resolve_moa_preset
+
+ slot = {"provider": "openrouter", "model": "test/actor"}
+ _write_config(hermes_home, {"moa": {"presets": {name: {"reference_models": [slot], "aggregator": slot}}}})
+ for field, value, expected in (("fanout", "every_n:5", "every_n:5"), ("reference_timeout", "900", 900)):
+ assert not _set_notice(_moa_key(name, field), value, capsys), field
+ get_config_value(_moa_key(name, field), as_json=True)
+ captured = capsys.readouterr()
+ assert json.loads(captured.out) == expected
+ assert NOTICE not in captured.err, field
+ preset = resolve_moa_preset(load_config()["moa"], name)
+ assert (preset["fanout"], preset["reference_timeout"]) == ("every_n:5", 900)
+
+
+ pytest.param("moa.presets.default.fanoutt", id="field-typo"),
+ pytest.param("moa.presets.review.reference_timeot", id="field-typo-custom"),
+ pytest.param(r"moa.presets.review\.v2.fanoutt", id="field-typo-escaped-name"),
+ pytest.param("moa.presets.review.unexpected.fanout", id="extra-segment-before-field"),
+ pytest.param("moa.presets.review.fanout.unexpected", id="extra-segment-after-field"),
+])
+def test_moa_non_field_paths_still_notice(key, hermes_home, monkeypatch, capsys):
+ # Only the schema walk and the MoA rule decide these keys; skip the plugin scan they would fall through to.
+ monkeypatch.setattr("hermes_cli.config_runtime_keys.plugin_declared_config_keys", frozenset)
+ assert _set_notice(key, "5", capsys)
+
+
def test_every_selection_category_key_set_is_what_read_selection_returns(hermes_home, capsys):
from tools.tool_backend_helpers import read_selection, selection_config_keys
diff --git a/tests/hermes_cli/test_moa_config_key_validation.py b/tests/hermes_cli/test_moa_config_key_validation.py
deleted file mode 100644
index 6161c063986..00000000000
--- a/tests/hermes_cli/test_moa_config_key_validation.py
+++ /dev/null
@@ -1,111 +0,0 @@
-"""Supported MoA preset keys survive CLI writes and reach the executing advisor loop."""
-
-import json
-from types import SimpleNamespace
-
-import pytest
-
-from hermes_cli.config import get_config_value, load_config, set_config_value
-from hermes_cli.moa_config import resolve_moa_preset
-
-
-def moa_home(tmp_path, monkeypatch):
- monkeypatch.setenv("HERMES_HOME", str(tmp_path))
- (tmp_path / ".env").touch()
- return tmp_path
-
-
[email protected]("name", ["default", "review", "review.v2", "code review"])
-def test_configured_cadence_and_timeout_reach_moa(name, moa_home, monkeypatch, capsys):
- (moa_home / "config.yaml").write_text(
- json.dumps(
- {
- "moa": {
- "presets": {
- name: {
- "reference_models": [
- {"provider": "openrouter", "model": "test/advisor"}
- ],
- "aggregator": {
- "provider": "openrouter",
- "model": "test/actor",
- },
- }
- }
- }
- }
- ),
- encoding="utf-8",
- )
- prefix = "moa.presets." + name.replace(".", r"\.")
- set_config_value(f"{prefix}.fanout", "every_n:5")
- set_config_value(f"{prefix}.reference_timeout", "900")
- written = capsys.readouterr()
- preset = resolve_moa_preset(load_config()["moa"], name)
- assert preset["fanout"] == "every_n:5"
- assert preset["reference_timeout"] == 900
-
- reads = []
- for field, expected in (("fanout", "every_n:5"), ("reference_timeout", 900)):
- get_config_value(f"{prefix}.{field}", as_json=True)
- captured = capsys.readouterr()
- assert json.loads(captured.out) == expected
- reads.append(captured.err)
-
- calls = []
-
- def fake_call(**kwargs):
- calls.append(kwargs)
- message = SimpleNamespace(content="advice", tool_calls=[])
- return SimpleNamespace(
- choices=[SimpleNamespace(message=message, finish_reason="stop")],
- usage=None,
- model="test/model",
- )
-
- monkeypatch.setattr("agent.moa_loop.call_llm", fake_call)
- from agent.moa_loop import MoAChatCompletions
-
- facade = MoAChatCompletions(name)
- messages = [{"role": "user", "content": "Review the task"}]
- reference_counts = []
- for iteration in range(6):
- facade.create(messages=list(messages), tools=[])
- reference_counts.append(sum(call["task"] == "moa_reference" for call in calls))
- messages.extend(
- [
- {
- "role": "assistant",
- "content": "",
- "tool_calls": [
- {
- "id": f"c{iteration}",
- "function": {"name": "inspect", "arguments": "{}"},
- }
- ],
- },
- {
- "role": "tool",
- "tool_call_id": f"c{iteration}",
- "content": f"result {iteration}",
- },
- ]
- )
- assert reference_counts == [1, 1, 1, 1, 1, 2]
- assert all(
- call["timeout"] == 900 for call in calls if call["task"] == "moa_reference"
- )
- assert "not a recognized config key" not in written.out + written.err + "".join(
- reads
- )
-
-
[email protected]("name", ["default", "review", "review.v2"])
- "field", ["fanoutt", "reference_timeot", "unexpected.fanout", "fanout.unexpected"]
-)
-def test_preset_typos_still_warn(name, field, moa_home, capsys):
- prefix = "moa.presets." + name.replace(".", r"\.")
- set_config_value(f"{prefix}.{field}", "5")
- assert "not a recognized config key" in capsys.readouterr().out