Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

e2e: auto-route journeys with a stub decision service

agent-runner35382f56d687a16
diff --git a/core/tests/e2e/follow_the_conversation/harness.py b/core/tests/e2e/follow_the_conversation/harness.py
index 8f7d723d0..1a4bc58df 100644
--- a/core/tests/e2e/follow_the_conversation/harness.py
+++ b/core/tests/e2e/follow_the_conversation/harness.py
@@ -3,6 +3,7 @@
setup register test agents, a second Mattermost person, and the rooms
playground add a fresh room with all three agents, for trying it by hand
agents run the scripted agents (they reply "I got ..." when addressed)
+ decider run a stub decision service that picks an agent by keyword
run play every scenario as people in Mattermost and write a report
`run.sh` beside this file drives the whole thing against a fresh local stack;
@@ -26,6 +27,7 @@ import uuid
from pathlib import Path
import aiohttp
+from aiohttp import web
REPO = Path(__file__).resolve().parents[4]
HERE = Path(
@@ -36,6 +38,7 @@ RECEIVED = HERE / "received.jsonl"
SERVER_LOG = HERE / "switch-core.log"
SWITCH = "http://localhost:8000"
MM = "http://localhost:8065"
+DECIDER_PORT = int(os.environ.get("E2E_DECIDER_PORT", "8091"))
AGENTS = {
"helper": "Answers questions about the Switch repo and its local setup.",
"reviewer": "Reviews code changes and pull requests.",
@@ -174,10 +177,11 @@ async def setup() -> None:
"addressing": "owner_only",
},
)
- for name in ("helper", "reviewer"):
+ # An agent can't change its own settings, so each is changed by the other.
+ for name, by in (("helper", "reviewer"), ("reviewer", "helper")):
await op(
http,
- helper,
+ state["agents"][by],
"update_agent_detail",
{"agent_id": state["agents"][name]["id"], "addressing": "anyone"},
)
@@ -225,6 +229,7 @@ async def setup() -> None:
rooms = {
"follow-ups": ["helper", "reviewer", "private-bot"],
"main-agent": ["helper", "reviewer", "private-bot"],
+ "auto-route": ["helper", "reviewer", "private-bot"],
}
for key, members in rooms.items():
created = await op(
@@ -389,6 +394,8 @@ def why(message_id: str, name: str) -> str:
"""The server's own reason this agent got the message, read from its log."""
pattern = f"message={message_id} agent={name} "
for line in reversed(SERVER_LOG.read_text().splitlines()[-4000:]):
+ if "Decision routing:" in line and f"{pattern}outcome=delivered" in line:
+ return "via the decision service (nobody was tagged)"
if "Follow the conversation:" in line and pattern in line:
route = re.search(r"route=(\S+)", line).group(1)
reason = line.split("reason=", 1)[1]
@@ -434,6 +441,58 @@ async def agents() -> None:
await asyncio.gather(*(run_agent(n, a) for n, a in state["agents"].items()))
+# ── Stub decision service ───────────────────────────────────────────────────
+
+# Keywords to the agent the stub picks, first match wins. A real decision
+# service reads the room's context; the stub only needs to be predictable.
+DECIDER_RULES = [
+ (("review", "pull request", " pr "), "reviewer"),
+ (("setup", "set up", "run ", "tests", "just "), "helper"),
+]
+
+
+def stub_pick(text: str, candidates: list[str]) -> str | None:
+ lowered = f" {text.lower()} "
+ for words, agent in DECIDER_RULES:
+ if agent in candidates and any(word in lowered for word in words):
+ return agent
+ return None
+
+
+async def decider() -> None:
+ """Answer Switch's decision requests the way the real service does, by
+ keyword, and log each request so the report can show what Switch sent."""
+ log = HERE / "decider.jsonl"
+
+ async def decide(request: web.Request) -> web.Response:
+ body = await request.json()
+ candidates = [a["name"] for a in body["agents"]]
+ pick = stub_pick(body["message"]["text"], candidates)
+ with log.open("a") as f:
+ f.write(json.dumps({"request": body, "pick": pick}) + "\n")
+ return web.json_response(
+ {
+ "agent": pick,
+ "model_choice": pick or "none",
+ "confidence": 0.99 if pick else 0.9,
+ "threshold": 0.6,
+ "reason": "stub: keyword match" if pick else "stub: no keyword",
+ "model": "stub",
+ "latency_ms": 1.0,
+ }
+ )
+
+ app = web.Application()
+ app.router.add_post("/decide", decide)
+ runner = web.AppRunner(app)
+ await runner.setup()
+ await web.TCPSite(runner, "127.0.0.1", DECIDER_PORT).start()
+ print(
+ f"stub decision service on http://127.0.0.1:{DECIDER_PORT}/decide", flush=True
+ )
+ await asyncio.Event().wait()
+
+
# ── Scenarios ───────────────────────────────────────────────────────────────
# Each step: (who, text, where, expected agents that receive it as addressed).
@@ -625,6 +684,60 @@ SCENARIOS = [
("user", "and which tests are slow?", "thread:f1", {"helper"}, None),
],
},
+ {
+ "id": "S15",
+ "room": "follow-ups",
+ "title": "Without !auto-route, an untagged question still reaches nobody",
+ "steps": [("bob", "how do I run the local setup?", "top", set(), None)],
+ },
+ {
+ "id": "S16",
+ "room": "auto-route",
+ "title": "!auto-route on, then an untagged question reaches the agent the decision service picks",
+ "steps": [
+ ("cmd", "user", "!auto-route on", "Automatic routing is **on**"),
+ (
+ "user",
+ "how do I run the local setup?",
+ "top",
+ {"helper"},
+ "answer:helper=r1",
+ ),
+ ],
+ },
+ {
+ "id": "S17",
+ "room": "auto-route",
+ "title": '"thanks" and people talking to each other reach nobody',
+ "steps": [
+ ("user", "thanks!", "top", set(), None),
+ ("bob", "lunch at 1 anyone?", "top", set(), None),
+ ],
+ },
+ {
+ "id": "S18",
+ "room": "auto-route",
+ "title": "In an auto-route room, the decision service decides thread replies, not the last agent to speak",
+ "steps": [
+ (
+ "user",
+ "can you review my change to the readme?",
+ "thread:r1",
+ {"reviewer"},
+ "answer:reviewer",
+ ),
+ ("user", "and how do I run the tests?", "thread:r1", {"helper"}, None),
+ ],
+ },
+ {
+ "id": "S19",
+ "room": "auto-route",
+ "title": "!auto-route off: untagged questions reach nobody again",
+ "steps": [
+ ("cmd", "user", "!auto-route off", "Automatic routing is **off**"),
+ ("bob", "how do I set up the database?", "top", set(), None),
+ ],
+ },
]
@@ -765,7 +878,9 @@ async def run() -> None:
def write_report(results, events, log_text, transcripts, state) -> int:
"""Write report.md and return how many scenarios failed."""
ftc_lines = [
- line for line in log_text.splitlines() if "Follow the conversation:" in line
+ line
+ for line in log_text.splitlines()
+ if "Follow the conversation:" in line or "Decision routing:" in line
]
out = ["# Follow the conversation: end-to-end proof", ""]
out.append(
@@ -816,7 +931,12 @@ def write_report(results, events, log_text, transcripts, state) -> int:
for line in ftc_lines:
if any(f"message={mid} " in line for mid in ids):
logs_for_sc.append(
- f"step {i}: " + line.split("Follow the conversation: ", 1)[1]
+ f"step {i}: "
+ + re.split(
+ r"(?:Follow the conversation|Decision routing): ",
+ line,
+ maxsplit=1,
+ )[1]
)
body.append("")
body.append("Server log lines for these messages:")
@@ -860,7 +980,11 @@ def write_report(results, events, log_text, transcripts, state) -> int:
if __name__ == "__main__":
asyncio.run(
- {"setup": setup, "agents": agents, "run": run, "playground": playground}[
- sys.argv[1]
- ]()
+ {
+ "setup": setup,
+ "agents": agents,
+ "decider": decider,
+ "run": run,
+ "playground": playground,
+ }[sys.argv[1]]()
)
diff --git a/core/tests/e2e/follow_the_conversation/run.sh b/core/tests/e2e/follow_the_conversation/run.sh
index 2755b8c0c..21dbed955 100755
--- a/core/tests/e2e/follow_the_conversation/run.sh
+++ b/core/tests/e2e/follow_the_conversation/run.sh
@@ -5,6 +5,10 @@
# just e2e-follow-ups --video the same, plus a captioned demo video (needs ffmpeg)
# just e2e-follow-ups --keep leave the stack and a playground room running afterwards
#
+# Automatic routing (!auto-route) asks a stub decision service on :8091 that
+# picks an agent by keyword. Set E2E_DECISION_URL to a real one's /decide
+# endpoint to play the same journeys against it instead.
+#
# It starts its own Postgres and Mattermost (Docker Compose project
# "switch-e2e") and its own switch-core on :8000, so stop your local stack
# first (`just down`, and any `just run`). Everything it writes goes to
@@ -27,7 +31,10 @@ done
cd "$REPO"
[ -f .env ] || { echo "No .env: run 'just init-env' first." >&2; exit 1; }
-for port in 5432 8000 8065 8081; do
+DECISION_URL="${E2E_DECISION_URL:-}"
+PORTS=(5432 8000 8065 8081)
+[ -z "$DECISION_URL" ] && PORTS+=(8091)
+for port in "${PORTS[@]}"; do
if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
echo "Port $port is in use. Stop your local stack first ('just down', and any 'just run')." >&2
exit 1
@@ -42,18 +49,21 @@ set -a; source .env; set +a
COMPOSE=(docker compose -p switch-e2e -f deploy/local/docker-compose.yml --project-directory .)
SERVER_PID=""
AGENTS_PID=""
+DECIDER_PID=""
stop_server() { [ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null && wait "$SERVER_PID" 2>/dev/null; SERVER_PID=""; return 0; }
stop_agents() { [ -n "$AGENTS_PID" ] && kill "$AGENTS_PID" 2>/dev/null; AGENTS_PID=""; return 0; }
+stop_decider() { [ -n "$DECIDER_PID" ] && kill "$DECIDER_PID" 2>/dev/null; DECIDER_PID=""; return 0; }
cleanup() {
if [ "$KEEP" = 0 ]; then
- stop_agents; stop_server
+ stop_agents; stop_server; stop_decider
"${COMPOSE[@]}" down -v >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
start_server() { # $1 = follow-up window in seconds
- FEATURE_FLAGS_ENABLED=follow_the_conversation \
+ FEATURE_FLAGS_ENABLED=follow_the_conversation,decision_routing \
+ DECISION_ENDPOINT_URL="${DECISION_URL:-http://127.0.0.1:8091/decide}" \
FOLLOW_THE_CONVERSATION_WINDOW_SECONDS="$1" \
GATEWAY_COOKIE_SECURE=false \
uv run --project core python -m switch_core.main >> "$OUT/switch-core.log" 2>&1 &
@@ -69,6 +79,10 @@ start_agents() {
}
rm -rf "$OUT"; mkdir -p "$OUT"
+if [ -z "$DECISION_URL" ]; then
+ uv run --project core python "$HERE/harness.py" decider > "$OUT/decider.log" 2>&1 &
+ DECIDER_PID=$!
+fi
echo "Starting a fresh stack (Postgres, Mattermost)…"
"${COMPOSE[@]}" down -v >/dev/null 2>&1 || true
"${COMPOSE[@]}" up -d --build >/dev/null