Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(locomo): restrict built-in evaluation tools to OpenViking retrieval (#5761)

Jiajie - He/him/his76511554eee2
diff --git a/benchmark/locomo/vikingbot/run_eval.py b/benchmark/locomo/vikingbot/run_eval.py
index 79cf54c7c..59362b6cb 100644
--- a/benchmark/locomo/vikingbot/run_eval.py
+++ b/benchmark/locomo/vikingbot/run_eval.py
@@ -255,10 +255,18 @@ def run_vikingbot_chat(
env = os.environ.copy()
if bot_log_file:
env["VIKINGBOT_LOG_FILE"] = bot_log_file
+ # LoCoMo disables general-purpose tools and OpenViking writes and exports.
+ chat_cmd = ["vikingbot", "chat"]
+ for tool_name in (
+ "web_search", "web_fetch", "exec", "read_file", "write_file", "edit_file",
+ "list_dir", "spawn", "message", "generate_image", "openviking_export",
+ "openviking_memory_commit", "openviking_add_resource",
+ ):
+ chat_cmd.extend(["--disable-tool", tool_name])
# 先执行 /new 命令清除会话
if sender_peer_id:
- new_cmd = ["vikingbot", "chat"]
+ new_cmd = chat_cmd.copy()
if config:
new_cmd.extend(["--config", config])
new_cmd.extend(
@@ -288,7 +296,7 @@ def run_vikingbot_chat(
else:
input = f"Answer the question directly: {question}"
- cmd = ["vikingbot", "chat"]
+ cmd = chat_cmd.copy()
if config:
cmd.extend(["--config", config])
cmd.extend(["-m", input, "-e"])
diff --git a/bot/vikingbot/channels/chat.py b/bot/vikingbot/channels/chat.py
index 4989f006a..566450698 100644
--- a/bot/vikingbot/channels/chat.py
+++ b/bot/vikingbot/channels/chat.py
@@ -47,12 +47,14 @@ class ChatChannel(BaseChannel):
markdown: bool = True,
logs: bool = False,
sender: str | None = None,
+ disabled_tools: list[str] | None = None,
):
super().__init__(config, bus, workspace_path)
self.session_id = session_id
self.markdown = markdown
self.logs = logs
self.sender = sender
+ self.disabled_tools = disabled_tools or []
self._response_received = asyncio.Event()
self._last_response: str | None = None
@@ -156,7 +158,7 @@ class ChatChannel(BaseChannel):
self._last_response = None
sender_id = self.sender or "user"
- metadata = {}
+ metadata = {"disabled_tools": self.disabled_tools}
memory_peers = getattr(self.config, "memory_peer", None)
if memory_peers:
metadata["memory_peers"] = memory_peers
diff --git a/bot/vikingbot/channels/single_turn.py b/bot/vikingbot/channels/single_turn.py
index 129b09451..f0770312c 100644
--- a/bot/vikingbot/channels/single_turn.py
+++ b/bot/vikingbot/channels/single_turn.py
@@ -46,12 +46,14 @@ class SingleTurnChannel(BaseChannel):
markdown: bool = True,
eval: bool = False,
sender: str | None = None,
+ disabled_tools: list[str] | None = None,
):
super().__init__(config, bus, workspace_path)
self.message = message
self.session_id = session_id
self.markdown = markdown
self.sender = sender
+ self.disabled_tools = disabled_tools or []
self._response_received = asyncio.Event()
self._last_response: str | None = None
self._eval = eval
@@ -62,7 +64,7 @@ class SingleTurnChannel(BaseChannel):
# Send the message
sender_id = self.sender or "user"
- metadata = {}
+ metadata = {"disabled_tools": self.disabled_tools}
memory_peers = getattr(self.config, "memory_peer", None)
if memory_peers:
metadata["memory_peers"] = memory_peers
diff --git a/bot/vikingbot/cli/commands.py b/bot/vikingbot/cli/commands.py
index 5f6232375..c4f62cc38 100644
--- a/bot/vikingbot/cli/commands.py
+++ b/bot/vikingbot/cli/commands.py
@@ -10,7 +10,7 @@ import sys
import time
import warnings
from pathlib import Path
-from typing import Any, Optional
+from typing import Annotated, Any, Optional
# --- stdout-safety guard (must run before any openviking/vikingbot imports) ---
@@ -882,6 +882,7 @@ def prepare_agent_channel(
sender: str | None = None,
memory_peer: list[str] | None = None,
memory_user: list[str] | None = None,
+ disabled_tools: list[str] | None = None,
):
"""Prepare channel for agent command."""
from vikingbot.channels.chat import ChatChannel, ChatChannelConfig
@@ -903,6 +904,7 @@ def prepare_agent_channel(
markdown=markdown,
eval=eval,
sender=sender,
+ disabled_tools=disabled_tools,
)
channels.add_channel(channel)
else:
@@ -919,6 +921,7 @@ def prepare_agent_channel(
markdown=markdown,
logs=logs,
sender=sender,
+ disabled_tools=disabled_tools,
)
channels.add_channel(channel)
@@ -952,6 +955,7 @@ def chat(
"--memory-user",
help="Deprecated legacy OpenViking user ID for root-key memory fanout",
),
+ disabled_tools: Annotated[list[str] | None, typer.Option("--disable-tool", help="Disable a tool (repeatable)")] = None,
):
"""Interact with the agent directly."""
path = Path(config_path).expanduser() if config_path is not None else None
@@ -1011,6 +1015,7 @@ def chat(
sender,
memory_peer,
memory_user,
+ disabled_tools=disabled_tools,
)
agent_loop = prepare_agent_loop(
config, bus, session_manager, cron, quiet=is_single_turn, eval=eval