Skip to content

Your lightweight open-source personal agent.

feat(heartbeat): custom evaluator prompt

Arthur K.dcb37259fad4
diff --git a/docs/configuration.md b/docs/configuration.md
index a23d9c1f5..9f7d9ce94 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2031,6 +2031,10 @@ The heartbeat job is backed by the same cron service as user-created reminders.
| `gateway.heartbeat.keepRecentMessages` | `8` | Number of recent heartbeat-session messages to retain after each run. |
| `gateway.restartMode` | `auto` | Restart strategy for `/restart`: `auto` uses `spawn` on Windows foreground runs and `exec` elsewhere. Use `exit` with Windows service wrappers such as WinSW or nssm so the service manager owns the restart. |
+### Custom heartbeat evaluator prompt
+
+The notification gate runs on a built-in system prompt. Advanced users can override it, but you rarely need to — it's strongly advised to first read the evaluator code and the default `evaluator.md`. To override, drop your prompt at `<workspace>/prompts/evaluator.md`. It must still instruct the model to call the `evaluate_notification` tool; otherwise the gate fails closed and stays silent.
+
## Subagent Concurrency
diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py
index c1df2c91c..f15aac43a 100644
--- a/nanobot/cli/commands.py
+++ b/nanobot/cli/commands.py
@@ -77,7 +77,7 @@ from nanobot.cli.stream import StreamRenderer, ThinkingSpinner # noqa: E402
from nanobot.config.paths import get_workspace_path, is_default_workspace # noqa: E402
from nanobot.config.schema import Config # noqa: E402
from nanobot.security.network import is_loopback_host # noqa: E402
-from nanobot.utils.evaluator import evaluate_response # noqa: E402
+from nanobot.utils.evaluator import evaluate_response, resolve_evaluator_prompt # noqa: E402
from nanobot.utils.helpers import sync_workspace_templates # noqa: E402
from nanobot.utils.restart import ( # noqa: E402
consume_restart_notice_from_env,
@@ -1859,21 +1859,29 @@ def _run_gateway(
finally:
if isinstance(message_tool, MessageTool) and suppress_token is not None:
message_tool.reset_suppress_delivery(suppress_token)
- response = resp.content if resp else ""
+
+ if not resp or not resp.content:
+ return
+
+ response = resp.content
# Keep a small tail of heartbeat history so the loop stays bounded.
session = agent.sessions.get_or_create("heartbeat")
session.retain_recent_legal_suffix(hb_cfg.keep_recent_messages)
agent.sessions.save(session)
- if not response:
- return None
+ evaluator_prompt = resolve_evaluator_prompt(config.workspace_path)
# Fail closed: stay silent on evaluator failure instead of notifying.
should_notify = await evaluate_response(
- response, prompt, agent.provider, agent.model,
+ response=response,
+ task_context=prompt,
+ provider=agent.provider,
+ model=agent.model,
+ evaluator_prompt=evaluator_prompt,
default_notify=False,
)
+
if should_notify:
logger.info("Heartbeat: completed, delivering response")
await _deliver_to_channel(
diff --git a/nanobot/command/builtin.py b/nanobot/command/builtin.py
index e2320df60..534da6d65 100644
--- a/nanobot/command/builtin.py
+++ b/nanobot/command/builtin.py
@@ -143,6 +143,14 @@ BUILTIN_COMMAND_SPECS: tuple[BuiltinCommandSpec, ...] = (
"[init]",
accepts_args=True,
),
+ BuiltinCommandSpec(
+ "/evaluator-prompt",
+ "Heartbeat evaluator",
+ "Customize the heartbeat notification gate prompt for this workspace.",
+ "file-text",
+ "[init]",
+ accepts_args=True,
+ ),
BuiltinCommandSpec(
"/skill",
"List skills",
@@ -515,6 +523,64 @@ async def cmd_dream_prompt(ctx: CommandContext) -> OutboundMessage:
)
+async def cmd_evaluator_prompt(ctx: CommandContext) -> OutboundMessage:
+ """Show or set up the workspace heartbeat evaluator prompt."""
+ from nanobot.utils.evaluator import (
+ default_evaluator_prompt,
+ evaluator_prompt_file,
+ has_evaluator_prompt_override,
+ )
+
+ workspace = ctx.loop.context.memory.workspace
+ path = evaluator_prompt_file(workspace)
+ display_path = path.relative_to(workspace).as_posix()
+ args = ctx.args.strip().lower()
+
+ if args == "init":
+ try:
+ prompt_exists_with_content = path.exists() and (
+ not path.is_file() or bool(path.read_text(encoding="utf-8").strip())
+ )
+ except OSError:
+ prompt_exists_with_content = True
+ if prompt_exists_with_content:
+ content = (
+ f"Heartbeat evaluator prompt already exists at `{display_path}`.\n\n"
+ "Edit that file, or delete/empty it to return to nanobot's default."
+ )
+ else:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(default_evaluator_prompt() + "\n", encoding="utf-8")
+ content = (
+ f"Created heartbeat evaluator prompt at `{display_path}`.\n\n"
+ "Edit that file to control when the heartbeat notification gate speaks. "
+ "It must still instruct the model to call the `evaluate_notification` tool, "
+ "otherwise the gate fails closed and stays silent. "
+ "Delete or empty it to return to nanobot's default."
+ )
+ elif args:
+ content = "Usage: /evaluator-prompt [init]"
+ elif has_evaluator_prompt_override(workspace):
+ content = (
+ "Heartbeat evaluator prompt: custom for this workspace\n\n"
+ f"- Path: `{display_path}`\n"
+ "- Delete or empty this file to return to nanobot's default."
+ )
+ else:
+ content = (
+ "Heartbeat evaluator prompt: nanobot default\n\n"
+ f"- Editable file: `{display_path}`\n"
+ "- Run `/evaluator-prompt init` to create an editable copy."
+ )
+
+ return OutboundMessage(
+ channel=ctx.msg.channel,
+ chat_id=ctx.msg.chat_id,
+ content=content,
+ metadata={**dict(ctx.msg.metadata or {}), "render_as": "text"},
+ )
+
+
def _format_dream_no_input_message() -> str:
return "\n".join([
"Dream has no conversation history to process yet.",
@@ -954,6 +1020,8 @@ def register_builtin_commands(router: CommandRouter) -> None:
router.prefix("/dream-restore ", cmd_dream_restore)
router.exact("/dream-prompt", cmd_dream_prompt)
router.prefix("/dream-prompt ", cmd_dream_prompt)
+ router.exact("/evaluator-prompt", cmd_evaluator_prompt)
+ router.prefix("/evaluator-prompt ", cmd_evaluator_prompt)
router.exact("/skill", cmd_skill)
router.exact("/help", cmd_help)
router.exact("/pairing", cmd_pairing)
diff --git a/nanobot/templates/prompts/README.md b/nanobot/templates/prompts/README.md
index 0084d1f0c..b141127b0 100644
--- a/nanobot/templates/prompts/README.md
+++ b/nanobot/templates/prompts/README.md
@@ -1,11 +1,25 @@
-# Dream Memory Instructions
+# Prompt Overrides
-This folder is for plain-language instructions that tell Dream how to organize memory in this workspace.
+This folder holds plain-language prompt overrides for this workspace.
-Most users do not need to edit anything here. To guide Dream differently for this workspace, run:
+## Dream memory
+
+`dream.md` tells Dream how to organize memory in this workspace. Most users do not need to touch it. To create an editable copy, run:
```text
/dream-prompt init
```
That creates `prompts/dream.md`. Edit it in plain Markdown. Delete or empty it to return to nanobot's default memory behavior.
+
+## Heartbeat evaluator
+
+`evaluator.md` overrides the system prompt for the heartbeat notification gate — the model that decides whether a heartbeat result is worth delivering. This is an advanced override; you rarely need it. Before editing, read the evaluator code and the default `evaluator.md`.
+
+To create an editable copy, run:
+
+```text
+/evaluator-prompt init
+```
+
+That creates `prompts/evaluator.md`. It must still instruct the model to call the `evaluate_notification` tool; otherwise the gate fails closed and stays silent. Delete or empty the file to return to the built-in prompt.
diff --git a/nanobot/utils/evaluator.py b/nanobot/utils/evaluator.py
index 07206f991..ac4dc1637 100644
--- a/nanobot/utils/evaluator.py
+++ b/nanobot/utils/evaluator.py
@@ -6,15 +6,58 @@ LLM call to decide whether the result warrants notifying the user.
from __future__ import annotations
+from contextlib import suppress
+from pathlib import Path
from typing import TYPE_CHECKING
from loguru import logger
+from nanobot.utils.helpers import truncate_text
from nanobot.utils.prompt_templates import render_template
if TYPE_CHECKING:
from nanobot.providers.base import LLMProvider
+# Cap for a workspace-local heartbeat evaluator prompt override.
+EVALUATOR_PROMPT_MAX_CHARS = 32_000
+
+
+def evaluator_prompt_file(workspace: Path) -> Path:
+ """Path to the workspace-local heartbeat evaluator prompt override."""
+ return workspace / "prompts" / "evaluator.md"
+
+
+def has_evaluator_prompt_override(workspace: Path) -> bool:
+ """True when the workspace defines a non-empty evaluator prompt override."""
+ with suppress(OSError):
+ path = evaluator_prompt_file(workspace)
+ return path.is_file() and bool(path.read_text(encoding="utf-8").strip())
+ return False
+
+
+def default_evaluator_prompt() -> str:
+ """The built-in heartbeat notification-gate system prompt."""
+ return render_template("agent/evaluator.md", part="system", strip=True)
+
+
+def resolve_evaluator_prompt(workspace: Path) -> str:
+ """Return the active evaluator prompt: workspace override or built-in default.
+
+ Oversized overrides are truncated so a runaway file cannot blow up the
+ evaluator call.
+ """
+ with suppress(OSError):
+ text = evaluator_prompt_file(workspace).read_text(encoding="utf-8").rstrip()
+ if text:
+ if len(text) > EVALUATOR_PROMPT_MAX_CHARS:
+ logger.warning(
+ "Workspace heartbeat evaluator prompt exceeds {} chars ({}); truncating.",
+ EVALUATOR_PROMPT_MAX_CHARS, len(text),
+ )
+ return truncate_text(text, EVALUATOR_PROMPT_MAX_CHARS)
+ return text
+ return default_evaluator_prompt()
+
_EVALUATE_TOOL = [
{
"type": "function",
@@ -44,17 +87,19 @@ async def evaluate_response(
task_context: str,
provider: LLMProvider,
model: str,
- default_notify: bool = True,
+ evaluator_prompt: str,
+ default_notify: bool = False,
) -> bool:
"""Decide whether a heartbeat result should be delivered to the user.
On any failure, falls back to ``default_notify``. Heartbeat passes
``False`` to fail closed.
"""
+
try:
llm_response = await provider.chat_with_retry(
messages=[
- {"role": "system", "content": render_template("agent/evaluator.md", part="system")},
+ {"role": "system", "content": evaluator_prompt},
{"role": "user", "content": render_template(
"agent/evaluator.md",
part="user",
@@ -64,7 +109,7 @@ async def evaluate_response(
],
tools=_EVALUATE_TOOL,
model=model,
- max_tokens=256,
+ max_tokens=4096,
temperature=0.0,
)
diff --git a/tests/agent/test_evaluator.py b/tests/agent/test_evaluator.py
index acefe40e1..faf375220 100644
--- a/tests/agent/test_evaluator.py
+++ b/tests/agent/test_evaluator.py
@@ -31,17 +31,26 @@ def _eval_tool_call(should_notify: bool, reason: str = "") -> LLMResponse:
)
+_EVAL_PROMPT = "You are a notification gate. Call evaluate_notification."
+
+
@pytest.mark.asyncio
async def test_should_notify_true() -> None:
provider = DummyProvider([_eval_tool_call(True, "user asked to be reminded")])
- result = await evaluate_response("Task completed with results", "check emails", provider, "m")
+ result = await evaluate_response(
+ "Task completed with results", "check emails", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT,
+ )
assert result is True
@pytest.mark.asyncio
async def test_should_notify_false() -> None:
provider = DummyProvider([_eval_tool_call(False, "routine check, nothing new")])
- result = await evaluate_response("All clear, no updates", "check status", provider, "m")
+ result = await evaluate_response(
+ "All clear, no updates", "check status", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT,
+ )
assert result is False
@@ -52,14 +61,20 @@ async def test_fallback_on_error() -> None:
raise RuntimeError("provider down")
provider = FailingProvider([])
- result = await evaluate_response("some response", "some task", provider, "m")
+ result = await evaluate_response(
+ "some response", "some task", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT, default_notify=True,
+ )
assert result is True
@pytest.mark.asyncio
async def test_no_tool_call_fallback() -> None:
provider = DummyProvider([LLMResponse(content="I think you should notify", tool_calls=[])])
- result = await evaluate_response("some response", "some task", provider, "m")
+ result = await evaluate_response(
+ "some response", "some task", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT, default_notify=True,
+ )
assert result is True
@@ -70,12 +85,18 @@ async def test_fail_closed_on_error() -> None:
raise RuntimeError("provider down")
provider = FailingProvider([])
- result = await evaluate_response("some", "task", provider, "m", default_notify=False)
+ result = await evaluate_response(
+ "some", "task", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT, default_notify=False,
+ )
assert result is False
@pytest.mark.asyncio
async def test_fail_closed_on_no_tool_call() -> None:
provider = DummyProvider([LLMResponse(content="text only", tool_calls=[])])
- result = await evaluate_response("some", "task", provider, "m", default_notify=False)
+ result = await evaluate_response(
+ "some", "task", provider, "m",
+ evaluator_prompt=_EVAL_PROMPT, default_notify=False,
+ )
assert result is False
diff --git a/tests/command/test_builtin_evaluator_prompt.py b/tests/command/test_builtin_evaluator_prompt.py
new file mode 100644
index 000000000..76f91da37
--- /dev/null
+++ b/tests/command/test_builtin_evaluator_prompt.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from nanobot.bus.events import InboundMessage
+from nanobot.command.builtin import (
+ build_help_text,
+ builtin_command_palette,
+ cmd_evaluator_prompt,
+)
+from nanobot.command.router import CommandContext
+from nanobot.utils.evaluator import default_evaluator_prompt
+
+
+def _make_ctx(tmp_path, raw: str = "/evaluator-prompt", args: str = "") -> CommandContext:
+ msg = InboundMessage(channel="cli", sender_id="u1", chat_id="direct", content=raw)
+ loop = SimpleNamespace(context=SimpleNamespace(memory=SimpleNamespace(workspace=tmp_path)))
+ return CommandContext(msg=msg, session=None, key=msg.session_key, raw=raw, args=args, loop=loop)
+
+
+async def test_evaluator_prompt_reports_default_prompt(tmp_path) -> None:
+ out = await cmd_evaluator_prompt(_make_ctx(tmp_path))
+
+ assert "Heartbeat evaluator prompt: nanobot default" in out.content
+ assert "prompts/evaluator.md" in out.content
+ assert str(tmp_path) not in out.content
+ assert "/evaluator-prompt init" in out.content
+
+
+async def test_evaluator_prompt_init_copies_default_prompt(tmp_path) -> None:
+ ctx = _make_ctx(tmp_path, "/evaluator-prompt init", "init")
+
+ out = await cmd_evaluator_prompt(ctx)
+
+ prompt_file = tmp_path / "prompts" / "evaluator.md"
+ assert "Created heartbeat evaluator prompt" in out.content
+ assert "prompts/evaluator.md" in out.content
+ assert str(tmp_path) not in out.content
+ assert "evaluate_notification" in out.content
+ assert prompt_file.read_text(encoding="utf-8") == default_evaluator_prompt() + "\n"
+
+
+async def test_evaluator_prompt_init_does_not_overwrite_existing_prompt(tmp_path) -> None:
+ prompt_file = tmp_path / "prompts" / "evaluator.md"
+ prompt_file.parent.mkdir()
+ prompt_file.write_text("custom", encoding="utf-8")
+ ctx = _make_ctx(tmp_path, "/evaluator-prompt init", "init")
+
+ out = await cmd_evaluator_prompt(ctx)
+
+ assert "already exists" in out.content
+ assert "prompts/evaluator.md" in out.content
+ assert str(tmp_path) not in out.content
+ assert prompt_file.read_text(encoding="utf-8") == "custom"
+
+
+async def test_evaluator_prompt_init_recreates_empty_prompt(tmp_path) -> None:
+ prompt_file = tmp_path / "prompts" / "evaluator.md"
+ prompt_file.parent.mkdir()
+ prompt_file.write_text(" \n", encoding="utf-8")
+ ctx = _make_ctx(tmp_path, "/evaluator-prompt init", "init")
+
+ out = await cmd_evaluator_prompt(ctx)
+
+ assert "Created heartbeat evaluator prompt" in out.content
+ assert prompt_file.read_text(encoding="utf-8") == default_evaluator_prompt() + "\n"
+
+
+async def test_evaluator_prompt_reports_override(tmp_path) -> None:
+ prompt_file = tmp_path / "prompts" / "evaluator.md"
+ prompt_file.parent.mkdir()
+ prompt_file.write_text("custom", encoding="utf-8")
+
+ out = await cmd_evaluator_prompt(_make_ctx(tmp_path))
+
+ assert "Heartbeat evaluator prompt: custom for this workspace" in out.content
+ assert "prompts/evaluator.md" in out.content
+
+
+async def test_evaluator_prompt_rejects_unknown_args(tmp_path) -> None:
+ out = await cmd_evaluator_prompt(_make_ctx(tmp_path, "/evaluator-prompt nope", "nope"))
+
+ assert out.content == "Usage: /evaluator-prompt [init]"
+
+
+def test_evaluator_prompt_command_in_help_and_palette() -> None:
+ palette = builtin_command_palette()
+ entry = next(item for item in palette if item["command"] == "/evaluator-prompt")
+
+ assert entry["arg_hint"] == "[init]"
+ assert entry["lifecycle"] == "side_channel"
+ assert entry["accepts_args"] is True
+ assert "/evaluator-prompt [init]" in build_help_text()