Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(agents): never re-ask an answered question, and mark defaults as not chosen (#1036)

Ruben Lütke53847c2e5eb5
diff --git a/dodaios/agent_chat/questions.py b/dodaios/agent_chat/questions.py
index 1140db3033..fa562289d1 100644
--- a/dodaios/agent_chat/questions.py
+++ b/dodaios/agent_chat/questions.py
@@ -27,6 +27,12 @@ the recommended option(s), or an empty answer for a ``text`` question with no
options. The person may also skip one question (``declined``), answer from
the chat composer (``message``), or move on so that older open questions
fall back to their recommendations (``moved_on``).
+
+A question is never put to the person twice. When the agent asks again what
+the person already answered (the same question with the same options) in this
+turn or within ``RECALL_WINDOW_S``, that answer comes back without a new card
+(``recalled``), and a fallback (timeout, skip) never replaces it. A fallback
+answer always tells the agent that the person did not choose it.
"""
from __future__ import annotations
@@ -60,6 +66,13 @@ DECLINED: Final[str] = "declined"
MESSAGE: Final[str] = "message"
#: The person sent a message instead of answering: the recommendation applies.
MOVED_ON: Final[str] = "moved_on"
+#: The person answered this same question earlier; no new card was shown.
+RECALLED: Final[str] = "recalled"
+#: Sources where nobody picked the answer: the agent's own recommendation applies.
+DEFAULTED: Final[frozenset[str]] = frozenset({TIMEOUT, SKIPPED, MOVED_ON, UNATTENDED})
+
+#: How long an answer from an earlier turn still answers the same question.
+RECALL_WINDOW_S: Final[float] = 3600.0
#: Question kinds.
SINGLE: Final[str] = "single"
@@ -149,7 +162,7 @@ class QuestionAnswer:
@property
def by_person(self) -> bool:
- return self.source in (PERSON, MESSAGE)
+ return self.source in (PERSON, MESSAGE, RECALLED)
def to_payload(self) -> dict[str, Any]:
payload: dict[str, Any] = {
@@ -378,6 +391,106 @@ def answer_json(spec: QuestionSpec, raw: Any) -> QuestionAnswer:
)
+def same_question(spec: QuestionSpec) -> tuple[str, str, frozenset[str]]:
+ """What makes two questions the same: their wording, kind and options."""
+ wording = " ".join(spec.question.casefold().split()).rstrip(" ?!.:")
+ return wording, spec.kind, frozenset(o.label.casefold() for o in spec.options)
+
+
+def recalled(
+ spec: QuestionSpec, earlier: QuestionSpec, answer: QuestionAnswer
+) -> QuestionAnswer:
+ """The person's earlier ``answer`` to ``earlier``, as an answer to ``spec``.
+
+ The options may stand in another order (the recommendation leads), so a
+ picked option is found again by its label.
+ """
+ index = {o.label.casefold(): i for i, o in enumerate(spec.options)}
+
+ def moved(old: int) -> int | None:
+ if not 0 <= old < len(earlier.options):
+ return None
+ return index.get(earlier.options[old].label.casefold())
+
+ picks = tuple(sorted(p for p in (moved(i) for i in answer.option_indices) if p is not None))
+ option = moved(answer.option_index) if answer.option_index is not None else None
+ return QuestionAnswer(answer.answer, option, RECALLED, picks)
+
+
+def answer_from_payload(row: Any) -> QuestionAnswer | None:
+ """One stored answer (``QuestionAnswer.to_payload``) read back, or ``None``."""
+ if not isinstance(row, dict):
+ return None
+ option = row.get("option_index")
+ picks = row.get("option_indices")
+ return QuestionAnswer(
+ str(row.get("answer") or ""),
+ option if isinstance(option, int) and not isinstance(option, bool) else None,
+ str(row.get("source") or PERSON),
+ tuple(i for i in picks if isinstance(i, int)) if isinstance(picks, list) else (),
+ )
+
+
+Answered = dict[tuple[str, str, frozenset[str]], tuple[QuestionSpec, QuestionAnswer]]
+
+
+def note_answers(
+ found: Answered,
+ specs: tuple[QuestionSpec, ...] | list[QuestionSpec],
+ answers: list[QuestionAnswer | None] | tuple[QuestionAnswer | None, ...],
+) -> None:
+ """Add the person's own answers among ``answers`` to ``found`` (newest wins)."""
+ for spec, answer in zip(specs, answers, strict=False):
+ if answer is not None and answer.by_person and (answer.answer or answer.option_indices):
+ found[same_question(spec)] = (spec, answer)
+
+
+def person_answers(events: list[dict[str, Any]], *, turn_id: str, since_ms: int) -> Answered:
+ """The person's answers on the cards in ``events``: every card of ``turn_id``
+ and every other card opened at or after ``since_ms``."""
+ cards: dict[str, tuple[QuestionSpec, ...]] = {}
+ found: Answered = {}
+ for event in events:
+ kind = event.get("kind")
+ payload = event.get("payload") or {}
+ question_id = str(payload.get("question_id") or "")
+ if kind == "question_required":
+ if payload.get("turn_id") != turn_id and int(event.get("ts_ms") or 0) < since_ms:
+ continue
+ try:
+ cards[question_id] = parse_questions({"questions": payload.get("questions") or []})
+ except ValueError: # a corrupt stored card answers nothing
+ continue
+ elif kind in ("question_progress", "question_resolved") and question_id in cards:
+ rows = payload.get("answers")
+ if isinstance(rows, list):
+ note_answers(found, cards[question_id], [answer_from_payload(r) for r in rows])
+ return found
+
+
+def recall(
+ specs: tuple[QuestionSpec, ...], found: Answered
+) -> list[QuestionAnswer | None]:
+ """Per question, the person's earlier answer to the same question, else ``None``."""
+ out: list[QuestionAnswer | None] = []
+ for spec in specs:
+ hit = found.get(same_question(spec))
+ out.append(recalled(spec, hit[0], hit[1]) if hit else None)
+ return out
+
+
+def for_agent(answer: QuestionAnswer) -> dict[str, Any]:
+ """What the agent must know about who chose ``answer``."""
+ if answer.source in DEFAULTED:
+ return {
+ "person_answered": False,
+ "note": "The person did not answer; this is your default. Never say they chose it.",
+ }
+ if answer.source == RECALLED:
+ return {"person_answered": True, "note": "The person already answered this earlier."}
+ return {}
+
+
def as_line(answer: str) -> str:
"""A ``multi`` answer on one line, for a sentence that lists several answers."""
return ", ".join(line.strip() for line in answer.splitlines() if line.strip())
@@ -386,6 +499,7 @@ def as_line(answer: str) -> str:
__all__ = [
"CANCELLED",
"DECLINED",
+ "DEFAULTED",
"KINDS",
"MESSAGE",
"MOVED_ON",
@@ -398,6 +512,8 @@ __all__ = [
"MIN_OPTIONS",
"PERSON",
"QUESTION_TIMEOUT_S",
+ "RECALLED",
+ "RECALL_WINDOW_S",
"SKIPPED",
"TIMEOUT",
"UNATTENDED",
@@ -406,9 +522,16 @@ __all__ = [
"QuestionSpec",
"TooManyQuestions",
"answer_from",
+ "answer_from_payload",
"answer_json",
"as_line",
"declined_answer",
+ "for_agent",
+ "note_answers",
"parse_questions",
+ "person_answers",
+ "recall",
+ "recalled",
"recommended_answer",
+ "same_question",
]
diff --git a/dodaios/agent_chat/runner_cli.py b/dodaios/agent_chat/runner_cli.py
index bdbcdc9a5b..6b9f0b05be 100644
--- a/dodaios/agent_chat/runner_cli.py
+++ b/dodaios/agent_chat/runner_cli.py
@@ -85,6 +85,7 @@ from dodaios.agent_chat.events import make_event
from dodaios.agent_chat.permissions import default_permission, normalize_permission
from dodaios.agent_chat.questions import (
CANCELLED,
+ DEFAULTED,
UNATTENDED,
TooManyQuestions,
as_line,
@@ -3356,11 +3357,16 @@ def _keep_going_prompt(
*,
receipt_hint: str | None = None,
) -> str:
- err = (error or "the last tool call failed").strip()[:500]
review = f"{receipt_hint}\n\n" if receipt_hint else ""
+ # Only a real error names the last call; a receipt hint alone is about an
+ # earlier call that no later call settled.
+ head = (
+ f"The last tool call was cancelled or failed:\n{error.strip()[:500]}"
+ if error and error.strip()
+ else "An earlier tool call failed and no later call settled it."
+ )
return (
- "The last tool call was cancelled or failed:\n"
- f"{err}\n\n"
+ f"{head}\n\n"
f"{KEEP_GOING_ON_TOOL_FAILURE}\n\n"
f"{review}"
"Original request:\n"
@@ -4328,8 +4334,12 @@ async def _drive_cli(
# A multi-select answer reads the way the CLI's own picker writes one:
# the chosen labels joined by commas.
# A skipped question (empty answer) still gets words the CLI can read.
+ # A default the person never picked says so, so the model cannot claim
+ # the person chose it.
picked = {
- str(row.get("question") or ""): as_line(a.answer) or "No preference; decide yourself."
+ str(row.get("question") or ""): (
+ as_line(a.answer) or "No preference; decide yourself."
+ ) + (" (the person did not answer; your default)" if a.source in DEFAULTED else "")
for row, a in zip(rows, answers, strict=True)
}
return {"behavior": "allow", "updatedInput": {**tool_input, "answers": picked}}
diff --git a/dodaios/agent_chat/service.py b/dodaios/agent_chat/service.py
index e1cb3a7fb2..a5d6a7f9d5 100644
--- a/dodaios/agent_chat/service.py
+++ b/dodaios/agent_chat/service.py
@@ -52,6 +52,7 @@ from dodaios.agent_chat.questions import (
MESSAGE,
MOVED_ON,
QUESTION_TIMEOUT_S,
+ RECALL_WINDOW_S,
SKIPPED,
TIMEOUT,
QuestionAnswer,
@@ -60,6 +61,9 @@ from dodaios.agent_chat.questions import (
answer_from,
answer_json,
declined_answer,
+ note_answers,
+ person_answers,
+ recall,
recommended_answer,
)
from dodaios.agent_chat.runner_api import TurnHandle, run_api_turn, supports_api_runner
@@ -2558,12 +2562,20 @@ class AgentChatService:
run = self._running.get(session_id)
if run is None or run.task is None or run.task.done():
raise RuntimeError("this chat has no running turn to ask in")
- if run.asks >= MAX_ASKS_PER_TURN:
- raise TooManyQuestions(f"this turn already asked {run.asks} times")
- run.asks += 1
question_id = uuid.uuid4().hex
loop = asyncio.get_running_loop()
open_q = _OpenQuestion(session_id, run.turn_id, specs, loop.create_future())
+ # A question the person already answered is never put to them again:
+ # its answer comes back, and the card shows only what is still open.
+ open_q.answers[:] = recall(specs, self._answered(session_id, run.turn_id))
+ if open_q.done:
+ open_q.delivered = True
+ open_q.result.set_result(list(open_q.answers))
+ self._questions[question_id] = open_q
+ return question_id
+ if run.asks >= MAX_ASKS_PER_TURN:
+ raise TooManyQuestions(f"this turn already asked {run.asks} times")
+ run.asks += 1
self._questions[question_id] = open_q
await self._emit(
session_id,
@@ -2697,16 +2709,34 @@ class AgentChatService:
except Exception: # noqa: BLE001 — the outcome is already delivered to the waiter
log.warning("agent chat: could not record question %s", question_id, exc_info=True)
- @staticmethod
- def _final_answers(open_q: _OpenQuestion) -> list[QuestionAnswer]:
+ def _answered(self, session_id: str, turn_id: str) -> dict[Any, Any]:
+ """The person's answers in this turn and within ``RECALL_WINDOW_S``."""
+ found = person_answers(
+ self.store.list_events(session_id, tail=self._TURN_PROMPT_TAIL),
+ turn_id=turn_id,
+ since_ms=int((time.time() - RECALL_WINDOW_S) * 1000),
+ )
+ for open_q in getattr(self, "_questions", {}).values():
+ if open_q.session_id == session_id:
+ note_answers(found, open_q.specs, open_q.answers)
+ return found
+
+ def _final_answers(self, open_q: _OpenQuestion) -> list[QuestionAnswer]:
+ source = open_q.closing or TIMEOUT
+ # A fallback never overrides what the person already answered elsewhere.
+ earlier: list[QuestionAnswer | None] = [None] * len(open_q.specs)
+ if source != CANCELLED and any(a is None for a in open_q.answers):
+ try:
+ earlier = recall(open_q.specs, self._answered(open_q.session_id, open_q.turn_id))
+ except Exception: # noqa: BLE001 — an unreadable log leaves the defaults
+ log.warning("agent chat: earlier answers unavailable", exc_info=True)
answers: list[QuestionAnswer] = []
- for spec, answer in zip(open_q.specs, open_q.answers, strict=True):
+ for spec, answer, before in zip(open_q.specs, open_q.answers, earlier, strict=True):
if answer is None:
- source = open_q.closing or TIMEOUT
answer = (
QuestionAnswer("", None, CANCELLED)
if source == CANCELLED
- else recommended_answer(spec, source)
+ else before or recommended_answer(spec, source)
)
answers.append(answer)
return answers
diff --git a/dodaios/agent_chat/task_recovery.py b/dodaios/agent_chat/task_recovery.py
index 582ecee67e..d577bedfc8 100644
--- a/dodaios/agent_chat/task_recovery.py
+++ b/dodaios/agent_chat/task_recovery.py
@@ -43,6 +43,47 @@ _TERMINAL = (
"rate-limited",
)
_DISCOVERY = frozenset({"toolsearch", "list_tools", "tools-list", "search_tools"})
+# A call refused before it ran (its arguments did not parse or validate) had
+# no effect and no target of its own: the next successful call of the same
+# tool is its corrected retry.
+_NOT_RUN = (
+ "not valid json",
+ "invalid json",
+ "jsondecodeerror",
+ "argument validation",
+ "inputvalidationerror",
+ "input validation error",
+ "was not invoked",
+ "invalid arguments",
+ "invalid params",
+ "invalid questions",
+ "is a required property",
+ "unexpected keyword argument",
+)
+# Meta tools that call another tool by name; a refused call names it only in
+# its (unparsed) arguments, while the retry reports the inner tool itself.
+_WRAPPERS = frozenset({"tool_call", "call_tool", "use_tool", "invoke_tool"})
+_INNER_NAME = re.compile(r"""['"](?:name|tool|tool_name)['"]\s*:\s*['"]([\w.:/-]{1,160})['"]""")
+
+
+def _bare(name: str) -> str:
+ return name.rsplit("__", 1)[-1].lower()
+
+
+def _names(name: str, arguments: Any) -> frozenset[str]:
+ """The tool a call is about: itself, or for a meta tool the tools it names.
+
+ A meta call that names its inner tools is only about them, so a later
+ meta call of another tool does not settle it.
+ """
+ if _bare(name) not in _WRAPPERS:
+ return frozenset({_bare(name)})
+ inner: set[str] = set()
+ values = arguments.values() if isinstance(arguments, dict) else [arguments]
+ for value in values:
+ raw = value if isinstance(value, str) else json.dumps(value, default=str)
+ inner.update(_bare(found) for found in _INNER_NAME.findall(raw[:8000]))
+ return frozenset(inner or {_bare(name)})
def blocks_automatic_recovery(text: str | None) -> bool:
@@ -51,10 +92,17 @@ def blocks_automatic_recovery(text: str | None) -> bool:
@dataclass
class ToolRecovery:
- """A successful unrelated action cannot erase an unfinished part of a task."""
+ """A successful unrelated action cannot erase an unfinished part of a task.
+
+ A success of the same action settles its failure, and a success of the
+ same tool settles a call of that tool that was refused before it ran.
+ """
calls: dict[str, tuple[str, str]] = field(default_factory=dict)
unresolved: dict[tuple[str, str], str] = field(default_factory=dict)
+ #: Failed calls that never ran (malformed arguments), with the tools they named.
+ not_run: dict[tuple[str, str], frozenset[str]] = field(default_factory=dict)
+ names: dict[str, frozenset[str]] = field(default_factory=dict)
declined: bool = False
blocked: bool = False
had_tool_calls: bool = False
@@ -69,19 +117,20 @@ class ToolRecovery:
str(payload.get("name") or "tool"),
hashlib.sha256(arguments.encode("utf-8")).hexdigest(),
)
+ self.names[call_id] = _names(self.calls[call_id][0], payload.get("input"))
return
if event.get("kind") != "tool_result":
return
key = self.calls.pop(call_id, (str(payload.get("name") or "tool"), call_id))
name = key[0]
+ names = self.names.pop(call_id, frozenset({_bare(name)}))
# Descriptions, provider bodies and retrieved instructions are never
# copied into a recovery prompt. Only a registered tool name and a
# fixed diagnostic code can cross that boundary.
raw = payload.get("output")
text = raw if isinstance(raw, str) else json.dumps(raw, default=str)
text = text[:8000]
- bare = name.rsplit("__", 1)[-1].lower()
- discovery = bare in _DISCOVERY
+ discovery = _bare(name) in _DISCOVERY
missing = discovery and text.lstrip().startswith("No matching deferred tools found")
if payload.get("is_error") or missing:
if blocks_automatic_recovery(text):
@@ -89,8 +138,17 @@ class ToolRecovery:
self.unresolved.pop(key, None)
else:
self.unresolved[key] = "discovery_empty" if missing else "tool_failed"
+ low = text.lower()
+ if any(marker in low for marker in _NOT_RUN):
+ self.not_run[key] = names
+ else:
+ self.not_run.pop(key, None)
else:
self.unresolved.pop(key, None)
+ # The corrected retry settles every refused call of the same tool.
+ for stale in [k for k, named in self.not_run.items() if named & names]:
+ self.not_run.pop(stale, None)
+ self.unresolved.pop(stale, None)
def hint(self) -> str | None:
if self.declined or self.blocked or not self.unresolved:
diff --git a/dodaios/agent_chat/turn_completion.py b/dodaios/agent_chat/turn_completion.py
index 34596539de..4565275e92 100644
--- a/dodaios/agent_chat/turn_completion.py
+++ b/dodaios/agent_chat/turn_completion.py
@@ -15,7 +15,7 @@ import re
import time
from typing import Any
-from .questions import CANCELLED, MAX_ASKS_PER_TURN
+from .questions import CANCELLED, MAX_ASKS_PER_TURN, for_agent
from .secret_requests import asks_for_secret, claims_stored_secret
from .task_recovery import ToolRecovery
@@ -216,7 +216,9 @@ class TurnCompletion:
for spec, answer in zip(specs, answers, strict=True):
if answer.source == CANCELLED:
raise asyncio.CancelledError
- rows.append({"question": spec.question, **answer.to_payload()})
+ rows.append(
+ {"question": spec.question, **answer.to_payload(), **for_agent(answer)}
+ )
self.answered.add(question_id)
if rows:
return (
diff --git a/dodaios/agent_chat/turn_prompts.py b/dodaios/agent_chat/turn_prompts.py
index 54472dc1e8..63b17caaaf 100644
--- a/dodaios/agent_chat/turn_prompts.py
+++ b/dodaios/agent_chat/turn_prompts.py
@@ -39,11 +39,11 @@ from dodaios.agent_chat.questions import (
MAX_QUESTIONS,
MIN_OPTIONS,
MOVED_ON,
- PERSON,
SKIPPED,
QuestionAnswer,
QuestionSpec,
answer_from,
+ answer_from_payload,
as_line,
parse_questions,
recommended_answer,
@@ -142,23 +142,7 @@ class OpenPlan:
def _answers_from(raw: Any, count: int) -> list[QuestionAnswer | None]:
rows = raw if isinstance(raw, list) else []
- out: list[QuestionAnswer | None] = []
- for index in range(count):
- row = rows[index] if index < len(rows) else None
- if not isinstance(row, dict):
- out.append(None)
- continue
- option = row.get("option_index")
- picks = row.get("option_indices")
- out.append(
- QuestionAnswer(
- str(row.get("answer") or ""),
- option if isinstance(option, int) else None,
- str(row.get("source") or PERSON),
- tuple(i for i in picks if isinstance(i, int)) if isinstance(picks, list) else (),
- )
- )
- return out
+ return [answer_from_payload(rows[i] if i < len(rows) else None) for i in range(count)]
def newest_open_ask(events: Sequence[dict[str, Any]]) -> OpenAsk | None:
diff --git a/dodaios/society/ask_tool.py b/dodaios/society/ask_tool.py
index 14a04d9f86..606092f5f5 100644
--- a/dodaios/society/ask_tool.py
+++ b/dodaios/society/ask_tool.py
@@ -31,12 +31,14 @@ from dodaios.agent_chat.questions import (
MIN_OPTIONS,
MOVED_ON,
QUESTION_TIMEOUT_S,
+ RECALLED,
SKIPPED,
TIMEOUT,
UNATTENDED,
QuestionAnswer,
QuestionSpec,
TooManyQuestions,
+ for_agent,
parse_questions,
recommended_answer,
)
@@ -276,13 +278,15 @@ def _result(specs: tuple[QuestionSpec, ...], answers: list[QuestionAnswer]) -> T
row["option_description"] = description
elif answer.by_person and not answer.option_indices:
row["typed"] = True
+ row.update(for_agent(answer))
rows.append(row)
output: dict[str, Any] = {"answers": rows}
sources = {a.source for a in answers}
if TIMEOUT in sources:
output["note"] = (
- f"The user stopped answering for {_MINUTES} minutes, so your recommendations "
- "were applied to the open questions. Continue with them and briefly mention it."
+ f"The user did not answer for {_MINUTES} minutes, so your recommendations were "
+ "applied to the open questions. The user did not choose them: continue with "
+ "them and say you went with your default."
)
elif SKIPPED in sources:
output["note"] = (
@@ -294,6 +298,11 @@ def _result(specs: tuple[QuestionSpec, ...], answers: list[QuestionAnswer]) -> T
"The user replied in the chat instead of on the card: read that message as "
"their answer. Questions they did not address took your recommendations."
)
+ if sources == {RECALLED}:
+ output["note"] = (
+ "The user already answered these questions, so no new card was shown. "
+ "Continue with their answers and do not ask again."
+ )
if DECLINED in sources:
skipped = "The user skipped a question (empty answer): decide it yourself."
output["note"] = f"{output['note']} {skipped}" if "note" in output else skipped
diff --git a/tests/unit/agent_chat/test_questions_no_repeat.py b/tests/unit/agent_chat/test_questions_no_repeat.py
new file mode 100644
index 0000000000..f9398b35aa
--- /dev/null
+++ b/tests/unit/agent_chat/test_questions_no_repeat.py
@@ -0,0 +1,326 @@
+"""A question is never put to the person twice, and a default is never their choice.
+
+Replays the Hermes run where the person answered "green", a stale receipt of an
+earlier malformed call restarted the agent, the agent asked again and the
+second card's timeout default ("red") was reported as the person's choice.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from dodaios.agent_chat import dodaios_harness, runner_cli
+from dodaios.agent_chat.events import make_event
+from dodaios.agent_chat.questions import (
+ PERSON,
+ RECALL_WINDOW_S,
+ RECALLED,
+ TIMEOUT,
+ parse_questions,
+)
+from dodaios.agent_chat.runner_api import TurnHandle
+from dodaios.agent_chat.service import AgentChatService, _Running
+from dodaios.agent_chat.store import AgentChatStore
+from dodaios.agent_chat.task_recovery import ToolRecovery
+from dodaios.agent_chat.turn_completion import TurnCompletion
+from dodaios.core.protocols import ChatTurn, current_chat_turn
+from dodaios.society.ask_tool import AskUserTool
+
+_COLOUR: dict[str, Any] = {
+ "question": "What is your favorite color?",
+ "options": [{"label": "red"}, {"label": "green"}],
+ "recommendation_reason": "No preference known.",
+}
+_SIZE: dict[str, Any] = {
+ "question": "Which size?",
+ "options": [{"label": "S"}, {"label": "M"}, {"label": "L"}],
+ "recommendation_reason": "Most common.",
+}
+
+# The live receipts: a meta call whose arguments did not parse, then the
+# corrected call, which the runtime reports under the inner tool's name.
+_MALFORMED = (
+ '[{"arguments": {"question": "What is your favorite color?", "options": ["red", '
+ '"green"], "required": True}, "name": "mcp__dodaios__society_ask_user"}]'
+)
+_LIVE_RECEIPTS: list[tuple[str, str, Any, str, bool]] = [
+ ("c1", "tool_search", {"queries": ["ask user"]}, "matches", False),
+ (
+ "c2", "tool_call", {"calls": _MALFORMED},
+ "tool_call failed: tool_call 'calls' is not valid JSON: Expecting value", True,
+ ),
+ (
+ "c3", "mcp__dodaios__society_ask_user", {"question": "What is your favorite color?"},
+ '{"answers": [{"answer": "green", "source": "person"}]}', False,
+ ),
+]
+
+
+def _feed(tracker: Any, receipts: list[tuple[str, str, Any, str, bool]]) -> None:
+ for call_id, name, args, output, error in receipts:
+ tracker.observe(make_event("tool_call", {"call_id": call_id, "name": name, "input": args}))
+ tracker.observe(
+ make_event("tool_result", {"call_id": call_id, "output": output, "is_error": error})
+ )
+
+
+# ------------------------------------------------------------ (a) stale receipts
+
+
+def test_a_corrected_retry_settles_the_malformed_meta_call() -> None:
+ tracker = ToolRecovery()
+ _feed(tracker, _LIVE_RECEIPTS[:2])
+ assert "tool_call" in (tracker.hint() or "")
+ _feed(tracker, _LIVE_RECEIPTS[2:])
+ assert tracker.hint() is None
+
+
+def test_a_corrected_retry_of_the_same_tool_settles_its_refused_call() -> None:
+ tracker = ToolRecovery()
+ _feed(tracker, [
+ ("c1", "society_ask_user", {"question": "?"}, "invalid questions: no options", True),
+ ("c2", "society_ask_user", _COLOUR, "answered", False),
+ ])
+ assert tracker.hint() is None
+
+
+def test_a_meta_call_of_another_tool_does_not_settle_a_refused_meta_call() -> None:
+ tracker = ToolRecovery()
+ _feed(tracker, [
+ _LIVE_RECEIPTS[1],
+ ("c3", "tool_call", {"calls": '[{"name": "mcp__dodaios__read_file"}]'}, "ok", False),
+ ])
+ assert "tool_call" in (tracker.hint() or "")
+ _feed(tracker, [
+ ("c4", "tool_call", {"calls": '[{"name": "mcp__dodaios__society_ask_user"}]'}, "ok", False),
+ ])
+ assert tracker.hint() is None
+
+
+def test_a_real_failure_is_not_settled_by_another_target() -> None:
+ tracker = ToolRecovery()
+ _feed(tracker, [
+ ("c1", "Write", {"path": "a.txt"}, "disk full", True),
+ ("c2", "Write", {"path": "b.txt"}, "saved", False),
+ ("c3", "tool_call", {"calls": '{"name": "Write"'}, "not valid JSON", True),
+ ("c4", "Read", {"path": "b.txt"}, "contents", False),
+ ])
+ hint = tracker.hint() or ""
+ assert "for Write" in hint and "for tool_call" in hint
+
+
+def test_a_receipt_hint_alone_does_not_claim_the_last_call_failed() -> None:
+ prompt = runner_cli._keep_going_prompt("Task", None, receipt_hint="hint")
+ assert "The last tool call was cancelled or failed" not in prompt
+ assert "An earlier tool call failed" in prompt
+ assert "boom" in runner_cli._keep_going_prompt("Task", "boom")
+
+
+def turn(monkeypatch: pytest.MonkeyPatch):
+ events: list[dict[str, Any]] = []
+ session = SimpleNamespace(
+ surface="society", session_id="society:qa", vendor_session="v", account_id=""
+ )
+
+ async def emit(event: dict[str, Any]) -> None:
+ events.append(event)
+
+ async def allow(*_args: Any) -> str:
+ return "allow"
+
+ async def briefing(*_args: Any) -> str:
+ return "Agent instructions"
+
+ async def identity(**_kwargs: Any) -> dodaios_harness.Identity:
+ return dodaios_harness.Identity(session.session_id, "Agent", "Agent")
+
+ async def language(*_args: Any) -> str:
+ return "en"
+
+ monkeypatch.setattr(runner_cli, "_surface_identity", briefing)
+ monkeypatch.setattr(dodaios_harness, "build_identity", identity)
+ monkeypatch.setattr("dodaios.society.reply_preference.resolve_agent_reply_language", language)
+ handle = TurnHandle(
+ session=session, turn_id="t1", emit=emit, request_approval=allow,
+ cancel=asyncio.Event(), surface="society", stance="ask",
+ )
+ origin = ChatTurn(session.session_id, "t1", "Ask me which colour I prefer.", True, "trace")
+ token = current_chat_turn.set(origin)
+ try:
+ yield handle, events, origin
+ finally:
+ current_chat_turn.reset(token)
+
+
[email protected]("runner", ["hermes-cli", "openclaw-cli", "claude-cli", "codex-cli"])
+async def test_an_answered_question_never_restarts_the_turn(turn, monkeypatch, runner) -> None:
+ handle, events, origin = turn
+ attempts: list[str] = []
+
+ async def once(handle: Any, prompt: str, *_args: Any, **_kwargs: Any) -> Any:
+ attempts.append(prompt)
+ for call_id, name, args, output, error in _LIVE_RECEIPTS:
+ await handle.emit(
+ make_event("tool_call", {"call_id": call_id, "name": name, "input": args})
+ )
+ await handle.emit(
+ make_event(
+ "tool_result", {"call_id": call_id, "output": output, "is_error": error}
+ )
+ )
+ return runner_cli._Outcome("done", None, {}, None, "v")
+
+ monkeypatch.setattr(runner_cli, "_run_cli_once", once)
+ await runner_cli.run_cli_turn(handle, origin.user_text, runner, identity=True)
+ assert len(attempts) == 1
+ assert [e["payload"]["status"] for e in events if e["kind"] == "turn_finished"] == ["done"]
+
+
+# ------------------------------------------------- (b) and (c) the card itself
+
+
+async def _service(tmp_path: Path, turn_id: str = "turn-1") -> tuple[AgentChatService, str]:
+ svc = AgentChatService(AgentChatStore(":memory:"), assistant_name=lambda: "Testo")
+ session = svc.store.create_session(
+ provider="fakeprov", model="m", effort="high", cwd=str(tmp_path)
+ )
+ _start_turn(svc, session.session_id, turn_id)
+ return svc, session.session_id
+
+
+def _start_turn(svc: AgentChatService, sid: str, turn_id: str) -> None:
+ old = svc._running.get(sid)
+ if old is not None and old.task is not None:
+ old.task.cancel()
+ run = _Running(turn_id, asyncio.Event())
+ run.task = asyncio.create_task(asyncio.sleep(3600))
+ svc._running[sid] = run
+
+
+async def _next(q: asyncio.Queue, kind: str) -> dict[str, Any]:
+ while True:
+ event = await asyncio.wait_for(q.get(), timeout=5)
+ if event["kind"] == kind:
+ return event["payload"]
+
+
+async def _answer_green(svc: AgentChatService, sid: str, q: asyncio.Queue) -> None:
+ asking = asyncio.create_task(svc.ask_questions(sid, parse_questions(_COLOUR)))
+ card = await _next(q, "question_required")
+ assert svc.resolve_question(sid, card["question_id"], index=0, option_index=1)
+ (answer,) = await asking
+ assert (answer.answer, answer.source) == ("green", PERSON)
+ await _next(q, "question_resolved")
+
+
+def _tool(svc: AgentChatService, sid: str) -> AskUserTool:
+ runtime = SimpleNamespace(
+ chat_service=lambda: svc, cached_agent=lambda _id: SimpleNamespace(name="QA")
+ )
+ return AskUserTool(runtime, "qa", session_id=sid)
+
+
+async def test_the_same_question_again_returns_the_answer_without_a_card(tmp_path) -> None:
+ svc, sid = await _service(tmp_path)
+ q = svc.subscribe(sid)
+ await _answer_green(svc, sid, q)
+ asks = svc._running[sid].asks
+ # Asked again (the recommendation now leads): no card, the person's answer.
+ result = await _tool(svc, sid).execute({**_COLOUR, "recommended": 1}, None)
+ (row,) = result.output["answers"]
+ assert (row["answer"], row["source"], row["person_answered"]) == ("green", RECALLED, True)
+ assert row["recommended"] is True # green leads the re-asked card
+ assert "do not ask again" in result.output["note"]
+ assert svc._running[sid].asks == asks
+ kinds = []
+ while not q.empty():
+ kinds.append(q.get_nowait()["kind"])
+ assert "question_required" not in kinds and svc.pending_questions(sid) == []
+ svc._running[sid].task.cancel()
+
+
+async def test_only_the_unanswered_questions_stay_open(tmp_path) -> None:
+ svc, sid = await _service(tmp_path)
+ q = svc.subscribe(sid)
+ await _answer_green(svc, sid, q)
+ series = parse_questions({"questions": [_COLOUR, _SIZE]})
+ asking = asyncio.create_task(svc.ask_questions(sid, series))
+ card = await _next(q, "question_required")
+ progress = await _next(q, "question_progress")
+ assert progress["answers"][0]["source"] == RECALLED and progress["answers"][1] is None
+ assert svc.resolve_question(sid, card["question_id"], index=1, option_index=2)
+ first, second = await asking
+ assert (first.answer, second.answer) == ("green", "L")
+ svc._running[sid].task.cancel()
+
+
+async def test_an_earlier_turn_counts_only_within_the_window(tmp_path, monkeypatch) -> None:
+ svc, sid = await _service(tmp_path)
+ q = svc.subscribe(sid)
+ await _answer_green(svc, sid, q)
+ _start_turn(svc, sid, "turn-2")
+ svc._questions.clear() # the first turn's cards closed with it
+ (again,) = await svc.ask_questions(sid, parse_questions(_COLOUR))
+ assert (again.answer, again.source) == ("green", RECALLED)
+ _start_turn(svc, sid, "turn-3")
+ svc._questions.clear()
+ later = time.time() + RECALL_WINDOW_S + 60
+ monkeypatch.setattr("dodaios.agent_chat.service.time.time", lambda: later)
+ (fresh,) = await svc.ask_questions(sid, parse_questions(_COLOUR), timeout_s=0.05)
+ assert fresh.source == TIMEOUT
+ svc._running[sid].task.cancel()
+
+
+async def test_a_timeout_never_overrides_what_the_person_answered(tmp_path) -> None:
+ svc, sid = await _service(tmp_path)
+ q = svc.subscribe(sid)
+ specs = parse_questions(_COLOUR)
+ first = asyncio.create_task(svc.ask_questions(sid, specs))
+ card = await _next(q, "question_required")
+ second = asyncio.create_task(svc.ask_questions(sid, specs, timeout_s=0.3))
+ await _next(q, "question_required")
+ assert svc.resolve_question(sid, card["question_id"], index=0, option_index=1)
+ await first
+ (late,) = await second
+ assert (late.answer, late.source) == ("green", RECALLED)
+ svc._running[sid].task.cancel()
+
+
+async def test_a_timeout_default_tells_the_agent_nobody_chose_it(tmp_path) -> None:
+ svc, sid = await _service(tmp_path)
+ tool = _tool(svc, sid)
+ original = svc.open_questions
+
+ async def quick(*args: Any, **kwargs: Any) -> str:
+ return await original(*args, **{**kwargs, "timeout_s": 0.05})
+
+ svc.open_questions = quick # type: ignore[method-assign]
+ result = await tool.execute(dict(_COLOUR), None)
+ (row,) = result.output["answers"]
+ assert (row["answer"], row["source"], row["person_answered"]) == ("red", TIMEOUT, False)
+ assert "Never say they chose it" in row["note"]
+ assert "did not choose them" in result.output["note"]
+ svc._running[sid].task.cancel()
+
+
+async def test_the_continuation_marks_a_default_as_not_the_persons(tmp_path) -> None:
+ svc, sid = await _service(tmp_path)
+ qid = await svc.open_questions(sid, parse_questions(_COLOUR), timeout_s=0.05)
+ await svc.wait_questions(sid, qid, 5)
+ svc._questions[qid].delivered = False # the runner never read it
+ handle = SimpleNamespace(
+ session=SimpleNamespace(session_id=sid), turn_id="turn-1", cancel=asyncio.Event()
+ )
+ completion = TurnCompletion(svc, handle, "Ask me", allow_correction=False)
+ completion.finish_event = make_event("turn_finished", {"status": "done"})
+ prompt = await completion.next_prompt()
+ assert prompt is not None and '"person_answered": false' in prompt
+ assert "Never say they chose it" in prompt
+ svc._running[sid].task.cancel()