Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

fix(onboarding): keep going when a setup turn returns prose or a null hint (#378)

Rufusfd3e7bb7dbd8
diff --git a/packages/core/openexecutive/onboarding/interview.py b/packages/core/openexecutive/onboarding/interview.py
index c84967fa..7d2eb7e6 100644
--- a/packages/core/openexecutive/onboarding/interview.py
+++ b/packages/core/openexecutive/onboarding/interview.py
@@ -39,7 +39,7 @@ import logging
from dataclasses import dataclass
from typing import Any
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from openexecutive.agents.onboarding_interviewer import (
ONBOARDING_INTERVIEWER_AGENT_ID,
@@ -78,6 +78,8 @@ _MAX_TOKENS = 8000
# changing this one does not change that one.
_REPAIR_ECHO_CHARS = 2000
+_MAX_PROSE_QUESTION_CHARS = 1000
+
ASK_TOOL_NAME = "ask_clarifying_question"
EMIT_TOOL_NAME = "emit_company_draft"
@@ -153,6 +155,13 @@ class Question(BaseModel):
question: str
hint: str = ""
+ @field_validator("hint", mode="before")
+ @classmethod
+ def _no_null_hint(cls, value: Any) -> Any:
+ # Models sometimes send "hint": null for "no example"; that is not a
+ # reason to throw the whole question away.
+ return "" if value is None else value
+
@dataclass(frozen=True)
class DraftError:
@@ -468,6 +477,16 @@ def _extract_tool_call(response: Any) -> tuple[str, dict[str, Any]]:
raise InterviewError("The setup assistant did not return a usable response.")
+def _prose(response: Any) -> str:
+ """The text blocks of a response, joined; empty when there are none."""
+ parts = [
+ str(getattr(b, "text", "") or "").strip()
+ for b in getattr(response, "content", []) or []
+ if getattr(b, "type", None) == "text"
+ ]
+ return "\n\n".join(p for p in parts if p)
+
+
def replay_transcript(
transcript: list[Turn], continue_prompt: str
) -> list[dict[str, Any]]:
@@ -690,7 +709,44 @@ async def advance(
iteration=attempt,
)
- name, raw = _extract_tool_call(response)
+ try:
+ name, raw = _extract_tool_call(response)
+ except InterviewError:
+ # No usable tool call (the model answered in prose, or named another
+ # tool). Block types only in the log: the text can hold the user's
+ # financials.
+ logger.error(
+ "onboarding interview: no tool call (stop_reason=%s, blocks=%s)",
+ getattr(response, "stop_reason", None),
+ [getattr(b, "type", None) for b in getattr(response, "content", []) or []],
+ )
+ prose = _prose(response)
+ if attempt == 0:
+ messages = [
+ *messages,
+ {"role": "assistant", "content": prose or "(no answer)"},
+ {
+ "role": "user",
+ "content": (
+ "Call exactly one tool: "
+ + (EMIT_TOOL_NAME if must_draft else f"{ASK_TOOL_NAME} or {EMIT_TOOL_NAME}")
+ + ", now."
+ ),
+ },
+ ]
+ continue
+ if (
+ prose
+ and not must_draft
+ and len(prose) <= _MAX_PROSE_QUESTION_CHARS
+ and prose.endswith("?")
+ ):
+ # The model asked in plain words twice running: the person can
+ # still answer it, so show it rather than a dead end. A long
+ # reply, or one that does not end in a question mark, is
+ # reasoning or a half-made draft: it is never shown.
+ return Question(question=prose)
+ raise
if name == ASK_TOOL_NAME:
if must_draft:
@@ -724,7 +780,11 @@ async def advance(
return question
except (ValidationError, ValueError) as exc:
logger.error(
- "onboarding interview: malformed question (%s)", type(exc).__name__
+ "onboarding interview: malformed question (%s, fields=%s)",
+ type(exc).__name__,
+ [".".join(map(str, e["loc"])) for e in exc.errors()]
+ if isinstance(exc, ValidationError)
+ else [],
)
if attempt == 0:
continue
diff --git a/packages/core/tests/unit/test_onboarding_interview.py b/packages/core/tests/unit/test_onboarding_interview.py
index faaa3615..7cec94fa 100644
--- a/packages/core/tests/unit/test_onboarding_interview.py
+++ b/packages/core/tests/unit/test_onboarding_interview.py
@@ -725,3 +725,90 @@ async def test_transcript_never_ends_on_an_assistant_turn(
roles = [m["role"] for m in provider.calls[0]["messages"]]
assert roles[-1] == "user", f"trailing assistant prefill: {roles}"
assert all(a != b for a, b in zip(roles, roles[1:]))
+
+
+async def test_null_hint_keeps_the_question(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = _ScriptedProvider(
+ [_tool_response(iv.ASK_TOOL_NAME, {"question": "Who runs ops?", "hint": None})]
+ )
+ _install(monkeypatch, provider)
+ got = await iv.advance(_opening(), questions_asked=0)
+ assert isinstance(got, iv.Question) and got.question == "Who runs ops?" and got.hint == ""
+ assert len(provider.calls) == 1
+
+
+async def test_a_prose_reply_is_retried_once_then_still_works(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ prose = SimpleNamespace(content=[SimpleNamespace(type="text", text="Who runs ops?")])
+ provider = _ScriptedProvider(
+ [prose, _tool_response(iv.ASK_TOOL_NAME, {"question": "Who runs ops?"})]
+ )
+ _install(monkeypatch, provider)
+ got = await iv.advance(_opening(), questions_asked=0)
+ assert isinstance(got, iv.Question)
+ assert len(provider.calls) == 2
+ # The retry nudges the model and keeps roles alternating.
+ roles = [m["role"] for m in provider.calls[1]["messages"]]
+ assert roles == ["user", "assistant", "user"]
+
+
+async def test_prose_twice_becomes_the_question(monkeypatch: pytest.MonkeyPatch) -> None:
+ prose = SimpleNamespace(content=[SimpleNamespace(type="text", text="Who are your customers?")])
+ provider = _ScriptedProvider([prose, prose])
+ _install(monkeypatch, provider)
+ got = await iv.advance(_opening(), questions_asked=0)
+ assert isinstance(got, iv.Question) and got.question == "Who are your customers?"
+
+
+async def test_no_tool_and_no_text_still_errors(monkeypatch: pytest.MonkeyPatch) -> None:
+ empty = SimpleNamespace(content=[])
+ provider = _ScriptedProvider([empty, empty])
+ _install(monkeypatch, provider)
+ with pytest.raises(iv.InterviewError):
+ await iv.advance(_opening(), questions_asked=0)
+
+
+async def test_long_prose_is_not_shown_as_a_question(monkeypatch: pytest.MonkeyPatch) -> None:
+ long = SimpleNamespace(content=[SimpleNamespace(type="text", text="x" * 1001)])
+ provider = _ScriptedProvider([long, long])
+ _install(monkeypatch, provider)
+ with pytest.raises(iv.InterviewError):
+ await iv.advance(_opening(), questions_asked=0)
+
+
+async def test_prose_twice_while_drafting_still_errors(monkeypatch: pytest.MonkeyPatch) -> None:
+ prose = SimpleNamespace(content=[SimpleNamespace(type="text", text="Who runs ops?")])
+ provider = _ScriptedProvider([prose, prose])
+ _install(monkeypatch, provider)
+ with pytest.raises(iv.InterviewError):
+ await iv.advance(_opening(), questions_asked=0, force_draft=True)
+
+
+async def test_short_prose_that_asks_nothing_is_not_a_question(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ note = SimpleNamespace(content=[SimpleNamespace(type="text", text="I have enough to draft.")])
+ provider = _ScriptedProvider([note, note])
+ _install(monkeypatch, provider)
+ with pytest.raises(iv.InterviewError):
+ await iv.advance(_opening(), questions_asked=0)
+
+
+async def test_prose_with_a_mid_text_question_mark_is_not_shown(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ text = "Is that right? Anyway, I would draft: Northwind, 12 staff."
+ note = SimpleNamespace(content=[SimpleNamespace(type="text", text=text)])
+ provider = _ScriptedProvider([note, note])
+ _install(monkeypatch, provider)
+ with pytest.raises(iv.InterviewError):
+ await iv.advance(_opening(), questions_asked=0)