Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(dictation): Prompt Mode names the language, drafts, then reads itself back

rubenluetke10-beep1343db83db63
diff --git a/jarvis/dictation/prompt_mode.py b/jarvis/dictation/prompt_mode.py
index 07b9930db5..505a7e1aa4 100644
--- a/jarvis/dictation/prompt_mode.py
+++ b/jarvis/dictation/prompt_mode.py
@@ -113,6 +113,15 @@ log = logging.getLogger(__name__)
#: a second — no pass over the transcript worth the name — reworded phrasing
#: the user had chosen deliberately, thinned the context they had given, and
#: closed every message with a sign-off the receiving agent has no use for.
+#:
+#: The language rule is a POINT OF THE ANALYSIS and both worked examples are
+#: shown, one German and one English. A first cut of v4 kept only the English
+#: pair, trusting the written rule to carry the language; measured live on
+#: Groq the same day, a German dictation came back as a fluent English
+#: message. A shown example outweighs a written rule, so the examples have to
+#: disagree with each other about language for the rule to be readable at
+#: all — and naming the language in writing before composing is what makes
+#: the model commit to it.
PROMPT_MODE_PROMPT_VERSION: Final[int] = 4
#: The status a successful Prompt Mode delivery reports on the history row.
@@ -132,41 +141,59 @@ _DEFAULT_TIMEOUT_MS: Final[int] = 12_000
_MIN_TIMEOUT_MS: Final[int] = 4_000
_MAX_TIMEOUT_MS: Final[int] = 20_000
-# The budget covers BOTH blocks: the analysis (300-900 tokens on a real
-# dictation) and the message itself. Generous on purpose — a cut-off answer
-# is rejected as truncated and costs the whole pass. Temperature 0: this is
-# rewriting, not writing.
-_MAX_OUTPUT_TOKENS: Final[int] = 3_000
+# The budget covers ALL THREE blocks: the analysis (300-900 tokens on a real
+# dictation), the draft, and the message the read-back corrects. Generous on
+# purpose — a cut-off answer is rejected as truncated and costs the whole
+# pass. Temperature 0: this is rewriting, not writing.
+_MAX_OUTPUT_TOKENS: Final[int] = 4_000
_TEMPERATURE: Final[float] = 0.0
-# The two blocks of the answer. Parsed with a tolerant regex rather than an
+# The three blocks of the answer. Parsed with tolerant regexes rather than an
# XML parser: the content is prose the model wrote, it may contain a stray
-# angle bracket, and a strict parser would throw away a perfectly good
-# message over one. Case-insensitive and whitespace-tolerant for the same
-# reason — a fast model writes ``<Prompt>`` often enough to matter.
+# angle bracket, and a strict parser would throw away a perfectly good message
+# over one. Case-insensitive and whitespace-tolerant for the same reason — a
+# fast model writes ``<Prompt>`` often enough to matter.
+#
+# ``<prompt>`` is LAST in the answer, so the regex takes the last one: a model
+# that names the tag inside its analysis ("...then the <prompt> block...")
+# would otherwise hand us its worksheet.
_ANALYSIS_BLOCK_RE: Final[re.Pattern[str]] = re.compile(
r"<\s*analysis\s*>(.*?)<\s*/\s*analysis\s*>", re.IGNORECASE | re.DOTALL
)
-_PROMPT_BLOCK_RE: Final[re.Pattern[str]] = re.compile(
- r"<\s*prompt\s*>(.*?)(?:<\s*/\s*prompt\s*>|\Z)", re.IGNORECASE | re.DOTALL
+_PROMPT_OPEN_RE: Final[re.Pattern[str]] = re.compile(r"<\s*prompt\s*>", re.IGNORECASE)
+_PROMPT_CLOSE_RE: Final[re.Pattern[str]] = re.compile(r"<\s*/\s*prompt\s*>", re.IGNORECASE)
+# The fallbacks, most recent boundary first: everything after the draft is the
+# corrected message, everything after the analysis is at worst draft + message.
+_CLOSE_FALLBACKS: Final[tuple[re.Pattern[str], ...]] = (
+ re.compile(r"<\s*/\s*draft\s*>", re.IGNORECASE),
+ re.compile(r"<\s*/\s*analysis\s*>", re.IGNORECASE),
)
-_ANALYSIS_CLOSE_RE: Final[re.Pattern[str]] = re.compile(r"<\s*/\s*analysis\s*>", re.IGNORECASE)
# Any leftover tag of ours the model echoed inside the message it wrote.
_OUR_TAGS_RE: Final[re.Pattern[str]] = re.compile(
- r"<\s*/?\s*(?:analysis|prompt)\s*>", re.IGNORECASE
+ r"<\s*/?\s*(?:analysis|draft|prompt)\s*>", re.IGNORECASE
)
-# A courtesy sign-off on the last line. v3 guaranteed one in code; v4 removes
-# it, and this cuts the one a model adds on its own — the message ends on its
-# last piece of substance, because the reader is a coding agent that gains
-# nothing from being thanked. Anchored to the LAST line and matched short: a
-# paragraph that merely contains "thanks" is the user's own wording and stays.
-_SIGN_OFF_LINE_RE: Final[re.Pattern[str]] = re.compile(
- r"^\W{0,3}(?:\w+\s+){0,4}?"
+# A courtesy sign-off. v3 guaranteed one in code; v4 removes it, and this cuts
+# the one a model adds on its own — the message ends on its last piece of
+# substance, because the reader is a coding agent that gains nothing from
+# being thanked.
+#
+# It comes in two shapes and both have been measured live on 2026-08-27: a
+# line of its own ("...\n\nDanke!") and a sentence tacked onto the end of the
+# last paragraph ("...lasse ihn unverändert. Danke dir."). Same vocabulary,
+# two anchors — the second one needs a sentence boundary in front of it, which
+# also keeps a message that IS a thank-you from being erased.
+_SIGN_OFF_WORDS: Final[str] = (
r"(?:thanks?|thank\s+you|dank(?:e|esch[öo]n)?|gracias|merci|grazie|obrigad[oa]|"
r"bedankt|tak|tack|takk|kiitos|dzi[eę]kuj[eę]|d[ěe]kuji|спасибо|ありがとう|谢谢|"
r"regards|cheers|gr[üu][ßs]e)"
- r"[\w\s,!.…]{0,40}$",
+)
+_SIGN_OFF_LINE_RE: Final[re.Pattern[str]] = re.compile(
+ rf"^\W{{0,3}}(?:\w+\s+){{0,4}}?{_SIGN_OFF_WORDS}[\w\s,!.…]{{0,40}}$",
+ re.IGNORECASE,
+)
+_SIGN_OFF_SENTENCE_RE: Final[re.Pattern[str]] = re.compile(
+ rf"(?<=[.!?…])[ \t]+(?:\w+\s+){{0,3}}?{_SIGN_OFF_WORDS}[\w\s,]{{0,30}}[.!…]*\s*$",
re.IGNORECASE,
)
@@ -180,48 +207,79 @@ to an AI coding agent. The user will paste what you write into that agent \
themselves, so it has to read like a well-written request from a colleague: \
complete, calm, and no longer than it needs to be.
-ANSWER IN TWO BLOCKS, IN THIS ORDER, AND NEVER SKIP THE FIRST.
+ANSWER IN THREE BLOCKS, IN THIS ORDER, AND NEVER SKIP ONE.
<analysis>
...your working pass over the transcript, worked through item by item...
</analysis>
+<draft>
+...your first version of the message...
+</draft>
<prompt>
-...the message, and nothing else...
+...the message again, corrected by the read-back, and nothing else...
</prompt>
-PHASE 1 - THE ANALYSIS. Work through all six points below in writing before \
+PHASE 1 - THE ANALYSIS. Work through all seven points below in writing before \
you compose a single sentence of the message. This is not a summary and it is \
not for the user; nobody will read it. It exists so that you have actually \
been through the transcript by the time you write. Take the time it needs - a \
careful pass here is the whole difference between a message that carries what \
the user said and one that carries your impression of it.
-1. TASKS. Number every separate thing the user asked for, in the order \
+1. LANGUAGE. Name the language the transcript is in, and quote the four or \
+five words that told you. THE MESSAGE IS WRITTEN IN THAT LANGUAGE, whatever \
+language these instructions or the examples below happen to be in. A German \
+transcript gets a German message, a Spanish one a Spanish message. Note also \
+which form of address the user used, and whether that language distinguishes \
+a familiar from a formal "you".
+2. TASKS. Number every separate thing the user asked for, in the order \
spoken. People dictate two, three, five small tasks in one breath, and the \
small one is dropped far more often than the big one. Write the count.
-2. LITERALS. List every file name, folder, path, symbol, function, class, \
+3. LITERALS. List every file name, folder, path, symbol, function, class, \
setting, product name, error message, quoted string, number, version and \
proper name the user said - exactly as they said it. Each of these must \
appear in the message, spelled the same way.
-3. THE SITUATION. What does the user see, what did they expect instead, and \
-where does it happen? Quote the phrase they used for it. This is the most \
-valuable thing a transcript holds and the first thing a hasty writer throws \
-away.
-4. LIMITS. Every restriction, preference, exclusion or bound they stated: \
-"only", "never", "not the live app", "keep the wording", "for now", a \
-deadline, a scope they drew.
-5. WORDING TO KEEP. The phrases the user chose that carry meaning and must \
-survive into the message unchanged - a term of art, a name they use for \
-something, a distinction they drew, a word they repeated. Also note which \
-form of address they used, and whether their language distinguishes a \
-familiar from a formal "you".
-6. NOT SAID. What is missing, ambiguous, or a pointer you cannot resolve \
+4. THE SITUATION. What does the user see, what did they expect instead, and \
+where does it happen? Quote the transcript verbatim for each of those, in the \
+transcript's own language. This is the most valuable thing a transcript holds \
+and the first thing a hasty writer throws away.
+5. LIMITS. Every restriction, preference, exclusion or bound they stated, each \
+one quoted verbatim: "only", "never", "not the live app", "keep the wording", \
+"for now", a deadline, a scope they drew.
+6. WORDING TO KEEP. Quote the phrases the user chose that carry meaning and \
+must survive into the message unchanged - a term of art, a name they use for \
+something, a distinction they drew, a word they repeated, a hedge like "I \
+think" that must not harden into a finding. Beside each, name what would be \
+lost if you replaced it with a synonym.
+7. NOT SAID. What is missing, ambiguous, or a pointer you cannot resolve \
("that one", "the second option"). These are left OUT of the message. Never \
guess a file, a cause or a mechanism to fill a gap.
-PHASE 2 - THE MESSAGE. Write it against the analysis. Then check it back \
-against points 1 and 2 before you close the block: every task present, every \
-literal present.
+PHASE 2 - THE DRAFT. Write the message against the analysis, in the \
+``<draft>`` block. This is a first version, not the answer.
+
+PHASE 3 - THE READ-BACK. Go through the draft once against the seven \
+questions below and write the corrected message into the ``<prompt>`` block. \
+When the draft was already right, the two blocks are simply the same text - \
+that is a fine outcome and not a reason to change anything for the sake of \
+it. Do NOT write out what you found; the corrections go into the message, not \
+next to it.
+
+a. Is it in the language from point 1?
+b. Is every task from point 2 there, none merged and none dropped?
+c. Is every literal from point 3 there, spelled the way the user said it?
+d. Does any sentence state something the user did not say? A cause they only \
+suspected, a fix they never named, a "so that it works everywhere again" \
+nobody asked for. Cut it - a gap the transcript leaves is left open.
+e. Did you replace a word of theirs from point 6 with one of your own? Put \
+theirs back.
+f. Does any sentence say again what the sentence before it already said, in \
+other words? One task must not appear twice - once as a description of the \
+problem and once as an instruction. Keep the clearer of the two and delete \
+the other outright.
+g. Does it end on the last piece of substance? Cut a closing line of thanks, \
+a "Danke dir" tacked onto the last sentence, a sign-off, an offer to help \
+further - even when the user said it themselves at the end of the dictation.
THE USER'S OWN WORDS ARE THE DEFAULT; REWRITING IS THE EXCEPTION. You are \
cleaning up a transcript, not authoring a document. Change a word only when \
@@ -270,10 +328,11 @@ sure that" clause that repeats the sentence before it, no summary. A sentence \
that adds no fact the message does not already have is left out.
PLAIN TEXT, SAME LANGUAGE. No markdown: no headings, no "##", no bullet \
-marks, no bold, no code fences, no labels like "Task:" or "Context:". Write \
-in the language the transcript is in; if it mixes languages, keep the mix. \
-Names, identifiers, paths and quoted strings stay exactly as spoken. Use the \
-ordinary hyphen-minus and ordinary spaces: no typographic hyphens, no \
+marks, no bold, no code fences, no labels like "Task:" or "Context:". Write in \
+the language you named in point 1 - these instructions are in English and that \
+means nothing about the message; if the transcript mixes languages, keep the \
+mix. Names, identifiers, paths and quoted strings stay exactly as spoken. Use \
+the ordinary hyphen-minus and ordinary spaces: no typographic hyphens, no \
non-breaking spaces, no two spaces at the end of a line.
SOUND LIKE THE PERSON WHO SPOKE, writing to a colleague - not like a ticket \
@@ -297,8 +356,10 @@ preamble, no quotes, no comment of your own.
{FORBIDDEN_SUBJECTS_RULE}
-One worked example of the shape. The analysis is shown short here; yours is \
-as long as the transcript needs.
+Two worked examples, one per language, because THE MESSAGE FOLLOWS THE \
+TRANSCRIPT AND NOT THE EXAMPLE. Measured live 2026-08-27: with only the \
+English pair shown, a German dictation came back as an English message. The \
+analyses are shown short here; yours is as long as the transcript needs.
Transcript: "okay so um the login page is broken again when you type a wrong \
password it just shows a blank screen instead of the error message i think \
@@ -306,19 +367,31 @@ it's in the auth handler file can you have a look and fix it so the message \
shows up like it used to and also rename the save button to submit"
Answer:
<analysis>
-1. TASKS (2): (a) fix the blank screen on a wrong password so the error \
+1. LANGUAGE: English ("okay so um", "can you have a look"). The message is \
+written in English. Informal, second person.
+2. TASKS (2): (a) fix the blank screen on a wrong password so the error \
message shows again; (b) rename the save button to submit.
-2. LITERALS: "login page", "auth handler file", "save", "submit", "wrong \
+3. LITERALS: "login page", "auth handler file", "save", "submit", "wrong \
password", "blank screen", "error message".
-3. SITUATION: typing a wrong password shows a blank screen instead of the \
-error message; the user says it is broken "again" and that it used to work.
-4. LIMITS: none stated.
-5. WORDING TO KEEP: "broken again", "I think it's in" (a guess, not a \
-finding - do not harden it into a cause), "like it used to". Informal, second \
-person.
-6. NOT SAID: which file exactly, what changed, no repro steps beyond the \
+4. SITUATION: "it just shows a blank screen instead of the error message", on \
+a wrong password. "broken again" and "like it used to" say it worked before.
+5. LIMITS: none stated.
+6. WORDING TO KEEP: "broken again" (not "is failing" - it has failed before); \
+"I think it's in" (a guess, not a finding - "the cause is in" would send the \
+agent off certain about something the user only suspects); "like it used to" \
+(names the old behaviour as the target).
+7. NOT SAID: which file exactly, what changed, no repro steps beyond the \
wrong password.
</analysis>
+<draft>
+The login page is failing again: on a wrong password the error message does \
+not appear and the screen stays blank. The cause is in the auth handler file.
+
+Please fix it so the error message is shown again and users get proper \
+feedback on a failed login.
+
+Also, please rename the save button to submit.
+</draft>
<prompt>
The login page is broken again: when you type a wrong password it just shows \
a blank screen instead of the error message. I think it's in the auth handler \
@@ -327,6 +400,42 @@ file.
Please have a look and fix it so the error message shows up like it used to.
Also, please rename the save button to submit.
+</prompt>
+
+Transcript: "ähm also die Indikatoren funktionieren nicht mehr richtig ich \
+möchte dass man sieht ob die Session gerade arbeitet oder nicht weil es ist \
+zum Beispiel so dass angezeigt wird sie ist fertig obwohl sie noch arbeitet \
+oder andersrum und live aktualisieren tun die sich auch nicht"
+Answer:
+<analysis>
+1. LANGUAGE: German ("ähm also", "ich möchte dass", "oder andersrum"). The \
+message is written in German. "du"-Form is not shown either way, so the \
+familiar form is used.
+2. TASKS (1): repair the session indicators.
+3. LITERALS: "Indikatoren", "Session".
+4. SITUATION: "angezeigt wird sie ist fertig obwohl sie noch arbeitet oder \
+andersrum", and "live aktualisieren tun die sich auch nicht".
+5. LIMITS: none stated.
+6. WORDING TO KEEP: "funktionieren nicht mehr richtig" - they EXIST and are \
+broken, so this is a repair and never "implementiere Indikatoren"; "live" is \
+the user's word for the update behaviour.
+7. NOT SAID: since when, which screen, no file named.
+</analysis>
+<draft>
+Die Statusanzeigen der Sessions sind fehlerhaft: Eine laufende Session wird \
+als abgeschlossen dargestellt und umgekehrt, und eine Aktualisierung in \
+Echtzeit findet nicht statt.
+
+Bitte implementiere eine zuverlässige Anzeige des Session-Status, die sich \
+automatisch aktualisiert, damit die Übersicht wieder verwendbar ist.
+</draft>
+<prompt>
+Die Indikatoren, die anzeigen, ob eine Session gerade arbeitet, funktionieren \
+nicht mehr richtig: Eine Session wird als fertig angezeigt, obwohl sie noch \
+arbeitet, und andersrum. Live aktualisieren sie sich auch nicht.
+
+Bitte bring die Indikatoren wieder in Ordnung, sodass man sieht, ob eine \
+Session gerade arbeitet, und sodass sie sich live aktualisieren.
</prompt>\
"""
@@ -423,17 +532,18 @@ def build_prompt_mode_prompt(protected_terms: Sequence[str] = ()) -> str:
def extract_prompt_block(answer: str) -> str:
- """The message out of a two-block answer, tolerant of a ragged one.
-
- In order of preference: the content of ``<prompt>`` (closing tag optional,
- because a model that ran out of budget still wrote a usable message up to
- that point — the truncation guard is what decides whether it is usable);
- otherwise everything after ``</analysis>``, which is the same message with
- the opening tag forgotten; otherwise the whole answer, on the assumption
- that the model ignored the two-block instruction and wrote the message
- directly. That last case then faces the same guards as before, so a model
- that answers with its analysis is caught by the markdown guard rather than
- handing the user a numbered worksheet.
+ """The message out of a three-block answer, tolerant of a ragged one.
+
+ In order of preference: the content of the LAST ``<prompt>`` (closing tag
+ optional, because a model that ran out of budget still wrote a usable
+ message up to that point — the truncation guard is what decides whether it
+ is usable); otherwise everything after ``</draft>`` and then everything
+ after ``</analysis>``, which is the same message with an opening tag
+ forgotten; otherwise the whole answer, on the assumption that the model
+ ignored the block instruction and wrote the message directly. That last
+ case then faces the same guards as before, so a model that answers with
+ its worksheet is caught by the markdown guard rather than handing the user
+ a numbered list.
Whatever comes out is stripped of any of our own tags the model echoed
inside it.
@@ -441,12 +551,19 @@ def extract_prompt_block(answer: str) -> str:
body = str(answer or "").strip()
if not body:
return ""
- match = _PROMPT_BLOCK_RE.search(body)
- if match:
- return _OUR_TAGS_RE.sub("", match.group(1)).strip()
- close = _ANALYSIS_CLOSE_RE.search(body)
- if close:
- return _OUR_TAGS_RE.sub("", body[close.end() :]).strip()
+ opens = list(_PROMPT_OPEN_RE.finditer(body))
+ if opens:
+ # The LAST opening tag, not the last complete block: a model that
+ # names the tag while thinking ("...then I fill the <prompt> block...")
+ # opens one inside its analysis and never closes it, and a match that
+ # started there would swallow the draft along with the message.
+ rest = body[opens[-1].end() :]
+ close = _PROMPT_CLOSE_RE.search(rest)
+ return _OUR_TAGS_RE.sub("", rest[: close.start()] if close else rest).strip()
+ for pattern in _CLOSE_FALLBACKS:
+ closes = list(pattern.finditer(body))
+ if closes:
+ return _OUR_TAGS_RE.sub("", body[closes[-1].end() :]).strip()
return _OUR_TAGS_RE.sub("", body).strip()
@@ -484,27 +601,36 @@ def normalize_prompt_text(text: str) -> str:
def ends_with_sign_off(text: str) -> bool:
- """Whether the LAST line of *text* is a short courtesy sign-off."""
- lines = [line.strip() for line in str(text or "").strip().splitlines() if line.strip()]
- if not lines:
+ """Whether *text* closes on a courtesy sign-off, as a line or a sentence."""
+ body = str(text or "").strip()
+ if not body:
return False
- return bool(_SIGN_OFF_LINE_RE.match(lines[-1]))
+ lines = [line.strip() for line in body.splitlines() if line.strip()]
+ if lines and _SIGN_OFF_LINE_RE.match(lines[-1]):
+ return True
+ return bool(_SIGN_OFF_SENTENCE_RE.search(body))
def strip_closing_sign_off(text: str) -> str:
- """*text* with a trailing courtesy line removed.
+ """*text* with a trailing courtesy line or sentence removed.
The prompt asks for a message that ends on its substance; this is the
guarantee, the way ``ensure_closing_thanks`` was the guarantee of the
- opposite in v3. Repeated, because a model that writes "Danke!" under
- "Viele Grüße" has written two. Never strips the whole message: a
- transcript that IS a thank-you note keeps its only line.
+ opposite in v3. Loops, because a model that writes "Danke!" under "Viele
+ Grüße" has written two, and because cutting the last line can expose
+ another underneath it. Never strips the whole message: a line of thanks
+ that is all there is stays, and a sentence is only cut when a finished
+ sentence stands in front of it.
Runs AFTER the truncation guard, for the reason that guard exists — a
message that stopped mid-sentence must be caught as damage, not tidied.
"""
body = str(text or "").strip()
while True:
+ trimmed = _SIGN_OFF_SENTENCE_RE.sub("", body).strip()
+ if trimmed != body and trimmed:
+ body = trimmed
+ continue
lines = body.splitlines()
kept = [line.strip() for line in lines if line.strip()]
if len(kept) < 2 or not _SIGN_OFF_LINE_RE.match(kept[-1]):
diff --git a/tests/unit/dictation/test_prompt_mode.py b/tests/unit/dictation/test_prompt_mode.py
index a96af51369..9ef2317942 100644
--- a/tests/unit/dictation/test_prompt_mode.py
+++ b/tests/unit/dictation/test_prompt_mode.py
@@ -7,9 +7,11 @@ Five things are worth pinning, and none of them is "the model writes well":
ritual, no reasoning echo — so the two rule texts are imported from the
blueprint and asserted to be the very same strings.
2. **The writer THINKS before it answers (v4, 2026-08-27).** A model spends
- time by emitting tokens, so the answer is two blocks: a written pass over
- the transcript, then the message. The pass is parsed off and thrown away,
- and a model that skips it is the v3 defect this revision exists to fix.
+ time by emitting tokens, so the answer is three blocks: a written pass over
+ the transcript, a draft, and the message the read-back corrects. Only the
+ last one is delivered, and a model that skips the first is the v3 defect
+ this revision exists to fix. Measured live the same day: 5.2 s on
+ gemini-3.5-flash-lite, which is the bar, against 0.4-0.9 s for v3.
3. **The maintainer's bar: plain text, the transcript's language, the
speaker's own words, several tasks dictated at once all surviving, and
nothing after the last piece of substance.** Each is a rule in the prompt
@@ -68,16 +70,28 @@ GOOD = (
"please fix that.\n\n"
"Also rename the save button to submit."
)
+#: What the draft looked like before the read-back: the hedge hardened into a
+#: finding, and a promise the transcript never made.
+DRAFT = (
+ "The login page is failing again. The cause is in AuthHandler.\n\n"
+ "Also rename the save button to submit, so the wording is consistent "
+ "everywhere."
+)
ANALYSIS = (
- "1. TASKS (2): fix the blank screen; rename the save button to submit.\n"
- "2. LITERALS: login page, AuthHandler, save, submit.\n"
- "3. SITUATION: a wrong password shows a blank screen.\n"
- "4. LIMITS: none stated.\n"
- "5. WORDING TO KEEP: broken again, I think it's in.\n"
- "6. NOT SAID: which file exactly."
+ "1. LANGUAGE: English. The message is written in English.\n"
+ "2. TASKS (2): fix the blank screen; rename the save button to submit.\n"
+ "3. LITERALS: login page, AuthHandler, save, submit.\n"
+ "4. SITUATION: a wrong password shows a blank screen.\n"
+ "5. LIMITS: none stated.\n"
+ "6. WORDING TO KEEP: broken again, I think it's in.\n"
+ "7. NOT SAID: which file exactly."
+)
+#: What a well-behaved model returns: the working pass, a draft, the message.
+ANSWER = (
+ f"<analysis>\n{ANALYSIS}\n</analysis>\n"
+ f"<draft>\n{DRAFT}\n</draft>\n"
+ f"<prompt>\n{GOOD}\n</prompt>"
)
-#: What a well-behaved model returns: the working pass, then the message.
-ANSWER = f"<analysis>\n{ANALYSIS}\n</analysis>\n<prompt>\n{GOOD}\n</prompt>"
class FakeChain:
@@ -146,44 +160,91 @@ def test_the_prompt_carries_the_blueprints_rules_verbatim() -> None:
# --------------------------------------------------------------------------- #
-def test_the_prompt_demands_an_analysis_before_the_message() -> None:
+def test_the_prompt_demands_an_analysis_and_a_read_back() -> None:
"""The whole point of v4. v3 asked for the message directly and a fast
model wrote it in under a second — no pass over the transcript at all."""
system = build_prompt_mode_prompt()
- assert "ANSWER IN TWO BLOCKS" in system
- assert "NEVER SKIP THE FIRST" in system
- assert "<analysis>" in system and "</analysis>" in system
- assert "<prompt>" in system and "</prompt>" in system
+ assert "ANSWER IN THREE BLOCKS" in system
+ assert "NEVER SKIP ONE" in system
+ for tag in ("<analysis>", "</analysis>", "<draft>", "</draft>",
+ "<prompt>", "</prompt>"):
+ assert tag in system, tag
assert "PHASE 1 - THE ANALYSIS" in system
- assert "PHASE 2 - THE MESSAGE" in system
+ assert "PHASE 2 - THE DRAFT" in system
+ assert "PHASE 3 - THE READ-BACK" in system
-def test_the_analysis_names_all_six_points_it_must_work_through() -> None:
+def test_the_analysis_names_all_seven_points_it_must_work_through() -> None:
system = build_prompt_mode_prompt()
- for point in ("1. TASKS", "2. LITERALS", "3. THE SITUATION", "4. LIMITS",
- "5. WORDING TO KEEP", "6. NOT SAID"):
+ for point in ("1. LANGUAGE", "2. TASKS", "3. LITERALS", "4. THE SITUATION",
+ "5. LIMITS", "6. WORDING TO KEEP", "7. NOT SAID"):
assert point in system, point
assert "before you compose a single sentence" in system
-def test_only_the_message_block_is_delivered() -> None:
+def test_the_read_back_asks_the_six_questions_that_catch_the_known_failures() -> None:
+ """Each one is a defect measured live rather than a checklist item: the
+ wrong language, a merged task, a dropped file name, a claim the user never
+ made, a word swapped for a synonym, a courtesy line."""
+ system = build_prompt_mode_prompt()
+ assert "Is it in the language from point 1?" in system
+ assert "none merged and none dropped" in system
+ assert "spelled the way the user said it" in system
+ assert "state something the user did not say" in system
+ assert "Put theirs back" in system
+ assert "say again what the sentence before it already said" in system
+ assert "Cut a closing line of thanks" in system
+ # ...including one the user said themselves; the reader is an agent.
+ assert "even when the user said it themselves" in system
+ # The read-back is a correction, not a report the user has to read past.
+ assert "not next to it" in system
+
+
+def test_the_language_rule_is_a_point_of_the_analysis() -> None:
+ """Measured live 2026-08-27: with the rule written but not worked through,
+ and only an English example shown, a German dictation came back English."""
+ system = build_prompt_mode_prompt()
+ assert "THE MESSAGE IS WRITTEN IN THAT LANGUAGE" in system
+ assert "A German transcript gets a German message" in system
+ assert "these instructions are in English and that means nothing" in system
+
+
+def test_only_the_final_message_is_delivered() -> None:
assert extract_prompt_block(ANSWER) == GOOD
-def test_a_forgotten_opening_tag_still_yields_the_message() -> None:
- """A fast model drops the opening tag often enough to matter; everything
- after the analysis is the message either way."""
- ragged = f"<analysis>\n{ANALYSIS}\n</analysis>\n{GOOD}"
+def test_the_draft_is_never_delivered() -> None:
+ """It is the version the read-back exists to correct. Handing it over
+ would make the whole third phase decorative."""
+ out = extract_prompt_block(ANSWER)
+ assert "The cause is in" not in out
+ assert "so the wording is consistent everywhere" not in out
+
+
+def test_a_forgotten_opening_tag_still_yields_the_final_message() -> None:
+ """A fast model drops a tag often enough to matter; everything after the
+ draft is the corrected message either way."""
+ ragged = f"<analysis>\n{ANALYSIS}\n</analysis>\n<draft>\n{DRAFT}\n</draft>\n{GOOD}"
assert extract_prompt_block(ragged) == GOOD
def test_a_missing_closing_tag_still_yields_the_message() -> None:
"""The answer ran out of budget mid-message. What it wrote comes out and
faces the truncation guard, which is the thing that decides."""
- cut = f"<analysis>\n{ANALYSIS}\n</analysis>\n<prompt>\n{GOOD}"
+ cut = f"<analysis>\n{ANALYSIS}\n</analysis>\n<draft>\n{DRAFT}\n</draft>\n<prompt>\n{GOOD}"
assert extract_prompt_block(cut) == GOOD
+def test_the_tag_named_inside_the_analysis_does_not_win() -> None:
+ """A model that writes "...then the <prompt> block..." while thinking would
+ otherwise hand us its worksheet. The LAST block is the answer."""
+ chatty = (
+ "<analysis>\n1. LANGUAGE: English. Next I fill the <prompt> block.\n</analysis>\n"
+ f"<draft>\n{DRAFT}\n</draft>\n<prompt>\n{GOOD}\n</prompt>"
+ )
+ assert extract_prompt_block(chatty) == GOOD
+
+
def test_an_answer_with_no_blocks_at_all_is_taken_whole() -> None:
"""The v3 shape. It is not thrown away — it faces the same guards it
always faced, so a good message still gets through."""
@@ -255,13 +316,34 @@ def test_the_prompt_describes_the_shape_of_the_message() -> None:
assert "no typographic hyphens" in system
-def test_the_prompt_shows_one_worked_two_block_answer() -> None:
- """A fast model follows a shown shape far better than a described one, and
- what has to be shown now is the SHAPE OF THE ANSWER, not of the message."""
+def test_the_prompt_shows_one_worked_answer_per_language_it_ships_for() -> None:
+ """A shown example outweighs a written rule, so the examples have to
+ DISAGREE about language for the language rule to be readable at all. With
+ only the English pair, a German dictation came back English (live,
+ 2026-08-27). Each shows the full three-block shape, draft included."""
+ system = build_prompt_mode_prompt()
+ assert system.count("Transcript: ") == 2
+ assert system.count("Answer:\n<analysis>") == 2
+ # The closing tag, not the opening one: the phase description names
+ # ``<draft>`` in running text as well, and only a real block closes.
+ assert system.count("</draft>") == 3, "one per example plus the shape itself"
+ assert "1. LANGUAGE: German" in system
+ assert "1. LANGUAGE: English" in system
+ # The examples are the LAST thing the model reads before the protected
+ # terms — nothing may follow them and dilute the shape they set.
+ assert prompt_mode._SYSTEM_PROMPT.rstrip().endswith("</prompt>")
+
+
+def test_each_shown_draft_carries_the_defect_its_read_back_removes() -> None:
+ """An example whose draft already equals its message teaches nothing about
+ the third phase. The English pair hardens a hedge into a finding; the
+ German one turns a repair into "implementiere"."""
system = build_prompt_mode_prompt()
- assert system.count("Transcript: ") == 1
- assert "Answer:\n<analysis>" in system
- assert system.rstrip().endswith("</prompt>") or "</prompt>" in system
+ assert "The cause is in the auth handler file" in system
+ assert "Bitte implementiere eine zuverlässige Anzeige" in system
+ # ...and neither survives into the message beside it.
+ assert "I think it's in the auth handler" in system
+ assert "Bitte bring die Indikatoren wieder in Ordnung" in system
def test_the_prompt_version_names_the_v4_revision() -> None:
@@ -461,7 +543,7 @@ def test_the_budget_covers_the_analysis_as_well_as_the_message() -> None:
source = inspect.getsource(prompt_mode._call_chain)
assert "max_output_tokens=_MAX_OUTPUT_TOKENS" in source
- assert prompt_mode._MAX_OUTPUT_TOKENS >= 3_000
+ assert prompt_mode._MAX_OUTPUT_TOKENS >= 4_000
assert prompt_mode._TEMPERATURE == 0.0
@@ -665,6 +747,23 @@ def test_a_trailing_thanks_is_cut() -> None:
assert strip_closing_sign_off("Please fix the page.\n\nThanks!") == "Please fix the page."
+ ("written", "kept"),
+ [
+ ("Bitte setze retry_count auf 10. Danke dir.", "Bitte setze retry_count auf 10."),
+ ("Fix it. Thanks a lot!", "Fix it."),
+ ("Please have a look. Thanks in advance.", "Please have a look."),
+ ],
+)
+def test_a_thanks_tacked_onto_the_last_sentence_is_cut(written: str, kept: str) -> None:
+ """Measured live 2026-08-27 on gpt-oss: the model did not put its thanks on
+ a line of its own, it appended it to the last paragraph — where a
+ line-anchored guard never sees it. The user had said "danke dir" at the end
+ of the dictation, and the writer carried it through."""
+ assert ends_with_sign_off(written) is True
+ assert strip_closing_sign_off(written) == kept
+
+
def test_a_stacked_sign_off_is_cut_to_the_last_piece_of_substance() -> None:
"""A model that writes "Danke!" under "Viele Grüße" has written two."""
assert strip_closing_sign_off("Bitte repariere die Seite.\n\nViele Grüße\n\nDanke!") == (