Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(agent-core): validate checkpoint output (#2364)
Call White334d8a70b6d8
diff --git a/core/agent_core/agent/checkpoint.py b/core/agent_core/agent/checkpoint.pyindex 2a1392601..5ab0662c0 100644--- a/core/agent_core/agent/checkpoint.py+++ b/core/agent_core/agent/checkpoint.py@@ -35,11 +35,14 @@ CHECKPOINT_TOOL_POLICY: Mapping[str, Rule] = MappingProxyType(})+#: Each ends by saying where the checkpoint belongs: a model denied a file write may otherwise keep trying one.DENIED = ("This is a checkpoint turn: tools that act outside your own scratch space are unavailable. "- "Write the checkpoint now."+ "Write the checkpoint as your reply text, not into a file."+)+BUDGET_USED = (+ "This is a checkpoint turn and its tool budget is used up. Write the checkpoint as your reply text, not into a file.")-BUDGET_USED = "This is a checkpoint turn and its tool budget is used up. Write the checkpoint now."@dataclass(frozen=True)diff --git a/core/agent_core/agent/loop.py b/core/agent_core/agent/loop.pyindex f5652d01e..4db506ed2 100644--- a/core/agent_core/agent/loop.py+++ b/core/agent_core/agent/loop.py@@ -84,6 +84,7 @@ from core.agent_core.harness.context import (budget,checkpoint_max_tokens,checkpoint_request,+ checkpoint_problem,checkpoint_text,clear_edit,clearable_results,@@ -1511,8 +1512,13 @@ class Agent:origin = message.originif message.stop_reason == "stop" and not message.tool_calls:checkpoint = checkpoint_text(message)+ problem = checkpoint_problem(checkpoint) if checkpoint else Noneif not checkpoint:error = "The checkpoint turn ended without checkpoint text."+ elif problem is not None:+ # Not a checkpoint (a meta note, a partial one): it fails, and its text stays in the+ # audit only; the old context is kept (section 6).+ checkpoint, error = "", f"The checkpoint turn's reply is not a checkpoint: {problem}."breakif message.stop_reason != "tool_use" or not message.tool_calls:# Only a stop with text and no call is a checkpoint, and only a tool-use stop carries calls.diff --git a/core/agent_core/harness/context.py b/core/agent_core/harness/context.pyindex 558eb8af5..e3e0117be 100644--- a/core/agent_core/harness/context.py+++ b/core/agent_core/harness/context.py@@ -14,6 +14,7 @@ from __future__ import annotationsimport jsonimport math+import reimport timeimport unicodedatafrom itertools import accumulate@@ -39,6 +40,7 @@ from core.agent_core.messages import (usage_to_dict,)from core.agent_core.tools.base import ToolSpec+from core.reply_enhancer import strip_silent_blocks# --- constants (context.md) -----------------------------------------------------@@ -567,6 +569,32 @@ def checkpoint_text(message: AssistantMessage) -> str:)+#: The approved template's top-level section markers (section 11), whatever language the headings are in.+_SECTION = re.compile(r"^#[ \t]*([123])\.", re.M)+++def checkpoint_problem(reply: str) -> Optional[str]:+ """Why a checkpoint turn's reply is not a checkpoint, or None when it is (section 6), judged on what the user+ would see of it: the product's one silent-block grammar (``strip_silent_blocks``, which delivery uses) removes its+ ``<silent>`` notes first. Then the template's three top-level markers (``# 1.``, ``# 2.``, ``# 3.``) appear+ exactly once each, in order, each section with content under its heading (a line that is not itself a heading).+ The one check every checkpoint path applies."""+ visible = strip_silent_blocks(reply)+ markers = list(_SECTION.finditer(visible))+ numbers = [marker.group(1) for marker in markers]+ for number in "123":+ if number not in numbers:+ return f"section {number} is missing"+ if numbers != ["1", "2", "3"]:+ return "a section marker is repeated or out of order"+ starts = [marker.start() for marker in markers]+ for number, (start, end) in enumerate(zip(starts, [*starts[1:], len(visible)]), 1):+ body = visible[start:end].splitlines()[1:] # the heading line itself is not content+ if not any(line.strip() and not line.lstrip().startswith("#") for line in body):+ return f"section {number} is empty"+ return None++def _unique(items: Sequence[str]) -> list[str]:return list(dict.fromkeys(items))diff --git a/docs/plans/agent-core-contracts/context.md b/docs/plans/agent-core-contracts/context.mdindex d989ce17c..d664de789 100644--- a/docs/plans/agent-core-contracts/context.md+++ b/docs/plans/agent-core-contracts/context.md@@ -176,7 +176,8 @@ budget (invariant 2):| `bash` and every other tool | denied, never executed |- A denied call gets the error result `This is a checkpoint turn: tools that act outside your own scratch space are- unavailable. Write the checkpoint now.`+ unavailable. Write the checkpoint as your reply text, not into a file.` (a model denied a file write may otherwise+ keep trying one; the owner-approved request, §11, stays untouched).- Every request of the turn is budgeted on the route resolved for it (§1): its `max_tokens` is that route's`min(16,000, O)`.- The bound is the window, not `T`, the same way in threshold and rolling turns. Before each call,@@ -191,16 +192,24 @@ budget (invariant 2):denial and the used-up budget) are a few dozen tokens and are never cut: a truncation note would invite anothercall.- At most 5 tool rounds run. After them, or once `room` falls below the floor, the budget is closed: every call gets- `This is a checkpoint turn and its tool budget is used up. Write the checkpoint now.` before the table is+ `This is a checkpoint turn and its tool budget is used up. Write the checkpoint as your reply text, not into a+ file.` before the table isconsulted, so the audit's `rounds` can count one more response, the one answered that way. A response that stillcalls tools after that ends the turn as failed.- `write` and `edit` in a checkpoint turn keep the tools' arguments and success texts (C-7); their errors are theirown, short and without the C-7 advice to use `bash`, which the turn may not call. `edit` reuses the tools'matching (`edit_diff`), BOM, and line-ending handling. The C-7 tools themselves are unchanged.- The checkpoint is the text of a final response that stops with `stop` and calls no tool; only a `tool_use` stop- with calls continues the turn. Anything else (a response refused at admission, a length stop, a `tool_use` stop- without calls, calls under any other stop, an error, or no text) is a failure, and nothing enters the context;- so is a checkpoint whose row the host cannot complete (§10, invariant 4).+ with calls continues the turn. That text is a checkpoint only if one validator, the same for every checkpoint+ turn (normal and rolling), accepts it, judged on what the user would see of it: the product's one silent-block+ grammar (`core/reply_enhancer.strip_silent_blocks`, which delivery uses) removes its `<silent>` notes first. Then+ the template's three top-level section markers (`# 1.`, `# 2.`, `# 3.` at the start of a line, whatever language+ the headings are in) appear exactly once each, in order, each section with content under its heading (a line that+ is not itself a heading). A reply wrapped in, or made only of, a `<silent>` note, or with a section missing,+ repeated, out of order, or empty, is no checkpoint. Anything else (a response refused at admission, a length stop, a `tool_use` stop without calls, calls+ under any other stop, an error, no text, or a reply the validator rejects) is a failure: it counts as unproductive+ (§10), nothing enters the context, the old context is kept, and the ladder continues; the reply's text is only in+ the audit row. So is a checkpoint whose row the host cannot complete (§10, invariant 4).- The turn's messages, read from the attempt ledger (§10, invariant 4) with every failed or retried attempt'spartial and its usage, and the turn's tool results, are recorded once per checkpoint turn, on every exit but anabort (§10, invariant 4), outcome included, as an audit rowdiff --git a/tests/agent_core/agent/test_compaction.py b/tests/agent_core/agent/test_compaction.pyindex e6576c423..a66df38d8 100644--- a/tests/agent_core/agent/test_compaction.py+++ b/tests/agent_core/agent/test_compaction.py@@ -52,6 +52,7 @@ from core.agent_core.harness.context import (from core.agent_core.harness.projection import context_view, projectfrom core.agent_core.messages import (AssistantMessage,+ TextBlock,ThinkingBlock,ToolCallBlock,ToolResultMessage,@@ -76,7 +77,8 @@ from tests.agent_core.fakes import (#: T = 19,904 (section 1), the numbers every test here is sized against.SELECTION = ModelSelection(ENDPOINT, replace(FAKE_SELECTION.capabilities, context_window=64_000, input_limit=32_000))THRESHOLD = 19_904-CHECKPOINT = "# 1. Self and method\n- the checkpoint"+#: A checkpoint with the template's three sections (section 11), which the validator accepts.+CHECKPOINT = "# 1. Self and method\n- the checkpoint\n# 2. Goals and requirements\n- the goal\n# 3. Now and next\n- the next step"def tokens(count: int) -> str:@@ -98,6 +100,11 @@ def checkpoint_message(request) -> str:return next(text for text in texts if text.startswith("<context-checkpoint>"))+def bulky(count: int) -> str:+ """A checkpoint of the template's three sections, about ``count`` tokens long."""+ return f"# 1. Self and method\n- {tokens(count)}\n# 2. Goals and requirements\n- the goal\n# 3. Now and next\n- next"++def call(name: str, call_id: str, **arguments) -> list[Done]:return [Done(assistant(calls=[ToolCallBlock(call_id, name, arguments)]))]@@ -737,7 +744,7 @@ async def test_a_new_turn_on_a_large_english_conversation_uses_the_stored_anchorasync def test_the_request_right_after_a_checkpoint_is_measured_and_rolls_when_it_does_not_fit():- big = [[Done(assistant(tokens(16_000)))], [Done(assistant("short"))]]+ big = [[Done(assistant(bulky(16_000)))], [Done(assistant(CHECKPOINT))]]model = Model(reads(6), big)agent = make_agent(model, tools=[sized_reader(5_000, 5_000, 5_000, 2_000, 2_000, 2_000)])events = await run(agent)@@ -747,7 +754,7 @@ async def test_the_request_right_after_a_checkpoint_is_measured_and_rolls_when_iasync def test_two_ineffective_checkpoints_stop_compaction_for_the_run():- bloated = [[Done(assistant(tokens(15_000)))] for _ in range(2)]+ bloated = [[Done(assistant(bulky(15_000)))] for _ in range(2)]model = Model(reads(7), bloated)agent = make_agent(model, tools=[sized_reader(6_000, 6_000, 6_000, 6_000, 5_000, 5_000, 5_000)])events = await run(agent)@@ -794,7 +801,7 @@ async def test_the_ladder_rolls_when_no_fork_can_take_the_whole_context():store = InMemoryTranscriptStore()await _history_on_a_large_window(store, [5_000] * 9)before = await store.load("session")- model = Model([[Done(assistant("done"))]], [[Done(assistant("ROLL ONE"))], [Done(assistant("ROLL TWO"))]])+ model = Model([[Done(assistant("done"))]], [[Done(assistant(f"{CHECKPOINT} ROLL ONE"))], [Done(assistant(f"{CHECKPOINT} ROLL TWO"))]])agent = make_agent(model, store=store)events = await run(agent)first, second = model.checkpoint_requests@@ -815,6 +822,68 @@ async def test_the_ladder_rolls_when_no_fork_can_take_the_whole_context():assert events[-1].reason == "completed"+#: The reply of the E2E checkpoint turn that replaced a Session's history: a write the turn denied, then a meta note.+_E2E_WRITE = ToolCallBlock(+ "toolu_write",+ "write",+ {"path": "/home/avibe/.avibe/workdir/e2e-scratch/CHECKPOINT.md", "content": "# 1. Self and method\n- strict"},+)+_E2E_NOTE = (+ "<silent>The function call was rejected because this is a checkpoint turn. I have attempted to write the "+ "checkpoint but hit the tool budget. The checkpoint content was prepared and ready; it documents the seven "+ "codewords collected (f01\u2013f07).</silent>"+)+++async def test_a_checkpoint_turn_that_replies_with_a_meta_note_fails_and_keeps_the_context():+ # The E2E reply: the model tried to write the checkpoint to a file, was denied, then replied with a <silent> note.+ # That is no checkpoint: it fails, counts, keeps the old context, and its text lives only in the audit row.+ model = Model(+ growing(4),+ [[Done(AssistantMessage((_E2E_WRITE,), assistant().origin, "tool_use"))], [Done(assistant(_E2E_NOTE))]],+ )+ agent = make_agent(model, tools=[reader(tokens(6_000))])+ events = await run(agent)+ failed = [event for event in events if isinstance(event, CompactionFailed)]+ assert len(failed) == 1 and "not a checkpoint" in failed[0].error+ assert events[-1].reason == "completed" and agent._unproductive == 1+ rows = await agent.store.load("session")+ assert not [row for row in rows if row.kind == "compaction"]+ sent = model.conversation_requests[-1]+ assert sent.messages == project(rows[: len(rows) - 1]).messages # the old context, whole+ assert not any("<silent>" in (block.text or "") for message in sent.messages for block in message.content+ if isinstance(block, TextBlock))+ (audit,) = [payload for _, _, payload in agent.store.audits] # the checkpoint-turn audits+ assert audit["outcome"] == "failed" and _E2E_NOTE in json.dumps(audit["messages"], ensure_ascii=False)+ # The denial told the model where the checkpoint belongs.+ denied = model.checkpoint_requests[1].messages[-1]+ assert denied.content[0].text.endswith("Write the checkpoint as your reply text, not into a file.")+++async def test_a_checkpoint_with_translated_headings_is_accepted():+ translated = (+ "# 1. 自我与方法\n## 我在这里的工作方式\n- 一步只读一个文件\n"+ "# 2. 目标与要求\n## 目标\n- 收集 f01 到 f27 的口令\n"+ "# 3. 现在与下一步\n## 进度\n- 已读 f01–f04"+ )+ model = Model(growing(4), [[Done(assistant(translated))]])+ agent = make_agent(model, tools=[reader(tokens(6_000))])+ await run(agent)+ (compaction,) = [row for row in await agent.store.load("session") if row.kind == "compaction"]+ assert compaction.payload["checkpoint"] == translated+++async def test_a_checkpoint_missing_its_second_section_is_rejected():+ partial = "# 1. Self and method\n- the checkpoint\n# 3. Now and next\n- the next step"+ model = Model(growing(4), [[Done(assistant(partial))]])+ agent = make_agent(model, tools=[reader(tokens(6_000))])+ events = await run(agent)+ assert [event.error for event in events if isinstance(event, CompactionFailed)] == [+ "The checkpoint turn's reply is not a checkpoint: section 2 is missing."+ ]+ assert not [row for row in await agent.store.load("session") if row.kind == "compaction"]++async def test_a_turn_keeps_its_input_and_every_steer_verbatim_across_a_mid_turn_checkpoint():# The Turn's input and a steer it accepted mid-Turn both stay as they were, in order, ahead of the checkpoint: a# checkpoint that rewrote either ("read f10 to f18" as "continue with f19") would drift what the Turn answers.@@ -1207,7 +1276,7 @@ class FailingTransactionStore(InMemoryTranscriptStore):async def test_invariant_3_a_c9_transition_commits_in_one_transaction_or_not_at_all():# A checkpoint's row commits in one transaction or not at all.store = FailingTransactionStore(failures=1)- model = Model(reads(4), [[Done(assistant(tokens(15_000)))]])+ model = Model(reads(4), [[Done(assistant(bulky(15_000)))]])agent = make_agent(model, store=store, tools=[sized_reader(6_000, 6_000, 6_000, 6_000)])events = await run(agent)rows = await store.load("session")@@ -1215,7 +1284,7 @@ async def test_invariant_3_a_c9_transition_commits_in_one_transaction_or_not_at_assert not [row for row in rows if row.kind == "compaction"]assert [type(event).__name__ for event in events if "Compaction" in type(event).__name__] == ["CompactionStarted"]assert events[-1].reason == "error"- model = Model(reads(1), [[Done(assistant(tokens(15_000)))]])+ model = Model(reads(1), [[Done(assistant(bulky(15_000)))]])agent = make_agent(model, store=store, tools=[sized_reader(1)])await run(agent, row="again")assert store.transactions[-1] == ["compaction"]diff --git a/tests/agent_core/agent/test_context.py b/tests/agent_core/agent/test_context.pyindex a3e3a9c3a..67ab1694b 100644--- a/tests/agent_core/agent/test_context.py+++ b/tests/agent_core/agent/test_context.py@@ -22,6 +22,7 @@ from core.agent_core.harness.context import (ITEM_BYTES,budget,carried_skills,+ checkpoint_problem,checkpoint_request,clearable_results,TOOL_TRUNCATED,@@ -759,6 +760,39 @@ def test_a_displayed_path_or_skill_name_is_one_line_of_plain_text():assert payload["files_read"] == [hostile] # the row keeps the original+_VALID = "# 1. Self and method\n- terse\n# 2. Goals and requirements\n- fix it\n# 3. Now and next\n- read f05 next"+++ "reply,problem",+ [+ (_VALID, None),+ # The headings may be written in the user's language; the numbered markers carry the structure.+ ("# 1. 自我与方法\n- 简洁\n# 2. 目标与要求\n- 收集口令\n# 3. 现在与下一步\n- 下一步读 f05", None),+ (f"{_VALID}\n<silent>an aside</silent>", None),+ ("# 1. Self and method\n- terse\n# 3. Now and next\n- next", "section 2 is missing"),+ ("# 2. Goals\n- g\n# 1. Self\n- s\n# 3. Now\n- n", "a section marker is repeated or out of order"),+ # Every marker counts: a misplaced one is not skipped for a later copy, and none may repeat.+ ("# 1. S\n- s\n# 3. N\n- n\n# 2. G\n- g\n# 3. N\n- n", "a section marker is repeated or out of order"),+ ("# 1. S\n- s\n# 2. G\n- g\n# 2. G\n- g\n# 3. N\n- n", "a section marker is repeated or out of order"),+ ("# 1. Self and method\n# 2. Goals\n- g\n# 3. Now\n- n", "section 1 is empty"),+ ("# 1. Self\n## How I work here\n# 2. Goals\n- g\n# 3. Now\n- n", "section 1 is empty"),+ ("# 1. Self\n- s\n# 2. Goals\n- g\n# 3. Now and next", "section 3 is empty"),+ (f"<silent>\n{_VALID}\n</silent>", "section 1 is missing"),+ # The product's own silent grammar decides what is hidden, exactly as delivery would: any case, attributes,+ # a spaced closing tag; and a tag shown in a code span hides nothing.+ (f"<SILENT>\n{_VALID}\n</SILENT>", "section 1 is missing"),+ (f'<silent reason="private">\n{_VALID}\n</silent >', "section 1 is missing"),+ (f"{_VALID}\n- the reply grammar hides `<silent>` notes", None),+ (f"<silent>unclosed\n{_VALID}", "section 1 is missing"),+ ("<silent>The function call was rejected because this is a checkpoint turn.</silent>", "section 1 is missing"),+ ("", "section 1 is missing"),+ ],+)+def test_only_a_reply_with_the_three_sections_outside_any_silent_block_is_a_checkpoint(reply, problem):+ assert checkpoint_problem(reply) == problem++def test_the_checkpoint_request_is_the_owner_approved_prompt_verbatim():contract = CONTRACT.read_text()section = contract[contract.index("## 11. Checkpoint request") :]diff --git a/tests/test_avibe_agent_context.py b/tests/test_avibe_agent_context.pyindex 08934cb5a..3925196e3 100644--- a/tests/test_avibe_agent_context.py+++ b/tests/test_avibe_agent_context.py@@ -33,7 +33,8 @@ from tests.test_avibe_agent import ( # noqa: F401 (fixtures))from vibe.i18n import t as i18n_t-CHECKPOINT = "# 1. Self and method\n- the checkpoint"+#: A checkpoint with the template's three sections (section 11), which the validator accepts.+CHECKPOINT = "# 1. Self and method\n- the checkpoint\n# 2. Goals and requirements\n- the goal\n# 3. Now and next\n- the next step"def tokens(count: int) -> str: