Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: restore sticky checkpoint boundaries on tool-heavy threads (#9971)

alkinun193841e7591f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index e0b3c40e1f2..b324e8bec40 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -1069,6 +1069,208 @@ def _archive_content_on_branch(content, transcript: Optional[list[str]]) -> bool
return True
+def _row_replayed(message: dict) -> bool:
+ """Whether the client re-sends this reply, by `isAbandonedAssistantTurn`.
+
+ Status is not the test: a Stop that reached text is kept, and only a reply carrying
+ nothing is dropped -- along with the tool results its wire projection would expand to.
+ A turn that FINISHED on reasoning alone is a reply too, replayed as `reasoning_content`,
+ even though `_as_wire` strips reasoning and leaves it nothing to be matched on.
+ """
+ if message.get("role") != "assistant":
+ return True
+ content = message.get("content")
+ if isinstance(content, str):
+ return bool(content.strip())
+ reasoning = False
+ for part in content or ():
+ if not isinstance(part, dict):
+ continue
+ if part.get("type") == "text" and str(part.get("text") or "").strip():
+ return True
+ if part.get("type") in ("image", "image-url", "file"):
+ return True
+ reasoning = reasoning or part.get("type") == "reasoning"
+ if reasoning and not _row_ended_early(message):
+ return True
+ return bool(message.get("attachments"))
+
+
+def _row_ended_early(message: dict) -> bool:
+ """`assistantTurnEndedEarly`: the persisted marker, since status is session state."""
+ metadata = message.get("metadata")
+ metadata = metadata if isinstance(metadata, dict) else {}
+ custom = metadata.get("custom")
+ custom = custom if isinstance(custom, dict) else {}
+ return bool(metadata.get("incomplete") or custom.get("incomplete"))
+
+
+def _archive_branch_chain(
+ messages: list[dict], branch_messages: Optional[list[dict]]
+) -> Optional[list[dict]]:
+ """The stored parent chain selected by this wire branch, when ancestry can prove one."""
+ if not branch_messages:
+ return None
+ try:
+ from core.rag import conversation_archive
+
+ def _chain(probe):
+ return conversation_archive._active_chain(
+ messages,
+ probe,
+ fallback = False,
+ require_unique = True,
+ )
+
+ chain = _chain(branch_messages)
+ if not chain:
+ return None
+ # The unstored newest turn only matches on a SIBLING, so settled turns pick the branch.
+ settled = list(branch_messages)
+ while settled and settled[-1].get("role") == "user":
+ settled.pop()
+ if settled:
+ proof = _chain(settled)
+ if not proof:
+ return None
+ tip = proof[-1].get("id")
+ if not any(row.get("id") == tip for row in chain):
+ return None
+
+ # Rows past that tip ride on the unstored turns alone, so they may only carry
+ # THOSE turns' text: matching the whole request instead let an abandoned row
+ # in on a text repeated earlier in it. Keyed by ROLE too, as the branch match
+ # is, so a stored reply cannot be justified by a user turn of the same words.
+ def _keys(rows):
+ return [
+ (wire.get("role"), _archive_message_text(wire.get("content")))
+ for wire in conversation_archive._as_wire(list(rows))
+ ]
+
+ carried = set(_keys(branch_messages[len(settled) :]))
+ past_tip = False
+ for row in chain:
+ if past_tip:
+ checked = False
+ for key in _keys([row]):
+ if not key[1]:
+ continue
+ if key not in carried:
+ return None
+ checked = True
+ # Nothing comparable came back. Only a reply the client drops whole may
+ # ride on the unstored turns unchecked; one it re-sends -- an image or
+ # an attachment, or a tool call still awaiting its result -- is not
+ # proof of anything, so refuse rather than take it on trust.
+ if not checked and _row_replayed(row):
+ return None
+ past_tip = past_tip or row.get("id") == tip
+ return chain
+ except Exception:
+ return None
+
+
+def _archive_as_wire(messages: Optional[list[dict]]) -> list[dict]:
+ """Project persisted rows and wire messages into boundary-counting wire units."""
+ try:
+ from core.rag import conversation_archive
+ return conversation_archive._as_wire(list(messages or ()))
+ except Exception:
+ return list(messages or ())
+
+
+class _CompactionBranchState(NamedTuple):
+ message: dict
+ truncation: Optional[dict]
+ recorded: int
+
+
+def _compaction_branch_states(
+ stored: list[dict], branch_messages: Optional[list[dict]] = None
+) -> list[_CompactionBranchState]:
+ """Newest authoritative compaction state(s) on one request branch.
+
+ Multiple rows are returned only when Retry siblings are textually indistinguishable;
+ consumers must then choose the conservative result across all of them.
+ """
+ # Assistant rows only: an abandoned "Done" rides in on a live "not done yet".
+ branch = _archive_branch_transcript(branch_messages, ("assistant",))
+ if branch_messages and not branch:
+ return []
+
+ chain = _archive_branch_chain(stored, branch_messages)
+ candidates = [
+ message
+ for message in reversed(chain if chain is not None else stored)
+ if message.get("role") == "assistant"
+ and (chain is not None or _archive_content_on_branch(message.get("content"), branch))
+ ]
+ if not candidates:
+ return []
+
+ # Exact first: the branch check is a substring test, so "Done" matches "Not done yet".
+ if chain is None:
+ live = set(branch or ())
+ exact = [
+ message
+ for message in candidates
+ if _archive_message_text(message.get("content")) in live
+ ]
+ if exact:
+ candidates = exact
+
+ # A COMPLETED row with no truncation ends the epoch; only active/aborted are placeholders.
+ states = []
+ for message in candidates:
+ metadata = message.get("metadata")
+ metadata = metadata if isinstance(metadata, dict) else {}
+ custom = metadata.get("custom")
+ custom = custom if isinstance(custom, dict) else {}
+ truncation = metadata.get("contextTruncation") or custom.get("contextTruncation")
+
+ recorded = None
+ if isinstance(truncation, dict):
+ raw_recorded = truncation.get("boundary_messages")
+ if raw_recorded is None:
+ raw_recorded = truncation.get("dropped_messages")
+ if raw_recorded is not None:
+ try:
+ recorded = max(0, int(raw_recorded or 0))
+ except (TypeError, ValueError):
+ pass
+ if recorded is None and chain is not None:
+ # Skipping defers to the previous epoch, so it needs a proved branch: on the text
+ # path the rows are twins. A research row reports its run, not whether this fit.
+ research_row = (
+ (metadata.get("serverManaged") or custom.get("serverManaged"))
+ and (metadata.get("researchRunId") or custom.get("researchRunId"))
+ and (metadata.get("researchStatus") or custom.get("researchStatus"))
+ )
+ if research_row:
+ continue
+ status = metadata.get("generationStatus") or custom.get("generationStatus")
+ incomplete = metadata.get("incomplete") or custom.get("incomplete")
+ reason = incomplete.get("reason") if isinstance(incomplete, dict) else incomplete
+ active = status in {"queued", "running", "cancelling"}
+ aborted = status in {"cancelled", "failed"} or reason in {"cancelled", "interrupted"}
+ if active or (status != "completed" and aborted):
+ continue
+ if recorded is None:
+ states.append(_CompactionBranchState(message, None, 0))
+ continue
+ states.append(_CompactionBranchState(message, truncation, recorded))
+
+ if not states:
+ return []
+ if chain is not None:
+ # Parent ancestry orders repeated text on one linear history unambiguously.
+ return states[:1]
+ newest = _archive_message_text(states[0].message.get("content"))
+ return [
+ state for state in states if _archive_message_text(state.message.get("content")) == newest
+ ]
+
+
def _sticky_compaction_state(
thread_id: Optional[str],
branch_messages: Optional[list[dict]] = None,
@@ -1100,57 +1302,20 @@ def _sticky_compaction_state(
try:
from storage import studio_db
- # The stored rows are the whole DAG, so the newest assistant turn can belong to a
- # sibling branch left by Retry, whose boundary is sized for history this branch
- # does not have. Skip rows the request's own messages do not contain.
- # Assistant messages only: the rows being checked are assistant replies, and
- # against every role a short abandoned one ("Done") rides in on a live user
- # message that merely contains it ("not done yet"), taking its boundary with it.
- _branch = _archive_branch_transcript(branch_messages, ("assistant",))
- if branch_messages and not _branch:
- # A branch with no reply of its own has no boundary to restore.
- return 0, False
- candidates = [
- message
- for message in reversed(studio_db.list_chat_messages(thread_id) or [])
- if message.get("role") == "assistant"
- and _archive_content_on_branch(message.get("content"), _branch)
- ]
- if not candidates:
+ stored = list(studio_db.list_chat_messages(thread_id) or [])
+ states = _compaction_branch_states(stored, branch_messages)
+ if not states:
return 0, False
- # The newest on-branch assistant turn decides, except that the branch check is
- # textual, so two Retry siblings that both read "Done" are indistinguishable here
- # and the first match could apply a much deeper branch's boundary. Where the text
- # cannot separate them, take the SMALLEST boundary: too small costs one extra
- # compaction, too large evicts live history.
- # That check is also a substring test (an archived turn is matched against
- # fragments of itself), so an abandoned "Done" can ride in on a live "Not done
- # yet" and then decide the boundary alone. Prefer exact matches where any exist.
- _live = set(_branch or ())
- _exact = [
- message
- for message in candidates
- if _archive_message_text(message.get("content")) in _live
- ]
- if _exact:
- candidates = _exact
-
- newest = _archive_message_text(candidates[0].get("content"))
boundaries = []
origins = []
- for message in candidates:
- if _archive_message_text(message.get("content")) != newest:
- continue
- metadata = message.get("metadata") or {}
- if not isinstance(metadata, dict):
+ branch_wire = _archive_as_wire(branch_messages)
+ for state in states:
+ truncation = state.truncation
+ recorded = state.recorded
+ if truncation is None:
return 0, False
- truncation = metadata.get("contextTruncation") or (metadata.get("custom") or {}).get(
- "contextTruncation"
- )
- if not isinstance(truncation, dict):
- return 0, False
- # Only a fit that SUCCEEDED describes a boundary worth restoring.
+ # Only a SUCCEEDED fit describes a boundary, but an explicit failure still rules.
if not truncation.get("fits"):
return 0, False
# A boundary is valid only under the fit that will consume it, and the two
@@ -1186,18 +1351,14 @@ def _sticky_compaction_state(
# rows saved before the ratio was recorded, which keep replaying as they did.
if not recorded_checkpoint:
recorded_ratio = truncation.get("boundary_headroom_ratio")
- if (
- recorded_ratio is not None
- and abs(float(recorded_ratio) - requested_ratio) > 1e-9
- ):
+ try:
+ if (
+ recorded_ratio is not None
+ and abs(float(recorded_ratio) - requested_ratio) > 1e-9
+ ):
+ return 0, False
+ except (TypeError, ValueError):
return 0, False
- # Counted against the request's own transcript, which is what it is applied
- # to. `dropped_messages` is the fallback for turns saved before that was
- # recorded: equal for a single fit, too large for a turn that refit often.
- recorded = truncation.get("boundary_messages")
- if recorded is None:
- recorded = truncation.get("dropped_messages")
- recorded = max(0, int(recorded or 0))
# A count is only valid against the transcript it was counted on. Deleting an
# already-evicted turn shortens the front, and replaying the count then evicts
# that many LIVE messages instead. Re-derive it from the anchor's position on
@@ -1205,8 +1366,8 @@ def _sticky_compaction_state(
# (a repeated text, an edited turn) must not deepen the cut.
anchor = truncation.get("boundary_anchor")
if isinstance(anchor, str) and anchor:
- for index, message in enumerate(_branch_non_system(branch_messages)):
- if _anchor_text(message) == anchor[:_ANCHOR_TEXT_CHARS]:
+ for index, branch_message in enumerate(_branch_non_system(branch_wire)):
+ if _anchor_text(branch_message) == anchor[:_ANCHOR_TEXT_CHARS]:
recorded = min(recorded, index)
break
boundaries.append(recorded)
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 7099d410cc5..c4873586d9f 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -690,8 +690,15 @@ def _walk_from(by_id: dict, parent_of: dict, leaf) -> list[dict]:
return chain
-def _branch_seed(messages: list[dict], by_id: dict, parent_of: dict, branch) -> Optional[str]:
- """Which stored leaf the REQUEST is on, by matching its text. None when nothing does.
+def _branch_seed(
+ messages: list[dict],
+ by_id: dict,
+ parent_of: dict,
+ branch,
+ *,
+ require_unique: bool = False,
+) -> Optional[str]:
+ """Which stored endpoint the REQUEST proves by matching text. None when no row matches.
The newest stored row is not the branch the request is on. Switching to a sibling,
continuing there and switching back leaves the abandoned branch holding the greatest
@@ -707,19 +714,28 @@ def _branch_seed(messages: list[dict], by_id: dict, parent_of: dict, branch) ->
"""
if not branch:
return None
+
# A LIST, in order, not a set: sets lose repetition and ordering, so an abandoned
# sibling with the same distinct texts tied with the request's own branch and, being
# newer, won. A multiset fixes the repeat case but not the reordered one, so this
# scores an in-order run. System and developer messages are excluded: Unsloth's
# prepended chat and project instructions are not part of the stored chain.
+ def _key(message):
+ """What a message matches ON. Role-blind let a rewound, unstored "Continue." match
+ an ABANDONED assistant reply of the same text and adopt its boundary."""
+ text = _normalise_cased(_probe_text(message))
+ if not text:
+ return None
+ return (str(message.get("role") or ""), text) if require_unique else text
+
wanted = [
- text
- for text in (
- _normalise_cased(_probe_text(message))
+ key
+ for key in (
+ _key(message)
for message in _as_wire(branch)
if str(message.get("role") or "") not in ("system", "developer")
)
- if text
+ if key
]
if not wanted:
return None
@@ -736,8 +752,13 @@ def _branch_seed(messages: list[dict], by_id: dict, parent_of: dict, branch) ->
key = lambda identifier: order.get(identifier, -1),
reverse = True,
)
+ if require_unique and len(leaves) > _BRANCH_SEED_MAX_LEAVES:
+ # A capped search cannot prove that an unvisited sibling does not tie the winner.
+ return None
best = None
+ best_matched = None
best_score = 0
+ best_tied = False
# Rendered once per STORED ROW, not per leaf: `_as_wire` expands a row the same way
# whichever chain it is walked in.
rendered: dict = {}
@@ -745,9 +766,9 @@ def _branch_seed(messages: list[dict], by_id: dict, parent_of: dict, branch) ->
def _texts_of(record: dict) -> list:
identifier = record.get("id")
if identifier is None:
- return [_normalise_cased(_probe_text(m)) for m in _as_wire([record])]
+ return [_key(m) for m in _as_wire([record])]
if identifier not in rendered:
- rendered[identifier] = [_normalise_cased(_probe_text(m)) for m in _as_wire([record])]
+ rendered[identifier] = [_key(m) for m in _as_wire([record])]
return rendered[identifier]
for leaf in leaves[:_BRANCH_SEED_MAX_LEAVES]:
@@ -756,24 +777,39 @@ def _branch_seed(messages: list[dict], by_id: dict, parent_of: dict, branch) ->
# neither side is a subsequence of the other.
cursor = 0
score = 0
+ last_matched = None
for record in _walk_from(by_id, parent_of, leaf):
- for text in _texts_of(record):
- spots = where.get(text) if text else None
+ for key in _texts_of(record):
+ spots = where.get(key) if key else None
if not spots:
continue
index = bisect.bisect_left(spots, cursor)
if index < len(spots):
cursor = spots[index] + 1
score += 1
+ last_matched = record.get("id")
if score > best_score:
best, best_score = leaf, score
- if best_score >= len(wanted):
+ best_matched = last_matched
+ best_tied = False
+ elif require_unique and score == best_score and score > 0:
+ # Tied on the SAME endpoint is not ambiguity: two retries fork past the proof.
+ best_tied = best_tied or last_matched != best_matched
+ if best_score >= len(wanted) and not require_unique:
# Every message the request carries is on this chain; nothing can beat it.
break
+ if require_unique:
+ return None if best_tied else best_matched
return best
-def _active_chain(messages: list[dict], branch = None) -> list[dict]:
+def _active_chain(
+ messages: list[dict],
+ branch = None,
+ *,
+ fallback: bool = True,
+ require_unique: bool = False,
+) -> list[dict]:
"""The rows on ONE branch, oldest first, rather than the whole stored DAG.
`list_chat_messages` is an unfiltered SELECT ordered by time, but a thread is a tree:
@@ -789,26 +825,62 @@ def _active_chain(messages: list[dict], branch = None) -> list[dict]:
Walked newest leaf back to root, the same shape the frontend's `orderBySelectedBranch`
uses to decide what the model is actually shown. `parent_id` is missing on rows written
before that column, so the previous row stands in for it, which is exactly a flat list
- when nothing branches.
+ when nothing branches. ``fallback=False`` lets callers decline when the request cannot
+ seed a chain instead of silently reading the newest stored sibling. ``require_unique``
+ likewise declines when indistinguishable leaves tie for the best branch match and trims
+ the winner after the last stored row the request actually matched.
"""
if not messages:
return []
by_id: dict = {}
parent_of: dict = {}
+ # Rows whose parent is storage order rather than a real link. The root's absent parent
+ # is not synthesized: nothing stood in for it.
+ synthesized: set = set()
previous = None
for message in messages:
identifier = message.get("id")
if identifier is None:
continue
by_id[identifier] = message
- parent_of[identifier] = message.get("parentId") or message.get("parent_id") or previous
+ parent = message.get("parentId") or message.get("parent_id")
+ # A row that CARRIES the column and holds null is a root the client meant, as
+ # editing the first prompt makes. Only a row written before the column exists has
+ # nothing to say, and there storage order stands in. Under the legacy path the
+ # stand-in is unconditional, as it always was: `_transcript_positions` numbers
+ # turns off this chain, and rooting a null there renumbers the whole archive.
+ stated = require_unique and ("parentId" in message or "parent_id" in message)
+ if parent is None and previous is not None and not stated:
+ synthesized.add(identifier)
+ parent = previous
+ parent_of[identifier] = parent
previous = identifier
if not by_id:
- return list(messages)
+ return list(messages) if fallback else []
+ if require_unique and synthesized:
+ # Storage order is not ancestry. Where it strung INDISTINGUISHABLE rows into one
+ # chain it made the abandoned twin an ancestor of the live one, and the greedy trim
+ # stopped on the twin, replaying ITS deeper boundary instead of the safe vote
+ # across them. The twins need not be adjacent: the abandoned branch's own
+ # continuation sits between them. Threads the text can still tell apart are left
+ # alone, since declining on every invented link would refuse legacy ancestry that
+ # storage order does recover.
+ seen: set = set()
+ for message in messages:
+ if message.get("id") is None:
+ continue
+ key = (message.get("role"), _normalise_cased(_probe_text(message)))
+ if key[1] and key in seen:
+ return []
+ seen.add(key)
# The request's own branch when it can be found, the newest row when it cannot: empty
# positions empty every seat and send every turn to MAX + 1, which is worse than
# reading the wrong branch.
- seed = _branch_seed(messages, by_id, parent_of, branch) or messages[-1].get("id")
+ seed = _branch_seed(messages, by_id, parent_of, branch, require_unique = require_unique)
+ if seed is None:
+ if not fallback:
+ return []
+ seed = messages[-1].get("id")
return _walk_from(by_id, parent_of, seed) or list(messages)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 880db4f9175..e8e651325c9 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -4705,92 +4705,30 @@ def _thread_has_checkpoint(thread_id, branch_messages = None) -> bool:
cannot happen, and costs the request the tool-path guards that reject `n > 1` and
non-streaming ask/auto.
- Read from the assistant turn's own `contextTruncation`, the same metadata the sticky
- boundary uses, so no new state is stored and it survives a restart. The NEWEST turn
- answers: while an epoch is in force every fit records it on the turn it produced, so
- a thread mid-epoch still says yes, and a thread whose window grew until the whole
- branch fits again says no rather than forcing the loop open for ever.
+ Read through the sticky boundary's branch-state resolver, so no new state is stored and
+ admission cannot disagree about Retry siblings or wire-shaped tool turns. The newest
+ authoritative turn answers: while an epoch is in force every fit records it on the turn
+ it produced, while a completed boundary-less turn ends it.
"""
if not thread_id:
return False
try:
- from core.rag import conversation_archive
+ from core.inference.llama_cpp import _compaction_branch_states
from storage import studio_db
- # Scoped to the branch the request is on. The stored rows are the whole DAG, so a
- # Retry that forked BEFORE the epoch-recording turn leaves it on an abandoned
- # sibling; a thread-wide scan would then report a checkpoint for a branch that
- # never reset. Same filter the sticky boundary applies, for the same reason.
- # As dicts: on the ordinary completions path these are `ChatMessage` models, and
- # the archive helper reads them with `.get`, so it raised, the caller swallowed it
- # and every thread reported no checkpoint. A tools-off thread that HAD reset then
- # never reopened the loop, so the block's promise that the history is searchable
- # was false for the whole of that epoch.
- branch = conversation_archive.branch_message_texts(
- _as_plain_messages(branch_messages), ("assistant",)
- )
- if branch_messages and not branch:
- # A branch with no reply of its own never recorded an epoch. Without this the
- # filter below is skipped rather than applied and the scan goes thread-wide
- # again, which editing or regenerating the FIRST user turn hits by re-sending
- # [system, user]. `_sticky_compaction_boundary` returns 0 there.
- return False
- live = set(branch or ())
- rows = [
- message
- for message in reversed(studio_db.list_chat_messages(str(thread_id)) or [])
- if message.get("role") == "assistant"
- ]
- if branch:
- # Exact matches where any exist, substring only as the fallback: the branch
- # check is textual, so an abandoned short reply rides in on a longer live one
- # ("Done" against "Not done yet") and reopens the loop on the live branch. The
- # sticky boundary prefers exact matches for the same collision.
- exact = [
- message
- for message in rows
- if conversation_archive.message_text(message.get("content")) in live
- ]
- rows = exact or [
- message
- for message in rows
- if conversation_archive.content_on_branch(message.get("content"), branch)
- ]
- if not rows:
- return False
- # Rows the text cannot tell apart decide TOGETHER. Two Retry siblings can carry
- # byte-identical replies with only the abandoned one having reset, and taking the
- # first match reopened the loop on the branch that never did. Where the branch
- # check cannot separate them, choose the reading that leaves the request as it was,
- # as the sticky boundary's `min(boundaries)` does. The case this loses -- a real
- # epoch on the live sibling -- is one the sticky boundary declines to replay
- # anyway, so there is nothing for the tool to reach back to.
- newest = conversation_archive.message_text(rows[0].get("content"))
- twins = [
- message
- for message in rows
- if conversation_archive.message_text(message.get("content")) == newest
- ]
-
- def _checkpointed(message: dict) -> bool:
- metadata = message.get("metadata") or {}
- if not isinstance(metadata, dict):
- return False
- truncation = metadata.get("contextTruncation") or (metadata.get("custom") or {}).get(
- "contextTruncation"
- )
- return bool(isinstance(truncation, dict) and truncation.get("checkpoint"))
-
- # The NEWEST distinguishable state answers, and nothing older. While an epoch is
- # in force every fit records it on the turn it produced, so the newest row says
- # so too; once the window grows and the whole branch fits again the fit records
- # nothing, and scanning back to an older reset then forced the tool loop on for
- # the rest of the thread's life -- overriding enable_tools = false, and the n > 1
- # and non-streaming guards with it, to repair a compaction that no longer exists.
- return all(_checkpointed(message) for message in twins)
+ # As dicts: the completions path sends `ChatMessage`, the resolver uses `.get`.
+ states = _compaction_branch_states(
+ list(studio_db.list_chat_messages(str(thread_id)) or []),
+ _as_plain_messages(branch_messages),
+ )
+ return bool(states) and all(
+ state.truncation is not None
+ and state.truncation.get("fits")
+ and state.truncation.get("checkpoint")
+ for state in states
+ )
except Exception:
return False
- return False
async def _select_request_tools(
diff --git a/studio/backend/tests/test_checkpoint_compaction.py b/studio/backend/tests/test_checkpoint_compaction.py
index 24eccc2a495..7ae117aaaad 100644
--- a/studio/backend/tests/test_checkpoint_compaction.py
+++ b/studio/backend/tests/test_checkpoint_compaction.py
@@ -1416,6 +1416,658 @@ def _stub_studio_db(monkeypatch, messages):
monkeypatch.setitem(sys.modules, "storage.studio_db", module)
+def _checkpoint_metadata(boundary, **extra):
+ return {
+ "custom": {
+ "contextTruncation": {
+ "fits": True,
+ "checkpoint": True,
+ "boundary_messages": boundary,
+ **extra,
+ }
+ }
+ }
+
+
+def test_a_wire_shaped_tool_branch_restores_the_stored_rows_boundary(monkeypatch):
+ """One stored assistant row expands to call, result, and reply on the wire.
+
+ Requiring the row's combined text to occur inside one wire assistant message rejects
+ the live row: its result is a separate ``tool`` message. The durable parent chain says
+ which stored row the request descends from without weakening sibling isolation. The
+ anchor is also counted after that same wire expansion, not against three stored rows.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ call = {
+ "type": "tool-call",
+ "toolCallId": "call-1",
+ "toolName": "terminal",
+ "args": {"command": "printf TOOL-9915"},
+ "result": "TOOL-9915",
+ }
+ rows = [
+ {
+ "id": "user-1",
+ "parentId": None,
+ "role": "user",
+ "content": [{"type": "text", "text": "Run the diagnostic."}],
+ },
+ {
+ "id": "assistant-1",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": [call, {"type": "text", "text": "The diagnostic passed."}],
+ "metadata": _checkpoint_metadata(4, boundary_anchor = "What happened?"),
+ },
+ {
+ "id": "user-2",
+ "parentId": "assistant-1",
+ "role": "user",
+ "content": [{"type": "text", "text": "What happened?"}],
+ },
+ {
+ "id": "assistant-retry",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "An abandoned retry on a sibling branch.",
+ "metadata": _checkpoint_metadata(99),
+ },
+ {
+ "id": "user-retry",
+ "parentId": "assistant-retry",
+ "role": "user",
+ "content": "A sibling question the request did not select.",
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "Run the diagnostic."},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call-1",
+ "function": {
+ "name": "terminal",
+ "arguments": '{"command":"printf TOOL-9915"}',
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call-1", "content": "TOOL-9915"},
+ {"role": "assistant", "content": "The diagnostic passed."},
+ {"role": "user", "content": "What happened?"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", rows[:3]) == (4, True)
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (4, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+
+def test_parent_linked_identical_retry_siblings_keep_the_smaller_boundary(monkeypatch):
+ """A full text match is not proof when two stored leaves are indistinguishable."""
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ def _reply(identifier, boundary):
+ return {
+ "id": identifier,
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": _checkpoint_metadata(boundary),
+ }
+
+ rows = [
+ {"id": "user-1", "parentId": None, "role": "user", "content": "Do the work."},
+ _reply("assistant-live", 6),
+ # Newer in storage, but an abandoned Retry sibling indistinguishable on the wire.
+ _reply("assistant-abandoned", 18),
+ ]
+ branch = [
+ {"role": "user", "content": "Do the work."},
+ {"role": "assistant", "content": "Done."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (6, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+ # Stopping at the shared user parent selects neither Retry reply.
+ assert llama_cpp._sticky_compaction_state("t1", branch[:1]) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch[:1]) is False
+
+ # Text cannot identify the live sibling, so one non-checkpoint twin makes the claim unsafe.
+ rows[2]["metadata"]["custom"]["contextTruncation"].pop("checkpoint")
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_repeated_text_on_one_parent_chain_uses_only_the_newest_state(monkeypatch):
+ """Identical replies are chronological when durable ancestry identifies one chain."""
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {
+ "id": "user-1",
+ "parentId": None,
+ "role": "user",
+ "content": "First task.",
+ },
+ {
+ "id": "assistant-1",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "user-2",
+ "parentId": "assistant-1",
+ "role": "user",
+ "content": "Second task.",
+ },
+ {
+ "id": "assistant-2",
+ "parentId": "user-2",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": _checkpoint_metadata(5),
+ },
+ {
+ "id": "user-3",
+ "parentId": "assistant-2",
+ "role": "user",
+ "content": "What happened?",
+ },
+ ]
+ branch = [{"role": row["role"], "content": row["content"]} for row in rows]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (5, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+
+def test_authoritative_ancestry_stops_before_an_unmatched_stored_descendant(monkeypatch):
+ """An edited request proves its common prefix, not the old branch after the edit."""
+ from core.inference import checkpoint, llama_cpp
+ from core.rag import conversation_archive
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "user-1", "parentId": None, "role": "user", "content": "First task."},
+ {
+ "id": "assistant-1",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "The common-prefix reply.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "user-old",
+ "parentId": "assistant-1",
+ "role": "user",
+ "content": "The question before it was edited.",
+ },
+ {
+ "id": "assistant-old",
+ "parentId": "user-old",
+ "role": "assistant",
+ "content": "An unmatched old-branch reply.",
+ "metadata": _checkpoint_metadata(12),
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "First task."},
+ {"role": "assistant", "content": "The common-prefix reply."},
+ {"role": "user", "content": "The edited replacement question."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ # Archive callers retain their historical fallback; only authoritative reads trim.
+ assert conversation_archive._active_chain(rows, branch) == rows
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+ ("metadata", "expected"),
+ [
+ ({"custom": {"incomplete": {"reason": "cancelled"}}}, (6, True)),
+ ({"incomplete": {"reason": "interrupted"}}, (6, True)),
+ ({"generationStatus": "running", "serverManaged": True}, (6, True)),
+ (
+ {"researchRunId": "run-1", "researchStatus": "completed", "serverManaged": True},
+ (6, True),
+ ),
+ (
+ {"researchRunId": "run-1", "researchStatus": "failed", "serverManaged": True},
+ (6, True),
+ ),
+ (
+ {"researchRunId": "run-1", "researchStatus": "cancelled", "serverManaged": True},
+ (6, True),
+ ),
+ (
+ {"custom": {"contextTruncation": {"fits": True, "boundary_messages": 4}}},
+ (0, False),
+ ),
+ (
+ {
+ "generationStatus": "completed",
+ "serverManaged": True,
+ "incomplete": {"reason": "length"},
+ },
+ (0, False),
+ ),
+ (
+ {
+ "custom": {
+ "generationStatus": "completed",
+ "incomplete": {"reason": "length"},
+ }
+ },
+ (0, False),
+ ),
+ ],
+ ids = [
+ "custom-cancelled-placeholder",
+ "top-level-interrupted-placeholder",
+ "top-level-active-placeholder",
+ "deep-research-completed",
+ "deep-research-failed",
+ "deep-research-cancelled",
+ "newer-rolling-state",
+ "top-level-completed-at-length",
+ "custom-completed-at-length",
+ ],
+)
+def test_the_newest_authoritative_state_controls_the_old_epoch(monkeypatch, metadata, expected):
+ """Only active or aborted boundary-less placeholders defer to the prior epoch."""
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "user-1", "parentId": None, "role": "user", "content": "First question."},
+ {
+ "id": "assistant-1",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "The epoch started here.",
+ "metadata": _checkpoint_metadata(6),
+ },
+ {"id": "user-2", "parentId": "assistant-1", "role": "user", "content": "Continue."},
+ {
+ "id": "assistant-2",
+ "parentId": "user-2",
+ "role": "assistant",
+ "content": "The newest reply.",
+ "metadata": metadata,
+ },
+ {
+ "id": "user-3",
+ "parentId": "assistant-2",
+ "role": "user",
+ "content": "Continue again.",
+ },
+ ]
+ branch = [{"role": row["role"], "content": row["content"]} for row in rows]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == expected
+ assert inference_routes._thread_has_checkpoint("t1", branch) is expected[1]
+
+
+def test_a_cancelled_epoch_boundary_is_found_through_its_stored_descendant(monkeypatch):
+ """A cancelled reply can be absent from wire history and remain on the parent chain."""
+ from core.inference import checkpoint, llama_cpp
+ from core.rag import conversation_archive
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "user-1", "parentId": None, "role": "user", "content": "First question."},
+ {
+ "id": "assistant-1",
+ "parentId": "user-1",
+ "role": "assistant",
+ "content": "The old epoch reply.",
+ "metadata": _checkpoint_metadata(6),
+ },
+ {"id": "user-2", "parentId": "assistant-1", "role": "user", "content": "More work."},
+ {
+ "id": "assistant-2",
+ "parentId": "user-2",
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool-call",
+ "toolCallId": "call-cancelled",
+ "toolName": "terminal",
+ "args": {"command": "sleep 30"},
+ "provenance": {"source": "local"},
+ }
+ ],
+ "metadata": {
+ "incomplete": {"reason": "cancelled"},
+ **_checkpoint_metadata(12, checkpoint_started = True),
+ },
+ },
+ {
+ "id": "user-3",
+ "parentId": "assistant-2",
+ "role": "user",
+ "content": "Continue after stopping.",
+ },
+ ]
+ # The adapter omits an unfinished local card, but user-3 durably descends from its row.
+ assert conversation_archive._as_wire([rows[3]]) == []
+ branch = [
+ {"role": "user", "content": "First question."},
+ {"role": "assistant", "content": "The old epoch reply."},
+ {"role": "user", "content": "More work."},
+ {"role": "user", "content": "Continue after stopping."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (12, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+
+def test_retrying_the_newest_turn_twice_still_resolves_the_proved_branch(monkeypatch):
+ """Leaves that fork BELOW what the request proves cannot make the answer ambiguous.
+
+ Both siblings trim to the same stored row, so scoring them as a tie discarded the
+ branch the request had just proved and dropped the thread back on the text path,
+ which is the path a tool-heavy row cannot survive.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Run the diagnostic."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool-call",
+ "toolCallId": "c1",
+ "toolName": "terminal",
+ "args": {"command": "probe"},
+ "result": "PROBE-9915",
+ },
+ {"type": "text", "text": "The diagnostic passed."},
+ ],
+ "metadata": _checkpoint_metadata(4),
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "Run the diagnostic."},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "c1", "function": {"name": "terminal", "arguments": "{}"}}],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": "PROBE-9915"},
+ {"role": "assistant", "content": "The diagnostic passed."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ for retry in range(3):
+ rows.append(
+ {
+ "id": f"fu{retry}",
+ "parentId": "a1",
+ "role": "user",
+ "content": f"A retried follow-up {retry}.",
+ }
+ )
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (4, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+
+def test_the_unstored_newest_turn_cannot_move_the_request_to_a_sibling(monkeypatch):
+ """The turn being answered is stored only once the reply completes.
+
+ A sibling that already carries that text is the one place it can be matched, so
+ scoring it there handed this request the sibling's deeper boundary.
+ """
+ from core.inference import checkpoint, llama_cpp
+
+ def _reply(identifier, boundary):
+ return {
+ "id": identifier,
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": _checkpoint_metadata(boundary),
+ }
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Do the work."},
+ _reply("a-live", 6),
+ _reply("a-abandoned", 18),
+ {"id": "u-abandoned", "parentId": "a-abandoned", "role": "user", "content": "Continue."},
+ ]
+ branch = [
+ {"role": "user", "content": "Do the work."},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Continue."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (6, True)
+
+
+def test_an_indistinguishable_placeholder_twin_is_not_dropped_from_the_vote(monkeypatch):
+ """Skipping a placeholder defers to the epoch before it, which needs a proved branch.
+
+ Where only text separates two Retry siblings, dropping the unreadable one leaves the
+ other deciding alone, and it is the abandoned one whose boundary then gets replayed.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ for status in ("cancelled", "running"):
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Do the work."},
+ {
+ "id": "a-live",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": status},
+ },
+ {
+ "id": "a-abandoned",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": _checkpoint_metadata(18),
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "Do the work."},
+ {"role": "assistant", "content": "Done."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_rewound_turn_does_not_match_an_assistant_reply_of_the_same_text(monkeypatch):
+ """Ancestry is proved by text, so the match has to agree about the role too.
+
+ Rewinding and typing "Continue." matched the ABANDONED continuation's assistant reply
+ of that text, and the request adopted a boundary its own branch never had.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Do the work."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "The shared reply.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {"id": "u2", "parentId": "a1", "role": "user", "content": "Take the next step."},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "Continue.",
+ "metadata": _checkpoint_metadata(18),
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "Do the work."},
+ {"role": "assistant", "content": "The shared reply."},
+ {"role": "user", "content": "Continue."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_chain_that_skips_past_the_settled_proof_is_refused(monkeypatch):
+ """A role-compatible match on the unstored turn can still land on a sibling.
+
+ The abandoned continuation ends in a user row reading "Continue." too, so the chain ran
+ to it and dragged the abandoned reply's deeper boundary along. Rows past the settled
+ tip must carry text the request actually sent; a row that renders nothing (an
+ unfinished tool card, the cancelled-epoch case) is not evidence either way.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Do the work."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "The shared reply.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {"id": "u2", "parentId": "a1", "role": "user", "content": "Take the next step."},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "Abandoned reply.",
+ "metadata": _checkpoint_metadata(18),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue."},
+ ]
+ branch = [
+ {"role": "user", "content": "Do the work."},
+ {"role": "assistant", "content": "The shared reply."},
+ {"role": "user", "content": "Continue."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_repeated_text_earlier_in_the_request_cannot_admit_an_abandoned_row(monkeypatch):
+ """Rows past the settled tip are justified by the unstored turns, and only those.
+
+ Checking them against the whole request let the abandoned continuation in on "Q" and
+ "Same", which the request does carry, but earlier, as the turns it rewound TO.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Q"},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Same",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {"id": "u2", "parentId": "a1", "role": "user", "content": "Q"},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "Same",
+ "metadata": _checkpoint_metadata(18),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue"},
+ ]
+ branch = [
+ {"role": "user", "content": "Q"},
+ {"role": "assistant", "content": "Same"},
+ {"role": "user", "content": "Continue"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_research_row_is_recognised_under_custom_metadata(monkeypatch):
+ """The archive accepts the research keys in either place, so this must too."""
+ from core.inference import checkpoint, llama_cpp
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "First question."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "The epoch reply.",
+ "metadata": _checkpoint_metadata(6),
+ },
+ {"id": "u2", "parentId": "a1", "role": "user", "content": "Continue."},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "The research row.",
+ "metadata": {
+ "custom": {
+ "serverManaged": True,
+ "researchRunId": "run-1",
+ "researchStatus": "completed",
+ }
+ },
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue again."},
+ ]
+ branch = [{"role": row["role"], "content": row["content"]} for row in rows]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (6, True)
+
+
def test_a_boundary_is_not_replayed_after_the_context_policy_changes(monkeypatch):
"""A policy switch has to discard the old depth, not just select another fitter.
@@ -1746,6 +2398,12 @@ def test_the_tool_loop_reopens_only_where_an_epoch_actually_happened(monkeypatch
_thread({"fits": True, "dropped_messages": 12, "checkpoint": True})
assert inference_routes._thread_has_checkpoint("t1") is True
+ # A refused fit records diagnostics, not an epoch that can be searched.
+ _thread({"fits": False, "dropped_messages": 12, "checkpoint": True})
+ assert inference_routes._thread_has_checkpoint("t1") is False
+
+ _thread({"fits": True, "dropped_messages": 12, "checkpoint": True})
+
# ...but only for the branch the request is on. A Retry that forked BEFORE the
# epoch-recording turn leaves it on an abandoned sibling, and a thread-wide scan would
# report a checkpoint for a branch that never reset.
@@ -2865,3 +3523,316 @@ def test_holding_the_block_whole_does_not_freeze_it_on_the_first_epoch():
assert seen_rounds[0] == 4, "the carried block survives while it fits"
assert seen_rounds[-1] == 0, "and ages out instead of holding every slot forever"
+
+
+def test_every_epoch_the_writer_records_carries_a_count_the_reader_can_use():
+ """`_thread_has_checkpoint` now demands a resolved boundary, not just the flag.
+
+ The old gate read `checkpoint` alone, so a record with the flag and no count still
+ admitted the search tool. Nothing released can write that pair -- this pins it, since
+ a writer that dropped the count would silently close the loop on a live epoch.
+ """
+ import ast
+ import inspect
+
+ tree = ast.parse(inspect.getsource(checkpoint))
+ counts = {"dropped_messages", "boundary_messages"}
+ seen = 0
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Dict):
+ continue
+ keys = {
+ k.value for k in node.keys if isinstance(k, ast.Constant) and isinstance(k.value, str)
+ }
+ if "checkpoint" not in keys:
+ continue
+ seen += 1
+ assert keys & counts, f"epoch record without a count: {sorted(keys)}"
+
+ assert seen, "no epoch record found; the invariant would pass vacuously"
+
+
+def test_a_cancelled_reply_that_reached_text_is_still_validated(monkeypatch):
+ """Stop is not omission: the client re-sends a partial reply, so it must match.
+
+ `isAbandonedAssistantTurn` (chat-adapter.ts) drops an assistant turn only when it
+ carries no text, image or attachment -- cancellation alone does not drop it. Exempting
+ every cancelled row from the post-tip check let this abandoned "Partial" ride in on
+ "Continue" and hand the request its boundary instead of the epoch that really ended.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Q"},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "A1",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "a2",
+ "parentId": "a1",
+ "role": "assistant",
+ "content": "Partial",
+ "metadata": {
+ "generationStatus": "cancelled",
+ "incomplete": {"reason": "cancelled"},
+ **_checkpoint_metadata(21),
+ },
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue"},
+ ]
+ branch = [
+ {"role": "user", "content": "Q"},
+ {"role": "assistant", "content": "A1"},
+ {"role": "user", "content": "Continue"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_storage_order_does_not_prove_ancestry_between_indistinguishable_rows(monkeypatch):
+ """Rows written before `parentId` fall back to storage order, which is not ancestry.
+
+ Two identical Retry replies with no links become one artificial chain, the abandoned
+ one stored first becomes the live one's ancestor, and the trim stops on it. Its deeper
+ boundary then replayed instead of the conservative vote, evicting live history.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ twin = "The same reply, twice."
+ rows = [
+ {"id": "u1", "role": "user", "content": "First question."},
+ {"id": "sib", "role": "assistant", "content": twin, "metadata": _checkpoint_metadata(55)},
+ # The abandoned branch's own continuation separates the twins in storage, so an
+ # adjacent-only comparison misses them.
+ {"id": "u2", "role": "user", "content": "An abandoned follow-up."},
+ {"id": "live", "role": "assistant", "content": twin, "metadata": _checkpoint_metadata(7)},
+ ]
+ branch = [
+ {"role": "user", "content": "First question."},
+ {"role": "assistant", "content": twin},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ # The twin vote, not the row storage happened to put first.
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (7, True)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is True
+
+
+def test_a_stored_reply_is_not_justified_by_a_user_turn_of_the_same_words(monkeypatch):
+ """The carried set is keyed by role, as the branch match is.
+
+ A text-only set let a stored assistant "Continue" pass on the live USER "Continue",
+ so an abandoned chain matching the trailing user turns carried its boundary in.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Start."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {"id": "u2", "parentId": "a1", "role": "user", "content": "Continue"},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "Continue",
+ "metadata": _checkpoint_metadata(30),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Next"},
+ ]
+ branch = [
+ {"role": "user", "content": "Start."},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Continue"},
+ {"role": "user", "content": "Next"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_completed_tool_turn_past_the_tip_is_validated_by_its_results(monkeypatch):
+ """A finished tool call is re-sent, calls and results, so it is not exempt.
+
+ Exempting every tool-only row let an abandoned tool exchange through unchecked; the
+ cancelled case stays exempt because its calls never returned and render nothing.
+ """
+ from core.inference import checkpoint, llama_cpp
+
+ def _call(identifier, *, result):
+ part = {
+ "type": "tool-call",
+ "toolCallId": identifier,
+ "toolName": "terminal",
+ "args": {"command": "probe"},
+ }
+ if result is not None:
+ part["result"] = result
+ return part
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Start."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "a2",
+ "parentId": "a1",
+ "role": "assistant",
+ "content": [_call("call-done", result = "an abandoned tool result")],
+ "metadata": _checkpoint_metadata(30),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue"},
+ ]
+ branch = [
+ {"role": "user", "content": "Start."},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Continue"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+
+
+def test_a_replayed_row_that_renders_no_text_is_refused_rather_than_trusted(monkeypatch):
+ """An image-only reply is re-sent but has nothing to compare, so it proves nothing."""
+ from core.inference import checkpoint, llama_cpp
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Start."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "a2",
+ "parentId": "a1",
+ "role": "assistant",
+ "content": [{"type": "image", "image": "data:image/png;base64,AAAA"}],
+ "metadata": _checkpoint_metadata(30),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue"},
+ ]
+ branch = [
+ {"role": "user", "content": "Start."},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Continue"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+
+
+def test_a_second_explicit_root_is_not_wired_onto_the_branch_before_it(monkeypatch):
+ """Editing the first prompt makes a real second root; storage order must not fuse them.
+
+ Standing the previous row in for every null parent joined the abandoned root's branch
+ onto the live one, so the walk reached back into it and could restore its boundary.
+ """
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "The first wording."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "An abandoned reply.",
+ "metadata": _checkpoint_metadata(44),
+ },
+ # The edit: a root of its own, not a child of the branch it replaced.
+ {"id": "u2", "parentId": None, "role": "user", "content": "The second wording."},
+ {
+ "id": "a2",
+ "parentId": "u2",
+ "role": "assistant",
+ "content": "The live reply.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ ]
+ branch = [
+ {"role": "user", "content": "The second wording."},
+ {"role": "assistant", "content": "The live reply."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ chain = llama_cpp._archive_branch_chain(rows, branch)
+ assert chain is not None and [row["id"] for row in chain] == ["u2", "a2"]
+ # The live turn recorded nothing, so the epoch is over; 44 is the abandoned root's.
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_completed_reasoning_only_reply_is_replayed_so_it_must_match(monkeypatch):
+ """`isAbandonedAssistantTurn` keeps a turn that finished on reasoning alone.
+
+ `_as_wire` strips reasoning, so such a row offers no key. Treating it as dropped let
+ an abandoned one past the tip unchecked and carried its boundary onto the live branch.
+ """
+ from core.inference import checkpoint, llama_cpp
+
+ rows = [
+ {"id": "u1", "parentId": None, "role": "user", "content": "Start."},
+ {
+ "id": "a1",
+ "parentId": "u1",
+ "role": "assistant",
+ "content": "Done.",
+ "metadata": {"generationStatus": "completed"},
+ },
+ {
+ "id": "a2",
+ "parentId": "a1",
+ "role": "assistant",
+ "content": [{"type": "reasoning", "text": "An abandoned line of thought."}],
+ "metadata": _checkpoint_metadata(30),
+ },
+ {"id": "u3", "parentId": "a2", "role": "user", "content": "Continue"},
+ ]
+ branch = [
+ {"role": "user", "content": "Start."},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Continue"},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert llama_cpp._archive_branch_chain(rows, branch) is None
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)
+ # A reply that STOPPED on reasoning carries nothing and is dropped, so it stays exempt.
+ rows[2]["metadata"] = {"incomplete": {"reason": "cancelled"}, **_checkpoint_metadata(30)}
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (30, True)