Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(turns): suppress answered PocketBase worker-loss retries (#1570)

Bingxi Zhao (Frank)469f1e096bbe
diff --git a/deeptutor/api/routers/sessions.py b/deeptutor/api/routers/sessions.py
index 3b034626b..633c3364b 100644
--- a/deeptutor/api/routers/sessions.py
+++ b/deeptutor/api/routers/sessions.py
@@ -176,38 +176,53 @@ def _attach_orphaned_failed_turns(
turn is the safe fallback. A failed preflight with no saved user row is
deliberately omitted rather than attached to an unrelated question.
"""
- users = [message for message in messages if message.get("role") == "user"]
+ users = [
+ (index, message) for index, message in enumerate(messages) if message.get("role") == "user"
+ ]
for turn in turns:
turn_id = str(turn.get("turn_id") or turn.get("id") or "")
started = float(turn.get("created_at") or 0)
finished = float(turn.get("finished_at") or turn.get("updated_at") or 0)
- user = next(
+ match = next(
(
- message
- for message in users
+ (index, message)
+ for index, message in users
if str((message.get("metadata") or {}).get("turn_id") or "") == turn_id
),
None,
)
- if user is None:
- user = next(
+ if match is None:
+ match = next(
(
- message
- for message in users
+ (index, message)
+ for index, message in users
if started <= float(message.get("created_at") or 0) <= finished
),
None,
)
- if user is None:
+ if match is None:
continue
+ user_index, user = match
+ next_user_index = next((index for index, _ in users if index > user_index), len(messages))
# A subsequent retry may have answered this same user row. Its real
# assistant bubble wins; showing an earlier failure beside it would
- # tell the learner that the answered question is still broken.
+ # tell the learner that the answered question is still broken. Legacy
+ # PocketBase rows have no parent link, so use their linear position
+ # only when the parent field is absent, and only after the failure.
if any(
message.get("role") == "assistant"
- and str(message.get("parent_message_id") or "") == str(user.get("id") or "")
- and float(message.get("created_at") or 0) >= started
- for message in messages
+ and (
+ (
+ str(message.get("parent_message_id") or "") == str(user.get("id") or "")
+ and float(message.get("created_at") or 0) >= started
+ )
+ or (
+ "parent_message_id" not in message
+ and user_index < index < next_user_index
+ and float(message.get("created_at") or 0) >= finished
+ )
+ )
+ for index, message in enumerate(messages)
):
continue
metadata = user.get("metadata")
diff --git a/deeptutor/services/session/pocketbase_store.py b/deeptutor/services/session/pocketbase_store.py
index a91c0d4a4..cfac09ac5 100644
--- a/deeptutor/services/session/pocketbase_store.py
+++ b/deeptutor/services/session/pocketbase_store.py
@@ -42,6 +42,9 @@ _VALID_ID = re.compile(r"^[a-zA-Z0-9_-]+$")
_ACTIVE_TURN_STATUSES = frozenset({"queued", "running", "waiting_input"})
_TERMINAL_TURN_STATUSES = frozenset({"completed", "failed", "cancelled"})
_ALL_TURN_STATUSES = _ACTIVE_TURN_STATUSES | _TERMINAL_TURN_STATUSES
+# The deployed PocketBase messages collection has no parent column. Keep the
+# link inside its JSON metadata until the collection schema can be upgraded.
+_PARENT_MESSAGE_METADATA_KEY = "_parent_message_id"
def _captured_store_context(method):
@@ -860,12 +863,11 @@ class PocketBaseSessionStore:
metadata: dict[str, Any] | None = None,
parent_message_id: int | str | None = None,
) -> int | str:
- # ``parent_message_id`` is accepted to match the protocol shape but is
- # not yet wired through PocketBase storage — branching only works on
- # the SQLite backend today.
- _ = parent_message_id
sid = _validate_id(session_id, "session_id")
now = time.time()
+ stored_metadata = dict(metadata or {})
+ if parent_message_id is not None:
+ stored_metadata[_PARENT_MESSAGE_METADATA_KEY] = str(parent_message_id)
def _add():
if _find_session_record(_pb(), sid, _current_user_id()) is None:
@@ -877,7 +879,7 @@ class PocketBaseSessionStore:
"capability": capability or "",
"events_json": events or [],
"attachments_json": attachments or [],
- "metadata_json": metadata or {},
+ "metadata_json": stored_metadata,
"msg_created_at": now,
}
record = _pb().collection("messages").create(payload)
@@ -1200,7 +1202,10 @@ class PocketBaseSessionStore:
]
def _message_record_to_dict(self, record: Any) -> dict[str, Any]:
- return {
+ raw_metadata = _json_loads(getattr(record, "metadata_json", None), {})
+ metadata = dict(raw_metadata) if isinstance(raw_metadata, dict) else {}
+ parent_message_id = metadata.pop(_PARENT_MESSAGE_METADATA_KEY, None)
+ message = {
"id": getattr(record, "id", ""),
"session_id": getattr(record, "session_id", ""),
"role": getattr(record, "role", ""),
@@ -1208,9 +1213,12 @@ class PocketBaseSessionStore:
"capability": getattr(record, "capability", "") or "",
"events": _json_loads(getattr(record, "events_json", None), []),
"attachments": _json_loads(getattr(record, "attachments_json", None), []),
- "metadata": _json_loads(getattr(record, "metadata_json", None), {}),
+ "metadata": metadata,
"created_at": _to_float(getattr(record, "msg_created_at", None)),
}
+ if parent_message_id is not None:
+ message["parent_message_id"] = parent_message_id
+ return message
# ------------------------------------------------------------------
# Turns
diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.py
index e5828543a..16d7f2884 100644
--- a/deeptutor/services/session/turns/executor.py
+++ b/deeptutor/services/session/turns/executor.py
@@ -1099,6 +1099,21 @@ class TurnExecutor:
parent_message_id=branch_parent_id,
metadata=assistant_provider_metadata,
)
+ elif is_regenerate and payload.get("regenerated_from_message_id") is not None:
+ # Regenerate reuses the saved user row. PocketBase ids are
+ # strings, so they cannot travel through the SQLite-only
+ # parent_message_id request field, but the assistant still
+ # needs an explicit link to hide an older failed attempt.
+ assistant_message_id = await self.store.add_message(
+ session_id=session_id,
+ role="assistant",
+ content=assistant_content,
+ capability=capability_name,
+ events=[],
+ attachments=generated_attachments or None,
+ parent_message_id=payload["regenerated_from_message_id"],
+ metadata=assistant_provider_metadata,
+ )
else:
assistant_message_id = await self.store.add_message(
session_id=session_id,
diff --git a/tests/services/session/test_pocketbase_isolation.py b/tests/services/session/test_pocketbase_isolation.py
index 3a0884423..022d5e805 100644
--- a/tests/services/session/test_pocketbase_isolation.py
+++ b/tests/services/session/test_pocketbase_isolation.py
@@ -17,6 +17,7 @@ import re
import pytest
+from deeptutor.api.routers.sessions import _attach_orphaned_failed_turns
from deeptutor.multi_user.context import reset_current_user, set_current_user
from deeptutor.multi_user.models import CurrentUser, UserScope
from deeptutor.services.session.pocketbase_store import PocketBaseSessionStore
@@ -255,6 +256,83 @@ async def test_get_session_404s_for_other_user(fake_pb) -> None:
assert own is not None and own["session_id"] == "s_secret"
+async def test_failed_turn_disappears_after_answered_pocketbase_retry(fake_pb) -> None:
+ store = PocketBaseSessionStore()
+ with as_user("alice"):
+ session = await store.create_session(session_id="s_retry")
+ turn = await store.begin_turn(session["id"], capability="chat")
+ user_id = await store.add_message(
+ session["id"], "user", "B", metadata={"turn_id": turn["id"]}
+ )
+ assert await store.transition_turn(
+ turn["id"],
+ "failed",
+ error="Worker lost",
+ failure_code="worker_lost",
+ retryable=True,
+ )
+ detail = await store.get_session_with_messages(session["id"])
+ assert detail is not None
+ _attach_orphaned_failed_turns(
+ detail["messages"], await store.list_orphaned_failed_turns(session["id"])
+ )
+ assert detail["messages"][0]["metadata"]["orphaned_failed_turn"]["turn_id"] == turn["id"]
+
+ answer_id = await store.add_message(
+ session["id"], "assistant", "The answer is B.", parent_message_id=user_id
+ )
+ detail = await store.get_session_with_messages(session["id"])
+ assert detail is not None
+ answer = detail["messages"][1]
+ assert answer["id"] == answer_id
+ assert answer["parent_message_id"] == user_id
+ assert "_parent_message_id" not in answer["metadata"]
+ _attach_orphaned_failed_turns(
+ detail["messages"], await store.list_orphaned_failed_turns(session["id"])
+ )
+ assert "orphaned_failed_turn" not in detail["messages"][0]["metadata"]
+
+
+async def test_legacy_pocketbase_answer_suppresses_only_its_own_failed_user(fake_pb) -> None:
+ store = PocketBaseSessionStore()
+ with as_user("alice"):
+ session = await store.create_session(session_id="s_legacy_retry")
+ turn = await store.begin_turn(session["id"], capability="chat")
+ await store.add_message(session["id"], "user", "First", metadata={"turn_id": turn["id"]})
+ assert await store.transition_turn(
+ turn["id"], "failed", error="Worker lost", failure_code="worker_lost"
+ )
+ # Older PocketBase rows have no parent metadata. A reply after the
+ # failed turn and before the next user row is still an answered retry.
+ await store.add_message(session["id"], "assistant", "First answered")
+ detail = await store.get_session_with_messages(session["id"])
+ assert detail is not None
+ _attach_orphaned_failed_turns(
+ detail["messages"], await store.list_orphaned_failed_turns(session["id"])
+ )
+ assert "orphaned_failed_turn" not in detail["messages"][0]["metadata"]
+
+ second = await store.create_session(session_id="s_other_answer")
+ second_turn = await store.begin_turn(second["id"], capability="chat")
+ await store.add_message(
+ second["id"], "user", "Still unanswered", metadata={"turn_id": second_turn["id"]}
+ )
+ assert await store.transition_turn(
+ second_turn["id"], "failed", error="Worker lost", failure_code="worker_lost"
+ )
+ await store.add_message(second["id"], "user", "A different question")
+ await store.add_message(second["id"], "assistant", "Different answer")
+ detail = await store.get_session_with_messages(second["id"])
+ assert detail is not None
+ _attach_orphaned_failed_turns(
+ detail["messages"], await store.list_orphaned_failed_turns(second["id"])
+ )
+ assert (
+ detail["messages"][0]["metadata"]["orphaned_failed_turn"]["turn_id"]
+ == second_turn["id"]
+ )
+
+
async def test_mutations_are_scoped_to_owner(fake_pb) -> None:
store = PocketBaseSessionStore()
with as_user("alice"):