Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): 避免后序归档阻塞队列 Worker (#3944)

Qin Haojiec96fbcb85fe4
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 0b6ae0605..e18ab92cb 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -792,6 +792,10 @@ class TaskTracker:
while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
await asyncio.sleep(0.05)
+ def has_work(self, task_id: str) -> bool:
+ """Return whether a task still owns durable or active queue work."""
+ return self._work_index.has_work(task_id)
+
def register_running_task(self, task_id: str) -> None:
"""Register the current asyncio task so cancellation can interrupt it."""
active_task = asyncio.current_task()
diff --git a/openviking/session/session.py b/openviking/session/session.py
index a10438cf1..4447118bd 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -68,7 +68,6 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
-_ARCHIVE_WAIT_POLL_SECONDS = 0.1
_PHASE2_QUEUE_WAIT_TIMEOUT_SECONDS = 1800.0
_MEMORY_EXTRACTION_MAX_RETRIES = 3
_MEMORY_EXTRACTION_RETRY_BASE_DELAY_SECONDS = 1.0
@@ -639,8 +638,6 @@ class Session:
self._agent_evolution_enabled_provider = agent_evolution_enabled_provider
self._usage_reporter = usage_reporter
- logger.info(f"Session created: {self.session_id} for user {self.user}")
-
async def load(self):
"""Load session data from storage."""
if self._loaded:
@@ -655,39 +652,38 @@ class Session:
for line in content.strip().split("\n")
if line.strip()
]
- logger.info(f"Session loaded: {self.session_id} ({len(self._messages)} messages)")
except Exception as exc:
if not _is_storage_not_found(exc):
raise
logger.debug(f"Session {self.session_id} not found, starting fresh")
- # Restore compression_index (scan history directory)
- try:
- history_items = await self._viking_fs.ls(f"{self._session_uri}/history", ctx=self.ctx)
- archive_indices = [
- int(match.group(1))
- for item in history_items
- if (match := re.fullmatch(r"archive_(\d+)", item["name"]))
- ]
- if archive_indices:
- max_index = max(archive_indices)
- self._compression.compression_index = max_index
- self._stats.compression_count = len(archive_indices)
- logger.debug(f"Restored compression_index: {max_index}")
- except Exception as exc:
- if not _is_storage_not_found(exc):
- raise
-
# Load .meta.json
try:
meta_content = await self._viking_fs.read_file(
f"{self._session_uri}/.meta.json", ctx=self.ctx
)
self._meta = SessionMeta.from_dict(json.loads(meta_content))
+ self._compression.compression_index = max(0, int(self._meta.commit_count))
+ self._stats.compression_count = self._compression.compression_index
except Exception as exc:
if not _is_storage_not_found(exc):
raise
# Old session without meta — derive from existing data
+ try:
+ history_items = await self._viking_fs.ls(
+ f"{self._session_uri}/history", ctx=self.ctx
+ )
+ archive_indices = [
+ int(match.group(1))
+ for item in history_items
+ if (match := re.fullmatch(r"archive_(\d+)", item["name"]))
+ ]
+ if archive_indices:
+ self._compression.compression_index = max(archive_indices)
+ self._stats.compression_count = len(archive_indices)
+ except Exception as exc:
+ if not _is_storage_not_found(exc):
+ raise
self._meta.commit_count = self._compression.compression_index
self._meta.total_message_count = None
@@ -1681,6 +1677,28 @@ class Session:
if await self._archive_file_exists(archive_uri, ".failed.json"):
return False
+ queue_message = marker.get("queue_message")
+ task_id = queue_message.get("task_id") if isinstance(queue_message, dict) else None
+ from openviking.service.task_tracker import get_task_tracker
+
+ tracker = get_task_tracker()
+ if not task_id or not tracker.has_work(str(task_id)):
+ error = "Phase 1 has no QueueFS work to resume"
+ await self._write_failed_marker(
+ archive_uri,
+ stage="phase1_recovery",
+ error=error,
+ lease_ref=lease,
+ )
+ if task_id:
+ await tracker.fail(
+ str(task_id),
+ error,
+ account_id=self.ctx.account_id,
+ user_id=self.ctx.user.user_id,
+ )
+ return False
+
try:
if not marker:
raise ValueError("Phase 1 metadata is missing")
@@ -1788,7 +1806,7 @@ class Session:
Phase 1 (Archive prep, path-lock protected): Split messages into
archive/retain parts, persist a recoverable intent and archive raw,
- enqueue Phase 2, then atomically publish the retained root state with
+ enqueue Phase 2, then publish the retained root state with
``phase1.status=ready``. Uses a distributed filesystem lock across
workers and processes.
Phase 2 (Memory extraction): Runs through the persistent QueueFS queue.
@@ -1936,11 +1954,19 @@ class Session:
effective_memory_types=set(effective_memory_types),
)
- archive_refs = await self._list_archive_refs()
self._compression.compression_index = max(
- [archive["index"] for archive in archive_refs],
- default=0,
+ self._compression.compression_index,
+ int(self._meta.commit_count),
)
+ while await self._viking_fs.exists(
+ (
+ f"{self._session_uri}/history/"
+ f"archive_{self._compression.compression_index + 1:03d}"
+ ),
+ ctx=self.ctx,
+ ):
+ self._compression.compression_index += 1
+
if not self._messages:
self._meta.pending_tokens = 0
self._remember_retention_policy(
@@ -2036,9 +2062,6 @@ class Session:
)
phase1_stage = "phase1_persist"
try:
- # Persist the full intent before any root rewrite. Queueing while
- # holding the same session lock lets a consumer either observe
- # final ready metadata or recover the exact interrupted state.
await self._write_phase1_marker(
archive_uri,
queue_message=queue_msg.to_dict(),
@@ -2085,11 +2108,6 @@ class Session:
queue_msg.to_dict(),
)
- # Register after the durable queue write but before publishing
- # ready. The consumer is blocked by this session lock until
- # ready, so it cannot complete the task before registration;
- # if registration fails, the queued consumer will observe the
- # Phase 1 failure marker and create/fail the task itself.
phase1_stage = "task_tracker_create"
await get_task_tracker().create(
"session_commit",
@@ -2175,7 +2193,7 @@ class Session:
error="session commit cancelled",
)
- async def resume_queued_commit(self, msg: "SessionCommitMsg") -> None:
+ async def resume_queued_commit(self, msg: "SessionCommitMsg") -> bool:
"""Run one durable Phase 2 job from its archived messages."""
from openviking.service.task_tracker import get_task_tracker
@@ -2195,14 +2213,14 @@ class Session:
raise
else:
if task.status.value == "completed":
- return
+ return True
await tracker.complete(
msg.task_id,
{"session_id": self.session_id, "archive_uri": msg.archive_uri},
account_id=self.ctx.account_id,
user_id=self.ctx.user.user_id,
)
- return
+ return True
try:
failed = json.loads(
@@ -2218,7 +2236,7 @@ class Session:
account_id=self.ctx.account_id,
user_id=self.ctx.user.user_id,
)
- return
+ return True
if not await self._ensure_phase1_ready(msg.archive_uri):
try:
@@ -2237,7 +2255,10 @@ class Session:
account_id=self.ctx.account_id,
user_id=self.ctx.user.user_id,
)
- return
+ return True
+
+ if not await self._can_run_archive(self._archive_index_from_uri(msg.archive_uri)):
+ return False
archive_error = ""
try:
@@ -2262,7 +2283,7 @@ class Session:
account_id=self.ctx.account_id,
user_id=self.ctx.user.user_id,
)
- return
+ return True
queued_policy = MemoryPolicy.from_dict(msg.memory_policy)
archive_meta = await self._read_archive_meta(msg.archive_uri)
@@ -2306,6 +2327,7 @@ class Session:
record_auto_commit_success=msg.record_auto_commit_success,
event_search_tags=list(msg.event_search_tags or []),
)
+ return True
async def _run_usage_reporting(
self,
@@ -2363,7 +2385,6 @@ class Session:
archive_index = self._archive_index_from_uri(archive_uri)
try:
- await self._wait_for_previous_archive_done(archive_index)
(
messages,
coverage_start_archive,
@@ -2637,11 +2658,10 @@ class Session:
*extraction_tasks,
return_exceptions=True,
)
- # The archive outcome is binary: if ANY Phase 2 step
- # still fails after retries, no .done coverage is
- # published. Successful memory steps keep a message-ID
- # progress marker solely to make later raw replay
- # idempotent.
+ # The archive outcome is binary: if any Phase 2 step
+ # still fails after retries, no .done marker is
+ # published. Successful steps retain message IDs so the
+ # same archive can resume without repeating them.
extraction_error: Optional[BaseException] = None
for label, result in zip(extraction_labels, _results, strict=True):
if isinstance(result, Exception):
@@ -3029,9 +3049,7 @@ class Session:
``failed`` archive no longer replays its raw messages. Cumulative v2
checkpoints are restored from the terminal archive alone; only a
terminal legacy v1 delta checkpoint invokes an older-history compatibility
- scan. Phase 2 coverage bookkeeping is unaffected because it keeps using
- the full ``_scan_archive_states()`` scan. Public
- ``pre_archive_abstracts`` stay empty; abstracts are not read.
+ scan. Public ``pre_archive_abstracts`` stay empty; abstracts are not read.
"""
archive_refs = await self._list_archive_refs()
newer_pending: List[Dict[str, Any]] = []
@@ -3850,10 +3868,8 @@ class Session:
) -> List[Message]:
"""Return pending/failed raw messages not covered by a completed archive.
- Kept as the RFC #3330 recovery helper. ``get_session_context`` no longer
- calls it because that read path stops at the newest terminal archive
- instead of walking the full history; Phase 2 roll-forward still replays
- uncovered failed archives through ``_prepare_phase2_archive_messages``.
+ Kept as the RFC #3330 compatibility helper. Current context assembly
+ and Phase 2 both skip failed archive raw messages.
"""
states = states if states is not None else await self._scan_archive_states()
covered = self._covered_archive_ids(states)
@@ -3884,116 +3900,72 @@ class Session:
raise ValueError(f"Invalid archive URI: {archive_uri}")
return int(match.group(1))
- async def _wait_for_previous_archive_done(self, archive_index: int) -> bool:
- """Wait until every earlier archive reaches a terminal state."""
+ async def _can_run_archive(self, archive_index: int) -> bool:
+ """Resolve an orphaned direct predecessor before this Archive runs."""
if archive_index <= 1 or not self._viking_fs:
return True
- while True:
- earlier_states = [
- state for state in await self._scan_archive_states() if state.index < archive_index
- ]
- pending_states = [state for state in earlier_states if state.state == "pending"]
- if not pending_states:
- non_completed = [state for state in earlier_states if state.state == "failed"]
- if non_completed:
- logger.info(
- "Earlier archives reached terminal non-completed states; "
- "continuing with raw replay: %s",
- [state.archive_id for state in non_completed],
- )
- return True
+ predecessor_uri = (
+ f"{self._session_uri}/history/archive_{archive_index - 1:03d}"
+ )
+ if not await self._viking_fs.exists(predecessor_uri, ctx=self.ctx):
+ return True
+ if await self._archive_terminal_state(predecessor_uri) != "pending":
+ return True
- # A new-format intent without ready status may be left by a
- # process interruption. Reconcile it under the session lock rather
- # than waiting forever for a queue item that may never have existed.
- reconciled = False
- for state in pending_states:
- phase1 = await self._read_phase1_meta(state.archive_uri)
- if not phase1 or phase1.get("status") == "ready":
- continue
- await self._ensure_phase1_ready(state.archive_uri)
- reconciled = True
- if reconciled:
- continue
- await asyncio.sleep(_ARCHIVE_WAIT_POLL_SECONDS)
+ phase1 = await self._read_phase1_meta(predecessor_uri)
+ if not phase1:
+ return False
+ if phase1.get("status") != "ready":
+ return not await self._ensure_phase1_ready(predecessor_uri)
+
+ queue_message = phase1.get("queue_message")
+ task_id = queue_message.get("task_id") if isinstance(queue_message, dict) else None
+ if not task_id:
+ return False
+
+ from openviking.service.task_tracker import get_task_tracker
+
+ tracker = get_task_tracker()
+ if tracker.has_work(str(task_id)):
+ return False
+
+ error = "Session commit queue work is missing"
+ await self._write_failed_marker(
+ predecessor_uri,
+ stage="queue_missing",
+ error=error,
+ )
+ await tracker.fail(
+ str(task_id),
+ error,
+ account_id=self.ctx.account_id,
+ user_id=self.ctx.user.user_id,
+ )
+ logger.warning(
+ "Skipped orphaned Session archive without QueueFS work: %s",
+ predecessor_uri,
+ )
+ return True
async def _prepare_phase2_archive_messages(
self,
archive_uri: str,
current_messages: List[Message],
) -> tuple[List[Message], str, str, List[str], Dict[str, set[str]]]:
- """Roll earlier failed raw archives into current Phase 2 input.
-
- Pending archives are never replayed: their own Phase 2 job still owns
- them, and this method is called only after all earlier directories have
- reached a terminal state.
- """
- current_index = self._archive_index_from_uri(archive_uri)
- states = await self._scan_archive_states()
- covered = self._covered_archive_ids(states)
- replay_states = [
- state
- for state in states
- if state.index < current_index
- and state.archive_id not in covered
- and state.state == "failed"
- ]
-
- combined: List[Message] = []
+ """Prepare only the current archive, preserving its retry progress."""
+ current_archive_id = archive_uri.rstrip("/").split("/")[-1]
completed_memory_steps: Dict[str, set[str]] = {}
- for state in replay_states:
- # A terminally-failed earlier archive can have a missing or corrupt
- # messages.jsonl (legacy "no messages" data, or #3417's own
- # archive_read terminal path). Tolerate that exactly like
- # _get_uncovered_archive_messages does, otherwise every subsequent
- # commit's Phase 2 re-raises here and stays permanently poisoned.
- # Real storage failures still propagate.
- try:
- replayed = await self._read_archive_messages(state.archive_uri)
- except _ArchiveMessagesCorruptError:
- logger.warning(
- "Skipping failed archive %s in Phase-2 replay: messages.jsonl is corrupt",
- state.archive_uri,
- )
- continue
- except Exception as exc:
- if not _is_storage_not_found(exc):
- raise
- logger.warning(
- "Skipping failed archive %s in Phase-2 replay: messages.jsonl is missing",
- state.archive_uri,
- )
- continue
- combined.extend(replayed)
- marker = state.failed
- self._merge_completed_memory_steps(
- completed_memory_steps,
- marker.get("completed_memory_steps") if marker else None,
- )
- state_meta = await self._read_archive_meta(state.archive_uri)
- self._merge_completed_memory_steps(
- completed_memory_steps,
- state_meta.get("completed_memory_steps"),
- )
- combined.extend(current_messages)
- combined = self._stable_deduplicate_messages(combined)
-
- # A restarted queue item can resume the current archive after a process
- # died between a successful memory step and the final .done write.
current_meta = await self._read_archive_meta(archive_uri)
self._merge_completed_memory_steps(
completed_memory_steps,
current_meta.get("completed_memory_steps"),
)
-
- coverage_start_index = min([current_index] + [state.index for state in replay_states])
- covered_failed = [state.archive_id for state in replay_states if state.state == "failed"]
return (
- combined,
- f"archive_{coverage_start_index:03d}",
- f"archive_{current_index:03d}",
- covered_failed,
+ current_messages,
+ current_archive_id,
+ current_archive_id,
+ [],
completed_memory_steps,
)
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index e8d61817e..53bcb08d1 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -82,6 +82,8 @@ class QueueManager:
ADD_RESOURCE = "AddResource"
SESSION_COMMIT = "SessionCommit"
USER_DELETION = "UserDeletion"
+ # A deferred archive re-enqueues itself; throttle the next scheduling round.
+ _SESSION_COMMIT_POLL_INTERVAL = 1.0
def __init__(
self,
@@ -213,6 +215,11 @@ class QueueManager:
"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
+ poll_interval = (
+ self._SESSION_COMMIT_POLL_INTERVAL
+ if queue.name == self.SESSION_COMMIT
+ else self._poll_interval
+ )
try:
if max_concurrent > 1:
loop.run_until_complete(
@@ -226,12 +233,14 @@ class QueueManager:
data = loop.run_until_complete(queue.dequeue())
if data is not None:
logger.debug("[QueueManager] Dequeued message from %s", queue.name)
+ if queue.name == self.SESSION_COMMIT:
+ stop_event.wait(poll_interval)
else:
- stop_event.wait(self._poll_interval)
+ stop_event.wait(poll_interval)
except Exception as e:
logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")
traceback.print_exc()
- stop_event.wait(self._poll_interval)
+ stop_event.wait(poll_interval)
finally:
loop.close()
@@ -242,6 +251,11 @@ class QueueManager:
A Semaphore caps inflight tasks at max_concurrent.
"""
+ poll_interval = (
+ self._SESSION_COMMIT_POLL_INTERVAL
+ if queue.name == self.SESSION_COMMIT
+ else self._poll_interval
+ )
sem = asyncio.Semaphore(max_concurrent)
active_tasks: Set[asyncio.Task] = set()
@@ -283,7 +297,7 @@ class QueueManager:
f"(active={len(active_tasks)})"
)
- await asyncio.sleep(self._poll_interval)
+ await asyncio.sleep(poll_interval)
# Drain remaining in-flight tasks on shutdown (with timeout)
if active_tasks:
diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.py
index ca0a03d66..9af90cbe6 100644
--- a/openviking/storage/queuefs/session_commit_processor.py
+++ b/openviking/storage/queuefs/session_commit_processor.py
@@ -44,7 +44,7 @@ class SessionCommitProcessor(DequeueHandlerBase):
)
return msg, ctx
- async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:
+ async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> bool:
# Bind a root observability context so Phase-2 extraction VLM/embedding
# token events are attributed to the committing account/user rather than
# "__unknown__" (mirrors SemanticProcessor.on_dequeue). Must bind inside
@@ -81,10 +81,19 @@ class SessionCommitProcessor(DequeueHandlerBase):
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
- return
+ return True
await session.load()
with bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id):
- await session.resume_queued_commit(msg)
+ processed = await session.resume_queued_commit(msg)
+ if not processed:
+ from openviking.storage.queuefs import QueueManager, get_queue_manager
+
+ await get_queue_manager().enqueue(
+ QueueManager.SESSION_COMMIT,
+ msg.to_dict(),
+ )
+ self.report_requeue()
+ return processed
finally:
reset_root_observability_context(root_context_token)
@@ -128,7 +137,7 @@ class SessionCommitProcessor(DequeueHandlerBase):
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
self.report_error(str(exc), data)
return None
- future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(
+ future: concurrent.futures.Future[bool] = asyncio.run_coroutine_threadsafe(
self._process(msg, ctx),
self._service_loop,
)
diff --git a/tests/server/conftest.py b/tests/server/conftest.py
index c5028fc68..18a22268d 100644
--- a/tests/server/conftest.py
+++ b/tests/server/conftest.py
@@ -22,6 +22,7 @@ from openviking.server.app import create_app
from openviking.server.config import ServerConfig
from openviking.server.identity import RequestContext, Role
from openviking.service.core import OpenVikingService
+from openviking.service.task_tracker import get_task_tracker
from openviking.storage.queuefs import QueueManager, SessionCommitMsg, get_queue_manager
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config.embedding_config import EmbeddingConfig
@@ -89,11 +90,21 @@ def _install_session_commit_queue_fallback(service: OpenVikingService, monkeypat
"""Execute SessionCommit jobs when MockLocalAGFS cannot dequeue QueueFS entries."""
queue_manager = get_queue_manager()
original_enqueue = queue_manager.enqueue
+ queued_commit_tasks: set[str] = set()
+ tracker = get_task_tracker()
+ original_has_work = tracker.has_work
+ monkeypatch.setattr(
+ tracker,
+ "has_work",
+ lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),
+ )
async def enqueue_with_session_commit_fallback(queue_name, data):
if queue_name != QueueManager.SESSION_COMMIT:
return await original_enqueue(queue_name, data)
+ queued_commit_tasks.add(data["task_id"])
+
async def process_commit() -> None:
await asyncio.sleep(0)
msg = SessionCommitMsg(**data)
@@ -106,8 +117,20 @@ def _install_session_commit_queue_fallback(service: OpenVikingService, monkeypat
msg.session_id,
session_uri=msg.session_uri,
)
+ while True:
+ phase1 = await queued_session._read_phase1_meta(msg.archive_uri)
+ if phase1.get("status") == "ready" or await queued_session._archive_file_exists(
+ msg.archive_uri,
+ ".failed.json",
+ ):
+ break
+ await asyncio.sleep(0)
await queued_session.load()
- await queued_session.resume_queued_commit(msg)
+ try:
+ while not await queued_session.resume_queued_commit(msg):
+ await asyncio.sleep(0)
+ finally:
+ queued_commit_tasks.discard(msg.task_id)
asyncio.create_task(process_commit())
return data["task_id"]
diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py
index 8c5638db2..78167f840 100644
--- a/tests/server/test_api_sessions.py
+++ b/tests/server/test_api_sessions.py
@@ -1170,7 +1170,8 @@ async def test_delete_session(client: httpx.AsyncClient):
json=_message_request("user", content="ensure persisted"),
)
# Compress to persist
- await client.post(f"/api/v1/sessions/{session_id}/commit")
+ commit_resp = await client.post(f"/api/v1/sessions/{session_id}/commit")
+ await _wait_for_task(client, commit_resp.json()["result"]["task_id"])
resp = await client.delete(f"/api/v1/sessions/{session_id}")
assert resp.status_code == 200
diff --git a/tests/session/conftest.py b/tests/session/conftest.py
index 4866b9758..0e153346d 100644
--- a/tests/session/conftest.py
+++ b/tests/session/conftest.py
@@ -12,6 +12,7 @@ import pytest_asyncio
from openviking.message import TextPart, ToolPart
from openviking.server.identity import RequestContext
from openviking.service.core import OpenVikingService
+from openviking.service.task_tracker import get_task_tracker
from openviking.session import Session
from openviking.storage.queuefs import QueueManager, SessionCommitMsg, get_queue_manager
from openviking.utils.time_utils import get_current_timestamp
@@ -28,11 +29,22 @@ async def client(
queue_manager = get_queue_manager()
original_enqueue = queue_manager.enqueue
commit_tasks = []
+ queued_commit_tasks: set[str] = set()
+ tracker = get_task_tracker()
+ original_has_work = tracker.has_work
+
+ monkeypatch.setattr(
+ tracker,
+ "has_work",
+ lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),
+ )
async def enqueue_with_session_commit_fallback(queue_name, data):
if queue_name != QueueManager.SESSION_COMMIT:
return await original_enqueue(queue_name, data)
+ queued_commit_tasks.add(data["task_id"])
+
async def process_commit():
message = SessionCommitMsg(**data)
queued_session = service.sessions.session(
@@ -49,7 +61,11 @@ async def client(
break
await asyncio.sleep(0)
await queued_session.load()
- await queued_session.resume_queued_commit(message)
+ try:
+ while not await queued_session.resume_queued_commit(message):
+ await asyncio.sleep(0)
+ finally:
+ queued_commit_tasks.discard(message.task_id)
commit_tasks.append(asyncio.create_task(process_commit()))
return data["task_id"]
diff --git a/tests/session/test_session_context.py b/tests/session/test_session_context.py
index 1a3a17a30..8e45f0452 100644
--- a/tests/session/test_session_context.py
+++ b/tests/session/test_session_context.py
@@ -44,11 +44,22 @@ async def client(
queue_manager = get_queue_manager()
original_enqueue = queue_manager.enqueue
+ commit_tasks: list[asyncio.Task] = []
+ queued_commit_tasks: set[str] = set()
+ tracker = get_task_tracker()
+ original_has_work = tracker.has_work
+ monkeypatch.setattr(
+ tracker,
+ "has_work",
+ lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),
+ )
async def enqueue_with_session_commit_fallback(queue_name, data):
if queue_name != QueueManager.SESSION_COMMIT:
return await original_enqueue(queue_name, data)
+ queued_commit_tasks.add(data["task_id"])
+
async def process_commit():
message = SessionCommitMsg(**data)
queued_session = client(session_id=message.session_id)
@@ -58,13 +69,19 @@ async def client(
break
await asyncio.sleep(0)
await queued_session.load()
- await queued_session.resume_queued_commit(message)
+ try:
+ while not await queued_session.resume_queued_commit(message):
+ await asyncio.sleep(0)
+ finally:
+ queued_commit_tasks.discard(message.task_id)
- asyncio.create_task(process_commit())
+ commit_tasks.append(asyncio.create_task(process_commit()))
return data["task_id"]
monkeypatch.setattr(queue_manager, "enqueue", enqueue_with_session_commit_fallback)
yield client
+ if commit_tasks:
+ await asyncio.gather(*commit_tasks, return_exceptions=True)
def _estimate_tokens(text: str) -> int:
diff --git a/tests/session/test_session_retention_integration.py b/tests/session/test_session_retention_integration.py
index 596ef0310..0979be0d2 100644
--- a/tests/session/test_session_retention_integration.py
+++ b/tests/session/test_session_retention_integration.py
@@ -117,11 +117,7 @@ async def test_session_context_enforces_hard_budget_without_mutating_archive_raw
async def test_context_stops_at_newest_terminal_without_replaying_older_failed_raw(
client,
):
- """The read path stops at archive_002 and never replays archive_001 raw.
-
- Phase 2 still treats the uncovered failed archive as replayable; only
- ``get_session_context`` stops at the newest terminal.
- """
+ """The read path stops at archive_002 and never replays archive_001 raw."""
session = client(session_id="failed_and_wm_disabled_archive_test")
await session.ensure_exists()
await _write_archive(
@@ -260,7 +256,7 @@ async def test_later_coverage_absorbs_failed_raw_and_stable_deduplicates_root(
assert [message["id"] for message in context["messages"]] == ["u1"]
-async def test_phase2_replays_failed_but_not_completed_without_overview(
+async def test_phase2_skips_failed_and_completed_archives_without_replay(
client,
):
session = client(session_id="phase2_failed_replay_test")
@@ -288,75 +284,57 @@ async def test_phase2_replays_failed_but_not_completed_without_overview(
[_text_message("u3", "user", "current three")],
)
- assert [message.id for message in messages] == ["u1", "u3"]
- assert start == "archive_001"
+ assert [message.id for message in messages] == ["u3"]
+ assert start == "archive_003"
assert end == "archive_003"
- assert failed == ["archive_001"]
+ assert failed == []
assert completed_steps == {}
-async def test_phase2_never_replays_an_earlier_pending_archive(
- client,
-):
- session = client(session_id="phase2_pending_not_replayed_test")
- await session.ensure_exists()
- await _write_archive(session, 1, [_text_message("u1", "user", "pending one")])
- await _write_archive(
- session,
- 2,
- [_text_message("u2", "user", "failed two")],
- failed=True,
- )
- current_uri = await _write_archive(
- session,
- 3,
- [_text_message("u3", "user", "current three")],
- )
-
- messages, start, end, failed, _completed_steps = await session._prepare_phase2_archive_messages(
- current_uri,
- [_text_message("u3", "user", "current three")],
- )
-
- assert [message.id for message in messages] == ["u2", "u3"]
- assert start == "archive_002"
- assert end == "archive_003"
- assert failed == ["archive_002"]
-
-
-async def test_phase2_waits_for_all_earlier_pending_archives(
+async def test_phase2_defers_live_predecessor_and_skips_missing_queue_work(
client,
monkeypatch,
):
- session = client(session_id="phase2_wait_all_pending_test")
+ session = client(session_id="phase2_predecessor_queue_work_test")
await session.ensure_exists()
- first_uri = await _write_archive(
+ await _write_archive(
session,
1,
[_text_message("u1", "user", "pending one")],
+ meta={
+ "phase1": {
+ "status": "ready",
+ "queue_message": {"task_id": "task-1"},
+ }
+ },
)
- await _write_archive(
+ second_uri = await _write_archive(
session,
2,
- [_text_message("u2", "user", "completed two")],
- overview="# Summary\n\ncomplete",
- done={"starting_message_id": "u2", "ending_message_id": "u2"},
+ [_text_message("u2", "user", "pending two")],
+ meta={
+ "phase1": {
+ "status": "ready",
+ "queue_message": {"task_id": "task-2"},
+ }
+ },
)
- monkeypatch.setattr("openviking.session.session._ARCHIVE_WAIT_POLL_SECONDS", 0.01)
+ tracker = get_task_tracker()
+ live_work = {"task-1", "task-2"}
+ monkeypatch.setattr(tracker, "has_work", live_work.__contains__)
- waiter = asyncio.create_task(session._wait_for_previous_archive_done(3))
- await asyncio.sleep(0.03)
- assert not waiter.done()
+ assert not await session._can_run_archive(3)
- await session._viking_fs.write_file(
- f"{first_uri}/.failed.json",
- json.dumps({"error": "synthetic terminal failure"}),
- ctx=session.ctx,
+ live_work.remove("task-2")
+ assert await session._can_run_archive(3)
+ failed = json.loads(
+ await session._viking_fs.read_file(f"{second_uri}/.failed.json", ctx=session.ctx)
)
- assert await asyncio.wait_for(waiter, timeout=0.5)
+ assert failed["stage"] == "queue_missing"
+ assert not await session._viking_fs.exists(f"{session.uri}/history/archive_001/.failed.json")
-async def test_missing_previous_archive_directory_does_not_block_phase2(
+async def test_missing_previous_archive_directory_allows_phase2(
client,
):
session = client(session_id="missing_previous_archive_test")
@@ -367,13 +345,10 @@ async def test_missing_previous_archive_directory_does_not_block_phase2(
[_text_message("u2", "user", "archive two")],
)
- assert await asyncio.wait_for(
- session._wait_for_previous_archive_done(2),
- timeout=0.5,
- )
+ assert await session._can_run_archive(2)
-async def test_phase2_roll_forward_writes_coverage_and_calls_existing_summary_once(
+async def test_phase2_processes_only_current_archive(
client,
monkeypatch,
):
@@ -398,7 +373,7 @@ async def test_phase2_roll_forward_writes_coverage_and_calls_existing_summary_on
async def fake_summary(messages, latest_archive_overview=""):
assert latest_archive_overview == ""
seen_message_ids.append([message.id for message in messages])
- return "# Session Summary\n\n## Current State\nBoth rounds are covered."
+ return "# Session Summary\n\n## Current State\nCurrent archive is covered."
config = SimpleNamespace(
memory=SimpleNamespace(
@@ -431,77 +406,16 @@ async def test_phase2_roll_forward_writes_coverage_and_calls_existing_summary_on
done = json.loads(await session._viking_fs.read_file(f"{current_uri}/.done", ctx=session.ctx))
task = await get_task_tracker().get(task_id)
context = await session.get_session_context()
- assert seen_message_ids == [["u1", "u2"]]
+ assert seen_message_ids == [["u2"]]
assert task.result["agent_evolution_enabled"] is True
- assert done["coverage_start_archive"] == "archive_001"
+ assert done["coverage_start_archive"] == "archive_002"
assert done["coverage_end_archive"] == "archive_002"
- assert done["covered_failed_archives"] == ["archive_001"]
- assert context["latest_archive_overview"].endswith("Both rounds are covered.")
+ assert done["covered_failed_archives"] == []
+ assert context["latest_archive_overview"].endswith("Current archive is covered.")
assert context["messages"] == []
-async def test_phase2_rolls_forward_done_archive_with_missing_required_overview(
- client,
- monkeypatch,
-):
- session = client(session_id="invalid_done_roll_forward_test")
- await session.ensure_exists()
- await _write_archive(
- session,
- 1,
- [_text_message("u1", "user", "raw from invalid completed archive")],
- done={
- "starting_message_id": "u1",
- "ending_message_id": "u1",
- "working_memory_enabled": True,
- },
- )
- current = _text_message("u2", "user", "current archive")
- current_uri = await _write_archive(session, 2, [current])
- seen_message_ids: list[list[str]] = []
-
- async def fake_summary(messages, latest_archive_overview=""):
- assert latest_archive_overview == ""
- seen_message_ids.append([message.id for message in messages])
- return "# Session Summary\n\n## Current State\nInvalid archive was recovered."
-
- config = SimpleNamespace(
- memory=SimpleNamespace(
- extraction_enabled=False,
- session_skill_extraction_enabled=False,
- )
- )
- monkeypatch.setattr("openviking.session.session.get_openviking_config", lambda: config)
- monkeypatch.setattr(session, "_generate_archive_summary_async", fake_summary)
-
- task_id = str(uuid4())
- await get_task_tracker().create(
- "session_commit",
- resource_id=session.session_id,
- account_id=session.ctx.account_id,
- user_id=session.ctx.user.user_id,
- task_id=task_id,
- )
- await session._run_memory_extraction(
- task_id=task_id,
- archive_uri=current_uri,
- messages=[current],
- usage_records=[],
- first_message_id=current.id,
- last_message_id=current.id,
- memory_policy={"working_memory": {"enabled": True}},
- )
-
- done = json.loads(await session._viking_fs.read_file(f"{current_uri}/.done", ctx=session.ctx))
- context = await session.get_session_context()
- assert seen_message_ids == [["u1", "u2"]]
- assert done["coverage_start_archive"] == "archive_001"
- assert done["covered_failed_archives"] == ["archive_001"]
- assert context["latest_archive_overview"].endswith("Invalid archive was recovered.")
- assert context["messages"] == []
-
-
-async def test_roll_forward_does_not_repeat_completed_memory_step_messages(
+async def test_phase2_retry_does_not_repeat_completed_current_archive_steps(
client,
monkeypatch,
):
@@ -516,14 +430,19 @@ async def test_roll_forward_does_not_repeat_completed_memory_step_messages(
meta={"completed_memory_steps": {"long_term": ["u1"]}},
)
current = _text_message("u2", "user", "extract this once")
- current_uri = await _write_archive(session, 2, [current])
+ current_uri = await _write_archive(
+ session,
+ 2,
+ [current],
+ meta={"completed_memory_steps": {"long_term": ["u2"]}},
+ )
summary_inputs: list[list[str]] = []
long_term_inputs: list[list[str]] = []
async def fake_summary(messages, latest_archive_overview=""):
assert latest_archive_overview == ""
summary_inputs.append([message.id for message in messages])
- return "# Session Summary\n\n## Current State\nBoth messages are summarized."
+ return "# Session Summary\n\n## Current State\nCurrent message is summarized."
async def fake_long_term(*, messages, **_kwargs):
long_term_inputs.append([message.id for message in messages])
@@ -567,9 +486,9 @@ async def test_roll_forward_does_not_repeat_completed_memory_step_messages(
)
done = json.loads(await session._viking_fs.read_file(f"{current_uri}/.done", ctx=session.ctx))
- assert summary_inputs == [["u1", "u2"]]
- assert long_term_inputs == [["u2"]]
- assert done["completed_memory_steps"]["long_term"] == ["u1", "u2"]
+ assert summary_inputs == [["u2"]]
+ assert long_term_inputs == []
+ assert done["completed_memory_steps"]["long_term"] == ["u2"]
async def test_completed_partial_turn_inserts_checkpoint_after_user_anchor(
@@ -1833,14 +1752,20 @@ async def test_phase1_enqueues_before_root_rewrite_and_publishes_ready_last(
)
result = await session.commit_async()
-
- assert observations == [(["archive me"], "preparing")]
+ session.add_message("user", [TextPart("wait for predecessor")])
+ second = await session.commit_async()
+ assert observations == [
+ (["archive me"], "preparing"),
+ (["wait for predecessor"], "preparing"),
+ ]
assert (await session._read_phase1_meta(result["archive_uri"]))["status"] == "ready"
+ assert (await session._read_phase1_meta(second["archive_uri"]))["status"] == "ready"
assert await session._read_live_messages_strict() == []
async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(
client,
+ monkeypatch,
):
session = client(session_id="phase1_reconcile_durable_root_test")
await session.ensure_exists()
@@ -1864,6 +1789,7 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(
min_raw_tail_steps=1,
)
await session._write_to_agfs_async(messages=[original[1]])
+ monkeypatch.setattr(get_task_tracker(), "has_work", lambda task_id: task_id == "synthetic")
assert await session._ensure_phase1_ready(archive_uri)
assert (await session._read_phase1_meta(archive_uri))["status"] == "ready"
@@ -1895,7 +1821,11 @@ async def test_interrupted_phase1_before_root_rewrite_becomes_failed(
assert not await session._ensure_phase1_ready(archive_uri)
states = await session._scan_archive_states()
context = await session.get_session_context()
+ failed = json.loads(
+ await session._viking_fs.read_file(f"{archive_uri}/.failed.json", ctx=session.ctx)
+ )
assert states[0].state == "failed"
+ assert failed["error"] == "Phase 1 has no QueueFS work to resume"
assert [message["id"] for message in context["messages"]] == ["u1"]
diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.py
index ebccb77c3..7b9bce2b7 100644
--- a/tests/storage/test_session_commit_processor_identity.py
+++ b/tests/storage/test_session_commit_processor_identity.py
@@ -22,8 +22,9 @@ from openviking_cli.session.user_id import UserIdentifier
class _FakeSession:
- def __init__(self, captured: dict) -> None:
+ def __init__(self, captured: dict, processed: bool = True) -> None:
self._captured = captured
+ self._processed = processed
async def exists(self) -> bool:
return True
@@ -31,18 +32,20 @@ class _FakeSession:
async def load(self) -> None:
return None
- async def resume_queued_commit(self, msg) -> None:
+ async def resume_queued_commit(self, msg) -> bool:
root = get_root_observability_context()
self._captured["account_id"] = root.account_id if root else None
self._captured["user_id"] = root.user_id if root else None
+ return self._processed
class _FakeSessionService:
- def __init__(self, captured: dict) -> None:
+ def __init__(self, captured: dict, processed: bool = True) -> None:
self._captured = captured
+ self._processed = processed
def session(self, ctx, session_id, session_uri=None):
- return _FakeSession(self._captured)
+ return _FakeSession(self._captured, self._processed)
class _MemoryVikingFS:
@@ -88,15 +91,26 @@ async def test_process_binds_committing_identity_to_root_context():
assert captured["user_id"] == "alice"
-async def test_process_resets_root_context_after_completion():
+async def test_process_requeues_deferred_commit_and_resets_root_context(monkeypatch):
+ queued = []
+
+ class _QueueManager:
+ async def enqueue(self, queue_name, data):
+ queued.append((queue_name, data))
+
processor = SessionCommitProcessor(
- _FakeSessionService({}),
+ _FakeSessionService({}, processed=False),
asyncio.get_running_loop(),
)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.get_queue_manager",
+ lambda: _QueueManager(),
+ )
ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.USER)
await processor._process(_make_msg(), ctx)
+ assert queued == [("SessionCommit", _make_msg().to_dict())]
assert get_root_observability_context() is None
diff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.py
index 01c7b2b8c..cdebae0f7 100644
--- a/tests/unit/session/test_session_commit_resume.py
+++ b/tests/unit/session/test_session_commit_resume.py
@@ -35,6 +35,7 @@ class _TaskStore:
class _MemoryVikingFS:
def __init__(self, files):
self.files = files
+ self._async_agfs = AsyncMock()
def _uri_to_path(self, uri, ctx=None):
return "/local/session-1"
@@ -44,7 +45,7 @@ class _MemoryVikingFS:
raise FileNotFoundError(uri)
return self.files[uri]
- async def write_file(self, uri, content, ctx=None):
+ async def write_file(self, uri, content, ctx=None, lease_ref=None):
self.files[uri] = content
@@ -72,6 +73,7 @@ async def test_resume_queued_commit_continues_phase2(monkeypatch):
session_uri=session_uri,
archive_uri=archive_uri,
user={"account_id": "default", "user_id": "default"},
+ memory_policy={"memory_types": []},
)
try:
diff --git a/tests/unit/session/test_session_phase2_replay_tolerance.py b/tests/unit/session/test_session_phase2_replay_tolerance.py
deleted file mode 100644
index db2229ea1..000000000
--- a/tests/unit/session/test_session_phase2_replay_tolerance.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Regression tests for Phase-2 archive replay tolerance (follow-up to #3417).
-
-#3417 made ``Session._read_archive_messages`` strict: it now raises on a
-missing/corrupt ``messages.jsonl`` instead of returning ``[]``. The read path
-(``_get_uncovered_archive_messages``) gained not-found tolerance, but the
-Phase-2 commit replay path (``_prepare_phase2_archive_messages``) did not.
-
-A terminally-failed earlier archive whose ``messages.jsonl`` is missing would
-therefore make every subsequent commit's Phase-2 replay raise, terminal-fail
-the current archive, and permanently poison the session's memory extraction.
-"""
-
-from unittest.mock import AsyncMock
-
-import pytest
-
-from openviking.session.session import ArchiveState, Session
-
-
-class _MemoryVikingFS:
- def __init__(self, files):
- self.files = files
-
- def _uri_to_path(self, uri, ctx=None):
- return "/local/session-1"
-
- async def read_file(self, uri, ctx=None):
- if uri not in self.files:
- raise FileNotFoundError(uri)
- value = self.files[uri]
- if isinstance(value, Exception):
- raise value
- return value
-
- async def write_file(self, uri, content, ctx=None):
- self.files[uri] = content
-
-
-def _session(files, states, monkeypatch):
- session_uri = "viking://user/sessions/session-1"
- session = Session(
- viking_fs=_MemoryVikingFS(files),
- session_id="session-1",
- session_uri=session_uri,
- )
- monkeypatch.setattr(session, "_scan_archive_states", AsyncMock(return_value=states))
- return session, session_uri
-
-
-def _failed_then_pending(session_uri):
- """archive_001 = terminally failed (missing messages), archive_002 = current pending."""
- a1 = f"{session_uri}/history/archive_001"
- a2 = f"{session_uri}/history/archive_002"
- states = [
- ArchiveState(
- archive_id="archive_001",
- archive_uri=a1,
- index=1,
- state="failed",
- failed={"stage": "archive_read", "error": "no messages"},
- ),
- ArchiveState(
- archive_id="archive_002",
- archive_uri=a2,
- index=2,
- state="pending",
- ),
- ]
- return a1, a2, states
-
-
-async def test_phase2_replay_skips_failed_archive_with_missing_messages(monkeypatch):
- session_uri = "viking://user/sessions/session-1"
- a1, a2, states = _failed_then_pending(session_uri)
- # archive_001/messages.jsonl is absent -> not-found; archive_002 has no
- # replay reads of its own (current messages are passed in directly).
- session, _ = _session({}, states, monkeypatch)
-
- from openviking.message import Message, TextPart
-
- current = [Message(id="cur", role="user", parts=[TextPart("hi")])]
-
- combined, _start, _end, covered_failed, _steps = await session._prepare_phase2_archive_messages(
- a2, current
- )
-
- # The poisoned archive is skipped; the current commit still gets its own
- # messages and can complete Phase 2 instead of being terminal-failed.
- assert [m.id for m in combined] == ["cur"]
- # The skipped archive is still reported as covered_failed so the current
- # archive's .done marks it covered — clearing the poison permanently
- # instead of re-reading (and skipping) it on every future commit.
- assert covered_failed == ["archive_001"]
-
- # Asymmetry check: the read path already tolerates the exact same scenario.
- uncovered = await session._get_uncovered_archive_messages(states)
- assert uncovered == []
-
-
-async def test_phase2_replay_reraises_real_storage_failure(monkeypatch):
- session_uri = "viking://user/sessions/session-1"
- a1, a2, states = _failed_then_pending(session_uri)
- # A real storage failure (not a not-found) reading archive_001 must still
- # propagate so it is not silently swallowed.
- boom = RuntimeError("storage backend unavailable")
- session, _ = _session({f"{a1}/messages.jsonl": boom}, states, monkeypatch)
-
- from openviking.message import Message, TextPart
-
- current = [Message(id="cur", role="user", parts=[TextPart("hi")])]
-
- with pytest.raises(RuntimeError, match="storage backend unavailable"):
- await session._prepare_phase2_archive_messages(a2, current)