Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(channels): preserve recovery order and adapter ownership
Xubin Ren182ada682581
diff --git a/docs/automations.md b/docs/automations.mdindex f53c375da..c5f3ca980 100644--- a/docs/automations.md+++ b/docs/automations.md@@ -55,6 +55,8 @@ The gateway saves final scheduled messages before it accepts them into the outgoThis includes explicit `message` calls in existing tasks. Each recipient has a separate deliveryrecord. Progress, reasoning, and streaming events do not enter this queue. WebUI streaming andsaved chat history keep their existing behavior.+External channels receive one final scheduled reply instead of a streamed reply. Ordinary chat+streaming is unchanged.If a channel reports that it has not sent any part of a message, the gateway retries every30 seconds, for up to one hour after queue acceptance. It sends the saved message to the saved@@ -78,7 +80,7 @@ Look for `Channel delivery` in the gateway log. The run ID links execution to ea| `unknown` | An interrupted or failed attempt might have reached the service; no automatic retry |The queue is stored in `delivery/outbox.sqlite3`, beside the active config file. Include this-directory in instance backups. It holds at most 256 pending messages, with a 1 MiB serialized limit+directory in instance backups; stop the gateway before copying it. It holds at most 256 pending messages, with a 1 MiB serialized limitper message. A full queue or storage error rejects the enqueue operation instead of reporting itas accepted. Terminal records discard message text and keep status for up to seven days, cappedat 1,000 records. Existing job and session files are not converted. Older versions ignore thisdiff --git a/nanobot/channels/linear/runtime.py b/nanobot/channels/linear/runtime.pyindex 3929d2904..e3ab0b6f5 100644--- a/nanobot/channels/linear/runtime.py+++ b/nanobot/channels/linear/runtime.py@@ -13,6 +13,7 @@ from pathlib import Pathfrom typing import Any, castfrom urllib.parse import urlparse+from nanobot.bus.delivery import ChannelNotReadyErrorfrom nanobot.bus.events import InboundMessage, OutboundMessagefrom nanobot.bus.outbound_events import ContextCompactionEvent, ProgressEvent, RetryWaitEventfrom nanobot.bus.queue import MessageBus@@ -167,6 +168,8 @@ class LinearChannel(BaseChannel):ephemeral=True,)return+ if self._client is None:+ raise ChannelNotReadyError("Linear HTTP client is not initialized")body = msg.contentif msg.media:attachments = await self._attachment_markdown(msg)diff --git a/nanobot/channels/manager.py b/nanobot/channels/manager.pyindex f994368e0..1f21b7d5b 100644--- a/nanobot/channels/manager.py+++ b/nanobot/channels/manager.py@@ -37,6 +37,8 @@ from nanobot.channels.contracts import (from nanobot.channels.outbox import RETRY_SECONDS, DeliveryOutboxfrom nanobot.channels.registry import channel_default_enabledfrom nanobot.config.schema import Config+from nanobot.security.workspace_access import WorkspaceScopeResolver+from nanobot.security.workspace_policy import resolve_allowed_pathfrom nanobot.utils.restart import (RestartNotice,consume_restart_notice_from_env,@@ -799,6 +801,24 @@ class ChannelManager:if session is None or session.generation != request.session_generation:self._outbox.finish(msg, "failed", "session_replaced")return+ if msg.media:+ # An accepted local path can become a symlink during an outage.+ # Recheck the current source policy before the adapter opens it.+ scope = WorkspaceScopeResolver(+ self.config.workspace_path, self.config.tools.restrict_to_workspace,+ ).for_turn(channel=request.session_key.split(":", 1)[0],+ message_metadata={}, session_metadata=session.metadata)+ try:+ for path in msg.media:+ if not path.startswith(("http://", "https://")):+ resolve_allowed_path(+ path, workspace=scope.project_path,+ allowed_root=scope.project_path if scope.restrict_to_workspace else None,+ extra_allowed_roots=[self._config_path.parent / "media"],+ )+ except (OSError, ValueError):+ self._outbox.finish(msg, "failed", "attachment_access_denied")+ returnif not self._outbox.claim(msg):returntry:diff --git a/nanobot/channels/napcat/runtime.py b/nanobot/channels/napcat/runtime.pyindex f2db7621d..515d9d870 100644--- a/nanobot/channels/napcat/runtime.py+++ b/nanobot/channels/napcat/runtime.py@@ -19,7 +19,7 @@ from pydantic import Fieldfrom websockets.asyncio.client import ClientConnectionfrom websockets.asyncio.client import connect as ws_connect-from nanobot.bus.delivery import ChannelNotReadyError+from nanobot.bus.delivery import ChannelNotReadyError, DeliveryRejectedErrorfrom nanobot.bus.events import OutboundMessagefrom nanobot.bus.queue import MessageBusfrom nanobot.channels.base import BaseChannel@@ -448,8 +448,7 @@ class NapcatChannel(BaseChannel):kind, _, target = msg.chat_id.partition(":")if kind not in ("private", "group") or not target:- logger.error("napcat: invalid chat_id '{}'", msg.chat_id)- return+ raise DeliveryRejectedError("napcat: invalid chat_id")segments: list[dict[str, Any]] = []for ref in msg.media or []:diff --git a/nanobot/channels/outbox.py b/nanobot/channels/outbox.pyindex 0d610c001..2a6b0aef5 100644--- a/nanobot/channels/outbox.py+++ b/nanobot/channels/outbox.py@@ -39,18 +39,24 @@ class DeliveryOutbox:fd = os.open(self.path, os.O_CREAT | os.O_RDWR, 0o600)os.close(fd)db = sqlite3.connect(self.path)+ db.execute("PRAGMA secure_delete=ON")db.row_factory = sqlite3.Rowtry:with db:if not self._initialized:db.execute("""CREATE TABLE IF NOT EXISTS deliveries (id TEXT PRIMARY KEY, run_id TEXT NOT NULL, channel TEXT NOT NULL,+ chat_id TEXT NOT NULL,payload TEXT, status TEXT NOT NULL, created REAL NOT NULL,updated REAL NOT NULL, next_attempt REAL NOT NULL, error TEXT)""")# A crash during send cannot tell us whether the service accepted it.+ interrupted = db.execute("SELECT id, run_id, channel FROM deliveries WHERE status='sending'").fetchall()db.execute("""UPDATE deliveries SET status='unknown', payload=NULL,error='interrupted_send', updated=? WHERE status='sending'""", (time.time(),))+ for row in interrupted:+ logger.warning("Channel delivery: run={} delivery={} channel={} status=unknown error=interrupted_send",+ row["run_id"], row["id"], row["channel"])yield dbself._initialized = Truefinally:@@ -79,16 +85,18 @@ class DeliveryOutbox:).fetchone()[0]if count >= MAX_PENDING:raise RuntimeError("Scheduled delivery queue is full; check channel connectivity")- db.execute("INSERT INTO deliveries VALUES (?, ?, ?, ?, 'pending', ?, ?, ?, NULL)",- (delivery_id, msg.delivery.run_id, msg.channel, payload, now, now, now))+ db.execute("INSERT INTO deliveries VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, ?, NULL)",+ (delivery_id, msg.delivery.run_id, msg.channel, msg.chat_id, payload, now, now, now))msg.delivery_id = delivery_idself._log(msg, "pending")@staticmethoddef _prune(db: sqlite3.Connection, now: float) -> None:- db.execute("""UPDATE deliveries SET status='failed', payload=NULL,+ expired = db.execute("""UPDATE deliveries SET status='failed', payload=NULL,error='expired', updated=? WHERE status='pending' AND created<=?""",- (now, now - MAX_AGE_SECONDS))+ (now, now - MAX_AGE_SECONDS)).rowcount+ if expired:+ logger.warning("{} scheduled deliveries expired; check the instance delivery store", expired)db.execute("DELETE FROM deliveries WHERE status NOT IN ('pending','sending') AND updated<?",(now - RETENTION_SECONDS,))db.execute("""DELETE FROM deliveries WHERE id IN (@@ -116,17 +124,23 @@ class DeliveryOutbox:with self._connect() as db:self._prune(db, now)changed = db.execute("""UPDATE deliveries SET status='sending', updated=?- WHERE id=? AND status='pending' AND next_attempt<=?""",+ WHERE id=? AND status='pending' AND next_attempt<=?+ AND NOT EXISTS (SELECT 1 FROM deliveries AS older+ WHERE older.channel=deliveries.channel AND older.chat_id=deliveries.chat_id+ AND older.rowid<deliveries.rowid AND older.status IN ('pending','sending'))""",(now, msg.delivery_id, now)).rowcountreturn changed == 1def finish(self, msg: OutboundMessage, status: DeliveryStatus, error: str | None = None) -> None:now = time.time()with self._connect() as db:- db.execute("""UPDATE deliveries SET status=?, error=?, updated=?, next_attempt=?,- payload=CASE WHEN ?='pending' THEN payload ELSE NULL END WHERE id=?""",- (status, error, now, now + RETRY_SECONDS, status, msg.delivery_id))- self._log(msg, status, error)+ changed = db.execute("""UPDATE deliveries SET status=?, error=?, updated=?, next_attempt=?,+ payload=CASE WHEN ?='pending' THEN payload ELSE NULL END+ WHERE id=? AND status IN ('pending','sending')""",+ (status, error, now, now + RETRY_SECONDS, status, msg.delivery_id)).rowcount+ self._prune(db, now)+ if changed:+ self._log(msg, status, error)@staticmethoddef _log(msg: OutboundMessage, status: DeliveryStatus, error: str | None = None) -> None:diff --git a/nanobot/channels/signal/runtime.py b/nanobot/channels/signal/runtime.pyindex c05d51ee9..fd9243e05 100644--- a/nanobot/channels/signal/runtime.py+++ b/nanobot/channels/signal/runtime.py@@ -17,6 +17,7 @@ from typing import Any, TypedDict, castimport httpxfrom pydantic import Field, computed_field, field_validator+from nanobot.bus.delivery import ChannelNotReadyErrorfrom nanobot.bus.events import InboundMessage, OutboundMessagefrom nanobot.bus.outbound_events import ProgressEventfrom nanobot.bus.queue import MessageBus@@ -571,6 +572,8 @@ class SignalChannel(BaseChannel):msg.event.notify or self.show_compaction_notices):return+ if not self._http:+ raise ChannelNotReadyError("Signal client not connected")is_progress_message = isinstance(msg.event, ProgressEvent)try:plain_text, text_styles = _markdown_to_signal(msg.content)diff --git a/nanobot/channels/telegram/runtime.py b/nanobot/channels/telegram/runtime.pyindex 64d15069e..de82cfb13 100644--- a/nanobot/channels/telegram/runtime.py+++ b/nanobot/channels/telegram/runtime.py@@ -802,7 +802,7 @@ class TelegramChannel(BaseChannel):with suppress(asyncio.TimeoutError):await asyncio.wait_for(self._app_ready.wait(), APP_RESTART_SEND_WAIT_SECONDS)if not self._app_ready.is_set() or self._app is None:- raise RuntimeError("Telegram application is restarting; message not delivered")+ raise ChannelNotReadyError("Telegram application is restarting; message not delivered")return self._appdef _note_poll_ok(self) -> None:diff --git a/nanobot/channels/telegram/tests/test_telegram_channel.py b/nanobot/channels/telegram/tests/test_telegram_channel.pyindex 6d99cba27..13b17d8c2 100644--- a/nanobot/channels/telegram/tests/test_telegram_channel.py+++ b/nanobot/channels/telegram/tests/test_telegram_channel.py@@ -12,6 +12,7 @@ try:except ImportError:pytest.skip("Telegram dependencies not installed (python-telegram-bot)", allow_module_level=True)+from nanobot.bus.delivery import ChannelNotReadyErrorfrom nanobot.bus.events import OutboundMessagefrom nanobot.bus.outbound_events import ProgressEventfrom nanobot.bus.queue import MessageBus@@ -621,14 +622,15 @@ async def test_send_during_rebuild_fails_instead_of_dropping(monkeypatch) -> Nonchannel._app = Nonemsg = OutboundMessage(channel="telegram", chat_id="123", content="hello")- with pytest.raises(RuntimeError, match="restarting"):+ with pytest.raises(ChannelNotReadyError, match="restarting"):await channel.send(msg)with pytest.raises(RuntimeError, match="restarting"):await channel.send_delta("123", "hello", stream_id="s1")- # Stopped: nothing to deliver, so stay quiet.+ # A stopped channel must not report a final message as delivered.channel._running = False- await channel.send(msg)+ with pytest.raises(ChannelNotReadyError, match="not running"):+ await channel.send(msg)await channel.send_delta("123", "hello", stream_id="s1")diff --git a/nanobot/channels/weixin/runtime.py b/nanobot/channels/weixin/runtime.pyindex 7998ac312..35d09f593 100644--- a/nanobot/channels/weixin/runtime.py+++ b/nanobot/channels/weixin/runtime.py@@ -2082,7 +2082,13 @@ class WeixinChannel(BaseChannel):)delivery_state.completed_parts.add(text_part)completed = True- except WeixinQuotaError:+ except WeixinQuotaError as error:+ if msg.delivery_id is not None:+ # The durable outbox owns scheduled results. Do not also retain+ # one in the channel's independently replayed, volatile queue.+ if not delivery_state.completed_parts:+ raise ChannelNotReadyError("WeChat needs a fresh inbound context") from error+ raiseif not is_progress:self._defer_outbound(msg)self.logger.warning(diff --git a/nanobot/channels/weixin/tests/test_weixin_channel.py b/nanobot/channels/weixin/tests/test_weixin_channel.pyindex 2acd14b91..85f26a815 100644--- a/nanobot/channels/weixin/tests/test_weixin_channel.py+++ b/nanobot/channels/weixin/tests/test_weixin_channel.py@@ -9,6 +9,7 @@ from unittest.mock import AsyncMockimport httpximport pytest+from nanobot.bus.delivery import ChannelNotReadyErrorfrom nanobot.bus.outbound_events import ProgressEventfrom nanobot.bus.queue import MessageBusfrom nanobot.channels.weixin import runtime as weixin_mod@@ -546,9 +547,7 @@ async def test_send_without_context_token_raises() -> None:channel._send_text = AsyncMock()with pytest.raises(RuntimeError, match="context_token missing"):- await channel.send(- type("Msg", (), {"chat_id": "unknown-user", "content": "pong", "media": [], "metadata": {}})()- )+ await channel.send(_make_outbound_msg(chat_id="unknown-user", content="pong"))channel._send_text.assert_not_awaited()@@ -1482,6 +1481,26 @@ def _make_outbound_msg(chat_id: str = "wx-user", content: str = "", media: list)[email protected]("partial", [False, True])+async def test_scheduled_quota_failure_has_only_one_recovery_owner(partial):+ channel, _bus = _make_channel()+ channel._client = object()+ channel._token = "token"+ channel._refresh_context_token_if_stale = AsyncMock(return_value="context")+ channel._get_typing_ticket = AsyncMock(return_value="")+ channel._send_media_part = AsyncMock()+ channel._send_text = AsyncMock(side_effect=weixin_mod.WeixinQuotaError("sendmessage"))+ msg = _make_outbound_msg(content="result", media=["image.png"] if partial else [])+ msg.delivery_id = "scheduled-result"++ expected = weixin_mod.WeixinQuotaError if partial else ChannelNotReadyError+ with pytest.raises(expected):+ await channel.send(msg)++ assert channel._deferred_outbound == {}+ assert channel._send_media_part.await_count == int(partial)++@pytest.mark.asyncioasync def test_send_media_timeout_error_propagates_without_text_fallback() -> None:"""httpx.TimeoutException during media send must re-raise immediately,diff --git a/nanobot/cron/bound_runner.py b/nanobot/cron/bound_runner.pyindex a5339b989..3f10053cf 100644--- a/nanobot/cron/bound_runner.py+++ b/nanobot/cron/bound_runner.py@@ -53,6 +53,10 @@ def _bound_session_delivery_context(source_label=source_label,))+ else:+ # External scheduled results need one recoverable final message, not a+ # transient stream whose completion marker cannot be replayed safely.+ metadata.pop("_wants_stream", None)return channel, chat_id, metadatadiff --git a/tests/channels/test_scheduled_delivery.py b/tests/channels/test_scheduled_delivery.pyindex 0c922355f..cc6bd50ff 100644--- a/tests/channels/test_scheduled_delivery.py+++ b/tests/channels/test_scheduled_delivery.py@@ -2,6 +2,7 @@import asyncioimport json+import osimport sqlite3from functools import partialfrom unittest.mock import AsyncMock, MagicMock@@ -11,7 +12,12 @@ import pytestfrom nanobot.agent.loop import AgentLoopfrom nanobot.agent.tools.context import RequestContext, request_contextfrom nanobot.agent.tools.message import MessageTool-from nanobot.bus.delivery import DELIVERY_RUN_META, ChannelNotReadyError, DeliveryRequest+from nanobot.bus.delivery import (+ DELIVERY_RUN_META,+ ChannelNotReadyError,+ DeliveryRejectedError,+ DeliveryRequest,+)from nanobot.bus.events import OutboundMessagefrom nanobot.bus.outbound_events import ProgressEventfrom nanobot.bus.queue import MessageBus@@ -19,6 +25,7 @@ from nanobot.channels import outbox as outbox_modulefrom nanobot.channels.base import BaseChannelfrom nanobot.channels.manager import ChannelManagerfrom nanobot.channels.outbox import DeliveryOutbox+from nanobot.channels.registry import load_channel_pluginfrom nanobot.channels.whatsapp.runtime import WhatsAppChannelfrom nanobot.config.schema import Configfrom nanobot.cron.bound_runner import run_bound_cron_job@@ -54,8 +61,8 @@ async def drain(manager):await asyncio.sleep(0.01)[email protected]("legacy_message", [False, True])-async def test_cron_offline_restart_and_reconnect_do_not_rerun_model(tmp_path, monkeypatch, legacy_message):[email protected]("legacy_message,one_shot", [(False, False), (True, False), (True, True)])+async def test_cron_offline_restart_and_reconnect_do_not_rerun_model(tmp_path, monkeypatch, legacy_message, one_shot):monkeypatch.setattr(outbox_module, "RETRY_SECONDS", 0)workspace = tmp_path / "workspace"workspace.mkdir()@@ -93,13 +100,18 @@ async def test_cron_offline_restart_and_reconnect_do_not_rerun_model(tmp_path, mmanager.channels["whatsapp"] = channelmanager._dispatch_task = asyncio.create_task(manager._dispatch_outbound())# Legacy channel/to fields still normalize through the existing binding migration.- job = cron.add_job("Reminder", CronSchedule(kind="every", every_ms=86400000),+ schedule = CronSchedule(kind="at", at_ms=4102444800000) if one_shot else CronSchedule(kind="every", every_ms=86400000)+ job = cron.add_job("Reminder", schedule,"Use message to send Diagnostic result" if legacy_message else "Return Diagnostic result",- deliver=True, channel="whatsapp", to="synthetic")+ deliver=True, channel="whatsapp", to="synthetic", delete_after_run=one_shot,+ channel_meta={"_wants_stream": True})try:assert await asyncio.wait_for(cron.run_job(job.id), 15)await drain(manager)- assert cron.list_jobs()[0].state.last_status == "ok"+ if one_shot:+ assert cron.list_jobs() == []+ else:+ assert cron.list_jobs()[0].state.last_status == "ok"assert len(rows(manager._outbox)) == 1assert rows(manager._outbox)[0]["status"] == "pending"channel._client.send_message.assert_not_called()@@ -123,7 +135,8 @@ async def test_cron_offline_restart_and_reconnect_do_not_rerun_model(tmp_path, massert channel._client.send_message.await_count == 1assert len(calls) == count# Existing jobs still load after delivery state is added beside them.- assert CronService(cron.store_path).list_jobs()[0].payload.message == job.payload.message+ old_jobs = CronService(cron.store_path).list_jobs()+ assert old_jobs == [] if one_shot else old_jobs[0].payload.message == job.payload.messagefinally:await manager.stop_all()cron.stop()@@ -172,6 +185,7 @@ async def test_legacy_broadcast_has_independent_targets_and_no_foreign_thread_measync def test_queue_acceptance_survives_restart_but_inflight_is_unknown(tmp_path):manager = make_manager(tmp_path)pending, inflight = message(), message(content="second")+ inflight.chat_id = "other-destination"await manager.bus.publish_outbound(pending)await manager.bus.publish_outbound(inflight)assert manager._outbox.claim(inflight)@@ -198,7 +212,8 @@ async def test_queue_bounds_expiry_and_untracked_events(tmp_path, monkeypatch):assert manager._outbox.ready() == []assert rows(manager._outbox)[0]["error"] == "expired"assert not manager._outbox.claim(first)- assert manager._outbox.path.stat().st_mode & 0o777 == 0o600+ if os.name != "nt":+ assert manager._outbox.path.stat().st_mode & 0o777 == 0o600@pytest.mark.parametrize("operation", ["delete", "reset"])@@ -218,3 +233,78 @@ async def test_pending_result_cannot_send_after_source_session_is_replaced(tmp_passert rows(manager._outbox)[0]["error"] == "session_replaced"finally:await sessions.state.aclose()+++async def test_recovery_keeps_destination_order_and_does_not_block_other_chats(tmp_path, monkeypatch):+ monkeypatch.setattr(outbox_module, "RETRY_SECONDS", 0)+ manager = make_manager(tmp_path)+ first, second, other = message(content="first"), message(content="second"), message(content="other")+ other.chat_id = "another-chat"+ for msg in (first, second, other):+ await manager.bus.publish_outbound(msg)+ await manager.bus.consume_outbound()+ channel = MagicMock(spec=BaseChannel, send=AsyncMock(side_effect=ChannelNotReadyError()))+ await manager._send_scheduled(channel, first)+ channel.send.side_effect = None+ await manager._send_scheduled(channel, second)+ await manager._send_scheduled(channel, other)+ assert [call.args[0].content for call in channel.send.await_args_list] == ["first", "other"]+ manager.channels["whatsapp"] = channel+ await manager._recover_scheduled()+ await drain(manager)+ assert [call.args[0].content for call in channel.send.await_args_list] == ["first", "other", "first", "second"]+ # A stale queued copy must not downgrade the durable successful result.+ manager._outbox.finish(first, "failed", "channel_disabled")+ assert all(row["status"] == "sent" for row in rows(manager._outbox))+++async def test_relocated_instance_finds_queue_and_disabled_channel_does_not_send(tmp_path):+ original = tmp_path / "original"+ manager = make_manager(original)+ await manager.bus.publish_outbound(message())+ moved = tmp_path / "moved"+ original.rename(moved)+ restarted = make_manager(moved)+ await restarted._recover_scheduled()+ assert rows(restarted._outbox)[0]["error"] == "channel_disabled"+ assert not original.exists()+++async def test_recovery_rechecks_attachment_workspace_boundary(tmp_path):+ workspace = tmp_path / "workspace"+ workspace.mkdir()+ sessions = SessionManager(workspace, sessions_root=tmp_path / "sessions")+ session = await sessions.state.get("whatsapp:synthetic")+ manager = make_manager(tmp_path, sessions)+ manager.config.agents.defaults.workspace = str(workspace)+ manager.config.tools.restrict_to_workspace = True+ attachment = workspace / "report.txt"+ attachment.write_text("report")+ msg = message(generation=session.generation)+ msg.media = [str(attachment)]+ channel = MagicMock(spec=BaseChannel, send=AsyncMock())+ try:+ await manager.bus.publish_outbound(msg)+ attachment.unlink()+ private = tmp_path / "private.txt"+ private.write_text("not authorized")+ attachment.symlink_to(private)+ await manager._send_scheduled(channel, msg)+ channel.send.assert_not_called()+ assert rows(manager._outbox)[0]["error"] == "attachment_access_denied"+ finally:+ await sessions.state.aclose()++[email protected]("name", [+ "whatsapp", "telegram", "discord", "slack", "matrix", "mattermost", "feishu",+ "qq", "wecom", "weixin", "napcat", "msteams", "linear", "signal", "dingtalk",+ "mochat", "email",+])+async def test_unavailable_adapter_does_not_report_success(name, monkeypatch):+ cls = load_channel_plugin(name).load_channel_class()+ channel = cls(cls.default_config(), MessageBus())+ if name == "dingtalk":+ monkeypatch.setattr(channel, "_get_access_token", AsyncMock(return_value=None))+ with pytest.raises((ChannelNotReadyError, DeliveryRejectedError)):+ await channel.send(OutboundMessage(name, "synthetic", "result"))