Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(telegram): stop reporting a self-healing poll conflict as an error
trohitg484295517630
diff --git a/server/nodes/telegram/_service.py b/server/nodes/telegram/_service.pyindex dc7eb72d..2a94962d 100644--- a/server/nodes/telegram/_service.py+++ b/server/nodes/telegram/_service.py@@ -42,11 +42,12 @@ import asyncioimport loggingimport mimetypesimport os+import timefrom datetime import datetimefrom typing import Any, Awaitable, Callable, Dict, Optionalfrom telegram import Bot, Update-from telegram.error import BadRequest, NetworkError+from telegram.error import BadRequest, Conflict, NetworkError, RetryAfterfrom telegram.ext import Application, ContextTypes, MessageHandler, filtersfrom telegram.helpers import escape_markdown@@ -68,6 +69,11 @@ logger = logging.getLogger(__name__)_TG_TEXT_LIMIT = 4096_TG_CAPTION_LIMIT = 1024+# How often a recurring, self-healing polling condition may re-log. Telegram+# re-emits Conflict on every getUpdates retry (~6s) for as long as another+# consumer holds the token, which is unactionable after the first line.+_POLLING_WARN_INTERVAL_S = 60.0+def _tg_len(text: str) -> int:"""Length as Telegram counts it: UTF-16 code units, not code points.@@ -372,6 +378,9 @@ class TelegramService(ServiceSingleton):self._polling_task: Optional[asyncio.Task] = Noneself._bot_info: Dict[str, Any] = {}self._owner_chat_id: Optional[int] = None+ # Throttle state for recurring polling conditions (see _log_throttled).+ self._polling_error_counts: Dict[str, int] = {}+ self._polling_error_last_log: Dict[str, float] = {}@classmethodasync def reset_instance(cls): # type: ignore[override]@@ -596,9 +605,53 @@ class TelegramService(ServiceSingleton):# Polling loop# =========================================================================+ def _log_throttled(self, key: str, message: str) -> None:+ """Log ``message`` at WARNING at most once per interval per ``key``.++ Suppressed repeats are counted and reported on the next emission, so+ a condition that recurs on every poll retry is visible without+ flooding the console.+ """+ now = time.monotonic()+ last = self._polling_error_last_log.get(key)+ if last is not None and (now - last) < _POLLING_WARN_INTERVAL_S:+ self._polling_error_counts[key] = self._polling_error_counts.get(key, 0) + 1+ return+ suppressed = self._polling_error_counts.pop(key, 0)+ if suppressed and last is not None:+ message = f"{message} (repeated {suppressed}x in the last {int(now - last)}s)"+ self._polling_error_last_log[key] = now+ logger.warning(message)+def _on_polling_error(self, error) -> None:+ """python-telegram-bot ``error_callback`` for the getUpdates loop.++ Everything reaching here is a *polling attempt* failure that PTB+ retries on its own — the bot stays connected and the loop keeps+ running. Only genuinely unexpected errors deserve ERROR; the two+ common transients would otherwise repeat every few seconds forever+ and read as fatal when nothing is broken.+ """if isinstance(error, NetworkError):logger.debug(f"[Telegram] Network error during polling (auto-retrying): {error}")+ elif isinstance(error, Conflict):+ # Telegram allows exactly one getUpdates consumer per token.+ # Emitted on every retry (~6s) for as long as the other consumer+ # holds the slot, so it is throttled: the first line is the+ # actionable one, the rest are noise.+ self._log_throttled(+ "conflict",+ "[Telegram] Another getUpdates consumer holds this bot token, so "+ "inbound messages are paused. Polling keeps retrying and recovers "+ "on its own once the other consumer stops. The bot stays "+ "connected and sending still works. Usual causes: a second dev "+ "server or deployed instance using the same token, or the previous "+ "run's long poll still draining (up to 30s after a restart).",+ )+ elif isinstance(error, RetryAfter):+ # Flow control, not a failure: PTB sleeps for the requested+ # window and resumes.+ logger.debug(f"[Telegram] Rate limited during polling (auto-retrying): {error}")else:logger.error(f"[Telegram] Polling error: {error}")@@ -617,7 +670,10 @@ class TelegramService(ServiceSingleton):logger.info("[Telegram] Polling cancelled")raiseexcept Exception as e:- logger.error(f"[Telegram] Polling error: {e}")+ # Distinct wording from _on_polling_error on purpose: reaching+ # here means the loop itself died and the bot is now marked+ # disconnected, which the retryable per-attempt failures are not.+ logger.error(f"[Telegram] Polling loop stopped, bot disconnected: {e}")self._connected = Falseawait self._broadcast_status()@@ -947,7 +1003,7 @@ class TelegramService(ServiceSingleton):head, tail = _split_head(caption, _TG_CAPTION_LIMIT) if caption else ("", "")if tail:logger.info(- "[Telegram] Caption is %d units (cap %d) — remainder threaded as a follow-up",+ "[Telegram] Caption is %d units (cap %d); remainder threaded as a follow-up",_tg_len(caption or ""),_TG_CAPTION_LIMIT,)diff --git a/server/tests/nodes/test_telegram_service.py b/server/tests/nodes/test_telegram_service.pyindex f4b09020..f9c33633 100644--- a/server/tests/nodes/test_telegram_service.py+++ b/server/tests/nodes/test_telegram_service.py@@ -424,6 +424,83 @@ def test_detail_extraction_failure_does_not_drop_the_message():assert data.get("voice") is None+# ---------------------------------------------------------------------------+# polling error severity+# ---------------------------------------------------------------------------+++def test_conflict_is_a_throttled_warning_not_an_error(caplog):+ """A second getUpdates consumer is self-healing, so it must not read as+ fatal — and it must not re-log on every ~6s retry."""+ from telegram.error import Conflict++ service = TelegramService()+ with caplog.at_level("DEBUG", logger="nodes.telegram._service"):+ for _ in range(25):+ service._on_polling_error(Conflict("terminated by other getUpdates request"))++ conflict_records = [r for r in caplog.records if "getUpdates consumer" in r.message]+ assert len(conflict_records) == 1, "25 retries must collapse to one line"+ assert conflict_records[0].levelname == "WARNING"+ # The operator needs to know it recovers by itself and that sending works.+ assert "recovers" in conflict_records[0].message+ assert not [r for r in caplog.records if r.levelname == "ERROR"]+++def test_suppressed_conflicts_are_counted_on_the_next_emission(caplog, monkeypatch):+ from telegram.error import Conflict+ from nodes.telegram import _service as mod++ service = TelegramService()+ clock = {"t": 1000.0}+ monkeypatch.setattr(mod.time, "monotonic", lambda: clock["t"])++ with caplog.at_level("DEBUG", logger="nodes.telegram._service"):+ for _ in range(5):+ service._on_polling_error(Conflict("x"))+ clock["t"] += mod._POLLING_WARN_INTERVAL_S + 1+ service._on_polling_error(Conflict("x"))++ lines = [r.message for r in caplog.records if "getUpdates consumer" in r.message]+ assert len(lines) == 2+ assert "repeated 4x" in lines[1]+++def test_network_and_rate_limit_stay_debug(caplog):+ from telegram.error import NetworkError, RetryAfter++ service = TelegramService()+ with caplog.at_level("DEBUG", logger="nodes.telegram._service"):+ service._on_polling_error(NetworkError("connection reset"))+ service._on_polling_error(RetryAfter(30))++ assert not [r for r in caplog.records if r.levelname in ("WARNING", "ERROR")]+++def test_unexpected_polling_errors_still_log_at_error(caplog):+ service = TelegramService()+ with caplog.at_level("DEBUG", logger="nodes.telegram._service"):+ service._on_polling_error(RuntimeError("something genuinely broken"))++ errors = [r for r in caplog.records if r.levelname == "ERROR"]+ assert len(errors) == 1+ assert "something genuinely broken" in errors[0].message+++def test_transient_and_fatal_polling_messages_are_distinguishable():+ """Both paths used to emit the identical '[Telegram] Polling error:'+ string despite opposite consequences — one retries, the other marks the+ bot disconnected."""+ import inspect++ from nodes.telegram._service import TelegramService as Svc++ callback_src = inspect.getsource(Svc._on_polling_error)+ loop_src = inspect.getsource(Svc._run_polling)+ assert "bot disconnected" in loop_src+ assert "bot disconnected" not in callback_src++def test_media_content_types_covers_every_downloadable_kind():assert _MEDIA_CONTENT_TYPES == {"photo",