Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(logging): a failed workflow task logs at ERROR (1b.29)
trohitg5635ac6c43c0
diff --git a/docs-internal/TEMPORAL_ARCHITECTURE.md b/docs-internal/TEMPORAL_ARCHITECTURE.mdindex d0b2092f..e7f3c770 100644--- a/docs-internal/TEMPORAL_ARCHITECTURE.md+++ b/docs-internal/TEMPORAL_ARCHITECTURE.md@@ -78,7 +78,10 @@ This invokes `run_standalone_worker()` from `services/temporal/worker.py`. It al(`services/chat/relay.py`): the worker has no sockets of its own, so the chat run events and the `chat.updated` /`approval_lifecycle` broadcasts its activities make go to the backend over `/ws/internal`([Chat Protocol → Standalone workers](./chat_protocol.md#standalone-workers)). Other broadcasts from its activities-(node status, agent progress) still reach only its own process.+(node status, agent progress) still reach only its own process. Run as a module, it configures logging like the+backend (`configure_logging`), including the filters on the SDK's loggers: a failed workflow task, which the SDK logs+at WARNING, is raised to ERROR (`core/logging.WorkflowTaskFailureFilter`), because it fails again on every retry+until the code changes.## System Architecturediff --git a/server/core/logging.py b/server/core/logging.pyindex 17bd810a..84487156 100644--- a/server/core/logging.py+++ b/server/core/logging.py@@ -285,6 +285,21 @@ class ExpectedLLMRetryFilter(logging.Filter):return True+class WorkflowTaskFailureFilter(logging.Filter):+ """Log a failed workflow task at ERROR.++ The SDK logs it at WARNING ("Failed activation on workflow ..."), yet a+ workflow task that fails fails again on every retry until the code+ changes: the workflow is stuck, not slowed down.+ """++ def filter(self, record: logging.LogRecord) -> bool:+ if getattr(record, "__temporal_error_identifier", None) == "WorkflowTaskFailure":+ record.levelno = logging.ERROR+ record.levelname = logging.getLevelName(logging.ERROR)+ return True++def configure_logging(settings: Settings) -> None:"""Configure structured logging based on settings.@@ -335,6 +350,9 @@ def configure_logging(settings: Settings) -> None:activity_logger = logging.getLogger("temporalio.activity")if not any(isinstance(item, ExpectedLLMRetryFilter) for item in activity_logger.filters):activity_logger.addFilter(ExpectedLLMRetryFilter())+ workflow_logger = logging.getLogger("temporalio.worker._workflow_instance")+ if not any(isinstance(item, WorkflowTaskFailureFilter) for item in workflow_logger.filters):+ workflow_logger.addFilter(WorkflowTaskFailureFilter())# Configure structlog. ``merge_contextvars`` pulls fields bound via# ``structlog.contextvars.bind_contextvars`` (or the :func:`log_context`diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex ac2ec4f8..a6828f0d 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -1020,4 +1020,10 @@ async def create_worker(if __name__ == "__main__":# Allow running worker standalone# Usage: python -m services.temporal.worker+ from core.config import Settings+ from core.logging import configure_logging++ # The embedded worker logs through main.py's configuration; a standalone+ # one needs its own (levels, and the filters on the SDK's loggers).+ configure_logging(Settings())asyncio.run(run_standalone_worker())diff --git a/server/tests/core_config/test_llm_retry_logging.py b/server/tests/core_config/test_llm_retry_logging.pyindex c8981456..e589d38b 100644--- a/server/tests/core_config/test_llm_retry_logging.py+++ b/server/tests/core_config/test_llm_retry_logging.py@@ -55,6 +55,7 @@ def test_configured_sdk_logger_installs_filter_once_and_applies_it(monkeypatch):activity_logger = logging.getLogger("temporalio.activity")monkeypatch.setattr(activity_logger, "filters", [])+ monkeypatch.setattr(logging.getLogger("temporalio.worker._workflow_instance"), "filters", [])monkeypatch.setattr(logging_module.logging, "basicConfig", Mock())monkeypatch.setattr(logging_module.structlog, "configure", Mock())for name in ("httpx", "httpcore"):diff --git a/server/tests/core_config/test_workflow_task_logging.py b/server/tests/core_config/test_workflow_task_logging.pynew file mode 100644index 00000000..9ad4fcb7--- /dev/null+++ b/server/tests/core_config/test_workflow_task_logging.py@@ -0,0 +1,50 @@+"""A failed workflow task is logged at ERROR (1b.29): the workflow is stuck."""++import logging+from types import SimpleNamespace+from unittest.mock import Mock++from core.logging import WorkflowTaskFailureFilter++_SDK_LOGGER = "temporalio.worker._workflow_instance"+++def record(identifier=None):+ entry = logging.LogRecord(_SDK_LOGGER, logging.WARNING, __file__, 1, "Failed activation on workflow", (), None)+ if identifier:+ # The SDK passes it through ``extra``; the attribute name is literal.+ setattr(entry, "__temporal_error_identifier", identifier)+ return entry+++def test_a_failed_workflow_task_is_raised_to_error():+ entry = record("WorkflowTaskFailure")+ assert WorkflowTaskFailureFilter().filter(entry)+ assert entry.levelno == logging.ERROR+ assert entry.levelname == "ERROR"+++def test_other_records_keep_their_level():+ for entry in (record(), record("ActivityFailure")):+ assert WorkflowTaskFailureFilter().filter(entry)+ assert entry.levelno == logging.WARNING+ assert entry.levelname == "WARNING"+++def test_configured_sdk_logger_installs_the_filter_once(monkeypatch):+ import core.logging as logging_module++ workflow_logger = logging.getLogger(_SDK_LOGGER)+ monkeypatch.setattr(workflow_logger, "filters", [])+ monkeypatch.setattr(logging.getLogger("temporalio.activity"), "filters", [])+ monkeypatch.setattr(logging_module.logging, "basicConfig", Mock())+ monkeypatch.setattr(logging_module.structlog, "configure", Mock())+ for name in ("httpx", "httpcore"):+ monkeypatch.setattr(logging.getLogger(name), "level", logging.NOTSET)+ settings = SimpleNamespace(log_level="INFO", log_file=None, log_format="json")+ logging_module.configure_logging(settings)+ logging_module.configure_logging(settings)+ assert len(workflow_logger.filters) == 1+ entry = record("WorkflowTaskFailure")+ assert workflow_logger.filter(entry)+ assert entry.levelno == logging.ERROR