Skip to content

Self-improving AI Employees for everyone

fix(logging): a failed workflow task logs at ERROR (1b.29)

trohitg5635ac6c43c0
diff --git a/docs-internal/TEMPORAL_ARCHITECTURE.md b/docs-internal/TEMPORAL_ARCHITECTURE.md
index d0b2092f..e7f3c770 100644
--- a/docs-internal/TEMPORAL_ARCHITECTURE.md
+++ b/docs-internal/TEMPORAL_ARCHITECTURE.md
@@ -78,7 +78,10 @@ This invokes `run_standalone_worker()` from `services/temporal/worker.py`. It al
(`services/chat/relay.py`): the worker has no sockets of its own, so the chat run events and the `chat.updated` /
`approval_lifecycle` broadcasts its activities make go to the backend over `/ws/internal`
([Chat Protocol → Standalone workers](./chat_protocol.md#standalone-workers)). Other broadcasts from its activities
-(node status, agent progress) still reach only its own process.
+(node status, agent progress) still reach only its own process. Run as a module, it configures logging like the
+backend (`configure_logging`), including the filters on the SDK's loggers: a failed workflow task, which the SDK logs
+at WARNING, is raised to ERROR (`core/logging.WorkflowTaskFailureFilter`), because it fails again on every retry
+until the code changes.
## System Architecture
diff --git a/server/core/logging.py b/server/core/logging.py
index 17bd810a..84487156 100644
--- a/server/core/logging.py
+++ b/server/core/logging.py
@@ -285,6 +285,21 @@ class ExpectedLLMRetryFilter(logging.Filter):
return True
+class WorkflowTaskFailureFilter(logging.Filter):
+ """Log a failed workflow task at ERROR.
+
+ The SDK logs it at WARNING ("Failed activation on workflow ..."), yet a
+ workflow task that fails fails again on every retry until the code
+ changes: the workflow is stuck, not slowed down.
+ """
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ if getattr(record, "__temporal_error_identifier", None) == "WorkflowTaskFailure":
+ record.levelno = logging.ERROR
+ record.levelname = logging.getLevelName(logging.ERROR)
+ return True
+
+
def configure_logging(settings: Settings) -> None:
"""Configure structured logging based on settings.
@@ -335,6 +350,9 @@ def configure_logging(settings: Settings) -> None:
activity_logger = logging.getLogger("temporalio.activity")
if not any(isinstance(item, ExpectedLLMRetryFilter) for item in activity_logger.filters):
activity_logger.addFilter(ExpectedLLMRetryFilter())
+ workflow_logger = logging.getLogger("temporalio.worker._workflow_instance")
+ if not any(isinstance(item, WorkflowTaskFailureFilter) for item in workflow_logger.filters):
+ workflow_logger.addFilter(WorkflowTaskFailureFilter())
# Configure structlog. ``merge_contextvars`` pulls fields bound via
# ``structlog.contextvars.bind_contextvars`` (or the :func:`log_context`
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index ac2ec4f8..a6828f0d 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -1020,4 +1020,10 @@ async def create_worker(
if __name__ == "__main__":
# Allow running worker standalone
# Usage: python -m services.temporal.worker
+ from core.config import Settings
+ from core.logging import configure_logging
+
+ # The embedded worker logs through main.py's configuration; a standalone
+ # one needs its own (levels, and the filters on the SDK's loggers).
+ configure_logging(Settings())
asyncio.run(run_standalone_worker())
diff --git a/server/tests/core_config/test_llm_retry_logging.py b/server/tests/core_config/test_llm_retry_logging.py
index c8981456..e589d38b 100644
--- a/server/tests/core_config/test_llm_retry_logging.py
+++ b/server/tests/core_config/test_llm_retry_logging.py
@@ -55,6 +55,7 @@ def test_configured_sdk_logger_installs_filter_once_and_applies_it(monkeypatch):
activity_logger = logging.getLogger("temporalio.activity")
monkeypatch.setattr(activity_logger, "filters", [])
+ monkeypatch.setattr(logging.getLogger("temporalio.worker._workflow_instance"), "filters", [])
monkeypatch.setattr(logging_module.logging, "basicConfig", Mock())
monkeypatch.setattr(logging_module.structlog, "configure", Mock())
for name in ("httpx", "httpcore"):
diff --git a/server/tests/core_config/test_workflow_task_logging.py b/server/tests/core_config/test_workflow_task_logging.py
new file mode 100644
index 00000000..9ad4fcb7
--- /dev/null
+++ b/server/tests/core_config/test_workflow_task_logging.py
@@ -0,0 +1,50 @@
+"""A failed workflow task is logged at ERROR (1b.29): the workflow is stuck."""
+
+import logging
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+from core.logging import WorkflowTaskFailureFilter
+
+_SDK_LOGGER = "temporalio.worker._workflow_instance"
+
+
+def record(identifier=None):
+ entry = logging.LogRecord(_SDK_LOGGER, logging.WARNING, __file__, 1, "Failed activation on workflow", (), None)
+ if identifier:
+ # The SDK passes it through ``extra``; the attribute name is literal.
+ setattr(entry, "__temporal_error_identifier", identifier)
+ return entry
+
+
+def test_a_failed_workflow_task_is_raised_to_error():
+ entry = record("WorkflowTaskFailure")
+ assert WorkflowTaskFailureFilter().filter(entry)
+ assert entry.levelno == logging.ERROR
+ assert entry.levelname == "ERROR"
+
+
+def test_other_records_keep_their_level():
+ for entry in (record(), record("ActivityFailure")):
+ assert WorkflowTaskFailureFilter().filter(entry)
+ assert entry.levelno == logging.WARNING
+ assert entry.levelname == "WARNING"
+
+
+def test_configured_sdk_logger_installs_the_filter_once(monkeypatch):
+ import core.logging as logging_module
+
+ workflow_logger = logging.getLogger(_SDK_LOGGER)
+ monkeypatch.setattr(workflow_logger, "filters", [])
+ monkeypatch.setattr(logging.getLogger("temporalio.activity"), "filters", [])
+ monkeypatch.setattr(logging_module.logging, "basicConfig", Mock())
+ monkeypatch.setattr(logging_module.structlog, "configure", Mock())
+ for name in ("httpx", "httpcore"):
+ monkeypatch.setattr(logging.getLogger(name), "level", logging.NOTSET)
+ settings = SimpleNamespace(log_level="INFO", log_file=None, log_format="json")
+ logging_module.configure_logging(settings)
+ logging_module.configure_logging(settings)
+ assert len(workflow_logger.filters) == 1
+ entry = record("WorkflowTaskFailure")
+ assert workflow_logger.filter(entry)
+ assert entry.levelno == logging.ERROR