Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(observability): stop Temporal spans flooding the log at INFO
trohitgb625c6b3b2b0
diff --git a/server/core/config.py b/server/core/config.pyindex 3c6a4dc8..4550bbb5 100644--- a/server/core/config.py+++ b/server/core/config.py@@ -171,7 +171,7 @@ class Settings(BaseSettings):le=65535,)# Web UI port. ``temporal server start-dev`` defaults this to- # ``--port + 1000``; declared explicitly via ``--ui-port`` (5683 in+ # ``--port + 1000``; declared explicitly via ``--ui-port`` (5680 in# the serial 5678-block) so the binding is intentional and surfaces# in status snapshots.temporal_ui_port: int = Field(@@ -311,6 +311,17 @@ class Settings(BaseSettings):env="LOG_FILE_BACKUP_COUNT",ge=0,)+ # OpenTelemetry console span export. Off unless explicitly enabled or+ # LOG_LEVEL=DEBUG (see ``tracing_console_spans_enabled``): the Temporal+ # SDK TracingInterceptor emits a span per Signal / Query / Update /+ # Activity on both the caller and handler side, which floods the+ # operator log at INFO with per-RPC lines.+ tracing_console_spans: bool = Field(default=False, env="TRACING_CONSOLE_SPANS")++ @property+ def tracing_console_spans_enabled(self) -> bool:+ """Console spans are debug output; DEBUG turns them on implicitly."""+ return self.tracing_console_spans or self.log_level.upper() == "DEBUG"# Rate Limitingrate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")diff --git a/server/core/tracing.py b/server/core/tracing.pyindex 0a030f7a..fcbea676 100644--- a/server/core/tracing.py+++ b/server/core/tracing.py@@ -10,8 +10,17 @@ want to emit spans do::with tracer.start_as_current_span("namespace.operation"):...-In dev, ``ConsoleSpanExporter`` is wired with a compact one-line-``formatter`` so each completed span shows up as::+Span export to the console is **opt-in**. Spans are always recorded (so+an OTLP exporter can be attached without touching call sites), but the+``ConsoleSpanExporter`` is only wired when ``console_spans=True`` — set+by ``LOG_LEVEL=DEBUG`` or ``TRACING_CONSOLE_SPANS=true``. At INFO the+Temporal SDK's ``TracingInterceptor`` alone emits a span per+Signal/Query/Update/Activity on both the caller and handler side, which+buries the operator log in per-RPC lines that carry no operational+signal.++When enabled, a compact one-line ``formatter`` renders each completed+span as::[span] broadcaster.refresh_telegram 4012ms path=auto_reconnect connected=False@@ -58,9 +67,21 @@ def _compact_span_formatter(span: ReadableSpan) -> str:return " ".join(parts) + "\n"-def init_tracing(service_name: str = "opencompany-backend") -> None:+def init_tracing(+ service_name: str = "opencompany-backend",+ *,+ console_spans: bool = False,+) -> None:"""Configure the global TracerProvider.+ Args:+ service_name: ``service.name`` resource attribute.+ console_spans: Attach the console exporter. Off by default —+ span-per-RPC output is a debugging tool, not operator+ signal. The provider is still installed either way, so+ ``trace.get_tracer(__name__)`` call sites keep working and+ an OTLP exporter can be added without code changes.+Idempotent — safe to call multiple times. Subsequent calls areno-ops so tests / reloads do not accumulate processors."""@@ -69,8 +90,9 @@ def init_tracing(service_name: str = "opencompany-backend") -> None:returnprovider = TracerProvider(resource=Resource.create({SERVICE_NAME: service_name}))- # Compact one-line console output instead of the default 30-line- # JSON dump per span — keeps interactive logs readable.- provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter(formatter=_compact_span_formatter)))+ if console_spans:+ # Compact one-line console output instead of the default 30-line+ # JSON dump per span — keeps interactive logs readable.+ provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter(formatter=_compact_span_formatter)))trace.set_tracer_provider(provider)_INITIALIZED = Truediff --git a/server/main.py b/server/main.pyindex 28c7446f..8fb8f4f0 100644--- a/server/main.py+++ b/server/main.py@@ -52,7 +52,7 @@ from core.tracing import init_tracingsettings = Settings()configure_logging(settings)-init_tracing()+init_tracing(console_spans=settings.tracing_console_spans_enabled)logger = get_logger(__name__)_startup_log("Importing DI container + all services...")diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex ca294ff4..3b0e65a0 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -238,7 +238,11 @@ class TemporalWorkerManager:max_concurrent_workflow_tasks=10,graceful_shutdown_timeout=_graceful_shutdown_timeout(),identity=_worker_identity(self.task_queue),- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],# Wave 18.2: sticky cache sized by deployment mode so# cached workflows skip Event-History replay without# blowing laptop RAM.@@ -498,7 +502,11 @@ class TemporalWorkerPool:activities=activities,graceful_shutdown_timeout=_graceful_shutdown_timeout(),identity=_worker_identity(queue),- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],# Wave 18.1: per-queue activities/second ceiling.max_activities_per_second=rate_limit,# Wave 18.3: activity-only workers need a small@@ -545,7 +553,7 @@ class TemporalWorkerPool:async def run_standalone_worker(- server_address: str = "localhost:5682",+ server_address: str | None = None,namespace: str = "default",task_queue: str = "machina-tasks",pool_size: int = 100,@@ -560,11 +568,19 @@ async def run_standalone_worker(python -m services.temporal.workerArgs:- server_address: Temporal server address+ server_address: Temporal server address. Defaults to+ ``TEMPORAL_SERVER_ADDRESS`` (canonical value in ``.env.template``+ via ``core.env_defaults``) so a bare+ ``python -m services.temporal.worker`` dials the same server+ as the backend.namespace: Temporal namespacetask_queue: Task queue to pollpool_size: Connection pool size"""+ if server_address is None:+ from core.env_defaults import env_value++ server_address = env_value("TEMPORAL_SERVER_ADDRESS")logger.info("Starting standalone Temporal worker",server_address=server_address,@@ -634,7 +650,11 @@ async def run_standalone_worker(max_concurrent_activities=pool_size,max_concurrent_workflow_tasks=10,graceful_shutdown_timeout=_graceful_shutdown_timeout(),- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],)logger.info("Worker running. Press Ctrl+C to stop.")@@ -694,7 +714,11 @@ async def create_worker(max_concurrent_activities=100,max_concurrent_workflow_tasks=10,graceful_shutdown_timeout=_graceful_shutdown_timeout(),- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],)