AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
debug(backend/chat): Add timing logging to chat stream generation mechanism (#12019)
Reinier van der Leer6467f6734f14
diff --git a/autogpt_platform/backend/backend/api/features/chat/db.py b/autogpt_platform/backend/backend/api/features/chat/db.pyindex d34b4e5b07..303ea0a698 100644--- a/autogpt_platform/backend/backend/api/features/chat/db.py+++ b/autogpt_platform/backend/backend/api/features/chat/db.py@@ -45,10 +45,7 @@ async def create_chat_session(successfulAgentRuns=SafeJson({}),successfulAgentSchedules=SafeJson({}),)- return await PrismaChatSession.prisma().create(- data=data,- include={"Messages": True},- )+ return await PrismaChatSession.prisma().create(data=data)async def update_chat_session(diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 3e731d86ac..74e6e8ba1e 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -266,12 +266,38 @@ async def stream_chat_post("""import asyncio+ import time++ stream_start_time = time.perf_counter()++ # Base log metadata (task_id added after creation)+ log_meta = {"component": "ChatStream", "session_id": session_id}+ if user_id:+ log_meta["user_id"] = user_id++ logger.info(+ f"[TIMING] stream_chat_post STARTED, session={session_id}, "+ f"user={user_id}, message_len={len(request.message)}",+ extra={"json_fields": log_meta},+ )session = await _validate_and_get_session(session_id, user_id)+ logger.info(+ f"[TIMING] session validated in {(time.perf_counter() - stream_start_time)*1000:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "duration_ms": (time.perf_counter() - stream_start_time) * 1000,+ }+ },+ )# Create a task in the stream registry for reconnection supporttask_id = str(uuid_module.uuid4())operation_id = str(uuid_module.uuid4())+ log_meta["task_id"] = task_id++ task_create_start = time.perf_counter()await stream_registry.create_task(task_id=task_id,session_id=session_id,@@ -280,14 +306,46 @@ async def stream_chat_post(tool_name="chat",operation_id=operation_id,)+ logger.info(+ f"[TIMING] create_task completed in {(time.perf_counter() - task_create_start)*1000:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "duration_ms": (time.perf_counter() - task_create_start) * 1000,+ }+ },+ )# Background task that runs the AI generation independently of SSE connectionasync def run_ai_generation():+ import time as time_module++ gen_start_time = time_module.perf_counter()+ logger.info(+ f"[TIMING] run_ai_generation STARTED, task={task_id}, session={session_id}, user={user_id}",+ extra={"json_fields": log_meta},+ )+ first_chunk_time, ttfc = None, None+ chunk_count = 0try:# Emit a start event with task_id for reconnectionstart_chunk = StreamStart(messageId=task_id, taskId=task_id)await stream_registry.publish_chunk(task_id, start_chunk)+ logger.info(+ f"[TIMING] StreamStart published at {(time_module.perf_counter() - gen_start_time)*1000:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": (time_module.perf_counter() - gen_start_time)+ * 1000,+ }+ },+ )+ logger.info(+ "[TIMING] Calling stream_chat_completion",+ extra={"json_fields": log_meta},+ )async for chunk in chat_service.stream_chat_completion(session_id,request.message,@@ -296,54 +354,202 @@ async def stream_chat_post(session=session, # Pass pre-fetched session to avoid double-fetchcontext=request.context,):+ chunk_count += 1+ if first_chunk_time is None:+ first_chunk_time = time_module.perf_counter()+ ttfc = first_chunk_time - gen_start_time+ logger.info(+ f"[TIMING] FIRST AI CHUNK at {ttfc:.2f}s, type={type(chunk).__name__}",+ extra={+ "json_fields": {+ **log_meta,+ "chunk_type": type(chunk).__name__,+ "time_to_first_chunk_ms": ttfc * 1000,+ }+ },+ )# Write to Redis (subscribers will receive via XREAD)await stream_registry.publish_chunk(task_id, chunk)- # Mark task as completed+ gen_end_time = time_module.perf_counter()+ total_time = (gen_end_time - gen_start_time) * 1000+ logger.info(+ f"[TIMING] run_ai_generation FINISHED in {total_time/1000:.1f}s; "+ f"task={task_id}, session={session_id}, "+ f"ttfc={ttfc or -1:.2f}s, n_chunks={chunk_count}",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time,+ "time_to_first_chunk_ms": (+ ttfc * 1000 if ttfc is not None else None+ ),+ "n_chunks": chunk_count,+ }+ },+ )+await stream_registry.mark_task_completed(task_id, "completed")except Exception as e:+ elapsed = time_module.perf_counter() - gen_start_timelogger.error(- f"Error in background AI generation for session {session_id}: {e}"+ f"[TIMING] run_ai_generation ERROR after {elapsed:.2f}s: {e}",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": elapsed * 1000,+ "error": str(e),+ }+ },)await stream_registry.mark_task_completed(task_id, "failed")# Start the AI generation in a background taskbg_task = asyncio.create_task(run_ai_generation())await stream_registry.set_task_asyncio_task(task_id, bg_task)+ setup_time = (time.perf_counter() - stream_start_time) * 1000+ logger.info(+ f"[TIMING] Background task started, setup={setup_time:.1f}ms",+ extra={"json_fields": {**log_meta, "setup_time_ms": setup_time}},+ )# SSE endpoint that subscribes to the task's streamasync def event_generator() -> AsyncGenerator[str, None]:+ import time as time_module++ event_gen_start = time_module.perf_counter()+ logger.info(+ f"[TIMING] event_generator STARTED, task={task_id}, session={session_id}, "+ f"user={user_id}",+ extra={"json_fields": log_meta},+ )subscriber_queue = None+ first_chunk_yielded = False+ chunks_yielded = 0try:# Subscribe to the task stream (this replays existing messages + live updates)+ subscribe_start = time_module.perf_counter()+ logger.info(+ "[TIMING] Calling subscribe_to_task",+ extra={"json_fields": log_meta},+ )subscriber_queue = await stream_registry.subscribe_to_task(task_id=task_id,user_id=user_id,last_message_id="0-0", # Get all messages from the beginning)+ subscribe_time = (time_module.perf_counter() - subscribe_start) * 1000+ logger.info(+ f"[TIMING] subscribe_to_task completed in {subscribe_time:.1f}ms, "+ f"queue_ok={subscriber_queue is not None}",+ extra={+ "json_fields": {+ **log_meta,+ "duration_ms": subscribe_time,+ "queue_obtained": subscriber_queue is not None,+ }+ },+ )if subscriber_queue is None:+ logger.info(+ "[TIMING] subscriber_queue is None, yielding finish",+ extra={"json_fields": log_meta},+ )yield StreamFinish().to_sse()yield "data: [DONE]\n\n"return# Read from the subscriber queue and yield to SSE+ logger.info(+ "[TIMING] Starting to read from subscriber_queue",+ extra={"json_fields": log_meta},+ )while True:try:+ queue_wait_start = time_module.perf_counter()chunk = await asyncio.wait_for(subscriber_queue.get(), timeout=30.0)+ queue_wait_time = (+ time_module.perf_counter() - queue_wait_start+ ) * 1000+ chunks_yielded += 1++ if not first_chunk_yielded:+ first_chunk_yielded = True+ elapsed = time_module.perf_counter() - event_gen_start+ logger.info(+ f"[TIMING] FIRST CHUNK from queue at {elapsed:.2f}s, "+ f"type={type(chunk).__name__}, "+ f"wait={queue_wait_time:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "chunk_type": type(chunk).__name__,+ "elapsed_ms": elapsed * 1000,+ "queue_wait_ms": queue_wait_time,+ }+ },+ )+ elif chunks_yielded % 50 == 0:+ logger.info(+ f"[TIMING] Chunk #{chunks_yielded}, "+ f"type={type(chunk).__name__}",+ extra={+ "json_fields": {+ **log_meta,+ "chunk_number": chunks_yielded,+ "chunk_type": type(chunk).__name__,+ }+ },+ )+yield chunk.to_sse()# Check for finish signalif isinstance(chunk, StreamFinish):+ total_time = time_module.perf_counter() - event_gen_start+ logger.info(+ f"[TIMING] StreamFinish received in {total_time:.2f}s; "+ f"n_chunks={chunks_yielded}",+ extra={+ "json_fields": {+ **log_meta,+ "chunks_yielded": chunks_yielded,+ "total_time_ms": total_time * 1000,+ }+ },+ )breakexcept asyncio.TimeoutError:# Send heartbeat to keep connection alive+ logger.info(+ f"[TIMING] Heartbeat timeout, chunks_so_far={chunks_yielded}",+ extra={+ "json_fields": {**log_meta, "chunks_so_far": chunks_yielded}+ },+ )yield StreamHeartbeat().to_sse()except GeneratorExit:+ logger.info(+ f"[TIMING] GeneratorExit (client disconnected), chunks={chunks_yielded}",+ extra={+ "json_fields": {+ **log_meta,+ "chunks_yielded": chunks_yielded,+ "reason": "client_disconnect",+ }+ },+ )pass # Client disconnected - background task continuesexcept Exception as e:- logger.error(f"Error in SSE stream for task {task_id}: {e}")+ elapsed = (time_module.perf_counter() - event_gen_start) * 1000+ logger.error(+ f"[TIMING] event_generator ERROR after {elapsed:.1f}ms: {e}",+ extra={+ "json_fields": {**log_meta, "elapsed_ms": elapsed, "error": str(e)}+ },+ )finally:# Unsubscribe when client disconnects or stream ends to prevent resource leakif subscriber_queue is not None:@@ -357,6 +563,18 @@ async def stream_chat_post(exc_info=True,)# AI SDK protocol termination - always yield even if unsubscribe fails+ total_time = time_module.perf_counter() - event_gen_start+ logger.info(+ f"[TIMING] event_generator FINISHED in {total_time:.2f}s; "+ f"task={task_id}, session={session_id}, n_chunks={chunks_yielded}",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time * 1000,+ "chunks_yielded": chunks_yielded,+ }+ },+ )yield "data: [DONE]\n\n"return StreamingResponse(@@ -425,7 +643,7 @@ async def stream_chat_get("Chat stream completed",extra={"session_id": session_id,- "chunk_count": chunk_count,+ "n_chunks": chunk_count,"first_chunk_type": first_chunk_type,},)diff --git a/autogpt_platform/backend/backend/api/features/chat/service.py b/autogpt_platform/backend/backend/api/features/chat/service.pyindex 06da6bdf2b..da18421b98 100644--- a/autogpt_platform/backend/backend/api/features/chat/service.py+++ b/autogpt_platform/backend/backend/api/features/chat/service.py@@ -371,21 +371,45 @@ async def stream_chat_completion(ValueError: If max_context_messages is exceeded"""+ completion_start = time.monotonic()++ # Build log metadata for structured logging+ log_meta = {"component": "ChatService", "session_id": session_id}+ if user_id:+ log_meta["user_id"] = user_id+logger.info(- f"Streaming chat completion for session {session_id} for message {message} and user id {user_id}. Message is user message: {is_user_message}"+ f"[TIMING] stream_chat_completion STARTED, session={session_id}, user={user_id}, "+ f"message_len={len(message) if message else 0}, is_user={is_user_message}",+ extra={+ "json_fields": {+ **log_meta,+ "message_len": len(message) if message else 0,+ "is_user_message": is_user_message,+ }+ },)# Only fetch from Redis if session not provided (initial call)if session is None:+ fetch_start = time.monotonic()session = await get_chat_session(session_id, user_id)+ fetch_time = (time.monotonic() - fetch_start) * 1000logger.info(- f"Fetched session from Redis: {session.session_id if session else 'None'}, "- f"message_count={len(session.messages) if session else 0}"+ f"[TIMING] get_chat_session took {fetch_time:.1f}ms, "+ f"n_messages={len(session.messages) if session else 0}",+ extra={+ "json_fields": {+ **log_meta,+ "duration_ms": fetch_time,+ "n_messages": len(session.messages) if session else 0,+ }+ },)else:logger.info(- f"Using provided session object: {session.session_id}, "- f"message_count={len(session.messages)}"+ f"[TIMING] Using provided session, messages={len(session.messages)}",+ extra={"json_fields": {**log_meta, "n_messages": len(session.messages)}},)if not session:@@ -406,17 +430,25 @@ async def stream_chat_completion(# Track user message in PostHogif is_user_message:+ posthog_start = time.monotonic()track_user_message(user_id=user_id,session_id=session_id,message_length=len(message),)+ posthog_time = (time.monotonic() - posthog_start) * 1000+ logger.info(+ f"[TIMING] track_user_message took {posthog_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": posthog_time}},+ )+ upsert_start = time.monotonic()+ session = await upsert_chat_session(session)+ upsert_time = (time.monotonic() - upsert_start) * 1000logger.info(- f"Upserting session: {session.session_id} with user id {session.user_id}, "- f"message_count={len(session.messages)}"+ f"[TIMING] upsert_chat_session took {upsert_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": upsert_time}},)- session = await upsert_chat_session(session)assert session, "Session not found"# Generate title for new sessions on first user message (non-blocking)@@ -454,7 +486,13 @@ async def stream_chat_completion(asyncio.create_task(_update_title())# Build system prompt with business understanding+ prompt_start = time.monotonic()system_prompt, understanding = await _build_system_prompt(user_id)+ prompt_time = (time.monotonic() - prompt_start) * 1000+ logger.info(+ f"[TIMING] _build_system_prompt took {prompt_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": prompt_time}},+ )# Initialize variables for streamingassistant_response = ChatMessage(@@ -483,9 +521,18 @@ async def stream_chat_completion(text_block_id = str(uuid_module.uuid4())# Yield message start+ setup_time = (time.monotonic() - completion_start) * 1000+ logger.info(+ f"[TIMING] Setup complete, yielding StreamStart at {setup_time:.1f}ms",+ extra={"json_fields": {**log_meta, "setup_time_ms": setup_time}},+ )yield StreamStart(messageId=message_id)try:+ logger.info(+ "[TIMING] Calling _stream_chat_chunks",+ extra={"json_fields": log_meta},+ )async for chunk in _stream_chat_chunks(session=session,tools=tools,@@ -893,9 +940,21 @@ async def _stream_chat_chunks(SSE formatted JSON response objects"""+ import time as time_module++ stream_chunks_start = time_module.perf_counter()model = config.model- logger.info("Starting pure chat stream")+ # Build log metadata for structured logging+ log_meta = {"component": "ChatService", "session_id": session.session_id}+ if session.user_id:+ log_meta["user_id"] = session.user_id++ logger.info(+ f"[TIMING] _stream_chat_chunks STARTED, session={session.session_id}, "+ f"user={session.user_id}, n_messages={len(session.messages)}",+ extra={"json_fields": {**log_meta, "n_messages": len(session.messages)}},+ )messages = session.to_openai_messages()if system_prompt:@@ -906,12 +965,18 @@ async def _stream_chat_chunks(messages = [system_message] + messages# Apply context window management+ context_start = time_module.perf_counter()context_result = await _manage_context_window(messages=messages,model=model,api_key=config.api_key,base_url=config.base_url,)+ context_time = (time_module.perf_counter() - context_start) * 1000+ logger.info(+ f"[TIMING] _manage_context_window took {context_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": context_time}},+ )if context_result.error:if "System prompt dropped" in context_result.error:@@ -946,9 +1011,19 @@ async def _stream_chat_chunks(while retry_count <= MAX_RETRIES:try:+ elapsed = (time_module.perf_counter() - stream_chunks_start) * 1000+ retry_info = (+ f" (retry {retry_count}/{MAX_RETRIES})" if retry_count > 0 else ""+ )logger.info(- f"Creating OpenAI chat completion stream..."- f"{f' (retry {retry_count}/{MAX_RETRIES})' if retry_count > 0 else ''}"+ f"[TIMING] Creating OpenAI stream at {elapsed:.1f}ms{retry_info}",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": elapsed,+ "retry_count": retry_count,+ }+ },)# Build extra_body for OpenRouter tracing and PostHog analytics@@ -965,6 +1040,7 @@ async def _stream_chat_chunks(:128] # OpenRouter limit+ api_call_start = time_module.perf_counter()stream = await client.chat.completions.create(model=model,messages=cast(list[ChatCompletionMessageParam], messages),@@ -974,6 +1050,11 @@ async def _stream_chat_chunks(stream_options=ChatCompletionStreamOptionsParam(include_usage=True),extra_body=extra_body,)+ api_init_time = (time_module.perf_counter() - api_call_start) * 1000+ logger.info(+ f"[TIMING] OpenAI stream object returned in {api_init_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": api_init_time}},+ )# Variables to accumulate tool callstool_calls: list[dict[str, Any]] = []@@ -984,10 +1065,13 @@ async def _stream_chat_chunks(# Track if we've started the text blocktext_started = False+ first_content_chunk = True+ chunk_count = 0# Process the streamchunk: ChatCompletionChunkasync for chunk in stream:+ chunk_count += 1if chunk.usage:yield StreamUsage(promptTokens=chunk.usage.prompt_tokens,@@ -1010,6 +1094,23 @@ async def _stream_chat_chunks(if not text_started and text_block_id:yield StreamTextStart(id=text_block_id)text_started = True+ # Log timing for first content chunk+ if first_content_chunk:+ first_content_chunk = False+ ttfc = (+ time_module.perf_counter() - api_call_start+ ) * 1000+ logger.info(+ f"[TIMING] FIRST CONTENT CHUNK at {ttfc:.1f}ms "+ f"(since API call), n_chunks={chunk_count}",+ extra={+ "json_fields": {+ **log_meta,+ "time_to_first_chunk_ms": ttfc,+ "n_chunks": chunk_count,+ }+ },+ )# Stream the text deltatext_response = StreamTextDelta(id=text_block_id or "",@@ -1066,7 +1167,21 @@ async def _stream_chat_chunks(toolName=tool_calls[idx]["function"]["name"],)emitted_start_for_idx.add(idx)- logger.info(f"Stream complete. Finish reason: {finish_reason}")+ stream_duration = time_module.perf_counter() - api_call_start+ logger.info(+ f"[TIMING] OpenAI stream COMPLETE, finish_reason={finish_reason}, "+ f"duration={stream_duration:.2f}s, "+ f"n_chunks={chunk_count}, n_tool_calls={len(tool_calls)}",+ extra={+ "json_fields": {+ **log_meta,+ "stream_duration_ms": stream_duration * 1000,+ "finish_reason": finish_reason,+ "n_chunks": chunk_count,+ "n_tool_calls": len(tool_calls),+ }+ },+ )# Yield all accumulated tool calls after the stream is complete# This ensures all tool call arguments have been fully received@@ -1086,6 +1201,12 @@ async def _stream_chat_chunks(# Re-raise to trigger retry logic in the parent functionraise+ total_time = (time_module.perf_counter() - stream_chunks_start) * 1000+ logger.info(+ f"[TIMING] _stream_chat_chunks COMPLETED in {total_time/1000:.1f}s; "+ f"session={session.session_id}, user={session.user_id}",+ extra={"json_fields": {**log_meta, "total_time_ms": total_time}},+ )yield StreamFinish()returnexcept Exception as e:diff --git a/autogpt_platform/backend/backend/api/features/chat/stream_registry.py b/autogpt_platform/backend/backend/api/features/chat/stream_registry.pyindex 88a5023e2b..509d20d9f4 100644--- a/autogpt_platform/backend/backend/api/features/chat/stream_registry.py+++ b/autogpt_platform/backend/backend/api/features/chat/stream_registry.py@@ -104,6 +104,24 @@ async def create_task(Returns:The created ActiveTask instance (metadata only)"""+ import time++ start_time = time.perf_counter()++ # Build log metadata for structured logging+ log_meta = {+ "component": "StreamRegistry",+ "task_id": task_id,+ "session_id": session_id,+ }+ if user_id:+ log_meta["user_id"] = user_id++ logger.info(+ f"[TIMING] create_task STARTED, task={task_id}, session={session_id}, user={user_id}",+ extra={"json_fields": log_meta},+ )+task = ActiveTask(task_id=task_id,session_id=session_id,@@ -114,10 +132,18 @@ async def create_task()# Store metadata in Redis+ redis_start = time.perf_counter()redis = await get_redis_async()+ redis_time = (time.perf_counter() - redis_start) * 1000+ logger.info(+ f"[TIMING] get_redis_async took {redis_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": redis_time}},+ )+meta_key = _get_task_meta_key(task_id)op_key = _get_operation_mapping_key(operation_id)+ hset_start = time.perf_counter()await redis.hset( # type: ignore[misc]meta_key,mapping={@@ -131,12 +157,22 @@ async def create_task("created_at": task.created_at.isoformat(),},)+ hset_time = (time.perf_counter() - hset_start) * 1000+ logger.info(+ f"[TIMING] redis.hset took {hset_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": hset_time}},+ )+await redis.expire(meta_key, config.stream_ttl)# Create operation_id -> task_id mapping for webhook lookupsawait redis.set(op_key, task_id, ex=config.stream_ttl)- logger.debug(f"Created task {task_id} for session {session_id}")+ total_time = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] create_task COMPLETED in {total_time:.1f}ms; task={task_id}, session={session_id}",+ extra={"json_fields": {**log_meta, "total_time_ms": total_time}},+ )return task@@ -156,26 +192,60 @@ async def publish_chunk(Returns:The Redis Stream message ID"""+ import time++ start_time = time.perf_counter()+ chunk_type = type(chunk).__name__chunk_json = chunk.model_dump_json()message_id = "0-0"+ # Build log metadata+ log_meta = {+ "component": "StreamRegistry",+ "task_id": task_id,+ "chunk_type": chunk_type,+ }+try:redis = await get_redis_async()stream_key = _get_task_stream_key(task_id)# Write to Redis Stream for persistence and real-time delivery+ xadd_start = time.perf_counter()raw_id = await redis.xadd(stream_key,{"data": chunk_json},maxlen=config.stream_max_length,)+ xadd_time = (time.perf_counter() - xadd_start) * 1000message_id = raw_id if isinstance(raw_id, str) else raw_id.decode()# Set TTL on stream to match task metadata TTLawait redis.expire(stream_key, config.stream_ttl)++ total_time = (time.perf_counter() - start_time) * 1000+ # Only log timing for significant chunks or slow operations+ if (+ chunk_type+ in ("StreamStart", "StreamFinish", "StreamTextStart", "StreamTextEnd")+ or total_time > 50+ ):+ logger.info(+ f"[TIMING] publish_chunk {chunk_type} in {total_time:.1f}ms (xadd={xadd_time:.1f}ms)",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time,+ "xadd_time_ms": xadd_time,+ "message_id": message_id,+ }+ },+ )except Exception as e:+ elapsed = (time.perf_counter() - start_time) * 1000logger.error(- f"Failed to publish chunk for task {task_id}: {e}",+ f"[TIMING] Failed to publish chunk {chunk_type} after {elapsed:.1f}ms: {e}",+ extra={"json_fields": {**log_meta, "elapsed_ms": elapsed, "error": str(e)}},exc_info=True,)@@ -200,24 +270,61 @@ async def subscribe_to_task(An asyncio Queue that will receive stream chunks, or None if task not foundor user doesn't have access"""+ import time++ start_time = time.perf_counter()++ # Build log metadata+ log_meta = {"component": "StreamRegistry", "task_id": task_id}+ if user_id:+ log_meta["user_id"] = user_id++ logger.info(+ f"[TIMING] subscribe_to_task STARTED, task={task_id}, user={user_id}, last_msg={last_message_id}",+ extra={"json_fields": {**log_meta, "last_message_id": last_message_id}},+ )++ redis_start = time.perf_counter()redis = await get_redis_async()meta_key = _get_task_meta_key(task_id)meta: dict[Any, Any] = await redis.hgetall(meta_key) # type: ignore[misc]+ hgetall_time = (time.perf_counter() - redis_start) * 1000+ logger.info(+ f"[TIMING] Redis hgetall took {hgetall_time:.1f}ms",+ extra={"json_fields": {**log_meta, "duration_ms": hgetall_time}},+ )if not meta:- logger.debug(f"Task {task_id} not found in Redis")+ elapsed = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] Task not found in Redis after {elapsed:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": elapsed,+ "reason": "task_not_found",+ }+ },+ )return None# Note: Redis client uses decode_responses=True, so keys are stringstask_status = meta.get("status", "")task_user_id = meta.get("user_id", "") or None+ log_meta["session_id"] = meta.get("session_id", "")# Validate ownership - if task has an owner, requester must matchif task_user_id:if user_id != task_user_id:logger.warning(- f"User {user_id} denied access to task {task_id} "- f"owned by {task_user_id}"+ f"[TIMING] Access denied: user {user_id} tried to access task owned by {task_user_id}",+ extra={+ "json_fields": {+ **log_meta,+ "task_owner": task_user_id,+ "reason": "access_denied",+ }+ },)return None@@ -225,7 +332,19 @@ async def subscribe_to_task(stream_key = _get_task_stream_key(task_id)# Step 1: Replay messages from Redis Stream+ xread_start = time.perf_counter()messages = await redis.xread({stream_key: last_message_id}, block=0, count=1000)+ xread_time = (time.perf_counter() - xread_start) * 1000+ logger.info(+ f"[TIMING] Redis xread (replay) took {xread_time:.1f}ms, status={task_status}",+ extra={+ "json_fields": {+ **log_meta,+ "duration_ms": xread_time,+ "task_status": task_status,+ }+ },+ )replayed_count = 0replay_last_id = last_message_id@@ -244,19 +363,48 @@ async def subscribe_to_task(except Exception as e:logger.warning(f"Failed to replay message: {e}")- logger.debug(f"Task {task_id}: replayed {replayed_count} messages")+ logger.info(+ f"[TIMING] Replayed {replayed_count} messages, last_id={replay_last_id}",+ extra={+ "json_fields": {+ **log_meta,+ "n_messages_replayed": replayed_count,+ "replay_last_id": replay_last_id,+ }+ },+ )# Step 2: If task is still running, start stream listener for live updatesif task_status == "running":+ logger.info(+ "[TIMING] Task still running, starting _stream_listener",+ extra={"json_fields": {**log_meta, "task_status": task_status}},+ )listener_task = asyncio.create_task(- _stream_listener(task_id, subscriber_queue, replay_last_id)+ _stream_listener(task_id, subscriber_queue, replay_last_id, log_meta))# Track listener task for cleanup on unsubscribe_listener_tasks[id(subscriber_queue)] = (task_id, listener_task)else:# Task is completed/failed - add finish marker+ logger.info(+ f"[TIMING] Task already {task_status}, adding StreamFinish",+ extra={"json_fields": {**log_meta, "task_status": task_status}},+ )await subscriber_queue.put(StreamFinish())+ total_time = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] subscribe_to_task COMPLETED in {total_time:.1f}ms; task={task_id}, "+ f"n_messages_replayed={replayed_count}",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time,+ "n_messages_replayed": replayed_count,+ }+ },+ )return subscriber_queue@@ -264,6 +412,7 @@ async def _stream_listener(task_id: str,subscriber_queue: asyncio.Queue[StreamBaseResponse],last_replayed_id: str,+ log_meta: dict | None = None,) -> None:"""Listen to Redis Stream for new messages using blocking XREAD.@@ -274,10 +423,27 @@ async def _stream_listener(task_id: Task ID to listen forsubscriber_queue: Queue to deliver messages tolast_replayed_id: Last message ID from replay (continue from here)+ log_meta: Structured logging metadata"""+ import time++ start_time = time.perf_counter()++ # Use provided log_meta or build minimal one+ if log_meta is None:+ log_meta = {"component": "StreamRegistry", "task_id": task_id}++ logger.info(+ f"[TIMING] _stream_listener STARTED, task={task_id}, last_id={last_replayed_id}",+ extra={"json_fields": {**log_meta, "last_replayed_id": last_replayed_id}},+ )+queue_id = id(subscriber_queue)# Track the last successfully delivered message ID for recovery hintslast_delivered_id = last_replayed_id+ messages_delivered = 0+ first_message_time = None+ xread_count = 0try:redis = await get_redis_async()@@ -287,9 +453,39 @@ async def _stream_listener(while True:# Block for up to 30 seconds waiting for new messages# This allows periodic checking if task is still running+ xread_start = time.perf_counter()+ xread_count += 1messages = await redis.xread({stream_key: current_id}, block=30000, count=100)+ xread_time = (time.perf_counter() - xread_start) * 1000++ if messages:+ msg_count = sum(len(msgs) for _, msgs in messages)+ logger.info(+ f"[TIMING] xread #{xread_count} returned {msg_count} messages in {xread_time:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "xread_count": xread_count,+ "n_messages": msg_count,+ "duration_ms": xread_time,+ }+ },+ )+ elif xread_time > 1000:+ # Only log timeouts (30s blocking)+ logger.info(+ f"[TIMING] xread #{xread_count} timeout after {xread_time:.1f}ms",+ extra={+ "json_fields": {+ **log_meta,+ "xread_count": xread_count,+ "duration_ms": xread_time,+ "reason": "timeout",+ }+ },+ )if not messages:# Timeout - check if task is still running@@ -326,10 +522,30 @@ async def _stream_listener()# Update last delivered ID on successful deliverylast_delivered_id = current_id+ messages_delivered += 1+ if first_message_time is None:+ first_message_time = time.perf_counter()+ elapsed = (first_message_time - start_time) * 1000+ logger.info(+ f"[TIMING] FIRST live message at {elapsed:.1f}ms, type={type(chunk).__name__}",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": elapsed,+ "chunk_type": type(chunk).__name__,+ }+ },+ )except asyncio.TimeoutError:logger.warning(- f"Subscriber queue full for task {task_id}, "- f"message delivery timed out after {QUEUE_PUT_TIMEOUT}s"+ f"[TIMING] Subscriber queue full, delivery timed out after {QUEUE_PUT_TIMEOUT}s",+ extra={+ "json_fields": {+ **log_meta,+ "timeout_s": QUEUE_PUT_TIMEOUT,+ "reason": "queue_full",+ }+ },)# Send overflow error with recovery infotry:@@ -351,15 +567,44 @@ async def _stream_listener(# Stop listening on finishif isinstance(chunk, StreamFinish):+ total_time = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] StreamFinish received in {total_time/1000:.1f}s; delivered={messages_delivered}",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time,+ "messages_delivered": messages_delivered,+ }+ },+ )returnexcept Exception as e:- logger.warning(f"Error processing stream message: {e}")+ logger.warning(+ f"Error processing stream message: {e}",+ extra={"json_fields": {**log_meta, "error": str(e)}},+ )except asyncio.CancelledError:- logger.debug(f"Stream listener cancelled for task {task_id}")+ elapsed = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] _stream_listener CANCELLED after {elapsed:.1f}ms, delivered={messages_delivered}",+ extra={+ "json_fields": {+ **log_meta,+ "elapsed_ms": elapsed,+ "messages_delivered": messages_delivered,+ "reason": "cancelled",+ }+ },+ )raise # Re-raise to propagate cancellationexcept Exception as e:- logger.error(f"Stream listener error for task {task_id}: {e}")+ elapsed = (time.perf_counter() - start_time) * 1000+ logger.error(+ f"[TIMING] _stream_listener ERROR after {elapsed:.1f}ms: {e}",+ extra={"json_fields": {**log_meta, "elapsed_ms": elapsed, "error": str(e)}},+ )# On error, send finish to unblock subscribertry:await asyncio.wait_for(@@ -368,10 +613,24 @@ async def _stream_listener()except (asyncio.TimeoutError, asyncio.QueueFull):logger.warning(- f"Could not deliver finish event for task {task_id} after error"+ "Could not deliver finish event after error",+ extra={"json_fields": log_meta},)finally:# Clean up listener task mapping on exit+ total_time = (time.perf_counter() - start_time) * 1000+ logger.info(+ f"[TIMING] _stream_listener FINISHED in {total_time/1000:.1f}s; task={task_id}, "+ f"delivered={messages_delivered}, xread_count={xread_count}",+ extra={+ "json_fields": {+ **log_meta,+ "total_time_ms": total_time,+ "messages_delivered": messages_delivered,+ "xread_count": xread_count,+ }+ },+ )_listener_tasks.pop(queue_id, None)