:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import time
import asyncio
from collections.abc import AsyncGenerator
from typing import Any
from uuid import uuid4
from django.conf import settings
import structlog
import posthoganalytics
from opentelemetry import trace
from prometheus_client import Histogram
from temporalio.client import WorkflowExecutionStatus, WorkflowHandle
from temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy
from temporalio.exceptions import WorkflowAlreadyStartedError
from posthog.schema import AssistantEventType, FailureMessage
from posthog.temporal.ai.base import AgentBaseWorkflow
from posthog.temporal.common.client import async_connect
from products.posthog_ai.backend.models.assistant import Conversation
from ee.hogai.core.workflow_payload import get_workflow_input_size_estimates
from ee.hogai.queue import ConversationQueueStore
from ee.hogai.stream.redis_stream import (
CONVERSATION_STREAM_MAX_LENGTH,
CONVERSATION_STREAM_TIMEOUT,
ApprovalEvent,
ConversationEvent,
ConversationRedisStream,
GenerationStatusEvent,
MessageEvent,
StreamError,
StreamEvent,
UpdateEvent,
get_conversation_stream_key,
)
from ee.hogai.utils.types import AssistantOutput
logger = structlog.get_logger(__name__)
_tracer = trace.get_tracer(__name__)
STREAM_DJANGO_EVENT_LOOP_LATENCY_HISTOGRAM = Histogram(
"posthog_ai_stream_django_event_loop_latency_seconds",
"Time from receiving chunk from Temporal to yielding it in Django event loop",
buckets=[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, float("inf")],
)
class AgentExecutor:
"""Manages executing an agent workflow and streaming the output."""
def __init__(
self,
conversation: Conversation,
stream_key: str | None = None,
timeout: int = CONVERSATION_STREAM_TIMEOUT,
max_length: int = CONVERSATION_STREAM_MAX_LENGTH,
reconnectable: bool = True,
) -> None:
self._conversation = conversation
if stream_key is None:
stream_key = get_conversation_stream_key(conversation.id)
self._redis_stream = ConversationRedisStream(stream_key, timeout=timeout, max_length=max_length)
self._workflow_id = f"conversation-{conversation.id}"
self._reconnectable = reconnectable
async def astream(self, workflow: type[AgentBaseWorkflow], inputs: Any) -> AsyncGenerator[AssistantOutput, Any]:
"""Stream agent workflow updates from Redis stream.
Args:
workflow: Agent temporal workflow class
inputs: Agent temporal workflow inputs
Returns:
AssistantOutput generator
"""
# If this is a reconnection attempt, we resume streaming
if self._conversation.status != Conversation.Status.IDLE and self._reconnectable:
if hasattr(inputs, "message") and inputs.message is not None:
raise ValueError("Cannot resume streaming with a new message")
async for chunk in self.stream_conversation():
yield chunk
else:
# Otherwise, process the new message (new generation) or resume generation (no new message)
async for chunk in self.start_workflow(workflow, inputs):
yield chunk
async def start_workflow(
self, workflow: type[AgentBaseWorkflow], inputs: Any
) -> AsyncGenerator[AssistantOutput, Any]:
# Capture the failure outcome inside the span and yield outside it — yielding from
# an async generator while a span is "current" leaks that span into the consumer
# task's contextvars. See stream_conversation for the longer note.
failed = False
with _tracer.start_as_current_span(
"posthog_ai.executor.start_workflow",
attributes={
"posthog_ai.conversation_id": str(self._conversation.id),
"posthog_ai.workflow_id": self._workflow_id,
"posthog_ai.workflow_class": workflow.__name__,
},
):
try:
# Delete the stream to ensure we start fresh
# since there might be a stale stream from a previous conversation gone wrong
with _tracer.start_as_current_span("posthog_ai.executor.delete_stale_stream"):
await self._redis_stream.delete_stream()
with _tracer.start_as_current_span("posthog_ai.executor.temporal_connect"):
client = await async_connect()
handle = await self._start_workflow_with_retry(client, workflow, inputs)
# Wait for the workflow to start running before streaming
is_workflow_running = await self._wait_for_workflow_to_start(handle)
if not is_workflow_running:
raise Exception(f"Workflow failed to start within timeout: {self._workflow_id}")
except Exception as e:
posthoganalytics.capture_exception(e, properties={"tag": "max_ai"})
logger.exception(
"Error starting workflow",
error=e,
workflow_input_estimated_bytes=get_workflow_input_size_estimates(inputs),
)
failed = True
if failed:
yield self._failure_message()
return
async for chunk in self.stream_conversation():
yield chunk
async def _start_workflow_with_retry(
self, client: Any, workflow: type[AgentBaseWorkflow], inputs: Any, max_retries: int = 3
) -> WorkflowHandle:
"""Start a workflow with retry logic for handling race conditions.
When a workflow is in the process of completing, there can be a brief window
where starting a new workflow with the same ID fails even with USE_EXISTING policy.
This method handles that by waiting for the existing workflow to complete and retrying.
"""
span = trace.get_current_span()
for attempt in range(max_retries):
try:
with _tracer.start_as_current_span(
"posthog_ai.executor.temporal_start_workflow",
attributes={"posthog_ai.attempt": attempt},
):
handle = await client.start_workflow(
workflow.run,
inputs,
id=self._workflow_id,
task_queue=settings.MAX_AI_TASK_QUEUE,
id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
)
span.set_attribute("posthog_ai.start_workflow.attempts", attempt + 1)
return handle
except WorkflowAlreadyStartedError:
if attempt < max_retries - 1:
# Get handle to existing workflow and wait for it to complete
logger.info(
"Workflow already started, waiting for completion",
workflow_id=self._workflow_id,
attempt=attempt + 1,
)
existing_handle = client.get_workflow_handle(workflow_id=self._workflow_id)
try:
# Wait for the existing workflow to complete (with timeout)
await asyncio.wait_for(existing_handle.result(), timeout=5.0)
except TimeoutError:
# If timeout, the workflow is still running - return the existing handle
logger.info(
"Existing workflow still running, using existing handle",
workflow_id=self._workflow_id,
)
span.set_attribute("posthog_ai.start_workflow.attempts", attempt + 1)
return existing_handle
except Exception:
# Workflow completed (possibly with error), retry starting a new one
pass
# Small delay before retry
await asyncio.sleep(0.1)
else:
raise
# This should be unreachable, but mypy needs it
raise Exception(f"Failed to start workflow after {max_retries} attempts")
async def _wait_for_workflow_to_start(self, handle: WorkflowHandle) -> bool:
"""Wait for the workflow to start running.
Args:
handle: Temporal workflow handle
Returns:
True if workflow started running, False otherwise
"""
max_attempts = 10 * 60 # 60 seconds total with 0.1s sleep
attempts = 0
with _tracer.start_as_current_span("posthog_ai.executor.wait_for_workflow_start") as span:
while attempts < max_attempts:
description = await handle.describe()
if description.status is None:
attempts += 1
await asyncio.sleep(0.1)
elif description.status == WorkflowExecutionStatus.RUNNING:
# Temporal only has one Open execution status, see: https://docs.temporal.io/workflow-execution
span.set_attribute("posthog_ai.poll_attempts", attempts)
span.set_attribute("posthog_ai.outcome", "running")
return True
else:
span.set_attribute("posthog_ai.poll_attempts", attempts)
span.set_attribute("posthog_ai.outcome", "not_running")
return False
span.set_attribute("posthog_ai.poll_attempts", attempts)
span.set_attribute("posthog_ai.outcome", "timeout")
return False
async def stream_conversation(self) -> AsyncGenerator[AssistantOutput, Any]:
"""Stream conversation updates from Redis stream.
Returns:
AssistantOutput generator
"""
chunk_count = 0
# Use start_span + use_span (without making the span "current" across yields).
# `start_as_current_span` inside an async generator attaches the span to the running
# task's contextvars; when the generator yields, the consumer task resumes with that
# context still set, leaking this span as the consumer's "current span". We scope
# the current-span context only around the non-yielding bookkeeping so child spans
# still parent correctly, while yields run outside the attached context.
# See https://github.com/open-telemetry/opentelemetry-python/issues/2606.
span = _tracer.start_span(
"posthog_ai.executor.stream_conversation",
attributes={"posthog_ai.conversation_id": str(self._conversation.id)},
)
try:
try:
with trace.use_span(span, end_on_exit=False):
# Wait for stream to be created
is_stream_available = await self._redis_stream.wait_for_stream()
if not is_stream_available:
raise StreamError(
"Stream for this conversation not available - Temporal workflow might have failed"
)
last_chunk_time = time.time()
read_iter = self._redis_stream.read_stream().__aiter__()
while True:
with trace.use_span(span, end_on_exit=False):
try:
chunk = await read_iter.__anext__()
except StopAsyncIteration:
break
message = await self._redis_stream_to_assistant_output(chunk)
temporal_to_code_latency = last_chunk_time - chunk.timestamp
if temporal_to_code_latency > 0:
STREAM_DJANGO_EVENT_LOOP_LATENCY_HISTOGRAM.observe(temporal_to_code_latency)
last_chunk_time = time.time()
chunk_count += 1
if message:
yield message
except Exception as e:
with trace.use_span(span, end_on_exit=False):
posthoganalytics.capture_exception(e, properties={"tag": "max_ai"})
logger.exception("Error streaming conversation", error=e)
yield self._failure_message()
finally:
with trace.use_span(span, end_on_exit=False):
span.set_attribute("posthog_ai.stream_conversation.chunks", chunk_count)
await self._redis_stream.delete_stream()
span.end()
async def _redis_stream_to_assistant_output(self, message: StreamEvent) -> AssistantOutput | None:
"""Convert Redis stream event to Assistant output.
Args:
message: event from Redis stream
Returns:
AssistantOutput or None
"""
if isinstance(message.event, MessageEvent):
return (AssistantEventType.MESSAGE, message.event.payload)
elif isinstance(message.event, ConversationEvent):
with _tracer.start_as_current_span(
"posthog_ai.executor.conversation_aget",
attributes={"posthog_ai.conversation_id": str(message.event.payload)},
):
# nosemgrep: idor-lookup-without-team (ID from internal Redis stream, caller validates user+team ownership)
conversation = await Conversation.objects.select_related("user").aget(id=message.event.payload)
return (AssistantEventType.CONVERSATION, conversation)
elif isinstance(message.event, UpdateEvent):
return (AssistantEventType.UPDATE, message.event.payload)
elif isinstance(message.event, GenerationStatusEvent):
return (AssistantEventType.STATUS, message.event.payload)
elif isinstance(message.event, ApprovalEvent):
return (AssistantEventType.APPROVAL, message.event.payload)
else:
return None
def _failure_message(self) -> AssistantOutput:
"""Returns a failure message as an Assistant output."""
failure_message = FailureMessage(
content="Oops! Something went wrong. Please try again.",
id=str(uuid4()),
)
return (AssistantEventType.MESSAGE, failure_message)
async def cancel_workflow(self) -> None:
"""Cancel the current conversation and clean up resources.
This cancels the main conversation workflow, any running subagent workflows,
any queued message workflows, and clears the cache queue.
The main workflow cancel is non-fatal because the main workflow may have already
completed after spawning a queued workflow. In that case, we still need to cancel
the queued workflows and clear the cache queue.
"""
self._conversation.status = Conversation.Status.CANCELING
await self._conversation.asave(update_fields=["status", "updated_at"])
try:
client = await async_connect()
# Cancel the main conversation workflow.
# This may fail if the workflow already completed (e.g. after spawning a queued workflow),
# but we must continue to cancel queued workflows and clean up.
try:
handle = client.get_workflow_handle(workflow_id=self._workflow_id)
await handle.cancel()
except Exception as e:
logger.warning(
"Failed to cancel main workflow (may have already completed)",
workflow_id=self._workflow_id,
conversation_id=str(self._conversation.id),
error=str(e),
)
# Cancel any running subagent workflows for this conversation
await self._cancel_subagent_workflows(client)
# Cancel any running queued message workflows for this conversation
await self._cancel_queue_workflows(client)
# Clear the cache queue so no new queued workflows are spawned
queue_store = ConversationQueueStore(str(self._conversation.id))
await queue_store.clear_async()
await self._redis_stream.delete_stream()
finally:
self._conversation.status = Conversation.Status.IDLE
await self._conversation.asave(update_fields=["status", "updated_at"])
async def _cancel_subagent_workflows(self, client) -> None:
"""Cancel all running subagent workflows for this conversation.
Subagent workflows have IDs in the format: subagent-{conversation_id}-{tool_call_id}
Args:
client: Temporal client
"""
# Query for all running subagent workflows for this conversation
subagent_prefix = f"subagent-{self._conversation.id}-"
query = f'WorkflowId STARTS_WITH "{subagent_prefix}" AND ExecutionStatus = "Running"'
try:
async for workflow in client.list_workflows(query=query):
try:
subagent_handle = client.get_workflow_handle(workflow_id=workflow.id)
await subagent_handle.cancel()
except Exception as e:
# Log but don't fail if a single subagent cancellation fails
logger.warning(
"Failed to cancel subagent workflow",
workflow_id=workflow.id,
conversation_id=str(self._conversation.id),
error=str(e),
)
except Exception as e:
# Log but don't fail the main cancellation if listing subagents fails
logger.warning(
"Failed to list subagent workflows for cancellation",
conversation_id=str(self._conversation.id),
error=str(e),
)
async def _cancel_queue_workflows(self, client) -> None:
"""Cancel all running queued message workflows for this conversation."""
queue_prefix = f"conversation-{self._conversation.id}-queued-"
query = f'WorkflowId STARTS_WITH "{queue_prefix}" AND ExecutionStatus = "Running"'
try:
async for workflow in client.list_workflows(query=query):
try:
queue_handle = client.get_workflow_handle(workflow_id=workflow.id)
await queue_handle.cancel()
except Exception as e:
logger.warning(
"Failed to cancel queued workflow",
workflow_id=workflow.id,
conversation_id=str(self._conversation.id),
error=str(e),
)
except Exception as e:
logger.warning(
"Failed to list queued workflows for cancellation",
conversation_id=str(self._conversation.id),
error=str(e),
)