Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.7 KB
Raw Blame
import time
import asyncio
from collections.abc import AsyncGenerator
from typing import Any
from uuid import uuid4

from django.conf import settings

import structlog
import posthoganalytics
from opentelemetry import trace
from prometheus_client import Histogram
from temporalio.client import WorkflowExecutionStatus, WorkflowHandle
from temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy
from temporalio.exceptions import WorkflowAlreadyStartedError

from posthog.schema import AssistantEventType, FailureMessage

from posthog.temporal.ai.base import AgentBaseWorkflow
from posthog.temporal.common.client import async_connect

from products.posthog_ai.backend.models.assistant import Conversation

from ee.hogai.core.workflow_payload import get_workflow_input_size_estimates
from ee.hogai.queue import ConversationQueueStore
from ee.hogai.stream.redis_stream import (
    CONVERSATION_STREAM_MAX_LENGTH,
    CONVERSATION_STREAM_TIMEOUT,
    ApprovalEvent,
    ConversationEvent,
    ConversationRedisStream,
    GenerationStatusEvent,
    MessageEvent,
    StreamError,
    StreamEvent,
    UpdateEvent,
    get_conversation_stream_key,
)
from ee.hogai.utils.types import AssistantOutput

logger = structlog.get_logger(__name__)
_tracer = trace.get_tracer(__name__)

STREAM_DJANGO_EVENT_LOOP_LATENCY_HISTOGRAM = Histogram(
    "posthog_ai_stream_django_event_loop_latency_seconds",
    "Time from receiving chunk from Temporal to yielding it in Django event loop",
    buckets=[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, float("inf")],
)


class AgentExecutor:
    """Manages executing an agent workflow and streaming the output."""

    def __init__(
        self,
        conversation: Conversation,
        stream_key: str | None = None,
        timeout: int = CONVERSATION_STREAM_TIMEOUT,
        max_length: int = CONVERSATION_STREAM_MAX_LENGTH,
        reconnectable: bool = True,
    ) -> None:
        self._conversation = conversation
        if stream_key is None:
            stream_key = get_conversation_stream_key(conversation.id)
        self._redis_stream = ConversationRedisStream(stream_key, timeout=timeout, max_length=max_length)
        self._workflow_id = f"conversation-{conversation.id}"
        self._reconnectable = reconnectable

    async def astream(self, workflow: type[AgentBaseWorkflow], inputs: Any) -> AsyncGenerator[AssistantOutput, Any]:
        """Stream agent workflow updates from Redis stream.

        Args:
            workflow: Agent temporal workflow class
            inputs: Agent temporal workflow inputs

        Returns:
            AssistantOutput generator
        """
        # If this is a reconnection attempt, we resume streaming
        if self._conversation.status != Conversation.Status.IDLE and self._reconnectable:
            if hasattr(inputs, "message") and inputs.message is not None:
                raise ValueError("Cannot resume streaming with a new message")
            async for chunk in self.stream_conversation():
                yield chunk
        else:
            # Otherwise, process the new message (new generation) or resume generation (no new message)
            async for chunk in self.start_workflow(workflow, inputs):
                yield chunk

    async def start_workflow(
        self, workflow: type[AgentBaseWorkflow], inputs: Any
    ) -> AsyncGenerator[AssistantOutput, Any]:
        # Capture the failure outcome inside the span and yield outside it — yielding from
        # an async generator while a span is "current" leaks that span into the consumer
        # task's contextvars. See stream_conversation for the longer note.
        failed = False
        with _tracer.start_as_current_span(
            "posthog_ai.executor.start_workflow",
            attributes={
                "posthog_ai.conversation_id": str(self._conversation.id),
                "posthog_ai.workflow_id": self._workflow_id,
                "posthog_ai.workflow_class": workflow.__name__,
            },
        ):
            try:
                # Delete the stream to ensure we start fresh
                # since there might be a stale stream from a previous conversation gone wrong
                with _tracer.start_as_current_span("posthog_ai.executor.delete_stale_stream"):
                    await self._redis_stream.delete_stream()

                with _tracer.start_as_current_span("posthog_ai.executor.temporal_connect"):
                    client = await async_connect()

                handle = await self._start_workflow_with_retry(client, workflow, inputs)

                # Wait for the workflow to start running before streaming
                is_workflow_running = await self._wait_for_workflow_to_start(handle)
                if not is_workflow_running:
                    raise Exception(f"Workflow failed to start within timeout: {self._workflow_id}")

            except Exception as e:
                posthoganalytics.capture_exception(e, properties={"tag": "max_ai"})
                logger.exception(
                    "Error starting workflow",
                    error=e,
                    workflow_input_estimated_bytes=get_workflow_input_size_estimates(inputs),
                )
                failed = True

        if failed:
            yield self._failure_message()
            return

        async for chunk in self.stream_conversation():
            yield chunk

    async def _start_workflow_with_retry(
        self, client: Any, workflow: type[AgentBaseWorkflow], inputs: Any, max_retries: int = 3
    ) -> WorkflowHandle:
        """Start a workflow with retry logic for handling race conditions.

        When a workflow is in the process of completing, there can be a brief window
        where starting a new workflow with the same ID fails even with USE_EXISTING policy.
        This method handles that by waiting for the existing workflow to complete and retrying.
        """
        span = trace.get_current_span()
        for attempt in range(max_retries):
            try:
                with _tracer.start_as_current_span(
                    "posthog_ai.executor.temporal_start_workflow",
                    attributes={"posthog_ai.attempt": attempt},
                ):
                    handle = await client.start_workflow(
                        workflow.run,
                        inputs,
                        id=self._workflow_id,
                        task_queue=settings.MAX_AI_TASK_QUEUE,
                        id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
                        id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
                    )
                span.set_attribute("posthog_ai.start_workflow.attempts", attempt + 1)
                return handle
            except WorkflowAlreadyStartedError:
                if attempt < max_retries - 1:
                    # Get handle to existing workflow and wait for it to complete
                    logger.info(
                        "Workflow already started, waiting for completion",
                        workflow_id=self._workflow_id,
                        attempt=attempt + 1,
                    )
                    existing_handle = client.get_workflow_handle(workflow_id=self._workflow_id)
                    try:
                        # Wait for the existing workflow to complete (with timeout)
                        await asyncio.wait_for(existing_handle.result(), timeout=5.0)
                    except TimeoutError:
                        # If timeout, the workflow is still running - return the existing handle
                        logger.info(
                            "Existing workflow still running, using existing handle",
                            workflow_id=self._workflow_id,
                        )
                        span.set_attribute("posthog_ai.start_workflow.attempts", attempt + 1)
                        return existing_handle
                    except Exception:
                        # Workflow completed (possibly with error), retry starting a new one
                        pass
                    # Small delay before retry
                    await asyncio.sleep(0.1)
                else:
                    raise
        # This should be unreachable, but mypy needs it
        raise Exception(f"Failed to start workflow after {max_retries} attempts")

    async def _wait_for_workflow_to_start(self, handle: WorkflowHandle) -> bool:
        """Wait for the workflow to start running.

        Args:
            handle: Temporal workflow handle

        Returns:
            True if workflow started running, False otherwise
        """
        max_attempts = 10 * 60  # 60 seconds total with 0.1s sleep
        attempts = 0

        with _tracer.start_as_current_span("posthog_ai.executor.wait_for_workflow_start") as span:
            while attempts < max_attempts:
                description = await handle.describe()
                if description.status is None:
                    attempts += 1
                    await asyncio.sleep(0.1)
                elif description.status == WorkflowExecutionStatus.RUNNING:
                    # Temporal only has one Open execution status, see: https://docs.temporal.io/workflow-execution
                    span.set_attribute("posthog_ai.poll_attempts", attempts)
                    span.set_attribute("posthog_ai.outcome", "running")
                    return True
                else:
                    span.set_attribute("posthog_ai.poll_attempts", attempts)
                    span.set_attribute("posthog_ai.outcome", "not_running")
                    return False

            span.set_attribute("posthog_ai.poll_attempts", attempts)
            span.set_attribute("posthog_ai.outcome", "timeout")
            return False

    async def stream_conversation(self) -> AsyncGenerator[AssistantOutput, Any]:
        """Stream conversation updates from Redis stream.

        Returns:
            AssistantOutput generator
        """
        chunk_count = 0
        # Use start_span + use_span (without making the span "current" across yields).
        # `start_as_current_span` inside an async generator attaches the span to the running
        # task's contextvars; when the generator yields, the consumer task resumes with that
        # context still set, leaking this span as the consumer's "current span". We scope
        # the current-span context only around the non-yielding bookkeeping so child spans
        # still parent correctly, while yields run outside the attached context.
        # See https://github.com/open-telemetry/opentelemetry-python/issues/2606.
        span = _tracer.start_span(
            "posthog_ai.executor.stream_conversation",
            attributes={"posthog_ai.conversation_id": str(self._conversation.id)},
        )
        try:
            try:
                with trace.use_span(span, end_on_exit=False):
                    # Wait for stream to be created
                    is_stream_available = await self._redis_stream.wait_for_stream()
                    if not is_stream_available:
                        raise StreamError(
                            "Stream for this conversation not available - Temporal workflow might have failed"
                        )
                    last_chunk_time = time.time()
                read_iter = self._redis_stream.read_stream().__aiter__()
                while True:
                    with trace.use_span(span, end_on_exit=False):
                        try:
                            chunk = await read_iter.__anext__()
                        except StopAsyncIteration:
                            break
                        message = await self._redis_stream_to_assistant_output(chunk)

                        temporal_to_code_latency = last_chunk_time - chunk.timestamp
                        if temporal_to_code_latency > 0:
                            STREAM_DJANGO_EVENT_LOOP_LATENCY_HISTOGRAM.observe(temporal_to_code_latency)
                        last_chunk_time = time.time()
                        chunk_count += 1

                    if message:
                        yield message
            except Exception as e:
                with trace.use_span(span, end_on_exit=False):
                    posthoganalytics.capture_exception(e, properties={"tag": "max_ai"})
                    logger.exception("Error streaming conversation", error=e)
                yield self._failure_message()
        finally:
            with trace.use_span(span, end_on_exit=False):
                span.set_attribute("posthog_ai.stream_conversation.chunks", chunk_count)
                await self._redis_stream.delete_stream()
            span.end()

    async def _redis_stream_to_assistant_output(self, message: StreamEvent) -> AssistantOutput | None:
        """Convert Redis stream event to Assistant output.

        Args:
            message: event from Redis stream

        Returns:
            AssistantOutput or None
        """
        if isinstance(message.event, MessageEvent):
            return (AssistantEventType.MESSAGE, message.event.payload)
        elif isinstance(message.event, ConversationEvent):
            with _tracer.start_as_current_span(
                "posthog_ai.executor.conversation_aget",
                attributes={"posthog_ai.conversation_id": str(message.event.payload)},
            ):
                # nosemgrep: idor-lookup-without-team (ID from internal Redis stream, caller validates user+team ownership)
                conversation = await Conversation.objects.select_related("user").aget(id=message.event.payload)
            return (AssistantEventType.CONVERSATION, conversation)
        elif isinstance(message.event, UpdateEvent):
            return (AssistantEventType.UPDATE, message.event.payload)
        elif isinstance(message.event, GenerationStatusEvent):
            return (AssistantEventType.STATUS, message.event.payload)
        elif isinstance(message.event, ApprovalEvent):
            return (AssistantEventType.APPROVAL, message.event.payload)
        else:
            return None

    def _failure_message(self) -> AssistantOutput:
        """Returns a failure message as an Assistant output."""
        failure_message = FailureMessage(
            content="Oops! Something went wrong. Please try again.",
            id=str(uuid4()),
        )
        return (AssistantEventType.MESSAGE, failure_message)

    async def cancel_workflow(self) -> None:
        """Cancel the current conversation and clean up resources.

        This cancels the main conversation workflow, any running subagent workflows,
        any queued message workflows, and clears the cache queue.

        The main workflow cancel is non-fatal because the main workflow may have already
        completed after spawning a queued workflow. In that case, we still need to cancel
        the queued workflows and clear the cache queue.
        """
        self._conversation.status = Conversation.Status.CANCELING
        await self._conversation.asave(update_fields=["status", "updated_at"])

        try:
            client = await async_connect()

            # Cancel the main conversation workflow.
            # This may fail if the workflow already completed (e.g. after spawning a queued workflow),
            # but we must continue to cancel queued workflows and clean up.
            try:
                handle = client.get_workflow_handle(workflow_id=self._workflow_id)
                await handle.cancel()
            except Exception as e:
                logger.warning(
                    "Failed to cancel main workflow (may have already completed)",
                    workflow_id=self._workflow_id,
                    conversation_id=str(self._conversation.id),
                    error=str(e),
                )

            # Cancel any running subagent workflows for this conversation
            await self._cancel_subagent_workflows(client)
            # Cancel any running queued message workflows for this conversation
            await self._cancel_queue_workflows(client)

            # Clear the cache queue so no new queued workflows are spawned
            queue_store = ConversationQueueStore(str(self._conversation.id))
            await queue_store.clear_async()

            await self._redis_stream.delete_stream()
        finally:
            self._conversation.status = Conversation.Status.IDLE
            await self._conversation.asave(update_fields=["status", "updated_at"])

    async def _cancel_subagent_workflows(self, client) -> None:
        """Cancel all running subagent workflows for this conversation.

        Subagent workflows have IDs in the format: subagent-{conversation_id}-{tool_call_id}

        Args:
            client: Temporal client
        """
        # Query for all running subagent workflows for this conversation
        subagent_prefix = f"subagent-{self._conversation.id}-"
        query = f'WorkflowId STARTS_WITH "{subagent_prefix}" AND ExecutionStatus = "Running"'

        try:
            async for workflow in client.list_workflows(query=query):
                try:
                    subagent_handle = client.get_workflow_handle(workflow_id=workflow.id)
                    await subagent_handle.cancel()
                except Exception as e:
                    # Log but don't fail if a single subagent cancellation fails
                    logger.warning(
                        "Failed to cancel subagent workflow",
                        workflow_id=workflow.id,
                        conversation_id=str(self._conversation.id),
                        error=str(e),
                    )
        except Exception as e:
            # Log but don't fail the main cancellation if listing subagents fails
            logger.warning(
                "Failed to list subagent workflows for cancellation",
                conversation_id=str(self._conversation.id),
                error=str(e),
            )

    async def _cancel_queue_workflows(self, client) -> None:
        """Cancel all running queued message workflows for this conversation."""
        queue_prefix = f"conversation-{self._conversation.id}-queued-"
        query = f'WorkflowId STARTS_WITH "{queue_prefix}" AND ExecutionStatus = "Running"'

        try:
            async for workflow in client.list_workflows(query=query):
                try:
                    queue_handle = client.get_workflow_handle(workflow_id=workflow.id)
                    await queue_handle.cancel()
                except Exception as e:
                    logger.warning(
                        "Failed to cancel queued workflow",
                        workflow_id=workflow.id,
                        conversation_id=str(self._conversation.id),
                        error=str(e),
                    )
        except Exception as e:
            logger.warning(
                "Failed to list queued workflows for cancellation",
                conversation_id=str(self._conversation.id),
                error=str(e),
            )