Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(posthog-ai): suggest the next reply in the composer (#130748)

Shy Alter583b2d41cc5d
diff --git a/frontend/src/scenes/max/components/QuestionInput.test.tsx b/frontend/src/scenes/max/components/QuestionInput.test.tsx
index 1b322ec42f1b..4058a9ff1cf0 100644
--- a/frontend/src/scenes/max/components/QuestionInput.test.tsx
+++ b/frontend/src/scenes/max/components/QuestionInput.test.tsx
@@ -10,7 +10,7 @@ import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
-import { composerSeedLogic } from 'products/posthog_ai/frontend/api/logics'
+import { composerSeedLogic, runStreamLogic } from 'products/posthog_ai/frontend/api/logics'
import { MAX_MESSAGE_LENGTH } from '../max-constants'
import { maxGlobalLogic } from '../maxGlobalLogic'
@@ -140,6 +140,51 @@ describe.each([
await waitFor(() => expect(slashCommandItem()).toBeInTheDocument())
})
+ describe('reply suggestions', () => {
+ const suggestion = 'Yes, open the pull request with the fix'
+ const hint = (): Element | null => document.querySelector('[data-attr="max-reply-suggestion"]')
+
+ beforeEach(() => {
+ const stream = runStreamLogic({ streamKey: maxLogicInstance.values.frontendConversationId })
+ act(() => {
+ stream.actions.pushHumanMessage('Why does the relay keep retrying?')
+ stream.actions.markTurnComplete()
+ stream.actions.setReplySuggestions({
+ turnIndex: 0,
+ replies: [suggestion],
+ })
+ })
+ })
+
+ it.each(['{Tab}', '{ArrowRight}'])('puts the suggestion in the input on %s without sending it', async (key) => {
+ const askMaxSpy = jest.spyOn(threadLogicInstance.actions, 'askMax')
+ const user = userEvent.setup()
+ const input = screen.getByRole('textbox') as HTMLTextAreaElement
+ await waitFor(() => expect(hint()).toHaveTextContent(suggestion))
+
+ input.focus()
+ await user.keyboard(key)
+
+ expect(input.value).toBe(suggestion)
+ expect(document.activeElement).toBe(input)
+ expect(hint()).not.toBeInTheDocument()
+ expect(askMaxSpy).not.toHaveBeenCalled()
+ })
+
+ it('hides the suggestion on Escape and leaves Tab to move focus', async () => {
+ const user = userEvent.setup()
+ const input = screen.getByRole('textbox') as HTMLTextAreaElement
+ await waitFor(() => expect(hint()).toBeInTheDocument())
+
+ input.focus()
+ await user.keyboard('{Escape}')
+ expect(hint()).not.toBeInTheDocument()
+
+ await user.keyboard('{Tab}')
+ expect(input.value).toBe('')
+ })
+ })
+
describe('message length limit', () => {
const sendButton = (): HTMLElement | null => document.querySelector('[data-attr="max-send-message"]')
diff --git a/frontend/src/scenes/max/components/QuestionInput.tsx b/frontend/src/scenes/max/components/QuestionInput.tsx
index bfe4bd7beaf1..5f7f7b0bd132 100644
--- a/frontend/src/scenes/max/components/QuestionInput.tsx
+++ b/frontend/src/scenes/max/components/QuestionInput.tsx
@@ -11,6 +11,8 @@ import { KeyboardShortcut } from 'lib/lemon-ui/KeyboardShortcut/KeyboardShortcut
import { cn } from 'lib/utils/css-classes'
import { AIConsentPopoverWrapper } from 'scenes/settings/organization/AIConsentPopoverWrapper'
+import { ReplySuggestionHint } from 'products/posthog_ai/frontend/api/composer'
+
import { ContextDisplay } from '../Context'
import { maxThreadLogic } from '../maxThreadLogic'
import {
@@ -134,6 +136,7 @@ export const QuestionInput = React.forwardRef<HTMLDivElement, QuestionInputProps
hasQuestion,
fillInHint,
showFillInHint,
+ replySuggestion,
promptLength,
isOverLengthLimit,
maxMessageLength,
@@ -241,7 +244,19 @@ export const QuestionInput = React.forwardRef<HTMLDivElement, QuestionInputProps
>
<SlashCommandAutocomplete visible={showAutocomplete} onClose={closeAutocomplete}>
<div className="relative w-full">
- {!inputValue && (
+ {replySuggestion !== null ? (
+ <div
+ id="textarea-hint"
+ className="text-tertiary absolute top-4 left-4 right-12 pointer-events-none"
+ >
+ <ReplySuggestionHint
+ suggestion={replySuggestion}
+ acceptHint={
+ <kbd className="KeyboardShortcut text-secondary">Tab to accept</kbd>
+ }
+ />
+ </div>
+ ) : !inputValue ? (
<div
id="textarea-hint"
className="text-secondary absolute top-4 left-4 text-sm pointer-events-none"
@@ -257,7 +272,7 @@ export const QuestionInput = React.forwardRef<HTMLDivElement, QuestionInputProps
placeholderCopy.text
)}
</div>
- )}
+ ) : null}
{/* Postfix cue after a fill-in suggestion's typed-in prefix. */}
{showFillInHint && (
<div className="absolute top-4 left-4 right-4 overflow-hidden pointer-events-none">
diff --git a/frontend/src/scenes/max/components/QuillQuestionInput.tsx b/frontend/src/scenes/max/components/QuillQuestionInput.tsx
index fd2b2969828e..11c17c3c01ed 100644
--- a/frontend/src/scenes/max/components/QuillQuestionInput.tsx
+++ b/frontend/src/scenes/max/components/QuillQuestionInput.tsx
@@ -4,12 +4,12 @@ import React, { ReactNode, useEffect, useRef, useState } from 'react'
import { IconArrowRight, IconCheck, IconPencil, IconPlus, IconStopFilled, IconTrash, IconX } from '@posthog/icons'
import { LemonTextArea } from '@posthog/lemon-ui'
-import { Button, Spinner, Switch, Text } from '@posthog/quill'
+import { Button, Kbd, Spinner, Switch, Text } from '@posthog/quill'
import { cn } from 'lib/utils/css-classes'
import { AIConsentPopoverWrapper } from 'scenes/settings/organization/AIConsentPopoverWrapper'
-import { QuillComposerLayout } from 'products/posthog_ai/frontend/api/composer'
+import { QuillComposerLayout, ReplySuggestionHint } from 'products/posthog_ai/frontend/api/composer'
import { ContextTags, ContextToolInfoTags, SandboxContextTags } from '../Context'
import { maxThreadLogic } from '../maxThreadLogic'
@@ -159,6 +159,7 @@ export const QuillQuestionInput = React.forwardRef<HTMLDivElement, QuillQuestion
handleQuestionChange,
fillInHint,
showFillInHint,
+ replySuggestion,
promptLength,
isOverLengthLimit,
maxMessageLength,
@@ -287,7 +288,18 @@ export const QuillQuestionInput = React.forwardRef<HTMLDivElement, QuillQuestion
chips={!isSharedThread && showContextUI ? <QuillContextChips /> : undefined}
field={
<>
- {!inputValue && (
+ {replySuggestion !== null ? (
+ <div
+ data-slot="composer-placeholder"
+ id="question-input-hint"
+ className="pointer-events-none absolute right-0 text-muted-foreground"
+ >
+ <ReplySuggestionHint
+ suggestion={replySuggestion}
+ acceptHint={<Kbd>Tab to accept</Kbd>}
+ />
+ </div>
+ ) : !inputValue ? (
<div
data-slot="composer-placeholder"
id="question-input-hint"
@@ -295,7 +307,7 @@ export const QuillQuestionInput = React.forwardRef<HTMLDivElement, QuillQuestion
>
{placeholderText}
</div>
- )}
+ ) : null}
{/* Postfix cue after a fill-in suggestion's typed-in prefix. */}
{showFillInHint && fillInHint && (
<div className="pointer-events-none absolute inset-x-0 top-2 overflow-hidden">
diff --git a/frontend/src/scenes/max/components/useQuestionInput.ts b/frontend/src/scenes/max/components/useQuestionInput.ts
index 60654fb9b815..d9101b29b8f2 100644
--- a/frontend/src/scenes/max/components/useQuestionInput.ts
+++ b/frontend/src/scenes/max/components/useQuestionInput.ts
@@ -27,6 +27,7 @@ export interface QuestionInputController {
hasQuestion: boolean
fillInHint: string | null
showFillInHint: boolean
+ replySuggestion: string | null
promptLength: number | null
isOverLengthLimit: boolean
maxMessageLength: number
@@ -106,6 +107,7 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
queueingEnabled,
queuedMessages,
queueSubmitting,
+ sandboxReplySuggestions,
} = useValues(maxThreadLogic)
const {
askMax,
@@ -123,6 +125,8 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
// user keeps typing a message that still starts with "/". "/" + Esc is a valid path.
const [autocompleteDismissed, setAutocompleteDismissed] = useState(false)
const [editingQueueId, setEditingQueueId] = useState<string | null>(null)
+ const [dismissedReplySuggestion, setDismissedReplySuggestion] = useState<string | null>(null)
+ const acceptedReplySuggestion = useRef<string | null>(null)
const displayQueuedMessages = useMemo(() => [...queuedMessages].reverse(), [queuedMessages])
// Hold the textarea value in local state so each keystroke is an isolated, cheap re-render
@@ -171,6 +175,13 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
// it rather than the prefix that happens to be on screen, and stop the animation writing
// the rest into the composer after the message has gone.
const content = typingSuggestion ?? prompt
+ if (acceptedReplySuggestion.current !== null) {
+ posthog.capture('posthog ai reply suggestion sent', {
+ conversation_id: conversation?.id,
+ edited: content.trim() !== acceptedReplySuggestion.current,
+ })
+ acceptedReplySuggestion.current = null
+ }
if (typingSuggestion) {
cancelSuggestionTyping()
}
@@ -191,6 +202,23 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
const hasQuestion = inputValue.trim().length > 0
// A fill-in suggestion typed its prefix in and is waiting for the user to complete it.
const showFillInHint = !!fillInHint
+ const replySuggestion =
+ !inputValue && !threadLoading && !isSharedThread && !showFillInHint && sandboxReplySuggestions
+ ? `${conversation?.id}:${sandboxReplySuggestions.turnIndex}` !== dismissedReplySuggestion
+ ? (sandboxReplySuggestions.replies[0] ?? null)
+ : null
+ : null
+ const replySuggestionTurn = replySuggestion !== null ? sandboxReplySuggestions?.turnIndex : undefined
+
+ useEffect(() => {
+ if (replySuggestionTurn !== undefined) {
+ posthog.capture('posthog ai reply suggestion shown', {
+ conversation_id: conversation?.id,
+ turn_index: replySuggestionTurn,
+ suggestion_count: sandboxReplySuggestions?.replies.length,
+ })
+ }
+ }, [replySuggestionTurn]) // oxlint-disable-line react-hooks/exhaustive-deps
const isQueueingSubmission = queueingEnabled && threadLoading && hasQuestion
const showStopButton = threadLoading && !isQueueingSubmission && !cancelLoading
@@ -264,6 +292,30 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
}
const handleKeyDown = (event: React.KeyboardEvent<HTMLTextAreaElement>): void => {
+ if (replySuggestion !== null && sandboxReplySuggestions) {
+ const noModifiers = !event.shiftKey && !event.altKey && !event.metaKey && !event.ctrlKey
+ if (noModifiers && (event.key === 'Tab' || event.key === 'ArrowRight')) {
+ event.preventDefault()
+ acceptedReplySuggestion.current = replySuggestion
+ setDismissedReplySuggestion(`${conversation?.id}:${sandboxReplySuggestions.turnIndex}`)
+ handleQuestionChange(replySuggestion)
+ posthog.capture('posthog ai reply suggestion accepted', {
+ conversation_id: conversation?.id,
+ turn_index: sandboxReplySuggestions.turnIndex,
+ key: event.key,
+ })
+ return
+ }
+ if (event.key === 'Escape') {
+ event.preventDefault()
+ setDismissedReplySuggestion(`${conversation?.id}:${sandboxReplySuggestions.turnIndex}`)
+ posthog.capture('posthog ai reply suggestion dismissed', {
+ conversation_id: conversation?.id,
+ turn_index: sandboxReplySuggestions.turnIndex,
+ })
+ return
+ }
+ }
if (event.key === 'ArrowUp' && !inputValue.trim() && queuedMessages.length > 0 && !editingQueueId) {
const target = event.currentTarget
const atStart = target.selectionStart === 0 && target.selectionEnd === 0
@@ -320,6 +372,7 @@ export function useQuestionInput({ textAreaRef, onSubmit }: UseQuestionInputOpti
hasQuestion,
fillInHint,
showFillInHint,
+ replySuggestion,
promptLength,
isOverLengthLimit,
maxMessageLength: MAX_MESSAGE_LENGTH,
diff --git a/frontend/src/scenes/max/maxThreadLogic.tsx b/frontend/src/scenes/max/maxThreadLogic.tsx
index 003d9fb34e2e..ffae2675bcee 100644
--- a/frontend/src/scenes/max/maxThreadLogic.tsx
+++ b/frontend/src/scenes/max/maxThreadLogic.tsx
@@ -76,6 +76,7 @@ import {
runStreamLogic,
WebSearchResultsByToolUseId,
} from 'products/posthog_ai/frontend/api/logics'
+import type { ReplySuggestions } from 'products/posthog_ai/frontend/api/types'
import { LogEntry, parseLogEvent } from 'products/posthog_ai/frontend/lib/parse-logs'
import { isPiTaskRuntime } from 'products/posthog_ai/frontend/types/taskTypes'
@@ -185,6 +186,7 @@ export interface maxThreadLogicValues {
sandboxAttachments: AttachedContext[] // posthogAiContextLogic
pendingSandboxPermissionRequest: PermissionRequestRecord | null // runStreamLogic
sandboxCurrentMode: string | null // runStreamLogic
+ sandboxReplySuggestions: ReplySuggestions | null // runStreamLogic
sceneId: string | null // sceneLogic
activeDangerousOperationApproval: {
payload: Record<string, any>
@@ -805,7 +807,11 @@ export const maxThreadLogic = kea<maxThreadLogicType>([
// BindLogic subtree (the input area renders for LangGraph conversations too, so they
// can't bind the keyed stream logic themselves).
runStreamLogic({ streamKey: conversationId, conversationId }),
- ['pendingPermissionRequest as pendingSandboxPermissionRequest', 'currentMode as sandboxCurrentMode'],
+ [
+ 'pendingPermissionRequest as pendingSandboxPermissionRequest',
+ 'currentMode as sandboxCurrentMode',
+ 'replySuggestions as sandboxReplySuggestions',
+ ],
],
actions: [
maxLogic({ panelId }),
diff --git a/products/posthog_ai/backend/receivers.py b/products/posthog_ai/backend/receivers.py
index af6fbbf3f0b1..9014c30dfb12 100644
--- a/products/posthog_ai/backend/receivers.py
+++ b/products/posthog_ai/backend/receivers.py
@@ -10,7 +10,8 @@ Every hook connects through the tasks facade so no model class crosses the produ
path too, so a chat can never fork back into LangGraph after it has been continued as a task.
- When tasks are read: a user without the sandbox runtime does not see the copies of their chats. The
copy is written for everyone so the switch is instant later, but until then their chats are chats.
-- After a turn completes: the end-of-turn suggestion is queued (``turn_suggestions/dispatch.py``).
+- After a turn completes: the end-of-turn suggestion is queued (``turn_suggestions/dispatch.py``), and so
+ are the suggested replies the composer offers (``reply_suggestions/dispatch.py``).
"""
import logging
@@ -23,6 +24,7 @@ from asgiref.sync import async_to_sync
from posthog.models import Team, User
from products.posthog_ai.backend.models.assistant import Conversation
+from products.posthog_ai.backend.reply_suggestions.dispatch import enqueue_reply_suggestions_on_turn_completed
from products.posthog_ai.backend.turn_suggestions.dispatch import enqueue_turn_suggestion_on_turn_completed
from products.tasks.backend.facade.task_run_signals import (
TaskOriginProduct,
@@ -64,6 +66,9 @@ def connect() -> None:
connect_task_run_turn_completed(
enqueue_turn_suggestion_on_turn_completed, dispatch_uid="posthog_ai_turn_suggestion_on_turn_completed"
)
+ connect_task_run_turn_completed(
+ enqueue_reply_suggestions_on_turn_completed, dispatch_uid="posthog_ai_reply_suggestions_on_turn_completed"
+ )
def copied_chats_hidden_without_sandbox_mode(team_id: int, user_id: int | None) -> Iterable[UUID]:
diff --git a/products/posthog_ai/backend/reply_suggestions/__init__.py b/products/posthog_ai/backend/reply_suggestions/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/posthog_ai/backend/reply_suggestions/dispatch.py b/products/posthog_ai/backend/reply_suggestions/dispatch.py
new file mode 100644
index 000000000000..ed8662695f21
--- /dev/null
+++ b/products/posthog_ai/backend/reply_suggestions/dispatch.py
@@ -0,0 +1,43 @@
+from typing import TYPE_CHECKING, Any
+
+from django.core.cache import cache
+
+import structlog
+
+from products.tasks.backend.facade.task_run_signals import TaskOriginProduct
+
+if TYPE_CHECKING:
+ from products.tasks.backend.models import TaskRun
+
+logger = structlog.get_logger(__name__)
+
+ENQUEUE_DEDUP_SECONDS = 1
+REPLY_SUGGESTIONS_EXPIRES_SECONDS = 30
+
+
+def enqueue_reply_suggestions(task_run: "TaskRun") -> bool:
+ try:
+ if task_run.origin_product != TaskOriginProduct.POSTHOG_AI:
+ return False
+ run_id = str(task_run.id)
+ reservation = f"posthog_ai_reply_suggestions_enqueue:{run_id}"
+ if not cache.add(reservation, True, timeout=ENQUEUE_DEDUP_SECONDS):
+ return False
+ from products.posthog_ai.backend.tasks import generate_reply_suggestions_task # noqa: PLC0415
+
+ try:
+ generate_reply_suggestions_task.apply_async(
+ kwargs={"run_id": run_id, "team_id": task_run.team_id},
+ expires=REPLY_SUGGESTIONS_EXPIRES_SECONDS,
+ )
+ except Exception:
+ cache.delete(reservation)
+ raise
+ except Exception:
+ logger.warning("posthog_ai_reply_suggestions_enqueue_failed", run_id=str(task_run.id), exc_info=True)
+ return False
+ return True
+
+
+def enqueue_reply_suggestions_on_turn_completed(sender: type, task_run: "TaskRun", **kwargs: Any) -> None:
+ enqueue_reply_suggestions(task_run)
diff --git a/products/posthog_ai/backend/reply_suggestions/service.py b/products/posthog_ai/backend/reply_suggestions/service.py
new file mode 100644
index 000000000000..4bfd344a7ba7
--- /dev/null
+++ b/products/posthog_ai/backend/reply_suggestions/service.py
@@ -0,0 +1,95 @@
+from typing import Literal
+
+from django.core.cache import cache
+
+import structlog
+
+from posthog.dataclasses import frozen
+
+from products.posthog_ai.backend.turn_suggestions.service import (
+ MAX_TRANSCRIPT_LOG_BYTES,
+ _history_is_whole,
+ load_web_conversation_run,
+)
+from products.posthog_ai.backend.turn_suggestions.transcript import build_turn_transcript
+from products.tasks.backend.facade.api import publish_task_run_stream_notification, read_task_run_history
+from products.tasks.backend.facade.reply_suggestions import ReplySuggestionAttribution, suggest_replies
+
+logger = structlog.get_logger(__name__)
+
+REPLY_SUGGESTIONS_FLAG = "posthog-ai-reply-suggestions"
+REPLY_SUGGESTIONS_METHOD = "_posthog/reply_suggestions"
+REPLY_SUGGESTIONS_AI_PRODUCT = "posthog_ai"
+REPLY_SUGGESTIONS_FEATURE = "reply_suggestions"
+REPLY_SUGGESTIONS_REASONING_EFFORT = "none"
+TURN_CLAIM_SECONDS = 60 * 60
+
+OutcomeStatus = Literal["emitted", "skipped", "failed"]
+
+
+@frozen
+class ReplySuggestionsOutcome:
+ status: OutcomeStatus
+ reason: str
+
+
+def _skipped(reason: str) -> ReplySuggestionsOutcome:
+ return ReplySuggestionsOutcome(status="skipped", reason=reason)
+
+
+def generate_reply_suggestions(run_id: str, team_id: int) -> ReplySuggestionsOutcome:
+ loaded = load_web_conversation_run(run_id, team_id, flag=REPLY_SUGGESTIONS_FLAG)
+ if isinstance(loaded, str):
+ return _skipped(loaded)
+ task_run, user = loaded.task_run, loaded.user
+ task = task_run.task
+ if not _history_is_whole(task_run):
+ return _skipped("history_truncated")
+
+ entries = read_task_run_history(task_run.id, task_run.task_id, task_run.team_id, max_bytes=MAX_TRANSCRIPT_LOG_BYTES)
+ if entries is None:
+ return _skipped("log_too_large")
+ transcript = build_turn_transcript(entries, assistant_text_limit=None)
+ if not transcript.human_messages:
+ return _skipped("no_user_message")
+ if not transcript.latest_turn_complete:
+ return _skipped("turn_in_progress")
+ if not transcript.assistant_text.strip():
+ return _skipped("no_answer")
+ turn_index = len(transcript.human_messages) - 1
+ if not cache.add(f"posthog_ai_reply_suggestions:{task.id}:{turn_index}", True, timeout=TURN_CLAIM_SECONDS):
+ return _skipped("turn_already_claimed")
+
+ from ee.hogai.utils.feature_flags import is_privacy_mode_enabled # noqa: PLC0415
+
+ suggestions = suggest_replies(
+ transcript.assistant_text,
+ ReplySuggestionAttribution(
+ ai_product=REPLY_SUGGESTIONS_AI_PRODUCT,
+ properties={"posthog_ai_feature": REPLY_SUGGESTIONS_FEATURE},
+ session_id=str(task.id),
+ distinct_id=str(user.distinct_id),
+ task_id=str(task.id),
+ run_id=str(task_run.id),
+ privacy_mode=is_privacy_mode_enabled(task_run.team),
+ ),
+ transcript.last_human_message,
+ reasoning_effort=REPLY_SUGGESTIONS_REASONING_EFFORT,
+ )
+ if not suggestions:
+ return _skipped("nothing_to_suggest")
+
+ delivery = publish_task_run_stream_notification(
+ task_run.id,
+ task_run.task_id,
+ task_run.team_id,
+ REPLY_SUGGESTIONS_METHOD,
+ {
+ "turnIndex": turn_index,
+ "suggestions": [{"label": s.label, "reply": s.reply} for s in suggestions],
+ },
+ persist=False,
+ )
+ if not delivery.live:
+ return ReplySuggestionsOutcome(status="failed", reason="publish_failed")
+ return ReplySuggestionsOutcome(status="emitted", reason=str(len(suggestions)))
diff --git a/products/posthog_ai/backend/tasks.py b/products/posthog_ai/backend/tasks.py
index 0809fea530af..72cf1dd3d025 100644
--- a/products/posthog_ai/backend/tasks.py
+++ b/products/posthog_ai/backend/tasks.py
@@ -1,9 +1,10 @@
import structlog
-from celery import shared_task
+from celery import Task, shared_task
from prometheus_client import Counter
from posthog.celery_queues import CeleryQueue
+from products.posthog_ai.backend.reply_suggestions.service import generate_reply_suggestions
from products.posthog_ai.backend.turn_suggestions.service import generate_turn_suggestion
logger = structlog.get_logger(__name__)
@@ -27,3 +28,37 @@ def generate_turn_suggestion_task(*, run_id: str, team_id: int) -> None:
status=outcome.status,
reason=outcome.reason,
)
+
+
+REPLY_SUGGESTIONS_OUTCOMES_TOTAL = Counter(
+ "posthog_ai_reply_suggestions_outcomes_total",
+ "End-of-turn reply suggestion jobs by outcome",
+ labelnames=["status", "reason"],
+)
+
+
+REPLY_SUGGESTIONS_SETTLE_RETRIES = 6
+REPLY_SUGGESTIONS_SETTLE_RETRY_SECONDS = 0.5
+
+
+@shared_task(
+ bind=True,
+ ignore_result=True,
+ queue=CeleryQueue.POSTHOG_AI.value,
+ soft_time_limit=45,
+ time_limit=60,
+ max_retries=REPLY_SUGGESTIONS_SETTLE_RETRIES,
+)
+def generate_reply_suggestions_task(self: Task, *, run_id: str, team_id: int) -> None:
+ outcome = generate_reply_suggestions(run_id, team_id)
+ if outcome.reason == "turn_in_progress" and self.request.retries < REPLY_SUGGESTIONS_SETTLE_RETRIES:
+ raise self.retry(countdown=REPLY_SUGGESTIONS_SETTLE_RETRY_SECONDS)
+ reason = outcome.reason if outcome.status != "emitted" else "emitted"
+ REPLY_SUGGESTIONS_OUTCOMES_TOTAL.labels(status=outcome.status, reason=reason).inc()
+ logger.info(
+ "posthog_ai_reply_suggestions_outcome",
+ run_id=run_id,
+ team_id=team_id,
+ status=outcome.status,
+ reason=outcome.reason,
+ )
diff --git a/products/posthog_ai/backend/tests/test_reply_suggestions_service.py b/products/posthog_ai/backend/tests/test_reply_suggestions_service.py
new file mode 100644
index 000000000000..6e15788cba68
--- /dev/null
+++ b/products/posthog_ai/backend/tests/test_reply_suggestions_service.py
@@ -0,0 +1,186 @@
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.core.cache import cache
+
+from celery.exceptions import Retry
+from parameterized import parameterized
+
+from products.posthog_ai.backend.reply_suggestions.dispatch import enqueue_reply_suggestions
+from products.posthog_ai.backend.reply_suggestions.service import (
+ REPLY_SUGGESTIONS_METHOD,
+ ReplySuggestionsOutcome,
+ generate_reply_suggestions,
+)
+from products.posthog_ai.backend.tasks import generate_reply_suggestions_task
+from products.tasks.backend.facade.api import TaskClientProvenance
+from products.tasks.backend.facade.contracts import StreamNotificationDelivery
+from products.tasks.backend.facade.reply_suggestions import SuggestedReply
+from products.tasks.backend.models import Task
+
+SERVICE = "products.posthog_ai.backend.reply_suggestions.service"
+OPEN_PR = SuggestedReply(label="Yes, open the PR", reply="Yes, open the pull request with the fix")
+
+
+def _notification(method: str, params: dict) -> dict:
+ return {"type": "notification", "notification": {"method": method, "params": params}}
+
+
+def _user_message(text: str) -> dict:
+ return _notification("_posthog/user_message", {"content": text})
+
+
+def _agent_final(text: str) -> dict:
+ return _notification(
+ "session/update",
+ {"update": {"sessionUpdate": "agent_message", "messageId": "m1", "content": {"type": "text", "text": text}}},
+ )
+
+
+def _turn_complete() -> dict:
+ return _notification("_posthog/turn_complete", {"stopReason": "end_turn"})
+
+
+TWO_TURNS = [
+ _user_message("How many signups last week?"),
+ _agent_final("There were 120 signups."),
+ _turn_complete(),
+ _user_message("Why did the relay keep retrying?"),
+ _agent_final("The retry loop never stops. Should I open a pull request with the fix?"),
+ _turn_complete(),
+]
+
+
+class TestGenerateReplySuggestions(BaseTest):
+ def setUp(self):
+ super().setUp()
+ cache.clear()
+ self.organization.is_ai_data_processing_approved = True
+ self.organization.save()
+ self.task = Task.objects.create(
+ team=self.team,
+ title="t",
+ description="d",
+ origin_product=Task.OriginProduct.POSTHOG_AI,
+ created_by=self.user,
+ )
+ self.task_run = self.task.create_run(mode="interactive")
+
+ patchers = {
+ "history": patch(f"{SERVICE}.read_task_run_history", return_value=TWO_TURNS),
+ "publish": patch(
+ f"{SERVICE}.publish_task_run_stream_notification",
+ return_value=StreamNotificationDelivery(live=True, persisted=False),
+ ),
+ "suggest": patch(f"{SERVICE}.suggest_replies", return_value=[OPEN_PR]),
+ "flag": patch(
+ "products.posthog_ai.backend.turn_suggestions.service.feature_enabled_or_false", return_value=True
+ ),
+ "privacy": patch("ee.hogai.utils.feature_flags.is_privacy_mode_enabled", return_value=True),
+ }
+ self.mocks = {name: patcher.start() for name, patcher in patchers.items()}
+ for patcher in patchers.values():
+ self.addCleanup(patcher.stop)
+
+ def _generate(self) -> ReplySuggestionsOutcome:
+ return generate_reply_suggestions(str(self.task_run.id), self.team.id)
+
+ def test_publishes_the_latest_turns_suggestions_to_the_live_stream_only(self):
+ assert self._generate() == ReplySuggestionsOutcome(status="emitted", reason="1")
+
+ answer, attribution, person_message = self.mocks["suggest"].call_args.args
+ assert answer == "The retry loop never stops. Should I open a pull request with the fix?"
+ assert person_message == "Why did the relay keep retrying?"
+ assert attribution.ai_product == "posthog_ai"
+ assert attribution.run_id == str(self.task_run.id)
+ assert attribution.privacy_mode is True
+ assert self.mocks["suggest"].call_args.kwargs == {"reasoning_effort": "none"}
+ publish = self.mocks["publish"].call_args
+ assert publish.args[3] == REPLY_SUGGESTIONS_METHOD
+ assert publish.args[4] == {
+ "turnIndex": 1,
+ "suggestions": [{"label": OPEN_PR.label, "reply": OPEN_PR.reply}],
+ }
+ assert publish.kwargs == {"persist": False}
+
+ def test_each_turn_calls_the_model_once_however_many_reports_arrive(self):
+ first = self._generate()
+ repeat = self._generate()
+ self.mocks["history"].return_value = [
+ *TWO_TURNS,
+ _user_message("And for logins?"),
+ _agent_final("Should I add a login event too?"),
+ _turn_complete(),
+ ]
+ next_turn = self._generate()
+
+ assert (first.status, repeat.reason, next_turn.status) == ("emitted", "turn_already_claimed", "emitted")
+ assert self.mocks["suggest"].call_count == 2
+ assert self.mocks["publish"].call_args.args[4]["turnIndex"] == 2
+
+ def test_simultaneous_reports_of_a_turn_queue_one_job(self):
+ with patch("products.posthog_ai.backend.tasks.generate_reply_suggestions_task.apply_async") as apply_async:
+ queued = [enqueue_reply_suggestions(self.task_run) for _ in range(3)]
+
+ assert queued == [True, False, False]
+ apply_async.assert_called_once()
+
+ def test_a_failed_enqueue_leaves_the_turn_to_the_next_report(self):
+ with patch(
+ "products.posthog_ai.backend.tasks.generate_reply_suggestions_task.apply_async",
+ side_effect=[ConnectionError("broker"), None],
+ ):
+ queued = [enqueue_reply_suggestions(self.task_run) for _ in range(2)]
+
+ assert queued == [False, True]
+
+ def test_a_long_answer_reaches_the_model_with_its_closing_question(self):
+ answer = "Step details. " * 600 + "Should I open a pull request with the fix?"
+ self.mocks["history"].return_value = [_user_message("Fix the relay"), _agent_final(answer), _turn_complete()]
+
+ self._generate()
+
+ assert self.mocks["suggest"].call_args.args[0].endswith("Should I open a pull request with the fix?")
+
+ @parameterized.expand(
+ [
+ ("answer_still_streaming", {"history": TWO_TURNS[:-1]}, "turn_in_progress"),
+ ("flag_off", {"flag": False}, "flag_off"),
+ ("nothing_to_suggest", {"suggest": []}, "nothing_to_suggest"),
+ ]
+ )
+ def test_skips_without_publishing(self, _name: str, returns: dict, reason: str):
+ for name, value in returns.items():
+ self.mocks[name].return_value = value
+
+ assert self._generate() == ReplySuggestionsOutcome(status="skipped", reason=reason)
+ self.mocks["publish"].assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("desktop_conversation", "client_provenance", TaskClientProvenance.POSTHOG_DESKTOP, "not_started_in_web"),
+ ("slack_conversation", "origin_product", Task.OriginProduct.SLACK, "not_posthog_ai_conversation"),
+ ]
+ )
+ def test_skips_conversations_the_web_composer_does_not_show(self, _name: str, field: str, value, reason: str):
+ setattr(self.task, field, value)
+ self.task.save()
+
+ assert self._generate() == ReplySuggestionsOutcome(status="skipped", reason=reason)
+ self.mocks["suggest"].assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("retries_while_the_last_frames_land", "turn_in_progress", True),
+ ("does_not_retry_a_settled_turn", "nothing_to_suggest", False),
+ ]
+ )
+ def test_task_retries_only_while_the_turn_settles(self, _name: str, reason: str, retries: bool):
+ outcome = ReplySuggestionsOutcome(status="skipped", reason=reason)
+ kwargs = {"run_id": str(self.task_run.id), "team_id": self.team.id}
+ with patch("products.posthog_ai.backend.tasks.generate_reply_suggestions", return_value=outcome):
+ if retries:
+ with self.assertRaises(Retry):
+ generate_reply_suggestions_task.apply(kwargs=kwargs)
+ else:
+ generate_reply_suggestions_task.apply(kwargs=kwargs)
diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py
index d18d427019f5..b55f74781f2e 100644
--- a/products/posthog_ai/backend/turn_suggestions/service.py
+++ b/products/posthog_ai/backend/turn_suggestions/service.py
@@ -64,10 +64,10 @@ def _skipped(reason: str) -> TurnSuggestionOutcome:
return TurnSuggestionOutcome(status="skipped", reason=reason)
-def _turn_suggestions_enabled(task_run: TaskRun, user: "User") -> bool:
+def _flag_enabled(flag: str, task_run: TaskRun, user: "User") -> bool:
organization_id = str(task_run.team.organization_id)
return feature_enabled_or_false(
- TURN_SUGGESTIONS_FLAG,
+ flag,
str(user.distinct_id),
groups={"organization": organization_id},
group_properties={"organization": {"id": organization_id}},
@@ -75,6 +75,38 @@ def _turn_suggestions_enabled(task_run: TaskRun, user: "User") -> bool:
)
+@frozen
+class WebConversationRun:
+ task_run: TaskRun
+ user: "User"
+
+
+def load_web_conversation_run(run_id: str, team_id: int, *, flag: str) -> WebConversationRun | str:
+ try:
+ task_run = TaskRun.objects.select_related("task__created_by", "team__organization").get(
+ id=run_id, team_id=team_id
+ )
+ except TaskRun.DoesNotExist:
+ return "run_missing"
+
+ task = task_run.task
+ if task.origin_product != Task.OriginProduct.POSTHOG_AI or task_run.mode != "interactive":
+ return "not_posthog_ai_conversation"
+ # Cards render only in the PostHog AI web app. Slack conversations carry the Slack origin and
+ # fail the check above; PostHog Desktop ones carry the Desktop client provenance.
+ if task.client_provenance == TaskClientProvenance.POSTHOG_DESKTOP:
+ return "not_started_in_web"
+ user = task.created_by
+ if user is None:
+ return "no_user"
+ # The judgment and the drafter send conversation text to third-party AI services.
+ if task_run.team.organization.is_ai_data_processing_approved is not True:
+ return "ai_data_processing_not_approved"
+ if not _flag_enabled(flag, task_run, user):
+ return "flag_off"
+ return WebConversationRun(task_run=task_run, user=user)
+
+
def _history_is_whole(task_run: TaskRun) -> bool:
"""Whether the resume chain the history reads reaches the conversation's first run.
@@ -158,28 +190,11 @@ def _capture_classified(
def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome:
- try:
- task_run = TaskRun.objects.select_related("task__created_by", "team__organization").get(
- id=run_id, team_id=team_id
- )
- except TaskRun.DoesNotExist:
- return _skipped("run_missing")
-
+ loaded = load_web_conversation_run(run_id, team_id, flag=TURN_SUGGESTIONS_FLAG)
+ if isinstance(loaded, str):
+ return _skipped(loaded)
+ task_run, user = loaded.task_run, loaded.user
task = task_run.task
- if task.origin_product != Task.OriginProduct.POSTHOG_AI or task_run.mode != "interactive":
- return _skipped("not_posthog_ai_conversation")
- # Cards render only in the PostHog AI web app. Slack conversations carry the Slack origin and
- # fail the check above; PostHog Desktop ones carry the Desktop client provenance.
- if task.client_provenance == TaskClientProvenance.POSTHOG_DESKTOP:
- return _skipped("not_started_in_web")
- user = task.created_by
- if user is None:
- return _skipped("no_user")
- # The judgment and the drafter send conversation text to third-party AI services.
- if task_run.team.organization.is_ai_data_processing_approved is not True:
- return _skipped("ai_data_processing_not_approved")
- if not _turn_suggestions_enabled(task_run, user):
- return _skipped("flag_off")
if not judge_configured():
logger.warning("posthog_ai_turn_suggestion_judge_not_configured", run_id=str(task_run.id))
return _skipped("judge_not_configured")
diff --git a/products/posthog_ai/backend/turn_suggestions/transcript.py b/products/posthog_ai/backend/turn_suggestions/transcript.py
index 9a5b5fd5e8ad..43cd4f474cb9 100644
--- a/products/posthog_ai/backend/turn_suggestions/transcript.py
+++ b/products/posthog_ai/backend/turn_suggestions/transcript.py
@@ -299,7 +299,9 @@ class HumanTurnCounter:
return True
-def build_turn_transcript(entries: Iterable[dict[str, Any]]) -> TurnTranscript:
+def build_turn_transcript(
+ entries: Iterable[dict[str, Any]], *, assistant_text_limit: int | None = ASSISTANT_TEXT_LIMIT
+) -> TurnTranscript:
"""Fold stream entries into the transcript of the turn after the last user message.
Human messages are counted the way the thread counts them: a ``_posthog/user_message`` frame, or a
@@ -394,8 +396,10 @@ def build_turn_transcript(entries: Iterable[dict[str, Any]]) -> TurnTranscript:
return TurnTranscript(
human_messages=tuple(human_messages),
latest_turn_complete=latest_turn_complete,
- assistant_text=truncate_text(
- _join_messages(assistant_messages), ASSISTANT_TEXT_LIMIT, collapse_whitespace=False
+ assistant_text=(
+ _join_messages(assistant_messages).strip()
+ if assistant_text_limit is None
+ else truncate_text(_join_messages(assistant_messages), assistant_text_limit, collapse_whitespace=False)
),
tool_calls=tuple(
TranscriptToolCall(
diff --git a/products/posthog_ai/frontend/api/composer.ts b/products/posthog_ai/frontend/api/composer.ts
index faf5e94be113..8cda44a44cb5 100644
--- a/products/posthog_ai/frontend/api/composer.ts
+++ b/products/posthog_ai/frontend/api/composer.ts
@@ -8,6 +8,7 @@
// deep `../components/*` paths. See ../README.md for the tier model and ../AGENTS.md for the coupling rule.
export { Composer } from '../components/composer/Composer'
+export { ReplySuggestionHint } from '../components/composer/ReplySuggestionHint'
export type {
ComposerRootProps,
ComposerFrameProps,
diff --git a/products/posthog_ai/frontend/api/types.ts b/products/posthog_ai/frontend/api/types.ts
index 78c420224c8e..1dba6bd80390 100644
--- a/products/posthog_ai/frontend/api/types.ts
+++ b/products/posthog_ai/frontend/api/types.ts
@@ -7,6 +7,7 @@ export type {
ProgressStep,
} from '../types/streamTypes'
export type { ToolCallMessage } from '../types/toolTypes'
+export type { ReplySuggestions } from '../utils/replySuggestions'
export type { Task, TaskAssigneeFilter, TaskListParams } from '../types/taskTypes'
export type {
ToolStreamEvent,
diff --git a/products/posthog_ai/frontend/components/composer/ReplySuggestionHint.tsx b/products/posthog_ai/frontend/components/composer/ReplySuggestionHint.tsx
new file mode 100644
index 000000000000..290277feffd5
--- /dev/null
+++ b/products/posthog_ai/frontend/components/composer/ReplySuggestionHint.tsx
@@ -0,0 +1,14 @@
+export function ReplySuggestionHint({
+ suggestion,
+ acceptHint,
+}: {
+ suggestion: string
+ acceptHint: JSX.Element
+}): JSX.Element {
+ return (
+ <span className="text-sm whitespace-pre-wrap break-words" data-attr="max-reply-suggestion">
+ {suggestion}
+ <span className="ml-2 inline-flex align-middle whitespace-nowrap">{acceptHint}</span>
+ </span>
+ )
+}
diff --git a/products/posthog_ai/frontend/logics/runInteractionLogic.test.ts b/products/posthog_ai/frontend/logics/runInteractionLogic.test.ts
index 2853b6336f7d..2bbe44a3f9b7 100644
--- a/products/posthog_ai/frontend/logics/runInteractionLogic.test.ts
+++ b/products/posthog_ai/frontend/logics/runInteractionLogic.test.ts
@@ -57,6 +57,7 @@ jest.mock('./runStreamLogic', () => {
setStubThinking: (thinking: boolean) => ({ thinking }),
setStubReady: (ready: boolean) => ({ ready }),
setStubClearSupported: (supported: boolean) => ({ supported }),
+ setStubReplySuggestions: (replySuggestions: unknown) => ({ replySuggestions }),
ingestPermissionRequest: (record: PermissionRequestRecord) => ({ record }),
markPermissionRequestResolved: (requestId: string) => ({ requestId }),
deliverPermission: (record: PermissionRequestRecord) => ({ record }),
@@ -135,6 +136,13 @@ jest.mock('./runStreamLogic', () => {
setCurrentMode: (_: string | null, { mode }: { mode: string }) => mode,
},
],
+ replySuggestions: [
+ null,
+ {
+ setStubReplySuggestions: (_: unknown, { replySuggestions }: { replySuggestions: unknown }) =>
+ replySuggestions,
+ },
+ ],
}),
])
return {
@@ -239,6 +247,45 @@ describe('runInteractionLogic', () => {
toolEvents?.unmount()
})
+ describe('reply suggestions', () => {
+ const REPLY = 'Yes, write the snippet'
+ const offer = (turnIndex: number): void =>
+ (
+ stream.actions as unknown as { setStubReplySuggestions: (value: unknown) => void }
+ ).setStubReplySuggestions({ turnIndex, replies: [REPLY] })
+
+ it.each([
+ ['offers the first reply when the agent waits', () => {}, REPLY],
+ ['hides it while the agent is working', () => setThinking(true), null],
+ ['hides it for the turn the user dismissed', () => logic.actions.dismissReplySuggestion(2), null],
+ ['hides it once the user accepted it', () => logic.actions.acceptReplySuggestion(REPLY, 'Tab'), null],
+ ['hides it while a message is sending', () => logic.actions.setSending(true), null],
+ ])('%s', (_name, act, expected) => {
+ offer(2)
+ act()
+ expect(logic.values.replySuggestion).toBe(expected)
+ })
+
+ it('shows the next turn after a dismissed one', () => {
+ offer(2)
+ logic.actions.dismissReplySuggestion(2)
+ offer(3)
+ expect(logic.values.replySuggestion).toBe(REPLY)
+ })
+
+ it.each([
+ [REPLY, false],
+ [`${REPLY} in TypeScript`, true],
+ ])('reports whether the accepted reply was sent as %s', async (sent, edited) => {
+ const capture = jest.spyOn(posthog, 'capture')
+ offer(2)
+ logic.actions.acceptReplySuggestion(REPLY, 'Tab')
+ await expectLogic(logic, () => stream.actions.pushHumanMessage(sent)).toFinishAllListeners()
+ expect(capture).toHaveBeenCalledWith('posthog ai reply suggestion sent', { task_id: TASK_ID, edited })
+ expect(logic.values.acceptedReplySuggestion).toBeNull()
+ })
+ })
+
describe('file attachments', () => {
let attachments: ReturnType<typeof composerAttachmentsLogic.build>
diff --git a/products/posthog_ai/frontend/logics/runInteractionLogic.ts b/products/posthog_ai/frontend/logics/runInteractionLogic.ts
index 8b9af64334df..ba2f06eceb4a 100644
--- a/products/posthog_ai/frontend/logics/runInteractionLogic.ts
+++ b/products/posthog_ai/frontend/logics/runInteractionLogic.ts
@@ -57,6 +57,7 @@ import { rememberAttachmentPreview } from '../utils/attachmentPreviews'
import type { PendingAttachment } from '../utils/attachments'
import { extractCommentContexts, withCommentContexts } from '../utils/commentContext'
import { contextItemLine, wrapWithPosthogContext } from '../utils/posthogContextBlock'
+import type { ReplySuggestions } from '../utils/replySuggestions'
import { submitWithWarmRunRetry } from '../utils/warmRunSubmission'
import { chatContextLogic, chatSendItems } from './chatContextLogic'
import { codexBillingLogic, pickedCodexModelAccess, usesChatGptPlan } from './codexBillingLogic'
@@ -198,11 +199,13 @@ export interface runInteractionLogicValues {
isThinking: boolean // runStreamLogic
pendingPermissionRequest: PermissionRequestRecord | null // runStreamLogic
permissionResponseRequestIds: Set<string> // runStreamLogic
+ replySuggestions: ReplySuggestions | null // runStreamLogic
respondingToPermission: boolean // runStreamLogic
runOpening: boolean // runStreamLogic
runStarted: boolean // runStreamLogic
defaultEffort: string | null // taskRunDefaultsLogic
defaultModel: string | null // taskRunDefaultsLogic
+ acceptedReplySuggestion: string | null
canSend: boolean
clearing: boolean
composerActive: boolean
@@ -232,6 +235,7 @@ export interface runInteractionLogicValues {
consentBlockedSource: 'draft' | 'queue' | 'steer'
contextCandidates: AttachedContextItem[]
deferredSteerRequestId: string | null
+ dismissedReplySuggestionTurn: number | null
draftRecovery: DraftRecovery
effortOverride: string | null
hasUnresolvedApproval: boolean
@@ -247,6 +251,7 @@ export interface runInteractionLogicValues {
queueHeld: boolean
queueHold: QueueHold
queuedMessages: QueuedMessage[]
+ replySuggestion: string | null
selectedEffort: ReasoningEffortEnumApi
selectedMode: PermissionMode
selectedModel: string
@@ -386,6 +391,14 @@ export interface runInteractionLogicActions {
releaseApplyBackTargets: (streamKey: string) => {
streamKey: string
} // toolStreamEventsLogic
+ acceptReplySuggestion: (
+ reply: string,
+ key: string
+ ) => {
+ key: string
+ reply: string
+ turnIndex: number | null
+ }
applyPendingCommentText: () => {
value: true
}
@@ -395,6 +408,9 @@ export interface runInteractionLogicActions {
blockOnConsent: (source?: 'draft' | 'queue' | 'steer') => {
source: 'draft' | 'queue' | 'steer'
}
+ clearAcceptedReplySuggestion: () => {
+ value: true
+ }
clearConsentBlock: () => {
value: true
}
@@ -404,6 +420,9 @@ export interface runInteractionLogicActions {
clearQueue: () => {
value: true
}
+ dismissReplySuggestion: (turnIndex: number) => {
+ turnIndex: number
+ }
enableTaskDraftPersistence: (
userId: string,
projectId: number
@@ -439,6 +458,9 @@ export interface runInteractionLogicActions {
removeQueuedMessage: (id: string) => {
id: string
}
+ reportReplySuggestionShown: (turnIndex: number) => {
+ turnIndex: number
+ }
resetComposerForm: (values?: { draft: string }) => {
values?: {
draft: string
@@ -595,6 +617,12 @@ export interface runInteractionLogicMeta {
arg: any,
arg2: any
) => PermissionMode
+ replySuggestion: (
+ replySuggestions: ReplySuggestions | null,
+ dismissedReplySuggestionTurn: number | null,
+ isBusy: boolean,
+ isSubmitting: boolean
+ ) => string | null
isBusy: (
isThinking: boolean,
runOpening: boolean,
@@ -699,6 +727,7 @@ export const runInteractionLogic = kea<runInteractionLogicType>([
'permissionResponseRequestIds',
'bootstrappedRunId',
'bootstrappedTaskId',
+ 'replySuggestions',
],
attachedContextLogic,
['sentContextKeysByTask', 'seenContextLinesByTask'],
@@ -790,6 +819,14 @@ export const runInteractionLogic = kea<runInteractionLogicType>([
queueDeliveryFailed: true,
// Mirrors the queue editor's open/closed state into the logic, so an automatic drain waits for it.
setQueueEditing: (editing: boolean) => ({ editing }),
+ acceptReplySuggestion: (reply: string, key: string) => ({
+ reply,
+ key,
+ turnIndex: values.replySuggestions?.turnIndex ?? null,
+ }),
+ dismissReplySuggestion: (turnIndex: number) => ({ turnIndex }),
+ reportReplySuggestionShown: (turnIndex: number) => ({ turnIndex }),
+ clearAcceptedReplySuggestion: true,
// Pick the model / reasoning effort for the next message. Selection is held client-side only and
// synced to the running agent (via `set_config_option`) at send time — not on each pick. The backend
// doesn't persist live changes back to the run state, so the override is the source of truth.
@@ -831,6 +868,20 @@ export const runInteractionLogic = kea<runInteractionLogicType>([
clearQueue: () => false,
},
],
+ dismissedReplySuggestionTurn: [
+ null as number | null,
+ {
+ dismissReplySuggestion: (_, { turnIndex }) => turnIndex,
+ acceptReplySuggestion: (state, { turnIndex }) => turnIndex ?? state,
+ },
+ ],
+ acceptedReplySuggestion: [
+ null as string | null,
+ {
+ acceptReplySuggestion: (_, { reply }) => reply,
+ clearAcceptedReplySuggestion: () => null,
+ },
+ ],
consentBlockedSource: ['draft' as 'draft' | 'queue' | 'steer', { blockOnConsent: (_, { source }) => source }],
deferredSteerRequestId: [
null as string | null,
@@ -1125,6 +1176,18 @@ export const runInteractionLogic = kea<runInteractionLogicType>([
return picked ? resolveModeForRuntimeAdapter(adapter, picked) : getDefaultModeForRuntimeAdapter(adapter)
},
],
+ replySuggestion: [
+ (s) => [s.replySuggestions, s.dismissedReplySuggestionTurn, s.isBusy, s.isSubmitting],
+ (
+ replySuggestions: ReplySuggestions | null,
+ dismissedTurn: number | null,
+ isBusy: boolean,
+ isSubmitting: boolean
+ ): string | null =>
+ !replySuggestions || isBusy || isSubmitting || replySuggestions.turnIndex === dismissedTurn
+ ? null
+ : (replySuggestions.replies[0] ?? null),
+ ],
// The agent is actively working a turn — a follow-up typed now should stage rather than send.
isBusy: [
(s, p) => [s.isThinking, s.runOpening, s.runStarted, s.isTerminal, p.runId],
@@ -1804,6 +1867,33 @@ export const runInteractionLogic = kea<runInteractionLogicType>([
}),
listeners(taskDraftListeners),
listeners(({ actions, values, props, cache }) => ({
+ reportReplySuggestionShown: ({ turnIndex }) => {
+ posthog.capture('posthog ai reply suggestion shown', {
+ task_id: props.taskId,
+ turn_index: turnIndex,
+ suggestion_count: values.replySuggestions?.replies.length,
+ })
+ },
+ acceptReplySuggestion: ({ key, turnIndex }) => {
+ posthog.capture('posthog ai reply suggestion accepted', {
+ task_id: props.taskId,
+ turn_index: turnIndex,
+ key,
+ })
+ },
+ dismissReplySuggestion: ({ turnIndex }) => {
+ posthog.capture('posthog ai reply suggestion dismissed', { task_id: props.taskId, turn_index: turnIndex })
+ },
+ pushHumanMessage: ({ content }) => {
+ const accepted = values.acceptedReplySuggestion
+ if (accepted !== null) {
+ posthog.capture('posthog ai reply suggestion sent', {
+ task_id: props.taskId,
+ edited: content.trim() !== accepted,
+ })
+ actions.clearAcceptedReplySuggestion()
+ }
+ },
hydrateTaskDraft: () => {
const draft = values.composerForm.draft
const commentLogic = props.taskId ? taskCommentContextsLogic.findMounted({ taskId: props.taskId }) : null
diff --git a/products/posthog_ai/frontend/logics/runStreamLogic.test.ts b/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
index fdf36453aa2b..82d6e3a2a458 100644
--- a/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
+++ b/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
@@ -1206,6 +1206,15 @@ describe('runStreamLogic', () => {
})
})
+ describe('reply suggestions', () => {
+ it('keeps the newer turn when an older frame lands late', () => {
+ logic.actions.setReplySuggestions({ turnIndex: 3, replies: ['Newer'] })
+ logic.actions.setReplySuggestions({ turnIndex: 2, replies: ['Older'] })
+
+ expect(logic.values.replySuggestionsFrame).toEqual({ turnIndex: 3, replies: ['Newer'] })
+ })
+ })
+
describe('pushHumanMessage', () => {
it('appends a timestamped human_message item ordered before subsequently ingested assistant frames', async () => {
logic.actions.pushHumanMessage('hello agent')
diff --git a/products/posthog_ai/frontend/logics/runStreamLogic.ts b/products/posthog_ai/frontend/logics/runStreamLogic.ts
index 737d603dd1d2..d26fba3c54c7 100644
--- a/products/posthog_ai/frontend/logics/runStreamLogic.ts
+++ b/products/posthog_ai/frontend/logics/runStreamLogic.ts
@@ -95,6 +95,7 @@ import {
} from '../types/wireTypes'
import { extractContextBlockLines } from '../utils/posthogContextBlock'
import { reconcileThreadItems, reconcileToolInvocations } from '../utils/reconcileFoldedThread'
+import { type ReplySuggestions, parseReplySuggestionsParams } from '../utils/replySuggestions'
import { extractAgentToolName, getClaudeCodeMeta, resolveToolCall } from '../utils/toolResolver'
import { parseTurnSuggestionParams } from '../utils/turnSuggestions'
import { computeTurnTrailers } from '../utils/turnTrailers'
@@ -2667,6 +2668,8 @@ export interface runStreamLogicValues {
maxAttempts: number
phase: RecoveryPhase
} | null
+ replySuggestions: ReplySuggestions | null
+ replySuggestionsFrame: ReplySuggestions | null
resolvedPermissionRequestIds: Set<string>
respondingToPermission: boolean
runArtifacts: RunArtifacts
@@ -2958,6 +2961,9 @@ export interface runStreamLogicActions {
setPendingRunMessage: (message: PendingRunMessage | null) => {
message: PendingRunMessage | null
}
+ setReplySuggestions: (replySuggestions: ReplySuggestions | null) => {
+ replySuggestions: ReplySuggestions | null
+ }
setRunOpening: (opening: boolean) => {
opening: boolean
}
@@ -3031,6 +3037,11 @@ export interface runStreamLogicMeta {
bootstrappedTaskId: string | null,
historyTurnBase: number
) => TurnSuggestion | null
+ replySuggestions: (
+ replySuggestionsFrame: ReplySuggestions | null,
+ threadItems: ThreadItem[],
+ historyTurnBase: number
+ ) => ReplySuggestions | null
latestTurnTraceId: (threadItems: ThreadItem[]) => string | null
threadItems: (foldedThread: FoldedThread, showDebugLogs: boolean) => ThreadItem[]
hasThreadItems: (threadItems: ThreadItem[]) => boolean
@@ -3274,6 +3285,7 @@ export const runStreamLogic = kea<runStreamLogicType>([
/** Records an outcome this tab sent to the server, so the card closes before the frame echoes back. */
recordTurnSuggestionOutcome: (turnIndex: number, outcome: string) => ({ turnIndex, outcome }),
markTurnSuggestionAccepted: (turnIndex: number) => ({ turnIndex }),
+ setReplySuggestions: (replySuggestions: ReplySuggestions | null) => ({ replySuggestions }),
markTurnSuggestionExpanded: (turnIndex: number) => ({ turnIndex }),
/**
* Reopens the turn for a follow-up that started outside this composer, such as one sent from Slack
@@ -3694,6 +3706,17 @@ export const runStreamLogic = kea<runStreamLogicType>([
],
// The turn whose card this tab accepted. That card stays up to show what it created, while
// every other tab hides it so a second accept cannot create a duplicate.
+ replySuggestionsFrame: [
+ null as ReplySuggestions | null,
+ {
+ setReplySuggestions: (state, { replySuggestions }) =>
+ state && replySuggestions && replySuggestions.turnIndex < state.turnIndex
+ ? state
+ : replySuggestions,
+ pushHumanMessage: () => null,
+ reset: () => null,
+ },
+ ],
turnSuggestionAcceptedHere: [
null as number | null,
{
@@ -3887,6 +3910,21 @@ export const runStreamLogic = kea<runStreamLogicType>([
return muted || resolved ? null : offer
},
],
+ replySuggestions: [
+ (s) => [s.replySuggestionsFrame, s.threadItems, s.historyTurnBase],
+ (
+ frame: ReplySuggestions | null,
+ threadItems: ThreadItem[],
+ historyTurnBase: number
+ ): ReplySuggestions | null => {
+ if (!frame) {
+ return null
+ }
+ const latestPromptIndex =
+ historyTurnBase + threadItems.filter((item) => item.type === 'human_message').length - 1
+ return frame.turnIndex === latestPromptIndex ? frame : null
+ },
+ ],
latestTurnTraceId: [
(s) => [s.threadItems],
(threadItems: ThreadItem[]): string | null => {
@@ -4572,6 +4610,10 @@ export const runStreamLogic = kea<runStreamLogicType>([
actions.markRunStarted()
return
}
+ if (method === '_posthog/reply_suggestions') {
+ actions.setReplySuggestions(parseReplySuggestionsParams(notification.params))
+ return
+ }
if (method === '_posthog/turn_suggestion') {
// The log holds only outcomes this tab saw live; the ledger adds the ones sent while
// no tab of this conversation was open. Read once per task, when it first has an offer.
diff --git a/products/posthog_ai/frontend/scenes/TaskTracker/components/TaskRunComposer.tsx b/products/posthog_ai/frontend/scenes/TaskTracker/components/TaskRunComposer.tsx
index b7501fbe06d3..5cb8a0f9d99a 100644
--- a/products/posthog_ai/frontend/scenes/TaskTracker/components/TaskRunComposer.tsx
+++ b/products/posthog_ai/frontend/scenes/TaskTracker/components/TaskRunComposer.tsx
@@ -1,6 +1,8 @@
import { useActions, useValues } from 'kea'
import { router } from 'kea-router'
-import { type MutableRefObject, type RefObject, useEffect, useMemo, useRef, useState } from 'react'
+import { type KeyboardEvent, type MutableRefObject, type RefObject, useEffect, useMemo, useRef, useState } from 'react'
+
+import { Kbd } from '@posthog/quill-primitives'
import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
import { projectLogic } from 'scenes/projectLogic'
@@ -10,7 +12,7 @@ import { userLogic } from 'scenes/userLogic'
import { todayShellLogic } from '~/layout/today/todayShellLogic'
-import { Composer } from 'products/posthog_ai/frontend/api/composer'
+import { Composer, ReplySuggestionHint } from 'products/posthog_ai/frontend/api/composer'
import {
runInteractionLogic,
type RunInteractionLogicProps,
@@ -86,6 +88,8 @@ export function TaskRunComposer({
composerActive,
steerPending,
cancellationState,
+ replySuggestion,
+ replySuggestions,
} = useValues(runInteractionLogic(logicProps))
const { slashCommands, commandResult } = useValues(runSlashCommandsLogic(logicProps))
const { submitComposer, dismissCommandResult } = useActions(runSlashCommandsLogic(logicProps))
@@ -112,6 +116,9 @@ export function TaskRunComposer({
steerQueue,
submitAfterConsent,
setQueueEditing,
+ acceptReplySuggestion,
+ dismissReplySuggestion,
+ reportReplySuggestionShown,
} = useActions(runInteractionLogic(logicProps))
useEffect(() => {
@@ -121,6 +128,28 @@ export function TaskRunComposer({
}, [user?.uuid, currentProjectId, enableTaskDraftPersistence])
const draft = useDebouncedDraft(composerForm.draft, (value) => setComposerFormValues({ draft: value }))
+ const shownReplySuggestion = draft.value ? null : replySuggestion
+ const shownReplySuggestionTurn = shownReplySuggestion !== null ? replySuggestions?.turnIndex : undefined
+ useEffect(() => {
+ if (shownReplySuggestionTurn !== undefined) {
+ reportReplySuggestionShown(shownReplySuggestionTurn)
+ }
+ }, [shownReplySuggestionTurn, reportReplySuggestionShown])
+ const handleReplySuggestionKeys = (event: KeyboardEvent<HTMLTextAreaElement>): void => {
+ if (shownReplySuggestion === null || shownReplySuggestionTurn === undefined) {
+ return
+ }
+ const noModifiers = !event.shiftKey && !event.altKey && !event.metaKey && !event.ctrlKey
+ if (noModifiers && (event.key === 'Tab' || event.key === 'ArrowRight')) {
+ event.preventDefault()
+ draft.onChange(shownReplySuggestion)
+ acceptReplySuggestion(shownReplySuggestion, event.key)
+ } else if (event.key === 'Escape') {
+ event.preventDefault()
+ event.stopPropagation()
+ dismissReplySuggestion(shownReplySuggestionTurn)
+ }
+ }
useEffect(() => {
flushDraftRef.current = draft.flush
}, [flushDraftRef, draft.flush])
@@ -183,8 +212,19 @@ export function TaskRunComposer({
const field = (
<ComposerCommandMenu commands={slashCommands}>
<Composer.Field>
- <Composer.Placeholder>{placeholder}</Composer.Placeholder>
- <Composer.Textarea data-attr="sandbox-composer-input" autoFocus={autoFocus} onPaste={onPaste} />
+ {shownReplySuggestion !== null ? (
+ <Composer.Placeholder className="right-12">
+ <ReplySuggestionHint suggestion={shownReplySuggestion} acceptHint={<Kbd>Tab to accept</Kbd>} />
+ </Composer.Placeholder>
+ ) : (
+ <Composer.Placeholder>{placeholder}</Composer.Placeholder>
+ )}
+ <Composer.Textarea
+ data-attr="sandbox-composer-input"
+ autoFocus={autoFocus}
+ onPaste={onPaste}
+ onKeyDown={handleReplySuggestionKeys}
+ />
</Composer.Field>
</ComposerCommandMenu>
)
diff --git a/products/posthog_ai/frontend/utils/replySuggestions.ts b/products/posthog_ai/frontend/utils/replySuggestions.ts
new file mode 100644
index 000000000000..749c771fb43a
--- /dev/null
+++ b/products/posthog_ai/frontend/utils/replySuggestions.ts
@@ -0,0 +1,18 @@
+export interface ReplySuggestions {
+ turnIndex: number
+ replies: string[]
+}
+
+export function parseReplySuggestionsParams(params: unknown): ReplySuggestions | null {
+ if (!params || typeof params !== 'object') {
+ return null
+ }
+ const { turnIndex, suggestions } = params as Record<string, unknown>
+ if (typeof turnIndex !== 'number' || !Array.isArray(suggestions)) {
+ return null
+ }
+ const replies = suggestions
+ .map((suggestion) => (suggestion as Record<string, unknown> | null)?.reply)
+ .filter((reply): reply is string => typeof reply === 'string' && !!reply.trim())
+ return replies.length ? { turnIndex, replies } : null
+}
diff --git a/products/slack_app/backend/logic/reply_suggestions.py b/products/slack_app/backend/logic/reply_suggestions.py
index 5c63529c4ce4..609fb2fb97f3 100644
--- a/products/slack_app/backend/logic/reply_suggestions.py
+++ b/products/slack_app/backend/logic/reply_suggestions.py
@@ -3,17 +3,14 @@
import json
from typing import Any
-from django.conf import settings
from django.core.cache import cache
import structlog
-from openai.types.shared_params import ResponseFormatJSONSchema
from slack_sdk import WebClient
from posthog.dataclasses import frozen
-from posthog.llm.gateway_client import build_ai_gateway_openai_client
-from posthog.llm.semantic_enrichment import extract_json_object
-from posthog.slack.channels import clip_text
+
+from products.tasks.backend.facade.reply_suggestions import MAX_LABEL_CHARS, ReplySuggestionAttribution, SuggestedReply
logger = structlog.get_logger(__name__)
@@ -21,107 +18,11 @@ REPLY_SUGGESTION_ACTION_ID_PREFIX = "slack_app_reply_suggestion:"
REPLY_SUGGESTIONS_BLOCK_ID = "slack_app_reply_suggestions"
# Online evaluations scope to this call with `slack_app_classifier = reply_suggestions`.
REPLY_SUGGESTIONS_CLASSIFIER = "reply_suggestions"
+REPLY_SUGGESTIONS_AI_PRODUCT = "slack_app_routing"
# Only one message in a thread holds open buttons at a time. A newer message makes them stale.
OPEN_SUGGESTIONS_TTL_SECONDS = 60 * 60 * 24 * 7
-MAX_SUGGESTIONS = 3
-# Slack rejects a button label or an accessibility label over 75 characters, and the whole message
-# goes down with it. Slack cuts a label well before that, so the prompt asks for a short one.
-MAX_LABEL_CHARS = 75
-MAX_REPLY_CHARS = 300
-
-SUGGESTIONS_MODEL = "gpt-6-luna"
-# A reasoning model spends its thinking and the JSON reply from one token budget.
-SUGGESTIONS_MAX_COMPLETION_TOKENS = 2048
-# The activity that delivers the answer is sync and runs once, so the call must end well inside it.
-SUGGESTIONS_TIMEOUT_SECONDS = 15.0
-SUGGESTIONS_MAX_RETRIES = 0
-# The question that asks for a reply is nearly always at the end of an answer.
-ANSWER_HEAD_CHARS = 1500
-ANSWER_TAIL_CHARS = 4500
-# A person's message is short, and a long one is a pasted log the answer already covers.
-PERSON_MESSAGE_CHARS = 2000
-
-# The answer goes in its own message, so text in it cannot replace these instructions.
-REPLY_SUGGESTIONS_INSTRUCTIONS = """\
-You write reply buttons for a Slack thread. The next message holds the PostHog agent's answer in \
-<agent_answer>. When it is known, the message that the person sent before the answer is in \
-<person_message>. The person who reads the answer can send one of your replies with one click.
-
-Use the person's message to see what they want, who decides, and which reply they will most likely send. \
-Write replies only to what the answer asks or offers, not to the person's message.
-
-First decide if the answer waits for the person. It does when it asks for permission, offers a next step, \
-or asks the person to choose or to confirm. If the answer only reports a result, explains something, or asks \
-nothing, return an empty list. If the next step waits for someone other than the person, such as another \
-team, or for an event that has not happened yet, return an empty list. When you are not sure, return an \
-empty list.
-
-If the answer waits for the person, write up to 3 replies: the messages the person will most likely type next.
-
-Write each reply the way a person types a short Slack message to a colleague:
-- The person sees only the label, under a long answer, so the label is the whole reply at a glance. \
-Name the choice in its own words, never by its number or letter alone. Not "Option 2" or "Use option 2", \
-but "Stacked rows" or "Go with stacked rows".
-- Vary the form. Do not put the same word in front of every label, such as "Use" or "Yes". When the \
-choices are parallel, a label can be the name of the choice alone. A good set reads like three different \
-people answering, not like a form.
-- Make each reply a different choice. Do not offer two replies that mean the same thing.
-- Put the most likely reply first. When the answer recommends a choice, that reply is the most likely.
-- Write in the language of the answer.
-
-Keep to what the answer offers:
-- Offer only actions that the answer names or clearly implies. Do not invent new work.
-- If the answer asks two or more separate questions, do not write a reply for each combination of \
-answers. Write a reply only if it answers every question with a combination that the answer recommends. \
-If there is none, return an empty list.
-- If the answer offers to check something first and then make a change, such as a pull request or a code \
-or config change, keep the two steps apart. Put first a reply that asks only for the check, so the agent \
-reports what it finds and waits. A reply that also makes the change is a separate choice.
-- If the answer asks for information that only the person knows, such as a name, a value or a repository, \
-do not guess it. Offer only the replies that are complete without it, or return an empty list.
-- Do not write replies that only say thanks or close the conversation.
-- If the answer offers to fix a bug or an error, the reply that accepts the fix has the label \
-"Fix it, make no mistakes". Its reply starts with "Fix it, make no mistakes." and can add the step that the \
-answer names. Do not also offer a plain "Yes, fix it" reply. If the answer is not in English, translate the \
-label and the reply word for word, such as "Beheb es, ohne Fehler" in German, and do not reword them.
-- Do not use em dashes. Use a comma or a full stop.
-
-Each suggestion has two parts:
-- "label": the button text. Keep it under 30 characters, because Slack cuts longer labels.
-- "reply": the full message the person sends to the agent. Keep it under 150 characters. \
-It means the same as the label and can add the detail the label leaves out, such as the option number.
-
-Examples of good sets:
-- The answer asks "Should I open a pull request with the fix?": \
-[{"label": "Yes, open the PR", "reply": "Yes, open the pull request with the fix"}, \
-{"label": "Show me the diff first", "reply": "Show me the diff first, before you open anything"}]
-- The answer lists three layouts and asks which one: \
-[{"label": "Stacked rows", "reply": "Go with stacked rows (option 2)"}, \
-{"label": "The card with toggles", "reply": "The card with toggles (option 1), please"}, \
-{"label": "Persona tabs", "reply": "Persona tabs (option 3)"}]
-- The answer says "I can also break this down by acquisition source": \
-[{"label": "Break it down by source", "reply": "Yes, break it down by acquisition source"}, \
-{"label": "No need, this is enough", "reply": "No need, this is enough for now"}]
-
-Answer only with JSON, for example {"suggestions": [{"label": "...", "reply": "..."}]}."""
-
-
-@frozen
-class ReplySuggestionAttribution:
- trace_id: str | None = None
- session_id: str | None = None
- distinct_id: str | None = None
- task_id: str | None = None
- run_id: str | None = None
-
-
-@frozen
-class SuggestedReply:
- label: str
- reply: str
-
@frozen
class ClickedSuggestion:
@@ -131,109 +32,25 @@ class ClickedSuggestion:
trace_id: str | None
-_RESPONSE_FORMAT: ResponseFormatJSONSchema = {
- "type": "json_schema",
- "json_schema": {
- "name": "slack_app_reply_suggestions",
- "strict": True,
- "schema": {
- "type": "object",
- "properties": {
- "suggestions": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {"label": {"type": "string"}, "reply": {"type": "string"}},
- "required": ["label", "reply"],
- "additionalProperties": False,
- },
- }
- },
- "required": ["suggestions"],
- "additionalProperties": False,
- },
- },
-}
-
-
-def _clip_answer(answer: str) -> str:
- if len(answer) <= ANSWER_HEAD_CHARS + ANSWER_TAIL_CHARS:
- return answer
- return f"{answer[:ANSWER_HEAD_CHARS]}\n…\n{answer[-ANSWER_TAIL_CHARS:]}"
-
-
-def _user_content(answer_markdown: str, person_message: str | None) -> str:
- answer = f"<agent_answer>\n{_clip_answer(answer_markdown)}\n</agent_answer>"
- message = clip_text(" ".join((person_message or "").split()), PERSON_MESSAGE_CHARS)
- if not message:
- return answer
- return f"<person_message>\n{message}\n</person_message>\n\n{answer}"
-
-
-def _clean_suggestions(raw: object) -> list[SuggestedReply]:
- """A label Slack would reject is dropped, not cut, because a cut label can say something the reply does not."""
- if not isinstance(raw, list):
- return []
- suggestions: list[SuggestedReply] = []
- for item in raw:
- if not isinstance(item, dict):
- continue
- label = " ".join(str(item.get("label") or "").split())
- reply = " ".join(str(item.get("reply") or "").split()) or label
- if not label or len(label) > MAX_LABEL_CHARS or len(reply) > MAX_REPLY_CHARS:
- continue
- if any(s.label == label or s.reply == reply for s in suggestions):
- continue
- suggestions.append(SuggestedReply(label=label, reply=reply))
- return suggestions[:MAX_SUGGESTIONS]
-
-
-def request_reply_suggestions(
- answer_markdown: str, attribution: ReplySuggestionAttribution, person_message: str | None = None
-) -> list[SuggestedReply]:
- """Raises on a failed call, so the offline eval can tell a failure from an answer that asks nothing."""
- if not answer_markdown.strip():
- return []
- properties = {"slack_app_classifier": REPLY_SUGGESTIONS_CLASSIFIER}
- if attribution.task_id:
- properties["task_id"] = attribution.task_id
- if attribution.run_id:
- properties["run_id"] = attribution.run_id
- client = build_ai_gateway_openai_client(
- settings.AI_GATEWAY_API_KEY,
- ai_product="slack_app_routing",
- trace_id=attribution.trace_id,
- session_id=attribution.session_id,
- distinct_id=attribution.distinct_id,
- properties=properties,
- ).with_options(timeout=SUGGESTIONS_TIMEOUT_SECONDS, max_retries=SUGGESTIONS_MAX_RETRIES)
- extra: dict[str, Any] = {"user": attribution.distinct_id} if attribution.distinct_id else {}
- response = client.chat.completions.create(
- model=SUGGESTIONS_MODEL,
- messages=[
- {"role": "system", "content": REPLY_SUGGESTIONS_INSTRUCTIONS},
- {"role": "user", "content": _user_content(answer_markdown, person_message)},
- ],
- max_completion_tokens=SUGGESTIONS_MAX_COMPLETION_TOKENS,
- response_format=_RESPONSE_FORMAT,
- **extra,
+def slack_reply_attribution(
+ *,
+ trace_id: str | None = None,
+ session_id: str | None = None,
+ distinct_id: str | None = None,
+ task_id: str | None = None,
+ run_id: str | None = None,
+ privacy_mode: bool = False,
+) -> ReplySuggestionAttribution:
+ return ReplySuggestionAttribution(
+ ai_product=REPLY_SUGGESTIONS_AI_PRODUCT,
+ properties={"slack_app_classifier": REPLY_SUGGESTIONS_CLASSIFIER},
+ trace_id=trace_id,
+ session_id=session_id,
+ distinct_id=distinct_id,
+ task_id=task_id,
+ run_id=run_id,
+ privacy_mode=privacy_mode,
)
- # Tolerant parse on top of the schema, because the gateway does not apply a response
- # format the same way on every provider route.
- parsed = extract_json_object(response.choices[0].message.content or "") or {}
- return _clean_suggestions(parsed.get("suggestions"))
-
-
-def suggest_replies(
- answer_markdown: str, attribution: ReplySuggestionAttribution, person_message: str | None = None
-) -> list[SuggestedReply]:
- """Any failure returns no replies: a missing button costs the reader one typed reply, and a
- wrong one sends the agent an instruction nobody gave."""
- try:
- return request_reply_suggestions(answer_markdown, attribution, person_message)
- except Exception:
- logger.warning("slack_app_reply_suggestions_failed", exc_info=True)
- return []
def reply_suggestions_block(
diff --git a/products/slack_app/backend/slack_thread.py b/products/slack_app/backend/slack_thread.py
index 2285f8dab7e6..5845d4abfa72 100644
--- a/products/slack_app/backend/slack_thread.py
+++ b/products/slack_app/backend/slack_thread.py
@@ -19,13 +19,11 @@ from products.slack_app.backend.feature_flags import is_slack_app_forking_enable
from products.slack_app.backend.logic.feature_tips.offer import TipRequest, offer_feature_tip
from products.slack_app.backend.logic.interview_recruitment import FrustrationInviteRequest, offer_frustration_invite
from products.slack_app.backend.logic.reply_suggestions import (
- ReplySuggestionAttribution,
- SuggestedReply,
clear_open_suggestions,
remember_open_suggestions,
reply_suggestions_block,
reply_suggestions_fallback_text,
- suggest_replies,
+ slack_reply_attribution,
)
from products.slack_app.backend.services.slack_messages import (
RunFooter,
@@ -44,6 +42,7 @@ from products.slack_app.backend.services.slack_messages import (
turn_feedback_block,
)
from products.slack_app.backend.services.task_notifications import task_slack_notifications_muted
+from products.tasks.backend.facade.reply_suggestions import SuggestedReply, suggest_replies
if TYPE_CHECKING:
from posthog.models.user import User
@@ -1012,7 +1011,10 @@ class SlackThreadHandler:
def _suggestions_block(self, answer_markdown: str) -> tuple[list[SuggestedReply], dict[str, Any]] | None:
integration = self._get_integration()
posthog_user = self._actor_posthog_user
- attribution = ReplySuggestionAttribution(
+ from ee.hogai.utils.feature_flags import is_privacy_mode_enabled # noqa: PLC0415
+
+ attribution = slack_reply_attribution(
+ privacy_mode=is_privacy_mode_enabled(integration.team),
trace_id=self.turn_trace_id,
session_id=self._session_id(),
distinct_id=posthog_user.distinct_id if posthog_user else None,
diff --git a/products/slack_app/backend/tests/test_reply_suggestions.py b/products/slack_app/backend/tests/test_reply_suggestions.py
index 7419a8132a40..2bda2c46ccf3 100644
--- a/products/slack_app/backend/tests/test_reply_suggestions.py
+++ b/products/slack_app/backend/tests/test_reply_suggestions.py
@@ -16,16 +16,15 @@ from posthog.models.user import User
from products.slack_app.backend.logic.reply_suggestions import (
REPLY_SUGGESTIONS_BLOCK_ID,
- ReplySuggestionAttribution,
- SuggestedReply,
clear_open_suggestions,
remember_open_suggestions,
reply_suggestions_block,
- suggest_replies,
+ slack_reply_attribution,
)
from products.slack_app.backend.services.slack_messages import RunFooter
from products.slack_app.backend.slack_thread import SlackThreadContext, SlackThreadHandler
from products.slack_app.backend.tests.helpers import sign_slack_request
+from products.tasks.backend.facade.reply_suggestions import SuggestedReply, suggest_replies
ANSWER_BLOCK = {"type": "markdown", "text": "Should I open a pull request?"}
FOOTER_BLOCK = {"type": "context", "elements": [{"type": "mrkdwn", "text": "Claude Opus 5"}]}
@@ -33,68 +32,18 @@ OPEN_PR = SuggestedReply(label="Yes, open the PR", reply="Yes, open the pull req
KEEP_LOCAL = SuggestedReply(label="No, keep it local", reply="No, keep the change local for now")
-def _suggestion(label: str, reply: str) -> dict[str, str]:
- return {"label": label, "reply": reply}
-
-
def _completion(content: str) -> MagicMock:
response = MagicMock()
response.choices = [MagicMock(message=MagicMock(content=content))]
return response
-class TestSuggestReplies(SimpleTestCase):
- @parameterized.expand(
- [
- (
- "keeps_order_and_caps_count",
- json.dumps(
- {
- "suggestions": [
- _suggestion(OPEN_PR.label, OPEN_PR.reply),
- _suggestion(KEEP_LOCAL.label, KEEP_LOCAL.reply),
- _suggestion("Add tests", "Add tests for the empty case too"),
- _suggestion("Use option 2", "Use option 2"),
- ]
- }
- ),
- [OPEN_PR, KEEP_LOCAL, SuggestedReply(label="Add tests", reply="Add tests for the empty case too")],
- ),
- (
- # Slack rejects the whole message for one label over 75 characters.
- "drops_overlong_and_duplicate_labels",
- json.dumps(
- {
- "suggestions": [
- _suggestion("x" * 76, "x"),
- _suggestion(OPEN_PR.label, OPEN_PR.reply),
- _suggestion("Yes, open the PR", "Yes, open it"),
- _suggestion("", "No label"),
- ]
- }
- ),
- [OPEN_PR],
- ),
- (
- "a_missing_reply_sends_the_label",
- json.dumps({"suggestions": [_suggestion("Use option 2", "")]}),
- [SuggestedReply(label="Use option 2", reply="Use option 2")],
- ),
- ("answer_asks_nothing", json.dumps({"suggestions": []}), []),
- ("prose_instead_of_json", "Sure, here are some ideas", []),
- ("wrong_shape", json.dumps({"suggestions": "Yes"}), []),
- ]
- )
- def test_suggestions_from_model_reply(self, _name: str, content: str, expected: list[SuggestedReply]) -> None:
- with patch("products.slack_app.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
- build.return_value.with_options.return_value.chat.completions.create.return_value = _completion(content)
- assert suggest_replies("Should I open a PR?", ReplySuggestionAttribution()) == expected
-
+class TestSlackReplySuggestionGeneration(SimpleTestCase):
def test_generation_carries_what_online_evaluations_scope_and_join_on(self) -> None:
- attribution = ReplySuggestionAttribution(
+ attribution = slack_reply_attribution(
trace_id="trace-1", session_id="T1:C1:1.0", distinct_id="user-1", task_id="task-1", run_id="run-1"
)
- with patch("products.slack_app.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
+ with patch("products.tasks.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
build.return_value.with_options.return_value.chat.completions.create.return_value = _completion(
json.dumps({"suggestions": []})
)
@@ -110,23 +59,6 @@ class TestSuggestReplies(SimpleTestCase):
create = build.return_value.with_options.return_value.chat.completions.create
assert create.call_args.kwargs["user"] == "user-1"
- def test_model_failure_offers_nothing(self) -> None:
- with patch("products.slack_app.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
- build.return_value.with_options.return_value.chat.completions.create.side_effect = TimeoutError()
- assert suggest_replies("Should I open a PR?", ReplySuggestionAttribution()) == []
-
- def test_the_person_message_reaches_the_model_apart_from_the_answer(self) -> None:
- with patch("products.slack_app.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
- create = build.return_value.with_options.return_value.chat.completions.create
- create.return_value = _completion(json.dumps({"suggestions": []}))
- suggest_replies("Should I open a PR?", ReplySuggestionAttribution(), "Why does the retry never stop?")
-
- content = create.call_args.kwargs["messages"][1]["content"]
- assert content == (
- "<person_message>\nWhy does the retry never stop?\n</person_message>\n\n"
- "<agent_answer>\nShould I open a PR?\n</agent_answer>"
- )
-
@override_settings(DEBUG=True)
@patch("products.slack_app.backend.api.SlackIntegration")
@@ -272,18 +204,19 @@ class TestReplySuggestionPlacement(SimpleTestCase):
def setUp(self) -> None:
cache.clear()
+ @patch("ee.hogai.utils.feature_flags.is_privacy_mode_enabled", return_value=False)
@patch("products.slack_app.backend.slack_thread.capture_slack_event")
@patch("products.slack_app.backend.slack_thread.suggest_replies", return_value=[OPEN_PR])
@patch.object(SlackThreadHandler, "_actor_posthog_user", None)
@patch.object(SlackThreadHandler, "_get_integration")
@patch.object(SlackThreadHandler, "_get_client")
def test_plain_answer_gets_the_buttons_under_its_text(
- self, mock_get_client, mock_get_integration, mock_suggest, _mock_capture
+ self, mock_get_client, mock_get_integration, mock_suggest, _mock_capture, _mock_privacy
) -> None:
mock_client = MagicMock()
mock_client.chat_postMessage.return_value = {"ts": "2.0"}
mock_get_client.return_value = mock_client
- mock_get_integration.return_value = Integration(id=7, config={}, integration_id="T1")
+ mock_get_integration.return_value = Integration(id=7, config={}, integration_id="T1", team=Team(id=1))
handler = SlackThreadHandler(
SlackThreadContext(integration_id=7, channel="C001", thread_ts="1.0"),
RunFooter(model="claude-opus-5"),
diff --git a/products/slack_app/evals/eval_reply_suggestions.py b/products/slack_app/evals/eval_reply_suggestions.py
index 551328d70bbe..81b1f2157f80 100644
--- a/products/slack_app/evals/eval_reply_suggestions.py
+++ b/products/slack_app/evals/eval_reply_suggestions.py
@@ -16,9 +16,10 @@ from products.posthog_ai.eval_harness.config import BaseEvalCase
from products.posthog_ai.eval_harness.harness.context import EvalContext
from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
from products.posthog_ai.eval_harness.one_shot import OneShotEval
-from products.slack_app.backend.logic import reply_suggestions
+from products.slack_app.backend.logic.reply_suggestions import slack_reply_attribution
from products.slack_app.evals.offline_config import offline_suite
from products.slack_app.evals.scorers import REPLY_OFFER_KEY, NoUnaskedButtons, ReplyOfferMatch, SuggestedRepliesFit
+from products.tasks.backend.facade import reply_suggestions
SUITE_KIND = SuiteKind.ONE_SHOT
@@ -192,7 +193,7 @@ async def eval_reply_suggestions(ctx: EvalContext) -> None:
suggestions = await asyncio.to_thread(
reply_suggestions.request_reply_suggestions,
case.prompt,
- reply_suggestions.ReplySuggestionAttribution(),
+ slack_reply_attribution(),
person_message,
)
except Exception as error:
diff --git a/products/tasks/backend/facade/reply_suggestions.py b/products/tasks/backend/facade/reply_suggestions.py
new file mode 100644
index 000000000000..f61ee057f037
--- /dev/null
+++ b/products/tasks/backend/facade/reply_suggestions.py
@@ -0,0 +1,17 @@
+from products.tasks.backend.logic.reply_suggestions import (
+ MAX_LABEL_CHARS,
+ SUGGESTIONS_MODEL,
+ ReplySuggestionAttribution,
+ SuggestedReply,
+ request_reply_suggestions,
+ suggest_replies,
+)
+
+__all__ = [
+ "MAX_LABEL_CHARS",
+ "SUGGESTIONS_MODEL",
+ "ReplySuggestionAttribution",
+ "SuggestedReply",
+ "request_reply_suggestions",
+ "suggest_replies",
+]
diff --git a/products/tasks/backend/logic/reply_suggestions.py b/products/tasks/backend/logic/reply_suggestions.py
new file mode 100644
index 000000000000..5f11dc2b3e60
--- /dev/null
+++ b/products/tasks/backend/logic/reply_suggestions.py
@@ -0,0 +1,239 @@
+"""Suggested next replies to an agent answer that asks the reader for a permission, a choice, or a next step.
+
+Every surface that shows a task conversation can offer them: Slack as buttons, the web composer as a
+suggestion the person accepts with a key press.
+"""
+
+from collections.abc import Mapping
+from typing import Any
+
+from django.conf import settings
+
+import structlog
+from openai.types.shared_params import ResponseFormatJSONSchema
+
+from posthog.dataclasses import frozen
+from posthog.llm.gateway_client import build_ai_gateway_openai_client
+from posthog.llm.semantic_enrichment import extract_json_object
+from posthog.slack.channels import clip_text
+
+logger = structlog.get_logger(__name__)
+
+MAX_SUGGESTIONS = 3
+# Slack rejects a button label or an accessibility label over 75 characters, and the whole message
+# goes down with it. Slack cuts a label well before that, so the prompt asks for a short one.
+MAX_LABEL_CHARS = 75
+MAX_REPLY_CHARS = 300
+
+SUGGESTIONS_MODEL = "gpt-6-luna"
+# A reasoning model spends its thinking and the JSON reply from one token budget.
+SUGGESTIONS_MAX_COMPLETION_TOKENS = 2048
+# The activity that delivers the answer is sync and runs once, so the call must end well inside it.
+SUGGESTIONS_TIMEOUT_SECONDS = 15.0
+SUGGESTIONS_MAX_RETRIES = 0
+# The question that asks for a reply is nearly always at the end of an answer.
+ANSWER_HEAD_CHARS = 1500
+ANSWER_TAIL_CHARS = 4500
+# A person's message is short, and a long one is a pasted log the answer already covers.
+PERSON_MESSAGE_CHARS = 2000
+
+# The answer goes in its own message, so text in it cannot replace these instructions.
+REPLY_SUGGESTIONS_INSTRUCTIONS = """\
+You write reply buttons for a Slack thread. The next message holds the PostHog agent's answer in \
+<agent_answer>. When it is known, the message that the person sent before the answer is in \
+<person_message>. The person who reads the answer can send one of your replies with one click.
+
+Use the person's message to see what they want, who decides, and which reply they will most likely send. \
+Write replies only to what the answer asks or offers, not to the person's message.
+
+First decide if the answer waits for the person. It does when it asks for permission, offers a next step, \
+or asks the person to choose or to confirm. If the answer only reports a result, explains something, or asks \
+nothing, return an empty list. If the next step waits for someone other than the person, such as another \
+team, or for an event that has not happened yet, return an empty list. When you are not sure, return an \
+empty list.
+
+If the answer waits for the person, write up to 3 replies: the messages the person will most likely type next.
+
+Write each reply the way a person types a short Slack message to a colleague:
+- The person sees only the label, under a long answer, so the label is the whole reply at a glance. \
+Name the choice in its own words, never by its number or letter alone. Not "Option 2" or "Use option 2", \
+but "Stacked rows" or "Go with stacked rows".
+- Vary the form. Do not put the same word in front of every label, such as "Use" or "Yes". When the \
+choices are parallel, a label can be the name of the choice alone. A good set reads like three different \
+people answering, not like a form.
+- Make each reply a different choice. Do not offer two replies that mean the same thing.
+- Put the most likely reply first. When the answer recommends a choice, that reply is the most likely.
+- Write in the language of the answer.
+
+Keep to what the answer offers:
+- Offer only actions that the answer names or clearly implies. Do not invent new work.
+- If the answer asks two or more separate questions, do not write a reply for each combination of \
+answers. Write a reply only if it answers every question with a combination that the answer recommends. \
+If there is none, return an empty list.
+- If the answer offers to check something first and then make a change, such as a pull request or a code \
+or config change, keep the two steps apart. Put first a reply that asks only for the check, so the agent \
+reports what it finds and waits. A reply that also makes the change is a separate choice.
+- If the answer asks for information that only the person knows, such as a name, a value or a repository, \
+do not guess it. Offer only the replies that are complete without it, or return an empty list.
+- Do not write replies that only say thanks or close the conversation.
+- If the answer offers to fix a bug or an error, the reply that accepts the fix has the label \
+"Fix it, make no mistakes". Its reply starts with "Fix it, make no mistakes." and can add the step that the \
+answer names. Do not also offer a plain "Yes, fix it" reply. If the answer is not in English, translate the \
+label and the reply word for word, such as "Beheb es, ohne Fehler" in German, and do not reword them.
+- Do not use em dashes. Use a comma or a full stop.
+
+Each suggestion has two parts:
+- "label": the button text. Keep it under 30 characters, because Slack cuts longer labels.
+- "reply": the full message the person sends to the agent. Keep it under 150 characters. \
+It means the same as the label and can add the detail the label leaves out, such as the option number.
+
+Examples of good sets:
+- The answer asks "Should I open a pull request with the fix?": \
+[{"label": "Yes, open the PR", "reply": "Yes, open the pull request with the fix"}, \
+{"label": "Show me the diff first", "reply": "Show me the diff first, before you open anything"}]
+- The answer lists three layouts and asks which one: \
+[{"label": "Stacked rows", "reply": "Go with stacked rows (option 2)"}, \
+{"label": "The card with toggles", "reply": "The card with toggles (option 1), please"}, \
+{"label": "Persona tabs", "reply": "Persona tabs (option 3)"}]
+- The answer says "I can also break this down by acquisition source": \
+[{"label": "Break it down by source", "reply": "Yes, break it down by acquisition source"}, \
+{"label": "No need, this is enough", "reply": "No need, this is enough for now"}]
+
+Answer only with JSON, for example {"suggestions": [{"label": "...", "reply": "..."}]}."""
+
+
+@frozen
+class ReplySuggestionAttribution:
+ ai_product: str
+ properties: Mapping[str, str]
+ trace_id: str | None = None
+ session_id: str | None = None
+ distinct_id: str | None = None
+ task_id: str | None = None
+ run_id: str | None = None
+ privacy_mode: bool = False
+
+
+@frozen
+class SuggestedReply:
+ label: str
+ reply: str
+
+
+_RESPONSE_FORMAT: ResponseFormatJSONSchema = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "slack_app_reply_suggestions",
+ "strict": True,
+ "schema": {
+ "type": "object",
+ "properties": {
+ "suggestions": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"label": {"type": "string"}, "reply": {"type": "string"}},
+ "required": ["label", "reply"],
+ "additionalProperties": False,
+ },
+ }
+ },
+ "required": ["suggestions"],
+ "additionalProperties": False,
+ },
+ },
+}
+
+
+def _clip_answer(answer: str) -> str:
+ if len(answer) <= ANSWER_HEAD_CHARS + ANSWER_TAIL_CHARS:
+ return answer
+ return f"{answer[:ANSWER_HEAD_CHARS]}\n…\n{answer[-ANSWER_TAIL_CHARS:]}"
+
+
+def _user_content(answer_markdown: str, person_message: str | None) -> str:
+ answer = f"<agent_answer>\n{_clip_answer(answer_markdown)}\n</agent_answer>"
+ message = clip_text(" ".join((person_message or "").split()), PERSON_MESSAGE_CHARS)
+ if not message:
+ return answer
+ return f"<person_message>\n{message}\n</person_message>\n\n{answer}"
+
+
+def _clean_suggestions(raw: object) -> list[SuggestedReply]:
+ """A label Slack would reject is dropped, not cut, because a cut label can say something the reply does not."""
+ if not isinstance(raw, list):
+ return []
+ suggestions: list[SuggestedReply] = []
+ for item in raw:
+ if not isinstance(item, dict):
+ continue
+ label = " ".join(str(item.get("label") or "").split())
+ reply = " ".join(str(item.get("reply") or "").split()) or label
+ if not label or len(label) > MAX_LABEL_CHARS or len(reply) > MAX_REPLY_CHARS:
+ continue
+ if any(s.label == label or s.reply == reply for s in suggestions):
+ continue
+ suggestions.append(SuggestedReply(label=label, reply=reply))
+ return suggestions[:MAX_SUGGESTIONS]
+
+
+def request_reply_suggestions(
+ answer_markdown: str,
+ attribution: ReplySuggestionAttribution,
+ person_message: str | None = None,
+ *,
+ reasoning_effort: str | None = None,
+) -> list[SuggestedReply]:
+ """Raises on a failed call, so the offline eval can tell a failure from an answer that asks nothing."""
+ if not answer_markdown.strip():
+ return []
+ properties = dict(attribution.properties)
+ if attribution.task_id:
+ properties["task_id"] = attribution.task_id
+ if attribution.run_id:
+ properties["run_id"] = attribution.run_id
+ client = build_ai_gateway_openai_client(
+ settings.AI_GATEWAY_API_KEY,
+ ai_product=attribution.ai_product,
+ trace_id=attribution.trace_id,
+ session_id=attribution.session_id,
+ distinct_id=attribution.distinct_id,
+ properties=properties,
+ ).with_options(timeout=SUGGESTIONS_TIMEOUT_SECONDS, max_retries=SUGGESTIONS_MAX_RETRIES)
+ extra: dict[str, Any] = {"user": attribution.distinct_id} if attribution.distinct_id else {}
+ if reasoning_effort:
+ extra["reasoning_effort"] = reasoning_effort
+ if attribution.privacy_mode:
+ extra["extra_headers"] = {"X-PostHog-Privacy-Mode": "true"}
+ response = client.chat.completions.create(
+ model=SUGGESTIONS_MODEL,
+ messages=[
+ {"role": "system", "content": REPLY_SUGGESTIONS_INSTRUCTIONS},
+ {"role": "user", "content": _user_content(answer_markdown, person_message)},
+ ],
+ max_completion_tokens=SUGGESTIONS_MAX_COMPLETION_TOKENS,
+ response_format=_RESPONSE_FORMAT,
+ **extra,
+ )
+ # Tolerant parse on top of the schema, because the gateway does not apply a response
+ # format the same way on every provider route.
+ parsed = extract_json_object(response.choices[0].message.content or "") or {}
+ return _clean_suggestions(parsed.get("suggestions"))
+
+
+def suggest_replies(
+ answer_markdown: str,
+ attribution: ReplySuggestionAttribution,
+ person_message: str | None = None,
+ *,
+ reasoning_effort: str | None = None,
+) -> list[SuggestedReply]:
+ """Any failure returns no replies: a missing suggestion costs the reader one typed reply, and a
+ wrong one sends the agent an instruction nobody gave."""
+ try:
+ return request_reply_suggestions(
+ answer_markdown, attribution, person_message, reasoning_effort=reasoning_effort
+ )
+ except Exception:
+ logger.warning("reply_suggestions_failed", ai_product=attribution.ai_product, exc_info=True)
+ return []
diff --git a/products/tasks/backend/tests/test_reply_suggestions_model.py b/products/tasks/backend/tests/test_reply_suggestions_model.py
new file mode 100644
index 000000000000..6f3c54e1a058
--- /dev/null
+++ b/products/tasks/backend/tests/test_reply_suggestions_model.py
@@ -0,0 +1,98 @@
+import json
+
+from unittest.mock import MagicMock, patch
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.tasks.backend.logic.reply_suggestions import ReplySuggestionAttribution, SuggestedReply, suggest_replies
+
+ATTRIBUTION = ReplySuggestionAttribution(ai_product="posthog_ai", properties={})
+OPEN_PR = SuggestedReply(label="Yes, open the PR", reply="Yes, open the pull request with the fix")
+KEEP_LOCAL = SuggestedReply(label="No, keep it local", reply="No, keep the change local for now")
+
+
+def _suggestion(label: str, reply: str) -> dict[str, str]:
+ return {"label": label, "reply": reply}
+
+
+def _completion(content: str) -> MagicMock:
+ response = MagicMock()
+ response.choices = [MagicMock(message=MagicMock(content=content))]
+ return response
+
+
+class TestSuggestReplies(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "keeps_order_and_caps_count",
+ json.dumps(
+ {
+ "suggestions": [
+ _suggestion(OPEN_PR.label, OPEN_PR.reply),
+ _suggestion(KEEP_LOCAL.label, KEEP_LOCAL.reply),
+ _suggestion("Add tests", "Add tests for the empty case too"),
+ _suggestion("Use option 2", "Use option 2"),
+ ]
+ }
+ ),
+ [OPEN_PR, KEEP_LOCAL, SuggestedReply(label="Add tests", reply="Add tests for the empty case too")],
+ ),
+ (
+ # Slack rejects the whole message for one label over 75 characters.
+ "drops_overlong_and_duplicate_labels",
+ json.dumps(
+ {
+ "suggestions": [
+ _suggestion("x" * 76, "x"),
+ _suggestion(OPEN_PR.label, OPEN_PR.reply),
+ _suggestion("Yes, open the PR", "Yes, open it"),
+ _suggestion("", "No label"),
+ ]
+ }
+ ),
+ [OPEN_PR],
+ ),
+ (
+ "a_missing_reply_sends_the_label",
+ json.dumps({"suggestions": [_suggestion("Use option 2", "")]}),
+ [SuggestedReply(label="Use option 2", reply="Use option 2")],
+ ),
+ ("answer_asks_nothing", json.dumps({"suggestions": []}), []),
+ ("prose_instead_of_json", "Sure, here are some ideas", []),
+ ("wrong_shape", json.dumps({"suggestions": "Yes"}), []),
+ ]
+ )
+ def test_suggestions_from_model_reply(self, _name: str, content: str, expected: list[SuggestedReply]) -> None:
+ with patch("products.tasks.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
+ build.return_value.with_options.return_value.chat.completions.create.return_value = _completion(content)
+ assert suggest_replies("Should I open a PR?", ATTRIBUTION) == expected
+
+ @parameterized.expand([(False, None), (True, {"X-PostHog-Privacy-Mode": "true"})])
+ def test_privacy_mode_asks_the_gateway_not_to_record_the_prompt(self, privacy_mode: bool, headers) -> None:
+ attribution = ReplySuggestionAttribution(ai_product="posthog_ai", properties={}, privacy_mode=privacy_mode)
+ with patch("products.tasks.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
+ create = build.return_value.with_options.return_value.chat.completions.create
+ create.return_value = _completion(json.dumps({"suggestions": []}))
+ suggest_replies("Should I open a PR?", attribution)
+
+ assert create.call_args.kwargs.get("extra_headers") == headers
+
+ def test_model_failure_offers_nothing(self) -> None:
+ with patch("products.tasks.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
+ build.return_value.with_options.return_value.chat.completions.create.side_effect = TimeoutError()
+ assert suggest_replies("Should I open a PR?", ATTRIBUTION) == []
+
+ def test_the_person_message_reaches_the_model_apart_from_the_answer(self) -> None:
+ with patch("products.tasks.backend.logic.reply_suggestions.build_ai_gateway_openai_client") as build:
+ create = build.return_value.with_options.return_value.chat.completions.create
+ create.return_value = _completion(json.dumps({"suggestions": []}))
+ suggest_replies("Should I open a PR?", ATTRIBUTION, "Why does the retry never stop?")
+
+ content = create.call_args.kwargs["messages"][1]["content"]
+ assert content == (
+ "<person_message>\nWhy does the retry never stop?\n</person_message>\n\n"
+ "<agent_answer>\nShould I open a PR?\n</agent_answer>"
+ )