Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(slack-app): invite frustrated users to an interview (#123274)

Vojta Bartošea304fcf58b1
diff --git a/products/slack_app/backend/logic/feature_tips/state.py b/products/slack_app/backend/logic/feature_tips/state.py
index 521d9f5b9901..0edb5ed29c3b 100644
--- a/products/slack_app/backend/logic/feature_tips/state.py
+++ b/products/slack_app/backend/logic/feature_tips/state.py
@@ -41,6 +41,9 @@ class FeatureTipsState(BaseModel):
shown: dict[str, TipRecord] = {}
opted_out_at: datetime | None = None
last_shown_at: datetime | None = None
+ # Interview invitations share this row so they need no migration. They have their own limit.
+ interview_invited_at: datetime | None = None
+ interview_opted_out_at: datetime | None = None
def parse_state(raw: object) -> FeatureTipsState:
diff --git a/products/slack_app/backend/logic/interview_recruitment.py b/products/slack_app/backend/logic/interview_recruitment.py
index 0693c7684a3c..4f1192cdaf29 100644
--- a/products/slack_app/backend/logic/interview_recruitment.py
+++ b/products/slack_app/backend/logic/interview_recruitment.py
@@ -3,42 +3,111 @@
Every invitation links to ``BOOKING_URL``. A click arrives as a ``block_actions`` payload, which is
how the click is counted.
+Three surfaces show an invitation:
+
- The bad-rating modal always shows the link, because the reader already chose to tell us what went wrong.
- The Home tab shows a button when the ``slack-app-interview-recruitment`` flag selects the reader.
+- A private message under the answer offers the interview when the flag selects the reader and the
+ reader shows frustration: a thumbs-down reaction on an answer, or a message that the decision model
+ reads as unhappy with the assistant or PostHog. A reader gets at most one such message in ``INVITE_INTERVAL``, and
+ "Don't ask again" stops these messages only. The model runs only after every cheap gate passes, so
+ most answers cost no call.
"""
from __future__ import annotations
import json
+import uuid
+from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Literal, get_args
from django.http import HttpResponse
+from django.utils import timezone
import structlog
from posthog.dataclasses import frozen
-from posthog.models.integration import Integration
+from posthog.llm.managed_decision_model import DEFAULT_DECISION_MODEL
+from posthog.llm.system_one import ChoiceAnswer, ChoiceQuestion
+from posthog.llm.system_one_client import build_system_one_client
+from posthog.models.integration import Integration, SlackIntegration
from products.slack_app.backend.analytics import capture_slack_event, slack_distinct_id
from products.slack_app.backend.feature_flags import is_slack_app_interview_recruitment_enabled
-from products.slack_app.backend.services.slack_messages import section_block
+from products.slack_app.backend.logic.feature_tips.state import FeatureTipsState, get_state, update_state
+from products.slack_app.backend.models import SlackThreadTaskMapping
+from products.slack_app.backend.services.inbox_interactivity import post_response_url
+from products.slack_app.backend.services.slack_messages import post_slack_ephemeral, section_block
from products.slack_app.backend.services.slack_user_oauth import find_linked_posthog_user
+from products.slack_app.backend.temporal.activities.classifiers import CLASSIFIER_PROPERTY
if TYPE_CHECKING:
from posthog.models.user import User
logger = structlog.get_logger(__name__)
-InterviewSurface = Literal["home", "bad_rating_modal"]
+InterviewSurface = Literal["home", "bad_rating_modal", "frustration"]
INTERVIEW_SURFACES: frozenset[str] = frozenset(get_args(InterviewSurface))
+FrustrationSignal = Literal["message", "reaction"]
BOOKING_URL = "https://cal.com/vojta-bartos-wye08f/posthog-slack-app"
INVITE_MESSAGE = "Got 30 minutes to tell us how the PostHog Slack app works for you? Book a chat with the team."
MODAL_INVITE_MESSAGE = "Want to talk it through? Book 30 minutes with the team."
+# A first-time reader gets answers before any invitation, like the feature tips.
+MIN_INTERACTIONS = 3
+INVITE_INTERVAL = timedelta(days=30)
+# A tip this recent probably sits under the answer the reader just reacted to.
+TIP_GAP = timedelta(hours=1)
INTERVIEW_BOOK_ACTION_ID = "slack_app_interview:book"
-INTERVIEW_ACTION_IDS: frozenset[str] = frozenset({INTERVIEW_BOOK_ACTION_ID})
-_CLICK_EVENTS: dict[str, str] = {INTERVIEW_BOOK_ACTION_ID: "slack app interview invite clicked"}
+INTERVIEW_DISMISS_ACTION_ID = "slack_app_interview:dismiss"
+INTERVIEW_OPT_OUT_ACTION_ID = "slack_app_interview:opt_out"
+INTERVIEW_ACTION_IDS: frozenset[str] = frozenset(
+ {INTERVIEW_BOOK_ACTION_ID, INTERVIEW_DISMISS_ACTION_ID, INTERVIEW_OPT_OUT_ACTION_ID}
+)
+_CLICK_EVENTS: dict[str, str] = {
+ INTERVIEW_BOOK_ACTION_ID: "slack app interview invite clicked",
+ INTERVIEW_DISMISS_ACTION_ID: "slack app interview invite dismissed",
+ INTERVIEW_OPT_OUT_ACTION_ID: "slack app interview invites turned off",
+}
+INVITES_OFF_TEXT = "👋 Got it. I won't ask you again."
+
+# Not the `slack_app` product the other classifiers bill under: recruitment serves PostHog, so it
+# must not spend the team's AI credits.
+SENTIMENT_AI_PRODUCT = "slack_app_interview_recruitment"
+# The answer is already in the thread. A slow call only drops the invitation.
+SENTIMENT_TIMEOUT_SECONDS = 5.0
+# A wrong invitation to a calm person costs trust, and a missed one costs nothing, so the bar is high.
+# Tuned in products/slack_app/evals/eval_interview_sentiment.py.
+SENTIMENT_MIN_CONFIDENCE = 0.7
+SENTIMENT_QUESTION_ID = "sentiment"
+UNHAPPY = "unhappy"
+NOT_UNHAPPY = "not_unhappy"
+# The model needs the tone, not the whole message. This also keeps pasted logs out of the call.
+SENTIMENT_MESSAGE_MAX_CHARS = 500
+SENTIMENT_INSTRUCTIONS = (
+ "`message` is what a person wrote to the PostHog assistant in Slack. "
+ "Decide whether the person has an unhappy experience with the assistant or with PostHog. "
+ "Judge the tone and the intent, in any language. Do not judge the topic."
+)
+SENTIMENT_CRITERIA = {
+ UNHAPPY: "The person is frustrated, annoyed, angry, disappointed, or sarcastic about the assistant, "
+ "its answers, or PostHog. Examples: a complaint, a statement that it failed or is useless, "
+ "a repeated request after earlier failures, or a threat to stop using it.",
+ NOT_UNHAPPY: "The person asks, instructs, thanks, or reports something in a neutral or positive tone. "
+ "A calm correction, an urgent request, or a problem in their own product or with other people "
+ "is not unhappiness with the assistant.",
+}
+
+
+@frozen
+class SentimentVerdict:
+ choice: str
+ confidence: float
+
+ @property
+ def is_unhappy(self) -> bool:
+ return self.choice == UNHAPPY and self.confidence >= SENTIMENT_MIN_CONFIDENCE
def linked_posthog_user(integration: Integration, slack_user_id: str) -> User | None:
@@ -80,6 +149,184 @@ def invite_section(text: str, button: dict[str, Any]) -> dict[str, Any]:
return {**section_block(text), "accessory": button}
+@frozen
+class FrustrationInviteRequest:
+ integration: Integration
+ slack_user_id: str
+ posthog_user: User | None
+ channel: str
+ thread_ts: str
+ signal: FrustrationSignal
+ person_message: str | None = None
+ trace_id: str | None = None
+
+
+def offer_frustration_invite(request: FrustrationInviteRequest) -> bool:
+ """Post a private invitation under the answer when the reader shows frustration.
+
+ ``True`` means the invitation posted, so the caller sends no feature tip under the same answer.
+ """
+ integration = request.integration
+ workspace_id = integration.integration_id or ""
+ state = get_state(workspace_id, request.slack_user_id)
+ now = timezone.now()
+ # A reader who turned feature tips off asked for fewer prompts, so this prompt stays away too.
+ if state.opted_out_at is not None or not _is_invite_due(state):
+ return False
+ # A reaction arrives after the answer, so a tip may already sit under that answer. No answer gets two prompts.
+ if state.last_shown_at is not None and now - state.last_shown_at < TIP_GAP:
+ return False
+ if not is_recruiting(integration, request.slack_user_id, request.posthog_user):
+ return False
+ # The limit stops the count at the threshold, so a reader with a long history costs no full scan.
+ thread_count = (
+ SlackThreadTaskMapping.objects.filter(
+ slack_workspace_id=workspace_id, mentioning_slack_user_id=request.slack_user_id
+ )
+ .values("id")[:MIN_INTERACTIONS]
+ .count()
+ )
+ if thread_count < MIN_INTERACTIONS:
+ return False
+ if request.signal == "message" and not _is_unhappy(
+ request.person_message or "",
+ team_id=integration.team_id,
+ distinct_id=request.posthog_user.distinct_id if request.posthog_user is not None else None,
+ trace_id=request.trace_id,
+ ):
+ return False
+
+ # The slot is held before the post, so two answers that finish together cannot both send one.
+ reserved_at = _reserve_invite(workspace_id, request.slack_user_id)
+ if reserved_at is None:
+ return False
+ invitation_id = str(uuid.uuid4())
+ try:
+ post_slack_ephemeral(
+ SlackIntegration(integration).client,
+ channel=request.channel,
+ user=request.slack_user_id,
+ thread_ts=request.thread_ts,
+ text=INVITE_MESSAGE,
+ blocks=frustration_invite_blocks(integration.id, invitation_id, request.trace_id),
+ )
+ except Exception:
+ # The reader saw nothing, so the slot goes back.
+ logger.warning("slack_app_interview_invite_post_failed", exc_info=True)
+ _release_invite(workspace_id, request.slack_user_id, reserved_at)
+ return False
+ capture_slack_event(
+ integration,
+ "slack app interview invite shown",
+ slack_user_id=request.slack_user_id,
+ posthog_user=request.posthog_user,
+ surface="frustration",
+ invitation_id=invitation_id,
+ signal=request.signal,
+ ai_trace_id=request.trace_id,
+ )
+ return True
+
+
+def _is_invite_due(state: FeatureTipsState) -> bool:
+ if state.interview_opted_out_at is not None:
+ return False
+ return state.interview_invited_at is None or timezone.now() - state.interview_invited_at >= INVITE_INTERVAL
+
+
+def _reserve_invite(slack_workspace_id: str, slack_user_id: str) -> datetime | None:
+ """Hold this person's invitation slot, and return when. ``None`` means another invitation is too recent."""
+ now = timezone.now()
+
+ def change(state: FeatureTipsState) -> None:
+ if _is_invite_due(state):
+ state.interview_invited_at = now
+
+ state = update_state(slack_workspace_id, slack_user_id, change)
+ return now if state.interview_invited_at == now else None
+
+
+def _release_invite(slack_workspace_id: str, slack_user_id: str, reserved_at: datetime) -> None:
+ """Give back a slot that ``_reserve_invite`` held for an invitation the reader never saw."""
+
+ def change(state: FeatureTipsState) -> None:
+ # A later reservation is another invitation's slot, so only this one goes back.
+ if state.interview_invited_at == reserved_at:
+ state.interview_invited_at = None
+
+ try:
+ update_state(slack_workspace_id, slack_user_id, change)
+ except Exception:
+ logger.warning("slack_app_interview_invite_release_failed", exc_info=True)
+
+
+def classify_sentiment(
+ message: str, *, team_id: int, distinct_id: str | None, trace_id: str | None
+) -> SentimentVerdict | None:
+ """Ask the decision model whether the person is unhappy with the assistant. ``None`` means no verdict.
+
+ Raises when no System One server is configured or the call fails, so the caller can tell an
+ outage from a refusal.
+ """
+ text = message.strip()[:SENTIMENT_MESSAGE_MAX_CHARS]
+ if not text:
+ return None
+ # No TypeSafe fallback: the message is customer text, and TypeSafe is a third party.
+ client = build_system_one_client(
+ model=DEFAULT_DECISION_MODEL,
+ ai_product=SENTIMENT_AI_PRODUCT,
+ team_id=team_id,
+ distinct_id=distinct_id,
+ trace_id=trace_id,
+ properties={CLASSIFIER_PROPERTY: "interview_sentiment"},
+ timeout=SENTIMENT_TIMEOUT_SECONDS,
+ )
+ result = client.decide(
+ state={"message": text},
+ questions={
+ SENTIMENT_QUESTION_ID: ChoiceQuestion(instructions=SENTIMENT_INSTRUCTIONS, criteria=SENTIMENT_CRITERIA)
+ },
+ )
+ answer = result.answers.get(SENTIMENT_QUESTION_ID)
+ if not isinstance(answer, ChoiceAnswer):
+ return None
+ return SentimentVerdict(choice=answer.choice, confidence=answer.confidence)
+
+
+def _is_unhappy(message: str, *, team_id: int, distinct_id: str | None, trace_id: str | None) -> bool:
+ """Whether the message reads as unhappy. Any failure reads as no, because a missed invitation costs nothing."""
+ try:
+ verdict = classify_sentiment(message, team_id=team_id, distinct_id=distinct_id, trace_id=trace_id)
+ except Exception:
+ logger.warning("slack_app_interview_sentiment_classifier_failed", exc_info=True)
+ return False
+ return verdict is not None and verdict.is_unhappy
+
+
+def frustration_invite_blocks(integration_id: int, invitation_id: str, trace_id: str | None) -> list[dict[str, Any]]:
+ book = book_button(integration_id, "frustration", invitation_id=invitation_id, trace_id=trace_id)
+
+ def button(action_id: str, label: str) -> dict[str, Any]:
+ return {
+ "type": "button",
+ "action_id": action_id,
+ "text": {"type": "plain_text", "text": label},
+ "value": book["value"],
+ }
+
+ return [
+ section_block(INVITE_MESSAGE),
+ {
+ "type": "actions",
+ "elements": [
+ book,
+ button(INTERVIEW_DISMISS_ACTION_ID, "Not now"),
+ button(INTERVIEW_OPT_OUT_ACTION_ID, "Don't ask again"),
+ ],
+ },
+ ]
+
+
@frozen
class _InterviewClick:
action_id: str
@@ -113,9 +360,10 @@ def interview_integration_hint(payload: dict[str, Any]) -> int | None:
def handle_interview_action(payload: dict[str, Any]) -> HttpResponse:
- """Record a click on an invitation's booking button.
+ """Record a click on an invitation's buttons.
- Slack opens the booking page itself and sends this payload only so the click can be counted.
+ For the booking button, Slack opens the booking page itself and sends this payload only so the
+ click can be counted. The private message goes away after any click, because it did its job.
"""
clicked = _clicked(payload)
slack_user_id = (payload.get("user") or {}).get("id")
@@ -137,12 +385,24 @@ def handle_interview_action(payload: dict[str, Any]) -> HttpResponse:
if integration is None or surface not in INTERVIEW_SURFACES:
return HttpResponse(status=200)
+ response_url = payload.get("response_url") or ""
+ if clicked.action_id == INTERVIEW_OPT_OUT_ACTION_ID:
+
+ def change(state: FeatureTipsState) -> None:
+ state.interview_opted_out_at = timezone.now()
+
+ update_state(integration.integration_id or "", slack_user_id, change)
+ post_response_url(response_url, {"replace_original": True, "text": INVITES_OFF_TEXT})
+ elif response_url and (payload.get("container") or {}).get("is_ephemeral"):
+ post_response_url(response_url, {"delete_original": True})
+
capture_slack_event(
integration,
_CLICK_EVENTS[clicked.action_id],
slack_user_id=slack_user_id,
posthog_user=linked_posthog_user(integration, slack_user_id),
surface=surface,
+ invitation_id=value.get("invitation_id"),
task_run_id=value.get("run_id"),
ai_trace_id=value.get("trace_id"),
)
diff --git a/products/slack_app/backend/services/turn_feedback.py b/products/slack_app/backend/services/turn_feedback.py
index 482cb3a6f3ea..89377c8eb303 100644
--- a/products/slack_app/backend/services/turn_feedback.py
+++ b/products/slack_app/backend/services/turn_feedback.py
@@ -569,4 +569,28 @@ def handle_reaction_added(event: dict, slack_team_id: str, workspace_integration
},
)
record_feature_used(integration.integration_id, event.get("user"), SlackFeature.ANSWER_RATING)
+ if rating == "bad" and target.slack_user_id:
+ _offer_interview_after_reaction(target, channel, message.get("thread_ts") if message else None)
return REACTION_HANDLED
+
+
+def _offer_interview_after_reaction(target: _FeedbackTarget, channel: str, thread_ts: Any) -> None:
+ """A thumbs-down reaction opens no modal, so a private message carries the invitation instead.
+
+ The offer runs on Celery, because the flag check and the post make network calls inside Slack's
+ three-second ack budget.
+ """
+ # noqa reason: the task module imports api, which imports this module at load time.
+ from products.slack_app.backend.tasks import offer_interview_invite_after_reaction # noqa: PLC0415
+
+ try:
+ offer_interview_invite_after_reaction.delay(
+ team_id=target.integration.team_id,
+ integration_id=target.integration.id,
+ slack_user_id=target.slack_user_id,
+ channel=channel,
+ thread_ts=thread_ts if isinstance(thread_ts, str) and thread_ts else target.turn_id or "",
+ trace_id=target.trace_id,
+ )
+ except Exception:
+ logger.warning("slack_app_interview_invite_after_reaction_failed", run_id=target.run_id, exc_info=True)
diff --git a/products/slack_app/backend/slack_thread.py b/products/slack_app/backend/slack_thread.py
index 4234a2b6c860..64df3b26a475 100644
--- a/products/slack_app/backend/slack_thread.py
+++ b/products/slack_app/backend/slack_thread.py
@@ -17,6 +17,7 @@ from posthog.slack.markdown import SLACK_MARKDOWN_TEXT_MAX_LEN, slack_markdown_b
from products.slack_app.backend.analytics import capture_slack_event, slack_session_id
from products.slack_app.backend.feature_flags import is_slack_app_forking_enabled
from products.slack_app.backend.logic.feature_tips.offer import TipRequest, offer_feature_tip
+from products.slack_app.backend.logic.interview_recruitment import FrustrationInviteRequest, offer_frustration_invite
from products.slack_app.backend.logic.reply_suggestions import (
ReplySuggestionAttribution,
SuggestedReply,
@@ -1128,8 +1129,25 @@ class SlackThreadHandler:
)
def offer_feature_tip(self) -> None:
+ """Offer one private prompt under the answer: an interview invitation if the reader is frustrated, else a tip."""
if not self.actor_slack_user_id:
return
+ try:
+ if offer_frustration_invite(
+ FrustrationInviteRequest(
+ integration=self._get_integration(),
+ slack_user_id=self.actor_slack_user_id,
+ posthog_user=self._actor_posthog_user,
+ channel=self.context.channel,
+ thread_ts=self.context.thread_ts,
+ signal="message",
+ person_message=self.turn_person_message,
+ trace_id=self.turn_trace_id,
+ )
+ ):
+ return
+ except Exception:
+ logger.warning("slack_app_interview_invite_failed", exc_info=True)
try:
offer_feature_tip(
TipRequest(
diff --git a/products/slack_app/backend/tasks.py b/products/slack_app/backend/tasks.py
index 4b45b802dec9..a72bbcc31e79 100644
--- a/products/slack_app/backend/tasks.py
+++ b/products/slack_app/backend/tasks.py
@@ -18,6 +18,11 @@ from products.slack_app.backend.api import (
does_other_region_claim_workspace,
send_region_proxy_request,
)
+from products.slack_app.backend.logic.interview_recruitment import (
+ FrustrationInviteRequest,
+ linked_posthog_user,
+ offer_frustration_invite,
+)
from products.slack_app.backend.slack_link_unfurl import add_artifact_image_to_unfurl, start_unfurl_charts
from products.slack_app.backend.temporal.types import SlackUnfurlChartRequest
@@ -61,6 +66,10 @@ def refresh_user_github_repository_cache(*, user_integration_id: str) -> None:
github.sync_repository_cache(min_refresh_interval_seconds=REPO_SNAPSHOT_REFRESH_INTERVAL_SECONDS)
+def _load_integration(integration_id: int, team_id: int) -> Integration | None:
+ return Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()
+
+
@shared_task(ignore_result=True)
@with_team_scope()
def start_slack_unfurl_charts(
@@ -72,9 +81,7 @@ def start_slack_unfurl_charts(
unfurls: dict[str, Any],
charts: list[dict[str, Any]],
) -> None:
- integration = (
- Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()
- )
+ integration = _load_integration(integration_id, team_id)
user = User.objects.filter(id=user_id).first()
if integration is None or user is None:
return
@@ -99,9 +106,7 @@ def add_slack_unfurl_artifact_image(
url: str,
title: str,
) -> None:
- integration = (
- Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()
- )
+ integration = _load_integration(integration_id, team_id)
user = User.objects.filter(id=user_id).first()
if integration is None or user is None:
return
@@ -113,3 +118,30 @@ def add_slack_unfurl_artifact_image(
url=url,
title=title,
)
+
+
+@shared_task(ignore_result=True)
+@with_team_scope()
+def offer_interview_invite_after_reaction(
+ *,
+ team_id: int,
+ integration_id: int,
+ slack_user_id: str,
+ channel: str,
+ thread_ts: str,
+ trace_id: str | None,
+) -> None:
+ integration = _load_integration(integration_id, team_id)
+ if integration is None:
+ return
+ offer_frustration_invite(
+ FrustrationInviteRequest(
+ integration=integration,
+ slack_user_id=slack_user_id,
+ posthog_user=linked_posthog_user(integration, slack_user_id),
+ channel=channel,
+ thread_ts=thread_ts,
+ signal="reaction",
+ trace_id=trace_id,
+ )
+ )
diff --git a/products/slack_app/backend/tests/logic/test_interview_recruitment.py b/products/slack_app/backend/tests/logic/test_interview_recruitment.py
new file mode 100644
index 000000000000..7b997b9f0f85
--- /dev/null
+++ b/products/slack_app/backend/tests/logic/test_interview_recruitment.py
@@ -0,0 +1,147 @@
+import json
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from django.apps import apps
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.llm.system_one import ChoiceAnswer, SystemOneResult
+from posthog.models.integration import Integration
+from posthog.models.organization import Organization
+from posthog.models.team.team import Team
+
+from products.slack_app.backend.logic import interview_recruitment
+from products.slack_app.backend.logic.feature_tips.state import get_state, update_state
+from products.slack_app.backend.logic.interview_recruitment import (
+ BOOKING_URL,
+ INTERVIEW_OPT_OUT_ACTION_ID,
+ FrustrationInviteRequest,
+ FrustrationSignal,
+ handle_interview_action,
+ offer_frustration_invite,
+)
+from products.slack_app.backend.models import SlackThreadTaskMapping
+
+WORKSPACE = "T_INTERVIEW"
+SLACK_USER = "U_READER"
+
+
+def _verdict(choice: str, confidence: float) -> MagicMock:
+ client = MagicMock()
+ client.decide.return_value = SystemOneResult(
+ model="m",
+ answers={"sentiment": ChoiceAnswer(choice=choice, confidence=confidence, probabilities={})},
+ input_tokens=None,
+ )
+ return client
+
+
+class TestFrustrationInvite:
+ @pytest.fixture(autouse=True)
+ def setup(self):
+ organization = Organization.objects.create(name="Org")
+ self.team = Team.objects.create(organization=organization, name="Team")
+ self.integration = Integration.objects.create(
+ team=self.team, kind="slack", integration_id=WORKSPACE, sensitive_config={"access_token": "xoxb-test"}
+ )
+ with (
+ patch.object(interview_recruitment, "is_slack_app_interview_recruitment_enabled", return_value=True),
+ patch.object(interview_recruitment, "post_slack_ephemeral") as self.post,
+ patch.object(interview_recruitment, "SlackIntegration"),
+ patch.object(interview_recruitment, "capture_slack_event") as self.capture,
+ patch.object(interview_recruitment, "build_system_one_client", return_value=_verdict("unhappy", 0.9)),
+ ):
+ yield
+
+ def _threads(self, count: int) -> None:
+ Task = apps.get_model("tasks", "Task")
+ TaskRun = apps.get_model("tasks", "TaskRun")
+ for index in range(count):
+ task = Task.objects.create(team=self.team, title="t")
+ SlackThreadTaskMapping.objects.create(
+ team=self.team,
+ integration=self.integration,
+ slack_workspace_id=WORKSPACE,
+ channel="C1",
+ thread_ts=f"{index}.1",
+ task=task,
+ task_run=TaskRun.objects.create(team=self.team, task=task),
+ mentioning_slack_user_id=SLACK_USER,
+ )
+
+ def _offer(self, signal: FrustrationSignal = "message") -> bool:
+ return offer_frustration_invite(
+ FrustrationInviteRequest(
+ integration=self.integration,
+ slack_user_id=SLACK_USER,
+ posthog_user=None,
+ channel="C1",
+ thread_ts="1.1",
+ signal=signal,
+ person_message="this is the third time you got the funnel wrong",
+ )
+ )
+
+ def test_a_frustrated_reader_is_invited_once_a_month(self):
+ self._threads(3)
+
+ assert [self._offer(), self._offer()] == [True, False]
+ self.post.assert_called_once()
+ book = self.post.call_args.kwargs["blocks"][1]["elements"][0]
+ assert book["url"] == BOOKING_URL
+ assert self.capture.call_args.args[1] == "slack app interview invite shown"
+ assert self.capture.call_args.kwargs["surface"] == "frustration"
+
+ @parameterized.expand(
+ [
+ ("calm_message", 3, _verdict("not_unhappy", 0.95), None),
+ ("unsure_model", 3, _verdict("unhappy", 0.5), None),
+ ("new_reader", 2, _verdict("unhappy", 0.9), None),
+ ("tips_turned_off", 3, _verdict("unhappy", 0.9), "opted_out_at"),
+ ("invitations_turned_off", 3, _verdict("unhappy", 0.9), "interview_opted_out_at"),
+ ("tip_under_the_same_answer", 3, _verdict("unhappy", 0.9), "last_shown_at"),
+ ]
+ )
+ def test_no_invitation_when_a_gate_fails(self, _name, threads, client, stamped_field):
+ self._threads(threads)
+ if stamped_field:
+ update_state(WORKSPACE, SLACK_USER, lambda state: setattr(state, stamped_field, timezone.now()))
+
+ with patch.object(interview_recruitment, "build_system_one_client", return_value=client):
+ assert self._offer() is False
+ self.post.assert_not_called()
+
+ def test_a_failed_post_keeps_the_readers_slot(self):
+ self._threads(3)
+ self.post.side_effect = [RuntimeError("channel_not_found"), {"ok": True}]
+
+ assert [self._offer(), self._offer()] == [False, True]
+
+ def test_a_thumbs_down_reaction_needs_no_model(self):
+ self._threads(3)
+
+ with patch.object(interview_recruitment, "build_system_one_client", side_effect=RuntimeError) as model:
+ assert self._offer(signal="reaction") is True
+ model.assert_not_called()
+
+ def test_dont_ask_again_stops_every_later_invitation(self):
+ self._threads(3)
+ value = json.dumps({"integration_id": self.integration.id, "surface": "frustration"})
+
+ with patch.object(interview_recruitment, "post_response_url"):
+ handle_interview_action(
+ {
+ "team": {"id": WORKSPACE},
+ "user": {"id": SLACK_USER},
+ "response_url": "https://hooks.slack.com/actions/x",
+ "actions": [{"action_id": INTERVIEW_OPT_OUT_ACTION_ID, "value": value}],
+ }
+ )
+
+ assert get_state(WORKSPACE, SLACK_USER).interview_opted_out_at is not None
+ assert self.capture.call_args.args[1] == "slack app interview invites turned off"
+ assert self._offer(signal="reaction") is False
diff --git a/products/slack_app/evals/eval_interview_sentiment.py b/products/slack_app/evals/eval_interview_sentiment.py
new file mode 100644
index 000000000000..d7602a565751
--- /dev/null
+++ b/products/slack_app/evals/eval_interview_sentiment.py
@@ -0,0 +1,89 @@
+"""Is the person unhappy with the PostHog assistant in Slack?
+
+A yes, together with the cheap gates, sends a private invitation to book a research interview.
+`no_unasked_invite` is the number to watch: an invitation to a calm person reads as spam. A missed
+unhappy message only costs one invitation.
+
+To run:
+ hogli evals eval_interview_sentiment
+ hogli evals eval_interview_sentiment --eval useless_answer
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from products.posthog_ai.eval_harness.config import BaseEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
+from products.posthog_ai.eval_harness.one_shot import OneShotEval
+from products.slack_app.backend.logic import interview_recruitment
+from products.slack_app.evals.offline_config import offline_suite
+from products.slack_app.evals.scorers import INTERVIEW_SENTIMENT_KEY, InterviewSentimentMatch, NoUnaskedInvite
+
+SUITE_KIND = SuiteKind.ONE_SHOT
+
+
+def _case(name: str, prompt: str, unhappy: bool) -> BaseEvalCase:
+ return BaseEvalCase(name=name, prompt=prompt, expected={INTERVIEW_SENTIMENT_KEY: {"unhappy": unhappy}})
+
+
+UNHAPPY_CASES = [
+ _case("useless_answer", "this is useless, you didn't even look at the right dashboard", True),
+ _case("failed_again", "third time asking. the signup count is still wrong. can you actually check it?", True),
+ _case("sarcasm", "great, another chart that answers a question I never asked", True),
+ _case("angry_caps", "WHY does it keep opening a PR when I only asked a question??", True),
+ _case("giving_up", "honestly I'll just write the SQL myself, this bot never gets it right", True),
+ _case("slow_complaint", "you took ten minutes to tell me you can't find the event. what's the point", True),
+ _case("disappointed", "I expected more, the summary skipped every number I care about", True),
+ _case("product_complaint", "PostHog keeps logging me out and now the bot can't see my project either", True),
+ _case("non_english", "to je úplně špatně, zase jsi ignoroval ten projekt co jsem napsal", True),
+]
+
+NOT_UNHAPPY_CASES = [
+ _case("plain_question", "how many people signed up last week?", False),
+ _case("terse_instruction", "funnel from pageview to signed_up, last 30 days", False),
+ _case("calm_correction", "small thing: I meant the mobile project, not web. can you rerun it?", False),
+ _case("thanks", "perfect, that's exactly what I needed, thanks!", False),
+ _case("urgent_but_calm", "urgent please, the board meeting is in 20 minutes and I need churn by plan", False),
+ # Unhappy, but about their own product or other people, not about the assistant.
+ _case("own_product_bug", "our checkout is broken again and customers are furious, find the errors", False),
+ _case("annoyed_at_colleague", "ugh, someone deleted the onboarding dashboard. can you rebuild it?", False),
+ _case("negative_metric", "retention dropped badly this month, what changed?", False),
+]
+
+
+async def eval_interview_sentiment(ctx: EvalContext) -> None:
+ async def task(case: BaseEvalCase, task_ctx: EvalContext) -> dict:
+ if task_ctx.demo_data is None:
+ return {"unhappy": None, "error": "No demo data team to bill the call to"}
+ try:
+ verdict = await asyncio.to_thread(
+ interview_recruitment.classify_sentiment,
+ case.prompt,
+ team_id=task_ctx.demo_data.master_team_id,
+ distinct_id=None,
+ trace_id=None,
+ )
+ except Exception as error:
+ return {"unhappy": None, "error": f"{type(error).__name__}: {error}"}
+ # The same rule production applies: an unhappy verdict below the threshold counts as no.
+ unhappy = verdict is not None and verdict.is_unhappy
+ return {
+ "unhappy": unhappy,
+ "verdict": {"choice": verdict.choice, "confidence": verdict.confidence} if verdict else None,
+ "last_message": f"unhappy={unhappy}",
+ }
+
+ await OneShotEval(
+ experiment_name="slack-app-interview-sentiment",
+ offline_suite=offline_suite(
+ "slack_app/eval_interview_sentiment::eval_interview_sentiment",
+ metrics=[INTERVIEW_SENTIMENT_KEY, "no_unasked_invite"],
+ suite_kind=SUITE_KIND,
+ ),
+ cases=[*UNHAPPY_CASES, *NOT_UNHAPPY_CASES],
+ scorers=[InterviewSentimentMatch(), NoUnaskedInvite()],
+ task=task,
+ ctx=ctx,
+ )
diff --git a/products/slack_app/evals/offline_config.py b/products/slack_app/evals/offline_config.py
index a37ed60f777a..f8e0c7c0d4c8 100644
--- a/products/slack_app/evals/offline_config.py
+++ b/products/slack_app/evals/offline_config.py
@@ -14,6 +14,10 @@ _SCORERS: dict[str, dict[str, OfflineScorer]] = {
"followup_routing": OfflineScorer(kind="boolean", version=UUID("01a1557c-d1ca-7c68-96eb-5406fd8dde02")),
"no_unasked_wake": OfflineScorer(kind="boolean", version=UUID("01a1557c-dd52-78d0-a1f1-1854076c09ac")),
},
+ "interview_sentiment": {
+ "interview_sentiment": OfflineScorer(kind="boolean", version=UUID("01a1783a-9ee4-73a5-bbdf-fe91e943eb96")),
+ "no_unasked_invite": OfflineScorer(kind="boolean", version=UUID("01a1783a-9ee4-73a5-bbdf-fe92235d3e9c")),
+ },
"model_classifier": {
"exit_code_zero": OfflineScorer(kind="boolean", version=UUID("01a1557c-e620-710d-82d2-de63990000f1")),
"model_override_match": OfflineScorer(kind="boolean", version=UUID("01a1557c-ee78-7412-9b90-2ecf80591aa0")),
diff --git a/products/slack_app/evals/scorers.py b/products/slack_app/evals/scorers.py
index 38c18295b958..3a2e68252beb 100644
--- a/products/slack_app/evals/scorers.py
+++ b/products/slack_app/evals/scorers.py
@@ -560,3 +560,50 @@ class NoForcedTip(Scorer):
score=1.0 if got == "none" else 0.0,
metadata={"actual": got, "verdict": (output or {}).get("verdict")},
)
+
+
+INTERVIEW_SENTIMENT_KEY = "interview_sentiment"
+
+
+class InterviewSentimentMatch(Scorer):
+ def _name(self) -> str:
+ return INTERVIEW_SENTIMENT_KEY
+
+ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:
+ want = (expected or {}).get(INTERVIEW_SENTIMENT_KEY) or {}
+ if "unhappy" not in want:
+ return Score(name=self._name(), score=None, metadata={"reason": "No expectation for this case"})
+ if output and output.get("error"):
+ return Score(name=self._name(), score=0.0, metadata={"reason": output["error"]})
+
+ got = (output or {}).get("unhappy")
+ return Score(
+ name=self._name(),
+ score=1.0 if got == want["unhappy"] else 0.0,
+ metadata={"expected": want["unhappy"], "actual": got, "verdict": (output or {}).get("verdict")},
+ )
+
+
+class NoUnaskedInvite(Scorer):
+ """The expensive direction: an interview invitation to a person who is not unhappy reads as spam.
+
+ Skips on unhappy cases, so the score reads as a rate over the calm messages.
+ """
+
+ def _name(self) -> str:
+ return "no_unasked_invite"
+
+ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:
+ want = (expected or {}).get(INTERVIEW_SENTIMENT_KEY) or {}
+ if want.get("unhappy") is not False:
+ return Score(name=self._name(), score=None, metadata={"reason": "Case is unhappy"})
+ if output and output.get("error"):
+ # A failed call invites nobody, this scorer's passing answer, so it would hide an outage.
+ return Score(name=self._name(), score=None, metadata={"reason": output["error"]})
+
+ got = (output or {}).get("unhappy")
+ return Score(
+ name=self._name(),
+ score=1.0 if got is False else 0.0,
+ metadata={"actual": got, "verdict": (output or {}).get("verdict")},
+ )