:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(slack-app): invite frustrated users to an interview (#123274)
Vojta Bartošea304fcf58b1
diff --git a/products/slack_app/backend/logic/feature_tips/state.py b/products/slack_app/backend/logic/feature_tips/state.pyindex 521d9f5b9901..0edb5ed29c3b 100644--- a/products/slack_app/backend/logic/feature_tips/state.py+++ b/products/slack_app/backend/logic/feature_tips/state.py@@ -41,6 +41,9 @@ class FeatureTipsState(BaseModel):shown: dict[str, TipRecord] = {}opted_out_at: datetime | None = Nonelast_shown_at: datetime | None = None+ # Interview invitations share this row so they need no migration. They have their own limit.+ interview_invited_at: datetime | None = None+ interview_opted_out_at: datetime | None = Nonedef parse_state(raw: object) -> FeatureTipsState:diff --git a/products/slack_app/backend/logic/interview_recruitment.py b/products/slack_app/backend/logic/interview_recruitment.pyindex 0693c7684a3c..4f1192cdaf29 100644--- a/products/slack_app/backend/logic/interview_recruitment.py+++ b/products/slack_app/backend/logic/interview_recruitment.py@@ -3,42 +3,111 @@Every invitation links to ``BOOKING_URL``. A click arrives as a ``block_actions`` payload, which ishow the click is counted.+Three surfaces show an invitation:+- The bad-rating modal always shows the link, because the reader already chose to tell us what went wrong.- The Home tab shows a button when the ``slack-app-interview-recruitment`` flag selects the reader.+- A private message under the answer offers the interview when the flag selects the reader and the+ reader shows frustration: a thumbs-down reaction on an answer, or a message that the decision model+ reads as unhappy with the assistant or PostHog. A reader gets at most one such message in ``INVITE_INTERVAL``, and+ "Don't ask again" stops these messages only. The model runs only after every cheap gate passes, so+ most answers cost no call."""from __future__ import annotationsimport json+import uuid+from datetime import datetime, timedeltafrom typing import TYPE_CHECKING, Any, Literal, get_argsfrom django.http import HttpResponse+from django.utils import timezoneimport structlogfrom posthog.dataclasses import frozen-from posthog.models.integration import Integration+from posthog.llm.managed_decision_model import DEFAULT_DECISION_MODEL+from posthog.llm.system_one import ChoiceAnswer, ChoiceQuestion+from posthog.llm.system_one_client import build_system_one_client+from posthog.models.integration import Integration, SlackIntegrationfrom products.slack_app.backend.analytics import capture_slack_event, slack_distinct_idfrom products.slack_app.backend.feature_flags import is_slack_app_interview_recruitment_enabled-from products.slack_app.backend.services.slack_messages import section_block+from products.slack_app.backend.logic.feature_tips.state import FeatureTipsState, get_state, update_state+from products.slack_app.backend.models import SlackThreadTaskMapping+from products.slack_app.backend.services.inbox_interactivity import post_response_url+from products.slack_app.backend.services.slack_messages import post_slack_ephemeral, section_blockfrom products.slack_app.backend.services.slack_user_oauth import find_linked_posthog_user+from products.slack_app.backend.temporal.activities.classifiers import CLASSIFIER_PROPERTYif TYPE_CHECKING:from posthog.models.user import Userlogger = structlog.get_logger(__name__)-InterviewSurface = Literal["home", "bad_rating_modal"]+InterviewSurface = Literal["home", "bad_rating_modal", "frustration"]INTERVIEW_SURFACES: frozenset[str] = frozenset(get_args(InterviewSurface))+FrustrationSignal = Literal["message", "reaction"]BOOKING_URL = "https://cal.com/vojta-bartos-wye08f/posthog-slack-app"INVITE_MESSAGE = "Got 30 minutes to tell us how the PostHog Slack app works for you? Book a chat with the team."MODAL_INVITE_MESSAGE = "Want to talk it through? Book 30 minutes with the team."+# A first-time reader gets answers before any invitation, like the feature tips.+MIN_INTERACTIONS = 3+INVITE_INTERVAL = timedelta(days=30)+# A tip this recent probably sits under the answer the reader just reacted to.+TIP_GAP = timedelta(hours=1)INTERVIEW_BOOK_ACTION_ID = "slack_app_interview:book"-INTERVIEW_ACTION_IDS: frozenset[str] = frozenset({INTERVIEW_BOOK_ACTION_ID})-_CLICK_EVENTS: dict[str, str] = {INTERVIEW_BOOK_ACTION_ID: "slack app interview invite clicked"}+INTERVIEW_DISMISS_ACTION_ID = "slack_app_interview:dismiss"+INTERVIEW_OPT_OUT_ACTION_ID = "slack_app_interview:opt_out"+INTERVIEW_ACTION_IDS: frozenset[str] = frozenset(+ {INTERVIEW_BOOK_ACTION_ID, INTERVIEW_DISMISS_ACTION_ID, INTERVIEW_OPT_OUT_ACTION_ID}+)+_CLICK_EVENTS: dict[str, str] = {+ INTERVIEW_BOOK_ACTION_ID: "slack app interview invite clicked",+ INTERVIEW_DISMISS_ACTION_ID: "slack app interview invite dismissed",+ INTERVIEW_OPT_OUT_ACTION_ID: "slack app interview invites turned off",+}+INVITES_OFF_TEXT = "👋 Got it. I won't ask you again."++# Not the `slack_app` product the other classifiers bill under: recruitment serves PostHog, so it+# must not spend the team's AI credits.+SENTIMENT_AI_PRODUCT = "slack_app_interview_recruitment"+# The answer is already in the thread. A slow call only drops the invitation.+SENTIMENT_TIMEOUT_SECONDS = 5.0+# A wrong invitation to a calm person costs trust, and a missed one costs nothing, so the bar is high.+# Tuned in products/slack_app/evals/eval_interview_sentiment.py.+SENTIMENT_MIN_CONFIDENCE = 0.7+SENTIMENT_QUESTION_ID = "sentiment"+UNHAPPY = "unhappy"+NOT_UNHAPPY = "not_unhappy"+# The model needs the tone, not the whole message. This also keeps pasted logs out of the call.+SENTIMENT_MESSAGE_MAX_CHARS = 500+SENTIMENT_INSTRUCTIONS = (+ "`message` is what a person wrote to the PostHog assistant in Slack. "+ "Decide whether the person has an unhappy experience with the assistant or with PostHog. "+ "Judge the tone and the intent, in any language. Do not judge the topic."+)+SENTIMENT_CRITERIA = {+ UNHAPPY: "The person is frustrated, annoyed, angry, disappointed, or sarcastic about the assistant, "+ "its answers, or PostHog. Examples: a complaint, a statement that it failed or is useless, "+ "a repeated request after earlier failures, or a threat to stop using it.",+ NOT_UNHAPPY: "The person asks, instructs, thanks, or reports something in a neutral or positive tone. "+ "A calm correction, an urgent request, or a problem in their own product or with other people "+ "is not unhappiness with the assistant.",+}+++@frozen+class SentimentVerdict:+ choice: str+ confidence: float++ @property+ def is_unhappy(self) -> bool:+ return self.choice == UNHAPPY and self.confidence >= SENTIMENT_MIN_CONFIDENCEdef linked_posthog_user(integration: Integration, slack_user_id: str) -> User | None:@@ -80,6 +149,184 @@ def invite_section(text: str, button: dict[str, Any]) -> dict[str, Any]:return {**section_block(text), "accessory": button}+@frozen+class FrustrationInviteRequest:+ integration: Integration+ slack_user_id: str+ posthog_user: User | None+ channel: str+ thread_ts: str+ signal: FrustrationSignal+ person_message: str | None = None+ trace_id: str | None = None+++def offer_frustration_invite(request: FrustrationInviteRequest) -> bool:+ """Post a private invitation under the answer when the reader shows frustration.++ ``True`` means the invitation posted, so the caller sends no feature tip under the same answer.+ """+ integration = request.integration+ workspace_id = integration.integration_id or ""+ state = get_state(workspace_id, request.slack_user_id)+ now = timezone.now()+ # A reader who turned feature tips off asked for fewer prompts, so this prompt stays away too.+ if state.opted_out_at is not None or not _is_invite_due(state):+ return False+ # A reaction arrives after the answer, so a tip may already sit under that answer. No answer gets two prompts.+ if state.last_shown_at is not None and now - state.last_shown_at < TIP_GAP:+ return False+ if not is_recruiting(integration, request.slack_user_id, request.posthog_user):+ return False+ # The limit stops the count at the threshold, so a reader with a long history costs no full scan.+ thread_count = (+ SlackThreadTaskMapping.objects.filter(+ slack_workspace_id=workspace_id, mentioning_slack_user_id=request.slack_user_id+ )+ .values("id")[:MIN_INTERACTIONS]+ .count()+ )+ if thread_count < MIN_INTERACTIONS:+ return False+ if request.signal == "message" and not _is_unhappy(+ request.person_message or "",+ team_id=integration.team_id,+ distinct_id=request.posthog_user.distinct_id if request.posthog_user is not None else None,+ trace_id=request.trace_id,+ ):+ return False++ # The slot is held before the post, so two answers that finish together cannot both send one.+ reserved_at = _reserve_invite(workspace_id, request.slack_user_id)+ if reserved_at is None:+ return False+ invitation_id = str(uuid.uuid4())+ try:+ post_slack_ephemeral(+ SlackIntegration(integration).client,+ channel=request.channel,+ user=request.slack_user_id,+ thread_ts=request.thread_ts,+ text=INVITE_MESSAGE,+ blocks=frustration_invite_blocks(integration.id, invitation_id, request.trace_id),+ )+ except Exception:+ # The reader saw nothing, so the slot goes back.+ logger.warning("slack_app_interview_invite_post_failed", exc_info=True)+ _release_invite(workspace_id, request.slack_user_id, reserved_at)+ return False+ capture_slack_event(+ integration,+ "slack app interview invite shown",+ slack_user_id=request.slack_user_id,+ posthog_user=request.posthog_user,+ surface="frustration",+ invitation_id=invitation_id,+ signal=request.signal,+ ai_trace_id=request.trace_id,+ )+ return True+++def _is_invite_due(state: FeatureTipsState) -> bool:+ if state.interview_opted_out_at is not None:+ return False+ return state.interview_invited_at is None or timezone.now() - state.interview_invited_at >= INVITE_INTERVAL+++def _reserve_invite(slack_workspace_id: str, slack_user_id: str) -> datetime | None:+ """Hold this person's invitation slot, and return when. ``None`` means another invitation is too recent."""+ now = timezone.now()++ def change(state: FeatureTipsState) -> None:+ if _is_invite_due(state):+ state.interview_invited_at = now++ state = update_state(slack_workspace_id, slack_user_id, change)+ return now if state.interview_invited_at == now else None+++def _release_invite(slack_workspace_id: str, slack_user_id: str, reserved_at: datetime) -> None:+ """Give back a slot that ``_reserve_invite`` held for an invitation the reader never saw."""++ def change(state: FeatureTipsState) -> None:+ # A later reservation is another invitation's slot, so only this one goes back.+ if state.interview_invited_at == reserved_at:+ state.interview_invited_at = None++ try:+ update_state(slack_workspace_id, slack_user_id, change)+ except Exception:+ logger.warning("slack_app_interview_invite_release_failed", exc_info=True)+++def classify_sentiment(+ message: str, *, team_id: int, distinct_id: str | None, trace_id: str | None+) -> SentimentVerdict | None:+ """Ask the decision model whether the person is unhappy with the assistant. ``None`` means no verdict.++ Raises when no System One server is configured or the call fails, so the caller can tell an+ outage from a refusal.+ """+ text = message.strip()[:SENTIMENT_MESSAGE_MAX_CHARS]+ if not text:+ return None+ # No TypeSafe fallback: the message is customer text, and TypeSafe is a third party.+ client = build_system_one_client(+ model=DEFAULT_DECISION_MODEL,+ ai_product=SENTIMENT_AI_PRODUCT,+ team_id=team_id,+ distinct_id=distinct_id,+ trace_id=trace_id,+ properties={CLASSIFIER_PROPERTY: "interview_sentiment"},+ timeout=SENTIMENT_TIMEOUT_SECONDS,+ )+ result = client.decide(+ state={"message": text},+ questions={+ SENTIMENT_QUESTION_ID: ChoiceQuestion(instructions=SENTIMENT_INSTRUCTIONS, criteria=SENTIMENT_CRITERIA)+ },+ )+ answer = result.answers.get(SENTIMENT_QUESTION_ID)+ if not isinstance(answer, ChoiceAnswer):+ return None+ return SentimentVerdict(choice=answer.choice, confidence=answer.confidence)+++def _is_unhappy(message: str, *, team_id: int, distinct_id: str | None, trace_id: str | None) -> bool:+ """Whether the message reads as unhappy. Any failure reads as no, because a missed invitation costs nothing."""+ try:+ verdict = classify_sentiment(message, team_id=team_id, distinct_id=distinct_id, trace_id=trace_id)+ except Exception:+ logger.warning("slack_app_interview_sentiment_classifier_failed", exc_info=True)+ return False+ return verdict is not None and verdict.is_unhappy+++def frustration_invite_blocks(integration_id: int, invitation_id: str, trace_id: str | None) -> list[dict[str, Any]]:+ book = book_button(integration_id, "frustration", invitation_id=invitation_id, trace_id=trace_id)++ def button(action_id: str, label: str) -> dict[str, Any]:+ return {+ "type": "button",+ "action_id": action_id,+ "text": {"type": "plain_text", "text": label},+ "value": book["value"],+ }++ return [+ section_block(INVITE_MESSAGE),+ {+ "type": "actions",+ "elements": [+ book,+ button(INTERVIEW_DISMISS_ACTION_ID, "Not now"),+ button(INTERVIEW_OPT_OUT_ACTION_ID, "Don't ask again"),+ ],+ },+ ]++@frozenclass _InterviewClick:action_id: str@@ -113,9 +360,10 @@ def interview_integration_hint(payload: dict[str, Any]) -> int | None:def handle_interview_action(payload: dict[str, Any]) -> HttpResponse:- """Record a click on an invitation's booking button.+ """Record a click on an invitation's buttons.- Slack opens the booking page itself and sends this payload only so the click can be counted.+ For the booking button, Slack opens the booking page itself and sends this payload only so the+ click can be counted. The private message goes away after any click, because it did its job."""clicked = _clicked(payload)slack_user_id = (payload.get("user") or {}).get("id")@@ -137,12 +385,24 @@ def handle_interview_action(payload: dict[str, Any]) -> HttpResponse:if integration is None or surface not in INTERVIEW_SURFACES:return HttpResponse(status=200)+ response_url = payload.get("response_url") or ""+ if clicked.action_id == INTERVIEW_OPT_OUT_ACTION_ID:++ def change(state: FeatureTipsState) -> None:+ state.interview_opted_out_at = timezone.now()++ update_state(integration.integration_id or "", slack_user_id, change)+ post_response_url(response_url, {"replace_original": True, "text": INVITES_OFF_TEXT})+ elif response_url and (payload.get("container") or {}).get("is_ephemeral"):+ post_response_url(response_url, {"delete_original": True})+capture_slack_event(integration,_CLICK_EVENTS[clicked.action_id],slack_user_id=slack_user_id,posthog_user=linked_posthog_user(integration, slack_user_id),surface=surface,+ invitation_id=value.get("invitation_id"),task_run_id=value.get("run_id"),ai_trace_id=value.get("trace_id"),)diff --git a/products/slack_app/backend/services/turn_feedback.py b/products/slack_app/backend/services/turn_feedback.pyindex 482cb3a6f3ea..89377c8eb303 100644--- a/products/slack_app/backend/services/turn_feedback.py+++ b/products/slack_app/backend/services/turn_feedback.py@@ -569,4 +569,28 @@ def handle_reaction_added(event: dict, slack_team_id: str, workspace_integration},)record_feature_used(integration.integration_id, event.get("user"), SlackFeature.ANSWER_RATING)+ if rating == "bad" and target.slack_user_id:+ _offer_interview_after_reaction(target, channel, message.get("thread_ts") if message else None)return REACTION_HANDLED+++def _offer_interview_after_reaction(target: _FeedbackTarget, channel: str, thread_ts: Any) -> None:+ """A thumbs-down reaction opens no modal, so a private message carries the invitation instead.++ The offer runs on Celery, because the flag check and the post make network calls inside Slack's+ three-second ack budget.+ """+ # noqa reason: the task module imports api, which imports this module at load time.+ from products.slack_app.backend.tasks import offer_interview_invite_after_reaction # noqa: PLC0415++ try:+ offer_interview_invite_after_reaction.delay(+ team_id=target.integration.team_id,+ integration_id=target.integration.id,+ slack_user_id=target.slack_user_id,+ channel=channel,+ thread_ts=thread_ts if isinstance(thread_ts, str) and thread_ts else target.turn_id or "",+ trace_id=target.trace_id,+ )+ except Exception:+ logger.warning("slack_app_interview_invite_after_reaction_failed", run_id=target.run_id, exc_info=True)diff --git a/products/slack_app/backend/slack_thread.py b/products/slack_app/backend/slack_thread.pyindex 4234a2b6c860..64df3b26a475 100644--- a/products/slack_app/backend/slack_thread.py+++ b/products/slack_app/backend/slack_thread.py@@ -17,6 +17,7 @@ from posthog.slack.markdown import SLACK_MARKDOWN_TEXT_MAX_LEN, slack_markdown_bfrom products.slack_app.backend.analytics import capture_slack_event, slack_session_idfrom products.slack_app.backend.feature_flags import is_slack_app_forking_enabledfrom products.slack_app.backend.logic.feature_tips.offer import TipRequest, offer_feature_tip+from products.slack_app.backend.logic.interview_recruitment import FrustrationInviteRequest, offer_frustration_invitefrom products.slack_app.backend.logic.reply_suggestions import (ReplySuggestionAttribution,SuggestedReply,@@ -1128,8 +1129,25 @@ class SlackThreadHandler:)def offer_feature_tip(self) -> None:+ """Offer one private prompt under the answer: an interview invitation if the reader is frustrated, else a tip."""if not self.actor_slack_user_id:return+ try:+ if offer_frustration_invite(+ FrustrationInviteRequest(+ integration=self._get_integration(),+ slack_user_id=self.actor_slack_user_id,+ posthog_user=self._actor_posthog_user,+ channel=self.context.channel,+ thread_ts=self.context.thread_ts,+ signal="message",+ person_message=self.turn_person_message,+ trace_id=self.turn_trace_id,+ )+ ):+ return+ except Exception:+ logger.warning("slack_app_interview_invite_failed", exc_info=True)try:offer_feature_tip(TipRequest(diff --git a/products/slack_app/backend/tasks.py b/products/slack_app/backend/tasks.pyindex 4b45b802dec9..a72bbcc31e79 100644--- a/products/slack_app/backend/tasks.py+++ b/products/slack_app/backend/tasks.py@@ -18,6 +18,11 @@ from products.slack_app.backend.api import (does_other_region_claim_workspace,send_region_proxy_request,)+from products.slack_app.backend.logic.interview_recruitment import (+ FrustrationInviteRequest,+ linked_posthog_user,+ offer_frustration_invite,+)from products.slack_app.backend.slack_link_unfurl import add_artifact_image_to_unfurl, start_unfurl_chartsfrom products.slack_app.backend.temporal.types import SlackUnfurlChartRequest@@ -61,6 +66,10 @@ def refresh_user_github_repository_cache(*, user_integration_id: str) -> None:github.sync_repository_cache(min_refresh_interval_seconds=REPO_SNAPSHOT_REFRESH_INTERVAL_SECONDS)+def _load_integration(integration_id: int, team_id: int) -> Integration | None:+ return Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()++@shared_task(ignore_result=True)@with_team_scope()def start_slack_unfurl_charts(@@ -72,9 +81,7 @@ def start_slack_unfurl_charts(unfurls: dict[str, Any],charts: list[dict[str, Any]],) -> None:- integration = (- Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()- )+ integration = _load_integration(integration_id, team_id)user = User.objects.filter(id=user_id).first()if integration is None or user is None:return@@ -99,9 +106,7 @@ def add_slack_unfurl_artifact_image(url: str,title: str,) -> None:- integration = (- Integration.objects.select_related("team__organization").filter(id=integration_id, team_id=team_id).first()- )+ integration = _load_integration(integration_id, team_id)user = User.objects.filter(id=user_id).first()if integration is None or user is None:return@@ -113,3 +118,30 @@ def add_slack_unfurl_artifact_image(url=url,title=title,)+++@shared_task(ignore_result=True)+@with_team_scope()+def offer_interview_invite_after_reaction(+ *,+ team_id: int,+ integration_id: int,+ slack_user_id: str,+ channel: str,+ thread_ts: str,+ trace_id: str | None,+) -> None:+ integration = _load_integration(integration_id, team_id)+ if integration is None:+ return+ offer_frustration_invite(+ FrustrationInviteRequest(+ integration=integration,+ slack_user_id=slack_user_id,+ posthog_user=linked_posthog_user(integration, slack_user_id),+ channel=channel,+ thread_ts=thread_ts,+ signal="reaction",+ trace_id=trace_id,+ )+ )diff --git a/products/slack_app/backend/tests/logic/test_interview_recruitment.py b/products/slack_app/backend/tests/logic/test_interview_recruitment.pynew file mode 100644index 000000000000..7b997b9f0f85--- /dev/null+++ b/products/slack_app/backend/tests/logic/test_interview_recruitment.py@@ -0,0 +1,147 @@+import json++import pytest+from unittest.mock import MagicMock, patch++from django.apps import apps+from django.utils import timezone++from parameterized import parameterized++from posthog.llm.system_one import ChoiceAnswer, SystemOneResult+from posthog.models.integration import Integration+from posthog.models.organization import Organization+from posthog.models.team.team import Team++from products.slack_app.backend.logic import interview_recruitment+from products.slack_app.backend.logic.feature_tips.state import get_state, update_state+from products.slack_app.backend.logic.interview_recruitment import (+ BOOKING_URL,+ INTERVIEW_OPT_OUT_ACTION_ID,+ FrustrationInviteRequest,+ FrustrationSignal,+ handle_interview_action,+ offer_frustration_invite,+)+from products.slack_app.backend.models import SlackThreadTaskMapping++WORKSPACE = "T_INTERVIEW"+SLACK_USER = "U_READER"+++def _verdict(choice: str, confidence: float) -> MagicMock:+ client = MagicMock()+ client.decide.return_value = SystemOneResult(+ model="m",+ answers={"sentiment": ChoiceAnswer(choice=choice, confidence=confidence, probabilities={})},+ input_tokens=None,+ )+ return client+++class TestFrustrationInvite:+ @pytest.fixture(autouse=True)+ def setup(self):+ organization = Organization.objects.create(name="Org")+ self.team = Team.objects.create(organization=organization, name="Team")+ self.integration = Integration.objects.create(+ team=self.team, kind="slack", integration_id=WORKSPACE, sensitive_config={"access_token": "xoxb-test"}+ )+ with (+ patch.object(interview_recruitment, "is_slack_app_interview_recruitment_enabled", return_value=True),+ patch.object(interview_recruitment, "post_slack_ephemeral") as self.post,+ patch.object(interview_recruitment, "SlackIntegration"),+ patch.object(interview_recruitment, "capture_slack_event") as self.capture,+ patch.object(interview_recruitment, "build_system_one_client", return_value=_verdict("unhappy", 0.9)),+ ):+ yield++ def _threads(self, count: int) -> None:+ Task = apps.get_model("tasks", "Task")+ TaskRun = apps.get_model("tasks", "TaskRun")+ for index in range(count):+ task = Task.objects.create(team=self.team, title="t")+ SlackThreadTaskMapping.objects.create(+ team=self.team,+ integration=self.integration,+ slack_workspace_id=WORKSPACE,+ channel="C1",+ thread_ts=f"{index}.1",+ task=task,+ task_run=TaskRun.objects.create(team=self.team, task=task),+ mentioning_slack_user_id=SLACK_USER,+ )++ def _offer(self, signal: FrustrationSignal = "message") -> bool:+ return offer_frustration_invite(+ FrustrationInviteRequest(+ integration=self.integration,+ slack_user_id=SLACK_USER,+ posthog_user=None,+ channel="C1",+ thread_ts="1.1",+ signal=signal,+ person_message="this is the third time you got the funnel wrong",+ )+ )++ def test_a_frustrated_reader_is_invited_once_a_month(self):+ self._threads(3)++ assert [self._offer(), self._offer()] == [True, False]+ self.post.assert_called_once()+ book = self.post.call_args.kwargs["blocks"][1]["elements"][0]+ assert book["url"] == BOOKING_URL+ assert self.capture.call_args.args[1] == "slack app interview invite shown"+ assert self.capture.call_args.kwargs["surface"] == "frustration"++ @parameterized.expand(+ [+ ("calm_message", 3, _verdict("not_unhappy", 0.95), None),+ ("unsure_model", 3, _verdict("unhappy", 0.5), None),+ ("new_reader", 2, _verdict("unhappy", 0.9), None),+ ("tips_turned_off", 3, _verdict("unhappy", 0.9), "opted_out_at"),+ ("invitations_turned_off", 3, _verdict("unhappy", 0.9), "interview_opted_out_at"),+ ("tip_under_the_same_answer", 3, _verdict("unhappy", 0.9), "last_shown_at"),+ ]+ )+ def test_no_invitation_when_a_gate_fails(self, _name, threads, client, stamped_field):+ self._threads(threads)+ if stamped_field:+ update_state(WORKSPACE, SLACK_USER, lambda state: setattr(state, stamped_field, timezone.now()))++ with patch.object(interview_recruitment, "build_system_one_client", return_value=client):+ assert self._offer() is False+ self.post.assert_not_called()++ def test_a_failed_post_keeps_the_readers_slot(self):+ self._threads(3)+ self.post.side_effect = [RuntimeError("channel_not_found"), {"ok": True}]++ assert [self._offer(), self._offer()] == [False, True]++ def test_a_thumbs_down_reaction_needs_no_model(self):+ self._threads(3)++ with patch.object(interview_recruitment, "build_system_one_client", side_effect=RuntimeError) as model:+ assert self._offer(signal="reaction") is True+ model.assert_not_called()++ def test_dont_ask_again_stops_every_later_invitation(self):+ self._threads(3)+ value = json.dumps({"integration_id": self.integration.id, "surface": "frustration"})++ with patch.object(interview_recruitment, "post_response_url"):+ handle_interview_action(+ {+ "team": {"id": WORKSPACE},+ "user": {"id": SLACK_USER},+ "response_url": "https://hooks.slack.com/actions/x",+ "actions": [{"action_id": INTERVIEW_OPT_OUT_ACTION_ID, "value": value}],+ }+ )++ assert get_state(WORKSPACE, SLACK_USER).interview_opted_out_at is not None+ assert self.capture.call_args.args[1] == "slack app interview invites turned off"+ assert self._offer(signal="reaction") is Falsediff --git a/products/slack_app/evals/eval_interview_sentiment.py b/products/slack_app/evals/eval_interview_sentiment.pynew file mode 100644index 000000000000..d7602a565751--- /dev/null+++ b/products/slack_app/evals/eval_interview_sentiment.py@@ -0,0 +1,89 @@+"""Is the person unhappy with the PostHog assistant in Slack?++A yes, together with the cheap gates, sends a private invitation to book a research interview.+`no_unasked_invite` is the number to watch: an invitation to a calm person reads as spam. A missed+unhappy message only costs one invitation.++To run:+ hogli evals eval_interview_sentiment+ hogli evals eval_interview_sentiment --eval useless_answer+"""++from __future__ import annotations++import asyncio++from products.posthog_ai.eval_harness.config import BaseEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.posthog_ai.eval_harness.harness.requirements import SuiteKind+from products.posthog_ai.eval_harness.one_shot import OneShotEval+from products.slack_app.backend.logic import interview_recruitment+from products.slack_app.evals.offline_config import offline_suite+from products.slack_app.evals.scorers import INTERVIEW_SENTIMENT_KEY, InterviewSentimentMatch, NoUnaskedInvite++SUITE_KIND = SuiteKind.ONE_SHOT+++def _case(name: str, prompt: str, unhappy: bool) -> BaseEvalCase:+ return BaseEvalCase(name=name, prompt=prompt, expected={INTERVIEW_SENTIMENT_KEY: {"unhappy": unhappy}})+++UNHAPPY_CASES = [+ _case("useless_answer", "this is useless, you didn't even look at the right dashboard", True),+ _case("failed_again", "third time asking. the signup count is still wrong. can you actually check it?", True),+ _case("sarcasm", "great, another chart that answers a question I never asked", True),+ _case("angry_caps", "WHY does it keep opening a PR when I only asked a question??", True),+ _case("giving_up", "honestly I'll just write the SQL myself, this bot never gets it right", True),+ _case("slow_complaint", "you took ten minutes to tell me you can't find the event. what's the point", True),+ _case("disappointed", "I expected more, the summary skipped every number I care about", True),+ _case("product_complaint", "PostHog keeps logging me out and now the bot can't see my project either", True),+ _case("non_english", "to je úplně špatně, zase jsi ignoroval ten projekt co jsem napsal", True),+]++NOT_UNHAPPY_CASES = [+ _case("plain_question", "how many people signed up last week?", False),+ _case("terse_instruction", "funnel from pageview to signed_up, last 30 days", False),+ _case("calm_correction", "small thing: I meant the mobile project, not web. can you rerun it?", False),+ _case("thanks", "perfect, that's exactly what I needed, thanks!", False),+ _case("urgent_but_calm", "urgent please, the board meeting is in 20 minutes and I need churn by plan", False),+ # Unhappy, but about their own product or other people, not about the assistant.+ _case("own_product_bug", "our checkout is broken again and customers are furious, find the errors", False),+ _case("annoyed_at_colleague", "ugh, someone deleted the onboarding dashboard. can you rebuild it?", False),+ _case("negative_metric", "retention dropped badly this month, what changed?", False),+]+++async def eval_interview_sentiment(ctx: EvalContext) -> None:+ async def task(case: BaseEvalCase, task_ctx: EvalContext) -> dict:+ if task_ctx.demo_data is None:+ return {"unhappy": None, "error": "No demo data team to bill the call to"}+ try:+ verdict = await asyncio.to_thread(+ interview_recruitment.classify_sentiment,+ case.prompt,+ team_id=task_ctx.demo_data.master_team_id,+ distinct_id=None,+ trace_id=None,+ )+ except Exception as error:+ return {"unhappy": None, "error": f"{type(error).__name__}: {error}"}+ # The same rule production applies: an unhappy verdict below the threshold counts as no.+ unhappy = verdict is not None and verdict.is_unhappy+ return {+ "unhappy": unhappy,+ "verdict": {"choice": verdict.choice, "confidence": verdict.confidence} if verdict else None,+ "last_message": f"unhappy={unhappy}",+ }++ await OneShotEval(+ experiment_name="slack-app-interview-sentiment",+ offline_suite=offline_suite(+ "slack_app/eval_interview_sentiment::eval_interview_sentiment",+ metrics=[INTERVIEW_SENTIMENT_KEY, "no_unasked_invite"],+ suite_kind=SUITE_KIND,+ ),+ cases=[*UNHAPPY_CASES, *NOT_UNHAPPY_CASES],+ scorers=[InterviewSentimentMatch(), NoUnaskedInvite()],+ task=task,+ ctx=ctx,+ )diff --git a/products/slack_app/evals/offline_config.py b/products/slack_app/evals/offline_config.pyindex a37ed60f777a..f8e0c7c0d4c8 100644--- a/products/slack_app/evals/offline_config.py+++ b/products/slack_app/evals/offline_config.py@@ -14,6 +14,10 @@ _SCORERS: dict[str, dict[str, OfflineScorer]] = {"followup_routing": OfflineScorer(kind="boolean", version=UUID("01a1557c-d1ca-7c68-96eb-5406fd8dde02")),"no_unasked_wake": OfflineScorer(kind="boolean", version=UUID("01a1557c-dd52-78d0-a1f1-1854076c09ac")),},+ "interview_sentiment": {+ "interview_sentiment": OfflineScorer(kind="boolean", version=UUID("01a1783a-9ee4-73a5-bbdf-fe91e943eb96")),+ "no_unasked_invite": OfflineScorer(kind="boolean", version=UUID("01a1783a-9ee4-73a5-bbdf-fe92235d3e9c")),+ },"model_classifier": {"exit_code_zero": OfflineScorer(kind="boolean", version=UUID("01a1557c-e620-710d-82d2-de63990000f1")),"model_override_match": OfflineScorer(kind="boolean", version=UUID("01a1557c-ee78-7412-9b90-2ecf80591aa0")),diff --git a/products/slack_app/evals/scorers.py b/products/slack_app/evals/scorers.pyindex 38c18295b958..3a2e68252beb 100644--- a/products/slack_app/evals/scorers.py+++ b/products/slack_app/evals/scorers.py@@ -560,3 +560,50 @@ class NoForcedTip(Scorer):score=1.0 if got == "none" else 0.0,metadata={"actual": got, "verdict": (output or {}).get("verdict")},)+++INTERVIEW_SENTIMENT_KEY = "interview_sentiment"+++class InterviewSentimentMatch(Scorer):+ def _name(self) -> str:+ return INTERVIEW_SENTIMENT_KEY++ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:+ want = (expected or {}).get(INTERVIEW_SENTIMENT_KEY) or {}+ if "unhappy" not in want:+ return Score(name=self._name(), score=None, metadata={"reason": "No expectation for this case"})+ if output and output.get("error"):+ return Score(name=self._name(), score=0.0, metadata={"reason": output["error"]})++ got = (output or {}).get("unhappy")+ return Score(+ name=self._name(),+ score=1.0 if got == want["unhappy"] else 0.0,+ metadata={"expected": want["unhappy"], "actual": got, "verdict": (output or {}).get("verdict")},+ )+++class NoUnaskedInvite(Scorer):+ """The expensive direction: an interview invitation to a person who is not unhappy reads as spam.++ Skips on unhappy cases, so the score reads as a rate over the calm messages.+ """++ def _name(self) -> str:+ return "no_unasked_invite"++ def _run_eval_sync(self, output: dict | None, expected=None, **kwargs) -> Score:+ want = (expected or {}).get(INTERVIEW_SENTIMENT_KEY) or {}+ if want.get("unhappy") is not False:+ return Score(name=self._name(), score=None, metadata={"reason": "Case is unhappy"})+ if output and output.get("error"):+ # A failed call invites nobody, this scorer's passing answer, so it would hide an outage.+ return Score(name=self._name(), score=None, metadata={"reason": output["error"]})++ got = (output or {}).get("unhappy")+ return Score(+ name=self._name(),+ score=1.0 if got is False else 0.0,+ metadata={"actual": got, "verdict": (output or {}).get("verdict")},+ )