:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(conversations): let get ticket return more of the first message (#123892)
Chris Shine8cdbec7538a4
diff --git a/products/conversations/backend/api/internal.py b/products/conversations/backend/api/internal.pyindex 90581625f8f1..4725c55af065 100644--- a/products/conversations/backend/api/internal.py+++ b/products/conversations/backend/api/internal.py@@ -24,6 +24,7 @@ from posthog.models import Teamfrom posthog.scoped_service_jwt import ScopedServiceJwtPurposefrom products.conversations.backend.api.ticket_actions import (+ TicketActionGetQuerySerializer,TicketActionMessageResponseSerializer,TicketActionMessageSerializer,handle_ticket_get,@@ -57,14 +58,25 @@ class InternalTicketView(APIView):authentication_classes = [ConversationsTicketJWTAuthentication]permission_classes = [IsAuthenticated]+ @extend_schema(parameters=[TicketActionGetQuerySerializer])def get(self, request: Request, team_id: str, ticket_id: uuid.UUID) -> Response:team, error = _check_ticket_access(request, ticket_id)if error:return errorassert team is not NoneTICKET_ACTION_AUTH_COUNTER.labels(auth_method="scoped_jwt", http_method="get").inc()+ include_first_message = wants_first_customer_message_text(request)+ max_bytes = None+ if include_first_message:+ query = TicketActionGetQuerySerializer(data=request.query_params)+ if not query.is_valid():+ return Response({"error": query.errors}, status=status.HTTP_400_BAD_REQUEST)+ max_bytes = query.validated_data.get("first_customer_message_max_bytes")return handle_ticket_get(- team, ticket_id, include_first_customer_message_text=wants_first_customer_message_text(request)+ team,+ ticket_id,+ include_first_customer_message_text=include_first_message,+ first_customer_message_max_bytes=max_bytes,)def patch(self, request: Request, team_id: str, ticket_id: uuid.UUID) -> Response:diff --git a/products/conversations/backend/api/tests/test_external.py b/products/conversations/backend/api/tests/test_external.pyindex d18c0487ff82..258ae417cc10 100644--- a/products/conversations/backend/api/tests/test_external.py+++ b/products/conversations/backend/api/tests/test_external.py@@ -19,7 +19,7 @@ from posthog.test.api_keys import create_project_secret_api_keyfrom posthog.test.authentication_checks import covers_authenticationfrom products.conversations.backend.api.external import ExternalTicketProjectSecretAPIKeyAuthentication-from products.conversations.backend.api.ticket_actions import _truncate_bytes+from products.conversations.backend.api.ticket_actions import _truncate_bytes, _truncate_json_bytesfrom products.conversations.backend.models import Ticketfrom products.conversations.backend.models.constants import Priority, Status@@ -546,9 +546,12 @@ class TestExternalTicketAPI(BaseTest):Comment.objects.filter(pk=comment.pk).update(created_at=timezone.now() - timedelta(minutes=minutes_ago))return comment- def _get_first_customer_message_text(self):+ def _get_first_customer_message_text(self, max_bytes: int | None = None):"""GET the ticket with the opt-in flag and return the first_customer_message_text value."""- response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._auth_headers())+ params: dict[str, str | int] = {"include_first_customer_message_text": "true"}+ if max_bytes is not None:+ params["first_customer_message_max_bytes"] = max_bytes+ response = self.client.get(self.url, params, **self._auth_headers())self.assertEqual(response.status_code, status.HTTP_200_OK)return response.json()["first_customer_message_text"]@@ -660,17 +663,18 @@ class TestExternalTicketAPI(BaseTest):[# A cut message ends with an ellipsis in place of its last three characters, so the# 200-byte result is 197 "x" plus "...".- ("ascii", "x" * 900, "x" * 197 + "..."),+ ("ascii", "x" * 900, None, "x" * 197 + "..."),# 3 bytes per character, so the cap lands at 66 characters (198 bytes); the ellipsis# replaces the last three, leaving 63 characters plus "...". A character-based cap# would emit 600 bytes against the workflow variable budget.- ("multibyte", "あ" * 300, "あ" * 63 + "..."),+ ("multibyte", "あ" * 300, None, "あ" * 63 + "..."),+ ("length_ignored_on_this_route", "x" * 900, 3000, "x" * 197 + "..."),])- def test_get_ticket_truncates_first_message_to_200_bytes(self, _name, content, expected):+ def test_get_ticket_truncates_first_message_to_its_byte_budget(self, _name, content, max_bytes, expected):self._create_message(content, minutes_ago=10, item_context={"author_type": "customer"})- self.assertEqual(self._get_first_customer_message_text(), expected)+ self.assertEqual(self._get_first_customer_message_text(max_bytes), expected)def test_get_ticket_first_message_that_fits_has_no_ellipsis(self):# The ellipsis only marks a cut, so a message inside the budget is returned unchanged.@@ -1019,14 +1023,19 @@ class TestExternalTicketAPI(BaseTest):class TestTruncateBytes(SimpleTestCase):- def test_drops_a_joiner_the_cut_left_dangling(self):+ @parameterized.expand([("utf8_bytes", _truncate_bytes), ("json_bytes", _truncate_json_bytes)])+ def test_drops_a_joiner_the_cut_left_dangling(self, _name, truncate):# The cut lands inside the emoji following a joiner, so errors="ignore" drops the# partial character and would otherwise end the preview on the joiner itself.- truncated = _truncate_bytes("xxx" + "👨👩👧" * 20, 200)+ truncated = truncate("xxx" + "👨👩👧" * 20, 200)self.assertFalse(truncated.endswith(""))self.assertTrue(truncated.endswith("..."))self.assertLessEqual(len(truncated.encode("utf-8")), 200)- def test_returns_text_within_budget_unchanged(self):- self.assertEqual(_truncate_bytes("short enough", 200), "short enough")+ @parameterized.expand([("utf8_bytes", _truncate_bytes), ("json_bytes", _truncate_json_bytes)])+ def test_returns_text_within_budget_unchanged(self, _name, truncate):+ self.assertEqual(truncate("short enough", 200), "short enough")++ def test_json_budget_counts_escaped_quotes(self):+ self.assertEqual(_truncate_json_bytes('"' * 300, 200), '"' * 98 + "...")diff --git a/products/conversations/backend/api/tests/test_internal.py b/products/conversations/backend/api/tests/test_internal.pyindex 87b92fb10bd2..cec9ec3ee1e2 100644--- a/products/conversations/backend/api/tests/test_internal.py+++ b/products/conversations/backend/api/tests/test_internal.py@@ -1,9 +1,11 @@+import jsonimport uuidfrom datetime import timedeltafrom posthog.test.base import BaseTestfrom django.test import override_settings+from django.utils import timezonefrom parameterized import parameterizedfrom prometheus_client import REGISTRY@@ -20,9 +22,12 @@ from products.conversations.backend.api.internal import (CONVERSATIONS_TICKETS_PURPOSE,ConversationsTicketJWTAuthentication,)+from products.conversations.backend.api.ticket_actions import TICKET_RESPONSE_MAX_BYTESfrom products.conversations.backend.models import Ticketfrom products.conversations.backend.models.constants import WORKFLOW_AUTHOR_NAME, WORKFLOW_AUTHOR_TYPE, Status+_IMAGE_ATTACHMENT = ""+# Signed with this route's key but carrying another surface's audience — a token minted for a# different purpose must never authenticate here even when the signing key checks out.OTHER_AUDIENCE_PURPOSE = ScopedServiceJwtPurpose(@@ -69,16 +74,80 @@ class TestInternalTicketAPI(BaseTest):def test_get_returns_first_customer_message_text_when_opted_in(self):# The field is shared with the external route but gated behind a query param, so confirm# the JWT route reads that param and returns the preview through the same handler.+ self._create_customer_message("How do I export my data?")+ response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._headers())+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.json()["first_customer_message_text"], "How do I export my data?")++ def _create_customer_message(self, content: str) -> None:Comment.objects.create(team=self.team,scope="conversations_ticket",item_id=str(self.ticket.id),- content="How do I export my data?",+ content=content,item_context={"author_type": "customer"},)- response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._headers())++ def _get_first_message(self, max_bytes: int | str):+ return self.client.get(+ self.url,+ {"include_first_customer_message_text": "true", "first_customer_message_max_bytes": max_bytes},+ **self._headers(),+ )++ @parameterized.expand(+ [+ ("longer_budget", "x" * 9000, 2200, "x" * 2197 + "..."),+ ("clamped_up_to_the_preview", "x" * 900, 50, "x" * 197 + "..."),+ ("window_cut_is_marked", "It broke " + _IMAGE_ATTACHMENT * 70, 2200, "It broke..."),+ (+ "invisible_padding_stripped",+ "Refund\u034f\u034f please\u00ad now\u200b\u2060\ufeff ok"+ + "\u200c\u00a0" * 50+ + "\u200b\u200c" * 20+ + " می\u200cخواهم",+ 2200,+ "Refund please now ok می\u200cخواهم",+ ),+ ]+ )+ def test_get_returns_a_longer_first_message_when_asked(self, _name, content, max_bytes, expected):+ self._create_customer_message(content)++ response = self._get_first_message(max_bytes)+self.assertEqual(response.status_code, status.HTTP_200_OK)- self.assertEqual(response.json()["first_customer_message_text"], "How do I export my data?")+ self.assertEqual(response.json()["first_customer_message_text"], expected)++ def test_get_skips_a_first_message_that_is_only_a_cut_link(self):+ self._create_customer_message("[report.pdf](https://example.com/files/" + "a" * 5000 + ")")+ Comment.objects.filter(item_id=str(self.ticket.id)).update(created_at=timezone.now() - timedelta(minutes=10))+ self._create_customer_message("Actual question")++ response = self._get_first_message(2200)++ self.assertEqual(response.json()["first_customer_message_text"], "Actual question")++ def test_get_fits_a_longer_first_message_into_the_response_budget(self):+ self._create_customer_message("あ" * 3000)+ Ticket.objects.filter(id=self.ticket.id).update(email_subject="件名" * 100, last_message_text="あ" * 500)++ response = self._get_first_message(5000)++ self.assertEqual(response.status_code, status.HTTP_200_OK)+ body = response.json()+ self.assertLessEqual(+ len(json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()), TICKET_RESPONSE_MAX_BYTES+ )+ self.assertTrue(body["first_customer_message_text"].endswith("..."))++ def test_get_rejects_a_first_message_length_that_is_not_a_number(self):+ self._create_customer_message("Hello")++ response = self._get_first_message("lots")++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+ self.assertIn("first_customer_message_max_bytes", response.json()["error"])def test_post_sends_a_public_message(self):response = self.client.post(diff --git a/products/conversations/backend/api/ticket_actions.py b/products/conversations/backend/api/ticket_actions.pyindex 4b1d9606a0cd..6b9a116eae52 100644--- a/products/conversations/backend/api/ticket_actions.py+++ b/products/conversations/backend/api/ticket_actions.py@@ -2,13 +2,17 @@Two routes wrap the get and patch handlers with different authentication: the publicexternal route (legacy Team.secret_api_token bearer, api/external.py) and the internal-service route (scoped service JWT, api/internal.py). The worker selects between them per-environment by config presence (#82564), so those two handlers must behave identically.+service route (scoped service JWT, api/internal.py). The worker calls only the internal+route, and #82564 tracks retiring the external one. Until then the two behave identically,+except that only the internal route offers the longer first-message excerpt.Sending a message is JWT-only. The legacy route does not expose it."""import re+import jsonimport uuid+from collections.abc import Mapping+from itertools import accumulatefrom zoneinfo import ZoneInfo, ZoneInfoNotFoundErrorfrom django.db import connection, transaction@@ -18,6 +22,7 @@ from django.utils import timezonefrom rest_framework import serializers, statusfrom rest_framework.request import Requestfrom rest_framework.response import Response+from rest_framework.utils.encoders import JSONEncoderfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models import Tag, Team@@ -176,6 +181,11 @@ def workflow_trigger_from_request(request: Request) -> Trigger | None:# message can still cost roughly twice the figure below.FIRST_MESSAGE_PREVIEW_BYTES = 200+FIRST_MESSAGE_MAX_BYTES = 3000++# A longer excerpt fills only what the rest of the response leaves, to stay inside the worker's 5KB.+TICKET_RESPONSE_MAX_BYTES = 3584+# How far down the thread to look for a message with something quotable in it. Bounded so a# ticket that opens with a run of attachments cannot turn this into a scan of the whole thread.FIRST_MESSAGE_CANDIDATES = 5@@ -195,13 +205,6 @@ _LINK_MARKDOWN = re.compile(r"\[([^\[\]]*)\]\([^)]*\)")# ceiling is an order of magnitude wider than the preview it feeds.FIRST_MESSAGE_SCAN_CHARS = 2000-# The database fetches one character more than the scan window, never the whole body. The extra-# character is what lets `_quotable_text` still tell a cut body from a whole one: a body at or-# under the scan window comes back untouched, and only a longer one comes back at this length,-# which is strictly greater than the window. Fetch exactly the window and that test can no-# longer fire. Keep this one above FIRST_MESSAGE_SCAN_CHARS.-FIRST_MESSAGE_DB_FETCH_CHARS = FIRST_MESSAGE_SCAN_CHARS + 1-# Taking that window can cut an attachment in half, and a fragment like `]*)?)?$")# matches the same character set the outbound Slack formatter unescapes (posthog/comment/formatting.py)._MARKDOWN_ESCAPE = re.compile(r"\\([\\`*_{}\[\]()#+\-.!|])")+# Email padding characters. A non-joiner between letters stays, because Persian and Indic words need it.+_INVISIBLE_PADDING = re.compile(r"[\u00ad\u034f\u200b\u2060\ufeff]")+_PADDING_NON_JOINER = re.compile(r"(?:^|(?<=[\s\u200c]))\u200c|\u200c(?=[\s\u200c]|$)")+++def _strip_severed_link(window: str) -> str:+ # A link whose long URL runs past the window starts before the tail strip reaches it.+ opener = window.rfind("](")+ start = window.rfind("[", 0, opener) if opener != -1 and ")" not in window[opener:] else -1+ if start == -1:+ return window+ return window[: start - 1 if window[start - 1 : start] == "!" else start]-def _quotable_text(content: str, *, unescape_markdown: bool) -> str:- window = content[:FIRST_MESSAGE_SCAN_CHARS]++def _quotable_text(content: str, *, scan_chars: int, unescape_markdown: bool, strip_padding: bool = False) -> str:+ window = content[:scan_chars]# Only a window that actually cut the body can hold a severed construct. Applying this to a# whole short message would eat a trailing bracket the customer meant to type.- if len(content) > FIRST_MESSAGE_SCAN_CHARS:- window = _PARTIAL_MARKDOWN_TAIL.sub("", window)+ if len(content) > scan_chars:+ # Only the end can hold a severed attachment, and stripping the whole window is quadratic.+ head, tail = window[:-FIRST_MESSAGE_SCAN_CHARS], window[-FIRST_MESSAGE_SCAN_CHARS:]+ window = head + _PARTIAL_MARKDOWN_TAIL.sub("", tail)+ if head:+ window = _strip_severed_link(window)text = _LINK_MARKDOWN.sub(r"\1", _IMAGE_MARKDOWN.sub("", window))# Only the widget editor escapes the customer's punctuation on the way in (its escapeMarkdown),# so only a widget message is unescaped here. Email, Slack, Teams and imports store the text as# typed, where a backslash is the customer's own and must survive into the preview.if unescape_markdown:text = _MARKDOWN_ESCAPE.sub(r"\1", text)+ if strip_padding:+ text = _PADDING_NON_JOINER.sub("", _INVISIBLE_PADDING.sub("", text))return " ".join(text.split())@@ -247,10 +269,34 @@ def _truncate_bytes(text: str, max_bytes: int) -> str:return (truncated[:-3] if len(truncated) > 3 else "") + "..."-def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markdown: bool) -> str | None:+def _json_bytes(char: str) -> int:+ if char in '"\\':+ return 2+ if char < " ":+ return 6+ return len(char.encode("utf-8"))+++def _truncate_json_bytes(text: str, max_bytes: int, *, window_cut: bool = False) -> str:+ """Cut text to fit max_bytes once JSON-encoded, which is how the worker measures variables.++ window_cut says the scan window already cut the body, so the text ends with the ellipsis even+ when it fits. An AI step that reads it must not take a partial message for the whole one.+ """+ costs = [_json_bytes(char) for char in text]+ if not window_cut and sum(costs) <= max_bytes:+ return text+ kept = sum(1 for total in accumulate(costs) if total <= max_bytes - len("..."))+ return text[:kept].rstrip("\u200d") + "..."+++def _first_customer_message_text(+ team_id: int, ticket_id: str, *, max_bytes: int | None, unescape_markdown: bool+) -> str | None:"""Preview of what the customer first asked, so a workflow can remind them which ticket it- is writing about on channels that carry no email subject.+ is writing about on channels that carry no email subject. With max_bytes, a longer excerpt+ instead, so a workflow's AI steps screen and route on more than the opening line.Only the customer's own messages qualify, because a ticket can open with a team messagesuch as an agent composing outbound mail or a teammate's Slack post that seeded the@@ -263,6 +309,7 @@ def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markdunescape_markdown is on only for a widget ticket, whose editor markdown-escapes the customer'spunctuation. Other channels store the text as typed and must keep their backslashes."""+ scan_chars = FIRST_MESSAGE_SCAN_CHARS if max_bytes is None else max(FIRST_MESSAGE_SCAN_CHARS, 2 * max_bytes)candidates = (visible_ticket_messages(team_id, ticket_id).filter(item_context__author_type="customer")@@ -271,37 +318,67 @@ def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markd# Fetch only the scan window plus one character, so a 50,000-character email never# crosses the wire in full. The preview keeps the same value it would from the whole# body: the strip and truncation below never look past the window anyway.- .annotate(content_preview=Substr("content", 1, FIRST_MESSAGE_DB_FETCH_CHARS))+ # The extra character is how `_quotable_text` tells a cut body from a whole one.+ .annotate(content_preview=Substr("content", 1, scan_chars + 1))# Zendesk-imported messages carry second-resolution timestamps, so created_at alone can# tie. Tie-breaking on id keeps the preview stable from one call to the next..order_by("created_at", "id").values_list("content_preview", flat=True)[:FIRST_MESSAGE_CANDIDATES])for content in candidates:- quotable = _quotable_text(content or "", unescape_markdown=unescape_markdown)+ body = content or ""+ quotable = _quotable_text(+ body, scan_chars=scan_chars, unescape_markdown=unescape_markdown, strip_padding=max_bytes is not None+ )if quotable:- return _truncate_bytes(quotable, FIRST_MESSAGE_PREVIEW_BYTES) or None+ if max_bytes is None:+ return _truncate_bytes(quotable, FIRST_MESSAGE_PREVIEW_BYTES) or None+ return _truncate_json_bytes(quotable, max_bytes, window_cut=len(body) > scan_chars)return None+def _first_message_max_bytes(requested: int, payload: Mapping[str, object]) -> int:+ with_empty_excerpt = {**payload, "first_customer_message_text": ""}+ rest = len(json.dumps(with_empty_excerpt, cls=JSONEncoder, ensure_ascii=False, separators=(",", ":")).encode())+ return max(FIRST_MESSAGE_PREVIEW_BYTES, min(requested, FIRST_MESSAGE_MAX_BYTES, TICKET_RESPONSE_MAX_BYTES - rest))++def wants_first_customer_message_text(request: Request) -> bool:"""True when the caller opted into first_customer_message_text with ?include_...=true.- The CDP async function only appends the parameter when the workflow enabled it, so it- arrives as the literal "true" or not at all. Parsing both routes the same way keeps them- identical.+ The CDP async function only appends the parameter when the workflow enabled it or set a+ length, so it arrives as the literal "true" or not at all. Parsing both routes the same way+ keeps them identical."""return request.query_params.get("include_first_customer_message_text", "").lower() == "true"+class TicketActionGetQuerySerializer(serializers.Serializer):+ first_customer_message_max_bytes = serializers.IntegerField(+ required=False,+ help_text=(+ "Return up to this many bytes of the customer's first message instead of the "+ f"{FIRST_MESSAGE_PREVIEW_BYTES}-byte preview, counted after JSON encoding. Values are clamped "+ f"to {FIRST_MESSAGE_PREVIEW_BYTES}-{FIRST_MESSAGE_MAX_BYTES}, and the excerpt is shorter when "+ f"the rest of the ticket leaves less room under {TICKET_RESPONSE_MAX_BYTES} bytes, but never below "+ f"the {FIRST_MESSAGE_PREVIEW_BYTES}-byte preview. Only used "+ "with include_first_customer_message_text=true."+ ),+ )++def handle_ticket_get(- team: Team, ticket_id: str | uuid.UUID, *, include_first_customer_message_text: bool = False+ team: Team,+ ticket_id: str | uuid.UUID,+ *,+ include_first_customer_message_text: bool = False,+ first_customer_message_max_bytes: int | None = None,) -> Response:"""Fetch a ticket's data for an already-authenticated team.first_customer_message_text is opt-in: it runs an extra query, so a caller that does not ask- for it (the default) pays nothing and gets the response shape it always got. Only the CSAT- workflows request it.+ for it (the default) pays nothing and gets the response shape it always got. CSAT workflows+ request the short preview, and triage workflows ask for a longer excerpt."""if error := validate_ticket_id(ticket_id):return error@@ -361,7 +438,12 @@ def handle_ticket_get(if include_first_customer_message_text:payload["first_customer_message_text"] = _first_customer_message_text(- team.id, str(ticket.id), unescape_markdown=ticket.channel_source == Channel.WIDGET+ team.id,+ str(ticket.id),+ max_bytes=None+ if first_customer_message_max_bytes is None+ else _first_message_max_bytes(first_customer_message_max_bytes, payload),+ unescape_markdown=ticket.channel_source == Channel.WIDGET,)return Response(payload) # nosemgrep: api-response-must-match-schema