Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(conversations): let get ticket return more of the first message (#123892)

Chris Shine8cdbec7538a4
diff --git a/products/conversations/backend/api/internal.py b/products/conversations/backend/api/internal.py
index 90581625f8f1..4725c55af065 100644
--- a/products/conversations/backend/api/internal.py
+++ b/products/conversations/backend/api/internal.py
@@ -24,6 +24,7 @@ from posthog.models import Team
from posthog.scoped_service_jwt import ScopedServiceJwtPurpose
from products.conversations.backend.api.ticket_actions import (
+ TicketActionGetQuerySerializer,
TicketActionMessageResponseSerializer,
TicketActionMessageSerializer,
handle_ticket_get,
@@ -57,14 +58,25 @@ class InternalTicketView(APIView):
authentication_classes = [ConversationsTicketJWTAuthentication]
permission_classes = [IsAuthenticated]
+ @extend_schema(parameters=[TicketActionGetQuerySerializer])
def get(self, request: Request, team_id: str, ticket_id: uuid.UUID) -> Response:
team, error = _check_ticket_access(request, ticket_id)
if error:
return error
assert team is not None
TICKET_ACTION_AUTH_COUNTER.labels(auth_method="scoped_jwt", http_method="get").inc()
+ include_first_message = wants_first_customer_message_text(request)
+ max_bytes = None
+ if include_first_message:
+ query = TicketActionGetQuerySerializer(data=request.query_params)
+ if not query.is_valid():
+ return Response({"error": query.errors}, status=status.HTTP_400_BAD_REQUEST)
+ max_bytes = query.validated_data.get("first_customer_message_max_bytes")
return handle_ticket_get(
- team, ticket_id, include_first_customer_message_text=wants_first_customer_message_text(request)
+ team,
+ ticket_id,
+ include_first_customer_message_text=include_first_message,
+ first_customer_message_max_bytes=max_bytes,
)
def patch(self, request: Request, team_id: str, ticket_id: uuid.UUID) -> Response:
diff --git a/products/conversations/backend/api/tests/test_external.py b/products/conversations/backend/api/tests/test_external.py
index d18c0487ff82..258ae417cc10 100644
--- a/products/conversations/backend/api/tests/test_external.py
+++ b/products/conversations/backend/api/tests/test_external.py
@@ -19,7 +19,7 @@ from posthog.test.api_keys import create_project_secret_api_key
from posthog.test.authentication_checks import covers_authentication
from products.conversations.backend.api.external import ExternalTicketProjectSecretAPIKeyAuthentication
-from products.conversations.backend.api.ticket_actions import _truncate_bytes
+from products.conversations.backend.api.ticket_actions import _truncate_bytes, _truncate_json_bytes
from products.conversations.backend.models import Ticket
from products.conversations.backend.models.constants import Priority, Status
@@ -546,9 +546,12 @@ class TestExternalTicketAPI(BaseTest):
Comment.objects.filter(pk=comment.pk).update(created_at=timezone.now() - timedelta(minutes=minutes_ago))
return comment
- def _get_first_customer_message_text(self):
+ def _get_first_customer_message_text(self, max_bytes: int | None = None):
"""GET the ticket with the opt-in flag and return the first_customer_message_text value."""
- response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._auth_headers())
+ params: dict[str, str | int] = {"include_first_customer_message_text": "true"}
+ if max_bytes is not None:
+ params["first_customer_message_max_bytes"] = max_bytes
+ response = self.client.get(self.url, params, **self._auth_headers())
self.assertEqual(response.status_code, status.HTTP_200_OK)
return response.json()["first_customer_message_text"]
@@ -660,17 +663,18 @@ class TestExternalTicketAPI(BaseTest):
[
# A cut message ends with an ellipsis in place of its last three characters, so the
# 200-byte result is 197 "x" plus "...".
- ("ascii", "x" * 900, "x" * 197 + "..."),
+ ("ascii", "x" * 900, None, "x" * 197 + "..."),
# 3 bytes per character, so the cap lands at 66 characters (198 bytes); the ellipsis
# replaces the last three, leaving 63 characters plus "...". A character-based cap
# would emit 600 bytes against the workflow variable budget.
- ("multibyte", "あ" * 300, "あ" * 63 + "..."),
+ ("multibyte", "あ" * 300, None, "あ" * 63 + "..."),
+ ("length_ignored_on_this_route", "x" * 900, 3000, "x" * 197 + "..."),
]
)
- def test_get_ticket_truncates_first_message_to_200_bytes(self, _name, content, expected):
+ def test_get_ticket_truncates_first_message_to_its_byte_budget(self, _name, content, max_bytes, expected):
self._create_message(content, minutes_ago=10, item_context={"author_type": "customer"})
- self.assertEqual(self._get_first_customer_message_text(), expected)
+ self.assertEqual(self._get_first_customer_message_text(max_bytes), expected)
def test_get_ticket_first_message_that_fits_has_no_ellipsis(self):
# The ellipsis only marks a cut, so a message inside the budget is returned unchanged.
@@ -1019,14 +1023,19 @@ class TestExternalTicketAPI(BaseTest):
class TestTruncateBytes(SimpleTestCase):
- def test_drops_a_joiner_the_cut_left_dangling(self):
+ @parameterized.expand([("utf8_bytes", _truncate_bytes), ("json_bytes", _truncate_json_bytes)])
+ def test_drops_a_joiner_the_cut_left_dangling(self, _name, truncate):
# The cut lands inside the emoji following a joiner, so errors="ignore" drops the
# partial character and would otherwise end the preview on the joiner itself.
- truncated = _truncate_bytes("xxx" + "👨‍👩‍👧" * 20, 200)
+ truncated = truncate("xxx" + "👨‍👩‍👧" * 20, 200)
self.assertFalse(truncated.endswith("‍"))
self.assertTrue(truncated.endswith("..."))
self.assertLessEqual(len(truncated.encode("utf-8")), 200)
- def test_returns_text_within_budget_unchanged(self):
- self.assertEqual(_truncate_bytes("short enough", 200), "short enough")
+ @parameterized.expand([("utf8_bytes", _truncate_bytes), ("json_bytes", _truncate_json_bytes)])
+ def test_returns_text_within_budget_unchanged(self, _name, truncate):
+ self.assertEqual(truncate("short enough", 200), "short enough")
+
+ def test_json_budget_counts_escaped_quotes(self):
+ self.assertEqual(_truncate_json_bytes('"' * 300, 200), '"' * 98 + "...")
diff --git a/products/conversations/backend/api/tests/test_internal.py b/products/conversations/backend/api/tests/test_internal.py
index 87b92fb10bd2..cec9ec3ee1e2 100644
--- a/products/conversations/backend/api/tests/test_internal.py
+++ b/products/conversations/backend/api/tests/test_internal.py
@@ -1,9 +1,11 @@
+import json
import uuid
from datetime import timedelta
from posthog.test.base import BaseTest
from django.test import override_settings
+from django.utils import timezone
from parameterized import parameterized
from prometheus_client import REGISTRY
@@ -20,9 +22,12 @@ from products.conversations.backend.api.internal import (
CONVERSATIONS_TICKETS_PURPOSE,
ConversationsTicketJWTAuthentication,
)
+from products.conversations.backend.api.ticket_actions import TICKET_RESPONSE_MAX_BYTES
from products.conversations.backend.models import Ticket
from products.conversations.backend.models.constants import WORKFLOW_AUTHOR_NAME, WORKFLOW_AUTHOR_TYPE, Status
+_IMAGE_ATTACHMENT = "![image001.png](https://us.posthog.com/uploaded_media/0199a0b1-2c3d-4e5f-8a9b-0c1d2e3f4a5b)"
+
# Signed with this route's key but carrying another surface's audience — a token minted for a
# different purpose must never authenticate here even when the signing key checks out.
OTHER_AUDIENCE_PURPOSE = ScopedServiceJwtPurpose(
@@ -69,16 +74,80 @@ class TestInternalTicketAPI(BaseTest):
def test_get_returns_first_customer_message_text_when_opted_in(self):
# The field is shared with the external route but gated behind a query param, so confirm
# the JWT route reads that param and returns the preview through the same handler.
+ self._create_customer_message("How do I export my data?")
+ response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._headers())
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ self.assertEqual(response.json()["first_customer_message_text"], "How do I export my data?")
+
+ def _create_customer_message(self, content: str) -> None:
Comment.objects.create(
team=self.team,
scope="conversations_ticket",
item_id=str(self.ticket.id),
- content="How do I export my data?",
+ content=content,
item_context={"author_type": "customer"},
)
- response = self.client.get(self.url, {"include_first_customer_message_text": "true"}, **self._headers())
+
+ def _get_first_message(self, max_bytes: int | str):
+ return self.client.get(
+ self.url,
+ {"include_first_customer_message_text": "true", "first_customer_message_max_bytes": max_bytes},
+ **self._headers(),
+ )
+
+ @parameterized.expand(
+ [
+ ("longer_budget", "x" * 9000, 2200, "x" * 2197 + "..."),
+ ("clamped_up_to_the_preview", "x" * 900, 50, "x" * 197 + "..."),
+ ("window_cut_is_marked", "It broke " + _IMAGE_ATTACHMENT * 70, 2200, "It broke..."),
+ (
+ "invisible_padding_stripped",
+ "Refund\u034f\u034f please\u00ad now\u200b\u2060\ufeff ok"
+ + "\u200c\u00a0" * 50
+ + "\u200b\u200c" * 20
+ + " می\u200cخواهم",
+ 2200,
+ "Refund please now ok می\u200cخواهم",
+ ),
+ ]
+ )
+ def test_get_returns_a_longer_first_message_when_asked(self, _name, content, max_bytes, expected):
+ self._create_customer_message(content)
+
+ response = self._get_first_message(max_bytes)
+
self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(response.json()["first_customer_message_text"], "How do I export my data?")
+ self.assertEqual(response.json()["first_customer_message_text"], expected)
+
+ def test_get_skips_a_first_message_that_is_only_a_cut_link(self):
+ self._create_customer_message("[report.pdf](https://example.com/files/" + "a" * 5000 + ")")
+ Comment.objects.filter(item_id=str(self.ticket.id)).update(created_at=timezone.now() - timedelta(minutes=10))
+ self._create_customer_message("Actual question")
+
+ response = self._get_first_message(2200)
+
+ self.assertEqual(response.json()["first_customer_message_text"], "Actual question")
+
+ def test_get_fits_a_longer_first_message_into_the_response_budget(self):
+ self._create_customer_message("あ" * 3000)
+ Ticket.objects.filter(id=self.ticket.id).update(email_subject="件名" * 100, last_message_text="あ" * 500)
+
+ response = self._get_first_message(5000)
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ body = response.json()
+ self.assertLessEqual(
+ len(json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()), TICKET_RESPONSE_MAX_BYTES
+ )
+ self.assertTrue(body["first_customer_message_text"].endswith("..."))
+
+ def test_get_rejects_a_first_message_length_that_is_not_a_number(self):
+ self._create_customer_message("Hello")
+
+ response = self._get_first_message("lots")
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertIn("first_customer_message_max_bytes", response.json()["error"])
def test_post_sends_a_public_message(self):
response = self.client.post(
diff --git a/products/conversations/backend/api/ticket_actions.py b/products/conversations/backend/api/ticket_actions.py
index 4b1d9606a0cd..6b9a116eae52 100644
--- a/products/conversations/backend/api/ticket_actions.py
+++ b/products/conversations/backend/api/ticket_actions.py
@@ -2,13 +2,17 @@
Two routes wrap the get and patch handlers with different authentication: the public
external route (legacy Team.secret_api_token bearer, api/external.py) and the internal
-service route (scoped service JWT, api/internal.py). The worker selects between them per
-environment by config presence (#82564), so those two handlers must behave identically.
+service route (scoped service JWT, api/internal.py). The worker calls only the internal
+route, and #82564 tracks retiring the external one. Until then the two behave identically,
+except that only the internal route offers the longer first-message excerpt.
Sending a message is JWT-only. The legacy route does not expose it.
"""
import re
+import json
import uuid
+from collections.abc import Mapping
+from itertools import accumulate
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from django.db import connection, transaction
@@ -18,6 +22,7 @@ from django.utils import timezone
from rest_framework import serializers, status
from rest_framework.request import Request
from rest_framework.response import Response
+from rest_framework.utils.encoders import JSONEncoder
from posthog.exceptions_capture import capture_exception
from posthog.models import Tag, Team
@@ -176,6 +181,11 @@ def workflow_trigger_from_request(request: Request) -> Trigger | None:
# message can still cost roughly twice the figure below.
FIRST_MESSAGE_PREVIEW_BYTES = 200
+FIRST_MESSAGE_MAX_BYTES = 3000
+
+# A longer excerpt fills only what the rest of the response leaves, to stay inside the worker's 5KB.
+TICKET_RESPONSE_MAX_BYTES = 3584
+
# How far down the thread to look for a message with something quotable in it. Bounded so a
# ticket that opens with a run of attachments cannot turn this into a scan of the whole thread.
FIRST_MESSAGE_CANDIDATES = 5
@@ -195,13 +205,6 @@ _LINK_MARKDOWN = re.compile(r"\[([^\[\]]*)\]\([^)]*\)")
# ceiling is an order of magnitude wider than the preview it feeds.
FIRST_MESSAGE_SCAN_CHARS = 2000
-# The database fetches one character more than the scan window, never the whole body. The extra
-# character is what lets `_quotable_text` still tell a cut body from a whole one: a body at or
-# under the scan window comes back untouched, and only a longer one comes back at this length,
-# which is strictly greater than the window. Fetch exactly the window and that test can no
-# longer fire. Keep this one above FIRST_MESSAGE_SCAN_CHARS.
-FIRST_MESSAGE_DB_FETCH_CHARS = FIRST_MESSAGE_SCAN_CHARS + 1
-
# Taking that window can cut an attachment in half, and a fragment like `![shot.png](/uploaded`
# no longer matches the patterns above, so it would survive into the preview. This drops the
# unterminated construct a cut leaves behind. The run cannot cross a bracket, so a closed
@@ -217,19 +220,38 @@ _PARTIAL_MARKDOWN_TAIL = re.compile(r"!?\[[^\[\]]*(?:\](?:\([^)]*)?)?$")
# matches the same character set the outbound Slack formatter unescapes (posthog/comment/formatting.py).
_MARKDOWN_ESCAPE = re.compile(r"\\([\\`*_{}\[\]()#+\-.!|])")
+# Email padding characters. A non-joiner between letters stays, because Persian and Indic words need it.
+_INVISIBLE_PADDING = re.compile(r"[\u00ad\u034f\u200b\u2060\ufeff]")
+_PADDING_NON_JOINER = re.compile(r"(?:^|(?<=[\s\u200c]))\u200c|\u200c(?=[\s\u200c]|$)")
+
+
+def _strip_severed_link(window: str) -> str:
+ # A link whose long URL runs past the window starts before the tail strip reaches it.
+ opener = window.rfind("](")
+ start = window.rfind("[", 0, opener) if opener != -1 and ")" not in window[opener:] else -1
+ if start == -1:
+ return window
+ return window[: start - 1 if window[start - 1 : start] == "!" else start]
-def _quotable_text(content: str, *, unescape_markdown: bool) -> str:
- window = content[:FIRST_MESSAGE_SCAN_CHARS]
+
+def _quotable_text(content: str, *, scan_chars: int, unescape_markdown: bool, strip_padding: bool = False) -> str:
+ window = content[:scan_chars]
# Only a window that actually cut the body can hold a severed construct. Applying this to a
# whole short message would eat a trailing bracket the customer meant to type.
- if len(content) > FIRST_MESSAGE_SCAN_CHARS:
- window = _PARTIAL_MARKDOWN_TAIL.sub("", window)
+ if len(content) > scan_chars:
+ # Only the end can hold a severed attachment, and stripping the whole window is quadratic.
+ head, tail = window[:-FIRST_MESSAGE_SCAN_CHARS], window[-FIRST_MESSAGE_SCAN_CHARS:]
+ window = head + _PARTIAL_MARKDOWN_TAIL.sub("", tail)
+ if head:
+ window = _strip_severed_link(window)
text = _LINK_MARKDOWN.sub(r"\1", _IMAGE_MARKDOWN.sub("", window))
# Only the widget editor escapes the customer's punctuation on the way in (its escapeMarkdown),
# so only a widget message is unescaped here. Email, Slack, Teams and imports store the text as
# typed, where a backslash is the customer's own and must survive into the preview.
if unescape_markdown:
text = _MARKDOWN_ESCAPE.sub(r"\1", text)
+ if strip_padding:
+ text = _PADDING_NON_JOINER.sub("", _INVISIBLE_PADDING.sub("", text))
return " ".join(text.split())
@@ -247,10 +269,34 @@ def _truncate_bytes(text: str, max_bytes: int) -> str:
return (truncated[:-3] if len(truncated) > 3 else "") + "..."
-def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markdown: bool) -> str | None:
+def _json_bytes(char: str) -> int:
+ if char in '"\\':
+ return 2
+ if char < " ":
+ return 6
+ return len(char.encode("utf-8"))
+
+
+def _truncate_json_bytes(text: str, max_bytes: int, *, window_cut: bool = False) -> str:
+ """Cut text to fit max_bytes once JSON-encoded, which is how the worker measures variables.
+
+ window_cut says the scan window already cut the body, so the text ends with the ellipsis even
+ when it fits. An AI step that reads it must not take a partial message for the whole one.
+ """
+ costs = [_json_bytes(char) for char in text]
+ if not window_cut and sum(costs) <= max_bytes:
+ return text
+ kept = sum(1 for total in accumulate(costs) if total <= max_bytes - len("..."))
+ return text[:kept].rstrip("\u200d") + "..."
+
+
+def _first_customer_message_text(
+ team_id: int, ticket_id: str, *, max_bytes: int | None, unescape_markdown: bool
+) -> str | None:
"""
Preview of what the customer first asked, so a workflow can remind them which ticket it
- is writing about on channels that carry no email subject.
+ is writing about on channels that carry no email subject. With max_bytes, a longer excerpt
+ instead, so a workflow's AI steps screen and route on more than the opening line.
Only the customer's own messages qualify, because a ticket can open with a team message
such as an agent composing outbound mail or a teammate's Slack post that seeded the
@@ -263,6 +309,7 @@ def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markd
unescape_markdown is on only for a widget ticket, whose editor markdown-escapes the customer's
punctuation. Other channels store the text as typed and must keep their backslashes.
"""
+ scan_chars = FIRST_MESSAGE_SCAN_CHARS if max_bytes is None else max(FIRST_MESSAGE_SCAN_CHARS, 2 * max_bytes)
candidates = (
visible_ticket_messages(team_id, ticket_id)
.filter(item_context__author_type="customer")
@@ -271,37 +318,67 @@ def _first_customer_message_text(team_id: int, ticket_id: str, *, unescape_markd
# Fetch only the scan window plus one character, so a 50,000-character email never
# crosses the wire in full. The preview keeps the same value it would from the whole
# body: the strip and truncation below never look past the window anyway.
- .annotate(content_preview=Substr("content", 1, FIRST_MESSAGE_DB_FETCH_CHARS))
+ # The extra character is how `_quotable_text` tells a cut body from a whole one.
+ .annotate(content_preview=Substr("content", 1, scan_chars + 1))
# Zendesk-imported messages carry second-resolution timestamps, so created_at alone can
# tie. Tie-breaking on id keeps the preview stable from one call to the next.
.order_by("created_at", "id")
.values_list("content_preview", flat=True)[:FIRST_MESSAGE_CANDIDATES]
)
for content in candidates:
- quotable = _quotable_text(content or "", unescape_markdown=unescape_markdown)
+ body = content or ""
+ quotable = _quotable_text(
+ body, scan_chars=scan_chars, unescape_markdown=unescape_markdown, strip_padding=max_bytes is not None
+ )
if quotable:
- return _truncate_bytes(quotable, FIRST_MESSAGE_PREVIEW_BYTES) or None
+ if max_bytes is None:
+ return _truncate_bytes(quotable, FIRST_MESSAGE_PREVIEW_BYTES) or None
+ return _truncate_json_bytes(quotable, max_bytes, window_cut=len(body) > scan_chars)
return None
+def _first_message_max_bytes(requested: int, payload: Mapping[str, object]) -> int:
+ with_empty_excerpt = {**payload, "first_customer_message_text": ""}
+ rest = len(json.dumps(with_empty_excerpt, cls=JSONEncoder, ensure_ascii=False, separators=(",", ":")).encode())
+ return max(FIRST_MESSAGE_PREVIEW_BYTES, min(requested, FIRST_MESSAGE_MAX_BYTES, TICKET_RESPONSE_MAX_BYTES - rest))
+
+
def wants_first_customer_message_text(request: Request) -> bool:
"""True when the caller opted into first_customer_message_text with ?include_...=true.
- The CDP async function only appends the parameter when the workflow enabled it, so it
- arrives as the literal "true" or not at all. Parsing both routes the same way keeps them
- identical.
+ The CDP async function only appends the parameter when the workflow enabled it or set a
+ length, so it arrives as the literal "true" or not at all. Parsing both routes the same way
+ keeps them identical.
"""
return request.query_params.get("include_first_customer_message_text", "").lower() == "true"
+class TicketActionGetQuerySerializer(serializers.Serializer):
+ first_customer_message_max_bytes = serializers.IntegerField(
+ required=False,
+ help_text=(
+ "Return up to this many bytes of the customer's first message instead of the "
+ f"{FIRST_MESSAGE_PREVIEW_BYTES}-byte preview, counted after JSON encoding. Values are clamped "
+ f"to {FIRST_MESSAGE_PREVIEW_BYTES}-{FIRST_MESSAGE_MAX_BYTES}, and the excerpt is shorter when "
+ f"the rest of the ticket leaves less room under {TICKET_RESPONSE_MAX_BYTES} bytes, but never below "
+ f"the {FIRST_MESSAGE_PREVIEW_BYTES}-byte preview. Only used "
+ "with include_first_customer_message_text=true."
+ ),
+ )
+
+
def handle_ticket_get(
- team: Team, ticket_id: str | uuid.UUID, *, include_first_customer_message_text: bool = False
+ team: Team,
+ ticket_id: str | uuid.UUID,
+ *,
+ include_first_customer_message_text: bool = False,
+ first_customer_message_max_bytes: int | None = None,
) -> Response:
"""Fetch a ticket's data for an already-authenticated team.
first_customer_message_text is opt-in: it runs an extra query, so a caller that does not ask
- for it (the default) pays nothing and gets the response shape it always got. Only the CSAT
- workflows request it.
+ for it (the default) pays nothing and gets the response shape it always got. CSAT workflows
+ request the short preview, and triage workflows ask for a longer excerpt.
"""
if error := validate_ticket_id(ticket_id):
return error
@@ -361,7 +438,12 @@ def handle_ticket_get(
if include_first_customer_message_text:
payload["first_customer_message_text"] = _first_customer_message_text(
- team.id, str(ticket.id), unescape_markdown=ticket.channel_source == Channel.WIDGET
+ team.id,
+ str(ticket.id),
+ max_bytes=None
+ if first_customer_message_max_bytes is None
+ else _first_message_max_bytes(first_customer_message_max_bytes, payload),
+ unescape_markdown=ticket.channel_source == Channel.WIDGET,
)
return Response(payload) # nosemgrep: api-response-must-match-schema