:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(signals): bound the inbox report search count (#130124)
posthog[bot]d4bfab822252
diff --git a/products/signals/backend/scout_harness/serializers.py b/products/signals/backend/scout_harness/serializers.pyindex f8236c36834d..9ca1b3e39e50 100644--- a/products/signals/backend/scout_harness/serializers.py+++ b/products/signals/backend/scout_harness/serializers.py@@ -1028,7 +1028,7 @@ class ScratchpadEntrySerializer(serializers.Serializer):# pg_trgm pads a word where a literal separator bounds it, so `ai_` still yields the trigram "AI ".-_TRIGRAM_INDEXABLE_TEXT = re.compile(r"[^\W_]{3}|(?<=[\W_])[^\W_]{2}|[^\W_]{2}(?=[\W_])")+TRIGRAM_INDEXABLE_TEXT = re.compile(r"[^\W_]{3}|(?<=[\W_])[^\W_]{2}|[^\W_]{2}(?=[\W_])")class SearchMemoryQuerySerializer(serializers.Serializer):@@ -1104,7 +1104,7 @@ class SearchMemoryQuerySerializer(serializers.Serializer):def validate_text(self, value: str) -> str:# The trigram indexes extract nothing from a term without an indexable word, so Postgres# reads and upper-cases every body the team holds to answer it.- if value and not _TRIGRAM_INDEXABLE_TEXT.search(value):+ if value and not TRIGRAM_INDEXABLE_TEXT.search(value):raise serializers.ValidationError("Use 3 letters or digits in a row, or 2 next to a symbol such as `ai_`, for example a whole word or identifier. ""To read one known entry, pass `key` instead."diff --git a/products/signals/backend/test/test_signal_report_api.py b/products/signals/backend/test/test_signal_report_api.pyindex 30ac948f2bc5..3415c2af76ef 100644--- a/products/signals/backend/test/test_signal_report_api.py+++ b/products/signals/backend/test/test_signal_report_api.py@@ -91,6 +91,7 @@ from products.signals.backend.test.report_metric_test_fixtures import trends_metfrom products.signals.backend.throttles import ReportMetricRefreshThrottlefrom products.signals.backend.views import (PR_CI_STATUS_MAX_REPORTS,+ SignalReportListPagination,SignalReportStateRequestSerializer,SignalReportViewSet,classify_report_list_client,@@ -2606,35 +2607,71 @@ class TestSignalReportListAPI(APIBaseTest):@parameterized.expand([- ("short_first_page", 10, 0, 3, False, False),- ("short_last_page", 2, 2, 1, False, False),- ("exact_full_page", 3, 0, 3, False, False),- ("full_page_with_more", 2, 0, 2, False, True),- ("offset_past_the_end", 2, 5, 0, True, False),+ # name, headroom, limit, offset, rows, runs_count_query, count, capped, has_next+ ("short_first_page", 1000, 10, 0, 3, False, 3, False, False),+ ("short_last_page", 1000, 2, 2, 1, False, 3, False, False),+ ("exact_full_page", 1000, 3, 0, 3, True, 3, False, False),+ ("full_page_under_the_cap", 1000, 2, 0, 2, True, 3, False, True),+ ("offset_past_the_end", 1000, 2, 5, 0, True, 3, False, False),+ ("full_page_over_the_cap", 1, 1, 0, 1, True, 2, True, True),+ ("deep_offset_moves_the_cap", 1, 1, 1, 1, True, 3, False, True),])- def test_search_page_counts_without_a_second_scan(- self, _name: str, limit: int, offset: int, rows: int, runs_count_query: bool, has_next: bool+ def test_search_page_counts_only_up_to_a_cap(+ self,+ _name: str,+ headroom: int,+ limit: int,+ offset: int,+ rows: int,+ runs_count_query: bool,+ count: int,+ capped: bool,+ has_next: bool,) -> None:for i in range(3):self._create_report(title=f"Checkout failure {i}")self._create_report(title="Unrelated")- with CaptureQueriesContext(connection) as ctx:+ with (+ patch.object(SignalReportListPagination, "SEARCH_COUNT_HEADROOM", headroom),+ CaptureQueriesContext(connection) as ctx,+ ):response = self.client.get(self._list_url(search="checkout", ordering="-created_at", limit=limit, offset=offset))assert response.status_code == status.HTTP_200_OKbody = response.json()- assert body["count"] == 3+ assert (body["count"], body["count_is_capped"]) == (count, capped)newest_first = ["Checkout failure 2", "Checkout failure 1", "Checkout failure 0"]assert [r["title"] for r in body["results"]] == newest_first[offset : offset + rows]assert (body["next"] is not None) == has_nextcount_queries = [- q for q in ctx.captured_queries if 'COUNT(*) AS "__count" FROM "signals_signalreport"' in q["sql"]+ q["sql"]+ for q in ctx.captured_queries+ if q["sql"].startswith('SELECT COUNT(*) FROM (SELECT "signals_signalreport"."id"')]assert bool(count_queries) == runs_count_query+ assert all(f"LIMIT {offset + limit + headroom + 1}" in sql for sql in count_queries)++ def test_search_shape_goes_on_the_list_span_without_the_term(self) -> None:+ for i in range(3):+ self._create_report(title=f"Checkout failure {i}")+ span = MagicMock()++ with patch("products.signals.backend.views.trace.get_current_span", return_value=span):+ response = self.client.get(self._list_url(search="checkout", limit=1))++ assert response.status_code == status.HTTP_200_OK+ attributes = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}+ assert {key: value for key, value in attributes.items() if key.startswith("signals.reports.list.search.")} == {+ "signals.reports.list.search.length": 8,+ "signals.reports.list.search.trigram_indexable": True,+ "signals.reports.list.search.count_query": "bounded",+ "signals.reports.list.search.count_capped": False,+ }+ assert "checkout" not in {str(value).lower() for value in attributes.values()}@parameterized.expand([("plain", {}), ("search", {"search": "checkout"})])def test_page_sort_selects_ids_without_joining_one_to_one_rows(self, _name: str, params: dict[str, str]) -> None:diff --git a/products/signals/backend/views.py b/products/signals/backend/views.pyindex 5cd0254d6525..0ed841baa6ab 100644--- a/products/signals/backend/views.py+++ b/products/signals/backend/views.py@@ -26,7 +26,6 @@ from django.db.models import (Sum,Value,When,- Window,)from django.db.models.expressions import OrderByfrom django.db.models.functions import Coalesce@@ -246,7 +245,11 @@ from products.signals.backend.reviewer_correction_notes import ReviewerCorrectiofrom products.signals.backend.reviewer_pr_assignment import schedule_reviewer_pr_assignmentfrom products.signals.backend.reviewer_rules import ReviewerRules, ReviewerWriteError, StaleReviewerProposalfrom products.signals.backend.scout_authorship import inbox_hidden_report_filter-from products.signals.backend.scout_harness.serializers import EmitReportRequestSerializer, EmitReportResponseSerializer+from products.signals.backend.scout_harness.serializers import (+ TRIGRAM_INDEXABLE_TEXT,+ EmitReportRequestSerializer,+ EmitReportResponseSerializer,+)from products.signals.backend.scout_harness.tools.report_author import (AI_PROCESSING_NOT_APPROVED_REASON,USER_REPORT_DAILY_LIMIT,@@ -1340,49 +1343,53 @@ class SignalReportReingestionStatusSerializer(serializers.Serializer):class SignalReportListPagination(FacadeLimitOffsetPagination):+ # A full search page counts at most this many rows past its end, so `next` still works at any depth.+ SEARCH_COUNT_HEADROOM = 1000++ count_is_capped: bool = False+ # How a search page got its count, for the list span. None without a search.+ search_count_query: Literal["skipped", "bounded"] | None = None+def paginate_queryset(self, queryset, request, view=None):# Fetch the page before the count. A page that ends before the limit already gives the# total, so the second scan is not needed. A `search` filter rechecks the text of every- # candidate report, and that scan costs seconds in a large team.+ # candidate report, so a full search page counts only up to a cap past its end.self.limit = self.get_limit(request)if self.limit is None:return Noneself.offset = self.get_offset(request)self.request = request- page, window_count = self._page_by_ids(- queryset, self.offset, self.limit, with_window_count=bool(request.query_params.get("search"))- )+ searched = bool(request.query_params.get("search"))+ page = self._page_by_ids(queryset, self.offset, self.limit)if len(page) < self.limit and (page or self.offset == 0):self.count = self.offset + len(page)- elif window_count is not None:- self.count = window_count+ self.search_count_query = "skipped" if searched else None+ elif searched:+ cap = self.offset + self.limit + self.SEARCH_COUNT_HEADROOM+ bounded_count = queryset.order_by().values("pk")[: cap + 1].count()+ self.count_is_capped = bounded_count > cap+ self.count = min(bounded_count, cap)+ self.search_count_query = "bounded"else:self.count = self.get_count(queryset)if self.count > self.limit and self.template is not None:self.display_page_controls = Truereturn page- def _page_by_ids(- self, queryset, offset: int, limit: int, *, with_window_count: bool- ) -> tuple[list[SignalReport], int | None]:+ def _page_by_ids(self, queryset, offset: int, limit: int) -> list[SignalReport]:# The sort covers every matching report in the team, so it selects ids only. Full rows would- # carry the wide report columns and the refund and tracker issue joins through that sort, and- # COUNT(*) OVER () would buffer them and spill to disk. The full rows load for the page alone.- if with_window_count:- rows = list(- queryset.annotate(search_total=Window(Count("*"))).values_list("pk", "search_total")[- offset : offset + limit- ]- )- page_ids = [pk for pk, _ in rows]- window_count = rows[0][1] if rows else None- else:- page_ids = list(queryset.values_list("pk", flat=True)[offset : offset + limit])- window_count = None+ # carry the wide report columns and the refund and tracker issue joins through that sort.+ # The full rows load for the page alone.+ page_ids = list(queryset.values_list("pk", flat=True)[offset : offset + limit])if not page_ids:- return [], None+ return []reports = {report.pk: report for report in queryset.filter(pk__in=page_ids).order_by()}- return [reports[pk] for pk in page_ids if pk in reports], window_count+ return [reports[pk] for pk in page_ids if pk in reports]++ def get_paginated_response(self, data):+ response = super().get_paginated_response(data)+ response.data["count_is_capped"] = self.count_is_capped+ return responsedef get_paginated_response_schema(self, schema):paginated = super().get_paginated_response_schema(schema)@@ -1395,6 +1402,13 @@ class SignalReportListPagination(FacadeLimitOffsetPagination):"inbox_merge_key belongs to this ranker."),}+ paginated["properties"]["count_is_capped"] = {+ "type": "boolean",+ "description": (+ "True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. "+ "Only a search with a full page can cap the count."+ ),+ }paginated["properties"]["ranking_pill"] = {"type": "boolean","nullable": True,@@ -3140,6 +3154,13 @@ class SignalReportViewSet(list_span.set_attribute("signals.reports.list.count_only", count_only)list_span.set_attribute("signals.reports.list.include_source_metadata", include_source_metadata)list_span.set_attribute("signals.reports.list.cached_filters", self._inbox_cached_filters)+ # The search shape only. The term itself never goes on the span.+ search = request.query_params.get("search")+ if search:+ list_span.set_attribute("signals.reports.list.search.length", len(search))+ list_span.set_attribute(+ "signals.reports.list.search.trigram_indexable", bool(TRIGRAM_INDEXABLE_TEXT.search(search))+ )with tracer.start_as_current_span("signals.reports.list.queryset"):queryset = self.filter_queryset(self.get_queryset())@@ -3172,6 +3193,12 @@ class SignalReportViewSet(list_span.set_attribute("signals.reports.list.has_next_page", page_offset + len(report_ids) < total_count)+ search_count_query = getattr(self.paginator, "search_count_query", None)+ if search_count_query is not None:+ list_span.set_attribute("signals.reports.list.search.count_query", search_count_query)+ list_span.set_attribute(+ "signals.reports.list.search.count_capped", getattr(self.paginator, "count_is_capped", False)+ )data = self._render_report_rows(reports, include_source_metadata=include_source_metadata)self._capture_list_ranking_usage(total_count if page is not None else len(reports))diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.tsindex bd1f26bea6c3..4a8846681e9f 100644--- a/products/signals/frontend/generated/api.schemas.ts+++ b/products/signals/frontend/generated/api.schemas.ts@@ -2542,6 +2542,8 @@ export interface PaginatedSignalReportListListApi {* @nullable*/ranker?: string | null+ /** True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. Only a search with a full page can cap the count. */+ count_is_capped?: boolean/*** Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.* @nullablediff --git a/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.ts b/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.tsnew file mode 100644index 000000000000..8d9d0a54080c--- /dev/null+++ b/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.ts@@ -0,0 +1,12 @@+import { triageLabel } from './InboxTriageButton'++describe('triageLabel', () => {+ test.each([+ [1, false, false, 'Triage 1 report'],+ [1000, true, false, 'Triage 1,000+ reports'],+ [1000, true, true, 'Triage 1,000+'],+ [1, true, false, 'Triage 1+ reports'],+ ])('labels a count of %s (capped: %s, compact: %s) as %s', (count, countIsCapped, compact, expected) => {+ expect(triageLabel(count, countIsCapped, compact)).toEqual(expected)+ })+})diff --git a/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx b/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsxindex 6fb8bc83814c..4ae6adf75431 100644--- a/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx+++ b/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx@@ -17,14 +17,15 @@ const NOTHING_TO_TRIAGE = 'No reports in this view need a pull request'// pinned: autocapture selector. Triage usage insights filter clicks on this data-attr.const TRIAGE_DATA_ATTR = 'inbox-triage-mode'-function triageLabel(count: number | null, compact: boolean): string {+export function triageLabel(count: number | null, countIsCapped: boolean, compact: boolean): string {if (count === null) {return compact ? 'Triage' : 'Triage mode'}+ const shownCount = `${humanFriendlyNumber(count)}${countIsCapped ? '+' : ''}`if (compact) {- return `Triage ${humanFriendlyNumber(count)}`+ return `Triage ${shownCount}`}- return `Triage ${humanFriendlyNumber(count)} ${count === 1 ? 'report' : 'reports'}`+ return `Triage ${shownCount} ${count === 1 && !countIsCapped ? 'report' : 'reports'}`}/**@@ -34,10 +35,10 @@ function triageLabel(count: number | null, compact: boolean): string {* `compact` drops the hotkey hint for the narrow tab row. The scene binds `t` either way.*/export function InboxTriageButton({ compact = false }: { compact?: boolean }): JSX.Element {- const { count } = useValues(reportListLogic(sectionListLogicProps(INBOX_PRIMARY_REPORT_SECTION_KEY)))+ const { count, countIsCapped } = useValues(reportListLogic(sectionListLogicProps(INBOX_PRIMARY_REPORT_SECTION_KEY)))const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')const disabledReason = count === 0 ? NOTHING_TO_TRIAGE : undefined- const label = triageLabel(count, compact)+ const label = triageLabel(count, countIsCapped, compact)if (todayRailNav) {return (diff --git a/products/signals/frontend/inbox/logics/reportListLogic.ts b/products/signals/frontend/inbox/logics/reportListLogic.tsindex 94cc3f120d8c..c098445ee494 100644--- a/products/signals/frontend/inbox/logics/reportListLogic.ts+++ b/products/signals/frontend/inbox/logics/reportListLogic.ts@@ -21,6 +21,7 @@ import {signalsReportsSourceMetadataCreate,} from 'products/signals/frontend/generated/api'import type {+ PaginatedSignalReportListListApi,ReviewerRemovalReasonEnumApi,SignalReportMetricSnapshotsApi,} from 'products/signals/frontend/generated/api.schemas'@@ -69,14 +70,15 @@ export interface ReportListRequestContext {}/** A list response stamped with the query params and display context that produced it (see the loader). */-export type ReportListResponse = CountedPaginatedResponse<SignalReport> & {- /** The ranker the server used for `sort=recommended`. Null for every other sort. */- ranker?: string | null- /** Whether the client shows a ranking score pill on each card, for `sort=recommended`. Null for every other sort. */- ranking_pill?: boolean | null- requestParams?: Record<string, unknown>- requestContext?: ReportListRequestContext-}+export type ReportListResponse = CountedPaginatedResponse<SignalReport> &+ Pick<PaginatedSignalReportListListApi, 'count_is_capped'> & {+ /** The ranker the server used for `sort=recommended`. Null for every other sort. */+ ranker?: string | null+ /** Whether the client shows a ranking score pill on each card, for `sort=recommended`. Null for every other sort. */+ ranking_pill?: boolean | null+ requestParams?: Record<string, unknown>+ requestContext?: ReportListRequestContext+ }/*** The list request for the API. `created_window` stays a preset key in `listApiParams`, so the@@ -222,6 +224,7 @@ export interface reportListLogicValues {user: UserType | null // userLogicclaimSavingReportIds: string[]count: number | null+ countIsCapped: booleancountLoading: booleancountSettled: booleanhasMore: boolean@@ -487,6 +490,7 @@ export interface reportListLogicMeta {isLoaded: (reportsResponse: ReportListResponse | null) => booleanlivePrReportIds: (reports: SignalReport[]) => string[]totalCount: (reportsResponse: ReportListResponse | null) => number | null+ countIsCapped: (reportsResponse: ReportListResponse | null) => booleanloadedQueryKey: (reportsResponse: ReportListResponse | null) => string | nullcountSettled: (countLoading: boolean,@@ -902,6 +906,11 @@ export const reportListLogic = kea<reportListLogicType>([(s) => [s.reportsResponse],(reportsResponse: ReportListResponse | null): number | null => reportsResponse?.count ?? null,],+ // A search stops counting past a cap, so `count` is a lower bound when this is true.+ countIsCapped: [+ (s) => [s.reportsResponse],+ (reportsResponse: ReportListResponse | null): boolean => reportsResponse?.count_is_capped === true,+ ],// Stable key for the query that produced the loaded response. Unlike `listApiParams`, this// only changes when a response fetched with the new params actually lands, so consumers// never associate the new query with rows from the previous one.diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 2cb0051af0bd..eb44d44c910e 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -74774,6 +74774,8 @@ export namespace Schemas {* @nullable*/ranker?: string | null;+ /** True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. Only a search with a full page can cap the count. */+ count_is_capped?: boolean;/*** Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.* @nullable