Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(signals): bound the inbox report search count (#130124)

posthog[bot]d4bfab822252
diff --git a/products/signals/backend/scout_harness/serializers.py b/products/signals/backend/scout_harness/serializers.py
index f8236c36834d..9ca1b3e39e50 100644
--- a/products/signals/backend/scout_harness/serializers.py
+++ b/products/signals/backend/scout_harness/serializers.py
@@ -1028,7 +1028,7 @@ class ScratchpadEntrySerializer(serializers.Serializer):
# pg_trgm pads a word where a literal separator bounds it, so `ai_` still yields the trigram "AI ".
-_TRIGRAM_INDEXABLE_TEXT = re.compile(r"[^\W_]{3}|(?<=[\W_])[^\W_]{2}|[^\W_]{2}(?=[\W_])")
+TRIGRAM_INDEXABLE_TEXT = re.compile(r"[^\W_]{3}|(?<=[\W_])[^\W_]{2}|[^\W_]{2}(?=[\W_])")
class SearchMemoryQuerySerializer(serializers.Serializer):
@@ -1104,7 +1104,7 @@ class SearchMemoryQuerySerializer(serializers.Serializer):
def validate_text(self, value: str) -> str:
# The trigram indexes extract nothing from a term without an indexable word, so Postgres
# reads and upper-cases every body the team holds to answer it.
- if value and not _TRIGRAM_INDEXABLE_TEXT.search(value):
+ if value and not TRIGRAM_INDEXABLE_TEXT.search(value):
raise serializers.ValidationError(
"Use 3 letters or digits in a row, or 2 next to a symbol such as `ai_`, for example a whole word or identifier. "
"To read one known entry, pass `key` instead."
diff --git a/products/signals/backend/test/test_signal_report_api.py b/products/signals/backend/test/test_signal_report_api.py
index 30ac948f2bc5..3415c2af76ef 100644
--- a/products/signals/backend/test/test_signal_report_api.py
+++ b/products/signals/backend/test/test_signal_report_api.py
@@ -91,6 +91,7 @@ from products.signals.backend.test.report_metric_test_fixtures import trends_met
from products.signals.backend.throttles import ReportMetricRefreshThrottle
from products.signals.backend.views import (
PR_CI_STATUS_MAX_REPORTS,
+ SignalReportListPagination,
SignalReportStateRequestSerializer,
SignalReportViewSet,
classify_report_list_client,
@@ -2606,35 +2607,71 @@ class TestSignalReportListAPI(APIBaseTest):
@parameterized.expand(
[
- ("short_first_page", 10, 0, 3, False, False),
- ("short_last_page", 2, 2, 1, False, False),
- ("exact_full_page", 3, 0, 3, False, False),
- ("full_page_with_more", 2, 0, 2, False, True),
- ("offset_past_the_end", 2, 5, 0, True, False),
+ # name, headroom, limit, offset, rows, runs_count_query, count, capped, has_next
+ ("short_first_page", 1000, 10, 0, 3, False, 3, False, False),
+ ("short_last_page", 1000, 2, 2, 1, False, 3, False, False),
+ ("exact_full_page", 1000, 3, 0, 3, True, 3, False, False),
+ ("full_page_under_the_cap", 1000, 2, 0, 2, True, 3, False, True),
+ ("offset_past_the_end", 1000, 2, 5, 0, True, 3, False, False),
+ ("full_page_over_the_cap", 1, 1, 0, 1, True, 2, True, True),
+ ("deep_offset_moves_the_cap", 1, 1, 1, 1, True, 3, False, True),
]
)
- def test_search_page_counts_without_a_second_scan(
- self, _name: str, limit: int, offset: int, rows: int, runs_count_query: bool, has_next: bool
+ def test_search_page_counts_only_up_to_a_cap(
+ self,
+ _name: str,
+ headroom: int,
+ limit: int,
+ offset: int,
+ rows: int,
+ runs_count_query: bool,
+ count: int,
+ capped: bool,
+ has_next: bool,
) -> None:
for i in range(3):
self._create_report(title=f"Checkout failure {i}")
self._create_report(title="Unrelated")
- with CaptureQueriesContext(connection) as ctx:
+ with (
+ patch.object(SignalReportListPagination, "SEARCH_COUNT_HEADROOM", headroom),
+ CaptureQueriesContext(connection) as ctx,
+ ):
response = self.client.get(
self._list_url(search="checkout", ordering="-created_at", limit=limit, offset=offset)
)
assert response.status_code == status.HTTP_200_OK
body = response.json()
- assert body["count"] == 3
+ assert (body["count"], body["count_is_capped"]) == (count, capped)
newest_first = ["Checkout failure 2", "Checkout failure 1", "Checkout failure 0"]
assert [r["title"] for r in body["results"]] == newest_first[offset : offset + rows]
assert (body["next"] is not None) == has_next
count_queries = [
- q for q in ctx.captured_queries if 'COUNT(*) AS "__count" FROM "signals_signalreport"' in q["sql"]
+ q["sql"]
+ for q in ctx.captured_queries
+ if q["sql"].startswith('SELECT COUNT(*) FROM (SELECT "signals_signalreport"."id"')
]
assert bool(count_queries) == runs_count_query
+ assert all(f"LIMIT {offset + limit + headroom + 1}" in sql for sql in count_queries)
+
+ def test_search_shape_goes_on_the_list_span_without_the_term(self) -> None:
+ for i in range(3):
+ self._create_report(title=f"Checkout failure {i}")
+ span = MagicMock()
+
+ with patch("products.signals.backend.views.trace.get_current_span", return_value=span):
+ response = self.client.get(self._list_url(search="checkout", limit=1))
+
+ assert response.status_code == status.HTTP_200_OK
+ attributes = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
+ assert {key: value for key, value in attributes.items() if key.startswith("signals.reports.list.search.")} == {
+ "signals.reports.list.search.length": 8,
+ "signals.reports.list.search.trigram_indexable": True,
+ "signals.reports.list.search.count_query": "bounded",
+ "signals.reports.list.search.count_capped": False,
+ }
+ assert "checkout" not in {str(value).lower() for value in attributes.values()}
@parameterized.expand([("plain", {}), ("search", {"search": "checkout"})])
def test_page_sort_selects_ids_without_joining_one_to_one_rows(self, _name: str, params: dict[str, str]) -> None:
diff --git a/products/signals/backend/views.py b/products/signals/backend/views.py
index 5cd0254d6525..0ed841baa6ab 100644
--- a/products/signals/backend/views.py
+++ b/products/signals/backend/views.py
@@ -26,7 +26,6 @@ from django.db.models import (
Sum,
Value,
When,
- Window,
)
from django.db.models.expressions import OrderBy
from django.db.models.functions import Coalesce
@@ -246,7 +245,11 @@ from products.signals.backend.reviewer_correction_notes import ReviewerCorrectio
from products.signals.backend.reviewer_pr_assignment import schedule_reviewer_pr_assignment
from products.signals.backend.reviewer_rules import ReviewerRules, ReviewerWriteError, StaleReviewerProposal
from products.signals.backend.scout_authorship import inbox_hidden_report_filter
-from products.signals.backend.scout_harness.serializers import EmitReportRequestSerializer, EmitReportResponseSerializer
+from products.signals.backend.scout_harness.serializers import (
+ TRIGRAM_INDEXABLE_TEXT,
+ EmitReportRequestSerializer,
+ EmitReportResponseSerializer,
+)
from products.signals.backend.scout_harness.tools.report_author import (
AI_PROCESSING_NOT_APPROVED_REASON,
USER_REPORT_DAILY_LIMIT,
@@ -1340,49 +1343,53 @@ class SignalReportReingestionStatusSerializer(serializers.Serializer):
class SignalReportListPagination(FacadeLimitOffsetPagination):
+ # A full search page counts at most this many rows past its end, so `next` still works at any depth.
+ SEARCH_COUNT_HEADROOM = 1000
+
+ count_is_capped: bool = False
+ # How a search page got its count, for the list span. None without a search.
+ search_count_query: Literal["skipped", "bounded"] | None = None
+
def paginate_queryset(self, queryset, request, view=None):
# Fetch the page before the count. A page that ends before the limit already gives the
# total, so the second scan is not needed. A `search` filter rechecks the text of every
- # candidate report, and that scan costs seconds in a large team.
+ # candidate report, so a full search page counts only up to a cap past its end.
self.limit = self.get_limit(request)
if self.limit is None:
return None
self.offset = self.get_offset(request)
self.request = request
- page, window_count = self._page_by_ids(
- queryset, self.offset, self.limit, with_window_count=bool(request.query_params.get("search"))
- )
+ searched = bool(request.query_params.get("search"))
+ page = self._page_by_ids(queryset, self.offset, self.limit)
if len(page) < self.limit and (page or self.offset == 0):
self.count = self.offset + len(page)
- elif window_count is not None:
- self.count = window_count
+ self.search_count_query = "skipped" if searched else None
+ elif searched:
+ cap = self.offset + self.limit + self.SEARCH_COUNT_HEADROOM
+ bounded_count = queryset.order_by().values("pk")[: cap + 1].count()
+ self.count_is_capped = bounded_count > cap
+ self.count = min(bounded_count, cap)
+ self.search_count_query = "bounded"
else:
self.count = self.get_count(queryset)
if self.count > self.limit and self.template is not None:
self.display_page_controls = True
return page
- def _page_by_ids(
- self, queryset, offset: int, limit: int, *, with_window_count: bool
- ) -> tuple[list[SignalReport], int | None]:
+ def _page_by_ids(self, queryset, offset: int, limit: int) -> list[SignalReport]:
# The sort covers every matching report in the team, so it selects ids only. Full rows would
- # carry the wide report columns and the refund and tracker issue joins through that sort, and
- # COUNT(*) OVER () would buffer them and spill to disk. The full rows load for the page alone.
- if with_window_count:
- rows = list(
- queryset.annotate(search_total=Window(Count("*"))).values_list("pk", "search_total")[
- offset : offset + limit
- ]
- )
- page_ids = [pk for pk, _ in rows]
- window_count = rows[0][1] if rows else None
- else:
- page_ids = list(queryset.values_list("pk", flat=True)[offset : offset + limit])
- window_count = None
+ # carry the wide report columns and the refund and tracker issue joins through that sort.
+ # The full rows load for the page alone.
+ page_ids = list(queryset.values_list("pk", flat=True)[offset : offset + limit])
if not page_ids:
- return [], None
+ return []
reports = {report.pk: report for report in queryset.filter(pk__in=page_ids).order_by()}
- return [reports[pk] for pk in page_ids if pk in reports], window_count
+ return [reports[pk] for pk in page_ids if pk in reports]
+
+ def get_paginated_response(self, data):
+ response = super().get_paginated_response(data)
+ response.data["count_is_capped"] = self.count_is_capped
+ return response
def get_paginated_response_schema(self, schema):
paginated = super().get_paginated_response_schema(schema)
@@ -1395,6 +1402,13 @@ class SignalReportListPagination(FacadeLimitOffsetPagination):
"inbox_merge_key belongs to this ranker."
),
}
+ paginated["properties"]["count_is_capped"] = {
+ "type": "boolean",
+ "description": (
+ "True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. "
+ "Only a search with a full page can cap the count."
+ ),
+ }
paginated["properties"]["ranking_pill"] = {
"type": "boolean",
"nullable": True,
@@ -3140,6 +3154,13 @@ class SignalReportViewSet(
list_span.set_attribute("signals.reports.list.count_only", count_only)
list_span.set_attribute("signals.reports.list.include_source_metadata", include_source_metadata)
list_span.set_attribute("signals.reports.list.cached_filters", self._inbox_cached_filters)
+ # The search shape only. The term itself never goes on the span.
+ search = request.query_params.get("search")
+ if search:
+ list_span.set_attribute("signals.reports.list.search.length", len(search))
+ list_span.set_attribute(
+ "signals.reports.list.search.trigram_indexable", bool(TRIGRAM_INDEXABLE_TEXT.search(search))
+ )
with tracer.start_as_current_span("signals.reports.list.queryset"):
queryset = self.filter_queryset(self.get_queryset())
@@ -3172,6 +3193,12 @@ class SignalReportViewSet(
list_span.set_attribute(
"signals.reports.list.has_next_page", page_offset + len(report_ids) < total_count
)
+ search_count_query = getattr(self.paginator, "search_count_query", None)
+ if search_count_query is not None:
+ list_span.set_attribute("signals.reports.list.search.count_query", search_count_query)
+ list_span.set_attribute(
+ "signals.reports.list.search.count_capped", getattr(self.paginator, "count_is_capped", False)
+ )
data = self._render_report_rows(reports, include_source_metadata=include_source_metadata)
self._capture_list_ranking_usage(total_count if page is not None else len(reports))
diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.ts
index bd1f26bea6c3..4a8846681e9f 100644
--- a/products/signals/frontend/generated/api.schemas.ts
+++ b/products/signals/frontend/generated/api.schemas.ts
@@ -2542,6 +2542,8 @@ export interface PaginatedSignalReportListListApi {
* @nullable
*/
ranker?: string | null
+ /** True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. Only a search with a full page can cap the count. */
+ count_is_capped?: boolean
/**
* Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.
* @nullable
diff --git a/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.ts b/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.ts
new file mode 100644
index 000000000000..8d9d0a54080c
--- /dev/null
+++ b/products/signals/frontend/inbox/components/shell/InboxTriageButton.test.ts
@@ -0,0 +1,12 @@
+import { triageLabel } from './InboxTriageButton'
+
+describe('triageLabel', () => {
+ test.each([
+ [1, false, false, 'Triage 1 report'],
+ [1000, true, false, 'Triage 1,000+ reports'],
+ [1000, true, true, 'Triage 1,000+'],
+ [1, true, false, 'Triage 1+ reports'],
+ ])('labels a count of %s (capped: %s, compact: %s) as %s', (count, countIsCapped, compact, expected) => {
+ expect(triageLabel(count, countIsCapped, compact)).toEqual(expected)
+ })
+})
diff --git a/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx b/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx
index 6fb8bc83814c..4ae6adf75431 100644
--- a/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx
+++ b/products/signals/frontend/inbox/components/shell/InboxTriageButton.tsx
@@ -17,14 +17,15 @@ const NOTHING_TO_TRIAGE = 'No reports in this view need a pull request'
// pinned: autocapture selector. Triage usage insights filter clicks on this data-attr.
const TRIAGE_DATA_ATTR = 'inbox-triage-mode'
-function triageLabel(count: number | null, compact: boolean): string {
+export function triageLabel(count: number | null, countIsCapped: boolean, compact: boolean): string {
if (count === null) {
return compact ? 'Triage' : 'Triage mode'
}
+ const shownCount = `${humanFriendlyNumber(count)}${countIsCapped ? '+' : ''}`
if (compact) {
- return `Triage ${humanFriendlyNumber(count)}`
+ return `Triage ${shownCount}`
}
- return `Triage ${humanFriendlyNumber(count)} ${count === 1 ? 'report' : 'reports'}`
+ return `Triage ${shownCount} ${count === 1 && !countIsCapped ? 'report' : 'reports'}`
}
/**
@@ -34,10 +35,10 @@ function triageLabel(count: number | null, compact: boolean): string {
* `compact` drops the hotkey hint for the narrow tab row. The scene binds `t` either way.
*/
export function InboxTriageButton({ compact = false }: { compact?: boolean }): JSX.Element {
- const { count } = useValues(reportListLogic(sectionListLogicProps(INBOX_PRIMARY_REPORT_SECTION_KEY)))
+ const { count, countIsCapped } = useValues(reportListLogic(sectionListLogicProps(INBOX_PRIMARY_REPORT_SECTION_KEY)))
const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')
const disabledReason = count === 0 ? NOTHING_TO_TRIAGE : undefined
- const label = triageLabel(count, compact)
+ const label = triageLabel(count, countIsCapped, compact)
if (todayRailNav) {
return (
diff --git a/products/signals/frontend/inbox/logics/reportListLogic.ts b/products/signals/frontend/inbox/logics/reportListLogic.ts
index 94cc3f120d8c..c098445ee494 100644
--- a/products/signals/frontend/inbox/logics/reportListLogic.ts
+++ b/products/signals/frontend/inbox/logics/reportListLogic.ts
@@ -21,6 +21,7 @@ import {
signalsReportsSourceMetadataCreate,
} from 'products/signals/frontend/generated/api'
import type {
+ PaginatedSignalReportListListApi,
ReviewerRemovalReasonEnumApi,
SignalReportMetricSnapshotsApi,
} from 'products/signals/frontend/generated/api.schemas'
@@ -69,14 +70,15 @@ export interface ReportListRequestContext {
}
/** A list response stamped with the query params and display context that produced it (see the loader). */
-export type ReportListResponse = CountedPaginatedResponse<SignalReport> & {
- /** The ranker the server used for `sort=recommended`. Null for every other sort. */
- ranker?: string | null
- /** Whether the client shows a ranking score pill on each card, for `sort=recommended`. Null for every other sort. */
- ranking_pill?: boolean | null
- requestParams?: Record<string, unknown>
- requestContext?: ReportListRequestContext
-}
+export type ReportListResponse = CountedPaginatedResponse<SignalReport> &
+ Pick<PaginatedSignalReportListListApi, 'count_is_capped'> & {
+ /** The ranker the server used for `sort=recommended`. Null for every other sort. */
+ ranker?: string | null
+ /** Whether the client shows a ranking score pill on each card, for `sort=recommended`. Null for every other sort. */
+ ranking_pill?: boolean | null
+ requestParams?: Record<string, unknown>
+ requestContext?: ReportListRequestContext
+ }
/**
* The list request for the API. `created_window` stays a preset key in `listApiParams`, so the
@@ -222,6 +224,7 @@ export interface reportListLogicValues {
user: UserType | null // userLogic
claimSavingReportIds: string[]
count: number | null
+ countIsCapped: boolean
countLoading: boolean
countSettled: boolean
hasMore: boolean
@@ -487,6 +490,7 @@ export interface reportListLogicMeta {
isLoaded: (reportsResponse: ReportListResponse | null) => boolean
livePrReportIds: (reports: SignalReport[]) => string[]
totalCount: (reportsResponse: ReportListResponse | null) => number | null
+ countIsCapped: (reportsResponse: ReportListResponse | null) => boolean
loadedQueryKey: (reportsResponse: ReportListResponse | null) => string | null
countSettled: (
countLoading: boolean,
@@ -902,6 +906,11 @@ export const reportListLogic = kea<reportListLogicType>([
(s) => [s.reportsResponse],
(reportsResponse: ReportListResponse | null): number | null => reportsResponse?.count ?? null,
],
+ // A search stops counting past a cap, so `count` is a lower bound when this is true.
+ countIsCapped: [
+ (s) => [s.reportsResponse],
+ (reportsResponse: ReportListResponse | null): boolean => reportsResponse?.count_is_capped === true,
+ ],
// Stable key for the query that produced the loaded response. Unlike `listApiParams`, this
// only changes when a response fetched with the new params actually lands, so consumers
// never associate the new query with rows from the previous one.
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2cb0051af0bd..eb44d44c910e 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -74774,6 +74774,8 @@ export namespace Schemas {
* @nullable
*/
ranker?: string | null;
+ /** True when `count` stopped at a cap, so it is a lower bound and `next` keeps paging past it. Only a search with a full page can cap the count. */
+ count_is_capped?: boolean;
/**
* Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.
* @nullable