Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): fetch smaller pages in the scout memory panel (#122992)

posthog[bot]b222fa73a760
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx
index 895bb6706571..dacbdfc832d2 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx
@@ -5,7 +5,7 @@ import { Tooltip } from '@posthog/lemon-ui'
import { cn } from 'lib/utils/css-classes'
import { scoutFleetLogic } from '../../../logics/scoutFleetLogic'
-import { scratchpadLogic } from '../../../logics/scratchpadLogic'
+import { SCRATCHPAD_WINDOW_PAGE_SIZE, scratchpadLogic } from '../../../logics/scratchpadLogic'
import { SCOUT_ROSTER_WINDOW_LABEL } from '../../../utils/scoutRunsWindow'
type StatTone = 'muted' | 'warning' | 'danger'
@@ -51,7 +51,7 @@ function Stat({
export function ScoutsRosterStats(): JSX.Element {
const { pauseAttentionCounts, enabledCount, emittedFindingsSummary, fleetFindingsSummaryLoadedOnce } =
useValues(scoutFleetLogic)
- // Mounted here rather than by the fleet logic, so the 1,000-entry scratchpad read only happens
+ // Mounted here rather than by the fleet logic, so the scratchpad read only happens
// on the surfaces that show it, not on every inbox tab that mounts the fleet.
const { recentlyLearnedCount, recentlyLearnedCountCapped, entries: scratchpadEntries } = useValues(scratchpadLogic)
// A summary that hasn't landed is not zero runs. The number holds a dash until it does; a failed
@@ -103,7 +103,7 @@ export function ScoutsRosterStats(): JSX.Element {
label="learned"
tooltip={
recentlyLearnedCountCapped
- ? `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}. Only the newest 1,000 entries are counted, and all of them fall in this window.`
+ ? `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}. Only the newest ${SCRATCHPAD_WINDOW_PAGE_SIZE} entries are counted, and all of them fall in this window.`
: `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}.`
}
/>
diff --git a/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx b/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx
index 2b4270787ebc..3aa1684f0d26 100644
--- a/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx
+++ b/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx
@@ -262,7 +262,7 @@ function ScratchpadHeader({
settled on, things they ruled out, and the bookkeeping that keeps them from repeating themselves.
</p>
<div className="flex flex-wrap gap-x-6 gap-y-2">
- {/* The count is what's loaded, not what exists: the endpoint caps at 1,000 newest-first,
+ {/* The count is what's loaded, not what exists: the first page is the newest few hundred,
which on a busy project is a few hours. Labelling it with the span it covers is what
stops it reading as the whole memory. */}
<Stat
diff --git a/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts b/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts
index 0cc50512ffcc..9097cdb93d03 100644
--- a/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts
+++ b/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts
@@ -13,7 +13,7 @@ import type { ScratchpadEntryApi } from 'products/signals/frontend/generated/api
import { PANEL_LOAD_TIMEOUT_MS } from '../utils/panelLoadTimeout'
import { forgetUnreachableReports } from '../utils/unreachableReports'
import {
- SCRATCHPAD_FETCH_LIMIT,
+ SCRATCHPAD_WINDOW_PAGE_SIZE,
SCRATCHPAD_PREVIEW_CHARS,
describeLoadedSpan,
entriesForSkill,
@@ -185,15 +185,15 @@ describe('scratchpadLogic', () => {
...entry(`pattern:${i}`, 'note'),
updated_at: new Date().toISOString(),
})
- const full = Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => fresh(i))
+ const full = Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => fresh(i))
logic.actions.loadEntriesSuccess(full)
- expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_FETCH_LIMIT)
+ expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_WINDOW_PAGE_SIZE)
expect(logic.values.recentlyLearnedCountCapped).toBe(true)
// One stale entry in a full fetch means the window ended inside it, so the count is exact.
logic.actions.loadEntriesSuccess([...full.slice(1), WHOLE])
- expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_FETCH_LIMIT - 1)
+ expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_WINDOW_PAGE_SIZE - 1)
expect(logic.values.recentlyLearnedCountCapped).toBe(false)
logic.actions.loadEntriesSuccess(full.slice(0, 10))
@@ -336,11 +336,11 @@ describe('scratchpadLogic', () => {
})
// Older pages walk the unfiltered window with a cursor, and a search is a separate one-shot
- // read they never reach. A control offered during a search spends a page of up to a thousand
+ // read they never reach. A control offered during a search spends a page of up to a few hundred
// rows the table cannot show, and moves the header counts while the listed rows stay put.
it('offers no older pages while a search is active', async () => {
logic.actions.loadEntriesSuccess(
- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))
+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note'))
)
expect(logic.values.canLoadOlderEntries).toBe(true)
@@ -357,7 +357,7 @@ describe('scratchpadLogic', () => {
// older is left. A control left live over that gap pages from the window that just went away.
it('retires the older-page control while the first page reloads', async () => {
logic.actions.loadEntriesSuccess(
- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))
+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note'))
)
expect(logic.values.hasMoreOlderEntries).toBe(true)
@@ -406,7 +406,7 @@ describe('scratchpadLogic', () => {
// spinner just stops, and the reader reads a failed press as an empty memory.
it('records a failed older page and clears the record on the retry', async () => {
logic.actions.loadEntriesSuccess(
- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))
+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note'))
)
useMocks({ get: { [SCRATCHPAD_URL]: () => [500, {}] } })
@@ -441,12 +441,15 @@ describe('scratchpadLogic', () => {
})
logic.actions.loadEntries()
await expectLogic(logic).toFinishAllListeners()
+ // The unfiltered window is the wide read, so both pages ask for the smaller page.
+ expect(searchRequests[0].get('limit')).toEqual(String(SCRATCHPAD_WINDOW_PAGE_SIZE))
searchRequests = []
logic.actions.loadOlderEntries()
await expectLogic(logic).toFinishAllListeners()
expect(searchRequests[0].get('date_to')).toEqual(WHOLE.updated_at)
+ expect(searchRequests[0].get('limit')).toEqual(String(SCRATCHPAD_WINDOW_PAGE_SIZE))
expect(logic.values.olderEntries).toEqual([older])
expect(logic.values.windowEntries?.map((e) => e.key)).toEqual([TRUNCATED.key, WHOLE.key, older.key])
// A short page is the end of the memory, so the button goes away.
diff --git a/products/signals/frontend/inbox/logics/scratchpadLogic.ts b/products/signals/frontend/inbox/logics/scratchpadLogic.ts
index ce99d60e1996..176eaff16b49 100644
--- a/products/signals/frontend/inbox/logics/scratchpadLogic.ts
+++ b/products/signals/frontend/inbox/logics/scratchpadLogic.ts
@@ -15,10 +15,13 @@ import { isReportUnreachable, rememberUnreachableReport } from '../utils/unreach
// Search reruns the server-side ILIKE on every keystroke; debounce so typing doesn't
// fire a request per character.
const SEARCH_DEBOUNCE_MS = 300
-// `list` caps at 1000 newest-first with no pagination wrapper — pull the whole window in
-// one read and filter it client-side. Older memory arrives through `loadOlderEntries`, which
-// walks the endpoint's `date_to` cursor a page at a time.
+// `list` caps at 1000 newest-first with no pagination wrapper. Search and one scout's entries
+// are narrow reads, so they take the whole cap in one read and filter it client-side.
export const SCRATCHPAD_FETCH_LIMIT = 1000
+// The unfiltered window is the wide read: on a busy project 1000 previews cost Postgres megabytes
+// per panel open. Fetch a smaller first page, and let `loadOlderEntries` walk the endpoint's
+// `date_to` cursor for older memory a page at a time.
+export const SCRATCHPAD_WINDOW_PAGE_SIZE = 200
// Bodies are an unbounded TextField clamped at 50k chars on write, so a full-fat window of
// 1000 entries is a payload nobody needs: the ledger renders a one-line clamp until you open it.
// Pull previews for the list and fetch the one body you expand. Sized well past one line so
@@ -94,8 +97,8 @@ export function entriesForSkill(
/**
* How wide a span the loaded rows actually cover, e.g. "last 8 h". The header needs this because
- * 1,000 rows is a cap, not a total: on a busy project the window closes after a few hours, and a
- * bare "1,000 entries" reads as the whole memory.
+ * a page of rows is a cap, not a total: on a busy project the window closes after a few hours, and
+ * a bare "200 entries" reads as the whole memory.
*/
export function describeLoadedSpan(entries: ScratchpadEntryApi[] | null): string | null {
const stamps = (entries ?? []).map((entry) => entry.updated_at).filter((stamp): stamp is string => !!stamp)
@@ -406,7 +409,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
signalsScoutScratchpadSearch(
String(teamId),
{
- limit: SCRATCHPAD_FETCH_LIMIT,
+ limit: SCRATCHPAD_WINDOW_PAGE_SIZE,
content_max_chars: SCRATCHPAD_PREVIEW_CHARS,
...dateFromParam(values.timeFilter),
},
@@ -538,7 +541,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
// still answer after a reload started, and this is bumped at dispatch, so the listener
// sees the invalidation without waiting for the fresh page to arrive.
windowGeneration: [0, { loadEntries: (state: number) => state + 1 }],
- // Pages walked back past the 1,000-row cap with the endpoint's `date_to` cursor. A fresh
+ // Pages walked back past the first page with the endpoint's `date_to` cursor. A fresh
// first page invalidates them: it may already carry rows these pages hold.
olderEntries: [
[] as ScratchpadEntryApi[],
@@ -573,7 +576,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
hasMoreOlderEntries: [
false,
{
- loadEntriesSuccess: (_, { entries }) => entries.length >= SCRATCHPAD_FETCH_LIMIT,
+ loadEntriesSuccess: (_, { entries }) => entries.length >= SCRATCHPAD_WINDOW_PAGE_SIZE,
appendOlderEntries: (_, { hasMore }) => hasMore,
// A reload drops the walked pages, so the control goes with them until the fresh
// first page says whether anything older is left.
@@ -660,7 +663,9 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
recentlyLearnedCountCapped: [
(s) => [s.entries, s.recentlyLearnedCount],
(entries: ScratchpadEntryApi[] | null, recentlyLearnedCount: number): boolean =>
- entries !== null && entries.length >= SCRATCHPAD_FETCH_LIMIT && recentlyLearnedCount === entries.length,
+ entries !== null &&
+ entries.length >= SCRATCHPAD_WINDOW_PAGE_SIZE &&
+ recentlyLearnedCount === entries.length,
],
// What the ledger filters over: the search hits while a search is typed, the whole loaded
// window otherwise. Null while the applicable set is still loading.
@@ -845,7 +850,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
}
try {
const page = await signalsScoutScratchpadSearch(String(teamId), {
- limit: SCRATCHPAD_FETCH_LIMIT,
+ limit: SCRATCHPAD_WINDOW_PAGE_SIZE,
content_max_chars: SCRATCHPAD_PREVIEW_CHARS,
date_to: cursor,
...dateFromParam(values.timeFilter),
@@ -861,7 +866,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([
const seen = new Set(loaded.map((entry) => entry.key))
actions.appendOlderEntries(
page.filter((entry) => !seen.has(entry.key)),
- page.length >= SCRATCHPAD_FETCH_LIMIT
+ page.length >= SCRATCHPAD_WINDOW_PAGE_SIZE
)
} catch {
actions.loadOlderEntriesFailure()