:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): fetch smaller pages in the scout memory panel (#122992)
posthog[bot]b222fa73a760
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsxindex 895bb6706571..dacbdfc832d2 100644--- a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterStats.tsx@@ -5,7 +5,7 @@ import { Tooltip } from '@posthog/lemon-ui'import { cn } from 'lib/utils/css-classes'import { scoutFleetLogic } from '../../../logics/scoutFleetLogic'-import { scratchpadLogic } from '../../../logics/scratchpadLogic'+import { SCRATCHPAD_WINDOW_PAGE_SIZE, scratchpadLogic } from '../../../logics/scratchpadLogic'import { SCOUT_ROSTER_WINDOW_LABEL } from '../../../utils/scoutRunsWindow'type StatTone = 'muted' | 'warning' | 'danger'@@ -51,7 +51,7 @@ function Stat({export function ScoutsRosterStats(): JSX.Element {const { pauseAttentionCounts, enabledCount, emittedFindingsSummary, fleetFindingsSummaryLoadedOnce } =useValues(scoutFleetLogic)- // Mounted here rather than by the fleet logic, so the 1,000-entry scratchpad read only happens+ // Mounted here rather than by the fleet logic, so the scratchpad read only happens// on the surfaces that show it, not on every inbox tab that mounts the fleet.const { recentlyLearnedCount, recentlyLearnedCountCapped, entries: scratchpadEntries } = useValues(scratchpadLogic)// A summary that hasn't landed is not zero runs. The number holds a dash until it does; a failed@@ -103,7 +103,7 @@ export function ScoutsRosterStats(): JSX.Element {label="learned"tooltip={recentlyLearnedCountCapped- ? `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}. Only the newest 1,000 entries are counted, and all of them fall in this window.`+ ? `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}. Only the newest ${SCRATCHPAD_WINDOW_PAGE_SIZE} entries are counted, and all of them fall in this window.`: `Scratchpad entries your scouts wrote or refreshed in the ${SCOUT_ROSTER_WINDOW_LABEL}.`}/>diff --git a/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx b/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsxindex 2b4270787ebc..3aa1684f0d26 100644--- a/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx+++ b/products/signals/frontend/inbox/components/scratchpad/ScratchpadPanel.tsx@@ -262,7 +262,7 @@ function ScratchpadHeader({settled on, things they ruled out, and the bookkeeping that keeps them from repeating themselves.</p><div className="flex flex-wrap gap-x-6 gap-y-2">- {/* The count is what's loaded, not what exists: the endpoint caps at 1,000 newest-first,+ {/* The count is what's loaded, not what exists: the first page is the newest few hundred,which on a busy project is a few hours. Labelling it with the span it covers is whatstops it reading as the whole memory. */}<Statdiff --git a/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts b/products/signals/frontend/inbox/logics/scratchpadLogic.test.tsindex 0cc50512ffcc..9097cdb93d03 100644--- a/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts+++ b/products/signals/frontend/inbox/logics/scratchpadLogic.test.ts@@ -13,7 +13,7 @@ import type { ScratchpadEntryApi } from 'products/signals/frontend/generated/apiimport { PANEL_LOAD_TIMEOUT_MS } from '../utils/panelLoadTimeout'import { forgetUnreachableReports } from '../utils/unreachableReports'import {- SCRATCHPAD_FETCH_LIMIT,+ SCRATCHPAD_WINDOW_PAGE_SIZE,SCRATCHPAD_PREVIEW_CHARS,describeLoadedSpan,entriesForSkill,@@ -185,15 +185,15 @@ describe('scratchpadLogic', () => {...entry(`pattern:${i}`, 'note'),updated_at: new Date().toISOString(),})- const full = Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => fresh(i))+ const full = Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => fresh(i))logic.actions.loadEntriesSuccess(full)- expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_FETCH_LIMIT)+ expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_WINDOW_PAGE_SIZE)expect(logic.values.recentlyLearnedCountCapped).toBe(true)// One stale entry in a full fetch means the window ended inside it, so the count is exact.logic.actions.loadEntriesSuccess([...full.slice(1), WHOLE])- expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_FETCH_LIMIT - 1)+ expect(logic.values.recentlyLearnedCount).toBe(SCRATCHPAD_WINDOW_PAGE_SIZE - 1)expect(logic.values.recentlyLearnedCountCapped).toBe(false)logic.actions.loadEntriesSuccess(full.slice(0, 10))@@ -336,11 +336,11 @@ describe('scratchpadLogic', () => {})// Older pages walk the unfiltered window with a cursor, and a search is a separate one-shot- // read they never reach. A control offered during a search spends a page of up to a thousand+ // read they never reach. A control offered during a search spends a page of up to a few hundred// rows the table cannot show, and moves the header counts while the listed rows stay put.it('offers no older pages while a search is active', async () => {logic.actions.loadEntriesSuccess(- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note')))expect(logic.values.canLoadOlderEntries).toBe(true)@@ -357,7 +357,7 @@ describe('scratchpadLogic', () => {// older is left. A control left live over that gap pages from the window that just went away.it('retires the older-page control while the first page reloads', async () => {logic.actions.loadEntriesSuccess(- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note')))expect(logic.values.hasMoreOlderEntries).toBe(true)@@ -406,7 +406,7 @@ describe('scratchpadLogic', () => {// spinner just stops, and the reader reads a failed press as an empty memory.it('records a failed older page and clears the record on the retry', async () => {logic.actions.loadEntriesSuccess(- Array.from({ length: SCRATCHPAD_FETCH_LIMIT }, (_, i) => entry(`pattern:${i}`, 'note'))+ Array.from({ length: SCRATCHPAD_WINDOW_PAGE_SIZE }, (_, i) => entry(`pattern:${i}`, 'note')))useMocks({ get: { [SCRATCHPAD_URL]: () => [500, {}] } })@@ -441,12 +441,15 @@ describe('scratchpadLogic', () => {})logic.actions.loadEntries()await expectLogic(logic).toFinishAllListeners()+ // The unfiltered window is the wide read, so both pages ask for the smaller page.+ expect(searchRequests[0].get('limit')).toEqual(String(SCRATCHPAD_WINDOW_PAGE_SIZE))searchRequests = []logic.actions.loadOlderEntries()await expectLogic(logic).toFinishAllListeners()expect(searchRequests[0].get('date_to')).toEqual(WHOLE.updated_at)+ expect(searchRequests[0].get('limit')).toEqual(String(SCRATCHPAD_WINDOW_PAGE_SIZE))expect(logic.values.olderEntries).toEqual([older])expect(logic.values.windowEntries?.map((e) => e.key)).toEqual([TRUNCATED.key, WHOLE.key, older.key])// A short page is the end of the memory, so the button goes away.diff --git a/products/signals/frontend/inbox/logics/scratchpadLogic.ts b/products/signals/frontend/inbox/logics/scratchpadLogic.tsindex ce99d60e1996..176eaff16b49 100644--- a/products/signals/frontend/inbox/logics/scratchpadLogic.ts+++ b/products/signals/frontend/inbox/logics/scratchpadLogic.ts@@ -15,10 +15,13 @@ import { isReportUnreachable, rememberUnreachableReport } from '../utils/unreach// Search reruns the server-side ILIKE on every keystroke; debounce so typing doesn't// fire a request per character.const SEARCH_DEBOUNCE_MS = 300-// `list` caps at 1000 newest-first with no pagination wrapper — pull the whole window in-// one read and filter it client-side. Older memory arrives through `loadOlderEntries`, which-// walks the endpoint's `date_to` cursor a page at a time.+// `list` caps at 1000 newest-first with no pagination wrapper. Search and one scout's entries+// are narrow reads, so they take the whole cap in one read and filter it client-side.export const SCRATCHPAD_FETCH_LIMIT = 1000+// The unfiltered window is the wide read: on a busy project 1000 previews cost Postgres megabytes+// per panel open. Fetch a smaller first page, and let `loadOlderEntries` walk the endpoint's+// `date_to` cursor for older memory a page at a time.+export const SCRATCHPAD_WINDOW_PAGE_SIZE = 200// Bodies are an unbounded TextField clamped at 50k chars on write, so a full-fat window of// 1000 entries is a payload nobody needs: the ledger renders a one-line clamp until you open it.// Pull previews for the list and fetch the one body you expand. Sized well past one line so@@ -94,8 +97,8 @@ export function entriesForSkill(/*** How wide a span the loaded rows actually cover, e.g. "last 8 h". The header needs this because- * 1,000 rows is a cap, not a total: on a busy project the window closes after a few hours, and a- * bare "1,000 entries" reads as the whole memory.+ * a page of rows is a cap, not a total: on a busy project the window closes after a few hours, and+ * a bare "200 entries" reads as the whole memory.*/export function describeLoadedSpan(entries: ScratchpadEntryApi[] | null): string | null {const stamps = (entries ?? []).map((entry) => entry.updated_at).filter((stamp): stamp is string => !!stamp)@@ -406,7 +409,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([signalsScoutScratchpadSearch(String(teamId),{- limit: SCRATCHPAD_FETCH_LIMIT,+ limit: SCRATCHPAD_WINDOW_PAGE_SIZE,content_max_chars: SCRATCHPAD_PREVIEW_CHARS,...dateFromParam(values.timeFilter),},@@ -538,7 +541,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([// still answer after a reload started, and this is bumped at dispatch, so the listener// sees the invalidation without waiting for the fresh page to arrive.windowGeneration: [0, { loadEntries: (state: number) => state + 1 }],- // Pages walked back past the 1,000-row cap with the endpoint's `date_to` cursor. A fresh+ // Pages walked back past the first page with the endpoint's `date_to` cursor. A fresh// first page invalidates them: it may already carry rows these pages hold.olderEntries: [[] as ScratchpadEntryApi[],@@ -573,7 +576,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([hasMoreOlderEntries: [false,{- loadEntriesSuccess: (_, { entries }) => entries.length >= SCRATCHPAD_FETCH_LIMIT,+ loadEntriesSuccess: (_, { entries }) => entries.length >= SCRATCHPAD_WINDOW_PAGE_SIZE,appendOlderEntries: (_, { hasMore }) => hasMore,// A reload drops the walked pages, so the control goes with them until the fresh// first page says whether anything older is left.@@ -660,7 +663,9 @@ export const scratchpadLogic = kea<scratchpadLogicType>([recentlyLearnedCountCapped: [(s) => [s.entries, s.recentlyLearnedCount],(entries: ScratchpadEntryApi[] | null, recentlyLearnedCount: number): boolean =>- entries !== null && entries.length >= SCRATCHPAD_FETCH_LIMIT && recentlyLearnedCount === entries.length,+ entries !== null &&+ entries.length >= SCRATCHPAD_WINDOW_PAGE_SIZE &&+ recentlyLearnedCount === entries.length,],// What the ledger filters over: the search hits while a search is typed, the whole loaded// window otherwise. Null while the applicable set is still loading.@@ -845,7 +850,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([}try {const page = await signalsScoutScratchpadSearch(String(teamId), {- limit: SCRATCHPAD_FETCH_LIMIT,+ limit: SCRATCHPAD_WINDOW_PAGE_SIZE,content_max_chars: SCRATCHPAD_PREVIEW_CHARS,date_to: cursor,...dateFromParam(values.timeFilter),@@ -861,7 +866,7 @@ export const scratchpadLogic = kea<scratchpadLogicType>([const seen = new Set(loaded.map((entry) => entry.key))actions.appendOlderEntries(page.filter((entry) => !seen.has(entry.key)),- page.length >= SCRATCHPAD_FETCH_LIMIT+ page.length >= SCRATCHPAD_WINDOW_PAGE_SIZE)} catch {actions.loadOlderEntriesFailure()