Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: continue Brain cursor pages past same-timestamp ties (#10871)
Adam Eivy6bbba566edd4
diff --git a/server/routes/brainCollectionPagination.test.js b/server/routes/brainCollectionPagination.test.jsindex 2cdad38e60..401f3965e7 100644--- a/server/routes/brainCollectionPagination.test.js+++ b/server/routes/brainCollectionPagination.test.js@@ -311,4 +311,92 @@ describe('Brain collection pagination (synthetic thousands-record fixtures)', ()expect(res.body.items.every(item => item.status === 'active')).toBe(true);});});++ // Bulk captures/imports share one timestamp. Rows sort newest-first with the id+ // ascending as tiebreak, so a cursor must continue with the ids AFTER its own at+ // the same timestamp. These fixtures are isolated by a search token and removed+ // afterwards so the whole-collection counts above are unaffected.+ describe('Cursor continuation across same-timestamp ties', () => {+ const TIE_TOKEN = 'tiefixture';+ const TIE_BASE = BASE_TIME - 24 * 60 * 60 * 1000;+ // Three timestamp groups of 7 records, so ties cross every page boundary of size 5.+ const GROUPS = [2, 1, 0].map((n) => TIE_BASE + n * 60000); // newest first+ const PER_GROUP = 7;+ const tieIds = {};++ const seedTies = async (type, prefix, createRecord) => {+ tieIds[type] = [];+ for (const [rank, ms] of GROUPS.entries()) {+ for (let i = 0; i < PER_GROUP; i++) {+ // Newer groups get HIGHER ids, so ids are not monotonic with the timestamp+ // order (a plain key comparison would pick the wrong side of each tie).+ const id = `${prefix}-tie-${String((GROUPS.length - rank) * 10 + i).padStart(3, '0')}`;+ await seedRecord(type, id, createRecord(id, new Date(ms).toISOString()));+ tieIds[type].push({ id, rank });+ }+ }+ brainStorage.invalidateAllCaches();+ };++ // Newest timestamp first; within a timestamp, ids ascending.+ const expectedOrder = (type) => [...tieIds[type]]+ .sort((a, b) => (a.rank - b.rank) || a.id.localeCompare(b.id))+ .map(({ id }) => id);++ const walk = async (type, { limit = 5, onFirstPage } = {}) => {+ const url = `/api/brain/${type}?search=${TIE_TOKEN}&limit=${limit}`;+ const ids = [];+ let cursor = '';+ let pages = 0;+ let deleted = 0;+ do {+ const res = await request(app).get(`${url}&cursor=${encodeURIComponent(cursor)}`);+ expect(res.status).toBe(200);+ expect(res.body.total).toBe(GROUPS.length * PER_GROUP - deleted);+ ids.push(...res.body.items.map((item) => item.id));+ cursor = res.body.nextCursor;+ pages += 1;+ if (pages === 1 && onFirstPage) deleted = await onFirstPage(res.body);+ expect(pages).toBeLessThan(20);+ } while (cursor);+ return ids;+ };++ beforeAll(async () => {+ await seedTies('inbox', 'inbox', (id, at) => ({+ capturedText: `${TIE_TOKEN} note`, title: id, status: 'filed', capturedAt: at, createdAt: at, updatedAt: at+ }));+ await seedTies('projects', 'mem', (id, at) => ({+ name: `${TIE_TOKEN} ${id}`, title: `${TIE_TOKEN} ${id}`, status: 'active', createdAt: at, updatedAt: at+ }));+ });++ afterAll(async () => {+ for (const [type, rows] of Object.entries(tieIds)) {+ for (const { id } of rows) rmSync(join(getTempRoot(), 'brain', type, id), { recursive: true, force: true });+ }+ brainStorage.invalidateAllCaches();+ });++ it.each(['inbox', 'projects'])('%s: walking tied timestamps returns every record exactly once in sort order', async (type) => {+ const ids = await walk(type);+ expect(ids).toEqual(expectedOrder(type));+ });++ it.each(['inbox', 'projects'])('%s: deleting the boundary row does not strand the remaining records', async (type) => {+ const expected = expectedOrder(type);+ let removed = null;+ const ids = await walk(type, {+ onFirstPage: async (body) => {+ removed = body.items[body.items.length - 1].id;+ rmSync(join(getTempRoot(), 'brain', type, removed), { recursive: true, force: true });+ brainStorage.invalidateAllCaches();+ return 1;+ }+ });+ expect(removed).toBe(expected[4]);+ // The boundary row was served on page one; everything after it still arrives.+ expect(ids).toEqual(expected);+ });+ });});diff --git a/server/services/brainStorage.js b/server/services/brainStorage.jsindex b62b3ee43b..3158ecf47a 100644--- a/server/services/brainStorage.js+++ b/server/services/brainStorage.js@@ -796,6 +796,26 @@ async function appendJsonl(type, record) {// brain type. These wrappers keep the historical getInboxLog/createInboxLog/…// API (capturedAt sort, status counts) on top of the generic entity primitives.+// Cursor pages share one contract: rows sort newest-first (timestamp descending)+// with the id ascending as the tiebreak, and the cursor is the last row's+// `<zero-padded timestamp>_<id>` key. Continuation therefore keeps older rows+// and, at the cursor's own timestamp, only ids that sort AFTER the cursor id --+// comparing the encoded key as a string would pick the smaller ids instead and+// repeat/skip rows whenever a timestamp tie crosses a page boundary. The cursor+// row itself need not still exist. A malformed cursor yields the first page.+const cursorKeyOf = (id, timestampMs) => `${String(timestampMs).padStart(15, '0')}_${id}`;++function rowsAfterCursor(rows, cursor, timestampOf) {+ const match = /^(\d+)_(.*)$/s.exec(cursor);+ if (!match) return rows;+ const cursorMs = Number(match[1]);+ const cursorId = match[2];+ return rows.filter(([id, summary]) => {+ const ms = timestampOf(summary);+ return ms < cursorMs || (ms === cursorMs && id.localeCompare(cursorId) > 0);+ });+}+/*** One page of inbox entries (newest-first by capturedAt), optional status/search filters and cursor pagination.*/@@ -821,11 +841,9 @@ export async function getInboxPage(options = {}) {matching.sort(([idA, a], [idB, b]) => (b.capturedAtMs - a.capturedAtMs) || idA.localeCompare(idB));const total = matching.length;- const keyOf = (id, s) => `${String(s.capturedAtMs).padStart(15, '0')}_${id}`;-let remaining = matching;if (cursor) {- remaining = matching.filter(([id, s]) => keyOf(id, s) < cursor);+ remaining = rowsAfterCursor(matching, cursor, (s) => s.capturedAtMs);} else if (offset > 0) {remaining = matching.slice(offset);}@@ -837,7 +855,7 @@ export async function getInboxPage(options = {}) {const hasMore = remaining.length > boundedLimit;const nextCursor = hasMore && pageRows.length > 0- ? keyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1])+ ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].capturedAtMs): null;return {@@ -1256,11 +1274,9 @@ export async function getEntityPage(type, options = {}) {matching.sort(([idA, a], [idB, b]) => (b.sortKey - a.sortKey) || idA.localeCompare(idB));const total = matching.length;- const keyOf = (id, s) => `${String(s.sortKey).padStart(15, '0')}_${id}`;-let remaining = matching;if (cursor) {- remaining = matching.filter(([id, s]) => keyOf(id, s) < cursor);+ remaining = rowsAfterCursor(matching, cursor, (s) => s.sortKey);} else if (offset > 0) {remaining = matching.slice(offset);}@@ -1276,7 +1292,7 @@ export async function getEntityPage(type, options = {}) {const hasMore = remaining.length > boundedLimit;const nextCursor = hasMore && pageRows.length > 0- ? keyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1])+ ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].sortKey): null;return {