Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: continue Brain cursor pages past same-timestamp ties (#10871)

Adam Eivy6bbba566edd4
diff --git a/server/routes/brainCollectionPagination.test.js b/server/routes/brainCollectionPagination.test.js
index 2cdad38e60..401f3965e7 100644
--- a/server/routes/brainCollectionPagination.test.js
+++ b/server/routes/brainCollectionPagination.test.js
@@ -311,4 +311,92 @@ describe('Brain collection pagination (synthetic thousands-record fixtures)', ()
expect(res.body.items.every(item => item.status === 'active')).toBe(true);
});
});
+
+ // Bulk captures/imports share one timestamp. Rows sort newest-first with the id
+ // ascending as tiebreak, so a cursor must continue with the ids AFTER its own at
+ // the same timestamp. These fixtures are isolated by a search token and removed
+ // afterwards so the whole-collection counts above are unaffected.
+ describe('Cursor continuation across same-timestamp ties', () => {
+ const TIE_TOKEN = 'tiefixture';
+ const TIE_BASE = BASE_TIME - 24 * 60 * 60 * 1000;
+ // Three timestamp groups of 7 records, so ties cross every page boundary of size 5.
+ const GROUPS = [2, 1, 0].map((n) => TIE_BASE + n * 60000); // newest first
+ const PER_GROUP = 7;
+ const tieIds = {};
+
+ const seedTies = async (type, prefix, createRecord) => {
+ tieIds[type] = [];
+ for (const [rank, ms] of GROUPS.entries()) {
+ for (let i = 0; i < PER_GROUP; i++) {
+ // Newer groups get HIGHER ids, so ids are not monotonic with the timestamp
+ // order (a plain key comparison would pick the wrong side of each tie).
+ const id = `${prefix}-tie-${String((GROUPS.length - rank) * 10 + i).padStart(3, '0')}`;
+ await seedRecord(type, id, createRecord(id, new Date(ms).toISOString()));
+ tieIds[type].push({ id, rank });
+ }
+ }
+ brainStorage.invalidateAllCaches();
+ };
+
+ // Newest timestamp first; within a timestamp, ids ascending.
+ const expectedOrder = (type) => [...tieIds[type]]
+ .sort((a, b) => (a.rank - b.rank) || a.id.localeCompare(b.id))
+ .map(({ id }) => id);
+
+ const walk = async (type, { limit = 5, onFirstPage } = {}) => {
+ const url = `/api/brain/${type}?search=${TIE_TOKEN}&limit=${limit}`;
+ const ids = [];
+ let cursor = '';
+ let pages = 0;
+ let deleted = 0;
+ do {
+ const res = await request(app).get(`${url}&cursor=${encodeURIComponent(cursor)}`);
+ expect(res.status).toBe(200);
+ expect(res.body.total).toBe(GROUPS.length * PER_GROUP - deleted);
+ ids.push(...res.body.items.map((item) => item.id));
+ cursor = res.body.nextCursor;
+ pages += 1;
+ if (pages === 1 && onFirstPage) deleted = await onFirstPage(res.body);
+ expect(pages).toBeLessThan(20);
+ } while (cursor);
+ return ids;
+ };
+
+ beforeAll(async () => {
+ await seedTies('inbox', 'inbox', (id, at) => ({
+ capturedText: `${TIE_TOKEN} note`, title: id, status: 'filed', capturedAt: at, createdAt: at, updatedAt: at
+ }));
+ await seedTies('projects', 'mem', (id, at) => ({
+ name: `${TIE_TOKEN} ${id}`, title: `${TIE_TOKEN} ${id}`, status: 'active', createdAt: at, updatedAt: at
+ }));
+ });
+
+ afterAll(async () => {
+ for (const [type, rows] of Object.entries(tieIds)) {
+ for (const { id } of rows) rmSync(join(getTempRoot(), 'brain', type, id), { recursive: true, force: true });
+ }
+ brainStorage.invalidateAllCaches();
+ });
+
+ it.each(['inbox', 'projects'])('%s: walking tied timestamps returns every record exactly once in sort order', async (type) => {
+ const ids = await walk(type);
+ expect(ids).toEqual(expectedOrder(type));
+ });
+
+ it.each(['inbox', 'projects'])('%s: deleting the boundary row does not strand the remaining records', async (type) => {
+ const expected = expectedOrder(type);
+ let removed = null;
+ const ids = await walk(type, {
+ onFirstPage: async (body) => {
+ removed = body.items[body.items.length - 1].id;
+ rmSync(join(getTempRoot(), 'brain', type, removed), { recursive: true, force: true });
+ brainStorage.invalidateAllCaches();
+ return 1;
+ }
+ });
+ expect(removed).toBe(expected[4]);
+ // The boundary row was served on page one; everything after it still arrives.
+ expect(ids).toEqual(expected);
+ });
+ });
});
diff --git a/server/services/brainStorage.js b/server/services/brainStorage.js
index b62b3ee43b..3158ecf47a 100644
--- a/server/services/brainStorage.js
+++ b/server/services/brainStorage.js
@@ -796,6 +796,26 @@ async function appendJsonl(type, record) {
// brain type. These wrappers keep the historical getInboxLog/createInboxLog/…
// API (capturedAt sort, status counts) on top of the generic entity primitives.
+// Cursor pages share one contract: rows sort newest-first (timestamp descending)
+// with the id ascending as the tiebreak, and the cursor is the last row's
+// `<zero-padded timestamp>_<id>` key. Continuation therefore keeps older rows
+// and, at the cursor's own timestamp, only ids that sort AFTER the cursor id --
+// comparing the encoded key as a string would pick the smaller ids instead and
+// repeat/skip rows whenever a timestamp tie crosses a page boundary. The cursor
+// row itself need not still exist. A malformed cursor yields the first page.
+const cursorKeyOf = (id, timestampMs) => `${String(timestampMs).padStart(15, '0')}_${id}`;
+
+function rowsAfterCursor(rows, cursor, timestampOf) {
+ const match = /^(\d+)_(.*)$/s.exec(cursor);
+ if (!match) return rows;
+ const cursorMs = Number(match[1]);
+ const cursorId = match[2];
+ return rows.filter(([id, summary]) => {
+ const ms = timestampOf(summary);
+ return ms < cursorMs || (ms === cursorMs && id.localeCompare(cursorId) > 0);
+ });
+}
+
/**
* One page of inbox entries (newest-first by capturedAt), optional status/search filters and cursor pagination.
*/
@@ -821,11 +841,9 @@ export async function getInboxPage(options = {}) {
matching.sort(([idA, a], [idB, b]) => (b.capturedAtMs - a.capturedAtMs) || idA.localeCompare(idB));
const total = matching.length;
- const keyOf = (id, s) => `${String(s.capturedAtMs).padStart(15, '0')}_${id}`;
-
let remaining = matching;
if (cursor) {
- remaining = matching.filter(([id, s]) => keyOf(id, s) < cursor);
+ remaining = rowsAfterCursor(matching, cursor, (s) => s.capturedAtMs);
} else if (offset > 0) {
remaining = matching.slice(offset);
}
@@ -837,7 +855,7 @@ export async function getInboxPage(options = {}) {
const hasMore = remaining.length > boundedLimit;
const nextCursor = hasMore && pageRows.length > 0
- ? keyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1])
+ ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].capturedAtMs)
: null;
return {
@@ -1256,11 +1274,9 @@ export async function getEntityPage(type, options = {}) {
matching.sort(([idA, a], [idB, b]) => (b.sortKey - a.sortKey) || idA.localeCompare(idB));
const total = matching.length;
- const keyOf = (id, s) => `${String(s.sortKey).padStart(15, '0')}_${id}`;
-
let remaining = matching;
if (cursor) {
- remaining = matching.filter(([id, s]) => keyOf(id, s) < cursor);
+ remaining = rowsAfterCursor(matching, cursor, (s) => s.sortKey);
} else if (offset > 0) {
remaining = matching.slice(offset);
}
@@ -1276,7 +1292,7 @@ export async function getEntityPage(type, options = {}) {
const hasMore = remaining.length > boundedLimit;
const nextCursor = hasMore && pageRows.length > 0
- ? keyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1])
+ ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].sortKey)
: null;
return {