Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

16.9 KB
Raw Blame
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import { mkdtempSync, rmSync } from 'fs';
import { mkdir, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import express from 'express';
import { makePathsProxy } from '../lib/mockPathsDataRoot.js';
import { request } from '../lib/testHelper.js';

var tempRoot;
function getTempRoot() {
  if (!tempRoot) tempRoot = mkdtempSync(join(tmpdir(), 'brain-collection-test-'));
  return tempRoot;
}

// Redirect all data paths to tempRoot before module imports
vi.mock('../lib/fileUtils.js', async () => {
  const actual = await vi.importActual('../lib/fileUtils.js');
  return makePathsProxy({ ...actual, readJSONFile: vi.fn(actual.readJSONFile) }, { dataRoot: () => getTempRoot() });
});

vi.mock('../services/instanceIdentity.js', () => ({
  getInstanceId: () => Promise.resolve('test-instance'),
}));

vi.mock('../services/userActions.js', () => ({
  recordUserAction: vi.fn(async () => ({ id: 'evt' }))
}));

// Import after hoisting mocks
import * as brainStorage from '../services/brainStorage.js';
import brainRoutes from './brain.js';

const app = express();
app.use(express.json());
app.use('/api/brain', brainRoutes);

const TOTAL_INBOX = 1200;
const TOTAL_MEMORIES = 1500;
const TOTAL_ENTITIES = 60;
const ENTITY_TYPES = ['people', 'projects', 'ideas', 'admin'];
const BASE_TIME = Date.parse('2026-08-01T00:00:00.000Z');

async function seedRecord(type, id, record) {
  const dir = join(getTempRoot(), 'brain', type, id);
  await mkdir(dir);
  await writeFile(join(dir, 'index.json'), JSON.stringify({ id, ...record }));
}

async function seedRecords(type, count, createRecord) {
  const parent = join(getTempRoot(), 'brain', type);
  await mkdir(parent, { recursive: true });

  const batchSize = 48;
  for (let start = 0; start < count; start += batchSize) {
    const batchEnd = Math.min(start + batchSize, count);
    await Promise.all(Array.from({ length: batchEnd - start }, (_, offset) => {
      const i = start + offset;
      const id = `${type === 'inbox' ? 'inbox' : 'mem'}-${String(i).padStart(5, '0')}`;
      return seedRecord(type, id, createRecord(i, id));
    }));
  }
}

describe('Brain collection pagination (synthetic thousands-record fixtures)', () => {
  beforeAll(async () => {
    // Seed 1,200 inbox entries
    await seedRecords('inbox', TOTAL_INBOX, (i, id) => {
      const capturedAt = new Date(BASE_TIME + i * 60000).toISOString();
      const isTargetSearch = i === 15 || i === 480 || i === 1120;
      return {
        id,
        capturedText: isTargetSearch ? `Urgent needle task ${i}` : `Thought note entry #${i}`,
        title: isTargetSearch ? `Needle Title ${i}` : `Inbox Item ${i}`,
        status: i % 4 === 0 ? 'needs_review' : (i % 4 === 1 ? 'filed' : (i % 4 === 2 ? 'done' : 'classifying')),
        capturedAt,
        createdAt: capturedAt,
        updatedAt: capturedAt
      };
    });

    // Seed 1,500 memories
    await seedRecords('memories', TOTAL_MEMORIES, (i, id) => {
      const updatedAt = new Date(BASE_TIME + i * 60000).toISOString();
      const isTargetSearch = i === 22 || i === 655 || i === 1433;
      const longBody = `Memory full content #${i}: ` + 'Detailed reflective thoughts spanning multiple sentences. '.repeat(8);
      return {
        id,
        title: isTargetSearch ? `Target Query Memory ${i}` : `Daily Memory ${i}`,
        content: longBody,
        tags: [i % 50 === 0 ? 'rare-tag' : 'common', `topic-${i % 10}`],
        archived: i % 10 === 0, // 150 archived, 1350 unarchived
        createdAt: updatedAt,
        updatedAt
      };
    });

    for (const type of ENTITY_TYPES) {
      await seedRecords(type, TOTAL_ENTITIES, (i) => {
        const updatedAt = new Date(BASE_TIME + i * 60000).toISOString();
        return {
          name: `Example ${type} ${i}`,
          title: `Example ${type} ${i}`,
          status: i % 2 === 0 ? 'active' : 'done',
          createdAt: updatedAt,
          updatedAt
        };
      });
    }

    brainStorage.invalidateAllCaches();
  });

  // Thousands of per-record files: async removal keeps the worker loop free and the
  // explicit budget outlasts a loaded disk (default hookTimeout is 30s).
  afterAll(async () => {
    if (tempRoot) await rm(tempRoot, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
  }, 120_000);

  describe('Inbox pagination & bounds', () => {
    it('enforces default limit of 50 and returns complete collection counts', async () => {
      const res = await request(app).get('/api/brain/inbox');
      expect(res.status).toBe(200);
      expect(res.body.items).toHaveLength(50);
      expect(res.body.entries).toHaveLength(50);
      expect(res.body.total).toBe(TOTAL_INBOX);
      expect(res.body.nextCursor).toBeTruthy();
      expect(res.body.counts).toMatchObject({
        total: TOTAL_INBOX,
        needs_review: 300,
        filed: 300,
        done: 300,
        classifying: 300
      });
    });

    it('enforces server-enforced ceiling: serves up to max limit 100, refuses unbounded requests, and storage clamps', async () => {
      const atMax = await request(app).get('/api/brain/inbox?limit=100');
      expect(atMax.status).toBe(200);
      expect(atMax.body.items).toHaveLength(100);

      const overMax = await request(app).get('/api/brain/inbox?limit=5000');
      expect(overMax.status).toBe(400);

      // Storage layer clamps any direct/internal requests exceeding ceiling
      const directClamped = await brainStorage.getInboxPage({ limit: 5000 });
      expect(directClamped.items).toHaveLength(100);
    });

    it('pages continuously via stable cursors without duplicates or skips', async () => {
      const page1 = await request(app).get('/api/brain/inbox?limit=50');
      expect(page1.status).toBe(200);
      expect(page1.body.items).toHaveLength(50);
      const cursor1 = page1.body.nextCursor;
      expect(cursor1).toBeTruthy();

      const page2 = await request(app).get(`/api/brain/inbox?limit=50&cursor=${encodeURIComponent(cursor1)}`);
      expect(page2.status).toBe(200);
      expect(page2.body.items).toHaveLength(50);
      const cursor2 = page2.body.nextCursor;
      expect(cursor2).toBeTruthy();

      const ids1 = new Set(page1.body.items.map(e => e.id));
      const ids2 = new Set(page2.body.items.map(e => e.id));
      for (const id of ids2) {
        expect(ids1.has(id)).toBe(false);
      }

      // Order is newest-first
      const last1Time = Date.parse(page1.body.items[page1.body.items.length - 1].capturedAt);
      const first2Time = Date.parse(page2.body.items[0].capturedAt);
      expect(first2Time).toBeLessThanOrEqual(last1Time);
    });

    it('applies search across the complete 1,200 record collection, not just loaded pages', async () => {
      const res = await request(app).get('/api/brain/inbox?search=needle');
      expect(res.status).toBe(200);
      expect(res.body.total).toBe(3);
      expect(res.body.items).toHaveLength(3);
      expect(res.body.nextCursor).toBeNull();
      const ids = res.body.items.map(item => item.id);
      expect(ids).toContain('inbox-00015');
      expect(ids).toContain('inbox-00480');
      expect(ids).toContain('inbox-01120');
    });

    it('applies status filter across complete collection', async () => {
      const res = await request(app).get('/api/brain/inbox?status=needs_review&limit=25');
      expect(res.status).toBe(200);
      expect(res.body.total).toBe(300);
      expect(res.body.items).toHaveLength(25);
      expect(res.body.items.every(e => e.status === 'needs_review')).toBe(true);
      expect(res.body.nextCursor).toBeTruthy();
    });
  });

  describe('Memory collection pagination, compact projection, and compatibility', () => {
    it('returns compact row projections with truncated content when using cursor pagination', async () => {
      const res = await request(app).get('/api/brain/memories?cursor=&limit=25');
      expect(res.status).toBe(200);
      expect(res.body.items).toHaveLength(25);
      expect(res.body.total).toBe(1350); // 1500 minus 150 archived
      expect(res.body.nextCursor).toBeTruthy();

      // Verify compact row projection
      for (const item of res.body.items) {
        expect(item.content.length).toBeLessThanOrEqual(300);
        expect(item.contentTruncated).toBe(true);
      }
    });

    it('enforces server-enforced memory ceiling: serves up to 100, refuses over 100, and storage clamps', async () => {
      const atMax = await request(app).get('/api/brain/memories?cursor=&limit=100');
      expect(atMax.status).toBe(200);
      expect(atMax.body.items).toHaveLength(100);

      const overMax = await request(app).get('/api/brain/memories?cursor=&limit=500');
      expect(overMax.status).toBe(400);

      // Storage layer clamps any direct/internal requests exceeding ceiling
      const directClamped = await brainStorage.getEntityPage('memories', { limit: 500 });
      expect(directClamped.items).toHaveLength(100);
    });

    it('fetches full untruncated content on individual record detail endpoint', async () => {
      const listRes = await request(app).get('/api/brain/memories?cursor=&limit=1');
      const sample = listRes.body.items[0];
      expect(sample.contentTruncated).toBe(true);

      const detailRes = await request(app).get(`/api/brain/memories/${sample.id}`);
      expect(detailRes.status).toBe(200);
      expect(detailRes.body.id).toBe(sample.id);
      expect(detailRes.body.content.length).toBeGreaterThan(400);
      expect(detailRes.body.contentTruncated).toBeUndefined();
    });

    it('pages through memories with stable cursors and zero ID overlap', async () => {
      const page1 = await request(app).get('/api/brain/memories?cursor=&limit=25');
      const cursor1 = page1.body.nextCursor;
      const ids1 = new Set(page1.body.items.map(m => m.id));

      const page2 = await request(app).get(`/api/brain/memories?cursor=${encodeURIComponent(cursor1)}&limit=25`);
      expect(page2.status).toBe(200);
      expect(page2.body.items).toHaveLength(25);
      const ids2 = new Set(page2.body.items.map(m => m.id));

      for (const id of ids2) {
        expect(ids1.has(id)).toBe(false);
      }
    });

    it('applies memory search across complete collection', async () => {
      const res = await request(app).get('/api/brain/memories?search=Target%20Query');
      expect(res.status).toBe(200);
      expect(res.body.total).toBe(3);
      expect(res.body.items).toHaveLength(3);
      const titles = res.body.items.map(m => m.title);
      expect(titles).toContain('Target Query Memory 22');
      expect(titles).toContain('Target Query Memory 655');
      expect(titles).toContain('Target Query Memory 1433');
    });

    it('preserves legacy unpaginated array response for callers without cursor or search', async () => {
      const res = await request(app).get('/api/brain/memories');
      expect(res.status).toBe(200);
      expect(Array.isArray(res.body)).toBe(true);
      expect(res.body).toHaveLength(1500); // Legacy returns all records
    });

    it('preserves legacy limit/offset envelope for callers passing limit/offset without cursor or search', async () => {
      const res = await request(app).get('/api/brain/memories?limit=10&offset=5');
      expect(res.status).toBe(200);
      expect(res.body).toMatchObject({
        total: 1500,
        limit: 10,
        offset: 5
      });
      expect(res.body.memories).toHaveLength(10);
      expect(res.body.nextCursor).toBeUndefined();
    });
  });
  // The client wrappers (client/src/services/apiBrain.test.js) send `cursor=` on a
  // collection's first request; this pins what the router does with that URL.
  describe('First-page request from the Brain collection views (empty initial cursor)', () => {
    it.each([...ENTITY_TYPES, 'memories'])('%s: cursor mode returns a continuation that reaches every record', async (type) => {
      const total = type === 'memories' ? 1350 : TOTAL_ENTITIES;
      const first = await request(app).get(`/api/brain/${type}?cursor=&limit=25`);
      expect(first.status).toBe(200);
      expect(first.body.items).toHaveLength(25);
      expect(first.body.total).toBe(total);
      expect(first.body.nextCursor).toBeTruthy();
      expect(first.body.offset).toBeUndefined();

      let seen = first.body.items.map(item => item.id);
      let cursor = first.body.nextCursor;
      while (cursor) {
        const next = await request(app).get(`/api/brain/${type}?cursor=${encodeURIComponent(cursor)}&limit=25`);
        seen = seen.concat(next.body.items.map(item => item.id));
        cursor = next.body.nextCursor;
      }
      expect(new Set(seen).size).toBe(total);
    });

    it.each(ENTITY_TYPES)('%s: a request without cursor keeps the legacy offset envelope', async (type) => {
      const res = await request(app).get(`/api/brain/${type}?limit=25`);
      expect(res.body).toMatchObject({ total: TOTAL_ENTITIES, limit: 25, offset: 0 });
      expect(res.body.nextCursor).toBeUndefined();
    });

    it('combines the empty cursor with a status filter', async () => {
      const res = await request(app).get('/api/brain/projects?cursor=&limit=25&status=active');
      expect(res.body.total).toBe(TOTAL_ENTITIES / 2);
      expect(res.body.items.every(item => item.status === 'active')).toBe(true);
    });
  });

  // Bulk captures/imports share one timestamp. Rows sort newest-first with the id
  // ascending as tiebreak, so a cursor must continue with the ids AFTER its own at
  // the same timestamp. These fixtures are isolated by a search token and removed
  // afterwards so the whole-collection counts above are unaffected.
  describe('Cursor continuation across same-timestamp ties', () => {
    const TIE_TOKEN = 'tiefixture';
    const TIE_BASE = BASE_TIME - 24 * 60 * 60 * 1000;
    // Three timestamp groups of 7 records, so ties cross every page boundary of size 5.
    const GROUPS = [2, 1, 0].map((n) => TIE_BASE + n * 60000); // newest first
    const PER_GROUP = 7;
    const tieIds = {};

    const seedTies = async (type, prefix, createRecord) => {
      tieIds[type] = [];
      for (const [rank, ms] of GROUPS.entries()) {
        for (let i = 0; i < PER_GROUP; i++) {
          // Newer groups get HIGHER ids, so ids are not monotonic with the timestamp
          // order (a plain key comparison would pick the wrong side of each tie).
          const id = `${prefix}-tie-${String((GROUPS.length - rank) * 10 + i).padStart(3, '0')}`;
          await seedRecord(type, id, createRecord(id, new Date(ms).toISOString()));
          tieIds[type].push({ id, rank });
        }
      }
      brainStorage.invalidateAllCaches();
    };

    // Newest timestamp first; within a timestamp, ids ascending.
    const expectedOrder = (type) => [...tieIds[type]]
      .sort((a, b) => (a.rank - b.rank) || a.id.localeCompare(b.id))
      .map(({ id }) => id);

    const walk = async (type, { limit = 5, onFirstPage } = {}) => {
      const url = `/api/brain/${type}?search=${TIE_TOKEN}&limit=${limit}`;
      const ids = [];
      let cursor = '';
      let pages = 0;
      let deleted = 0;
      do {
        const res = await request(app).get(`${url}&cursor=${encodeURIComponent(cursor)}`);
        expect(res.status).toBe(200);
        expect(res.body.total).toBe(GROUPS.length * PER_GROUP - deleted);
        ids.push(...res.body.items.map((item) => item.id));
        cursor = res.body.nextCursor;
        pages += 1;
        if (pages === 1 && onFirstPage) deleted = await onFirstPage(res.body);
        expect(pages).toBeLessThan(20);
      } while (cursor);
      return ids;
    };

    beforeAll(async () => {
      await seedTies('inbox', 'inbox', (id, at) => ({
        capturedText: `${TIE_TOKEN} note`, title: id, status: 'filed', capturedAt: at, createdAt: at, updatedAt: at
      }));
      await seedTies('projects', 'mem', (id, at) => ({
        name: `${TIE_TOKEN} ${id}`, title: `${TIE_TOKEN} ${id}`, status: 'active', createdAt: at, updatedAt: at
      }));
    });

    afterAll(async () => {
      for (const [type, rows] of Object.entries(tieIds)) {
        for (const { id } of rows) rmSync(join(getTempRoot(), 'brain', type, id), { recursive: true, force: true });
      }
      brainStorage.invalidateAllCaches();
    });

    it.each(['inbox', 'projects'])('%s: walking tied timestamps returns every record exactly once in sort order', async (type) => {
      const ids = await walk(type);
      expect(ids).toEqual(expectedOrder(type));
    });

    it.each(['inbox', 'projects'])('%s: deleting the boundary row does not strand the remaining records', async (type) => {
      const expected = expectedOrder(type);
      let removed = null;
      const ids = await walk(type, {
        onFirstPage: async (body) => {
          removed = body.items[body.items.length - 1].id;
          rmSync(join(getTempRoot(), 'brain', type, removed), { recursive: true, force: true });
          brainStorage.invalidateAllCaches();
          return 1;
        }
      });
      expect(removed).toBe(expected[4]);
      // The boundary row was served on page one; everything after it still arrives.
      expect(ids).toEqual(expected);
    });
  });
});