Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.5 KB
Raw Blame
/**
 * Coverage guard for the Data Manager's CATEGORIES map (issue #3285).
 *
 * An unclassified `data/` directory doesn't merely render as "Unknown category"
 * — `categoryMeta()` also falls back to `archivable: false, deletable: false`,
 * so the row loses its Archive/Purge affordances entirely. That is how the
 * single largest directory on disk ended up being the one thing the cleanup
 * page refused to act on.
 *
 * This test enumerates the top-level `data/` directories the codebase can
 * create and fails when one of them has no CATEGORIES entry. Discovery is
 * derived, never hand-maintained, so a new data dir can't regress silently:
 *
 *   1. `PATHS` in lib/fileUtils.js — every value under `PATHS.data`.
 *   2. `DEFAULT_EXCLUDES` in services/backup.js — anchored rsync filter paths.
 *   3. Source scan of server/ + scripts/ for the ways a data dir is spelled
 *      outside PATHS: `dataPath('name')`, `join(PATHS.data, 'name')`,
 *      `` `${PATHS.data}/name` ``, `join(root, 'data', 'name')`, and the
 *      collection-store idiom `join(PATHS.data, TYPE)` with a const name.
 *   4. `data.reference/` — dirs seeded into `data/` on first install.
 */

import { describe, it, expect } from 'vitest';
import { readdirSync, readFileSync } from 'fs';
import { join, relative, sep } from 'path';
import { PATHS } from '../lib/fileUtils.js';
import { DEFAULT_EXCLUDES } from './backup.js';
import { CATEGORIES, UNKNOWN_CATEGORY_DESCRIPTION, purgeCategory, archiveCategory } from './dataManager.js';

// A path segment that is a file, not a directory (`settings.json`, `TASKS.md`).
const isFileSegment = (segment) => segment.includes('.');

// Every spelling of "a path under data/" that appears in the codebase. Missing
// one means a whole family of directories escapes the guard — `dataPath('x')`
// was the gap that let `data/spotify` and `data/youtube` slip through.
const DATA_DIR_PATTERNS = [
  /\bdataPath\(\s*'([a-z0-9._-]+)'/g,
  /PATHS\.data,\s*'([a-z0-9._-]+)'/g,
  /\$\{PATHS\.data\}\/([a-z0-9._-]+)/g,
  /\b(?:join|resolve)\(\s*[^,;]+,\s*'data',\s*'([a-z0-9._-]+)'/g
];

// `join(PATHS.data, TYPE)` — the collection-store idiom, where the directory
// name is a module constant rather than an inline literal. Capture the
// identifier, then resolve it against its declaration in the same file.
const INDIRECT_DATA_DIR_PATTERN = /PATHS\.data,\s*([A-Z][A-Z0-9_]*)\s*\)/g;

function resolveConstant(source, identifier) {
  const declaration = new RegExp(`\\bconst\\s+${identifier}\\s*=\\s*'([a-z0-9._-]+)'`);
  return source.match(declaration)?.[1] ?? null;
}

function walkJsFiles(dir, out = []) {
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
    const full = join(dir, entry.name);
    if (entry.isDirectory()) walkJsFiles(full, out);
    else if (entry.name.endsWith('.js') && !entry.name.includes('.test.')) out.push(full);
  }
  return out;
}

/** @returns {Map<string, string>} data dir name → where it was discovered */
function discoverDataDirs() {
  const found = new Map();
  const add = (name, source) => {
    if (!name || isFileSegment(name) || found.has(name)) return;
    found.set(name, source);
  };

  const dataRoot = PATHS.data + sep;
  for (const [key, value] of Object.entries(PATHS)) {
    if (typeof value !== 'string' || !value.startsWith(dataRoot)) continue;
    add(value.slice(dataRoot.length).split(sep)[0], `PATHS.${key}`);
  }

  for (const { path } of DEFAULT_EXCLUDES) {
    add(path.replace(/^\//, '').split('/')[0], 'backup.js DEFAULT_EXCLUDES');
  }

  for (const dir of ['server', 'scripts']) {
    for (const file of walkJsFiles(join(PATHS.root, dir))) {
      const source = readFileSync(file, 'utf8');
      for (const pattern of DATA_DIR_PATTERNS) {
        for (const match of source.matchAll(pattern)) {
          add(match[1], relative(PATHS.root, file));
        }
      }
      for (const match of source.matchAll(INDIRECT_DATA_DIR_PATTERN)) {
        add(resolveConstant(source, match[1]), relative(PATHS.root, file));
      }
    }
  }

  for (const entry of readdirSync(join(PATHS.root, 'data.reference'), { withFileTypes: true })) {
    if (entry.isDirectory()) add(entry.name, 'data.reference/');
  }

  return found;
}

describe('dataManager CATEGORIES coverage (#3285)', () => {
  it('discovers literal data paths without treating domain-name arrays as paths', () => {
    const source = `
      const a = join(rootDir, 'data', 'example-one');
      const b = path.resolve(
        tmpdir(),
        'data', 'example-two'
      );
      const districts = ['agents', 'data', 'federation'];
    `;
    expect(DATA_DIR_PATTERNS.flatMap((pattern) => [...source.matchAll(pattern)].map((match) => match[1])))
      .toEqual(['example-one', 'example-two']);
  });

  it('classifies every top-level data/ directory the codebase can create', () => {
    const discovered = discoverDataDirs();
    expect(discovered.size).toBeGreaterThan(20); // discovery itself must not silently break

    const unclassified = [...discovered]
      .filter(([name]) => !CATEGORIES[name])
      .map(([name, source]) => `${name} (from ${source})`);

    expect(unclassified, 'Add a CATEGORIES entry in server/services/dataManager.js for each of these').toEqual([]);
  });

  it('gives every category a label, a description, and explicit permission flags', () => {
    for (const [key, meta] of Object.entries(CATEGORIES)) {
      expect(meta.label, `${key} label`).toBeTruthy();
      expect(meta.description, `${key} description`).toBeTruthy();
      expect(meta.description, `${key} description`).not.toBe(UNKNOWN_CATEGORY_DESCRIPTION);
      expect(typeof meta.archivable, `${key} archivable`).toBe('boolean');
      expect(typeof meta.deletable, `${key} deletable`).toBe('boolean');
    }
  });
});

describe('dataManager purge scope (#3327)', () => {
  it('gives every purgeable category an explicit purgeScope and none to the protected ones', () => {
    for (const [key, meta] of Object.entries(CATEGORIES)) {
      if (meta.deletable) {
        expect(['items', 'category'], `${key} purgeScope`).toContain(meta.purgeScope);
      } else {
        expect(meta.purgeScope, `${key} is protected and must not carry a purgeScope`).toBeUndefined();
      }
    }
  });

  it('keeps the irreplaceable media categories out of the all-or-nothing purge', () => {
    // LoRA adapters are excluded from backup snapshots and cost hours of GPU
    // time to retrain — no purge affordance at all.
    expect(CATEGORIES.loras.deletable).toBe(false);
    // Uploads and renders that other records point at: reclaimable one at a time.
    expect(CATEGORIES.images.purgeScope).toBe('items');
    expect(CATEGORIES.videos.purgeScope).toBe('items');
    // Screenshots are task scratch — still wipeable whole.
    expect(CATEGORIES.screenshots.purgeScope).toBe('category');
  });

  // Issue #10912: these directories hold user-authored state with no other copy
  // (drafts and the send-attempt audit, todos and assessment reports). Each entry
  // records why it is protected; the table is the audit trail, not an inference.
  const USER_STATE_CATEGORIES = {
    messages: 'drafts, the delivery_unknown send audit, triage rules and account config',
    review: 'Review Hub todos, pending CoS action requests and security assessment reports',
    'image-render-usage': 'historical image accounting and deduplication identities',
    'image-render-usage-backfills': 'import plans, preservation fingerprints and recovery journals',
  };
  it.each(Object.entries(USER_STATE_CATEGORIES))(
    'never offers a purge for %s, which holds %s',
    (key) => {
      expect(CATEGORIES[key]).toMatchObject({ archivable: true, deletable: false });
      expect(CATEGORIES[key].purgeScope).toBeUndefined();
    }
  );

  it('protects durable backup admission authority from archive and both purge forms', async () => {
    expect(CATEGORIES['backup-admission']).toMatchObject({ archivable: false, deletable: false });
    await expect(archiveCategory('backup-admission')).rejects.toThrow(/is not archivable/);
    await expect(purgeCategory('backup-admission')).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('backup-admission', { subPath: 'cut' })).rejects.toThrow(/is not purgeable/);
  });

  // Audit cluster 04, decision 2: `data/beeper/` holds one subdirectory
  // (`attachments/`), so neither purge form was ever reachable — a category
  // wipe needs `purgeScope: 'category'` and a per-item purge refuses a
  // directory entry outright. The visible purge control was dead either way;
  // this category now reports size only and reclamation happens through the
  // attachment budget sweep and the per-conversation purge instead.
  it('reports the Beeper attachment mirror as size-only — no purge control that was never reachable', () => {
    expect(CATEGORIES.beeper.deletable).toBe(false);
    expect(CATEGORIES.beeper.purgeScope).toBeUndefined();
  });

  // These only exercise the refusal paths, which all return before purgeCategory
  // touches the filesystem — no test here may reach `rm` against the real data/.
  it('refuses a category-wide purge for item-scoped and protected categories', async () => {
    await expect(purgeCategory('images')).rejects.toThrow(/only supports per-item purge/);
    await expect(purgeCategory('videos')).rejects.toThrow(/only supports per-item purge/);
    await expect(purgeCategory('loras')).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('loras', { subPath: 'anything' })).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('beeper')).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('beeper', { subPath: 'attachments' })).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('peer-execution')).rejects.toThrow(/is not purgeable/);
    await expect(purgeCategory('peer-execution', { subPath: 'fixture-operation/evidence.json' })).rejects.toThrow(/is not purgeable/);
    for (const category of Object.keys(USER_STATE_CATEGORIES)) {
      for (const options of [undefined, { subPath: 'account' }]) {
        await expect(purgeCategory(category, options)).rejects.toMatchObject({ status: 403, code: 'CATEGORY_NOT_PURGEABLE' });
      }
    }
  });

  it('fails closed when a deletable category has no recognized purgeScope', async () => {
    const original = CATEGORIES.screenshots.purgeScope;
    CATEGORIES.screenshots.purgeScope = 'typo';
    await expect(purgeCategory('screenshots')).rejects.toThrow(/only supports per-item purge/);
    CATEGORIES.screenshots.purgeScope = original;
  });
});