Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

12.9 KB
Raw Blame
import { describe, it, expect } from 'vitest';

import { readFileSync } from 'node:fs';
import { join } from 'node:path';

import {
  MANIFEST_RELATIVE_PATH,
  REGENERATE_COMMAND,
  REPO_ROOT,
  buildStageCallSites,
  createStageScanDiagnostics,
  generateStageCallSites,
  readShippedStageKeys,
  readStageCallSitesManifest,
  serializeManifest,
} from './generate-prompt-stage-call-sites.js';
import { POSITION_INVARIANCE_FAILURE, shiftSourceText } from './lib/positionInvariance.js';

// Fixture sources exercise the scanner's contract without depending on the
// real tree, so a legitimate refactor of `server/` can't turn these into
// false failures.
const src = (path, source) => ({ path, source });

describe('prompt stage call-site scanner', () => {
  const shippedStageKeys = ['demo-alpha', 'demo-beta', 'demo-panel-one', 'demo-panel-two', 'demo-unused'];

  it('catches each direct call shape', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [
        src('server/services/a.js', "await getStage('demo-alpha');"),
        src('server/services/b.js', 'await buildPrompt("demo-beta", {});'),
        src('server/services/c.js', "await runStage('demo-alpha', {});"),
        src('server/services/d.js', "await runStagedLLM('demo-beta', {});"),
        src('server/services/e.js', "await runStageScopedInlineLLM('demo-alpha', body);"),
      ],
    });

    expect(index['demo-alpha']).toEqual(['server/services/a.js', 'server/services/c.js', 'server/services/e.js']);
    expect(index['demo-beta']).toEqual(['server/services/b.js', 'server/services/d.js']);
  });

  it('ignores a stage key that only appears in a comment', () => {
    // A JSDoc block naming `cd-plan` is prose, not a call site — listing that
    // file under "Referenced in" in the delete dialog would be a lie.
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [
        src('server/lib/note.js', "/**\n * The planner ('demo-alpha') writes this.\n */\nexport const x = 1;"),
        src('server/lib/line.js', "// falls back to 'demo-alpha' when unset\nexport const y = 2;"),
        src('server/lib/real.js', "getStage('demo-alpha');"),
      ],
    });

    expect(index['demo-alpha']).toEqual(['server/lib/real.js']);
  });

  it('does not mistake a URL inside a literal for the start of a comment', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/a.js', "const url = 'https://example.com/x';\ngetStage('demo-alpha');")],
    });

    expect(index['demo-alpha']).toEqual(['server/services/a.js']);
  });

  it('catches keys reached indirectly through a lookup table or module constant', () => {
    // The shapes a call-shape-only regex would miss: textStages.js's
    // `{ idea: 'pipeline-idea-expansion' }` map and liveDirector.js's
    // `const STAGE = 'writers-room-continue'`.
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [
        src('server/services/map.js', "const STAGES = { first: 'demo-alpha' };\nrunStage(STAGES[kind]);"),
        src('server/services/const.js', "const STAGE = 'demo-beta';\nawait buildPrompt(STAGE, {});"),
      ],
    });

    expect(index['demo-alpha']).toEqual(['server/services/map.js']);
    expect(index['demo-beta']).toEqual(['server/services/const.js']);
  });

  it('catches stages reached only through an interpolated prefix', () => {
    // `pipeline-panel-${personaId}` in readerPanel.js — no literal key exists
    // anywhere, so a literal-only scan would leave these deletable.
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/panel.js', 'const stage = (id) => `demo-panel-${id}`;')],
    });

    expect(index['demo-panel-one']).toEqual(['server/services/panel.js']);
    expect(index['demo-panel-two']).toEqual(['server/services/panel.js']);
    // The prefix itself is not a key, and unrelated stages stay untouched.
    expect(index['demo-panel-']).toBeUndefined();
    expect(index['demo-alpha']).toBeUndefined();
  });

  it('finds an interpolated prefix that is not at the start of the template', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/panel.js', 'const stage = `${scope}:demo-panel-${id}`;')],
    });

    expect(index['demo-panel-one']).toEqual(['server/services/panel.js']);
  });

  it('ignores an interpolated prefix inside a comment', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/panel.js', '// resolves `demo-panel-${id}` at runtime\nexport const x = 1;')],
    });

    expect(index['demo-panel-one']).toBeUndefined();
  });

  // #6624. The regex tokenizer this scanner replaced walked comments and
  // quoted spans in one alternation, so a backtick it could not attribute
  // opened a phantom template that ran to the NEXT backtick — here, one inside
  // a `//` comment two lines later — swallowing every call site in between.
  // The failure was stable rather than flaky, so the drift test reported it as
  // "the manifest is stale" and its advice ("regenerate and commit") would
  // have ACCEPTED a manifest missing a shipped stage's only reference.
  it('finds a call site after a backtick the old tokenizer could not attribute', () => {
    const desyncing = [
      'const stripTicks = (s) => s.replace(/`/g, "");',
      "export const build = (ctx) => buildPrompt('demo-alpha', ctx);",
      '// The helper above removes every ` from the rendered prompt.',
    ].join('\n');

    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/desync.js', desyncing)],
    });

    expect(index['demo-alpha']).toEqual(['server/services/desync.js']);
  });

  it('throws naming the file when a source will not parse', () => {
    expect(() =>
      buildStageCallSites({
        shippedStageKeys,
        sources: [src('server/services/broken.js', 'export const = ;')],
      }),
    ).toThrow(/server\/services\/broken\.js/);
  });

  it('does not match a longer key that merely starts with a shorter one', () => {
    const index = buildStageCallSites({
      shippedStageKeys: ['demo-alpha', 'demo-alpha-extended'],
      sources: [src('server/services/a.js', "getStage('demo-alpha-extended');")],
    });

    expect(index['demo-alpha']).toBeUndefined();
    expect(index['demo-alpha-extended']).toEqual(['server/services/a.js']);
  });

  it('records a call site for a key the shipped stage config never listed', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/a.js', "getStage('demo-unshipped');")],
    });

    expect(index['demo-unshipped']).toEqual(['server/services/a.js']);
  });

  it('resolves earlier literals and template prefixes against keys discovered in a later file', () => {
    const sources = [
      // biome-ignore lint/suspicious/noTemplateCurlyInString: Parse this interpolation as fixture source, not as the test's own template.
      src('server/services/a.js', "const stages = ['demo-unshipped', `demo-template-one`]; const pick = (id) => `demo-future-${id}`;"),
      src('server/services/z.js', "getStage('demo-unshipped'); buildPrompt('demo-template-one'); buildPrompt('demo-future-one');"),
    ];
    const build = (scanned) => buildStageCallSites({ shippedStageKeys: [], sources: scanned });

    expect(build(sources)).toEqual({
      'demo-future-one': ['server/services/a.js', 'server/services/z.js'],
      'demo-template-one': ['server/services/a.js', 'server/services/z.js'],
      'demo-unshipped': ['server/services/a.js', 'server/services/z.js'],
    });
    expect(build([...sources].reverse())).toEqual(build(sources));
  });

  it('names an over-budget synchronous phase with redacted timing, inventory, and resource evidence', () => {
    let clock = 0;
    const reports = [];
    const diagnostics = createStageScanDiagnostics({ budgetMs: 60000, now: () => clock, onProgress: (report) => reports.push(report) });
    diagnostics.checkpoint({ sources: 3, sourceBytes: 1234, scannedSources: 2, nodes: 42 });

    expect(() => diagnostics.measure('parse', () => { clock = 60001; }))
      .toThrow(/exceeded 60000ms during parse/);
    expect(reports).toEqual([expect.objectContaining({
      phase: 'parse', sources: 3, sourceBytes: 1234, scannedSources: 2, nodes: 42,
      elapsedMs: 60001, phaseMs: { parse: 60001 },
      cpuMs: expect.any(Number), sampledPeakHeapMiB: expect.any(Number), sampledPeakRssMiB: expect.any(Number),
    })]);
  });

  it('omits keys nothing references and dedupes repeated references', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [src('server/services/a.js', "getStage('demo-alpha');\nbuildPrompt('demo-alpha', {});")],
    });

    expect(index['demo-unused']).toBeUndefined();
    expect(index['demo-alpha']).toEqual(['server/services/a.js']);
  });

  it('sorts keys and paths so the manifest is byte-stable', () => {
    const index = buildStageCallSites({
      shippedStageKeys,
      sources: [
        src('server/services/z.js', "getStage('demo-beta');\ngetStage('demo-alpha');"),
        src('server/services/a.js', "getStage('demo-beta');"),
      ],
    });

    expect(Object.keys(index)).toEqual(['demo-alpha', 'demo-beta']);
    expect(index['demo-beta']).toEqual(['server/services/a.js', 'server/services/z.js']);
  });

  // This manifest already keys by stage and lists paths only, so it passes
  // today. The test is here to keep it that way: it states the rule as a
  // property, so a future scanner that starts recording where a call site sits
  // fails here rather than in someone's rebase. Same guard as the route
  // catalog's, which is why both call the one helper.
  it('builds a byte-identical index after every source line shifts', () => {
    const sources = [
      src('server/services/a.js', "await getStage('demo-alpha');"),
      src('server/services/b.js', 'await buildPrompt("demo-beta", {});'),
    ];
    const build = (scanned) => serializeManifest(buildStageCallSites({ shippedStageKeys, sources: scanned }));

    expect(
      build(sources.map(({ path, source }) => src(path, shiftSourceText(source)))),
      POSITION_INVARIANCE_FAILURE,
    ).toBe(build(sources));
  });
});

// The guard the whole generated-manifest posture exists for: adding a
// literal-key call site (or renaming a file that has one) without rerunning
// the generator leaves stages unprotected, and this fails until it's rerun.
describe('prompt stage call-site manifest', () => {
  // Keep the 60s deadline. Synchronous work can delay Vitest's timer, so the
  // diagnostic budget also checks each completed phase and identifies it.
  // Report elapsed/CPU time, inventory, and sampled memory without source text.
  it('matches a fresh scan of the tracked server sources', { timeout: 60000 }, () => {
    const stale = `${MANIFEST_RELATIVE_PATH} is stale — run \`${REGENERATE_COMMAND}\` and commit the result.`;
    const diagnostics = createStageScanDiagnostics({
      budgetMs: 60000,
      onProgress: (report) => console.log(`📜 Prompt-stage scan: ${JSON.stringify(report)}`),
    });
    try {
      const fresh = generateStageCallSites(REPO_ROOT, { diagnostics });
      diagnostics.measure('comparison', () => {
        // Structural first, so a genuine drift reports as a readable key diff…
        expect(fresh, stale).toEqual(readStageCallSitesManifest());
        // …then byte-exact, so a hand-edit that reformats the file is caught too.
        expect(readFileSync(join(REPO_ROOT, MANIFEST_RELATIVE_PATH), 'utf8'), stale)
          .toBe(serializeManifest(fresh));
      });
    } finally {
      diagnostics.finish();
    }
  });

  it('covers the shipped stages that features resolve by name', () => {
    const manifest = readStageCallSitesManifest();
    // A sample from the four call paths #3335 named — a scanner regression
    // that silently stopped matching one of them would still pass the drift
    // test above (both sides regenerate), so pin the expectation here.
    for (const key of [
      'pipeline-series-concept-judge',
      'pipeline-canon-describe-from-prose',
      'catalog-ideas-scenes-concepts',
      'soul-contradiction-detector',
      'pipeline-panel-editor',
    ]) {
      expect(manifest[key], `${key} lost its call-site entry`).toBeTruthy();
      expect(manifest[key].length).toBeGreaterThan(0);
    }

    // Most of the shipped catalog is reachable by name; a large drop means the
    // scan broke rather than that the code genuinely stopped calling stages.
    const shipped = readShippedStageKeys();
    const referenced = shipped.filter((key) => manifest[key]);
    expect(referenced.length).toBeGreaterThan(shipped.length * 0.75);
  });

  it('lists only tracked, repo-relative server paths', () => {
    for (const [key, paths] of Object.entries(readStageCallSitesManifest())) {
      for (const path of paths) {
        expect(path, `${key} -> ${path}`).toMatch(/^server\/[\w./-]+\.js$/);
        expect(path.endsWith('.test.js')).toBe(false);
      }
    }
  });
});