Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.1 KB
Raw Blame
import { describe, it, expect } from 'vitest';
import { clampToCharLimit, countWords, escapeRegExp, isNonBlankStr, isStr, trimTo, trimToClause } from './textUtils.js';
import { collectClientSources, collectServerSources, readClientSource, readServerSource } from './testHelper.js';

describe('countWords', () => {
  it('counts whitespace-separated tokens', () => {
    expect(countWords('one two three')).toBe(3);
    expect(countWords('one')).toBe(1);
  });

  it('collapses runs of mixed whitespace', () => {
    expect(countWords('  hello   world  ')).toBe(2);
    expect(countWords('one two\nthree\tfour')).toBe(4);
  });

  it('treats hyphenates and contractions as single words', () => {
    expect(countWords("don't stop now")).toBe(3);
    expect(countWords('hyphen-ated counts once')).toBe(3);
  });

  it('returns 0 for empty, whitespace-only, and non-string input', () => {
    expect(countWords('')).toBe(0);
    expect(countWords('   ')).toBe(0);
    expect(countWords(null)).toBe(0);
    expect(countWords(undefined)).toBe(0);
    expect(countWords(42)).toBe(0);
    expect(countWords({})).toBe(0);
  });
});

describe('trimTo', () => {
  it('trims and caps strings without coercing other values', () => {
    expect(trimTo('  bounded text  ', 7)).toBe('bounded');
    expect(trimTo(' short ', 20)).toBe('short');
    expect(trimTo(null, 20)).toBe('');
    expect(trimTo(42, 20)).toBe('');
  });
});

describe('isStr / isNonBlankStr', () => {
  it('isStr is the bare type check — an empty string is a string', () => {
    expect(isStr('')).toBe(true);
    expect(isStr('   ')).toBe(true);
    expect(isStr(new String('boxed'))).toBe(false);
    expect(isStr(null)).toBe(false);
    expect(isStr(42)).toBe(false);
  });

  it('isNonBlankStr rejects blank and non-string values without trimming what it accepts', () => {
    expect(isNonBlankStr('x')).toBe(true);
    expect(isNonBlankStr('  padded  ')).toBe(true);
    expect(isNonBlankStr('')).toBe(false);
    expect(isNonBlankStr('   ')).toBe(false);
    expect(isNonBlankStr('\n\t')).toBe(false);
    expect(isNonBlankStr(null)).toBe(false);
    expect(isNonBlankStr(0)).toBe(false);
  });
});

describe('clampToCharLimit', () => {
  it('passes text through when there is no cap or it already fits', () => {
    expect(clampToCharLimit('a neon alley', 800)).toEqual({ text: 'a neon alley', truncated: false });
    expect(clampToCharLimit('a neon alley', 0)).toEqual({ text: 'a neon alley', truncated: false });
    expect(clampToCharLimit(null, 10)).toEqual({ text: '', truncated: false });
  });

  it('cuts on the last sentence end when one sits deep in the allowance', () => {
    const { text, truncated } = clampToCharLimit('Wide shot of a rain-slick alley. Then the camera pushes in hard.', 45);
    expect(truncated).toBe(true);
    expect(text).toBe('Wide shot of a rain-slick alley.');
  });

  it('falls back to a word boundary rather than cutting mid-word', () => {
    // No sentence end deep enough in the allowance — a mid-word cut would
    // change what the tail asks the renderer for.
    expect(clampToCharLimit('alpha bravo charlie delta', 14)).toEqual({ text: 'alpha bravo', truncated: true });
  });

  it('does not cut mid-abbreviation — the bare lastIndexOf(".") cap ended on "Dr."', () => {
    // Regression for #6455: the old cap took the last "." in the window whenever it
    // sat past 60% of the budget, so an abbreviation deep in the string became the
    // cut point and the value ended on a hanging "Dr." — which a renderer reads as
    // a truncated name. trimToClause's COMMON_ABBREVIATION_RE guard rejects that
    // terminator and falls through to the whole-word boundary instead.
    expect(clampToCharLimit('We met the visiting consultant Dr. Vey at the clinic yesterday', 40))
      .toEqual({ text: 'We met the visiting consultant Dr. Vey', truncated: true });
  });

  it('does not cut mid-decimal', () => {
    // Same defect, decimal flavour: the old cap cut at the "." inside "3.5", handing
    // the renderer "…holds at 3." — a different number, not a shorter sentence.
    expect(clampToCharLimit('The reactor holds at 3.5 megawatts under load and the crew waits', 30))
      .toEqual({ text: 'The reactor holds at 3.5', truncated: true });
  });
});

describe('escapeRegExp', () => {
  it('escapes every RegExp metacharacter and nothing else', () => {
    expect(escapeRegExp('a.c')).toBe('a\\.c');
    expect(escapeRegExp('C++ (faction) [v1.0]')).toBe('C\\+\\+ \\(faction\\) \\[v1\\.0\\]');
    expect(escapeRegExp('a|b {x} ^$ *? \\')).toBe('a\\|b \\{x\\} \\^\\$ \\*\\? \\\\');
    expect(escapeRegExp('plain words 42')).toBe('plain words 42');
  });

  it('makes a metacharacter-laden token match only itself', () => {
    const token = 'C++ (faction) [v1.0]';
    expect(new RegExp(`^${escapeRegExp(token)}$`).test(token)).toBe(true);
    expect(new RegExp(`^${escapeRegExp('a.c')}$`).test('abc')).toBe(false);
  });

  // Seven of the migrated copies were `s.replace(...)`, which threw a TypeError on
  // a non-string; the shared helper coerces. Every migrated call site already
  // filters to strings upstream, so the change is unreachable today — but it is
  // the one semantic the migration altered, so it is pinned here rather than left
  // to be "fixed" back into a throw by someone reading only this module. Coercion
  // is deliberate: these callers splice user-supplied tokens (LoRA triggers,
  // character aliases, catalog labels) into `new RegExp(...)`, where a throw
  // surfaces as an opaque 500. The trade is that a stray non-string becomes a
  // literal 'null'/'42' pattern rather than an error, which is why callers must
  // keep filtering rather than lean on the coercion.
  it('coerces non-string input instead of throwing', () => {
    expect(escapeRegExp(null)).toBe('null');
    expect(escapeRegExp(undefined)).toBe('undefined');
    expect(escapeRegExp(42)).toBe('42');
    expect(escapeRegExp(1.5)).toBe('1\\.5');
  });
});

// The extraction of `escapeRegExp` into this module landed once and then rotted:
// twenty-odd server modules kept (or re-added) a private copy — some named
// `escapeRe`/`escapeRegex`, most just inlined at the call site — because nothing
// failed when they did, and the copies drifted (`s.replace` threw on a non-string
// where `String(s).replace` coerced). This guard is what makes the extraction
// stick: a fresh copy fails the suite instead of shipping.
//
// It keys on the escape IDIOM, not on the identifier, because every copy this repo
// ever grew was a byte-identical paste under a different name (or no name at all).
// The `'\\$&'` replacement is the spelling-independent half — it is what makes a
// `.replace` an escape rather than an edit, and after this migration it appears in
// exactly two source files repo-wide: `server/lib/textUtils.js` and its client
// mirror. Consequence: even quoting the idiom in a comment trips the guard —
// describe the rule in prose, or put the example in a textUtils.js, which is the
// file that owns it on each side.
//
// Scope: `collectServerSources` walks all of `server/` but skips `*.test.js`, so
// that half covers product code — a copy in a server test can't change what the
// server does, and this very file spells the idiom. `collectClientSources` walks
// `client/src/` and does NOT skip tests (nor `.jsx`), because nothing over there
// needs the exemption and a client test was one of the copies #5790 migrated.
const ESCAPE_IDIOMS = [
  // The self-referential replacement every copy of the escape uses.
  /'\\\\\$&'/,
  // The escape's character class, for a copy that assembles it differently.
  /\[\.\*\+\?\^\$\{\}\(\)\|\[\\\]\\\\\]/,
  // A copy that reorders the class but keeps the conventional name.
  /(?:^|[^\w$.])(?:const|let|var|function)\s+escapeRegExp\b/,
];

// How many times the most-spelled idiom in `idioms` appears in `source`.
const idiomCount = (idioms) => (source) => idioms
  .map((idiom) => source.match(new RegExp(idiom.source, 'g'))?.length ?? 0)
  .reduce((most, count) => Math.max(most, count), 0);

const escapeIdiomCount = idiomCount(ESCAPE_IDIOMS);

describe('no private escapeRegExp', () => {
  it('leaves lib/textUtils.js as the only RegExp-escape implementation under server/', () => {
    const offenders = collectServerSources()
      .filter((rel) => rel !== 'lib/textUtils.js')
      .filter((rel) => escapeIdiomCount(readServerSource(rel)) > 0);
    expect(
      offenders,
      `these re-inline the RegExp escape — import escapeRegExp from lib/textUtils.js instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  // The client half of the same guard, and it now allows NO exemption: since #6364
  // `client/src/lib/textUtils.js` re-exports this module rather than copying it, so
  // no file under `client/src/` spells the escape at all. Before that, every new
  // client caller copied the nearest one — five product modules and a test had done
  // so by #5790. `collectClientSources` counts `.jsx` and client TESTS too; see its
  // docstring.
  it('leaves no RegExp-escape implementation anywhere under client/src/', () => {
    const offenders = collectClientSources()
      .filter((rel) => escapeIdiomCount(readClientSource(rel)) > 0);
    expect(
      offenders,
      `these re-inline the RegExp escape — import escapeRegExp from lib/textUtils.js instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  // Both walks feed the same detector, so pin that it actually fires — an empty
  // offender list is equally what a walk returning nothing produces.
  it('detects a re-inlined copy under any of its spellings', () => {
    expect(escapeIdiomCount(readServerSource('lib/textUtils.js'))).toBeGreaterThan(0);
    // The client walk has no exempt file left to prove itself against, so pin it
    // on a synthetic copy instead — an empty offender list must mean "nobody
    // spells it", not "the walk read nothing".
    expect(collectClientSources().length).toBeGreaterThan(100);
    expect(escapeIdiomCount("const escapeRegExp = (s) => s;")).toBeGreaterThan(0);
    expect(escapeIdiomCount('const x = 1;')).toBe(0);
  });
});

// `countWords` had the same history as the escape: extracted here as "the
// canonical whitespace-token count", then re-spelled in five product modules and
// the client's formatters — whose copy justified itself with "the client cannot
// import from server/", untrue since #6364. Same remedy: key on the counting
// IDIOMS rather than the identifier, since the copies were pastes under three
// names and four under none. The whitespace-token match is how every named copy
// counted; split-and-drop-empties is how the inline ones did. A `matchAll` or a
// `re.exec` loop over the same class is NOT matched — those are the tokenizers
// that need each token's position (rapid reader, tab notation, clichés), and a
// tokenizer is not a count.
//
// A bare split-on-whitespace `.length` (no filter) is deliberately absent: it is
// the five-field cron check's spelling far more often than a word count, and the
// one word-count use (the brain digest's word cap) truncates with the same split,
// so it is at least internally consistent.
const WORD_COUNT_IDIOMS = [
  // The whitespace-token match used as a count.
  /\.match\(\/\\S\+\/g\)/,
  // The split-and-drop-empties count.
  /\.split\(\/\\s\+\/\)\.filter\(Boolean\)\.length/,
  // A helper re-declared under either conventional name.
  /(?:^|[^\w$.])(?:(?:const|let|var)\s+(?:countWords|wordCount)\s*=\s*(?:async\s*)?\(|function\s+(?:countWords|wordCount)\s*\()/,
];

const wordCountIdiomCount = idiomCount(WORD_COUNT_IDIOMS);

describe('no private countWords', () => {
  it('leaves lib/textUtils.js as the only whitespace word count under server/', () => {
    const offenders = collectServerSources()
      .filter((rel) => rel !== 'lib/textUtils.js')
      .filter((rel) => wordCountIdiomCount(readServerSource(rel)) > 0);
    expect(
      offenders,
      `these re-spell the word count — import countWords from lib/textUtils.js instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  // The client walk allows no exemption either: `client/src/lib/textUtils.js`
  // re-exports this module's count, and the formatters copy is gone.
  it('leaves no whitespace word count anywhere under client/src/', () => {
    const offenders = collectClientSources()
      .filter((rel) => wordCountIdiomCount(readClientSource(rel)) > 0);
    expect(
      offenders,
      `these re-spell the word count — import countWords from lib/textUtils.js instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  it('detects a re-spelled count under any of its idioms, and not a tokenizer', () => {
    expect(wordCountIdiomCount(readServerSource('lib/textUtils.js'))).toBeGreaterThan(0);
    expect(wordCountIdiomCount('const wordCount = (t) => t.split(/\\s+/).filter(Boolean).length;')).toBeGreaterThan(0);
    expect(wordCountIdiomCount('function countWords(text) { return 0; }')).toBeGreaterThan(0);
    expect(wordCountIdiomCount('for (const m of text.matchAll(/\\S+/g)) {}')).toBe(0);
    expect(wordCountIdiomCount('const isCron = value.trim().split(/\\s+/).length === 5;')).toBe(0);
    expect(wordCountIdiomCount('const wordCount = useMemo(() => countWords(body), [body]);')).toBe(0);
  });
});

// String predicates and `trimTo`-shaped bounders had the same history as the
// RegExp escape above. The only exported predicate lived in `storyBible.js`
// behind `crypto` / `fileUtils`, so 73 sanitizers grew a private copy: 17
// spelling the bare type check, 28 a non-empty / non-blank variant — under
// `isStr`, `hasText`, `isText`, `isUsable`, `isNonEmptyStr`, …, with half of
// FableLoom's `isStr` accepting `''` and the other half rejecting it — and 26
// re-rolling `trimTo` as `clampStr` / `trimString` / `str` / `text` / `trim`.
// #6837 moved every one onto `isStr` / `isNonBlankStr` / `trimTo` here; this
// guard is what keeps them here.
//
// It keys on the declared BODY, never the identifier, and every idiom is
// anchored on the statement's terminating `;` (or a function's closing brace).
// That anchoring is what leaves a DOMAIN predicate alone: one whose first line
// merely ends in `typeof x === 'string'` and continues with `&& RE.test(x)`,
// or one that adds any clause at all, never matches — only the pure generic
// bodies do. A one-line alias (`const hasText = isNonBlankStr;`) has no
// `typeof` and passes; a normalizer without `.slice(` is not a bounder.
const GENERIC_STRING_TAIL = String.raw`(?:\s*&&\s*(?:!!\2|\2(?:\.trim\(\))?\.length\s*>\s*0|\2\.trim\(\)\s*!==\s*''))?`;
const STRING_HELPER_IDIOMS = [
  // const isX = (v) => typeof v === 'string'[ && <generic non-empty / non-blank tail>];
  new RegExp(String.raw`^[ \t]*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*\(?\s*(\w+)\s*\)?\s*=>\s*typeof\s+\2\s*===\s*'string'${GENERIC_STRING_TAIL}\s*;[ \t]*$`, 'gm'),
  // function isX(v) { return typeof v === 'string'[ && <tail>]; }
  new RegExp(String.raw`^[ \t]*(?:export\s+)?function\s+(\w+)\s*\(\s*(\w+)\s*\)\s*\{\s*return\s+typeof\s+\2\s*===\s*'string'${GENERIC_STRING_TAIL}\s*;?\s*\}`, 'gm'),
  // const trimX = (v, max[ = DEFAULT]) => [(]typeof v === 'string' ? v.trim().slice(0, max) : ''[)];   (isStr(v) ? … as well)
  new RegExp(String.raw`^[ \t]*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*\(\s*(\w+)\s*,\s*(\w+)(?:\s*=\s*[^,)]+)?\s*\)\s*=>\s*\(?\s*(?:typeof\s+\2\s*===\s*'string'|isStr\(\s*\2\s*\))\s*\?\s*\2\.trim\(\)\.slice\(\s*0\s*,\s*\3\s*\)\s*:\s*''\s*\)?\s*;[ \t]*$`, 'gm'),
];

// Total declarations across all three idioms (a file carrying a predicate AND a
// bounder counts both) — unlike `idiomCount` above, which takes the max of
// alternative spellings of ONE idiom and drops the `m` flag these rely on.
const stringHelperDeclarationCount = (source) => STRING_HELPER_IDIOMS
  .reduce((sum, idiom) => sum + (source.match(new RegExp(idiom.source, idiom.flags))?.length ?? 0), 0);

describe('no private string predicate or bounder', () => {
  it('leaves lib/textUtils.js as the only string predicate or bounder under server/', () => {
    const offenders = collectServerSources()
      .filter((rel) => rel !== 'lib/textUtils.js')
      .filter((rel) => stringHelperDeclarationCount(readServerSource(rel)) > 0);
    expect(
      offenders,
      `these re-declare a string predicate or a trimTo-shaped bounder — import isStr / isNonBlankStr / trimTo from lib/textUtils.js instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  // The client mirror re-exports the predicates, so nothing over there needs one.
  it('leaves no string predicate or bounder anywhere under client/src/', () => {
    const offenders = collectClientSources()
      .filter((rel) => stringHelperDeclarationCount(readClientSource(rel)) > 0);
    expect(
      offenders,
      `these re-declare a string predicate or a trimTo-shaped bounder — import isStr / isNonBlankStr / trimTo from lib/textUtils instead: ${offenders.join(', ')}`
    ).toEqual([]);
  });

  it('detects a re-rolled copy under any of its spellings, and not a domain predicate', () => {
    // Positive control on the owner: isStr, isNonBlankStr, and the two-line trimTo.
    expect(stringHelperDeclarationCount(readServerSource('lib/textUtils.js'))).toBe(3);
    expect(collectClientSources().length).toBeGreaterThan(100);

    const fires = [
      "const isStr = (v) => typeof v === 'string';",
      "export const isString = value => typeof value === 'string';",
      "const isNonEmptyString = (value) => typeof value === 'string' && !!value;",
      "const isNonEmptyStr = (v) => typeof v === 'string' && v.length > 0;",
      "  const hasText = (value) => typeof value === 'string' && value.trim().length > 0;",
      "const filled = (v) => typeof v === 'string' && v.trim() !== '';",
      "export function isNonEmptyString(value) {\n  return typeof value === 'string' && value.trim().length > 0;\n}",
      "const clampStr = (v, max) => (typeof v === 'string' ? v.trim().slice(0, max) : '');",
      "const trimString = (value, max = MAX_PROMPT_LEN) =>\n  typeof value === 'string' ? value.trim().slice(0, max) : '';",
      "const asBoundedString = (value, max) => (\n  typeof value === 'string' ? value.trim().slice(0, max) : ''\n);",
      "const trimTo = (v, max) => (isStr(v) ? v.trim().slice(0, max) : '');",
    ];
    for (const copy of fires) expect(stringHelperDeclarationCount(copy), copy).toBe(1);

    const passes = [
      "const isHexHash = (v) => typeof v === 'string' && HEX_64.test(v);",
      "const isInstanceId = (id) => typeof id === 'string' && id.length > 0 && id.length <= MAX_ID_LEN;",
      "const validLibraryPath = (path) => typeof path === 'string'\n  && path.startsWith('/') && !path.includes('..');",
      "const hasPrompt = (prompt) => typeof prompt === 'string' && prompt.trim() !== '' && prompt !== '(no prompt)';",
      "const normalizeText = (text) => typeof text === 'string' ? text.trim() : '';",
      "const normalizeTitle = (title) => typeof title === 'string' ? title.trim().slice(0, MAX_TITLE_LENGTH) : '';",
      "const text = (value, max = 6000) => typeof value === 'string'\n  ? value.length > max ? `${value.slice(0, max)}\\n[truncated]` : value : '';",
      'const hasText = isNonBlankStr;',
      "const label = typeof name === 'string' ? name.trim().slice(0, 80) : '';",
    ];
    for (const source of passes) expect(stringHelperDeclarationCount(source), source).toBe(0);
  });
});

describe('trimToClause (boundary-aware prose cap)', () => {
  it('returns text untouched when it fits', () => {
    expect(trimToClause('A short logline.', 500)).toBe('A short logline.');
    expect(trimToClause('  trimmed  ', 500)).toBe('trimmed');
  });

  it('is empty for non-strings (matches trimTo)', () => {
    expect(trimToClause(null, 50)).toBe('');
    expect(trimToClause(undefined, 50)).toBe('');
    expect(trimToClause(42, 50)).toBe('');
  });

  it('clips at a sentence boundary when it preserves a meaningful share of the budget', () => {
    const text = 'First full sentence about the arc here. And then a second clause runs on well past the budget.';
    const out = trimToClause(text, 50);
    expect(out).toBe('First full sentence about the arc here.');
    expect(out.length).toBeLessThanOrEqual(50);
  });

  it('prefers the only complete sentence at 53% over a near-cap fragment', () => {
    const complete = `${'word '.repeat(52)}done.`; // 265 chars
    const tail = ` ${'continuation '.repeat(30)}`;
    const out = trimToClause(complete + tail, 500);
    expect(out).toBe(complete);
    expect(out.endsWith('.')).toBe(true);
  });

  it('does not treat abbreviations or decimals as sentence endings', () => {
    const text = `Dr. Example measured 3.5 units before the actual stop. ${'tail '.repeat(20)}`;
    const out = trimToClause(text, 70);
    expect(out).toBe('Dr. Example measured 3.5 units before the actual stop.');
  });

  it('keeps closing quotes and brackets attached to the sentence', () => {
    const text = `The operator said, "Stop now." ${'tail '.repeat(20)}`;
    expect(trimToClause(text, 50)).toBe('The operator said, "Stop now."');
  });

  it('backs off to a clause boundary when the field holds no sentence break', () => {
    // The shape a saturated 200-char transition label takes: one clause-chained
    // sentence, no terminator anywhere in budget. Cutting on the last whole word
    // leaves "...with no repayment lien, no" — a dangling half-clause the next
    // verification round reports as an incomplete record.
    const label = 'Proves with the validated block that the takeover cannot reach quorum, then closes the short and escrows the proceeds with no repayment lien, no personal withdrawal key, and no veto';
    const out = trimToClause(label, 160);
    expect(out).toBe('Proves with the validated block that the takeover cannot reach quorum, then closes the short and escrows the proceeds with no repayment lien');
    expect(out.length).toBeLessThanOrEqual(160);
    expect(/[,;:—–]$/.test(out)).toBe(false);
  });

  it('prefers a usable sentence break over a later clause break', () => {
    const text = 'The vote carries after a long night of argument. The crews ratify, the boosters transfer, and the manifest is certified.';
    expect(trimToClause(text, 70)).toBe('The vote carries after a long night of argument.');
  });

  it('ignores a clause break so early that honoring it would gut the field', () => {
    // Comma at ~8% of the budget: keeping only "Yes" is worse than the ragged
    // whole-word edge, so the word fallback still wins.
    const text = `Yes, ${'word '.repeat(40)}`;
    const out = trimToClause(text, 60);
    expect(out.startsWith('Yes, word')).toBe(true);
    expect(out.length).toBeGreaterThan(50);
  });

  it('never clips mid-word — falls back to a whole-word boundary on a run-on', () => {
    // No sentence terminator within budget → must still end on a complete word,
    // not "...tracing the brand an".
    const runon = 'JUNO risks her anonymity to be recognized as author while Caroline Marsh starts tracing the brand and the buzz';
    const out = trimToClause(runon, 60);
    expect(out.length).toBeLessThanOrEqual(60);
    expect(out.endsWith(' ')).toBe(false);
    // The last token is a whole word from the source (no partial word).
    const lastWord = out.split(' ').pop();
    expect(runon.split(' ')).toContain(lastWord);
  });

  it('never returns more than max characters', () => {
    const long = 'word '.repeat(400); // 2000 chars, no sentence breaks
    expect(trimToClause(long, 100).length).toBeLessThanOrEqual(100);
  });
});