Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.6 KB
Raw Blame
/**
 * Wave B (spec 1.4): content_hash computed as a resumable stream.
 *
 * Protects: `streamingContentHash(page) === contentHash(page)` for every page
 * and every segmentation of its regions on code-point boundaries (astral
 * characters, CRLF, control characters, quotes, backslashes, U+2028, empty
 * truth, empty timeline, tags in frontmatter or hoisted, ephemeral frontmatter
 * keys); resuming from a stored midstate after an append equals hashing from
 * scratch, for each resume rule; the stored state stays under 128 bytes; the
 * vendored SHA-256 equals node:crypto around every block boundary.
 * Fails when: the streamed envelope drifts from JSON.stringify's bytes (key
 * order, escaping, the `timeline || ''` default, the frontmatter/tags
 * normalization), the midstate does not round-trip, or the SHA-256 padding is
 * wrong at a boundary.
 * Why existing coverage misses it: test/content-hash-parity-3694.test.ts pins
 * contentHash's formula, not a second implementation of it.
 * Seams: none; pure functions.
 */
import { describe, expect, test } from 'bun:test';
import { createHash } from 'node:crypto';
import fc from 'fast-check';
import { contentHash } from '../../src/core/utils.ts';
import type { PageInput } from '../../src/core/types.ts';
import { CONTENT_HASH_STATE_MAX_BYTES, ContentHashStream, streamingContentHash } from '../../src/core/page-body/content-hash-stream.ts';
import { ResumableSha256, SHA256_STATE_MAX_BYTES } from '../../src/core/page-body/sha256-resumable.ts';

const NUM_RUNS = 500;
const unit = fc.constantFrom('a', 'Z', ' ', '"', '\\', '\r', '\n', '\r\n', '\t', '\u0000', '\u001f', '\u007f', '\u2028', '\u2029',
  'é', '中', '😀', '𝄞', '/', '<', '{', '}', ':');
const text = fc.string({ unit, maxLength: 60 });
const key = fc.constantFrom('captured_at', 'ingested_at', 'quarantine', 'trust_tier', 'atoms_scan_hash', 'tags', 'source', 'date', 'nested');
const value = fc.oneof(text, fc.integer(), fc.boolean(), fc.constant(null), fc.array(text, { maxLength: 3 }), fc.dictionary(text, text, { maxKeys: 2 }));
const page = fc.record({
  title: text,
  type: fc.constantFrom('note', 'person', 'concept'),
  compiled_truth: text,
  timeline: fc.option(text, { nil: undefined }),
  frontmatter: fc.option(fc.dictionary(key, value, { maxKeys: 5 }), { nil: undefined }),
  tags: fc.option(fc.array(text, { maxLength: 4 }), { nil: undefined }),
}) as fc.Arbitrary<PageInput>;

/** Split `s` at the given fractions, always on code-point boundaries. */
function segments(s: string, cuts: number[]): string[] {
  const points = Array.from(s);
  const at = [...new Set(cuts.map(c => Math.floor(c * (points.length + 1))))].sort((x, y) => x - y);
  const out: string[] = [];
  let prev = 0;
  for (const c of at) { out.push(points.slice(prev, c).join('')); prev = c; }
  out.push(points.slice(prev).join(''));
  return out;
}

const cuts = fc.array(fc.double({ min: 0, max: 1, noNaN: true, maxExcluded: true }), { maxLength: 6 });

describe('streamed content_hash', () => {
  test('equals contentHash for every page and segmentation', () => {
    fc.assert(fc.property(page, cuts, cuts, (p, truthCuts, timelineCuts) => {
      const stream = ContentHashStream.begin(p);
      for (const seg of segments(p.compiled_truth, truthCuts)) stream.pushTruth(seg);
      stream.beginTimeline();
      for (const seg of segments(p.timeline || '', timelineCuts)) stream.pushTimeline(seg);
      expect(stream.finish(p)).toBe(contentHash(p));
      expect(streamingContentHash(p)).toBe(contentHash(p));
    }), { numRuns: NUM_RUNS });
  });

  test('named shapes: empty regions, hoisted vs frontmatter tags, ephemeral keys', () => {
    const base: PageInput = { title: 'T', type: 'note', compiled_truth: '' };
    const shapes: PageInput[] = [
      base,
      { ...base, timeline: '' },
      { ...base, compiled_truth: '', timeline: 'only timeline' },
      { ...base, frontmatter: { tags: ['b', 'a'] } },
      { ...base, tags: ['b', 'a'] },
      { ...base, frontmatter: { captured_at: '2026-01-01', ingested_at: 'x', trust_tier: 'owner', atoms_scan_hash: 'abc', keep: 1 } },
      { ...base, compiled_truth: 'line\r\n"quoted" \\ back\u2028sep\u0000nul😀' },
      { ...base, frontmatter: { date: new Date(Date.UTC(2024, 5, 1)) } },
    ];
    for (const p of shapes) expect(streamingContentHash(p)).toBe(contentHash(p));
    expect(streamingContentHash(shapes[3]!)).toBe(streamingContentHash(shapes[4]!));
  });

  test('resume after append equals hashing from scratch, for each resume rule', () => {
    fc.assert(fc.property(page, text, text, (p, truthAppend, timelineAppend) => {
      const truth = p.compiled_truth;
      const timeline = p.timeline || '';
      const truthRow = ContentHashStream.begin(p).pushTruth(truth).exportState();
      const lastRow = ContentHashStream.fromState(truthRow).pushTimeline(timeline).exportState();

      // Append to the last region (the timeline): resume from the last row.
      const appendedTimeline = { ...p, timeline: timeline + timelineAppend };
      expect(ContentHashStream.fromState(lastRow).pushTimeline(timelineAppend).finish(p)).toBe(contentHash(appendedTimeline));

      // Append to the truth while the timeline is non-empty: resume from the last truth row, re-feed the timeline.
      const appendedTruth = { ...p, compiled_truth: truth + truthAppend };
      expect(ContentHashStream.fromState(truthRow).pushTruth(truthAppend).pushTimeline(timeline).finish(p)).toBe(contentHash(appendedTruth));

      // Frontmatter or tags change: resume from the last row with the new values.
      const retagged = { ...p, tags: ['zeta', 'alpha'], frontmatter: { ...(p.frontmatter ?? {}), added: truthAppend } };
      expect(ContentHashStream.fromState(lastRow).finish(retagged)).toBe(contentHash(retagged));

      // An empty timeline: the truth row is the last row.
      const noTimeline = { ...p, timeline: '', compiled_truth: truth + truthAppend };
      expect(ContentHashStream.fromState(truthRow).pushTruth(truthAppend).finish(p)).toBe(contentHash(noTimeline));
    }), { numRuns: NUM_RUNS });
  });

  test('finish leaves the stream usable and state stays under 128 bytes', () => {
    fc.assert(fc.property(page, (p) => {
      const stream = ContentHashStream.begin(p).pushTruth(p.compiled_truth);
      const before = stream.finish(p);
      expect(stream.finish(p)).toBe(before);
      expect(stream.exportState().length).toBeLessThanOrEqual(CONTENT_HASH_STATE_MAX_BYTES);
      expect(stream.pushTimeline(p.timeline || '').exportState().length).toBeLessThanOrEqual(128);
    }), { numRuns: NUM_RUNS });
    expect(() => ContentHashStream.begin({ title: 't', type: 'note' }).pushTimeline('x').pushTruth('y')).toThrow();
    expect(() => ContentHashStream.fromState(new Uint8Array([7, ...new Uint8Array(40)]))).toThrow();
  });
});

describe('ResumableSha256', () => {
  const native = (b: Uint8Array) => createHash('sha256').update(b).digest('hex');

  test('digest equals node:crypto around every block boundary, resumed at every split', () => {
    const lengths = new Set<number>();
    for (const k of [0, 1, 2, 3, 4, 8, 16]) for (const d of [-1, 0, 1]) { lengths.add(Math.max(0, 64 * k + d)); lengths.add(Math.max(0, 64 * k + 55 + d)); }
    for (let n = 0; n <= 130; n++) lengths.add(n);
    for (const n of lengths) {
      const data = new Uint8Array(n).map((_, i) => (i * 131 + n) & 0xff);
      const expected = native(data);
      expect(new ResumableSha256().update(data).digest()).toBe(expected);
      for (const split of [0, 1, n >> 1, Math.max(0, n - 1), n]) {
        const state = new ResumableSha256().update(data.subarray(0, split)).exportState();
        expect(state.length).toBeLessThanOrEqual(SHA256_STATE_MAX_BYTES);
        expect(ResumableSha256.fromState(state).update(data.subarray(split)).digest()).toBe(expected);
      }
    }
  });

  test('random lengths and chunkings', () => {
    fc.assert(fc.property(fc.uint8Array({ maxLength: 2000 }), fc.array(fc.nat(300), { maxLength: 8 }), (data, sizes) => {
      let hasher = new ResumableSha256();
      let at = 0;
      for (const size of sizes) {
        hasher = ResumableSha256.fromState(hasher.update(data.subarray(at, at + size)).exportState());
        at = Math.min(data.length, at + size);
      }
      hasher.update(data.subarray(at));
      expect(hasher.digest()).toBe(native(data));
    }), { numRuns: NUM_RUNS });
  });

  test('a state whose length disagrees with its byte count is refused', () => {
    const state = new ResumableSha256().update(new Uint8Array(70)).exportState();
    expect(() => ResumableSha256.fromState(state.subarray(0, state.length - 1))).toThrow();
    expect(() => ResumableSha256.fromState(new Uint8Array(10))).toThrow();
  });
});