Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
9.0 KB
Raw Blame
/**
 * Scripts written without spaces between words (Chinese, Japanese,
 * Korean, Thai, Lao, Khmer) must be measured per character, or a whole
 * paragraph counts as one "word" and lands in a single chunk that an
 * embedding provider rejects as oversized (issue #186). And whatever the
 * script, no chunk may exceed `maxTokens` - overlap included.
 */

import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { join } from "node:path";

import { test, expect } from "bun:test";

import { chunkMarkdown, type ChunkOptions } from "../../../src/core/search/chunker.ts";
import { CORPUS_OPTION_SETS, spaceSeparatedCorpus } from "../../helpers/chunker-corpus.ts";

const HAN_SENTENCE = "向量索引把每一篇笔记切成若干片段并分别计算嵌入以便语义检索";

/** A single Han paragraph with no whitespace and no line breaks. */
function hanParagraph(chars: number, punctuate: boolean): string {
  let out = "";
  while (out.length < chars) out += HAN_SENTENCE + (punctuate ? "。" : "");
  return out.slice(0, chars);
}

function allWithinCap(text: string, opts: Required<ChunkOptions>): void {
  const r = chunkMarkdown(text, "doc", opts);
  for (const c of r.chunks) {
    expect(c.tokenCount).toBeLessThanOrEqual(opts.maxTokens);
  }
}

/** Every chunk's content, overlap-free, with line breaks removed. */
function bodyText(text: string, opts: ChunkOptions): string {
  return chunkMarkdown(text, "doc", { ...opts, overlapTokens: 0 })
    .chunks.map((c) => c.content)
    .join("")
    .replace(/\n/g, "");
}

test("a 9,004-character Han paragraph is split into chunks within the cap", () => {
  const text = hanParagraph(9004, false);
  const r = chunkMarkdown(text, "zh");
  expect(r.chunks.length).toBeGreaterThan(1);
  for (const c of r.chunks) expect(c.tokenCount).toBeLessThanOrEqual(800);
  // Nothing is lost: the bodies, without overlap, rebuild the paragraph.
  expect(bodyText(text, {})).toBe(text);
});

test("each Han character is measured as its own token", () => {
  const r = chunkMarkdown("索引重建", "zh");
  expect(r.chunks[0]?.tokenCount).toBe(4);
  // Mixed text: the Latin word keeps counting as one word.
  const mixed = chunkMarkdown("使用 sqlite 索引", "zh");
  expect(mixed.chunks[0]?.tokenCount).toBe(1 + 1 + 1 + 2);
});

test("a long Japanese paragraph breaks after sentence punctuation", () => {
  const sentence = "検索索引はノートを小さな断片に分けて埋め込みを計算します。";
  const text = sentence.repeat(120);
  const r = chunkMarkdown(text, "ja", { maxTokens: 200, minTokens: 50, overlapTokens: 0 });
  expect(r.chunks.length).toBeGreaterThan(1);
  for (const c of r.chunks) {
    expect(c.tokenCount).toBeLessThanOrEqual(200);
    expect(c.content.endsWith("。")).toBe(true);
  }
});

test("astral-plane Han characters count once each and are never split", () => {
  // CJK Extension B lives above U+FFFF: two UTF-16 code units per character.
  let text = "";
  for (let i = 0; i < 3000; i++) text += String.fromCodePoint(0x20000 + (i % 500));
  const r = chunkMarkdown(text, "ext-b", { maxTokens: 100, minTokens: 10, overlapTokens: 30 });
  expect(r.chunks.length).toBeGreaterThan(1);
  for (const c of r.chunks) {
    expect(c.tokenCount).toBeLessThanOrEqual(100);
    // No lone surrogate at either end of a chunk.
    expect(/^[\uDC00-\uDFFF]|[\uD800-\uDBFF]$/.test(c.content)).toBe(false);
    expect([...c.content.replace(/\n/g, "")].length).toBe(c.tokenCount);
  }
  expect(bodyText(text, { maxTokens: 100, minTokens: 10 })).toBe(text);
});

test("Thai combining marks stay with the character they modify", () => {
  const word = "ภาษาไทยไม่เว้นวรรคระหว่างคำที่เขียนต่อกัน";
  const text = word.repeat(200);
  const r = chunkMarkdown(text, "th", { maxTokens: 64, minTokens: 8, overlapTokens: 16 });
  expect(r.chunks.length).toBeGreaterThan(1);
  for (const c of r.chunks) {
    expect(c.tokenCount).toBeLessThanOrEqual(64);
    for (const line of c.content.split("\n")) expect(/^\p{M}/u.test(line)).toBe(false);
  }
});

test("Cyrillic and other spaced scripts keep counting words, not letters", () => {
  const text = "Это тестовая строка с кириллицей. Ελληνικά κείμενα επίσης. مرحبا بالعالم";
  const r = chunkMarkdown(text, "mixed");
  expect(r.chunks[0]?.tokenCount).toBe(text.split(" ").length);
  // A long Cyrillic note is not fragmented letter by letter.
  const long = Array.from({ length: 400 }, (_, i) => `слово${i}`).join(" ");
  const chunks = chunkMarkdown(long, "ru", { maxTokens: 800, minTokens: 100, overlapTokens: 100 });
  expect(chunks.chunks.length).toBe(1);
  expect(chunks.chunks[0]?.tokenCount).toBe(400);
});

// ── the cap invariant ────────────────────────────────────────────────────

function mulberry32(seed: number): () => number {
  let a = seed >>> 0;
  return () => {
    a = (a + 0x6d2b79f5) >>> 0;
    let t = a;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

const PIECES = [
  () => hanParagraph(40, true),
  () => "検索索引はノートを断片に分けます。",
  () => "한국어 문장은 띄어쓰기를 사용합니다.",
  () => "ภาษาไทยไม่เว้นวรรค",
  () => String.fromCodePoint(0x20000, 0x2a6d6, 0x2b740),
  () => "the quick brown fox jumps over the lazy dog",
  () => "Привет мир, это заметка",
  () => "emoji 🙂👍🏽 and café",
];

/** A deterministic mixed-script document: headings, lists, code, one-line walls. */
function generatedDocument(seed: number): string {
  const rand = mulberry32(seed);
  const pick = () => PIECES[Math.floor(rand() * PIECES.length)]!();
  const blocks: string[] = [];
  if (seed % 3 === 0)
    blocks.push(
      `---\ntitle: ${pick()}\nsummary: ${Array.from({ length: 60 }, pick).join("")}\n---`,
    );
  const count = 3 + Math.floor(rand() * 12);
  for (let b = 0; b < count; b++) {
    const kind = rand();
    const len = 1 + Math.floor(rand() * (rand() < 0.3 ? 300 : 20));
    const run = () => Array.from({ length: len }, pick).join(rand() < 0.5 ? "" : " ");
    if (kind < 0.15) blocks.push(`## ${run().slice(0, 400)}`);
    else if (kind < 0.3)
      blocks.push(Array.from({ length: 1 + (len % 6) }, () => `- ${run()}`).join("\n"));
    else if (kind < 0.4) blocks.push("```\n" + run() + "\n" + run() + "\n```");
    else blocks.push(Array.from({ length: 1 + (len % 4) }, run).join("\n"));
  }
  return blocks.join("\n\n");
}

for (const overlapTokens of [0, 100]) {
  test(`no chunk exceeds maxTokens (overlap ${overlapTokens}) over generated mixed-script documents`, () => {
    for (let seed = 1; seed <= 40; seed++) {
      const text = generatedDocument(seed);
      allWithinCap(text, { maxTokens: 800, minTokens: 100, overlapTokens });
      allWithinCap(text, {
        maxTokens: 150,
        minTokens: 20,
        overlapTokens: Math.min(overlapTokens, 60),
      });
    }
  });
}

test("overlap alone never pushes a chunk over the cap", () => {
  // Long single-line paragraphs: the old overlap took a whole line "whatever
  // it costs" and prepended it, so the next chunk started over budget.
  const line = Array.from({ length: 700 }, (_, i) => `w${i}`).join(" ");
  const text = `${line}\n\n${line}\n\n${line}`;
  allWithinCap(text, { maxTokens: 800, minTokens: 100, overlapTokens: 100 });
  const r = chunkMarkdown(text, "doc", { maxTokens: 800, minTokens: 100, overlapTokens: 100 });
  // Overlap is still carried, just bounded: chunk two opens with the tail of chunk one.
  expect(r.chunks[1]?.content.startsWith("w600 ")).toBe(true);
});

// ── the space-separated path is unchanged ────────────────────────────────

interface BaselineEntry {
  readonly options: number;
  readonly doc: number;
  readonly withinCap: boolean;
  readonly digest: string;
}

test("space-separated notes chunk exactly as before the fix", () => {
  const baseline = JSON.parse(
    readFileSync(
      join(import.meta.dir, "../../fixtures/chunker/space-separated-baseline.json"),
      "utf8",
    ),
  ) as { entries: BaselineEntry[] };
  const corpus = spaceSeparatedCorpus();
  let pinned = 0;
  for (const e of baseline.entries) {
    const opts = CORPUS_OPTION_SETS[e.options]!;
    const r = chunkMarkdown(corpus[e.doc]!, `note-${e.doc}`, opts);
    const digest = createHash("sha256").update(JSON.stringify(r.chunks)).digest("hex");
    if (e.withinCap) {
      // Output that already respected the cap is byte-identical, hashes and all.
      expect({ doc: e.doc, options: e.options, digest }).toEqual({
        doc: e.doc,
        options: e.options,
        digest: e.digest,
      });
      pinned++;
    } else {
      // Output that broke the cap changes, and now respects it.
      for (const c of r.chunks) expect(c.tokenCount).toBeLessThanOrEqual(opts.maxTokens);
    }
  }
  expect(pinned).toBeGreaterThan(50);
});