Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.6 KB
Raw Blame
/**
 * Shared CJK (Chinese / Japanese / Korean) detection and handling primitives.
 *
 * Replaces the inline copy in `src/core/search/expansion.ts:58` and provides
 * a single source of truth for every downstream caller: slug grammar, chunker
 * word-counting, chunker delimiters, PGLite keyword fallback.
 *
 * Scope: BMP-only Unicode ranges that cover ~99% of real CJK content:
 *   - Han (CJK Unified Ideographs): U+4E00–U+9FFF
 *   - Hiragana: U+3040–U+309F
 *   - Katakana: U+30A0–U+30FF
 *   - Hangul Syllables: U+AC00–U+D7AF
 *
 * Out of scope (v0.32.7): Han extensions A/B/C, halfwidth katakana,
 * compatibility ideographs, compatibility Jamo, iteration marks (々/〇).
 * Filed as v0.33+ follow-up.
 */

export const CJK_SLUG_CHARS = '一-鿿぀-ゟ゠-ヿ가-힯';

export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`);

/**
 * Slug "word" character class (#3417): every script's letters, not just
 * Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any
 * regex composed from this string (without `u`, `\p{Ll}` silently matches
 * the literal chars `p`, `L`, `l`, `{`, `}`).
 *
 *   \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …)
 *   \p{Lm} modifier letters
 *   \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …)
 *   \p{M}  combining marks that survive the Latin accent-strip pass
 *          (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs)
 *   \p{N}  numbers (0-9, Arabic-Indic digits, …)
 *
 * Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment()
 * lowercases before filtering, so validators stay lowercase-canonical.
 *
 * Distinct from CJK_SLUG_CHARS above, which also drives the
 * countCJKAwareWords density heuristic — do NOT merge the two, or slug
 * grammar changes silently change chunking behavior.
 */
export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}';

/**
 * Unicode variation selectors (emoji VS1–VS16 U+FE00–FE0F, ideographic IVS
 * U+E0100–E01EF): Mn-category invisibles that survive the \p{M} keep above.
 * Both slug grammars — `sync.ts:slugifySegment` and its #4855 twin
 * `link-extraction.ts:normalizeBasename` — strip them so an emoji folder name
 * written with VS16 lands on the clean slug and its `[[wikilink]]` resolves.
 */
export const SLUG_VARIATION_SELECTORS_RE = /[\uFE00-\uFE0F\u{E0100}-\u{E01EF}]/gu;

/**
 * Page-slug segment grammar (no anchors): word-char lead, then word-char or
 * hyphen continuation. Single source for validatePageSlug (operations.ts),
 * SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug
 * validator shares one grammar (#738). Compose with the `u` flag — see
 * SLUG_WORD_CHARS.
 */
export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`;

export const CJK_SENTENCE_DELIMITERS = ['。', '!', '?']; // 。!?
export const CJK_CLAUSE_DELIMITERS = [';', ':', ',', '、']; // ;:,、

/**
 * Density threshold for switching word-count strategy. Below this CJK char
 * density, a doc is treated as Latin-mostly and stays whitespace-tokenized
 * (so a 5000-word English doc with one Japanese term doesn't get char-counted
 * and over-split). At or above, it's CJK-mostly.
 */
export const CJK_DENSITY_THRESHOLD = 0.30;

export function hasCJK(s: string): boolean {
  return CJK_RANGES_REGEX.test(s);
}

/**
 * CJK-aware "word" count. CJK languages aren't whitespace-tokenized, so a
 * paragraph of Chinese collapses to 1 word under /\S+/g and downstream chunkers
 * never split it (the 8192-token OpenAI embedding limit then rejects the chunk).
 *
 * Heuristic (per codex outside-voice C13): switch on CJK character density,
 * not mere presence. Below CJK_DENSITY_THRESHOLD the doc is Latin-dominant
 * and whitespace tokens are the right unit; at or above it's CJK-dominant
 * and char count is the right unit.
 */
export function countCJKAwareWords(s: string): number {
  if (s.length === 0) return 0;
  return wordCountOf(wordStats(s));
}

/**
 * The ONE density test behind countCJKAwareWords: true when CJK chars make
 * up at least CJK_DENSITY_THRESHOLD of the non-whitespace chars. Callers
 * that need to branch on "would countCJKAwareWords count chars here?" (the
 * chunker's overlap extractor) route through this so the two cannot drift.
 */
export function isCJKDominant(s: string): boolean {
  return statsAreCJKDominant(wordStats(s));
}

/**
 * Everything countCJKAwareWords needs, gathered in one pass over UTF-16
 * code units: non-whitespace units, BMP CJK units (the CJK_SLUG_CHARS
 * ranges), and `/\S+/g` runs. The first/last flags let two adjacent
 * strings' stats combine in O(1) (concatWordStats), so a caller that grows
 * a string piece by piece never recounts it.
 */
export interface WordStats {
  length: number;
  nonWhitespace: number;
  cjk: number;
  runs: number;
  startsNonWhitespace: boolean;
  endsNonWhitespace: boolean;
}

const CJK_UNIT_RANGES: ReadonlyArray<readonly [number, number]> = Array.from(
  CJK_SLUG_CHARS.matchAll(/(.)-(.)/g),
  m => [m[1]!.charCodeAt(0), m[2]!.charCodeAt(0)] as const,
);

const CJK_UNIT_MIN = Math.min(...CJK_UNIT_RANGES.map(([lo]) => lo));

function isCJKUnit(c: number): boolean {
  if (c < CJK_UNIT_MIN) return false;
  for (const [lo, hi] of CJK_UNIT_RANGES) if (c >= lo && c <= hi) return true;
  return false;
}

/** Exactly the code units JavaScript's `\s` matches. */
function isWhitespaceUnit(c: number): boolean {
  if (c <= 0x20) return c === 0x20 || (c >= 0x09 && c <= 0x0d);
  if (c < 0xa0) return false;
  return c === 0xa0 || c === 0x1680 || (c >= 0x2000 && c <= 0x200a)
    || c === 0x2028 || c === 0x2029 || c === 0x202f || c === 0x205f
    || c === 0x3000 || c === 0xfeff;
}

export function wordStats(s: string): WordStats {
  let nonWhitespace = 0;
  let cjk = 0;
  let runs = 0;
  let prevNonWhitespace = false;
  for (let i = 0; i < s.length; i++) {
    const c = s.charCodeAt(i);
    if (isWhitespaceUnit(c)) {
      prevNonWhitespace = false;
      continue;
    }
    nonWhitespace++;
    if (isCJKUnit(c)) cjk++;
    if (!prevNonWhitespace) runs++;
    prevNonWhitespace = true;
  }
  return {
    length: s.length,
    nonWhitespace,
    cjk,
    runs,
    startsNonWhitespace: s.length > 0 && !isWhitespaceUnit(s.charCodeAt(0)),
    endsNonWhitespace: prevNonWhitespace,
  };
}

/** Stats of `a + b` from the stats of `a` and `b`. */
export function concatWordStats(a: WordStats, b: WordStats): WordStats {
  return {
    length: a.length + b.length,
    nonWhitespace: a.nonWhitespace + b.nonWhitespace,
    cjk: a.cjk + b.cjk,
    runs: a.runs + b.runs - (a.endsNonWhitespace && b.startsNonWhitespace ? 1 : 0),
    startsNonWhitespace: a.length === 0 ? b.startsNonWhitespace : a.startsNonWhitespace,
    endsNonWhitespace: b.length === 0 ? a.endsNonWhitespace : b.endsNonWhitespace,
  };
}

export function statsAreCJKDominant(st: WordStats): boolean {
  if (st.nonWhitespace === 0) return false;
  return st.cjk / st.nonWhitespace >= CJK_DENSITY_THRESHOLD;
}

/** countCJKAwareWords of the string the stats describe. */
export function wordCountOf(st: WordStats): number {
  return statsAreCJKDominant(st) ? st.nonWhitespace : st.runs;
}

/**
 * LIKE-pattern escape for PGLite/Postgres `ILIKE ... ESCAPE '\'`.
 * Must escape backslash FIRST so the introduced backslashes aren't double-escaped.
 */
export function escapeLikePattern(s: string): string {
  return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
}

/**
 * Splits a CJK query into distinct, non-empty whitespace-delimited terms.
 *
 * In Korean and Japanese, word order is flexible and particles attach to nouns,
 * so the same fact or search intent frequently appears in varying token sequences
 * (e.g. "김대리 미팅" vs "미팅 김대리"). Splitting into individual terms allows
 * multi-term conjunction (AND matching) across chunk text regardless of word order.
 *
 * Returns deduplicated terms preserving the original order of appearance.
 */
export function splitCJKQueryTerms(query: string): string[] {
  const terms = query.split(/\s+/).filter(t => t.length > 0);
  return Array.from(new Set(terms));
}


/**
 * Korean postpositions (조사), longest first so `endsWith` takes the longest
 * particle ("에서" before "에"/"서", "으로" before "로").
 */
const KO_PARTICLES: readonly string[] = [
  '에서는', '에게서', '으로는', '으로서',
  '에서', '에게', '한테', '까지', '부터', '보다', '처럼', '마다',
  '조차', '라도', '이나', '으로', '로서',
  '은', '는', '이', '가', '을', '를', '에', '의', '로', '와', '과', '도', '만',
];

/** One-syllable stems ("회의" → "회") match almost everything, so a stem keeps 2+ characters. */
const KO_MIN_STEM_LENGTH = 2;

/**
 * The noun stem of a Korean term with a trailing particle ("혈당에서" →
 * "혈당"), or the term unchanged. Korean attaches particles to the noun, so
 * the substring match would otherwise miss a page that says "혈당 수치". The
 * stem is a substring of the inflected form, so matching the stem alone
 * covers both spellings. Stripping is lexical (고양이 → 고양), which only
 * widens a fallback match.
 */
export function stripKoreanParticle(term: string): string {
  for (const p of KO_PARTICLES) {
    if (term.length - p.length >= KO_MIN_STEM_LENGTH && term.endsWith(p)) return term.slice(0, -p.length);
    if (term.endsWith(p)) return term;
  }
  return term;
}

/**
 * The terms every CJK keyword builder matches: whitespace-split, Korean
 * particles stripped, deduplicated in order. The count, the keyword pages and
 * the chunk search all read this, so a count never disagrees with the rows.
 */
export function cjkMatchTerms(query: string): string[] {
  return Array.from(new Set(splitCJKQueryTerms(query).map(stripKoreanParticle)));
}