Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.3 KB
Raw Blame
import { describe, test, expect } from 'bun:test';
import {
  hasCJK,
  countCJKAwareWords,
  CJK_DENSITY_THRESHOLD,
  CJK_RANGES_REGEX,
  CJK_SLUG_CHARS,
  CJK_SENTENCE_DELIMITERS,
  CJK_CLAUSE_DELIMITERS,
  escapeLikePattern,
  splitCJKQueryTerms,
  stripKoreanParticle,
  cjkMatchTerms,
} from '../src/core/cjk.ts';

describe('hasCJK', () => {
  test('true on Han', () => {
    expect(hasCJK('品牌圣经')).toBe(true);
  });
  test('true on Hiragana', () => {
    expect(hasCJK('ひらがな')).toBe(true);
  });
  test('true on Katakana', () => {
    expect(hasCJK('カタカナ')).toBe(true);
  });
  test('true on Hangul Syllables', () => {
    expect(hasCJK('한글')).toBe(true);
  });
  test('false on ASCII', () => {
    expect(hasCJK('hello world')).toBe(false);
  });
  test('false on Latin-with-accents', () => {
    expect(hasCJK('café résumé')).toBe(false);
  });
  test('true on mixed CJK + ASCII', () => {
    expect(hasCJK('hello 世界')).toBe(true);
  });
  test('false on empty string', () => {
    expect(hasCJK('')).toBe(false);
  });
});

describe('countCJKAwareWords (30% density threshold)', () => {
  test('pure Chinese paragraph counts chars', () => {
    // 6 Han characters, all CJK → char count
    expect(countCJKAwareWords('品牌圣经测试用例')).toBe(8);
  });

  test('pure ASCII paragraph counts whitespace tokens', () => {
    expect(countCJKAwareWords('this is a normal english sentence')).toBe(6);
  });

  test('CJK-dominant mixed switches to char count', () => {
    // ~80% CJK by char count → char-count branch
    const s = '品牌圣经品牌圣经 is the brand';
    expect(countCJKAwareWords(s)).toBe(s.replace(/\s/g, '').length);
  });

  test('English doc with one Japanese term stays whitespace-tokenized', () => {
    // 1 CJK / ~50 non-whitespace chars = ~2%, well below 30%
    const s = 'the user wrote a long english blog post about マンガ and other interests';
    // Should NOT char-count (would be ~60). Should whitespace-tokenize.
    expect(countCJKAwareWords(s)).toBe((s.match(/\S+/g) || []).length);
  });

  test('exactly at 30% threshold uses CJK branch', () => {
    // 3 CJK chars + 7 ASCII non-ws chars = 10 total; 3/10 = 0.30 → CJK
    const s = '世界世 abcdefg';
    expect(countCJKAwareWords(s)).toBe(10);
  });

  test('just below threshold uses whitespace branch', () => {
    // 2 CJK + 8 ASCII = 10 total; 2/10 = 0.20 < 0.30 → whitespace
    const s = '世界 abcdefgh';
    expect(countCJKAwareWords(s)).toBe(2); // two whitespace-delimited tokens
  });

  test('empty string returns 0', () => {
    expect(countCJKAwareWords('')).toBe(0);
  });

  test('whitespace-only returns 0', () => {
    expect(countCJKAwareWords('   \n\t  ')).toBe(0);
  });
});

describe('constants', () => {
  test('CJK_DENSITY_THRESHOLD is 0.30', () => {
    expect(CJK_DENSITY_THRESHOLD).toBe(0.30);
  });

  test('CJK_RANGES_REGEX matches all four scripts', () => {
    expect(CJK_RANGES_REGEX.test('一')).toBe(true);
    expect(CJK_RANGES_REGEX.test('あ')).toBe(true);
    expect(CJK_RANGES_REGEX.test('カ')).toBe(true);
    expect(CJK_RANGES_REGEX.test('한')).toBe(true);
    expect(CJK_RANGES_REGEX.test('a')).toBe(false);
  });

  test('CJK_SLUG_CHARS can be embedded into a character class', () => {
    const re = new RegExp(`^[a-z0-9${CJK_SLUG_CHARS}]+$`);
    expect(re.test('品牌圣经')).toBe(true);
    expect(re.test('hello')).toBe(true);
    expect(re.test('hello-world')).toBe(false); // no hyphen in this test class
  });

  test('CJK_SENTENCE_DELIMITERS covers 。!?', () => {
    expect(CJK_SENTENCE_DELIMITERS).toEqual(['。', '!', '?']);
  });

  test('CJK_CLAUSE_DELIMITERS covers ;:,、', () => {
    expect(CJK_CLAUSE_DELIMITERS).toEqual([';', ':', ',', '、']);
  });
});

describe('escapeLikePattern', () => {
  test('escapes % and _', () => {
    expect(escapeLikePattern('100% off_today')).toBe('100\\% off\\_today');
  });
  test('escapes backslash first', () => {
    // 'a\b' → 'a\\b' (backslash doubled)
    expect(escapeLikePattern('a\\b')).toBe('a\\\\b');
  });
  test('escapes all three meta-chars together', () => {
    expect(escapeLikePattern('a\\%b_c')).toBe('a\\\\\\%b\\_c');
  });
  test('no-op on plain text', () => {
    expect(escapeLikePattern('hello world')).toBe('hello world');
  });
  test('no-op on CJK', () => {
    expect(escapeLikePattern('测试')).toBe('测试');
  });
});

describe('splitCJKQueryTerms', () => {
  test('splits query by whitespace into terms', () => {
    expect(splitCJKQueryTerms('김대리 미팅')).toEqual(['김대리', '미팅']);
    expect(splitCJKQueryTerms('미팅 김대리')).toEqual(['미팅', '김대리']);
  });

  test('deduplicates terms while preserving original order', () => {
    expect(splitCJKQueryTerms('김대리 미팅 김대리')).toEqual(['김대리', '미팅']);
    expect(splitCJKQueryTerms('미팅 김대리 미팅')).toEqual(['미팅', '김대리']);
    expect(splitCJKQueryTerms('a b a c b')).toEqual(['a', 'b', 'c']);
  });

  test('handles consecutive whitespace, tabs, and newlines', () => {
    expect(splitCJKQueryTerms('김대리   미팅 \t\n 2026')).toEqual(['김대리', '미팅', '2026']);
    expect(splitCJKQueryTerms('  \t  주간   보고서  \n ')).toEqual(['주간', '보고서']);
  });

  test('returns single term for single-word queries', () => {
    expect(splitCJKQueryTerms('김대리')).toEqual(['김대리']);
    expect(splitCJKQueryTerms(' 한글 ')).toEqual(['한글']);
  });

  test('returns empty array on empty or whitespace-only input', () => {
    expect(splitCJKQueryTerms('')).toEqual([]);
    expect(splitCJKQueryTerms('   ')).toEqual([]);
    expect(splitCJKQueryTerms('\t\n\r ')).toEqual([]);
  });

  test('handles mixed CJK and alphanumeric queries', () => {
    expect(splitCJKQueryTerms('한국어 日本語 中文 English 123')).toEqual([
      '한국어',
      '日本語',
      '中文',
      'English',
      '123',
    ]);
  });
});


describe('stripKoreanParticle (#6666)', () => {
  test('strips the longest trailing particle', () => {
    expect(stripKoreanParticle('혈당에서')).toBe('혈당');
    expect(stripKoreanParticle('검진을')).toBe('검진');
    expect(stripKoreanParticle('병원으로')).toBe('병원');
    expect(stripKoreanParticle('진료실에서')).toBe('진료실');
    expect(stripKoreanParticle('결과에서는')).toBe('결과');
  });

  test('keeps a word that only ends in a particle syllable', () => {
    expect(stripKoreanParticle('회의')).toBe('회의');
    expect(stripKoreanParticle('결과')).toBe('결과');
    expect(stripKoreanParticle('도로')).toBe('도로');
    expect(stripKoreanParticle('에서')).toBe('에서');
    expect(stripKoreanParticle('는')).toBe('는');
  });

  test('leaves Chinese, Japanese and ASCII terms alone', () => {
    expect(stripKoreanParticle('席位')).toBe('席位');
    expect(stripKoreanParticle('東京で')).toBe('東京で');
    expect(stripKoreanParticle('LDL')).toBe('LDL');
    expect(stripKoreanParticle('LDL수치가')).toBe('LDL수치');
  });
});

describe('cjkMatchTerms (#6666)', () => {
  test('stems each term and drops duplicates in order', () => {
    expect(cjkMatchTerms('혈당에서 검진을')).toEqual(['혈당', '검진']);
    expect(cjkMatchTerms('혈당 혈당에서 수치')).toEqual(['혈당', '수치']);
    expect(cjkMatchTerms('席位 升级')).toEqual(['席位', '升级']);
  });
});