The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(bengali): preserve decimal tokens
haelyra5e170e4886cf
diff --git a/pi/core/skills/bengali-nlp/SKILL.md b/pi/core/skills/bengali-nlp/SKILL.mdindex c16b41946..74a81067f 100644--- a/pi/core/skills/bengali-nlp/SKILL.md+++ b/pi/core/skills/bengali-nlp/SKILL.md@@ -131,8 +131,8 @@ import redef tokenize_bangla(text: str) -> list[str]:"""Basic Bengali tokenizer — split on whitespace and punctuation."""- # Remove Bengali and ASCII punctuation- text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)+ # Remove Bengali and ASCII punctuation, but preserve decimal points+ text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)return [token for token in text.split() if token]# For production, use a trained tokenizer:diff --git a/skills/bengali-nlp/SKILL.md b/skills/bengali-nlp/SKILL.mdindex c16b41946..74a81067f 100644--- a/skills/bengali-nlp/SKILL.md+++ b/skills/bengali-nlp/SKILL.md@@ -131,8 +131,8 @@ import redef tokenize_bangla(text: str) -> list[str]:"""Basic Bengali tokenizer — split on whitespace and punctuation."""- # Remove Bengali and ASCII punctuation- text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)+ # Remove Bengali and ASCII punctuation, but preserve decimal points+ text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)return [token for token in text.split() if token]# For production, use a trained tokenizer:diff --git a/tests/ci/bengali-language-surface.test.js b/tests/ci/bengali-language-surface.test.jsindex cb88d3153..06e3c1955 100644--- a/tests/ci/bengali-language-surface.test.js+++ b/tests/ci/bengali-language-surface.test.js@@ -143,6 +143,32 @@ function run() {});});+ test('Bengali tokenizer preserves decimal quantities', () => {+ const skill = read('skills/bengali-nlp/SKILL.md');+ const tokenizer = extractFencedBlocks(skill)+ .filter(block => block.language === 'python')+ .find(block => block.code.includes('def tokenize_bangla'));+ assert.ok(tokenizer, 'Expected the Bengali tokenizer example');++ const python = findPython();+ if (!python) {+ throw new SkipError('no Python 3 interpreter on PATH');+ }++ const check = [+ tokenizer.code,+ "assert tokenize_bangla('ওজন ৩.৫ কেজি। মূল্য 12.75 টাকা.') == ['ওজন', '৩.৫', 'কেজি', 'মূল্য', '12.75', 'টাকা']",+ ].join('\n');+ const result = spawnSync(python.command, [...python.prefix, '-c', check], {+ encoding: 'utf8',+ });+ assert.strictEqual(+ result.status,+ 0,+ `Bengali tokenizer changed decimal quantities:\n${result.stderr || result.stdout}`,+ );+ });+test('Bengali skill is exported and installable', () => {const agentYaml = read('agent.yaml');const modules = readJson('manifests/install-modules.json').modules;