Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(bengali): preserve decimal tokens

haelyra5e170e4886cf
diff --git a/pi/core/skills/bengali-nlp/SKILL.md b/pi/core/skills/bengali-nlp/SKILL.md
index c16b41946..74a81067f 100644
--- a/pi/core/skills/bengali-nlp/SKILL.md
+++ b/pi/core/skills/bengali-nlp/SKILL.md
@@ -131,8 +131,8 @@ import re
def tokenize_bangla(text: str) -> list[str]:
"""Basic Bengali tokenizer — split on whitespace and punctuation."""
- # Remove Bengali and ASCII punctuation
- text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+ # Remove Bengali and ASCII punctuation, but preserve decimal points
+ text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
return [token for token in text.split() if token]
# For production, use a trained tokenizer:
diff --git a/skills/bengali-nlp/SKILL.md b/skills/bengali-nlp/SKILL.md
index c16b41946..74a81067f 100644
--- a/skills/bengali-nlp/SKILL.md
+++ b/skills/bengali-nlp/SKILL.md
@@ -131,8 +131,8 @@ import re
def tokenize_bangla(text: str) -> list[str]:
"""Basic Bengali tokenizer — split on whitespace and punctuation."""
- # Remove Bengali and ASCII punctuation
- text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+ # Remove Bengali and ASCII punctuation, but preserve decimal points
+ text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
return [token for token in text.split() if token]
# For production, use a trained tokenizer:
diff --git a/tests/ci/bengali-language-surface.test.js b/tests/ci/bengali-language-surface.test.js
index cb88d3153..06e3c1955 100644
--- a/tests/ci/bengali-language-surface.test.js
+++ b/tests/ci/bengali-language-surface.test.js
@@ -143,6 +143,32 @@ function run() {
});
});
+ test('Bengali tokenizer preserves decimal quantities', () => {
+ const skill = read('skills/bengali-nlp/SKILL.md');
+ const tokenizer = extractFencedBlocks(skill)
+ .filter(block => block.language === 'python')
+ .find(block => block.code.includes('def tokenize_bangla'));
+ assert.ok(tokenizer, 'Expected the Bengali tokenizer example');
+
+ const python = findPython();
+ if (!python) {
+ throw new SkipError('no Python 3 interpreter on PATH');
+ }
+
+ const check = [
+ tokenizer.code,
+ "assert tokenize_bangla('ওজন ৩.৫ কেজি। মূল্য 12.75 টাকা.') == ['ওজন', '৩.৫', 'কেজি', 'মূল্য', '12.75', 'টাকা']",
+ ].join('\n');
+ const result = spawnSync(python.command, [...python.prefix, '-c', check], {
+ encoding: 'utf8',
+ });
+ assert.strictEqual(
+ result.status,
+ 0,
+ `Bengali tokenizer changed decimal quantities:\n${result.stderr || result.stdout}`,
+ );
+ });
+
test('Bengali skill is exported and installable', () => {
const agentYaml = read('agent.yaml');
const modules = readJson('manifests/install-modules.json').modules;