Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
skills: move description parsing and the word split into leaf modules
garrytanf68bf290821b
diff --git a/src/core/skill-catalog.ts b/src/core/skill-catalog.tsindex 27c4381a28..05d736bb92 100644--- a/src/core/skill-catalog.ts+++ b/src/core/skill-catalog.ts@@ -50,6 +50,8 @@ import {import { loadOrDeriveManifest, type ManifestEntry } from './skill-manifest.ts';import { loadSkillTriggerIndex, FRONTMATTER_SECTION } from './skill-trigger-index.ts';import { parseSkillFrontmatter } from './skill-frontmatter.ts';+import { oneLineDescription } from './skill-description.ts';+import { splitWords } from './skills/resolver/tokenize.ts';import { operationScopesAllowed } from './scope.ts';import { currentVerifiedLocalWriter } from './persistence/identity.ts';import { disabledOpsForPublishGates } from '../mcp/publish-gates.ts';@@ -378,72 +380,13 @@ function resolveSkillTools(// Frontmatter projection + description// ----------------------------------------------------------------------------/**- * Parse a `description:` from raw frontmatter (best-effort).- *- * #1711: handles YAML block scalars. `description: |` (literal) and- * `description: >` (folded), with optional chomping/indent indicators- * (`|-`, `>+`, `|2`), are parsed by reading the following indented lines and- * folding them into a single line for the one-line catalog. Pre-fix the regex- * captured the bare `|`/`>` indicator as the description, so block-scalar skills- * showed a literal "|" in the catalog instead of their text.- */-function parseDescriptionField(raw: string): string | undefined {- const lines = raw.split('\n');- for (let i = 0; i < lines.length; i++) {- const m = lines[i].match(/^description:[ \t]*(.*)$/);- if (!m) continue;- const inline = m[1].trim();-- // Block scalar indicator (`|`, `>`, with optional chomp `+`/`-` and indent digit).- if (/^[|>][+-]?\d*$/.test(inline)) {- const block: string[] = [];- let baseIndent: number | null = null;- for (let j = i + 1; j < lines.length; j++) {- const line = lines[j];- if (line.trim().length === 0) { block.push(''); continue; }- const indent = line.length - line.trimStart().length;- if (baseIndent === null) {- if (indent === 0) break; // no indented continuation- baseIndent = indent;- }- if (indent < baseIndent) break; // dedent ends the block- block.push(line.slice(baseIndent));- }- // Catalog descriptions are one line; collapse literal/folded whitespace.- const folded = block.join(' ').replace(/\s+/g, ' ').trim();- return folded.length > 0 ? folded : undefined;- }-- // Inline scalar: strip a matching pair of surrounding quotes.- const unq = inline.replace(/^(['"])([\s\S]*)\1$/, '$2').trim();- return unq.length > 0 ? unq : undefined;- }- return undefined;-}-/** Strip the leading `---\n...\n---` fence; return the prose body. */function stripFrontmatterFence(content: string): string {const m = content.match(/^---\n[\s\S]*?\n---\n?/);return m ? content.slice(m[0].length) : content;}-/** First non-empty, non-heading prose line of the body (description fallback). */-function firstProseLine(body: string): string {- for (const line of body.split('\n')) {- const t = line.trim();- if (t.length === 0) continue;- if (t.startsWith('#')) continue;- if (t.startsWith('<!--')) continue;- return t.replace(/^[*_>`-]+\s*/, '').trim();- }- return '';-}--/** description = frontmatter `description:` or first prose line, else ''. */-export function oneLineDescription(raw: string, body: string): string {- return parseDescriptionField(raw) ?? firstProseLine(body);-}+export { oneLineDescription };// ---------------------------------------------------------------------------// Catalog assembly@@ -488,7 +431,7 @@ const QUERY_RESULT_LIMIT_MAX = 100;/** Lowercased word tokens; hyphens and punctuation split, so `meeting-prep` is `meeting` + `prep`. */function queryTokens(text: string): string[] {- return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(t => t.length >= 2 && !QUERY_STOPWORDS.has(t));+ return splitWords(text).filter(t => t.length >= 2 && !QUERY_STOPWORDS.has(t));}/**diff --git a/src/core/skill-description.ts b/src/core/skill-description.tsnew file mode 100644index 0000000000..567e3dbac3--- /dev/null+++ b/src/core/skill-description.ts@@ -0,0 +1,61 @@+/**+ * One-line skill descriptions, as the skill catalog shows them: the+ * frontmatter `description:` (inline or a `|` / `>` block scalar folded to+ * one line), else the body's first prose line. Leaf module, shared by+ * `list_skills` (src/core/skill-catalog.ts), the bundled-skill reader+ * (src/core/skills/bundled.ts) and the skill resolver, so none of them has+ * to load the operations table to describe a skill.+ */++/**+ * `description: |` (literal) and `description: >` (folded), with optional+ * chomping/indent indicators (`|-`, `>+`, `|2`), are parsed by reading the+ * following indented lines and folding them into a single line.+ */+function parseDescriptionField(raw: string): string | undefined {+ const lines = raw.split('\n');+ for (let i = 0; i < lines.length; i++) {+ const m = lines[i].match(/^description:[ \t]*(.*)$/);+ if (!m) continue;+ const inline = m[1].trim();++ if (/^[|>][+-]?\d*$/.test(inline)) {+ const block: string[] = [];+ let baseIndent: number | null = null;+ for (let j = i + 1; j < lines.length; j++) {+ const line = lines[j];+ if (line.trim().length === 0) { block.push(''); continue; }+ const indent = line.length - line.trimStart().length;+ if (baseIndent === null) {+ if (indent === 0) break;+ baseIndent = indent;+ }+ if (indent < baseIndent) break;+ block.push(line.slice(baseIndent));+ }+ const folded = block.join(' ').replace(/\s+/g, ' ').trim();+ return folded.length > 0 ? folded : undefined;+ }++ const unq = inline.replace(/^(['"])([\s\S]*)\1$/, '$2').trim();+ return unq.length > 0 ? unq : undefined;+ }+ return undefined;+}++/** First non-empty, non-heading prose line of the body (description fallback). */+function firstProseLine(body: string): string {+ for (const line of body.split('\n')) {+ const t = line.trim();+ if (t.length === 0) continue;+ if (t.startsWith('#')) continue;+ if (t.startsWith('<!--')) continue;+ return t.replace(/^[*_>`-]+\s*/, '').trim();+ }+ return '';+}++/** description = frontmatter `description:` or first prose line, else ''. */+export function oneLineDescription(raw: string, body: string): string {+ return parseDescriptionField(raw) ?? firstProseLine(body);+}diff --git a/src/core/skills/bundled.ts b/src/core/skills/bundled.tsindex 07495fae7f..1416a8382b 100644--- a/src/core/skills/bundled.ts+++ b/src/core/skills/bundled.ts@@ -19,6 +19,7 @@*/import { createHash } from 'node:crypto';import { parseSkillFrontmatter } from '../skill-frontmatter.ts';+import { oneLineDescription } from '../skill-description.ts';import { BUNDLED_SKILLS, BUNDLED_SKILL_FILES, type BundledSkillMeta } from './bundled-skills.generated.ts';export type BundledSkill = BundledSkillMeta;@@ -45,7 +46,6 @@ export function bundledSkillTriggers(skill: BundledSkill): string[] {/** The skill's one-line description, as the skill catalog derives it. */export async function bundledSkillDescription(skill: BundledSkill): Promise<string> {- const { oneLineDescription } = await import('../skill-catalog.ts');const text = bundledSkillText(skill);return oneLineDescription(parseSkillFrontmatter(text)?.raw ?? '', text.replace(/^---\n[\s\S]*?\n---\n?/, '')).replace(/\s+/g, ' ').trim();}diff --git a/src/core/skills/resolver/tokenize.ts b/src/core/skills/resolver/tokenize.tsnew file mode 100644index 0000000000..7e95a235a2--- /dev/null+++ b/src/core/skills/resolver/tokenize.ts@@ -0,0 +1,63 @@+/**+ * The skill resolver's word rule, shared with `scoreSkillMatch`+ * (src/core/skill-catalog.ts), which keeps its own stopword list so+ * `list_skills` ranks exactly as before.+ *+ * `splitWords` is the Unicode split both rankers use: lowercase, then cut on+ * every run that is not a letter or a number. `lexTerms` is the BM25F term+ * rule frozen by the Step 1 offline eval (gbrain-evals+ * docs/benchmarks/2026-11-07-skill-ranker-offline, scripts/rank.ts `lex`):+ * drop pieces shorter than 2 code units and a fixed English stopword list,+ * then strip a few plural and tense suffixes. One deliberate difference:+ * a run of CJK characters (the ranges in src/core/cjk.ts) becomes+ * overlapping bigrams (a lone CJK character stays a unigram), where the+ * prototype kept the whole run as one token that almost never matches.+ * Text without CJK characters tokenizes exactly as the prototype did+ * (test/fixtures/skill-resolver/tokenizer-vectors.json pins it).+ *+ * Leaf module: no imports beyond the CJK ranges.+ */+import { CJK_SLUG_CHARS } from '../../cjk.ts';++export const TOKENIZER_ID = 'gbrain-skill-lex-v1';++const WORD_SPLIT = /[^\p{L}\p{N}]+/u;+const HAS_CJK = new RegExp(`[${CJK_SLUG_CHARS}]`);+const CJK_SEGMENTS = new RegExp(`[${CJK_SLUG_CHARS}]+|[^${CJK_SLUG_CHARS}]+`, 'g');++/** Lowercased pieces between non-letter, non-number runs (empty pieces included). */+export function splitWords(text: string): string[] {+ return text.toLowerCase().split(WORD_SPLIT);+}++export const STOPWORDS: ReadonlySet<string> = new Set(('a an the and or of to for in on at by with my me is it be this that from as are was were i you your we our us can could ' ++ 'would should will do does did have has had not no so if then than but about into over up out what which who how when where why please just also any all ' ++ 'some them they their there here its').split(' '));++export function stem(w: string): string {+ if (w.length > 4 && w.endsWith('ies')) return `${w.slice(0, -3)}y`;+ if (w.length > 4 && /(s|x|z|ch|sh)es$/.test(w)) return w.slice(0, -2);+ if (w.length > 3 && w.endsWith('s') && !/(ss|us|is)$/.test(w)) return w.slice(0, -1);+ if (w.length > 5 && w.endsWith('ing')) return w.slice(0, -3);+ if (w.length > 4 && w.endsWith('ed')) return w.slice(0, -2);+ return w;+}++function pushWord(out: string[], w: string): void {+ if (w.length >= 2 && !STOPWORDS.has(w)) out.push(stem(w));+}++/** BM25F terms of a text, in order (duplicates kept). */+export function lexTerms(text: string): string[] {+ const out: string[] = [];+ for (const piece of splitWords(text)) {+ if (!piece) continue;+ if (!HAS_CJK.test(piece)) { pushWord(out, piece); continue; }+ for (const seg of piece.match(CJK_SEGMENTS) ?? []) {+ if (!HAS_CJK.test(seg)) { pushWord(out, seg); continue; }+ if (seg.length === 1) { out.push(seg); continue; }+ for (let i = 0; i + 1 < seg.length; i++) out.push(seg.slice(i, i + 2));+ }+ }+ return out;+}