The design language that makes your AI harness better at design.
JavaScript
Slim the skill description and add an activation eval (#1048)
Paul Bakausef5dcdb04aaa
diff --git a/CLAUDE.md b/CLAUDE.mdindex 5959938b3..1e8e89b3e 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -194,6 +194,7 @@ The default suite does not cover everything. When a change touches one of these| `ENGINE_VERSION` bump | also `bun run test:live-e2e-accept-cleanup` | bills a provider API key || `ENGINE_VERSION` bump | `bun run test:live-svelte-adapter-deepseek` | bills DeepSeek || `SKILL.src.md` Setup, Setup-adjacent reference files, `ENGINE_VERSION` bump | `bun run test:skill-behavior` | ~5 min, bills all four provider keys |+| The skill's frontmatter `description` in `SKILL.src.md` | `bun run test:skill-activation` | ~3 min, about $1 (Anthropic + Google keys); baseline in `tests/skill-behavior/README.md` || `ENGINE_VERSION` bump | `bun run test:new-work-e2e` | Playwright, offline, no API cost || `plugin/`, `skill/agents/`, `scripts/build.js`, plugin manifest validator | `bun run test:plugin-e2e` | ~1 s; already in the default suite, needs the `claude` CLI |diff --git a/package.json b/package.jsonindex e9cf17fc0..e81a159e8 100644--- a/package.json+++ b/package.json@@ -66,6 +66,7 @@"test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e","test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent","test:skill-behavior": "node scripts/run-tests.mjs skill-behavior",+ "test:skill-activation": "node scripts/run-tests.mjs skill-activation","test:skill-workflow": "node scripts/run-tests.mjs skill-workflow","test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek","test:rust-changed": "node scripts/test-rust-changed.mjs",diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjsindex 8df8a7574..575623195 100644--- a/scripts/test-suites.mjs+++ b/scripts/test-suites.mjs@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = ['live-e2e-accept-cleanup','new-work-e2e','skill-behavior',+ 'skill-activation','skill-workflow','live-svelte-adapter-deepseek',];@@ -288,7 +289,9 @@ export const SUITES = {/^skill\/SKILL\.src\.md$/,/^skill\/reference\//,/^ENGINE_VERSION$/,- /^tests\/skill-behavior\//,+ // activation.eval.mjs and its fixtures belong to skill-activation, which+ // nothing here loads; editing them should not bill this lane.+ /^tests\/skill-behavior\/(?!activation[.-])/,],commands: [{runner: 'node',@@ -297,6 +300,20 @@ export const SUITES = {files: ['tests/skill-behavior/scenarios.test.mjs'],}],},+ 'skill-activation': {+ description: 'LLM-backed check that the skill description triggers on UI work and stays quiet otherwise.',+ optIn: true,+ triggers: [+ /^skill\/SKILL\.src\.md$/,+ /^tests\/skill-behavior\/(activation-cases|providers)\.mjs$/,+ ],+ commands: [{+ runner: 'node',+ timeoutMs: 900000,+ wallClockMs: 1_200_000,+ files: ['tests/skill-behavior/activation.eval.mjs'],+ }],+ },'skill-workflow': {description: 'Explicitly opt-in completed workflows with a preflighted browser.',optIn: true,@@ -306,7 +323,9 @@ export const SUITES = {/^skill\//,/^ENGINE_VERSION$/,/^tests\/skill-workflow\//,- /^tests\/skill-behavior\//,+ // activation.eval.mjs and its fixtures belong to skill-activation, which+ // nothing here loads; editing them should not bill this lane.+ /^tests\/skill-behavior\/(?!activation[.-])/,],commands: [{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },diff --git a/skill/SKILL.src.md b/skill/SKILL.src.mdindex 9b703a148..de0316613 100644--- a/skill/SKILL.src.md+++ b/skill/SKILL.src.md@@ -1,6 +1,6 @@---name: impeccable-description: "Use when the user wants to design, redesign, shape, critique, audit, polish, clarify, distill, harden, optimize, adapt, animate, colorize, extract, or otherwise improve a frontend interface. Covers websites, landing pages, dashboards, product UI, app shells, components, forms, settings, onboarding, and empty states. Handles UX review, visual hierarchy, information architecture, cognitive load, accessibility, performance, responsive behavior, theming, anti-patterns, typography, fonts, spacing, layout, alignment, color, motion, micro-interactions, UX copy, error states, edge cases, i18n, and reusable design systems or tokens. Also use for bland designs that need to become bolder or more delightful, loud designs that should become quieter, live browser iteration on UI elements, or ambitious visual effects that should feel technically extraordinary. Not for backend-only or non-UI tasks."+description: "Use when the user wants to design, redesign, shape, critique, audit, polish, simplify, harden, animate, or otherwise improve a frontend interface: websites, landing pages, dashboards, product UI, components, forms, settings, onboarding, empty states, and native iOS or Android screens. Handles UX review, hierarchy, layout, typography, color, motion, UX copy, accessibility, responsive behavior, edge cases, and capturing or documenting design systems (DESIGN.md, tokens). Also for making bland UI bolder, loud UI quieter, or ambitious visual effects. Not for backend-only or non-UI tasks."argument-hint: "[{{command_hint}}] [target]"user-invocable: trueallowed-tools:diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.mdindex 750425f66..617ee7e98 100644--- a/tests/skill-behavior/README.md+++ b/tests/skill-behavior/README.md@@ -35,6 +35,85 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5 IMPECCABLE_SKILL_BEHAVIOR_VERBOnode --test --test-timeout=600000 --test-name-pattern="scenario 6" tests/skill-behavior/scenarios.test.mjs```+## Skill activation (`activation.eval.mjs`, issue #375)++`bun run test:skill-activation` is a separate opt-in suite for the frontmatter+`description` in `skill/SKILL.src.md`, the one line every harness loads into+every session. It does not load the skill body. It simulates the selection+step: a system prompt lists twelve skills (impeccable plus eleven neighbours+such as frontend-dev, docs-writer, dataviz and brand-identity, from+`activation-cases.mjs`), the user message is one request, and the model has a+`Skill` tool beside Read, Edit, Write and Bash. An activation is a `Skill` call+naming `impeccable`; starting the work directly or loading another skill is a+miss.++The 54 cases are 34 positives (every command family, five document-shaped+requests, three native) and 20 negatives (clear non-UI work plus the #744 near+neighbours: frontend code with no visual decision, and visual work that is not+an interface). Cases ending in `_744` are the pinned #744 prompts.++```bash+bun run test:skill-activation # gate: current description, 2 models x 2 samples, about $1+node tests/skill-behavior/activation.eval.mjs # same run, report only+ACTIVATION_VARIANTS=current,previous ACTIVATION_SAMPLES=3 \+ node tests/skill-behavior/activation.eval.mjs # compare against the v4.5 description+ACTIVATION_DESCRIPTION_FILE=draft.txt ACTIVATION_VARIANTS=current,file \+ node tests/skill-behavior/activation.eval.mjs # try a draft before editing SKILL.src.md+```++The gate fails on any false activation, or when recall drops under 40%+(Sonnet) or 75% (Gemini), or document-shaped recall under 50%. Run it whenever+the description changes, and compare variants at three samples before shipping+a rewrite: single cells swing between 0/3 and 3/3 across runs, so read the+category totals, not individual cases.++### Baseline (2026-10-16, claude-sonnet-5 and gemini-3.7-flash, 3 samples, first action)++`previous` is the 118-word description shipped through v4.5 (301 tokens);+`current` is the 81-word rewrite that replaced it (211 tokens). "Other" is the+25 positives that are neither document-shaped nor native.++| Model | Description | Recall | Document | Native | Other | False positives |+|---|---|---|---|---|---|---|+| Sonnet | previous | 35/99 (35%) | 3/15 | 2/9 | 30/75 | 0/21 |+| Sonnet | current | 54/99 (55%) | 12/15 | 8/9 | 34/75 | 0/21 |+| Gemini | previous | 75/99 (76%) | 4/15 | 7/9 | 64/75 | 0/21 |+| Gemini | current | 88/99 (89%) | 13/15 | 9/9 | 66/75 | 0/21 |++The negatives in this round were the seven near neighbours; over all rounds+the 13 clear non-UI negatives never fired either (0/112 per model for+`previous`).++What the measurement found:++- The previous description never named DESIGN.md or native platforms. "Write+ a DESIGN.md", "Document our design system" and "Create a style guide" went to+ docs-writer or straight to work (0/3 on both models), which is #744's+ completion finding. SwiftUI and Compose requests scored 0/3 on Sonnet.+- Shorter is not free. A 53-word rewrite (`short` in the script) fixed those+ cases but lost the core UI asks it did not name: ambitious effects went from+ 3/3 to 0/3 on both models, and token extraction, form simplification,+ component and motion requests all dropped. Its non-document, non-native+ recall fell from 35/75 to 27/75 (Sonnet) and from 65/75 to 56/75 (Gemini). A+ 34-word version lost more. The verb enumeration does real work; the long+ topic list (cognitive load, i18n, anti-patterns, theming) did not.+- Sonnet decides at the first step. When its first action is not `Skill`, it+ explores with Bash or Read and almost never loads the skill afterwards (2 of+ about 135 runs with four steps and stubbed results). Polish, accessibility+ review, empty states, responsive fixes and edge-case hardening miss on+ Sonnet under every description tried; wording alone does not reach them.+- A direct UI performance ask (`ui_performance`, added after the baseline+ above) fired 0/3 on both models under both descriptions: dropping+ "performance" and "optimize" from the description lost nothing there, and+ naming them did not help either. Through the gate it costs about one+ point of recall.+- Gemini's misses mostly go to the frontend-dev neighbour, so results depend+ on the roster. The roster is a guess at a typical install, not a measurement.++This is a simulated harness, not Claude Code itself, and neither model is the+Haiku that #744 measured. Treat it as a regression gate and a way to compare+drafts, not as a prediction of production activation rates.+## How it works### Protocol versus full completiondiff --git a/tests/skill-behavior/activation-cases.mjs b/tests/skill-behavior/activation-cases.mjsnew file mode 100644index 000000000..a3d29d3dd--- /dev/null+++ b/tests/skill-behavior/activation-cases.mjs@@ -0,0 +1,93 @@+/**+ * Fixtures for the skill-activation eval (activation.eval.mjs).+ *+ * COMPETITORS is the rest of the simulated skill listing: realistic neighbours,+ * several deliberately close to the boundary (frontend-dev, docs-writer,+ * dataviz, brand-identity). impeccable is inserted in the middle of the list.+ *+ * CASES: expect true means impeccable should load, false means it should not.+ * Ids ending in _744 come from the pinned case set in issue #744 (verbatim).+ * cat groups cases for the per-category report; positives cover every command+ * family, document-shaped requests and native UI; negatives cover clear non-UI+ * work plus near neighbours (frontend code with no visual decision, or a visual+ * decision that is not an interface).+ */++export const COMPETITORS = [+ ['frontend-dev', 'Implement and debug frontend features in React, Vue, Svelte, or Next.js: components, state management, routing, data fetching, forms and validation, build tooling, and browser bugs. Use for writing or fixing frontend code.'],+ ['docs-writer', 'Write and improve technical documentation: READMEs, API references, guides, tutorials, changelogs, and docstrings. Use when the user asks to write, restructure, or edit documentation for a codebase or library.'],+ ['code-review', 'Review a diff, branch, or pull request for correctness bugs, security issues, and maintainability problems, and report findings by severity. Use when the user asks for a code review.'],+ ['test-writer', 'Write and fix unit, integration, and end-to-end tests (Jest, Vitest, pytest, Playwright). Use when the user asks to add tests, improve coverage, or fix failing tests.'],+ ['pdf', 'Read, create, merge, split, and fill PDF files, including form fields, OCR, watermarks, and text or table extraction. Use whenever a .pdf file is involved.'],+ ['xlsx', 'Create, read, and edit spreadsheets (.xlsx, .csv): formulas, formatting, cleaning messy tabular data, and charts inside workbooks. Use when a spreadsheet is the input or output.'],+ ['dataviz', 'Use before creating any chart, graph, plot, or data visualization in any medium: chart type choice, color palettes, axes, legends, and tooltips.'],+ ['sql-helper', 'Write, explain, and optimize SQL queries and schemas for Postgres, MySQL, and SQLite, including indexes, query plans, and migrations.'],+ ['ci-config', 'Create and debug CI/CD pipelines: GitHub Actions, GitLab CI, CircleCI, caching, matrices, secrets, and deployment workflows.'],+ ['brand-identity', 'Develop brand identity: naming, logos and wordmarks, brand voice, and brand guidelines for print and marketing collateral.'],+ ['security-review', 'Audit code for security vulnerabilities such as injection, auth flaws, secrets in source, and unsafe dependencies.'],+];++export const CASES = [+ // build / redesign+ { id: 'hero_direction_744', cat: 'build', expect: true, prompt: 'src/routes/landing.tsx is our hero and it reads like a wireframe - system font, one blue button, no hierarchy. Give it a real visual direction and build it: type, colour, spacing, the lot. Decide the direction yourself.' },+ { id: 'landing_new', cat: 'build', expect: true, prompt: 'Build a landing page for our new invoicing product in src/routes/index.tsx. Make it look great.' },+ { id: 'pricing_page', cat: 'build', expect: true, prompt: 'We need a pricing page with three tiers and a comparison table. Add it at src/routes/pricing.tsx.' },+ { id: 'redesign_site', cat: 'build', expect: true, prompt: 'Our marketing site looks like every other SaaS template. Redesign the homepage so it actually stands out.' },+ { id: 'forgettable_usage_744', cat: 'bolder', expect: true, prompt: 'src/routes/usage.tsx is the screen our customers stare at every day and it is competent and completely forgettable - three grey boxes and a bare table. Make it something they would remember, and build it.' },+ { id: 'quieter', cat: 'quieter', expect: true, prompt: 'The dashboard is way too loud: gradients everywhere, five accent colors, everything bold. Tone it down so it feels calm and professional.' },+ // review family+ { id: 'polish', cat: 'polish', expect: true, prompt: 'The checkout flow works but feels rough around the edges. Do a final pass on it before we ship tomorrow.' },+ { id: 'audit', cat: 'audit', expect: true, prompt: 'Audit the frontend in src/ for quality issues: accessibility, performance, theming, responsive problems. Give me a prioritized list.' },+ { id: 'critique', cat: 'critique', expect: true, prompt: 'Be honest: what is wrong with our signup page from a UX perspective? Screenshot is in docs/signup.png.' },+ { id: 'settings_rework_744', cat: 'layout', expect: true, prompt: 'Go through src/routes/settings.tsx and tell me where it falls down on hierarchy, grouping and the destructive action sitting next to Save, then rework it.' },+ { id: 'a11y_review', cat: 'a11y', expect: true, prompt: 'Check our modal and form components for accessibility problems: focus handling, contrast, labels, keyboard use.' },+ // craft topics+ { id: 'typography', cat: 'typography', expect: true, prompt: 'The fonts on our blog feel off. Pick a better type pairing and fix the scale and line lengths.' },+ { id: 'color', cat: 'color', expect: true, prompt: 'Our app is all grey. Add some color to it without making it garish, and give it a proper dark mode.' },+ { id: 'layout_spacing', cat: 'layout', expect: true, prompt: 'Everything on the account page feels cramped and misaligned. Fix the spacing and alignment.' },+ { id: 'motion', cat: 'motion', expect: true, prompt: 'Add some tasteful micro-interactions and transitions to the sidebar and cards so the app feels more alive.' },+ { id: 'ux_copy', cat: 'copy', expect: true, prompt: 'Rewrite the button labels, error messages and empty-form hints in the signup flow; right now they are confusing and robotic.' },+ { id: 'onboarding', cat: 'onboarding', expect: true, prompt: 'New users drop off after signup. Create a first-run onboarding experience for the web app that gets them to their first project.' },+ { id: 'empty_state_744', cat: 'onboarding', expect: true, prompt: 'A brand new workspace lands on src/routes/usage.tsx with no rows at all, so it just shows three zeroes and an empty table. Work out what someone arriving for the first time should see there instead, and build it.' },+ { id: 'dashboard', cat: 'build', expect: true, prompt: 'Build an admin dashboard showing signups, revenue and churn, with a recent-activity table.' },+ { id: 'responsive', cat: 'responsive', expect: true, prompt: 'The nav and the pricing cards break on phones: they overflow and the text is tiny. Fix it for mobile.' },+ { id: 'harden_errors', cat: 'harden', expect: true, prompt: 'Make the file upload component handle the ugly cases well: long filenames, errors, slow networks, 0 files, 500 files.' },+ { id: 'ui_performance', cat: 'optimize', expect: true, prompt: 'The product list stutters when you scroll and the filters feel sluggish on mid-range laptops. Make the interface feel fast.' },+ { id: 'clarify_distill', cat: 'distill', expect: true, prompt: 'The project creation form has 14 fields and people abandon it. Simplify it.' },+ { id: 'component', cat: 'build', expect: true, prompt: 'Make a nice-looking date range picker component for our design system.' },+ { id: 'overdrive', cat: 'overdrive', expect: true, prompt: 'I want the hero of our portfolio to have a jaw-dropping WebGL effect that reacts to the cursor.' },+ { id: 'live_iterate', cat: 'live', expect: true, prompt: 'Open the app in the browser and let me iterate on a few variants of the hero card visually.' },+ // document-shaped design requests (#744 weakness)+ { id: 'completion_744', cat: 'document', expect: true, prompt: 'Work out the visual system this codebase is already using - type scale, colour, spacing, component patterns - and write it down at the project root, in the file a later session would look for, so a new contributor can pick it up without asking.' },+ { id: 'design_md', cat: 'document', expect: true, prompt: 'Write a DESIGN.md for this app.' },+ { id: 'document_ds', cat: 'document', expect: true, prompt: 'Document our design system: colors, type, spacing, and the core components.' },+ { id: 'extract_tokens', cat: 'document', expect: true, prompt: 'Pull the hard-coded colors and spacing values out of our components into reusable design tokens.' },+ { id: 'style_guide', cat: 'document', expect: true, prompt: 'Create a style guide for the frontend so new screens stay consistent with the existing ones.' },+ // native+ { id: 'ios_screen', cat: 'native', expect: true, prompt: 'Our SwiftUI settings screen looks dated. Redesign it so it feels like a modern iOS app.' },+ { id: 'android_compose', cat: 'native', expect: true, prompt: 'Polish the Jetpack Compose home screen: spacing, typography and the bottom bar feel off.' },+ { id: 'rn_onboarding', cat: 'native', expect: true, prompt: 'Design the onboarding screens for our React Native app.' },++ // negatives: clear non-UI+ { id: 'backend_bug', cat: 'neg-backend', expect: false, prompt: 'POST /api/orders returns 500 when the cart has a discount code. Find and fix the bug in api/orders.ts.' },+ { id: 'sql', cat: 'neg-backend', expect: false, prompt: 'Write a SQL query that returns the top 10 customers by revenue for each month of 2025.' },+ { id: 'slow_query_744', cat: 'neg-backend', expect: false, prompt: 'This Postgres query for monthly revenue takes 12 seconds on 4M rows. Work out which index it is missing.' },+ { id: 'api_readme', cat: 'neg-docs', expect: false, prompt: 'Write a README for our REST API in api/: setup, auth, and example requests for each endpoint.' },+ { id: 'jsdoc', cat: 'neg-docs', expect: false, prompt: 'Add docstrings to every exported function in src/lib/.' },+ { id: 'unit_tests', cat: 'neg-tests', expect: false, prompt: 'Add unit tests for src/lib/pricing.ts; it has no coverage at all.' },+ { id: 'ci_config', cat: 'neg-ci', expect: false, prompt: 'Set up a GitHub Actions workflow that runs lint and tests on every pull request and caches node_modules.' },+ { id: 'refactor', cat: 'neg-refactor', expect: false, prompt: 'Refactor src/lib/api-client.ts to use async/await instead of promise chains and remove the duplicated retry logic.' },+ { id: 'pdf_form', cat: 'neg-pdf', expect: false, prompt: 'Fill out the W-9 PDF form in docs/w9.pdf with our company details from company.json.' },+ { id: 'data_analysis', cat: 'neg-data', expect: false, prompt: 'Analyze data/sales.csv and tell me which regions grew fastest last quarter.' },+ { id: 'code_review', cat: 'neg-review', expect: false, prompt: 'Review my current diff for bugs before I merge.' },+ { id: 'dep_upgrade', cat: 'neg-refactor', expect: false, prompt: 'Upgrade the project from React 18 to React 19 and fix whatever breaks in the build.' },+ { id: 'cli_script', cat: 'neg-backend', expect: false, prompt: 'Write a Python script that renames every photo in a folder by its EXIF date.' },+ // negatives: near neighbours from #744 (frontend code, no visual decision; or visual but not an interface)+ { id: 'undefined_email_744', cat: 'neg-near', expect: false, prompt: 'src/components/InviteForm.tsx throws "Cannot read properties of undefined (reading \'toLowerCase\')" when a member row arrives with no email. Fix the crash. Nothing on screen changes.' },+ { id: 'cta_instrumentation_744', cat: 'neg-near', expect: false, prompt: 'When the "Start free" button in src/routes/landing.tsx first scrolls into view, send a cta_seen event to our analytics with the visitor\'s plan tier. Use an IntersectionObserver and fire it once per session. Nothing on screen changes.' },+ { id: 'token_substitution_744', cat: 'neg-near', expect: false, prompt: 'Every literal in src/components/Badge.tsx already exists as a custom property in src/tokens.css. Swap the literals for the variables. This is a find-and-replace - do not add, remove or change a single value, and the rendered pixels must be identical afterwards.' },+ { id: 'double_submit_744', cat: 'neg-near', expect: false, prompt: 'The button in src/components/PlanPicker.tsx fires onChoose twice if you click it quickly, and the workspace ends up on whichever PATCH resolves last. Guard the handler so a second click while one is in flight does nothing. Nothing on screen changes.' },+ { id: 'wordmark_744', cat: 'neg-near', expect: false, prompt: 'Read brand/brief.md and give me three directions for the wordmark itself - the letterforms and the mark. Not where it sits on a page, the mark.' },+ { id: 'print_booklet_744', cat: 'neg-near', expect: false, prompt: 'Read print/investor-brief.md. For the printed pages, decide which chart types carry those four figures and what colour palette survives CMYK on uncoated stock at that dot gain. Print only - there is no screen version and there never will be.' },+ { id: 'redux_state', cat: 'neg-near', expect: false, prompt: 'Move the cart state in our React app from prop drilling to a Zustand store. No visual changes.' },+];diff --git a/tests/skill-behavior/activation.eval.mjs b/tests/skill-behavior/activation.eval.mjsnew file mode 100644index 000000000..99332d570--- /dev/null+++ b/tests/skill-behavior/activation.eval.mjs@@ -0,0 +1,331 @@+/**+ * Skill-activation eval: does the frontmatter `description` in+ * skill/SKILL.src.md make a model load the skill when it should, and only then?+ *+ * Harnesses put every installed skill's name and description into every+ * session; the model decides from that line alone whether to load the skill.+ * This eval simulates that decision. The system prompt lists twelve skills+ * (impeccable plus the eleven neighbours in activation-cases.mjs), the user+ * message is one request, and the model has the Skill tool plus Read, Edit,+ * Write and Bash. An activation is a `Skill` call naming `impeccable`.+ * Starting the work directly or routing to another skill is a miss; that is+ * what the misses in issue #744 looked like (Read, then Edit, no Skill call).+ *+ * It reports recall on the positive cases, the false-positive rate on the+ * negatives, per-category misses and per-case counts. Under `node --test` (the+ * `bun run test:skill-activation` suite) it also asserts a floor: zero false+ * positives, and recall at or above the per-model floors in GATES below, which+ * sit well under the measured baseline in tests/skill-behavior/README.md.+ *+ * Run:+ * bun run test:skill-activation # gate on the current description+ * node tests/skill-behavior/activation.eval.mjs # same eval, report only+ *+ * Knobs (environment):+ * ACTIVATION_MODELS=claude-sonnet-5,gemini-3.7-flash model lineup (default shown)+ * ACTIVATION_VARIANTS=current,previous descriptions to compare (see VARIANTS)+ * ACTIVATION_DESCRIPTION_FILE=path.txt add a draft description as variant "file"+ * ACTIVATION_SAMPLES=2 samples per case (default 2)+ * ACTIVATION_STEPS=1 >1 lets Read/Bash return stubs so the+ * model can explore before deciding+ * ACTIVATION_ONLY=id,id restrict to some case ids+ * ACTIVATION_CONCURRENCY=8+ * ACTIVATION_OUT=path.json write raw results+ * ACTIVATION_REPORT=path.json re-print a report from saved results+ *+ * Cost: the default run is 54 cases x 2 samples x 2 models = 216 single-step+ * calls, about 2k input tokens each: roughly $1 on Sonnet and cents on Gemini.+ * Each extra variant adds the same again. ACTIVATION_STEPS=4 roughly doubles+ * the Sonnet cost even with prompt caching. Keys come from the repo-root .env;+ * a model whose key is unset is skipped.+ */+import { generateText, stepCountIs, tool } from 'ai';+import { z } from 'zod';+import fs from 'node:fs';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { detectProvider, getModel, hasKey, PROVIDERS } from './providers.mjs';+import { CASES, COMPETITORS } from './activation-cases.mjs';++const __dirname = path.dirname(fileURLToPath(import.meta.url));+const REPO_ROOT = path.resolve(__dirname, '..', '..');++// ---------------------------------------------------------------------------+// Description variants+// ---------------------------------------------------------------------------++export function currentDescription() {+ const src = fs.readFileSync(path.join(REPO_ROOT, 'skill', 'SKILL.src.md'), 'utf8');+ const m = src.match(/^description:\s*"(.*)"\s*$/m);+ if (!m) throw new Error('could not read description from skill/SKILL.src.md');+ return m[1];+}++export const VARIANTS = {+ current: currentDescription(),+ // The 118-word description shipped through v4.5 (issue #375 baseline).+ previous: 'Use when the user wants to design, redesign, shape, critique, audit, polish, clarify, distill, harden, optimize, adapt, animate, colorize, extract, or otherwise improve a frontend interface. Covers websites, landing pages, dashboards, product UI, app shells, components, forms, settings, onboarding, and empty states. Handles UX review, visual hierarchy, information architecture, cognitive load, accessibility, performance, responsive behavior, theming, anti-patterns, typography, fonts, spacing, layout, alignment, color, motion, micro-interactions, UX copy, error states, edge cases, i18n, and reusable design systems or tokens. Also use for bland designs that need to become bolder or more delightful, loud designs that should become quieter, live browser iteration on UI elements, or ambitious visual effects that should feel technically extraordinary. Not for backend-only or non-UI tasks.',+ // The ~50-word rewrite measured for #375. Kept as the cautionary short form:+ // it gained the document and native cases but lost core UI asks.+ short: 'Design, redesign, critique, audit, or polish frontend UI: websites, landing pages, dashboards, app screens, components, and native iOS or Android views. Covers layout, typography, color, motion, UX copy, accessibility, responsive behavior, onboarding, empty states, and documenting design systems (DESIGN.md, tokens). Use when visual or UX quality matters. Not for backend or non-UI work.',+};+if (process.env.ACTIVATION_DESCRIPTION_FILE) {+ VARIANTS.file = fs.readFileSync(process.env.ACTIVATION_DESCRIPTION_FILE, 'utf8').trim();+}++// Floors for the node --test gate, per model family. The baseline (README)+// measured 55% (Sonnet) and 89% (Gemini) recall for the shipped description+// at n=3; the floors leave room for sampling noise at n=2.+const GATES = {+ anthropic: { recall: 0.4, documentRecall: 0.5 },+ google: { recall: 0.75, documentRecall: 0.5 },+ default: { recall: 0.4, documentRecall: 0.5 },+};++// ---------------------------------------------------------------------------+// Simulated harness+// ---------------------------------------------------------------------------++export function systemPrompt(impeccableDescription) {+ const skills = [...COMPETITORS.slice(0, 5), ['impeccable', impeccableDescription], ...COMPETITORS.slice(5)];+ const listing = skills.map(([n, d]) => `- ${n}: ${d}`).join('\n');+ return [+ 'You are a coding agent working in the user\'s repository at /work (a React + Vite web app with src/, a package.json and an api/ folder, unless the request says otherwise).',+ 'Use the tools available to you to complete the user\'s request.',+ '',+ 'Skills are packaged instructions for particular kinds of task. When a task matches one of the skills listed below, call the Skill tool to load it BEFORE doing anything else; its instructions replace your default approach. If no skill fits, just start the work with your other tools.',+ '',+ 'The following skills are available for use with the Skill tool:',+ '',+ listing,+ ].join('\n');+}++const STUB = {+ Read: ({ file_path }) => `// ${file_path}\nexport default function Page() {\n return (<main className="page">\n <h1>Title</h1>\n <p>Body copy.</p>\n <button className="btn">Continue</button>\n </main>);\n}\n`,+ Bash: () => 'README.md\napi/\npackage.json\nsrc/\n components/\n routes/\n lib/\n styles/\n tokens.css\nvite.config.ts',+};++// No `execute` on any tool by default, so generateText stops after the first+// step and the model's first decision is what gets scored.+const BASE_TOOLS = {+ Skill: tool({+ description: 'Load a skill by name. Use when the request matches one of the listed skills.',+ inputSchema: z.object({ skill: z.string().describe('Exact skill name from the listing'), args: z.string().optional() }),+ }),+ Read: tool({ description: 'Read a file from the repository.', inputSchema: z.object({ file_path: z.string() }) }),+ Edit: tool({ description: 'Replace a string in a file.', inputSchema: z.object({ file_path: z.string(), old_string: z.string(), new_string: z.string() }) }),+ Write: tool({ description: 'Write a file.', inputSchema: z.object({ file_path: z.string(), content: z.string() }) }),+ Bash: tool({ description: 'Run a shell command.', inputSchema: z.object({ command: z.string() }) }),+};++function toolsFor(steps) {+ if (steps <= 1) return BASE_TOOLS;+ return {+ ...BASE_TOOLS,+ Read: tool({ ...BASE_TOOLS.Read, execute: async (i) => STUB.Read(i) }),+ Bash: tool({ ...BASE_TOOLS.Bash, execute: async (i) => STUB.Bash(i) }),+ };+}++async function runOne(modelId, description, prompt, steps) {+ for (let attempt = 0; attempt < 4; attempt++) {+ try {+ const result = await generateText({+ model: getModel(modelId),+ instructions: { role: 'system', content: systemPrompt(description), providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } } },+ prompt,+ tools: toolsFor(steps),+ stopWhen: stepCountIs(steps),+ maxOutputTokens: 2048,+ });+ const calls = result.steps.flatMap((st) => st.toolCalls ?? []);+ // A response can load several skills (frontend-dev, then impeccable).+ // Any impeccable load counts; otherwise report the first skill named.+ const loaded = calls+ .filter((t) => t.toolName === 'Skill')+ .map((t) => String(t.input?.skill ?? '').replace(/^\//, ''));+ return {+ skill: loaded.includes('impeccable') ? 'impeccable' : (loaded[0] ?? null),+ skillsLoaded: loaded,+ firstTool: calls[0]?.toolName ?? 'text',+ steps: result.steps.length,+ usage: result.totalUsage ?? result.usage,+ };+ } catch (err) {+ if (attempt === 3) return { error: String(err?.message ?? err) };+ await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));+ }+ }+}++async function pool(jobs, n, fn) {+ let i = 0;+ const out = new Array(jobs.length);+ await Promise.all(Array.from({ length: n }, async () => {+ while (i < jobs.length) {+ const k = i++;+ out[k] = await fn(jobs[k]);+ }+ }));+ return out;+}++export function resolveOptions(env = process.env) {+ const list = (v, d) => (v || d).split(',').map((s) => s.trim()).filter(Boolean);+ const only = env.ACTIVATION_ONLY ? new Set(list(env.ACTIVATION_ONLY)) : null;+ return {+ models: list(env.ACTIVATION_MODELS, 'claude-sonnet-5,gemini-3.7-flash'),+ variants: list(env.ACTIVATION_VARIANTS, 'current'),+ samples: Number(env.ACTIVATION_SAMPLES || 2),+ steps: Number(env.ACTIVATION_STEPS || 1),+ concurrency: Number(env.ACTIVATION_CONCURRENCY || 8),+ cases: only ? CASES.filter((c) => only.has(c.id)) : CASES,+ };+}++export async function runEval({ models, variants, samples, steps, concurrency, cases }) {+ for (const v of variants) if (!VARIANTS[v]) throw new Error(`unknown variant "${v}" (have: ${Object.keys(VARIANTS).join(', ')})`);+ const jobs = [];+ for (const m of models) for (const v of variants) for (const c of cases) for (let s = 0; s < samples; s++) jobs.push({ m, v, c, s });+ return pool(jobs, concurrency, async (j) => ({+ model: j.m, variant: j.v, id: j.c.id, cat: j.c.cat, expect: j.c.expect, sample: j.s,+ ...(await runOne(j.m, VARIANTS[j.v], j.c.prompt, steps)),+ }));+}++// ---------------------------------------------------------------------------+// Scoring and report+// ---------------------------------------------------------------------------++const fired = (r) => r.skill === 'impeccable';++export function score(results, model, variant) {+ const rs = results.filter((r) => !r.error && r.model === model && r.variant === variant);+ const pos = rs.filter((r) => r.expect);+ const neg = rs.filter((r) => !r.expect);+ const doc = pos.filter((r) => r.cat === 'document');+ return {+ tp: pos.filter(fired).length, positives: pos.length,+ fp: neg.filter(fired).length, negatives: neg.length,+ docTp: doc.filter(fired).length, documents: doc.length,+ otherSkill: pos.filter((r) => r.skill && r.skill !== 'impeccable').length,+ errors: results.filter((r) => r.error && r.model === model && r.variant === variant).length,+ };+}++function pct(a, b) { return b ? `${((100 * a) / b).toFixed(0)}%` : 'n/a'; }++export function report(results, log = console.log) {+ const models = [...new Set(results.map((r) => r.model))];+ const variants = [...new Set(results.map((r) => r.variant))];+ const ids = new Set(results.map((r) => r.id));+ const cases = CASES.filter((c) => ids.has(c.id));+ const ok = results.filter((r) => !r.error);++ log('Descriptions:');+ for (const v of variants) log(` ${v}: ${VARIANTS[v] ? VARIANTS[v].split(/\s+/).length : '?'} words`);++ log('\nRecall on positives / false positives on negatives:');+ for (const m of models) for (const v of variants) {+ const s = score(results, m, v);+ log(` ${m.padEnd(18)} ${v.padEnd(9)} recall ${s.tp}/${s.positives} (${pct(s.tp, s.positives)}) document ${s.docTp}/${s.documents} FP ${s.fp}/${s.negatives} routed elsewhere ${s.otherSkill}${s.errors ? ` errors ${s.errors}` : ''}`);+ }++ log('\nPer-case activations (fired/samples):');+ const cols = models.flatMap((m) => variants.map((v) => ({ m, v, label: `${m.split('-')[0]}:${v}` })));+ log(` ${'case'.padEnd(26)} ${cols.map((c) => c.label.padStart(16)).join(' ')}`);+ for (const c of cases) {+ const cells = cols.map(({ m, v }) => {+ const rs = ok.filter((r) => r.model === m && r.variant === v && r.id === c.id);+ return `${rs.filter(fired).length}/${rs.length}`.padStart(16);+ });+ log(` ${((c.expect ? '+ ' : '- ') + c.id).padEnd(26)} ${cells.join(' ')}`);+ }++ log('\nMisses by category (positives):');+ const cats = [...new Set(cases.filter((c) => c.expect).map((c) => c.cat))];+ for (const m of models) for (const v of variants) {+ const misses = cats.map((cat) => {+ const rs = ok.filter((r) => r.model === m && r.variant === v && r.cat === cat && r.expect);+ const miss = rs.filter((r) => !fired(r)).length;+ return miss ? `${cat} ${miss}/${rs.length}` : null;+ }).filter(Boolean);+ log(` ${m} ${v}: ${misses.join(', ') || 'none'}`);+ }++ if (variants.length > 1) {+ const [base, ...rest] = variants;+ log(`\nCases where a variant differs from "${base}" (positives lost, negatives gained):`);+ for (const m of models) for (const v of rest) for (const c of cases) {+ const n = (vv) => ok.filter((r) => r.model === m && r.variant === vv && r.id === c.id).filter(fired).length;+ const a = n(base);+ const b = n(v);+ if ((c.expect && b < a) || (!c.expect && b > a)) log(` ${m} ${v} ${c.id}: ${base} ${a} -> ${v} ${b}`);+ }+ }++ log('\nUsage:');+ for (const m of models) {+ const rs = ok.filter((r) => r.model === m);+ const inTok = rs.reduce((s, r) => s + (r.usage?.inputTokens ?? 0), 0);+ const outTok = rs.reduce((s, r) => s + (r.usage?.outputTokens ?? 0), 0);+ log(` ${m}: ${rs.length} calls, ${inTok} input tokens, ${outTok} output tokens`);+ }+}++// ---------------------------------------------------------------------------+// Entry points: `node --test` (gated suite) or plain `node` (report only)+// ---------------------------------------------------------------------------++async function runAsTest() {+ const { describe, it } = await import('node:test');+ const assert = (await import('node:assert/strict')).default;+ const opts = resolveOptions();+ for (const modelId of opts.models) {+ const provider = detectProvider(modelId);+ describe(`skill activation :: ${modelId}`, () => {+ if (!hasKey(provider)) {+ it(`skipped, ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});+ return;+ }+ it(`clears the activation floor for: ${opts.variants.join(', ')}`, async () => {+ const results = await runEval({ ...opts, models: [modelId] });+ report(results, (line) => console.log(line));+ const gate = GATES[provider] ?? GATES.default;+ for (const v of opts.variants) {+ const s = score(results, modelId, v);+ assert.ok(s.errors <= Math.ceil(0.05 * (s.positives + s.negatives + s.errors)), `${v}: ${s.errors} calls errored`);+ assert.equal(s.fp, 0, `${v}: ${s.fp} false activations on negative cases`);+ assert.ok(s.tp / s.positives >= gate.recall, `${v}: recall ${pct(s.tp, s.positives)} is under the ${gate.recall * 100}% floor`);+ if (s.documents) {+ assert.ok(s.docTp / s.documents >= gate.documentRecall, `${v}: document-shaped recall ${s.docTp}/${s.documents} is under the ${gate.documentRecall * 100}% floor`);+ }+ }+ });+ });+ }+}++async function runAsScript() {+ if (process.env.ACTIVATION_REPORT) {+ report(JSON.parse(fs.readFileSync(process.env.ACTIVATION_REPORT, 'utf8')));+ return;+ }+ const opts = resolveOptions();+ opts.models = opts.models.filter((m) => {+ const ok = hasKey(detectProvider(m));+ if (!ok) console.error(`skip ${m}: ${PROVIDERS[detectProvider(m)].envKey} is unset`);+ return ok;+ });+ const total = opts.models.length * opts.variants.length * opts.cases.length * opts.samples;+ console.error(`${total} calls (${opts.models.join(', ')} x ${opts.variants.join(', ')} x ${opts.cases.length} cases x ${opts.samples})`);+ const results = await runEval(opts);+ if (process.env.ACTIVATION_OUT) fs.writeFileSync(process.env.ACTIVATION_OUT, JSON.stringify(results, null, 2));+ report(results);+}++if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {+ const entry = process.env.NODE_TEST_CONTEXT ? runAsTest() : runAsScript();+ entry.catch((e) => { console.error(e); process.exit(1); });+}