The design language that makes your AI harness better at design.
JavaScript
test: scenario 19 documentation case when the context launcher is denied (#791)
Matt Van Horn7a7579858c5a
diff --git a/tests/skill-behavior-harness.test.mjs b/tests/skill-behavior-harness.test.mjsindex f36f6a536..781702d35 100644--- a/tests/skill-behavior-harness.test.mjs+++ b/tests/skill-behavior-harness.test.mjs@@ -4,7 +4,7 @@ import fs from 'node:fs';import path from 'node:path';import { MockLanguageModelV3 } from 'ai/test';import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';-import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';+import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';@@ -186,6 +186,24 @@ it('headless behavior shells disable unattended decision pages and omit provider}});+it('documentation fallback requires an assistant warning before the first tool call after denial', () => {+ const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };+ const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };+ const warning = { role: 'assistant', content: 'Context loading did not run because the launcher was denied.' };+ const read = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'read', toolName: 'read', input: { path: 'reference/document.md' } }] };+ const write = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'write', toolName: 'write', input: { path: 'DESIGN.md' } }] };+ assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, warning, read, write]));+ assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, { role: 'assistant', content: [{ type: 'text', text: warning.content }, ...read.content] }, write]));+ for (const messages of [+ [call, denial, read, warning, write], // Reads first, warns only before the write.+ [call, denial, read, write, warning], // Final-only disclosure.+ [warning, call, denial, read], // Not a response to the actual denial.+ [call, denial, warning], // Warning with no follow-up tool call.+ ]) {+ assert.throws(() => assertLauncherDenialWarningBeforeNextTool(messages), assert.AssertionError);+ }+});+it('planning fallback requires an assistant warning between the denial and context reads', () => {const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.mdindex 19a14db3d..5e7456005 100644--- a/tests/skill-behavior/README.md+++ b/tests/skill-behavior/README.md@@ -321,7 +321,7 @@ results remain the completed measurements.| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic || 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic || 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |-| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |+| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, a denied-launcher planning-only case, and a denied-launcher documentation case (PRODUCT.md + index.html, no DESIGN.md) | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor; documentation requires successful document.md and source reads before any DESIGN.md write, with the denial disclosed before the first tool call after the denied launcher |## Setup launcher-failure branch (2026-09-06, PR #750)@@ -405,6 +405,20 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5,gpt-5.6-terra,gemini-3.7-flash,node --test --test-name-pattern='scenario 19:' tests/skill-behavior/scenarios.test.mjs```+### Degraded documentation case (#789)++A fourth scenario 19 case covers the documentation branch: launcher denied,+PRODUCT.md plus an incumbent `index.html`, prompt `/impeccable document`.+It requires an actual denied context attempt, successful `document.md` and+source reads before the DESIGN.md write, the denial disclosed before the+first tool call after the denied launcher, and an untouched PRODUCT.md.++Scenario 19 on this branch, all four cases, one run each on 2026-09-09:+4/4 on claude-sonnet-5, 4/4 on gpt-5.6-terra, 4/4 on gemini-3.7-flash. The+documentation case also passes with main's unchanged `SKILL.src.md` and+`init.md` plus this test file, so the coverage is the test, not a skill-text+change.+## Workflow-advice baseline (2026-09-05, PR #737)The four cases in scenarios 16-18 are new; prior scenario results do notdiff --git a/tests/skill-behavior/assertions.mjs b/tests/skill-behavior/assertions.mjsindex fc8ec5e32..236e9a3f1 100644--- a/tests/skill-behavior/assertions.mjs+++ b/tests/skill-behavior/assertions.mjs@@ -61,17 +61,25 @@ export function assertNewWorkLifecycle(trace, { target, redesign = false }) {export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;-export function assertPlanningFallbackWarning(responseMessages) {- const blocks = responseMessages.flatMap((message) =>+function responseBlocks(responseMessages) {+ return responseMessages.flatMap((message) =>(typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content).map((block) => ({ ...block, role: message.role })),);+}++function contextLauncherDenialIndex(blocks) {const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant'&& block.type === 'tool-call' && block.toolName === 'bash'&& /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId));- const denialIndex = blocks.findIndex((block) => block.role === 'tool'+ return blocks.findIndex((block) => block.role === 'tool'&& block.type === 'tool-result' && contextCalls.has(block.toolCallId)&& block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value));+}++export function assertPlanningFallbackWarning(responseMessages) {+ const blocks = responseBlocks(responseMessages);+ const denialIndex = contextLauncherDenialIndex(blocks);assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');const warningIndex = blocks.findIndex((block, index) => index > denialIndex&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));@@ -81,3 +89,16 @@ export function assertPlanningFallbackWarning(responseMessages) {assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex,'planning fallback must warn after denial and before reading project context, not only in the final response');}++export function assertLauncherDenialWarningBeforeNextTool(responseMessages) {+ const blocks = responseBlocks(responseMessages);+ const denialIndex = contextLauncherDenialIndex(blocks);+ assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');+ const warningIndex = blocks.findIndex((block, index) => index > denialIndex+ && block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));+ const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex+ && block.role === 'assistant' && block.type === 'tool-call');+ assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher');+ assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex,+ 'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write');+}diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjsindex d4c5c63a5..7057231f1 100644--- a/tests/skill-behavior/scenarios.test.mjs+++ b/tests/skill-behavior/scenarios.test.mjs@@ -28,7 +28,7 @@ import {ENGINE_MISSING_MESSAGE,} from './harness.mjs';import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';-import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';+import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';import { assertCompleted } from '../skill-workflow/assertions.mjs';import {PRODUCT_MD_SAMPLE,@@ -708,6 +708,35 @@ for (const modelId of resolveModelList()) {});}+ it('scenario 19: denied launcher requires document.md before writing DESIGN.md', async () => {+ const workspace = prepareWorkspace({ files: {+ 'PRODUCT.md': PRODUCT_MD_SAMPLE,+ 'index.html': MINIMAL_LANDING_HTML,+ } });+ try {+ const { trace, stepTexts, finishReason, responseMessages } = await runTurn({+ workspace,+ model,+ userPrompt: '/impeccable document. Record the incumbent design system from index.html into DESIGN.md.',+ maxSteps: 14,+ denyBash: true,+ });+ logTrace('S19', 'denied-launcher-document', modelId, trace, { finishReason, text: stepTexts.join('\n') });+ assert.notEqual(finishReason, 'length', 'a truncated response is not a completed documentation pass');+ assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt');+ const designWriteIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.some((p) => /(?:^|\/)DESIGN\.md$/.test(p)));+ assert.ok(designWriteIndex >= 0, 'must still produce DESIGN.md, not stop at the refusal');+ const documentReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && /(?:^|\/)reference\/document\.md$/.test(call.input.path));+ assert.ok(documentReadIndex >= 0 && documentReadIndex < designWriteIndex, 'reference/document.md must actually be read before DESIGN.md is written');+ const sourceReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && call.input.path.endsWith('index.html'));+ assert.ok(sourceReadIndex >= 0 && sourceReadIndex < designWriteIndex, 'the incumbent source must be read before DESIGN.md is written');+ assertLauncherDenialWarningBeforeNextTool(responseMessages);+ assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)PRODUCT\.md$/.test(p))), 'must not rewrite PRODUCT.md');+ } finally {+ cleanupWorkspace(workspace);+ }+ });+it('scenario 19: denied launcher keeps planning-only work read-only without craft-floor', async () => {const workspace = prepareWorkspace({ files: {'PRODUCT.md': PRODUCT_MD_SAMPLE,