Skip to content

The design language that makes your AI harness better at design.

JavaScript

test: scenario 19 documentation case when the context launcher is denied (#791)

Matt Van Horn7a7579858c5a
diff --git a/tests/skill-behavior-harness.test.mjs b/tests/skill-behavior-harness.test.mjs
index f36f6a536..781702d35 100644
--- a/tests/skill-behavior-harness.test.mjs
+++ b/tests/skill-behavior-harness.test.mjs
@@ -4,7 +4,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { MockLanguageModelV3 } from 'ai/test';
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';
-import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
+import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';
import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';
import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';
@@ -186,6 +186,24 @@ it('headless behavior shells disable unattended decision pages and omit provider
}
});
+it('documentation fallback requires an assistant warning before the first tool call after denial', () => {
+ const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
+ const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
+ const warning = { role: 'assistant', content: 'Context loading did not run because the launcher was denied.' };
+ const read = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'read', toolName: 'read', input: { path: 'reference/document.md' } }] };
+ const write = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'write', toolName: 'write', input: { path: 'DESIGN.md' } }] };
+ assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, warning, read, write]));
+ assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, { role: 'assistant', content: [{ type: 'text', text: warning.content }, ...read.content] }, write]));
+ for (const messages of [
+ [call, denial, read, warning, write], // Reads first, warns only before the write.
+ [call, denial, read, write, warning], // Final-only disclosure.
+ [warning, call, denial, read], // Not a response to the actual denial.
+ [call, denial, warning], // Warning with no follow-up tool call.
+ ]) {
+ assert.throws(() => assertLauncherDenialWarningBeforeNextTool(messages), assert.AssertionError);
+ }
+});
+
it('planning fallback requires an assistant warning between the denial and context reads', () => {
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md
index 19a14db3d..5e7456005 100644
--- a/tests/skill-behavior/README.md
+++ b/tests/skill-behavior/README.md
@@ -321,7 +321,7 @@ results remain the completed measurements.
| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic |
| 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic |
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
-| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
+| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, a denied-launcher planning-only case, and a denied-launcher documentation case (PRODUCT.md + index.html, no DESIGN.md) | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor; documentation requires successful document.md and source reads before any DESIGN.md write, with the denial disclosed before the first tool call after the denied launcher |
## Setup launcher-failure branch (2026-09-06, PR #750)
@@ -405,6 +405,20 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5,gpt-5.6-terra,gemini-3.7-flash,
node --test --test-name-pattern='scenario 19:' tests/skill-behavior/scenarios.test.mjs
```
+### Degraded documentation case (#789)
+
+A fourth scenario 19 case covers the documentation branch: launcher denied,
+PRODUCT.md plus an incumbent `index.html`, prompt `/impeccable document`.
+It requires an actual denied context attempt, successful `document.md` and
+source reads before the DESIGN.md write, the denial disclosed before the
+first tool call after the denied launcher, and an untouched PRODUCT.md.
+
+Scenario 19 on this branch, all four cases, one run each on 2026-09-09:
+4/4 on claude-sonnet-5, 4/4 on gpt-5.6-terra, 4/4 on gemini-3.7-flash. The
+documentation case also passes with main's unchanged `SKILL.src.md` and
+`init.md` plus this test file, so the coverage is the test, not a skill-text
+change.
+
## Workflow-advice baseline (2026-09-05, PR #737)
The four cases in scenarios 16-18 are new; prior scenario results do not
diff --git a/tests/skill-behavior/assertions.mjs b/tests/skill-behavior/assertions.mjs
index fc8ec5e32..236e9a3f1 100644
--- a/tests/skill-behavior/assertions.mjs
+++ b/tests/skill-behavior/assertions.mjs
@@ -61,17 +61,25 @@ export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
-export function assertPlanningFallbackWarning(responseMessages) {
- const blocks = responseMessages.flatMap((message) =>
+function responseBlocks(responseMessages) {
+ return responseMessages.flatMap((message) =>
(typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content)
.map((block) => ({ ...block, role: message.role })),
);
+}
+
+function contextLauncherDenialIndex(blocks) {
const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant'
&& block.type === 'tool-call' && block.toolName === 'bash'
&& /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId));
- const denialIndex = blocks.findIndex((block) => block.role === 'tool'
+ return blocks.findIndex((block) => block.role === 'tool'
&& block.type === 'tool-result' && contextCalls.has(block.toolCallId)
&& block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value));
+}
+
+export function assertPlanningFallbackWarning(responseMessages) {
+ const blocks = responseBlocks(responseMessages);
+ const denialIndex = contextLauncherDenialIndex(blocks);
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
@@ -81,3 +89,16 @@ export function assertPlanningFallbackWarning(responseMessages) {
assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex,
'planning fallback must warn after denial and before reading project context, not only in the final response');
}
+
+export function assertLauncherDenialWarningBeforeNextTool(responseMessages) {
+ const blocks = responseBlocks(responseMessages);
+ const denialIndex = contextLauncherDenialIndex(blocks);
+ assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
+ const warningIndex = blocks.findIndex((block, index) => index > denialIndex
+ && block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
+ const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex
+ && block.role === 'assistant' && block.type === 'tool-call');
+ assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher');
+ assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex,
+ 'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write');
+}
diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs
index d4c5c63a5..7057231f1 100644
--- a/tests/skill-behavior/scenarios.test.mjs
+++ b/tests/skill-behavior/scenarios.test.mjs
@@ -28,7 +28,7 @@ import {
ENGINE_MISSING_MESSAGE,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
-import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
+import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
import { assertCompleted } from '../skill-workflow/assertions.mjs';
import {
PRODUCT_MD_SAMPLE,
@@ -708,6 +708,35 @@ for (const modelId of resolveModelList()) {
});
}
+ it('scenario 19: denied launcher requires document.md before writing DESIGN.md', async () => {
+ const workspace = prepareWorkspace({ files: {
+ 'PRODUCT.md': PRODUCT_MD_SAMPLE,
+ 'index.html': MINIMAL_LANDING_HTML,
+ } });
+ try {
+ const { trace, stepTexts, finishReason, responseMessages } = await runTurn({
+ workspace,
+ model,
+ userPrompt: '/impeccable document. Record the incumbent design system from index.html into DESIGN.md.',
+ maxSteps: 14,
+ denyBash: true,
+ });
+ logTrace('S19', 'denied-launcher-document', modelId, trace, { finishReason, text: stepTexts.join('\n') });
+ assert.notEqual(finishReason, 'length', 'a truncated response is not a completed documentation pass');
+ assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt');
+ const designWriteIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.some((p) => /(?:^|\/)DESIGN\.md$/.test(p)));
+ assert.ok(designWriteIndex >= 0, 'must still produce DESIGN.md, not stop at the refusal');
+ const documentReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && /(?:^|\/)reference\/document\.md$/.test(call.input.path));
+ assert.ok(documentReadIndex >= 0 && documentReadIndex < designWriteIndex, 'reference/document.md must actually be read before DESIGN.md is written');
+ const sourceReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && call.input.path.endsWith('index.html'));
+ assert.ok(sourceReadIndex >= 0 && sourceReadIndex < designWriteIndex, 'the incumbent source must be read before DESIGN.md is written');
+ assertLauncherDenialWarningBeforeNextTool(responseMessages);
+ assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)PRODUCT\.md$/.test(p))), 'must not rewrite PRODUCT.md');
+ } finally {
+ cleanupWorkspace(workspace);
+ }
+ });
+
it('scenario 19: denied launcher keeps planning-only work read-only without craft-floor', async () => {
const workspace = prepareWorkspace({ files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,