An agentic skills framework & software development methodology that works.
fix(tests): stop the SDD skill test flaking on timing and prose case
Jesse Vincent0e13ad8222c6
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.shindex c94cfec9e..83217cdad 100755--- a/tests/claude-code/run-skill-tests.sh+++ b/tests/claude-code/run-skill-tests.sh@@ -25,7 +25,8 @@ fi# Parse command line argumentsVERBOSE=falseSPECIFIC_TEST=""-TIMEOUT=600 # Default 10 minute timeout per test+TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case+ # (test-subagent-driven-development.sh: 9 prompts x 90s each)RUN_INTEGRATION=falsewhile [[ $# -gt 0 ]]; do@@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; doecho "Options:"echo " --verbose, -v Show verbose output"echo " --test, -t NAME Run only the specified test"- echo " --timeout SECONDS Set timeout per test (default: 300)"+ echo " --timeout SECONDS Set timeout per test (default: 900)"echo " --integration, -i Run integration tests (slow, 10-30 min)"echo " --help, -h Show this help"echo ""diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.shindex 1b5ead3b4..9e187610d 100755--- a/tests/claude-code/test-helpers.sh+++ b/tests/claude-code/test-helpers.sh@@ -30,12 +30,14 @@ run_claude() {# Check if output contains a pattern# Usage: assert_contains "output" "pattern" "test name"+# Matching is case-insensitive: patterns are prose keywords, and models+# freely capitalize skill terms ("Do Not Trust", "Spec Compliance").assert_contains() {local output="$1"local pattern="$2"local test_name="${3:-test}"- if echo "$output" | grep -q "$pattern"; then+ if echo "$output" | grep -qi "$pattern"; thenecho " [PASS] $test_name"return 0else@@ -54,7 +56,7 @@ assert_not_contains() {local pattern="$2"local test_name="${3:-test}"- if echo "$output" | grep -q "$pattern"; then+ if echo "$output" | grep -qi "$pattern"; thenecho " [FAIL] $test_name"echo " Did not expect to find: $pattern"echo " In output:"@@ -74,7 +76,7 @@ assert_count() {local expected="$3"local test_name="${4:-test}"- local actual=$(echo "$output" | grep -c "$pattern" || echo "0")+ local actual=$(echo "$output" | grep -ci "$pattern" || echo "0")if [ "$actual" -eq "$expected" ]; thenecho " [PASS] $test_name (found $actual instances)"@@ -98,16 +100,20 @@ assert_order() {local test_name="${4:-test}"# Get line numbers where patterns appear- local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1)- local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1)+ local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1)+ local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1)if [ -z "$line_a" ]; thenecho " [FAIL] $test_name: pattern A not found: $pattern_a"+ echo " In output:"+ echo "$output" | sed 's/^/ /'return 1fiif [ -z "$line_b" ]; thenecho " [FAIL] $test_name: pattern B not found: $pattern_b"+ echo " In output:"+ echo "$output" | sed 's/^/ /'return 1fidiff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.shindex d8f3e10ce..151fc64d0 100755--- a/tests/claude-code/test-subagent-driven-development.sh+++ b/tests/claude-code/test-subagent-driven-development.sh@@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..."output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")-if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then+if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then: # passelseexit 1fi-if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then+if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then: # passelseexit 1