Skip to content

An agentic skills framework & software development methodology that works.

fix(tests): stop the SDD skill test flaking on timing and prose case

Jesse Vincent0e13ad8222c6
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh
index c94cfec9e..83217cdad 100755
--- a/tests/claude-code/run-skill-tests.sh
+++ b/tests/claude-code/run-skill-tests.sh
@@ -25,7 +25,8 @@ fi
# Parse command line arguments
VERBOSE=false
SPECIFIC_TEST=""
-TIMEOUT=600 # Default 10 minute timeout per test
+TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case
+ # (test-subagent-driven-development.sh: 9 prompts x 90s each)
RUN_INTEGRATION=false
while [[ $# -gt 0 ]]; do
@@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do
echo "Options:"
echo " --verbose, -v Show verbose output"
echo " --test, -t NAME Run only the specified test"
- echo " --timeout SECONDS Set timeout per test (default: 300)"
+ echo " --timeout SECONDS Set timeout per test (default: 900)"
echo " --integration, -i Run integration tests (slow, 10-30 min)"
echo " --help, -h Show this help"
echo ""
diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.sh
index 1b5ead3b4..9e187610d 100755
--- a/tests/claude-code/test-helpers.sh
+++ b/tests/claude-code/test-helpers.sh
@@ -30,12 +30,14 @@ run_claude() {
# Check if output contains a pattern
# Usage: assert_contains "output" "pattern" "test name"
+# Matching is case-insensitive: patterns are prose keywords, and models
+# freely capitalize skill terms ("Do Not Trust", "Spec Compliance").
assert_contains() {
local output="$1"
local pattern="$2"
local test_name="${3:-test}"
- if echo "$output" | grep -q "$pattern"; then
+ if echo "$output" | grep -qi "$pattern"; then
echo " [PASS] $test_name"
return 0
else
@@ -54,7 +56,7 @@ assert_not_contains() {
local pattern="$2"
local test_name="${3:-test}"
- if echo "$output" | grep -q "$pattern"; then
+ if echo "$output" | grep -qi "$pattern"; then
echo " [FAIL] $test_name"
echo " Did not expect to find: $pattern"
echo " In output:"
@@ -74,7 +76,7 @@ assert_count() {
local expected="$3"
local test_name="${4:-test}"
- local actual=$(echo "$output" | grep -c "$pattern" || echo "0")
+ local actual=$(echo "$output" | grep -ci "$pattern" || echo "0")
if [ "$actual" -eq "$expected" ]; then
echo " [PASS] $test_name (found $actual instances)"
@@ -98,16 +100,20 @@ assert_order() {
local test_name="${4:-test}"
# Get line numbers where patterns appear
- local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1)
- local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1)
+ local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1)
+ local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1)
if [ -z "$line_a" ]; then
echo " [FAIL] $test_name: pattern A not found: $pattern_a"
+ echo " In output:"
+ echo "$output" | sed 's/^/ /'
return 1
fi
if [ -z "$line_b" ]; then
echo " [FAIL] $test_name: pattern B not found: $pattern_b"
+ echo " In output:"
+ echo "$output" | sed 's/^/ /'
return 1
fi
diff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.sh
index d8f3e10ce..151fc64d0 100755
--- a/tests/claude-code/test-subagent-driven-development.sh
+++ b/tests/claude-code/test-subagent-driven-development.sh
@@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..."
output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")
-if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
+if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
: # pass
else
exit 1
fi
-if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then
+if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then
: # pass
else
exit 1