An agentic skills framework & software development methodology that works.
Add integration test for subagent-driven-development
Jesse Vincentfa946ae46527
diff --git a/tests/claude-code/README.md b/tests/claude-code/README.mdindex 177f791a1..e53647ba1 100644--- a/tests/claude-code/README.md+++ b/tests/claude-code/README.md@@ -13,11 +13,16 @@ This test suite verifies that skills are loaded correctly and Claude follows the## Running Tests-### Run all tests:+### Run all fast tests (recommended):```bash./run-skill-tests.sh```+### Run integration tests (slow, 10-30 minutes):+```bash+./run-skill-tests.sh --integration+```+### Run specific test:```bash./run-skill-tests.sh --test test-subagent-driven-development.sh@@ -28,9 +33,9 @@ This test suite verifies that skills are loaded correctly and Claude follows the./run-skill-tests.sh --verbose```-### Set custom timeout (default 300s):+### Set custom timeout:```bash-./run-skill-tests.sh --timeout 600+./run-skill-tests.sh --timeout 1800 # 30 minutes for integration tests```## Test Structure@@ -75,15 +80,40 @@ echo "=== All tests passed ==="## Current Tests-### test-subagent-driven-development.sh-Tests the updated subagent-driven-development workflow:-- Skill loading+### Fast Tests (run by default)++#### test-subagent-driven-development.sh+Tests skill content and requirements (~2 minutes):+- Skill loading and accessibility- Workflow ordering (spec compliance before code quality)-- Self-review requirements-- Plan reading efficiency (read once)-- Spec compliance reviewer skepticism-- Review loops-- Task context provision (full text, not file reading)+- Self-review requirements documented+- Plan reading efficiency documented+- Spec compliance reviewer skepticism documented+- Review loops documented+- Task context provision documented++### Integration Tests (use --integration flag)++#### test-subagent-driven-development-integration.sh+Full workflow execution test (~10-30 minutes):+- Creates real test project with Node.js setup+- Creates implementation plan with 2 tasks+- Executes plan using subagent-driven-development+- Verifies actual behaviors:+ - Plan read once at start (not per task)+ - Full task text provided in subagent prompts+ - Subagents perform self-review before reporting+ - Spec compliance review happens before code quality+ - Spec reviewer reads code independently+ - Working implementation is produced+ - Tests pass+ - Proper git commits created++**What it tests:**+- The workflow actually works end-to-end+- Our improvements are actually applied+- Subagents follow the skill correctly+- Final code is functional and tested## Adding New Testsdiff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.shindex a50d71d69..3e339fd3d 100755--- a/tests/claude-code/run-skill-tests.sh+++ b/tests/claude-code/run-skill-tests.sh@@ -26,6 +26,7 @@ fiVERBOSE=falseSPECIFIC_TEST=""TIMEOUT=300 # Default 5 minute timeout per test+RUN_INTEGRATION=falsewhile [[ $# -gt 0 ]]; docase $1 in@@ -41,17 +42,25 @@ while [[ $# -gt 0 ]]; doTIMEOUT="$2"shift 2;;+ --integration|-i)+ RUN_INTEGRATION=true+ shift+ ;;--help|-h)echo "Usage: $0 [options]"echo ""echo "Options:"- echo " --verbose, -v Show verbose output"- echo " --test, -t NAME Run only the specified test"- echo " --timeout SECONDS Set timeout per test (default: 300)"- echo " --help, -h Show this help"+ echo " --verbose, -v Show verbose output"+ echo " --test, -t NAME Run only the specified test"+ echo " --timeout SECONDS Set timeout per test (default: 300)"+ echo " --integration, -i Run integration tests (slow, 10-30 min)"+ echo " --help, -h Show this help"echo ""echo "Tests:"- echo " test-subagent-driven-development.sh Test subagent-driven-development workflow"+ echo " test-subagent-driven-development.sh Test skill loading and requirements"+ echo ""+ echo "Integration Tests (use --integration):"+ echo " test-subagent-driven-development-integration.sh Full workflow execution"exit 0;;*)@@ -62,11 +71,21 @@ while [[ $# -gt 0 ]]; doesacdone-# List of skill tests to run+# List of skill tests to run (fast unit tests)tests=("test-subagent-driven-development.sh")+# Integration tests (slow, full execution)+integration_tests=(+ "test-subagent-driven-development-integration.sh"+)++# Add integration tests if requested+if [ "$RUN_INTEGRATION" = true ]; then+ tests+=("${integration_tests[@]}")+fi+# Filter to specific test if requestedif [ -n "$SPECIFIC_TEST" ]; thentests=("$SPECIFIC_TEST")@@ -153,6 +172,12 @@ echo " Failed: $failed"echo " Skipped: $skipped"echo ""+if [ "$RUN_INTEGRATION" = false ] && [ ${#integration_tests[@]} -gt 0 ]; then+ echo "Note: Integration tests were not run (they take 10-30 minutes)."+ echo "Use --integration flag to run full workflow execution tests."+ echo ""+fi+if [ $failed -gt 0 ]; thenecho "STATUS: FAILED"exit 1diff --git a/tests/claude-code/test-subagent-driven-development-integration.sh b/tests/claude-code/test-subagent-driven-development-integration.shnew file mode 100755index 000000000..691d36be4--- /dev/null+++ b/tests/claude-code/test-subagent-driven-development-integration.sh@@ -0,0 +1,338 @@+#!/usr/bin/env bash+# Integration Test: subagent-driven-development workflow+# Actually executes a plan and verifies the new workflow behaviors+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+source "$SCRIPT_DIR/test-helpers.sh"++echo "========================================"+echo " Integration Test: subagent-driven-development"+echo "========================================"+echo ""+echo "This test executes a real plan using the skill and verifies:"+echo " 1. Plan is read once (not per task)"+echo " 2. Full task text provided to subagents"+echo " 3. Subagents perform self-review"+echo " 4. Spec compliance review before code quality"+echo " 5. Review loops when issues found"+echo " 6. Spec reviewer reads code independently"+echo ""+echo "WARNING: This test may take 10-30 minutes to complete."+echo ""++# Create test project+TEST_PROJECT=$(create_test_project)+echo "Test project: $TEST_PROJECT"++# Trap to cleanup+trap "cleanup_test_project $TEST_PROJECT" EXIT++# Set up minimal Node.js project+cd "$TEST_PROJECT"++cat > package.json <<'EOF'+{+ "name": "test-project",+ "version": "1.0.0",+ "type": "module",+ "scripts": {+ "test": "node --test"+ }+}+EOF++mkdir -p src test docs/plans++# Create a simple implementation plan+cat > docs/plans/implementation-plan.md <<'EOF'+# Test Implementation Plan++This is a minimal plan to test the subagent-driven-development workflow.++## Task 1: Create Add Function++Create a function that adds two numbers.++**File:** `src/math.js`++**Requirements:**+- Function named `add`+- Takes two parameters: `a` and `b`+- Returns the sum of `a` and `b`+- Export the function++**Implementation:**+```javascript+export function add(a, b) {+ return a + b;+}+```++**Tests:** Create `test/math.test.js` that verifies:+- `add(2, 3)` returns `5`+- `add(0, 0)` returns `0`+- `add(-1, 1)` returns `0`++**Verification:** `npm test`++## Task 2: Create Multiply Function++Create a function that multiplies two numbers.++**File:** `src/math.js` (add to existing file)++**Requirements:**+- Function named `multiply`+- Takes two parameters: `a` and `b`+- Returns the product of `a` and `b`+- Export the function+- DO NOT add any extra features (like power, divide, etc.)++**Implementation:**+```javascript+export function multiply(a, b) {+ return a * b;+}+```++**Tests:** Add to `test/math.test.js`:+- `multiply(2, 3)` returns `6`+- `multiply(0, 5)` returns `0`+- `multiply(-2, 3)` returns `-6`++**Verification:** `npm test`+EOF++# Initialize git repo+git init --quiet+git config user.email "[email protected]"+git config user.name "Test User"+git add .+git commit -m "Initial commit" --quiet++echo ""+echo "Project setup complete. Starting execution..."+echo ""++# Run Claude with subagent-driven-development+# Capture full output to analyze+OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"++# Note: We use a longer timeout since this is integration testing+timeout 1800 claude -p "$(cat <<'PROMPT'+I want you to execute the implementation plan at docs/plans/implementation-plan.md using the subagent-driven-development skill.++IMPORTANT: Follow the skill exactly. I will be verifying that you:+1. Read the plan once at the beginning+2. Provide full task text to subagents (don't make them read files)+3. Ensure subagents do self-review before reporting+4. Run spec compliance review before code quality review+5. Use review loops when issues are found++Begin now. Execute the plan.+PROMPT+)" > "$OUTPUT_FILE" 2>&1 || {+ echo "EXECUTION FAILED"+ cat "$OUTPUT_FILE"+ exit 1+}++echo ""+echo "Execution complete. Analyzing results..."+echo ""++# Read the output+OUTPUT=$(cat "$OUTPUT_FILE")++# Verification tests+FAILED=0++echo "=== Verification Tests ==="+echo ""++# Test 1: Plan should be read once at the beginning+echo "Test 1: Plan read once at beginning..."+if echo "$OUTPUT" | grep -q "Load Plan\|read.*plan\|extract.*tasks"; then+ # Check it's near the beginning (within first 20% of output)+ total_lines=$(echo "$OUTPUT" | wc -l)+ plan_line=$(echo "$OUTPUT" | grep -n "Load Plan\|read.*plan\|extract.*tasks" | head -1 | cut -d: -f1)+ threshold=$((total_lines / 5))++ if [ "$plan_line" -lt "$threshold" ]; then+ echo " [PASS] Plan read early (line $plan_line of $total_lines)"+ else+ echo " [FAIL] Plan not read early (line $plan_line of $total_lines)"+ FAILED=$((FAILED + 1))+ fi++ # Should NOT re-read for each task+ read_count=$(echo "$OUTPUT" | grep -c "read.*plan" || echo "0")+ if [ "$read_count" -le 3 ]; then # Allow some mentions but not per-task+ echo " [PASS] Plan not re-read per task ($read_count mentions)"+ else+ echo " [FAIL] Plan read too many times ($read_count mentions)"+ FAILED=$((FAILED + 1))+ fi+else+ echo " [FAIL] No evidence of plan loading"+ FAILED=$((FAILED + 1))+fi+echo ""++# Test 2: Full task text provided to subagents+echo "Test 2: Full task text provided to subagents..."+if echo "$OUTPUT" | grep -q "Task Description.*Requirements\|FULL TEXT"; then+ echo " [PASS] Task text appears to be provided in prompts"+else+ echo " [FAIL] No evidence of full task text in prompts"+ FAILED=$((FAILED + 1))+fi++# Should NOT make subagent read files+if echo "$OUTPUT" | grep -q "Read.*docs/plans/implementation-plan.md" | grep -v "I'm reading\|I read"; then+ echo " [FAIL] Subagent was told to read plan file"+ FAILED=$((FAILED + 1))+else+ echo " [PASS] Subagent not told to read plan file"+fi+echo ""++# Test 3: Subagents do self-review+echo "Test 3: Subagents perform self-review..."+if echo "$OUTPUT" | grep -qi "self-review\|self review\|reviewing my work\|look.*with fresh eyes"; then+ echo " [PASS] Self-review mentioned"++ # Check for self-review findings+ if echo "$OUTPUT" | grep -qi "completeness\|quality\|discipline"; then+ echo " [PASS] Self-review checklist items mentioned"+ else+ echo " [WARN] Self-review checklist items not clearly mentioned"+ fi+else+ echo " [FAIL] No evidence of self-review"+ FAILED=$((FAILED + 1))+fi+echo ""++# Test 4: Spec compliance review before code quality+echo "Test 4: Spec compliance review before code quality..."+spec_line=$(echo "$OUTPUT" | grep -ni "spec.*compliance.*review" | head -1 | cut -d: -f1)+code_line=$(echo "$OUTPUT" | grep -ni "code.*quality.*review" | head -1 | cut -d: -f1)++if [ -n "$spec_line" ] && [ -n "$code_line" ]; then+ if [ "$spec_line" -lt "$code_line" ]; then+ echo " [PASS] Spec compliance review before code quality (line $spec_line < $code_line)"+ else+ echo " [FAIL] Code quality before spec compliance (line $code_line < $spec_line)"+ FAILED=$((FAILED + 1))+ fi+else+ if [ -z "$spec_line" ]; then+ echo " [FAIL] No spec compliance review found"+ FAILED=$((FAILED + 1))+ fi+ if [ -z "$code_line" ]; then+ echo " [WARN] No code quality review found (might be acceptable if spec review caught issues)"+ fi+fi+echo ""++# Test 5: Spec reviewer is skeptical and reads code+echo "Test 5: Spec reviewer reads code independently..."+if echo "$OUTPUT" | grep -qi "do not trust.*report\|verify.*independently\|reading.*code\|inspecting.*implementation"; then+ echo " [PASS] Spec reviewer reads code independently"+else+ echo " [FAIL] No evidence of independent code verification"+ FAILED=$((FAILED + 1))+fi+echo ""++# Test 6: Implementation actually works+echo "Test 6: Implementation verification..."+if [ -f "$TEST_PROJECT/src/math.js" ]; then+ echo " [PASS] src/math.js created"++ if grep -q "export function add" "$TEST_PROJECT/src/math.js"; then+ echo " [PASS] add function exists"+ else+ echo " [FAIL] add function missing"+ FAILED=$((FAILED + 1))+ fi++ if grep -q "export function multiply" "$TEST_PROJECT/src/math.js"; then+ echo " [PASS] multiply function exists"+ else+ echo " [FAIL] multiply function missing"+ FAILED=$((FAILED + 1))+ fi+else+ echo " [FAIL] src/math.js not created"+ FAILED=$((FAILED + 1))+fi++if [ -f "$TEST_PROJECT/test/math.test.js" ]; then+ echo " [PASS] test/math.test.js created"+else+ echo " [FAIL] test/math.test.js not created"+ FAILED=$((FAILED + 1))+fi++# Try running tests+if cd "$TEST_PROJECT" && npm test > test-output.txt 2>&1; then+ echo " [PASS] Tests pass"+else+ echo " [FAIL] Tests failed"+ cat test-output.txt+ FAILED=$((FAILED + 1))+fi+echo ""++# Test 7: Git commits show proper workflow+echo "Test 7: Git commit history..."+commit_count=$(git -C "$TEST_PROJECT" log --oneline | wc -l)+if [ "$commit_count" -gt 2 ]; then # Initial + at least 2 task commits+ echo " [PASS] Multiple commits created ($commit_count total)"+else+ echo " [FAIL] Too few commits ($commit_count, expected >2)"+ FAILED=$((FAILED + 1))+fi+echo ""++# Test 8: Check for extra features (spec compliance should catch)+echo "Test 8: No extra features added (spec compliance)..."+if grep -q "export function divide\|export function power\|export function subtract" "$TEST_PROJECT/src/math.js" 2>/dev/null; then+ echo " [WARN] Extra features found (spec review should have caught this)"+ # Not failing on this as it tests reviewer effectiveness+else+ echo " [PASS] No extra features added"+fi+echo ""++# Summary+echo "========================================"+echo " Test Summary"+echo "========================================"+echo ""++if [ $FAILED -eq 0 ]; then+ echo "STATUS: PASSED"+ echo "All verification tests passed!"+ echo ""+ echo "The subagent-driven-development skill correctly:"+ echo " ✓ Reads plan once at start"+ echo " ✓ Provides full task text to subagents"+ echo " ✓ Enforces self-review"+ echo " ✓ Runs spec compliance before code quality"+ echo " ✓ Spec reviewer verifies independently"+ echo " ✓ Produces working implementation"+ exit 0+else+ echo "STATUS: FAILED"+ echo "Failed $FAILED verification tests"+ echo ""+ echo "Output saved to: $OUTPUT_FILE"+ echo ""+ echo "Review the output to see what went wrong."+ exit 1+fi