Skip to content

An agentic skills framework & software development methodology that works.

Add integration test for subagent-driven-development

Jesse Vincentfa946ae46527
diff --git a/tests/claude-code/README.md b/tests/claude-code/README.md
index 177f791a1..e53647ba1 100644
--- a/tests/claude-code/README.md
+++ b/tests/claude-code/README.md
@@ -13,11 +13,16 @@ This test suite verifies that skills are loaded correctly and Claude follows the
## Running Tests
-### Run all tests:
+### Run all fast tests (recommended):
```bash
./run-skill-tests.sh
```
+### Run integration tests (slow, 10-30 minutes):
+```bash
+./run-skill-tests.sh --integration
+```
+
### Run specific test:
```bash
./run-skill-tests.sh --test test-subagent-driven-development.sh
@@ -28,9 +33,9 @@ This test suite verifies that skills are loaded correctly and Claude follows the
./run-skill-tests.sh --verbose
```
-### Set custom timeout (default 300s):
+### Set custom timeout:
```bash
-./run-skill-tests.sh --timeout 600
+./run-skill-tests.sh --timeout 1800 # 30 minutes for integration tests
```
## Test Structure
@@ -75,15 +80,40 @@ echo "=== All tests passed ==="
## Current Tests
-### test-subagent-driven-development.sh
-Tests the updated subagent-driven-development workflow:
-- Skill loading
+### Fast Tests (run by default)
+
+#### test-subagent-driven-development.sh
+Tests skill content and requirements (~2 minutes):
+- Skill loading and accessibility
- Workflow ordering (spec compliance before code quality)
-- Self-review requirements
-- Plan reading efficiency (read once)
-- Spec compliance reviewer skepticism
-- Review loops
-- Task context provision (full text, not file reading)
+- Self-review requirements documented
+- Plan reading efficiency documented
+- Spec compliance reviewer skepticism documented
+- Review loops documented
+- Task context provision documented
+
+### Integration Tests (use --integration flag)
+
+#### test-subagent-driven-development-integration.sh
+Full workflow execution test (~10-30 minutes):
+- Creates real test project with Node.js setup
+- Creates implementation plan with 2 tasks
+- Executes plan using subagent-driven-development
+- Verifies actual behaviors:
+ - Plan read once at start (not per task)
+ - Full task text provided in subagent prompts
+ - Subagents perform self-review before reporting
+ - Spec compliance review happens before code quality
+ - Spec reviewer reads code independently
+ - Working implementation is produced
+ - Tests pass
+ - Proper git commits created
+
+**What it tests:**
+- The workflow actually works end-to-end
+- Our improvements are actually applied
+- Subagents follow the skill correctly
+- Final code is functional and tested
## Adding New Tests
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh
index a50d71d69..3e339fd3d 100755
--- a/tests/claude-code/run-skill-tests.sh
+++ b/tests/claude-code/run-skill-tests.sh
@@ -26,6 +26,7 @@ fi
VERBOSE=false
SPECIFIC_TEST=""
TIMEOUT=300 # Default 5 minute timeout per test
+RUN_INTEGRATION=false
while [[ $# -gt 0 ]]; do
case $1 in
@@ -41,17 +42,25 @@ while [[ $# -gt 0 ]]; do
TIMEOUT="$2"
shift 2
;;
+ --integration|-i)
+ RUN_INTEGRATION=true
+ shift
+ ;;
--help|-h)
echo "Usage: $0 [options]"
echo ""
echo "Options:"
- echo " --verbose, -v Show verbose output"
- echo " --test, -t NAME Run only the specified test"
- echo " --timeout SECONDS Set timeout per test (default: 300)"
- echo " --help, -h Show this help"
+ echo " --verbose, -v Show verbose output"
+ echo " --test, -t NAME Run only the specified test"
+ echo " --timeout SECONDS Set timeout per test (default: 300)"
+ echo " --integration, -i Run integration tests (slow, 10-30 min)"
+ echo " --help, -h Show this help"
echo ""
echo "Tests:"
- echo " test-subagent-driven-development.sh Test subagent-driven-development workflow"
+ echo " test-subagent-driven-development.sh Test skill loading and requirements"
+ echo ""
+ echo "Integration Tests (use --integration):"
+ echo " test-subagent-driven-development-integration.sh Full workflow execution"
exit 0
;;
*)
@@ -62,11 +71,21 @@ while [[ $# -gt 0 ]]; do
esac
done
-# List of skill tests to run
+# List of skill tests to run (fast unit tests)
tests=(
"test-subagent-driven-development.sh"
)
+# Integration tests (slow, full execution)
+integration_tests=(
+ "test-subagent-driven-development-integration.sh"
+)
+
+# Add integration tests if requested
+if [ "$RUN_INTEGRATION" = true ]; then
+ tests+=("${integration_tests[@]}")
+fi
+
# Filter to specific test if requested
if [ -n "$SPECIFIC_TEST" ]; then
tests=("$SPECIFIC_TEST")
@@ -153,6 +172,12 @@ echo " Failed: $failed"
echo " Skipped: $skipped"
echo ""
+if [ "$RUN_INTEGRATION" = false ] && [ ${#integration_tests[@]} -gt 0 ]; then
+ echo "Note: Integration tests were not run (they take 10-30 minutes)."
+ echo "Use --integration flag to run full workflow execution tests."
+ echo ""
+fi
+
if [ $failed -gt 0 ]; then
echo "STATUS: FAILED"
exit 1
diff --git a/tests/claude-code/test-subagent-driven-development-integration.sh b/tests/claude-code/test-subagent-driven-development-integration.sh
new file mode 100755
index 000000000..691d36be4
--- /dev/null
+++ b/tests/claude-code/test-subagent-driven-development-integration.sh
@@ -0,0 +1,338 @@
+#!/usr/bin/env bash
+# Integration Test: subagent-driven-development workflow
+# Actually executes a plan and verifies the new workflow behaviors
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+echo "========================================"
+echo " Integration Test: subagent-driven-development"
+echo "========================================"
+echo ""
+echo "This test executes a real plan using the skill and verifies:"
+echo " 1. Plan is read once (not per task)"
+echo " 2. Full task text provided to subagents"
+echo " 3. Subagents perform self-review"
+echo " 4. Spec compliance review before code quality"
+echo " 5. Review loops when issues found"
+echo " 6. Spec reviewer reads code independently"
+echo ""
+echo "WARNING: This test may take 10-30 minutes to complete."
+echo ""
+
+# Create test project
+TEST_PROJECT=$(create_test_project)
+echo "Test project: $TEST_PROJECT"
+
+# Trap to cleanup
+trap "cleanup_test_project $TEST_PROJECT" EXIT
+
+# Set up minimal Node.js project
+cd "$TEST_PROJECT"
+
+cat > package.json <<'EOF'
+{
+ "name": "test-project",
+ "version": "1.0.0",
+ "type": "module",
+ "scripts": {
+ "test": "node --test"
+ }
+}
+EOF
+
+mkdir -p src test docs/plans
+
+# Create a simple implementation plan
+cat > docs/plans/implementation-plan.md <<'EOF'
+# Test Implementation Plan
+
+This is a minimal plan to test the subagent-driven-development workflow.
+
+## Task 1: Create Add Function
+
+Create a function that adds two numbers.
+
+**File:** `src/math.js`
+
+**Requirements:**
+- Function named `add`
+- Takes two parameters: `a` and `b`
+- Returns the sum of `a` and `b`
+- Export the function
+
+**Implementation:**
+```javascript
+export function add(a, b) {
+ return a + b;
+}
+```
+
+**Tests:** Create `test/math.test.js` that verifies:
+- `add(2, 3)` returns `5`
+- `add(0, 0)` returns `0`
+- `add(-1, 1)` returns `0`
+
+**Verification:** `npm test`
+
+## Task 2: Create Multiply Function
+
+Create a function that multiplies two numbers.
+
+**File:** `src/math.js` (add to existing file)
+
+**Requirements:**
+- Function named `multiply`
+- Takes two parameters: `a` and `b`
+- Returns the product of `a` and `b`
+- Export the function
+- DO NOT add any extra features (like power, divide, etc.)
+
+**Implementation:**
+```javascript
+export function multiply(a, b) {
+ return a * b;
+}
+```
+
+**Tests:** Add to `test/math.test.js`:
+- `multiply(2, 3)` returns `6`
+- `multiply(0, 5)` returns `0`
+- `multiply(-2, 3)` returns `-6`
+
+**Verification:** `npm test`
+EOF
+
+# Initialize git repo
+git init --quiet
+git config user.email "[email protected]"
+git config user.name "Test User"
+git add .
+git commit -m "Initial commit" --quiet
+
+echo ""
+echo "Project setup complete. Starting execution..."
+echo ""
+
+# Run Claude with subagent-driven-development
+# Capture full output to analyze
+OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"
+
+# Note: We use a longer timeout since this is integration testing
+timeout 1800 claude -p "$(cat <<'PROMPT'
+I want you to execute the implementation plan at docs/plans/implementation-plan.md using the subagent-driven-development skill.
+
+IMPORTANT: Follow the skill exactly. I will be verifying that you:
+1. Read the plan once at the beginning
+2. Provide full task text to subagents (don't make them read files)
+3. Ensure subagents do self-review before reporting
+4. Run spec compliance review before code quality review
+5. Use review loops when issues are found
+
+Begin now. Execute the plan.
+PROMPT
+)" > "$OUTPUT_FILE" 2>&1 || {
+ echo "EXECUTION FAILED"
+ cat "$OUTPUT_FILE"
+ exit 1
+}
+
+echo ""
+echo "Execution complete. Analyzing results..."
+echo ""
+
+# Read the output
+OUTPUT=$(cat "$OUTPUT_FILE")
+
+# Verification tests
+FAILED=0
+
+echo "=== Verification Tests ==="
+echo ""
+
+# Test 1: Plan should be read once at the beginning
+echo "Test 1: Plan read once at beginning..."
+if echo "$OUTPUT" | grep -q "Load Plan\|read.*plan\|extract.*tasks"; then
+ # Check it's near the beginning (within first 20% of output)
+ total_lines=$(echo "$OUTPUT" | wc -l)
+ plan_line=$(echo "$OUTPUT" | grep -n "Load Plan\|read.*plan\|extract.*tasks" | head -1 | cut -d: -f1)
+ threshold=$((total_lines / 5))
+
+ if [ "$plan_line" -lt "$threshold" ]; then
+ echo " [PASS] Plan read early (line $plan_line of $total_lines)"
+ else
+ echo " [FAIL] Plan not read early (line $plan_line of $total_lines)"
+ FAILED=$((FAILED + 1))
+ fi
+
+ # Should NOT re-read for each task
+ read_count=$(echo "$OUTPUT" | grep -c "read.*plan" || echo "0")
+ if [ "$read_count" -le 3 ]; then # Allow some mentions but not per-task
+ echo " [PASS] Plan not re-read per task ($read_count mentions)"
+ else
+ echo " [FAIL] Plan read too many times ($read_count mentions)"
+ FAILED=$((FAILED + 1))
+ fi
+else
+ echo " [FAIL] No evidence of plan loading"
+ FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 2: Full task text provided to subagents
+echo "Test 2: Full task text provided to subagents..."
+if echo "$OUTPUT" | grep -q "Task Description.*Requirements\|FULL TEXT"; then
+ echo " [PASS] Task text appears to be provided in prompts"
+else
+ echo " [FAIL] No evidence of full task text in prompts"
+ FAILED=$((FAILED + 1))
+fi
+
+# Should NOT make subagent read files
+if echo "$OUTPUT" | grep -q "Read.*docs/plans/implementation-plan.md" | grep -v "I'm reading\|I read"; then
+ echo " [FAIL] Subagent was told to read plan file"
+ FAILED=$((FAILED + 1))
+else
+ echo " [PASS] Subagent not told to read plan file"
+fi
+echo ""
+
+# Test 3: Subagents do self-review
+echo "Test 3: Subagents perform self-review..."
+if echo "$OUTPUT" | grep -qi "self-review\|self review\|reviewing my work\|look.*with fresh eyes"; then
+ echo " [PASS] Self-review mentioned"
+
+ # Check for self-review findings
+ if echo "$OUTPUT" | grep -qi "completeness\|quality\|discipline"; then
+ echo " [PASS] Self-review checklist items mentioned"
+ else
+ echo " [WARN] Self-review checklist items not clearly mentioned"
+ fi
+else
+ echo " [FAIL] No evidence of self-review"
+ FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 4: Spec compliance review before code quality
+echo "Test 4: Spec compliance review before code quality..."
+spec_line=$(echo "$OUTPUT" | grep -ni "spec.*compliance.*review" | head -1 | cut -d: -f1)
+code_line=$(echo "$OUTPUT" | grep -ni "code.*quality.*review" | head -1 | cut -d: -f1)
+
+if [ -n "$spec_line" ] && [ -n "$code_line" ]; then
+ if [ "$spec_line" -lt "$code_line" ]; then
+ echo " [PASS] Spec compliance review before code quality (line $spec_line < $code_line)"
+ else
+ echo " [FAIL] Code quality before spec compliance (line $code_line < $spec_line)"
+ FAILED=$((FAILED + 1))
+ fi
+else
+ if [ -z "$spec_line" ]; then
+ echo " [FAIL] No spec compliance review found"
+ FAILED=$((FAILED + 1))
+ fi
+ if [ -z "$code_line" ]; then
+ echo " [WARN] No code quality review found (might be acceptable if spec review caught issues)"
+ fi
+fi
+echo ""
+
+# Test 5: Spec reviewer is skeptical and reads code
+echo "Test 5: Spec reviewer reads code independently..."
+if echo "$OUTPUT" | grep -qi "do not trust.*report\|verify.*independently\|reading.*code\|inspecting.*implementation"; then
+ echo " [PASS] Spec reviewer reads code independently"
+else
+ echo " [FAIL] No evidence of independent code verification"
+ FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 6: Implementation actually works
+echo "Test 6: Implementation verification..."
+if [ -f "$TEST_PROJECT/src/math.js" ]; then
+ echo " [PASS] src/math.js created"
+
+ if grep -q "export function add" "$TEST_PROJECT/src/math.js"; then
+ echo " [PASS] add function exists"
+ else
+ echo " [FAIL] add function missing"
+ FAILED=$((FAILED + 1))
+ fi
+
+ if grep -q "export function multiply" "$TEST_PROJECT/src/math.js"; then
+ echo " [PASS] multiply function exists"
+ else
+ echo " [FAIL] multiply function missing"
+ FAILED=$((FAILED + 1))
+ fi
+else
+ echo " [FAIL] src/math.js not created"
+ FAILED=$((FAILED + 1))
+fi
+
+if [ -f "$TEST_PROJECT/test/math.test.js" ]; then
+ echo " [PASS] test/math.test.js created"
+else
+ echo " [FAIL] test/math.test.js not created"
+ FAILED=$((FAILED + 1))
+fi
+
+# Try running tests
+if cd "$TEST_PROJECT" && npm test > test-output.txt 2>&1; then
+ echo " [PASS] Tests pass"
+else
+ echo " [FAIL] Tests failed"
+ cat test-output.txt
+ FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 7: Git commits show proper workflow
+echo "Test 7: Git commit history..."
+commit_count=$(git -C "$TEST_PROJECT" log --oneline | wc -l)
+if [ "$commit_count" -gt 2 ]; then # Initial + at least 2 task commits
+ echo " [PASS] Multiple commits created ($commit_count total)"
+else
+ echo " [FAIL] Too few commits ($commit_count, expected >2)"
+ FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 8: Check for extra features (spec compliance should catch)
+echo "Test 8: No extra features added (spec compliance)..."
+if grep -q "export function divide\|export function power\|export function subtract" "$TEST_PROJECT/src/math.js" 2>/dev/null; then
+ echo " [WARN] Extra features found (spec review should have caught this)"
+ # Not failing on this as it tests reviewer effectiveness
+else
+ echo " [PASS] No extra features added"
+fi
+echo ""
+
+# Summary
+echo "========================================"
+echo " Test Summary"
+echo "========================================"
+echo ""
+
+if [ $FAILED -eq 0 ]; then
+ echo "STATUS: PASSED"
+ echo "All verification tests passed!"
+ echo ""
+ echo "The subagent-driven-development skill correctly:"
+ echo " ✓ Reads plan once at start"
+ echo " ✓ Provides full task text to subagents"
+ echo " ✓ Enforces self-review"
+ echo " ✓ Runs spec compliance before code quality"
+ echo " ✓ Spec reviewer verifies independently"
+ echo " ✓ Produces working implementation"
+ exit 0
+else
+ echo "STATUS: FAILED"
+ echo "Failed $FAILED verification tests"
+ echo ""
+ echo "Output saved to: $OUTPUT_FILE"
+ echo ""
+ echo "Review the output to see what went wrong."
+ exit 1
+fi