Skip to content

An agentic skills framework & software development methodology that works.

fix: verify skill usage via session transcript not text output

Jesse Vincent24ca8cd9d5fd
diff --git a/tests/claude-code/test-subagent-driven-development-integration.sh b/tests/claude-code/test-subagent-driven-development-integration.sh
index e528519e7..422db4328 100755
--- a/tests/claude-code/test-subagent-driven-development-integration.sh
+++ b/tests/claude-code/test-subagent-driven-development-integration.sh
@@ -161,108 +161,57 @@ echo ""
echo "Execution complete. Analyzing results..."
echo ""
-# Read the output
-OUTPUT=$(cat "$OUTPUT_FILE")
+# Find the session transcript
+# Session files are in ~/.claude/projects/-<working-dir>/<session-id>.jsonl
+WORKING_DIR_ESCAPED=$(echo "$SCRIPT_DIR/../.." | sed 's/\//-/g' | sed 's/^-//')
+SESSION_DIR="$HOME/.claude/projects/$WORKING_DIR_ESCAPED"
-# Verification tests
-FAILED=0
-
-echo "=== Verification Tests ==="
-echo ""
-
-# Test 1: Plan should be read once at the beginning
-echo "Test 1: Plan read once at beginning..."
-if echo "$OUTPUT" | grep -q "Load Plan\|read.*plan\|extract.*tasks"; then
- # Check it's near the beginning (within first 20% of output)
- total_lines=$(echo "$OUTPUT" | wc -l)
- plan_line=$(echo "$OUTPUT" | grep -n "Load Plan\|read.*plan\|extract.*tasks" | head -1 | cut -d: -f1)
- threshold=$((total_lines / 5))
-
- if [ "$plan_line" -lt "$threshold" ]; then
- echo " [PASS] Plan read early (line $plan_line of $total_lines)"
- else
- echo " [FAIL] Plan not read early (line $plan_line of $total_lines)"
- FAILED=$((FAILED + 1))
- fi
+# Find the most recent session file (created during this test run)
+SESSION_FILE=$(find "$SESSION_DIR" -name "*.jsonl" -type f -mmin -60 2>/dev/null | sort -r | head -1)
- # Should NOT re-read for each task
- read_count=$(echo "$OUTPUT" | grep -c "read.*plan" || echo "0")
- if [ "$read_count" -le 3 ]; then # Allow some mentions but not per-task
- echo " [PASS] Plan not re-read per task ($read_count mentions)"
- else
- echo " [FAIL] Plan read too many times ($read_count mentions)"
- FAILED=$((FAILED + 1))
- fi
-else
- echo " [FAIL] No evidence of plan loading"
- FAILED=$((FAILED + 1))
+if [ -z "$SESSION_FILE" ]; then
+ echo "ERROR: Could not find session transcript file"
+ echo "Looked in: $SESSION_DIR"
+ exit 1
fi
+
+echo "Analyzing session transcript: $(basename "$SESSION_FILE")"
echo ""
-# Test 2: Full task text provided to subagents
-echo "Test 2: Full task text provided to subagents..."
-if echo "$OUTPUT" | grep -q "Task Description.*Requirements\|FULL TEXT"; then
- echo " [PASS] Task text appears to be provided in prompts"
-else
- echo " [FAIL] No evidence of full task text in prompts"
- FAILED=$((FAILED + 1))
-fi
+# Verification tests
+FAILED=0
-# Should NOT make subagent read files
-if echo "$OUTPUT" | grep -q "Read.*docs/plans/implementation-plan.md" | grep -v "I'm reading\|I read"; then
- echo " [FAIL] Subagent was told to read plan file"
- FAILED=$((FAILED + 1))
-else
- echo " [PASS] Subagent not told to read plan file"
-fi
+echo "=== Verification Tests ==="
echo ""
-# Test 3: Subagents do self-review
-echo "Test 3: Subagents perform self-review..."
-if echo "$OUTPUT" | grep -qi "self-review\|self review\|reviewing my work\|look.*with fresh eyes"; then
- echo " [PASS] Self-review mentioned"
-
- # Check for self-review findings
- if echo "$OUTPUT" | grep -qi "completeness\|quality\|discipline"; then
- echo " [PASS] Self-review checklist items mentioned"
- else
- echo " [WARN] Self-review checklist items not clearly mentioned"
- fi
+# Test 1: Skill was invoked
+echo "Test 1: Skill tool invoked..."
+if grep -q '"name":"Skill".*"skill":"superpowers:subagent-driven-development"' "$SESSION_FILE"; then
+ echo " [PASS] subagent-driven-development skill was invoked"
else
- echo " [FAIL] No evidence of self-review"
+ echo " [FAIL] Skill was not invoked"
FAILED=$((FAILED + 1))
fi
echo ""
-# Test 4: Spec compliance review before code quality
-echo "Test 4: Spec compliance review before code quality..."
-spec_line=$(echo "$OUTPUT" | grep -ni "spec.*compliance.*review" | head -1 | cut -d: -f1)
-code_line=$(echo "$OUTPUT" | grep -ni "code.*quality.*review" | head -1 | cut -d: -f1)
-
-if [ -n "$spec_line" ] && [ -n "$code_line" ]; then
- if [ "$spec_line" -lt "$code_line" ]; then
- echo " [PASS] Spec compliance review before code quality (line $spec_line < $code_line)"
- else
- echo " [FAIL] Code quality before spec compliance (line $code_line < $spec_line)"
- FAILED=$((FAILED + 1))
- fi
+# Test 2: Subagents were used (Task tool)
+echo "Test 2: Subagents dispatched..."
+task_count=$(grep -c '"name":"Task"' "$SESSION_FILE" || echo "0")
+if [ "$task_count" -ge 2 ]; then
+ echo " [PASS] $task_count subagents dispatched"
else
- if [ -z "$spec_line" ]; then
- echo " [FAIL] No spec compliance review found"
- FAILED=$((FAILED + 1))
- fi
- if [ -z "$code_line" ]; then
- echo " [WARN] No code quality review found (might be acceptable if spec review caught issues)"
- fi
+ echo " [FAIL] Only $task_count subagent(s) dispatched (expected >= 2)"
+ FAILED=$((FAILED + 1))
fi
echo ""
-# Test 5: Spec reviewer is skeptical and reads code
-echo "Test 5: Spec reviewer reads code independently..."
-if echo "$OUTPUT" | grep -qi "do not trust.*report\|verify.*independently\|reading.*code\|inspecting.*implementation"; then
- echo " [PASS] Spec reviewer reads code independently"
+# Test 3: TodoWrite was used for tracking
+echo "Test 3: Task tracking..."
+todo_count=$(grep -c '"name":"TodoWrite"' "$SESSION_FILE" || echo "0")
+if [ "$todo_count" -ge 1 ]; then
+ echo " [PASS] TodoWrite used $todo_count time(s) for task tracking"
else
- echo " [FAIL] No evidence of independent code verification"
+ echo " [FAIL] TodoWrite not used"
FAILED=$((FAILED + 1))
fi
echo ""