Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(ci): run AI code review across a configurable list of models (#4667)

LearningCircuitd830cbbaea41
diff --git a/.github/scripts/combine-ai-reviews.sh b/.github/scripts/combine-ai-reviews.sh
new file mode 100755
index 0000000000..bc03be63ca
--- /dev/null
+++ b/.github/scripts/combine-ai-reviews.sh
@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+#
+# Combine multiple per-model AI code reviews into a single sticky PR comment.
+#
+# This is the pure-assembly half of the AI Code Reviewer workflow
+# (.github/workflows/ai-code-reviewer.yml), split out so the non-trivial parts —
+# header/footer stripping, "Reviewer N" section assembly, label union, and the
+# pass/fail aggregation — can be unit-tested without any network or GitHub API
+# access. The workflow runs the models (network) and writes each reviewer's raw
+# stdout to resp_<i>.json and its exit code to code_<i>; this script turns those
+# into the comment body, label set, decision, and success count.
+#
+# Usage:
+# combine-ai-reviews.sh <work_dir> <model_1> [<model_2> ...]
+#
+# Reads, for i in 0..N-1 (N = number of model args):
+# <work_dir>/resp_<i>.json - raw stdout from ai-reviewer.sh for reviewer i
+# <work_dir>/code_<i> - that reviewer's integer exit code
+# <work_dir>/err_<i>.log - that reviewer's stderr (only shown in DEBUG)
+# The model names are used only for the per-reviewer log-group labels; the
+# posted comment stays anonymized ("Reviewer 1", "Reviewer 2", ...).
+#
+# Environment:
+# HEAD_SHA - head commit sha for the "Last reviewed at commit" line (opt)
+# DEBUG_MODE - "true" to echo raw responses + stderr to this script's stderr
+#
+# Writes (into <work_dir>):
+# comment_body.md - the combined sticky-comment body
+# labels.txt - deduped labels, one per line (may be empty)
+# decision.txt - "pass" or "fail"
+# success_count.txt - number of reviewers that produced a usable review
+#
+# Logs ::group:: sections and debug output to stderr (kept off stdout so the
+# output files are the only contract). Exit status is 0 even when every
+# reviewer failed — the caller inspects success_count.txt and decides whether
+# to fail the workflow. A non-zero exit means a usage error only.
+
+set -euo pipefail
+
+WORK_DIR="${1:-}"
+if [ -z "$WORK_DIR" ] || [ ! -d "$WORK_DIR" ]; then
+ echo "usage: $0 <work_dir> <model> [<model> ...]" >&2
+ exit 2
+fi
+shift
+MODELS=("$@")
+if [ "${#MODELS[@]}" -eq 0 ]; then
+ echo "error: at least one model name is required" >&2
+ exit 2
+fi
+
+DEBUG_MODE="${DEBUG_MODE:-false}"
+HEAD_SHA="${HEAD_SHA:-}"
+
+COMBINED_REVIEW=""
+ALL_LABELS=""
+ANY_FAIL="false"
+SUCCESS_COUNT=0
+
+# Assemble the combined comment in model order. Reviewers are anonymized as
+# "Reviewer N" — the model -> number mapping appears only in the (debug) logs
+# below, so readers judge the feedback, not the model.
+for i in "${!MODELS[@]}"; do
+ REVIEWER_NUM=$((i + 1))
+ EXIT_CODE="$(cat "$WORK_DIR/code_$i" 2>/dev/null || echo 1)"
+ # A missing/empty/garbage exit-code file counts as a failed reviewer rather
+ # than tripping the numeric comparison below (which would print "integer
+ # expression expected" and then fall through to wrongly treating the reviewer
+ # as a success).
+ [[ "$EXIT_CODE" =~ ^[0-9]+$ ]] || EXIT_CODE=1
+ AI_RESPONSE="$(cat "$WORK_DIR/resp_$i.json" 2>/dev/null || echo "")"
+ echo "::group::Reviewer #$REVIEWER_NUM (${MODELS[i]})" >&2
+
+ if [ "$DEBUG_MODE" = "true" ]; then
+ {
+ echo "=== RAW AI RESPONSE (reviewer #$REVIEWER_NUM, exit $EXIT_CODE) ==="
+ echo "$AI_RESPONSE"
+ echo "--- stderr ---"
+ cat "$WORK_DIR/err_$i.log" 2>/dev/null || true
+ echo "=== END RAW AI RESPONSE ==="
+ } >&2
+ fi
+
+ # One model failing (hard error, empty, or non-JSON) must not sink the others:
+ # record a short failure note and keep going. The caller fails the workflow
+ # only if *every* reviewer failed (success_count == 0). A reviewer that errors
+ # out never sets a verdict, so it can never flip the aggregate decision to
+ # "fail" — only a model that returns a valid "fail" verdict does. Sensitive raw
+ # output is never written to the comment.
+ # Require a JSON *object*: a bare `jq .` accepts any valid JSON, including a
+ # top-level string/array/number that a refusing or misbehaving model might
+ # emit ("I refuse to review this"). That would pass the gate and then crash
+ # the `.review` access below under `set -e`, sinking *every* reviewer and
+ # writing no output at all. `type == "object"` rejects non-objects so they
+ # degrade to a per-reviewer failure note, preserving the others.
+ if [ "$EXIT_CODE" -ne 0 ] || [ -z "$AI_RESPONSE" ] || ! echo "$AI_RESPONSE" | jq -e 'type == "object"' >/dev/null 2>&1; then
+ echo "⚠️ Reviewer #$REVIEWER_NUM did not return a usable review (exit $EXIT_CODE)" >&2
+ REVIEW_BODY="_This reviewer could not complete its review (see workflow logs for details)._"
+ LABELS=""
+ else
+ REVIEW_RAW=$(echo "$AI_RESPONSE" | jq -r '.review // "No review provided"')
+ # Trim whitespace with tr (the verdict is a single token). The previous
+ # `| xargs` aborted the entire script on an unbalanced quote in the value,
+ # which—inside this per-reviewer loop—would sink every reviewer.
+ DECISION=$(echo "$AI_RESPONSE" | jq -r '.fail_pass_workflow // "uncertain"' | tr -d '[:space:]')
+ LABELS=$(echo "$AI_RESPONSE" | jq -r '.labels_added[]? // empty')
+ SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
+ [ "$DECISION" = "fail" ] && ANY_FAIL="true"
+
+ # Each review opens with a "## AI Code Review" H2 and closes with the
+ # singular Friendly AI Reviewer footer. Under a "### 👤 Reviewer N"
+ # subheading the H2 is redundant and the footer would repeat once per model,
+ # so strip both here; a single (plural) footer is added to the combined
+ # comment below. The footer match keys on the stable "Review by [Friendly AI
+ # Reviewer]" prefix and the \s*\z tail absorbs trailing whitespace; if the
+ # upstream markers ever change and aren't found, the text is left intact
+ # (nothing is dropped).
+ REVIEW_BODY=$(printf '%s' "$REVIEW_RAW" \
+ | perl -0pe 's/\A\s*##\s*AI Code Review\s*\n+//' \
+ | perl -0pe 's/\n*---\s*\n\*Review by \[Friendly AI Reviewer\][^\n]*\n?\s*\z//s')
+ fi
+
+ # Lead each section with a blank line so the "### Reviewer N" heading always
+ # renders (command substitution strips trailing newlines, so the separator
+ # must precede the heading, not follow the previous body).
+ COMBINED_REVIEW=$(printf '%s\n\n### 👤 Reviewer %s\n\n%s' "$COMBINED_REVIEW" "$REVIEWER_NUM" "$REVIEW_BODY")
+ [ -n "$LABELS" ] && ALL_LABELS=$(printf '%s\n%s' "$ALL_LABELS" "$LABELS")
+ echo "::endgroup::" >&2
+done
+
+# Aggregate labels (union across reviewers) and the pass/fail decision (request
+# changes if ANY reviewer did).
+LABELS=$(printf '%s\n' "$ALL_LABELS" | sed '/^[[:space:]]*$/d' | sort -u)
+if [ "$ANY_FAIL" = "true" ]; then
+ DECISION="fail"
+else
+ DECISION="pass"
+fi
+
+REVIEWER_WORD="reviewers"
+[ "${#MODELS[@]}" -eq 1 ] && REVIEWER_WORD="reviewer"
+STICKY_MARKER="<!-- ai-code-review:sticky -->"
+FOOTER="*Reviews by [Friendly AI Reviewer](https://github.com/LearningCircuit/Friendly-AI-Reviewer) - made with ❤️*"
+# COMBINED_REVIEW already starts with "\n\n", so it follows the title directly
+# (no extra newline) to yield exactly one blank line before "### Reviewer 1".
+# The backticks in `%s` are intentional literal Markdown (inline-code the SHA),
+# not a command substitution — single quotes keep them literal on purpose.
+# shellcheck disable=SC2016
+COMMENT_BODY=$(printf '%s\n\n## 🤖 AI Code Review (%s %s)%s\n\n---\n%s\n\n_Last reviewed at commit `%s`_' \
+ "$STICKY_MARKER" "${#MODELS[@]}" "$REVIEWER_WORD" "$COMBINED_REVIEW" "$FOOTER" "${HEAD_SHA:0:7}")
+
+printf '%s' "$COMMENT_BODY" > "$WORK_DIR/comment_body.md"
+printf '%s' "$LABELS" > "$WORK_DIR/labels.txt"
+printf '%s' "$DECISION" > "$WORK_DIR/decision.txt"
+printf '%s' "$SUCCESS_COUNT" > "$WORK_DIR/success_count.txt"
diff --git a/.github/workflows/ai-code-reviewer.yml b/.github/workflows/ai-code-reviewer.yml
index b427db4416..a38b9942fb 100644
--- a/.github/workflows/ai-code-reviewer.yml
+++ b/.github/workflows/ai-code-reviewer.yml
@@ -66,7 +66,12 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- AI_MODEL: ${{ vars.AI_MODEL || 'moonshotai/kimi-k2-thinking' }}
+ # Comma-separated list of models. Each model produces one independent
+ # review, presented as "Reviewer 1", "Reviewer 2", ... in a single
+ # combined comment. Falls back to the shared AI_MODEL (also used by
+ # release.yml, which must stay a single model) when unset, so a
+ # one-entry list reproduces the original single-review behavior.
+ AI_REVIEW_MODELS: ${{ vars.AI_REVIEW_MODELS || vars.AI_MODEL || 'moonshotai/kimi-k2-thinking' }}
AI_TEMPERATURE: ${{ vars.AI_TEMPERATURE || '0.1' }}
AI_MAX_TOKENS: ${{ vars.AI_MAX_TOKENS || '64000' }}
MAX_DIFF_SIZE: ${{ vars.MAX_DIFF_SIZE || '800000' }}
@@ -78,80 +83,78 @@ jobs:
DEBUG_MODE: ${{ vars.DEBUG_MODE || 'false' }}
EVENT_ACTION: ${{ github.event.action }}
run: |
- # Run the AI reviewer and save response to file
echo "Running AI code review..."
- # Temporarily disable errexit to capture errors properly
- set +e
- AI_RESPONSE=$(bash ai-reviewer.sh < diff.txt)
- EXIT_CODE=$?
- set -e
-
- # Output raw response for debugging (only in debug mode)
- if [ "$DEBUG_MODE" = "true" ]; then
- echo "=== RAW AI RESPONSE FOR DEBUG ==="
- echo "Exit code: $EXIT_CODE"
- echo "$AI_RESPONSE"
- echo "=== END RAW AI RESPONSE ==="
- fi
-
- # Check if the script failed
- if [ $EXIT_CODE -ne 0 ]; then
- echo "❌ AI reviewer script failed with exit code $EXIT_CODE"
- echo ""
- echo "Error output:"
- echo "$AI_RESPONSE"
- exit 1
- fi
-
- # Check if AI_RESPONSE is empty
- if [ -z "$AI_RESPONSE" ]; then
- echo "❌ AI response is empty"
- exit 1
- fi
-
- # Parse JSON response
- if ! echo "$AI_RESPONSE" | jq . >/dev/null 2>&1; then
- echo "⚠️ AI response is not valid JSON. Cannot process review."
-
- # Log raw response for debugging (only in debug mode)
- if [ "$DEBUG_MODE" = "true" ]; then
- echo "=== DEBUG: Raw AI Response (first 2000 chars) ==="
- echo "$AI_RESPONSE" | head -c 2000
- echo ""
- echo "=== END DEBUG ==="
-
- # Try to extract JSON from thinking model response
- if echo "$AI_RESPONSE" | grep -q '"content"'; then
- echo "=== DEBUG: Attempting to extract content from response ==="
- EXTRACTED_CONTENT=$(echo "$AI_RESPONSE" | jq -r '.choices[0].message.content // empty' 2>/dev/null || echo "")
- if [ -n "$EXTRACTED_CONTENT" ]; then
- echo "Extracted content (first 1000 chars):"
- echo "$EXTRACTED_CONTENT" | head -c 1000
- echo ""
- fi
- fi
- fi
-
- # Don't post raw response as it may contain sensitive data
- echo "AI response could not be processed. Please check the workflow logs for debugging information (enable DEBUG_MODE to see details)." | gh pr comment "$PR_NUMBER" --repo "$REPO_FULL_NAME" -F -
+ # Parse the comma-separated model list. Each entry becomes one
+ # independent reviewer ("Reviewer 1", "Reviewer 2", ...). A single
+ # entry reproduces the original one-review behavior.
+ # `read` returns non-zero at EOF; the here-string always appends a
+ # trailing newline so it returns 0 here, but `|| true` keeps a future
+ # edge case from tripping `set -e`. An empty/whitespace AI_REVIEW_MODELS
+ # yields only empty entries, which the filter below drops, correctly
+ # falling through to the "No models configured" error.
+ IFS=',' read -ra RAW_MODELS <<< "$AI_REVIEW_MODELS" || true
+ MODELS=()
+ for m in "${RAW_MODELS[@]}"; do
+ # Trim surrounding whitespace with parameter expansion. (xargs would
+ # choke on a model id containing a quote and word-collapse internal
+ # spaces — model ids shouldn't have either, but this is robust.)
+ m="${m#"${m%%[![:space:]]*}"}" # strip leading whitespace
+ m="${m%"${m##*[![:space:]]}"}" # strip trailing whitespace
+ [ -n "$m" ] && MODELS+=("$m")
+ done
+ if [ "${#MODELS[@]}" -eq 0 ]; then
+ echo "❌ No models configured in AI_REVIEW_MODELS"
exit 1
fi
+ echo "Configured ${#MODELS[@]} reviewer model(s)"
+
+ # Launch every model concurrently so the job's wall-clock is the
+ # slowest single review, not the sum of all of them. Each reviewer
+ # writes its JSON response and its diagnostics to its own index-keyed
+ # files, so the "Reviewer N" numbering stays stable and the parallel
+ # output never interleaves. AI_MODEL is overridden per call; all other
+ # settings (temperature, token limits, diff size, context fetching)
+ # are shared.
+ pids=()
+ for i in "${!MODELS[@]}"; do
+ AI_MODEL="${MODELS[i]}" bash ai-reviewer.sh < diff.txt \
+ > "resp_$i.json" 2> "err_$i.log" &
+ pids[i]=$!
+ done
- # Extract fields from JSON
- REVIEW=$(echo "$AI_RESPONSE" | jq -r '.review // "No review provided"')
- DECISION=$(echo "$AI_RESPONSE" | jq -r '.fail_pass_workflow // "uncertain"' | tr -d '\n\r' | xargs)
- LABELS=$(echo "$AI_RESPONSE" | jq -r '.labels_added[]? // empty')
+ # Record each reviewer's exit code to code_<i> (a failing model must
+ # not abort the others, so disable errexit around wait).
+ for i in "${!MODELS[@]}"; do
+ set +e
+ wait "${pids[i]}"
+ echo $? > "code_$i"
+ set -e
+ done
- # Post (or update) the review as a single sticky PR comment. On each
+ # Assemble the combined comment with the extracted, unit-tested helper
+ # (.github/scripts/combine-ai-reviews.sh). It reads resp_<i>.json /
+ # code_<i> from the working dir and writes the comment body, label set,
+ # decision, and success count back out. Keeping the assembly in a
+ # script is what lets tests/ci/test_combine_ai_reviews.py cover the
+ # parsing, footer-stripping, label union, and pass/fail logic without
+ # any network or GitHub API access.
+ bash .github/scripts/combine-ai-reviews.sh "$PWD" "${MODELS[@]}"
+ COMMENT_BODY="$(cat comment_body.md)"
+ LABELS="$(cat labels.txt)"
+ DECISION="$(cat decision.txt)"
+ SUCCESS_COUNT="$(cat success_count.txt)"
+
+ # Post (or update) the reviews as a single sticky PR comment. On each
# push we edit the same comment in place instead of stacking a new one
# per push. The hidden marker identifies our comment specifically, so
# we never touch comments from humans or other bots. (Review comments
# made before this change lack the marker and are left as-is.)
# A commit-sha line lets reviewers tell which push the (edited-in-place)
# comment reflects, since GitHub only shows a vague "edited" marker.
+ # This literal MUST match the marker the helper script prepends to the
+ # comment body.
STICKY_MARKER="<!-- ai-code-review:sticky -->"
- COMMENT_BODY=$(printf "%s\n\n%s\n\n_Last reviewed at commit \`%s\`_" "$STICKY_MARKER" "$REVIEW" "${HEAD_SHA:0:7}")
# STICKY_MARKER is interpolated into the jq program below, so it must
# stay free of `"` and `\`. gh api's --jq takes no --arg, so we can't
# pass it as a jq variable; keeping the marker a literal constant is
@@ -170,9 +173,12 @@ jobs:
printf '%s' "$COMMENT_BODY" | gh pr comment "$PR_NUMBER" --repo "$REPO_FULL_NAME" -F -
fi
- # Add labels to PR
+ # Add labels to PR. Iterate line-by-line (labels.txt is one label per
+ # line) rather than word-splitting, so a multi-word label such as
+ # "good first issue" is treated as a single label, not three.
echo "Labels to add: $LABELS"
- for label in $LABELS; do
+ while IFS= read -r label; do
+ [ -n "$label" ] || continue
echo "Adding label: $label"
if gh label create "$label" --color "0366d6" --description "Auto-created by AI reviewer" 2>/dev/null; then
echo "Created new label: $label"
@@ -184,7 +190,7 @@ jobs:
else
echo "Failed to add label: $label"
fi
- done
+ done <<< "$LABELS"
# Handle workflow decision (after posting review and labels)
echo "AI decision: $DECISION"
@@ -204,7 +210,15 @@ jobs:
fi
fi
- echo "✅ AI review completed successfully"
+ # If every reviewer failed, the posted comment carries only failure
+ # notes — surface that as a red workflow run rather than a silent pass,
+ # matching the original single-review behavior on an unusable response.
+ if [ "$SUCCESS_COUNT" -eq 0 ]; then
+ echo "❌ All ${#MODELS[@]} reviewer(s) failed to produce a usable review."
+ exit 1
+ fi
+
+ echo "✅ AI review completed successfully ($SUCCESS_COUNT/${#MODELS[@]} reviewers)"
- name: Fail Workflow if Requested
if: env.FAIL_ON_REQUESTED_CHANGES == 'true' && env.DECISION == 'fail'
@@ -217,6 +231,9 @@ jobs:
if: always()
run: |
rm -f diff.txt
+ # Per-reviewer response/log/exit-code files from the parallel fan-out,
+ # plus the assembly helper's output files.
+ rm -f resp_*.json err_*.log code_* comment_body.md labels.txt decision.txt success_count.txt
# Only remove ai_response.txt if it exists (only created in debug mode)
if [ -f "ai_response.txt" ]; then
rm -f ai_response.txt
diff --git a/Dockerfile b/Dockerfile
index fe52f3cc64..91d1dc9d22 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -134,6 +134,13 @@ ARG DEBIAN_FRONTEND=noninteractive
# corresponding upgrade in the builder-base stage (top of file).
RUN apt-get update -o Acquire::Retries=3 && apt-get upgrade -y -o Acquire::Retries=3 \
&& apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+ # jq + perl: required by tests/ci/test_combine_ai_reviews.py, which exercises
+ # the AI-review combine helper (it uses jq and perl). Without them those tests
+ # silently skipped in this pytest lane, leaving the script with no enforced
+ # coverage. perl is normally present via perl-base, but install it explicitly
+ # so the test's fail-loud-in-CI tool check can never false-alarm.
+ jq \
+ perl \
xauth \
xvfb \
# Dependencies for Chromium
diff --git a/docs/AI_Code_Reviewer_Setup.md b/docs/AI_Code_Reviewer_Setup.md
index 541695077c..1a08091cc5 100644
--- a/docs/AI_Code_Reviewer_Setup.md
+++ b/docs/AI_Code_Reviewer_Setup.md
@@ -33,13 +33,32 @@ The review is posted as a single comprehensive comment on your pull request.
### 3. Configure Workflow (Optional)
-The workflow is pre-configured with sensible defaults, but you can customize it by editing `.github/workflows/ai-code-reviewer.yml`:
+The workflow is pre-configured with sensible defaults. You can override them **without editing the workflow** by adding GitHub **repository variables** (not secrets):
-- **AI_MODEL**: Change the AI model (see [OpenRouter models](https://openrouter.ai/models))
+1. Go to your repository → **Settings** → **Secrets and variables** → **Actions**
+2. Open the **Variables** tab → **New repository variable**
+3. Add any of:
+
+- **AI_REVIEW_MODELS**: Comma-separated list of models. Each model runs the reviewer independently, so you get one review per model in a single combined comment (see [Multiple reviewers](#multiple-reviewers)). When unset, falls back to `AI_MODEL`, then to the built-in default.
+- **AI_MODEL**: A single model (see [OpenRouter models](https://openrouter.ai/models)). Used by the reviewer when `AI_REVIEW_MODELS` is unset, and also by the release-notes workflow. Default: `moonshotai/kimi-k2-thinking`.
- **AI_TEMPERATURE**: Adjust randomness (default: `0.1` for consistent reviews)
-- **AI_MAX_TOKENS**: Maximum response length (default: `2000`)
+- **AI_MAX_TOKENS**: Maximum response length (default: `64000`)
- **MAX_DIFF_SIZE**: Maximum diff size in bytes (default: `800000` / 800KB)
+#### Multiple reviewers
+
+Set **AI_REVIEW_MODELS** to a comma-separated list to have several models review the same diff. For example:
+
+```
+minimax/minimax-m3, z-ai/glm-5.2
+```
+
+Notes:
+- Separate models with **commas** (spaces around them are fine and trimmed). A space-separated value is treated as a *single* model name and will fail.
+- Each model is shown anonymously as **Reviewer 1**, **Reviewer 2**, … (the model→reviewer mapping appears only in the workflow logs), so readers judge the feedback rather than the model.
+- All reviews are collected into one sticky PR comment. Labels are unioned across reviewers; if `FAIL_ON_REQUESTED_CHANGES` is enabled, the workflow fails when **any** reviewer requests changes. This is deliberately "any" rather than a majority vote — a single model catching a real blocker shouldn't be outvoted — and it only takes effect when you opt into `FAIL_ON_REQUESTED_CHANGES` (off by default). A reviewer that errors out (rather than returning a verdict) never counts as a "fail"; it just shows a note in its section.
+- Models run **in parallel**, so the job's wall-clock is the slowest single review rather than the sum. Each model still makes its own set of GitHub API calls for context, so a very long list means many concurrent calls; two or three models are well within limits.
+
## Usage
### Triggering AI Reviews
diff --git a/tests/ci/test_combine_ai_reviews.py b/tests/ci/test_combine_ai_reviews.py
new file mode 100644
index 0000000000..f276e744e0
--- /dev/null
+++ b/tests/ci/test_combine_ai_reviews.py
@@ -0,0 +1,426 @@
+"""Unit tests for .github/scripts/combine-ai-reviews.sh.
+
+The AI Code Reviewer workflow runs N models over a PR diff (network) and writes
+each reviewer's raw stdout to resp_<i>.json and exit code to code_<i>. The
+combine-ai-reviews.sh helper turns those files into the single sticky-comment
+body, the deduped label set, the pass/fail decision, and the success count.
+
+These tests exercise that pure-assembly logic directly with canned fixtures —
+no network, no GitHub API — so the non-trivial bash/jq/perl parts (comma-split
+already happens in the workflow; here it's header/footer stripping, anonymized
+"Reviewer N" section assembly, label union, and decision aggregation) are
+covered in CI.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+SCRIPT = (
+ Path(__file__).resolve().parents[2]
+ / ".github"
+ / "scripts"
+ / "combine-ai-reviews.sh"
+)
+
+_MISSING_TOOLS = [t for t in ("bash", "jq", "perl") if shutil.which(t) is None]
+_IN_CI = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS"))
+
+# Skip ONLY when the tools are missing locally. In CI they must be present
+# (jq+perl are installed into the test image), so we deliberately do NOT skip
+# there: if they're ever missing in CI these tests run and fail loudly instead
+# of silently skipping and leaving combine-ai-reviews.sh with zero enforced
+# coverage. Failing per-test (vs a module-level raise) keeps the failure
+# localized to this lane — see test_required_tools_present_in_ci for the
+# explicit, clearly-messaged guard.
+pytestmark = pytest.mark.skipif(
+ bool(_MISSING_TOOLS) and not _IN_CI,
+ reason=f"combine-ai-reviews.sh requires {_MISSING_TOOLS} (missing locally)",
+)
+
+
+def test_required_tools_present_in_ci():
+ """The test image must ship the tools the helper needs. If this fails in CI,
+ install the missing tool in the Dockerfile `ldr-test` stage — otherwise the
+ rest of this suite would be a silent no-op."""
+ if _IN_CI:
+ assert not _MISSING_TOOLS, (
+ f"CI test image is missing {_MISSING_TOOLS}; add it to the Dockerfile "
+ "ldr-test stage so combine-ai-reviews.sh keeps real CI coverage."
+ )
+
+
+FOOTER_SINGULAR = (
+ "*Review by [Friendly AI Reviewer]"
+ "(https://github.com/LearningCircuit/Friendly-AI-Reviewer) - made with ❤️*"
+)
+
+
+def make_response(body, decision="pass", labels=None, *, trailing=""):
+ """Build a realistic ai-reviewer.sh JSON response (review + verdict + labels)."""
+ review = f"## AI Code Review\n\n{body}\n\n---\n{FOOTER_SINGULAR}{trailing}"
+ return json.dumps(
+ {
+ "review": review,
+ "fail_pass_workflow": decision,
+ "labels_added": labels or [],
+ }
+ )
+
+
+def run_combine(work_dir, reviewers, *, head_sha="abc1234def", debug=False):
+ """Write fixtures for each reviewer and run the helper; return its outputs.
+
+ `reviewers` is a list of dicts: {"code": int, "resp": str}. Model names are
+ deliberately distinctive so tests can assert they never leak into the
+ (anonymized) comment body.
+ """
+ models = []
+ for i, rv in enumerate(reviewers):
+ (work_dir / f"code_{i}").write_text(str(rv["code"]))
+ (work_dir / f"resp_{i}.json").write_text(rv["resp"])
+ (work_dir / f"err_{i}.log").write_text("")
+ models.append(f"secret-model-{i}-DO-NOT-LEAK")
+
+ env = {"HEAD_SHA": head_sha, "DEBUG_MODE": "true" if debug else "false"}
+ result = subprocess.run(
+ ["bash", str(SCRIPT), str(work_dir), *models],
+ capture_output=True,
+ text=True,
+ env={**env, "PATH": __import__("os").environ["PATH"]},
+ )
+ assert result.returncode == 0, (
+ f"script failed ({result.returncode}): {result.stderr}"
+ )
+ return {
+ "models": models,
+ "comment": (work_dir / "comment_body.md").read_text(),
+ "labels": (work_dir / "labels.txt").read_text(),
+ "decision": (work_dir / "decision.txt").read_text(),
+ "success_count": (work_dir / "success_count.txt").read_text(),
+ "stderr": result.stderr,
+ }
+
+
+def test_two_reviewers_pass_and_fail(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {
+ "code": 0,
+ "resp": make_response(
+ "All good here.", "pass", ["enhancement"]
+ ),
+ },
+ {
+ "code": 0,
+ "resp": make_response("Race condition.", "fail", ["bug"]),
+ },
+ ],
+ )
+ c = out["comment"]
+ # Both anonymized sections present, in order.
+ assert "### 👤 Reviewer 1" in c
+ assert "### 👤 Reviewer 2" in c
+ assert c.index("Reviewer 1") < c.index("Reviewer 2")
+ # Bodies carried through.
+ assert "All good here." in c
+ assert "Race condition." in c
+ # Aggregation: any fail -> fail; labels unioned + sorted; both succeeded.
+ assert out["decision"] == "fail"
+ assert out["labels"].split() == ["bug", "enhancement"]
+ assert out["success_count"] == "2"
+
+
+def test_header_and_per_review_footer_are_stripped(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [{"code": 0, "resp": make_response("Body one.", "pass")}],
+ )
+ c = out["comment"]
+ # The per-review "## AI Code Review" H2 is removed (only the combined H2 with
+ # the reviewer count remains), and the singular per-review footer is gone.
+ assert "## 🤖 AI Code Review (1 reviewer)" in c
+ assert "## AI Code Review\n" not in c
+ assert "Review by [Friendly AI Reviewer]" not in c
+ # Exactly one combined (plural) footer is appended.
+ assert c.count("Reviews by [Friendly AI Reviewer]") == 1
+
+
+def test_footer_strip_tolerates_trailing_whitespace(tmp_path):
+ # LLMs sometimes append trailing newlines/spaces after the footer.
+ out = run_combine(
+ tmp_path,
+ [
+ {
+ "code": 0,
+ "resp": make_response("Body.", "pass", trailing="\n\n \n"),
+ }
+ ],
+ )
+ assert "Review by [Friendly AI Reviewer]" not in out["comment"]
+ # ...but the actual review body must survive (guard against an over-greedy
+ # strip that nukes everything).
+ assert "Body." in out["comment"]
+ assert out["success_count"] == "1"
+
+
+def test_failed_reviewer_degrades_without_sinking_others(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {
+ "code": 0,
+ "resp": make_response("Good review.", "pass", ["enhancement"]),
+ },
+ {"code": 1, "resp": ""}, # hard error: non-zero exit, empty stdout
+ {"code": 0, "resp": "this is not json"}, # garbage stdout
+ ],
+ )
+ c = out["comment"]
+ assert "Good review." in c
+ assert c.count("could not complete its review") == 2
+ # Two of three reviewers failed; the survivor still counts.
+ assert out["success_count"] == "1"
+ # A reviewer that errors out never flips the decision to fail.
+ assert out["decision"] == "pass"
+ # Failed reviewers contribute no labels.
+ assert out["labels"].split() == ["enhancement"]
+
+
+def test_all_reviewers_failed(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [{"code": 1, "resp": ""}, {"code": 1, "resp": ""}],
+ )
+ assert out["success_count"] == "0"
+ assert out["decision"] == "pass"
+ assert out["comment"].count("could not complete its review") == 2
+
+
+def test_labels_are_unioned_and_deduped(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {
+ "code": 0,
+ "resp": make_response("a", "pass", ["bug", "security"]),
+ },
+ {
+ "code": 0,
+ "resp": make_response("b", "pass", ["bug", "enhancement"]),
+ },
+ ],
+ )
+ assert out["labels"].split() == ["bug", "enhancement", "security"]
+
+
+def test_model_names_never_leak_into_comment(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {"code": 0, "resp": make_response("x", "pass")},
+ {"code": 0, "resp": make_response("y", "fail")},
+ ],
+ )
+ for model in out["models"]:
+ assert model not in out["comment"]
+
+
+def test_single_reviewer_uses_singular_wording_and_sticky_marker(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [{"code": 0, "resp": make_response("Solo.", "pass")}],
+ )
+ c = out["comment"]
+ assert "(1 reviewer)" in c
+ assert "(1 reviewers)" not in c
+ # Sticky marker must lead the body (the workflow searches for it verbatim).
+ assert c.startswith("<!-- ai-code-review:sticky -->")
+ # Commit sha line rendered with real backticks (not escaped backslashes).
+ assert "_Last reviewed at commit `abc1234`_" in c
+
+
+def test_blank_line_before_horizontal_rule(tmp_path):
+ # A "text\n---" with no blank line is a setext H2 in Markdown; ensure the
+ # divider before the footer is preceded by a blank line.
+ out = run_combine(
+ tmp_path,
+ [{"code": 0, "resp": make_response("Body.", "pass")}],
+ )
+ assert "\n\n---\n" in out["comment"]
+
+
+# --- Regression tests for resilience (one bad reviewer must not sink others) ---
+
+
+def test_valid_json_non_object_does_not_sink_other_reviewers(tmp_path):
+ # A model can return valid JSON that is a bare string/array (e.g. a refusal),
+ # not the expected object. That must degrade to a per-reviewer failure note,
+ # not crash the whole combine and lose the good review. (Regression: a bare
+ # `jq .` gate accepted non-objects, then `.review` errored under set -e.)
+ out = run_combine(
+ tmp_path,
+ [
+ {
+ "code": 0,
+ "resp": '"I refuse to review this"',
+ }, # valid JSON string
+ {
+ "code": 0,
+ "resp": '["also", "not", "an", "object"]',
+ }, # JSON array
+ {"code": 0, "resp": make_response("Real review.", "pass", ["bug"])},
+ ],
+ )
+ c = out["comment"]
+ assert "Real review." in c
+ assert c.count("could not complete its review") == 2
+ assert out["success_count"] == "1"
+ assert out["labels"].split() == ["bug"]
+
+
+def test_weird_verdict_string_does_not_crash(tmp_path):
+ # A verdict value containing a quote previously aborted the script via
+ # `| xargs` (unmatched quote). It must now be handled gracefully.
+ resp = json.dumps(
+ {
+ "review": f"## AI Code Review\n\nBody.\n\n---\n{FOOTER_SINGULAR}",
+ "fail_pass_workflow": "it's complicated",
+ "labels_added": [],
+ }
+ )
+ out = run_combine(tmp_path, [{"code": 0, "resp": resp}])
+ assert out["success_count"] == "1"
+ # A non-"fail" verdict (even a weird one) does not request changes.
+ assert out["decision"] == "pass"
+
+
+def test_empty_exit_code_file_counts_as_failed(tmp_path):
+ # An empty/unreadable code_<i> must be treated as a failed reviewer, not
+ # silently counted as a success (regression: `[ "" -ne 0 ]` errored and fell
+ # through to the success path).
+ out = run_combine(
+ tmp_path,
+ [{"code": "", "resp": make_response("Should be ignored.", "pass")}],
+ )
+ assert out["success_count"] == "0"
+ assert "could not complete its review" in out["comment"]
+ assert "Should be ignored." not in out["comment"]
+
+
+def test_non_numeric_exit_code_counts_as_failed(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [{"code": "boom", "resp": make_response("Ignored.", "pass")}],
+ )
+ assert out["success_count"] == "0"
+ assert "integer expression" not in out["stderr"]
+
+
+def test_mixed_pass_fail_error_aggregation(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {"code": 0, "resp": make_response("ok", "pass", ["enhancement"])},
+ {"code": 0, "resp": make_response("blocker", "fail", ["bug"])},
+ {"code": 1, "resp": ""}, # hard error
+ ],
+ )
+ # Only usable reviews count; the one real "fail" verdict drives the decision;
+ # labels union only from successful reviewers; exactly one failure note.
+ assert out["success_count"] == "2"
+ assert out["decision"] == "fail"
+ assert out["labels"].split() == ["bug", "enhancement"]
+ assert out["comment"].count("could not complete its review") == 1
+
+
+def test_valid_json_missing_review_field(tmp_path):
+ resp = json.dumps({"fail_pass_workflow": "pass", "labels_added": []})
+ out = run_combine(tmp_path, [{"code": 0, "resp": resp}])
+ # Object with no .review still counts as a (degenerate) success and renders
+ # the jq fallback text rather than crashing.
+ assert out["success_count"] == "1"
+ assert "No review provided" in out["comment"]
+
+
+def test_debug_mode_does_not_change_comment_and_keeps_model_in_stderr(tmp_path):
+ reviewers = [{"code": 0, "resp": make_response("Body.", "pass")}]
+ # Separate dirs so the two runs' fixtures/outputs don't collide.
+ plain_dir = tmp_path / "plain"
+ dbg_dir = tmp_path / "debug"
+ plain_dir.mkdir()
+ dbg_dir.mkdir()
+ plain = run_combine(plain_dir, reviewers)
+ dbg = run_combine(dbg_dir, reviewers, debug=True)
+ # The posted comment must be byte-identical regardless of DEBUG_MODE — debug
+ # content (raw response, model name) goes only to stderr, never the comment.
+ assert dbg["comment"] == plain["comment"]
+ # DEBUG must not perturb any of the machine-read outputs either.
+ assert dbg["decision"] == plain["decision"]
+ assert dbg["labels"] == plain["labels"]
+ assert dbg["success_count"] == plain["success_count"]
+ assert "RAW AI RESPONSE" in dbg["stderr"]
+ assert "RAW AI RESPONSE" not in plain["stderr"]
+ # Model name leaks only into stderr logs (anonymization), never the comment.
+ assert dbg["models"][0] in dbg["stderr"]
+ assert dbg["models"][0] not in dbg["comment"]
+
+
+def test_four_reviewers_keep_order(tmp_path):
+ out = run_combine(
+ tmp_path,
+ [
+ {"code": 0, "resp": make_response(f"body-{n}", "pass")}
+ for n in range(4)
+ ],
+ )
+ c = out["comment"]
+ positions = [c.index(f"Reviewer {n}") for n in range(1, 5)]
+ assert positions == sorted(positions)
+ for n in range(4):
+ assert f"body-{n}" in c
+
+
+def test_usage_errors_exit_nonzero(tmp_path):
+ # The script's only non-zero exits: bad work_dir and zero model args.
+ env = {**os.environ}
+ no_dir = subprocess.run(
+ ["bash", str(SCRIPT), str(tmp_path / "does-not-exist"), "m"],
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert no_dir.returncode == 2
+ no_models = subprocess.run(
+ ["bash", str(SCRIPT), str(tmp_path)],
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert no_models.returncode == 2
+
+
+def test_exit_code_with_trailing_newline(tmp_path):
+ # The workflow writes exit codes via `echo $? > code_i`, i.e. WITH a trailing
+ # newline ("0\n"), whereas the other tests use the bare "0" form. Confirm the
+ # real workflow shape is handled (command substitution strips the newline).
+ (tmp_path / "code_0").write_text("0\n")
+ (tmp_path / "resp_0.json").write_text(make_response("ok", "pass"))
+ (tmp_path / "err_0.log").write_text("")
+ result = subprocess.run(
+ ["bash", str(SCRIPT), str(tmp_path), "m"],
+ capture_output=True,
+ text=True,
+ env={
+ "HEAD_SHA": "abc1234def",
+ "DEBUG_MODE": "false",
+ "PATH": os.environ["PATH"],
+ },
+ )
+ assert result.returncode == 0, result.stderr
+ assert (tmp_path / "success_count.txt").read_text() == "1"