~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(ci): run AI code review across a configurable list of models (#4667)
LearningCircuitd830cbbaea41
diff --git a/.github/scripts/combine-ai-reviews.sh b/.github/scripts/combine-ai-reviews.shnew file mode 100755index 0000000000..bc03be63ca--- /dev/null+++ b/.github/scripts/combine-ai-reviews.sh@@ -0,0 +1,155 @@+#!/usr/bin/env bash+#+# Combine multiple per-model AI code reviews into a single sticky PR comment.+#+# This is the pure-assembly half of the AI Code Reviewer workflow+# (.github/workflows/ai-code-reviewer.yml), split out so the non-trivial parts —+# header/footer stripping, "Reviewer N" section assembly, label union, and the+# pass/fail aggregation — can be unit-tested without any network or GitHub API+# access. The workflow runs the models (network) and writes each reviewer's raw+# stdout to resp_<i>.json and its exit code to code_<i>; this script turns those+# into the comment body, label set, decision, and success count.+#+# Usage:+# combine-ai-reviews.sh <work_dir> <model_1> [<model_2> ...]+#+# Reads, for i in 0..N-1 (N = number of model args):+# <work_dir>/resp_<i>.json - raw stdout from ai-reviewer.sh for reviewer i+# <work_dir>/code_<i> - that reviewer's integer exit code+# <work_dir>/err_<i>.log - that reviewer's stderr (only shown in DEBUG)+# The model names are used only for the per-reviewer log-group labels; the+# posted comment stays anonymized ("Reviewer 1", "Reviewer 2", ...).+#+# Environment:+# HEAD_SHA - head commit sha for the "Last reviewed at commit" line (opt)+# DEBUG_MODE - "true" to echo raw responses + stderr to this script's stderr+#+# Writes (into <work_dir>):+# comment_body.md - the combined sticky-comment body+# labels.txt - deduped labels, one per line (may be empty)+# decision.txt - "pass" or "fail"+# success_count.txt - number of reviewers that produced a usable review+#+# Logs ::group:: sections and debug output to stderr (kept off stdout so the+# output files are the only contract). Exit status is 0 even when every+# reviewer failed — the caller inspects success_count.txt and decides whether+# to fail the workflow. A non-zero exit means a usage error only.++set -euo pipefail++WORK_DIR="${1:-}"+if [ -z "$WORK_DIR" ] || [ ! -d "$WORK_DIR" ]; then+ echo "usage: $0 <work_dir> <model> [<model> ...]" >&2+ exit 2+fi+shift+MODELS=("$@")+if [ "${#MODELS[@]}" -eq 0 ]; then+ echo "error: at least one model name is required" >&2+ exit 2+fi++DEBUG_MODE="${DEBUG_MODE:-false}"+HEAD_SHA="${HEAD_SHA:-}"++COMBINED_REVIEW=""+ALL_LABELS=""+ANY_FAIL="false"+SUCCESS_COUNT=0++# Assemble the combined comment in model order. Reviewers are anonymized as+# "Reviewer N" — the model -> number mapping appears only in the (debug) logs+# below, so readers judge the feedback, not the model.+for i in "${!MODELS[@]}"; do+ REVIEWER_NUM=$((i + 1))+ EXIT_CODE="$(cat "$WORK_DIR/code_$i" 2>/dev/null || echo 1)"+ # A missing/empty/garbage exit-code file counts as a failed reviewer rather+ # than tripping the numeric comparison below (which would print "integer+ # expression expected" and then fall through to wrongly treating the reviewer+ # as a success).+ [[ "$EXIT_CODE" =~ ^[0-9]+$ ]] || EXIT_CODE=1+ AI_RESPONSE="$(cat "$WORK_DIR/resp_$i.json" 2>/dev/null || echo "")"+ echo "::group::Reviewer #$REVIEWER_NUM (${MODELS[i]})" >&2++ if [ "$DEBUG_MODE" = "true" ]; then+ {+ echo "=== RAW AI RESPONSE (reviewer #$REVIEWER_NUM, exit $EXIT_CODE) ==="+ echo "$AI_RESPONSE"+ echo "--- stderr ---"+ cat "$WORK_DIR/err_$i.log" 2>/dev/null || true+ echo "=== END RAW AI RESPONSE ==="+ } >&2+ fi++ # One model failing (hard error, empty, or non-JSON) must not sink the others:+ # record a short failure note and keep going. The caller fails the workflow+ # only if *every* reviewer failed (success_count == 0). A reviewer that errors+ # out never sets a verdict, so it can never flip the aggregate decision to+ # "fail" — only a model that returns a valid "fail" verdict does. Sensitive raw+ # output is never written to the comment.+ # Require a JSON *object*: a bare `jq .` accepts any valid JSON, including a+ # top-level string/array/number that a refusing or misbehaving model might+ # emit ("I refuse to review this"). That would pass the gate and then crash+ # the `.review` access below under `set -e`, sinking *every* reviewer and+ # writing no output at all. `type == "object"` rejects non-objects so they+ # degrade to a per-reviewer failure note, preserving the others.+ if [ "$EXIT_CODE" -ne 0 ] || [ -z "$AI_RESPONSE" ] || ! echo "$AI_RESPONSE" | jq -e 'type == "object"' >/dev/null 2>&1; then+ echo "⚠️ Reviewer #$REVIEWER_NUM did not return a usable review (exit $EXIT_CODE)" >&2+ REVIEW_BODY="_This reviewer could not complete its review (see workflow logs for details)._"+ LABELS=""+ else+ REVIEW_RAW=$(echo "$AI_RESPONSE" | jq -r '.review // "No review provided"')+ # Trim whitespace with tr (the verdict is a single token). The previous+ # `| xargs` aborted the entire script on an unbalanced quote in the value,+ # which—inside this per-reviewer loop—would sink every reviewer.+ DECISION=$(echo "$AI_RESPONSE" | jq -r '.fail_pass_workflow // "uncertain"' | tr -d '[:space:]')+ LABELS=$(echo "$AI_RESPONSE" | jq -r '.labels_added[]? // empty')+ SUCCESS_COUNT=$((SUCCESS_COUNT + 1))+ [ "$DECISION" = "fail" ] && ANY_FAIL="true"++ # Each review opens with a "## AI Code Review" H2 and closes with the+ # singular Friendly AI Reviewer footer. Under a "### 👤 Reviewer N"+ # subheading the H2 is redundant and the footer would repeat once per model,+ # so strip both here; a single (plural) footer is added to the combined+ # comment below. The footer match keys on the stable "Review by [Friendly AI+ # Reviewer]" prefix and the \s*\z tail absorbs trailing whitespace; if the+ # upstream markers ever change and aren't found, the text is left intact+ # (nothing is dropped).+ REVIEW_BODY=$(printf '%s' "$REVIEW_RAW" \+ | perl -0pe 's/\A\s*##\s*AI Code Review\s*\n+//' \+ | perl -0pe 's/\n*---\s*\n\*Review by \[Friendly AI Reviewer\][^\n]*\n?\s*\z//s')+ fi++ # Lead each section with a blank line so the "### Reviewer N" heading always+ # renders (command substitution strips trailing newlines, so the separator+ # must precede the heading, not follow the previous body).+ COMBINED_REVIEW=$(printf '%s\n\n### 👤 Reviewer %s\n\n%s' "$COMBINED_REVIEW" "$REVIEWER_NUM" "$REVIEW_BODY")+ [ -n "$LABELS" ] && ALL_LABELS=$(printf '%s\n%s' "$ALL_LABELS" "$LABELS")+ echo "::endgroup::" >&2+done++# Aggregate labels (union across reviewers) and the pass/fail decision (request+# changes if ANY reviewer did).+LABELS=$(printf '%s\n' "$ALL_LABELS" | sed '/^[[:space:]]*$/d' | sort -u)+if [ "$ANY_FAIL" = "true" ]; then+ DECISION="fail"+else+ DECISION="pass"+fi++REVIEWER_WORD="reviewers"+[ "${#MODELS[@]}" -eq 1 ] && REVIEWER_WORD="reviewer"+STICKY_MARKER="<!-- ai-code-review:sticky -->"+FOOTER="*Reviews by [Friendly AI Reviewer](https://github.com/LearningCircuit/Friendly-AI-Reviewer) - made with ❤️*"+# COMBINED_REVIEW already starts with "\n\n", so it follows the title directly+# (no extra newline) to yield exactly one blank line before "### Reviewer 1".+# The backticks in `%s` are intentional literal Markdown (inline-code the SHA),+# not a command substitution — single quotes keep them literal on purpose.+# shellcheck disable=SC2016+COMMENT_BODY=$(printf '%s\n\n## 🤖 AI Code Review (%s %s)%s\n\n---\n%s\n\n_Last reviewed at commit `%s`_' \+ "$STICKY_MARKER" "${#MODELS[@]}" "$REVIEWER_WORD" "$COMBINED_REVIEW" "$FOOTER" "${HEAD_SHA:0:7}")++printf '%s' "$COMMENT_BODY" > "$WORK_DIR/comment_body.md"+printf '%s' "$LABELS" > "$WORK_DIR/labels.txt"+printf '%s' "$DECISION" > "$WORK_DIR/decision.txt"+printf '%s' "$SUCCESS_COUNT" > "$WORK_DIR/success_count.txt"diff --git a/.github/workflows/ai-code-reviewer.yml b/.github/workflows/ai-code-reviewer.ymlindex b427db4416..a38b9942fb 100644--- a/.github/workflows/ai-code-reviewer.yml+++ b/.github/workflows/ai-code-reviewer.yml@@ -66,7 +66,12 @@ jobs:env:GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}- AI_MODEL: ${{ vars.AI_MODEL || 'moonshotai/kimi-k2-thinking' }}+ # Comma-separated list of models. Each model produces one independent+ # review, presented as "Reviewer 1", "Reviewer 2", ... in a single+ # combined comment. Falls back to the shared AI_MODEL (also used by+ # release.yml, which must stay a single model) when unset, so a+ # one-entry list reproduces the original single-review behavior.+ AI_REVIEW_MODELS: ${{ vars.AI_REVIEW_MODELS || vars.AI_MODEL || 'moonshotai/kimi-k2-thinking' }}AI_TEMPERATURE: ${{ vars.AI_TEMPERATURE || '0.1' }}AI_MAX_TOKENS: ${{ vars.AI_MAX_TOKENS || '64000' }}MAX_DIFF_SIZE: ${{ vars.MAX_DIFF_SIZE || '800000' }}@@ -78,80 +83,78 @@ jobs:DEBUG_MODE: ${{ vars.DEBUG_MODE || 'false' }}EVENT_ACTION: ${{ github.event.action }}run: |- # Run the AI reviewer and save response to fileecho "Running AI code review..."- # Temporarily disable errexit to capture errors properly- set +e- AI_RESPONSE=$(bash ai-reviewer.sh < diff.txt)- EXIT_CODE=$?- set -e-- # Output raw response for debugging (only in debug mode)- if [ "$DEBUG_MODE" = "true" ]; then- echo "=== RAW AI RESPONSE FOR DEBUG ==="- echo "Exit code: $EXIT_CODE"- echo "$AI_RESPONSE"- echo "=== END RAW AI RESPONSE ==="- fi-- # Check if the script failed- if [ $EXIT_CODE -ne 0 ]; then- echo "❌ AI reviewer script failed with exit code $EXIT_CODE"- echo ""- echo "Error output:"- echo "$AI_RESPONSE"- exit 1- fi-- # Check if AI_RESPONSE is empty- if [ -z "$AI_RESPONSE" ]; then- echo "❌ AI response is empty"- exit 1- fi-- # Parse JSON response- if ! echo "$AI_RESPONSE" | jq . >/dev/null 2>&1; then- echo "⚠️ AI response is not valid JSON. Cannot process review."-- # Log raw response for debugging (only in debug mode)- if [ "$DEBUG_MODE" = "true" ]; then- echo "=== DEBUG: Raw AI Response (first 2000 chars) ==="- echo "$AI_RESPONSE" | head -c 2000- echo ""- echo "=== END DEBUG ==="-- # Try to extract JSON from thinking model response- if echo "$AI_RESPONSE" | grep -q '"content"'; then- echo "=== DEBUG: Attempting to extract content from response ==="- EXTRACTED_CONTENT=$(echo "$AI_RESPONSE" | jq -r '.choices[0].message.content // empty' 2>/dev/null || echo "")- if [ -n "$EXTRACTED_CONTENT" ]; then- echo "Extracted content (first 1000 chars):"- echo "$EXTRACTED_CONTENT" | head -c 1000- echo ""- fi- fi- fi-- # Don't post raw response as it may contain sensitive data- echo "AI response could not be processed. Please check the workflow logs for debugging information (enable DEBUG_MODE to see details)." | gh pr comment "$PR_NUMBER" --repo "$REPO_FULL_NAME" -F -+ # Parse the comma-separated model list. Each entry becomes one+ # independent reviewer ("Reviewer 1", "Reviewer 2", ...). A single+ # entry reproduces the original one-review behavior.+ # `read` returns non-zero at EOF; the here-string always appends a+ # trailing newline so it returns 0 here, but `|| true` keeps a future+ # edge case from tripping `set -e`. An empty/whitespace AI_REVIEW_MODELS+ # yields only empty entries, which the filter below drops, correctly+ # falling through to the "No models configured" error.+ IFS=',' read -ra RAW_MODELS <<< "$AI_REVIEW_MODELS" || true+ MODELS=()+ for m in "${RAW_MODELS[@]}"; do+ # Trim surrounding whitespace with parameter expansion. (xargs would+ # choke on a model id containing a quote and word-collapse internal+ # spaces — model ids shouldn't have either, but this is robust.)+ m="${m#"${m%%[![:space:]]*}"}" # strip leading whitespace+ m="${m%"${m##*[![:space:]]}"}" # strip trailing whitespace+ [ -n "$m" ] && MODELS+=("$m")+ done+ if [ "${#MODELS[@]}" -eq 0 ]; then+ echo "❌ No models configured in AI_REVIEW_MODELS"exit 1fi+ echo "Configured ${#MODELS[@]} reviewer model(s)"++ # Launch every model concurrently so the job's wall-clock is the+ # slowest single review, not the sum of all of them. Each reviewer+ # writes its JSON response and its diagnostics to its own index-keyed+ # files, so the "Reviewer N" numbering stays stable and the parallel+ # output never interleaves. AI_MODEL is overridden per call; all other+ # settings (temperature, token limits, diff size, context fetching)+ # are shared.+ pids=()+ for i in "${!MODELS[@]}"; do+ AI_MODEL="${MODELS[i]}" bash ai-reviewer.sh < diff.txt \+ > "resp_$i.json" 2> "err_$i.log" &+ pids[i]=$!+ done- # Extract fields from JSON- REVIEW=$(echo "$AI_RESPONSE" | jq -r '.review // "No review provided"')- DECISION=$(echo "$AI_RESPONSE" | jq -r '.fail_pass_workflow // "uncertain"' | tr -d '\n\r' | xargs)- LABELS=$(echo "$AI_RESPONSE" | jq -r '.labels_added[]? // empty')+ # Record each reviewer's exit code to code_<i> (a failing model must+ # not abort the others, so disable errexit around wait).+ for i in "${!MODELS[@]}"; do+ set +e+ wait "${pids[i]}"+ echo $? > "code_$i"+ set -e+ done- # Post (or update) the review as a single sticky PR comment. On each+ # Assemble the combined comment with the extracted, unit-tested helper+ # (.github/scripts/combine-ai-reviews.sh). It reads resp_<i>.json /+ # code_<i> from the working dir and writes the comment body, label set,+ # decision, and success count back out. Keeping the assembly in a+ # script is what lets tests/ci/test_combine_ai_reviews.py cover the+ # parsing, footer-stripping, label union, and pass/fail logic without+ # any network or GitHub API access.+ bash .github/scripts/combine-ai-reviews.sh "$PWD" "${MODELS[@]}"+ COMMENT_BODY="$(cat comment_body.md)"+ LABELS="$(cat labels.txt)"+ DECISION="$(cat decision.txt)"+ SUCCESS_COUNT="$(cat success_count.txt)"++ # Post (or update) the reviews as a single sticky PR comment. On each# push we edit the same comment in place instead of stacking a new one# per push. The hidden marker identifies our comment specifically, so# we never touch comments from humans or other bots. (Review comments# made before this change lack the marker and are left as-is.)# A commit-sha line lets reviewers tell which push the (edited-in-place)# comment reflects, since GitHub only shows a vague "edited" marker.+ # This literal MUST match the marker the helper script prepends to the+ # comment body.STICKY_MARKER="<!-- ai-code-review:sticky -->"- COMMENT_BODY=$(printf "%s\n\n%s\n\n_Last reviewed at commit \`%s\`_" "$STICKY_MARKER" "$REVIEW" "${HEAD_SHA:0:7}")# STICKY_MARKER is interpolated into the jq program below, so it must# stay free of `"` and `\`. gh api's --jq takes no --arg, so we can't# pass it as a jq variable; keeping the marker a literal constant is@@ -170,9 +173,12 @@ jobs:printf '%s' "$COMMENT_BODY" | gh pr comment "$PR_NUMBER" --repo "$REPO_FULL_NAME" -F -fi- # Add labels to PR+ # Add labels to PR. Iterate line-by-line (labels.txt is one label per+ # line) rather than word-splitting, so a multi-word label such as+ # "good first issue" is treated as a single label, not three.echo "Labels to add: $LABELS"- for label in $LABELS; do+ while IFS= read -r label; do+ [ -n "$label" ] || continueecho "Adding label: $label"if gh label create "$label" --color "0366d6" --description "Auto-created by AI reviewer" 2>/dev/null; thenecho "Created new label: $label"@@ -184,7 +190,7 @@ jobs:elseecho "Failed to add label: $label"fi- done+ done <<< "$LABELS"# Handle workflow decision (after posting review and labels)echo "AI decision: $DECISION"@@ -204,7 +210,15 @@ jobs:fifi- echo "✅ AI review completed successfully"+ # If every reviewer failed, the posted comment carries only failure+ # notes — surface that as a red workflow run rather than a silent pass,+ # matching the original single-review behavior on an unusable response.+ if [ "$SUCCESS_COUNT" -eq 0 ]; then+ echo "❌ All ${#MODELS[@]} reviewer(s) failed to produce a usable review."+ exit 1+ fi++ echo "✅ AI review completed successfully ($SUCCESS_COUNT/${#MODELS[@]} reviewers)"- name: Fail Workflow if Requestedif: env.FAIL_ON_REQUESTED_CHANGES == 'true' && env.DECISION == 'fail'@@ -217,6 +231,9 @@ jobs:if: always()run: |rm -f diff.txt+ # Per-reviewer response/log/exit-code files from the parallel fan-out,+ # plus the assembly helper's output files.+ rm -f resp_*.json err_*.log code_* comment_body.md labels.txt decision.txt success_count.txt# Only remove ai_response.txt if it exists (only created in debug mode)if [ -f "ai_response.txt" ]; thenrm -f ai_response.txtdiff --git a/Dockerfile b/Dockerfileindex fe52f3cc64..91d1dc9d22 100644--- a/Dockerfile+++ b/Dockerfile@@ -134,6 +134,13 @@ ARG DEBIAN_FRONTEND=noninteractive# corresponding upgrade in the builder-base stage (top of file).RUN apt-get update -o Acquire::Retries=3 && apt-get upgrade -y -o Acquire::Retries=3 \&& apt-get install -y --no-install-recommends -o Acquire::Retries=3 \+ # jq + perl: required by tests/ci/test_combine_ai_reviews.py, which exercises+ # the AI-review combine helper (it uses jq and perl). Without them those tests+ # silently skipped in this pytest lane, leaving the script with no enforced+ # coverage. perl is normally present via perl-base, but install it explicitly+ # so the test's fail-loud-in-CI tool check can never false-alarm.+ jq \+ perl \xauth \xvfb \# Dependencies for Chromiumdiff --git a/docs/AI_Code_Reviewer_Setup.md b/docs/AI_Code_Reviewer_Setup.mdindex 541695077c..1a08091cc5 100644--- a/docs/AI_Code_Reviewer_Setup.md+++ b/docs/AI_Code_Reviewer_Setup.md@@ -33,13 +33,32 @@ The review is posted as a single comprehensive comment on your pull request.### 3. Configure Workflow (Optional)-The workflow is pre-configured with sensible defaults, but you can customize it by editing `.github/workflows/ai-code-reviewer.yml`:+The workflow is pre-configured with sensible defaults. You can override them **without editing the workflow** by adding GitHub **repository variables** (not secrets):-- **AI_MODEL**: Change the AI model (see [OpenRouter models](https://openrouter.ai/models))+1. Go to your repository → **Settings** → **Secrets and variables** → **Actions**+2. Open the **Variables** tab → **New repository variable**+3. Add any of:++- **AI_REVIEW_MODELS**: Comma-separated list of models. Each model runs the reviewer independently, so you get one review per model in a single combined comment (see [Multiple reviewers](#multiple-reviewers)). When unset, falls back to `AI_MODEL`, then to the built-in default.+- **AI_MODEL**: A single model (see [OpenRouter models](https://openrouter.ai/models)). Used by the reviewer when `AI_REVIEW_MODELS` is unset, and also by the release-notes workflow. Default: `moonshotai/kimi-k2-thinking`.- **AI_TEMPERATURE**: Adjust randomness (default: `0.1` for consistent reviews)-- **AI_MAX_TOKENS**: Maximum response length (default: `2000`)+- **AI_MAX_TOKENS**: Maximum response length (default: `64000`)- **MAX_DIFF_SIZE**: Maximum diff size in bytes (default: `800000` / 800KB)+#### Multiple reviewers++Set **AI_REVIEW_MODELS** to a comma-separated list to have several models review the same diff. For example:++```+minimax/minimax-m3, z-ai/glm-5.2+```++Notes:+- Separate models with **commas** (spaces around them are fine and trimmed). A space-separated value is treated as a *single* model name and will fail.+- Each model is shown anonymously as **Reviewer 1**, **Reviewer 2**, … (the model→reviewer mapping appears only in the workflow logs), so readers judge the feedback rather than the model.+- All reviews are collected into one sticky PR comment. Labels are unioned across reviewers; if `FAIL_ON_REQUESTED_CHANGES` is enabled, the workflow fails when **any** reviewer requests changes. This is deliberately "any" rather than a majority vote — a single model catching a real blocker shouldn't be outvoted — and it only takes effect when you opt into `FAIL_ON_REQUESTED_CHANGES` (off by default). A reviewer that errors out (rather than returning a verdict) never counts as a "fail"; it just shows a note in its section.+- Models run **in parallel**, so the job's wall-clock is the slowest single review rather than the sum. Each model still makes its own set of GitHub API calls for context, so a very long list means many concurrent calls; two or three models are well within limits.+## Usage### Triggering AI Reviewsdiff --git a/tests/ci/test_combine_ai_reviews.py b/tests/ci/test_combine_ai_reviews.pynew file mode 100644index 0000000000..f276e744e0--- /dev/null+++ b/tests/ci/test_combine_ai_reviews.py@@ -0,0 +1,426 @@+"""Unit tests for .github/scripts/combine-ai-reviews.sh.++The AI Code Reviewer workflow runs N models over a PR diff (network) and writes+each reviewer's raw stdout to resp_<i>.json and exit code to code_<i>. The+combine-ai-reviews.sh helper turns those files into the single sticky-comment+body, the deduped label set, the pass/fail decision, and the success count.++These tests exercise that pure-assembly logic directly with canned fixtures —+no network, no GitHub API — so the non-trivial bash/jq/perl parts (comma-split+already happens in the workflow; here it's header/footer stripping, anonymized+"Reviewer N" section assembly, label union, and decision aggregation) are+covered in CI.+"""++import json+import os+import shutil+import subprocess+from pathlib import Path++import pytest++SCRIPT = (+ Path(__file__).resolve().parents[2]+ / ".github"+ / "scripts"+ / "combine-ai-reviews.sh"+)++_MISSING_TOOLS = [t for t in ("bash", "jq", "perl") if shutil.which(t) is None]+_IN_CI = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS"))++# Skip ONLY when the tools are missing locally. In CI they must be present+# (jq+perl are installed into the test image), so we deliberately do NOT skip+# there: if they're ever missing in CI these tests run and fail loudly instead+# of silently skipping and leaving combine-ai-reviews.sh with zero enforced+# coverage. Failing per-test (vs a module-level raise) keeps the failure+# localized to this lane — see test_required_tools_present_in_ci for the+# explicit, clearly-messaged guard.+pytestmark = pytest.mark.skipif(+ bool(_MISSING_TOOLS) and not _IN_CI,+ reason=f"combine-ai-reviews.sh requires {_MISSING_TOOLS} (missing locally)",+)+++def test_required_tools_present_in_ci():+ """The test image must ship the tools the helper needs. If this fails in CI,+ install the missing tool in the Dockerfile `ldr-test` stage — otherwise the+ rest of this suite would be a silent no-op."""+ if _IN_CI:+ assert not _MISSING_TOOLS, (+ f"CI test image is missing {_MISSING_TOOLS}; add it to the Dockerfile "+ "ldr-test stage so combine-ai-reviews.sh keeps real CI coverage."+ )+++FOOTER_SINGULAR = (+ "*Review by [Friendly AI Reviewer]"+ "(https://github.com/LearningCircuit/Friendly-AI-Reviewer) - made with ❤️*"+)+++def make_response(body, decision="pass", labels=None, *, trailing=""):+ """Build a realistic ai-reviewer.sh JSON response (review + verdict + labels)."""+ review = f"## AI Code Review\n\n{body}\n\n---\n{FOOTER_SINGULAR}{trailing}"+ return json.dumps(+ {+ "review": review,+ "fail_pass_workflow": decision,+ "labels_added": labels or [],+ }+ )+++def run_combine(work_dir, reviewers, *, head_sha="abc1234def", debug=False):+ """Write fixtures for each reviewer and run the helper; return its outputs.++ `reviewers` is a list of dicts: {"code": int, "resp": str}. Model names are+ deliberately distinctive so tests can assert they never leak into the+ (anonymized) comment body.+ """+ models = []+ for i, rv in enumerate(reviewers):+ (work_dir / f"code_{i}").write_text(str(rv["code"]))+ (work_dir / f"resp_{i}.json").write_text(rv["resp"])+ (work_dir / f"err_{i}.log").write_text("")+ models.append(f"secret-model-{i}-DO-NOT-LEAK")++ env = {"HEAD_SHA": head_sha, "DEBUG_MODE": "true" if debug else "false"}+ result = subprocess.run(+ ["bash", str(SCRIPT), str(work_dir), *models],+ capture_output=True,+ text=True,+ env={**env, "PATH": __import__("os").environ["PATH"]},+ )+ assert result.returncode == 0, (+ f"script failed ({result.returncode}): {result.stderr}"+ )+ return {+ "models": models,+ "comment": (work_dir / "comment_body.md").read_text(),+ "labels": (work_dir / "labels.txt").read_text(),+ "decision": (work_dir / "decision.txt").read_text(),+ "success_count": (work_dir / "success_count.txt").read_text(),+ "stderr": result.stderr,+ }+++def test_two_reviewers_pass_and_fail(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {+ "code": 0,+ "resp": make_response(+ "All good here.", "pass", ["enhancement"]+ ),+ },+ {+ "code": 0,+ "resp": make_response("Race condition.", "fail", ["bug"]),+ },+ ],+ )+ c = out["comment"]+ # Both anonymized sections present, in order.+ assert "### 👤 Reviewer 1" in c+ assert "### 👤 Reviewer 2" in c+ assert c.index("Reviewer 1") < c.index("Reviewer 2")+ # Bodies carried through.+ assert "All good here." in c+ assert "Race condition." in c+ # Aggregation: any fail -> fail; labels unioned + sorted; both succeeded.+ assert out["decision"] == "fail"+ assert out["labels"].split() == ["bug", "enhancement"]+ assert out["success_count"] == "2"+++def test_header_and_per_review_footer_are_stripped(tmp_path):+ out = run_combine(+ tmp_path,+ [{"code": 0, "resp": make_response("Body one.", "pass")}],+ )+ c = out["comment"]+ # The per-review "## AI Code Review" H2 is removed (only the combined H2 with+ # the reviewer count remains), and the singular per-review footer is gone.+ assert "## 🤖 AI Code Review (1 reviewer)" in c+ assert "## AI Code Review\n" not in c+ assert "Review by [Friendly AI Reviewer]" not in c+ # Exactly one combined (plural) footer is appended.+ assert c.count("Reviews by [Friendly AI Reviewer]") == 1+++def test_footer_strip_tolerates_trailing_whitespace(tmp_path):+ # LLMs sometimes append trailing newlines/spaces after the footer.+ out = run_combine(+ tmp_path,+ [+ {+ "code": 0,+ "resp": make_response("Body.", "pass", trailing="\n\n \n"),+ }+ ],+ )+ assert "Review by [Friendly AI Reviewer]" not in out["comment"]+ # ...but the actual review body must survive (guard against an over-greedy+ # strip that nukes everything).+ assert "Body." in out["comment"]+ assert out["success_count"] == "1"+++def test_failed_reviewer_degrades_without_sinking_others(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {+ "code": 0,+ "resp": make_response("Good review.", "pass", ["enhancement"]),+ },+ {"code": 1, "resp": ""}, # hard error: non-zero exit, empty stdout+ {"code": 0, "resp": "this is not json"}, # garbage stdout+ ],+ )+ c = out["comment"]+ assert "Good review." in c+ assert c.count("could not complete its review") == 2+ # Two of three reviewers failed; the survivor still counts.+ assert out["success_count"] == "1"+ # A reviewer that errors out never flips the decision to fail.+ assert out["decision"] == "pass"+ # Failed reviewers contribute no labels.+ assert out["labels"].split() == ["enhancement"]+++def test_all_reviewers_failed(tmp_path):+ out = run_combine(+ tmp_path,+ [{"code": 1, "resp": ""}, {"code": 1, "resp": ""}],+ )+ assert out["success_count"] == "0"+ assert out["decision"] == "pass"+ assert out["comment"].count("could not complete its review") == 2+++def test_labels_are_unioned_and_deduped(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {+ "code": 0,+ "resp": make_response("a", "pass", ["bug", "security"]),+ },+ {+ "code": 0,+ "resp": make_response("b", "pass", ["bug", "enhancement"]),+ },+ ],+ )+ assert out["labels"].split() == ["bug", "enhancement", "security"]+++def test_model_names_never_leak_into_comment(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {"code": 0, "resp": make_response("x", "pass")},+ {"code": 0, "resp": make_response("y", "fail")},+ ],+ )+ for model in out["models"]:+ assert model not in out["comment"]+++def test_single_reviewer_uses_singular_wording_and_sticky_marker(tmp_path):+ out = run_combine(+ tmp_path,+ [{"code": 0, "resp": make_response("Solo.", "pass")}],+ )+ c = out["comment"]+ assert "(1 reviewer)" in c+ assert "(1 reviewers)" not in c+ # Sticky marker must lead the body (the workflow searches for it verbatim).+ assert c.startswith("<!-- ai-code-review:sticky -->")+ # Commit sha line rendered with real backticks (not escaped backslashes).+ assert "_Last reviewed at commit `abc1234`_" in c+++def test_blank_line_before_horizontal_rule(tmp_path):+ # A "text\n---" with no blank line is a setext H2 in Markdown; ensure the+ # divider before the footer is preceded by a blank line.+ out = run_combine(+ tmp_path,+ [{"code": 0, "resp": make_response("Body.", "pass")}],+ )+ assert "\n\n---\n" in out["comment"]+++# --- Regression tests for resilience (one bad reviewer must not sink others) ---+++def test_valid_json_non_object_does_not_sink_other_reviewers(tmp_path):+ # A model can return valid JSON that is a bare string/array (e.g. a refusal),+ # not the expected object. That must degrade to a per-reviewer failure note,+ # not crash the whole combine and lose the good review. (Regression: a bare+ # `jq .` gate accepted non-objects, then `.review` errored under set -e.)+ out = run_combine(+ tmp_path,+ [+ {+ "code": 0,+ "resp": '"I refuse to review this"',+ }, # valid JSON string+ {+ "code": 0,+ "resp": '["also", "not", "an", "object"]',+ }, # JSON array+ {"code": 0, "resp": make_response("Real review.", "pass", ["bug"])},+ ],+ )+ c = out["comment"]+ assert "Real review." in c+ assert c.count("could not complete its review") == 2+ assert out["success_count"] == "1"+ assert out["labels"].split() == ["bug"]+++def test_weird_verdict_string_does_not_crash(tmp_path):+ # A verdict value containing a quote previously aborted the script via+ # `| xargs` (unmatched quote). It must now be handled gracefully.+ resp = json.dumps(+ {+ "review": f"## AI Code Review\n\nBody.\n\n---\n{FOOTER_SINGULAR}",+ "fail_pass_workflow": "it's complicated",+ "labels_added": [],+ }+ )+ out = run_combine(tmp_path, [{"code": 0, "resp": resp}])+ assert out["success_count"] == "1"+ # A non-"fail" verdict (even a weird one) does not request changes.+ assert out["decision"] == "pass"+++def test_empty_exit_code_file_counts_as_failed(tmp_path):+ # An empty/unreadable code_<i> must be treated as a failed reviewer, not+ # silently counted as a success (regression: `[ "" -ne 0 ]` errored and fell+ # through to the success path).+ out = run_combine(+ tmp_path,+ [{"code": "", "resp": make_response("Should be ignored.", "pass")}],+ )+ assert out["success_count"] == "0"+ assert "could not complete its review" in out["comment"]+ assert "Should be ignored." not in out["comment"]+++def test_non_numeric_exit_code_counts_as_failed(tmp_path):+ out = run_combine(+ tmp_path,+ [{"code": "boom", "resp": make_response("Ignored.", "pass")}],+ )+ assert out["success_count"] == "0"+ assert "integer expression" not in out["stderr"]+++def test_mixed_pass_fail_error_aggregation(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {"code": 0, "resp": make_response("ok", "pass", ["enhancement"])},+ {"code": 0, "resp": make_response("blocker", "fail", ["bug"])},+ {"code": 1, "resp": ""}, # hard error+ ],+ )+ # Only usable reviews count; the one real "fail" verdict drives the decision;+ # labels union only from successful reviewers; exactly one failure note.+ assert out["success_count"] == "2"+ assert out["decision"] == "fail"+ assert out["labels"].split() == ["bug", "enhancement"]+ assert out["comment"].count("could not complete its review") == 1+++def test_valid_json_missing_review_field(tmp_path):+ resp = json.dumps({"fail_pass_workflow": "pass", "labels_added": []})+ out = run_combine(tmp_path, [{"code": 0, "resp": resp}])+ # Object with no .review still counts as a (degenerate) success and renders+ # the jq fallback text rather than crashing.+ assert out["success_count"] == "1"+ assert "No review provided" in out["comment"]+++def test_debug_mode_does_not_change_comment_and_keeps_model_in_stderr(tmp_path):+ reviewers = [{"code": 0, "resp": make_response("Body.", "pass")}]+ # Separate dirs so the two runs' fixtures/outputs don't collide.+ plain_dir = tmp_path / "plain"+ dbg_dir = tmp_path / "debug"+ plain_dir.mkdir()+ dbg_dir.mkdir()+ plain = run_combine(plain_dir, reviewers)+ dbg = run_combine(dbg_dir, reviewers, debug=True)+ # The posted comment must be byte-identical regardless of DEBUG_MODE — debug+ # content (raw response, model name) goes only to stderr, never the comment.+ assert dbg["comment"] == plain["comment"]+ # DEBUG must not perturb any of the machine-read outputs either.+ assert dbg["decision"] == plain["decision"]+ assert dbg["labels"] == plain["labels"]+ assert dbg["success_count"] == plain["success_count"]+ assert "RAW AI RESPONSE" in dbg["stderr"]+ assert "RAW AI RESPONSE" not in plain["stderr"]+ # Model name leaks only into stderr logs (anonymization), never the comment.+ assert dbg["models"][0] in dbg["stderr"]+ assert dbg["models"][0] not in dbg["comment"]+++def test_four_reviewers_keep_order(tmp_path):+ out = run_combine(+ tmp_path,+ [+ {"code": 0, "resp": make_response(f"body-{n}", "pass")}+ for n in range(4)+ ],+ )+ c = out["comment"]+ positions = [c.index(f"Reviewer {n}") for n in range(1, 5)]+ assert positions == sorted(positions)+ for n in range(4):+ assert f"body-{n}" in c+++def test_usage_errors_exit_nonzero(tmp_path):+ # The script's only non-zero exits: bad work_dir and zero model args.+ env = {**os.environ}+ no_dir = subprocess.run(+ ["bash", str(SCRIPT), str(tmp_path / "does-not-exist"), "m"],+ capture_output=True,+ text=True,+ env=env,+ )+ assert no_dir.returncode == 2+ no_models = subprocess.run(+ ["bash", str(SCRIPT), str(tmp_path)],+ capture_output=True,+ text=True,+ env=env,+ )+ assert no_models.returncode == 2+++def test_exit_code_with_trailing_newline(tmp_path):+ # The workflow writes exit codes via `echo $? > code_i`, i.e. WITH a trailing+ # newline ("0\n"), whereas the other tests use the bare "0" form. Confirm the+ # real workflow shape is handled (command substitution strips the newline).+ (tmp_path / "code_0").write_text("0\n")+ (tmp_path / "resp_0.json").write_text(make_response("ok", "pass"))+ (tmp_path / "err_0.log").write_text("")+ result = subprocess.run(+ ["bash", str(SCRIPT), str(tmp_path), "m"],+ capture_output=True,+ text=True,+ env={+ "HEAD_SHA": "abc1234def",+ "DEBUG_MODE": "false",+ "PATH": os.environ["PATH"],+ },+ )+ assert result.returncode == 0, result.stderr+ assert (tmp_path / "success_count.txt").read_text() == "1"