~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(release): complete signal-ordered diff input + hallucination guard for AI release notes (#5027)
LearningCircuit2b5557dd7b1a
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex baa5a43773..5b1294a23e 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -726,12 +726,12 @@ jobs:NOTES_FILE="$(mktemp)"PROMPT_FILE="$(mktemp)"AI_RESPONSE_FILE="$(mktemp)"+ AI_SUMMARY_FILE="$(mktemp)"PR_BODIES_FILE="$(mktemp)"DIFF_FILE="$(mktemp)"- COMPARE_JSON="$(mktemp)"# ${DIFF_FILE}.clean is the staging path for the NUL-strip step;# rm -f tolerates the case where it doesn't exist.- trap 'rm -f "$NOTES_FILE" "$PROMPT_FILE" "$AI_RESPONSE_FILE" "$PR_BODIES_FILE" "$DIFF_FILE" "${DIFF_FILE}.clean" "$COMPARE_JSON"' EXIT+ trap 'rm -f "$NOTES_FILE" "$PROMPT_FILE" "$AI_RESPONSE_FILE" "$AI_SUMMARY_FILE" "$PR_BODIES_FILE" "$DIFF_FILE" "${DIFF_FILE}.clean"' EXIT# `set -e` does NOT exit on a failing command substitution in an# assignment, so wrap with explicit failure handling.@@ -808,11 +808,18 @@ jobs:# ---- Diff between previous release and current ----# /releases/latest excludes drafts and prereleases. Returns 404 if# there is no prior stable release — the AI then runs without a- # diff input. Filter noise (lockfiles, generated docs, SBOM, static- # assets, snapshot fixtures, binary patches) so the LLM sees signal.- # Hard cap diff size at 700,000 chars (~175k tokens) to leave- # headroom in Kimi K2 Thinking's 262k context window for the rest- # of the prompt + 64k output budget.+ # diff input. The diff is produced LOCALLY by git (shallow-fetch+ # of the previous tag), not by the compare API: the API silently+ # caps .files[] at 300 entries (the v1.8.1→v1.9.0 compare hit+ # exactly 300) and lists files alphabetically, so tail-truncation+ # dropped patches arbitrarily instead of by importance. The local+ # diff is complete, starts with a full --stat, and orders patch+ # bodies by signal (src/ first, largest change first) so the+ # 700,000-char budget (~175k tokens, headroom in Kimi K2+ # Thinking's 262k context for prompt + 64k output) trims the+ # least informative patches first. Same noise filter as before+ # (lockfiles, generated docs, SBOM, static assets, snapshots,+ # binaries).: > "$DIFF_FILE"PREV_TAG=""if PREV_TAG=$(gh api "/repos/${GITHUB_REPOSITORY}/releases/latest" --jq '.tag_name' 2>/dev/null) && [[ -n "$PREV_TAG" && "$PREV_TAG" != "null" ]]; then@@ -836,49 +843,145 @@ jobs:fifiif [[ -n "$PREV_TAG" ]]; then- echo "Fetching diff $PREV_TAG...$RELEASE_TAG"- if gh api "/repos/${GITHUB_REPOSITORY}/compare/${PREV_TAG}...${RELEASE_TAG}" > "$COMPARE_JSON" 2>/dev/null; then- # GitHub's compare API caps the .files[] array at 300 entries- # per response (it paginates commits, not files). Log when we- # hit that boundary so a future release's diff truncation can- # be diagnosed quickly.- compare_file_count=$(jq -r '.files | length' "$COMPARE_JSON" 2>/dev/null || echo 0)- if [[ "$compare_file_count" -ge 300 ]]; then- echo "WARNING: compare API returned $compare_file_count files — at the 300-file limit; some files may be omitted from the AI diff input"- fi- # Don't swallow jq stderr — if the filter fails (e.g. the- # compare API returned malformed JSON), we want the actual- # error in the runner log, not a silent empty diff.- if ! jq -r '.files[]- | select(.patch != null)- | select((.filename | test("(package-lock\\.json$|pdm\\.lock$|\\.min\\.(js|css)$|^docs/CONFIGURATION\\.md$|^docs/ci/workflow-status\\.md$|^sbom-|/__pycache__/|\\.pyc$|^docs/images/|^src/local_deep_research/web/static/|/(golden_master|snapshot)|\\.snap$)")) | not)- | "diff --git a/\(.filename) b/\(.filename)\n\(.patch)\n"' \- "$COMPARE_JSON" > "$DIFF_FILE"; then- echo "WARNING: jq failed processing compare API output — diff input will be empty"- : > "$DIFF_FILE"- fi- # Character-aware truncation. `head -c` is byte-oriented and- # can split multi-byte UTF-8 sequences (towncrier emits emoji- # headers 💥/🔒/✨/🐛 that appear in diffs of- # docs/release_notes/*.md). Splitting mid-emoji produces- # invalid UTF-8, which jq --rawfile then refuses to encode.- python3 - "$DIFF_FILE" 700000 'diff truncated at 700,000 chars' <<'PYEOF'+ echo "Fetching previous release tag $PREV_TAG for local diff"+ # Shallow-fetch just the previous tag's commit. NOTE:+ # actions/checkout configures a blob:none partial clone whenever+ # sparse-checkout is set, and this fetch inherits that filter —+ # the object store has the TREES of both commits but not the+ # blobs. The diffs below still work because git lazily+ # batch-fetches missing blobs from the promisor remote over+ # anonymous HTTPS (public repo; persist-credentials: false is+ # irrelevant to an unauthenticated fetch). If that lazy fetch+ # ever breaks (repo made private, anonymous throttling), the+ # python builder's git call exits non-zero and the guarded+ # `if !` below degrades to WARNING + empty diff — the release+ # itself is never blocked by this step.+ if git fetch --quiet --depth=1 origin "refs/tags/${PREV_TAG}:refs/tags/${PREV_TAG}"; then+ # Best-effort like every other AI input: a diff failure must+ # not fail the release, only shrink the prompt.+ if ! python3 - "$PREV_TAG" "$GITHUB_SHA" "$DIFF_FILE" 700000 <<'PYEOF'+ import re+ import subprocessimport sys- path, max_chars, marker = sys.argv[1], int(sys.argv[2]), sys.argv[3]- with open(path, encoding='utf-8', errors='replace') as f:- content = f.read()- if len(content) > max_chars:- content = content[:max_chars] + f"\n[... {marker} ...]\n"- with open(path, 'w', encoding='utf-8') as f:- f.write(content)- print(f"Truncated to {len(content)} chars", file=sys.stderr)++ prev, head, out_path, budget = (+ sys.argv[1],+ sys.argv[2],+ sys.argv[3],+ int(sys.argv[4]),+ )++ # Same noise filter the old compare-API jq expression used.+ NOISE = re.compile(+ r"(package-lock\.json$|pdm\.lock$|\.min\.(js|css)$"+ r"|^docs/CONFIGURATION\.md$|^docs/ci/workflow-status\.md$"+ r"|^sbom-|/__pycache__/|\.pyc$|^docs/images/"+ r"|^src/local_deep_research/web/static/"+ r"|/(golden_master|snapshot)|\.snap$)"+ )+++ def git(*args):+ # errors="replace" keeps undecodable bytes from raising;+ # --no-renames keeps numstat paths plain (no "{a => b}"+ # munging) so the per-file diff below can address them.+ return subprocess.run(+ ["git", "--no-pager", *args],+ check=True,+ capture_output=True,+ text=True,+ errors="replace",+ ).stdout+++ rng = f"{prev}..{head}"+ # The complete --stat goes first and is never truncated: even+ # when patch bodies below are dropped for budget, the model+ # always sees the full shape of the release.+ stat = git("diff", "--no-renames", "--stat=200", rng)++ # -z terminates entries with NUL and, crucially, disables+ # core.quotePath C-quoting: without it a path with non-ASCII or+ # special characters comes back as a quoted literal ("a\303\251")+ # that the per-file pathspec below silently fails to match —+ # the file would be counted as included while its patch is empty.+ files = []+ for entry in git("diff", "--no-renames", "--numstat", "-z", rng).split("\0"):+ if not entry:+ continue+ adds, dels, path = entry.split("\t", 2)+ if adds == "-" or NOISE.search(path): # binary or noise+ continue+ files.append((path, int(adds) + int(dels)))+++ # Signal order: source first, then everything else, then+ # tests/CI/docs; biggest change first within each group. Budget+ # overflow then drops the LEAST informative patches instead of+ # whatever sorted last alphabetically.+ def group(path):+ if path.startswith("src/"):+ return 0+ if path.startswith(("tests/", ".github/", "docs/")):+ return 2+ return 1+++ files.sort(key=lambda f: (group(f[0]), -f[1]))++ chunks = [f"===== DIFFSTAT ({prev}...{head}, complete) =====\n{stat}\n"]+ written = len(chunks[0])+ dropped = []+ unmatched = []+ for path, _ in files:+ # :(literal) keeps glob metacharacters in a filename ([ * ?)+ # from being interpreted as pathspec wildcards.+ patch = git("diff", "--no-renames", rng, "--", f":(literal){path}")+ if not patch:+ # A pathspec that matches nothing exits 0 with empty+ # output — e.g. a filename whose bytes don't round-trip+ # the errors="replace" UTF-8 decode (U+FFFD in the+ # decoded path no longer equals the on-disk bytes).+ # Without this branch the file would be COUNTED as+ # included while its patch is silently absent.+ unmatched.append(path)+ continue+ if written + len(patch) > budget:+ dropped.append(path)+ continue+ chunks.append(patch)+ written += len(patch)++ if dropped:+ shown = ", ".join(dropped[:20]) + (" ..." if len(dropped) > 20 else "")+ chunks.append(+ f"\n[... {len(dropped)} lower-priority file diffs omitted to fit "+ f"the {budget}-char budget: {shown} ...]\n"+ )+ if unmatched:+ shown = ", ".join(unmatched[:10]) + (" ..." if len(unmatched) > 10 else "")+ chunks.append(+ f"\n[... {len(unmatched)} file diffs unavailable (pathspec did not "+ f"match after decoding): {shown} ...]\n"+ )+ with open(out_path, "w", encoding="utf-8") as fh:+ fh.write("".join(chunks))+ print(+ f"Diff input: {len(files) - len(dropped) - len(unmatched)}/{len(files)} "+ f"file patches + complete diffstat, {written} chars"+ + (f"; {len(unmatched)} unmatched pathspec(s): {unmatched[:10]}" if unmatched else ""),+ file=sys.stderr,+ )PYEOF+ then+ echo "WARNING: local git diff failed — proceeding without diff"+ : > "$DIFF_FILE"+ fi# NUL bytes break jq --rawfile JSON encoding; strip them.tr -d '\000' < "$DIFF_FILE" > "${DIFF_FILE}.clean" && mv "${DIFF_FILE}.clean" "$DIFF_FILE"echo "Diff size: $(wc -c < "$DIFF_FILE") bytes"else- echo "WARNING: compare API call failed — proceeding without diff"- : > "$DIFF_FILE"+ echo "WARNING: could not fetch tag ${PREV_TAG} — proceeding without diff"fielseecho "No previous stable release found — skipping diff"@@ -990,6 +1093,91 @@ jobs:fiif [[ -n "$AI_SUMMARY" ]]; thenecho "AI summary generated (model=$AI_MODEL, ${#AI_SUMMARY} chars)"+ # Hallucination guard: every reference the model emitted+ # (#NNNN, bare pull/issues URL, GH-NNNN — all forms GitHub+ # renders as a reference link) must appear in the INPUT+ # sections of the prompt it was given.+ # The prompt merely INSTRUCTS the model never to invent PR+ # numbers; instructions are not enforcement, and this text+ # publishes sight-unseen. On any unknown reference, drop+ # the whole AI section loudly — the release proceeds with+ # towncrier notes + the auto PR list, the same degrade+ # path as a missing OPENROUTER_API_KEY. The check reads+ # $PROMPT_FILE post-truncation, i.e. exactly what the+ # model saw. It cannot catch invented BEHAVIOR claims,+ # only invented references — but those are the most+ # embarrassing and the only mechanically detectable kind.+ # All-numeric hex colors ("#123456") can false-positive;+ # that costs us the AI section, never a wrong note.+ printf '%s' "$AI_SUMMARY" > "$AI_SUMMARY_FILE"+ if ! python3 - "$PROMPT_FILE" "$AI_SUMMARY_FILE" <<'PYEOF'+ import os+ import re+ import sys++ with open(sys.argv[1], encoding="utf-8", errors="replace") as f:+ prompt = f.read()+ with open(sys.argv[2], encoding="utf-8", errors="replace") as f:+ summary = f.read()++ # Only the INPUT sections feed the allowed set. The instruction+ # text above the first "----- " header contains a literal+ # format example ("Reference PRs as `#1234`") that would+ # otherwise self-whitelist #1234 — a real PR in this repo — for+ # the single most predictable parroting artifact.+ cut = prompt.find("\n----- ")+ inputs = prompt[cut:] if cut != -1 else prompt++ # Both sides scan the same two LOCAL reference forms:+ # - "#NNNN" with a (?<![\w/]) lookbehind, so the cross-repo+ # shorthand "owner/repo#166" (rendered by GitHub as a link to+ # the OTHER repo) neither whitelists a local number on the+ # input side nor counts as a local claim on the output side.+ # The lookbehind also skips inline anchors whose "#" follows+ # a word char or slash ("page#4-setup", "docs/#4").+ # - pull/issues URLs scoped to THIS repository — a cross-repo+ # URL in a PR body must not whitelist its number locally, and+ # the model repeating one must not trip the guard.+ # GITHUB_REPOSITORY is always set in Actions; if it were ever+ # missing, the KeyError exits non-zero and the AI section is+ # dropped (safe direction).+ repo = re.escape(os.environ["GITHUB_REPOSITORY"])+ allowed = set(re.findall(r"(?<![\w/])#(\d+)\b", inputs))+ allowed |= set(re.findall(repo + r"/(?:pull|issues)/(\d+)\b", inputs))++ # Emitted references: capture local URL forms first, then strip+ # URLs from the summary so their numeric fragment anchors+ # ("https://ex.com/p#4-setup") can't count as prose references,+ # then scan the remaining prose for "#NNNN" and "GH-NNNN".+ # (Round 1 used a (?!-) lookahead against anchors, but that+ # silently removed any "#9999-suffixed" reference from+ # detection entirely — a false-NEGATIVE hole, the wrong+ # direction for a guard. URL-stripping kills the anchor+ # false-positive without blinding the scan.)+ url_refs = set(re.findall(repo + r"/(?:pull|issues)/(\d+)\b", summary))+ prose = re.sub(r"https?://\S+", " ", summary)+ emitted = set(re.findall(r"(?<![\w/])#(\d+)\b", prose))+ emitted |= url_refs+ emitted |= set(re.findall(r"\bGH-(\d+)\b", prose))++ unknown = sorted(emitted - allowed, key=int)+ if unknown:+ print(+ "AI summary references numbers absent from its input: "+ + ", ".join("#" + n for n in unknown),+ file=sys.stderr,+ )+ sys.exit(1)+ print(+ f"AI reference check passed: {len(emitted)} reference(s), "+ "all present in the prompt",+ file=sys.stderr,+ )+ PYEOF+ then+ echo "WARNING: AI summary contains references not present in its input — dropping AI section (release proceeds with towncrier notes + PR list)"+ AI_SUMMARY=""+ fielseecho "WARNING: AI response 2xx but no usable content field — skipping summary"# Diagnostic: dump the response shape so the next failure@@ -1125,8 +1313,11 @@ jobs:behind a change better than the title.- DIFF: the actual code change between the previous release and this one.- May be truncated. Use to disambiguate what a PR actually did or to catch- user-visible changes not well described in PR text.+ Starts with a complete diffstat of every changed file, followed by+ per-file patches ordered by importance (source code first, largest+ changes first); low-priority patches may be omitted to fit, but the+ diffstat is always complete. Use to disambiguate what a PR actually did+ or to catch user-visible changes not well described in PR text.Write a helpful narrative that goes at the top of the release page. Lead