Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Scan delivered source history instead of an empty merge log
pulse-triage[bot]3aaa967bba91
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex 83d4e8f72a..0e58a57457 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -48,17 +48,18 @@ jobs:tar -xzf "/tmp/${archive}" -C /tmp gitleaksinstall /tmp/gitleaks /usr/local/bin/gitleaks+ - name: Validate delivered-history secret scan+ run: python3 scripts/tests/test_ci_secret_scan.py+- name: Run gitleaks+ env:+ EXPECTED_HEAD: ${{ github.sha }}+ DIFF_BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}+ # A merge's latest log entry can have no patch. Scan all newly+ # delivered commits and its first-parent resolution, not just -1.+ # Missing/non-ancestor bases scan the complete checkout history.run: |- set -euo pipefail- gitleaks detect \- --source . \- --log-opts=-1 \- --no-banner \- --redact \- --exit-code 2 \- --report-format sarif \- --report-path results.sarif+ bash scripts/scan-ci-secrets.sh "$EXPECTED_HEAD" "$DIFF_BASE"# The secret scan above runs on every push including docs-only ones — this# workflow is the only per-push gitleaks pass, so it must not be pathdiff --git a/scripts/scan-ci-secrets.sh b/scripts/scan-ci-secrets.shnew file mode 100644index 0000000000..dd5d7cf49f--- /dev/null+++ b/scripts/scan-ci-secrets.sh@@ -0,0 +1,57 @@+#!/usr/bin/env bash+# Scan the delivered history, including merge-resolution bytes. Keep gitleaks'+# pinned detector, redaction, exact fingerprints and failure exits unchanged.+set -euo pipefail++expected_head="${1:-}"+base="${2:-}"+if [[ ! "$expected_head" =~ ^[0-9a-f]{40}$ ]]; then+ echo "::error::Secret scan requires an exact expected checkout SHA." >&2+ exit 1+fi+head=$(git rev-parse --verify 'HEAD^{commit}')+if [[ "$head" != "$expected_head" ]]; then+ echo "::error::Secret scan checkout does not match the expected source." >&2+ exit 1+fi+if [[ "$(git rev-parse --is-shallow-repository)" != false ]]; then+ echo "::error::Secret scan requires complete checkout history (fetch-depth: 0)." >&2+ exit 1+fi++# workflow_dispatch, new branches, missing objects and force-pushes do not+# establish an incremental range. Never silently narrow those cases to HEAD^.+revision="$head"+mode="complete checkout history"+if [[ "$base" =~ ^[0-9a-f]{40}$ && "$base" != 0000000000000000000000000000000000000000 && "$base" != "$head" ]] \+ && git cat-file -e "$base^{commit}" 2>/dev/null; then+ if git merge-base --is-ancestor "$base" "$head"; then+ revision="$base..$head"+ mode="delivered commit range"+ else+ status=$?+ if [[ "$status" != 1 ]]; then+ echo "::error::Secret scan could not establish checkout ancestry." >&2+ exit "$status"+ fi+ fi+fi++commits=$(git rev-list --count "$revision" --)+if [[ ! "$commits" =~ ^[1-9][0-9]*$ ]]; then+ echo "::error::Secret scan did not establish a nonempty history." >&2+ exit 1+fi+printf 'Secret scan scope: %s; %s reachable commits; head=%s\n' "$mode" "$commits" "$head"++# This changes merge *diffs*, not traversal: side-branch commits remain in the+# range, including credentials removed before the final tree. The first-parent+# merge diff also covers a resolution authored only in the merge itself.+gitleaks detect \+ --source . \+ --log-opts="--full-history --diff-merges=first-parent --no-ext-diff --no-textconv --no-renames $revision --" \+ --no-banner \+ --redact \+ --exit-code 2 \+ --report-format sarif \+ --report-path results.sarifdiff --git a/scripts/tests/test_ci_secret_scan.py b/scripts/tests/test_ci_secret_scan.pynew file mode 100644index 0000000000..64db791a59--- /dev/null+++ b/scripts/tests/test_ci_secret_scan.py@@ -0,0 +1,232 @@+#!/usr/bin/env python3+"""Exercise the production scan command against real isolated Git histories.++The detector double inspects the exact Git patch stream, not secret heuristics.+Normal CI still owns the pinned gitleaks detector's actual verdict.+"""++import json+import os+from pathlib import Path+import re+import shutil+import subprocess+import tempfile+import unittest+++ROOT = Path(__file__).resolve().parents[2]+SCRIPT = ROOT / "scripts/scan-ci-secrets.sh"+WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"+CANARY = "DELIVERED_HISTORY_FIXTURE_MARKER"++DETECTOR = r'''#!/usr/bin/env python3+import json, os, pathlib, shlex, subprocess, sys+args = sys.argv[1:]+opts = next(arg.split("=", 1)[1] for arg in args if arg.startswith("--log-opts="))+result = subprocess.run(["git", "log", "-p", "-U0", *shlex.split(opts)],+ capture_output=True, text=True)+pathlib.Path(os.environ["DETECTOR_RECORD"]).write_text(json.dumps({+ "args": args, "log_opts": opts, "git_exit": result.returncode,+ "marker_in_patch": "DELIVERED_HISTORY_FIXTURE_MARKER" in result.stdout,+}))+sys.exit(result.returncode or int(os.environ.get("DETECTOR_EXIT", "0")))+'''+++class DeliveredHistoryScanTest(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name)+ self.repo = self.root / "repo"+ self.repo.mkdir()+ self.bin = self.root / "bin"+ self.bin.mkdir()+ detector = self.bin / "gitleaks"+ detector.write_text(DETECTOR)+ detector.chmod(0o755)+ self.record = self.root / "detector.json"+ self.env = {**os.environ, "PATH": f"{self.bin}:{os.environ['PATH']}",+ "DETECTOR_RECORD": str(self.record)}+ self.git("init", "-q", "--initial-branch=main")+ self.write("fixture.txt", "original\n")+ self.base = self.commit("Base fixture")++ def git(self, *args):+ return subprocess.check_output([+ "git", "-c", "user.name=Secret scan fixture",+ "-c", "[email protected]", "-c", "core.hooksPath=/dev/null",+ *args,+ ], cwd=self.repo, stderr=subprocess.PIPE, text=True).strip()++ def write(self, name, value):+ path = self.repo / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(value)++ def commit(self, message):+ self.git("add", "-A")+ self.git("commit", "-q", "-m", message)+ return self.git("rev-parse", "HEAD")++ def scan(self, base=None, head=None, env=None):+ if self.record.exists():+ self.record.unlink()+ return subprocess.run(["bash", str(SCRIPT), head or self.git("rev-parse", "HEAD"),+ self.base if base is None else base],+ cwd=self.repo, env={**self.env, **(env or {})},+ capture_output=True, text=True, timeout=10)++ def admitted(self, result):+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ record = json.loads(self.record.read_text())+ self.assertEqual(record["git_exit"], 0)+ return record++ def test_entire_multicommit_push_includes_value_removed_before_head(self):+ self.write("fixture.txt", CANARY + "\n")+ self.commit("Introduced marker")+ self.write("fixture.txt", "removed before publication\n")+ head = self.commit("Removed marker")+ self.write("receipt.json", "{}\n")+ final = self.commit("Later evidence commit")+ self.assertNotIn(CANARY, self.git("log", "-p", "-U0", "-1"))+ record = self.admitted(self.scan())+ self.assertTrue(record["marker_in_patch"])+ self.assertIn(f"{self.base}..{final}", record["log_opts"])+ self.assertNotIn(head + "..", record["log_opts"])++ def test_merged_side_commits_are_scanned_even_when_merge_patch_is_empty(self):+ self.git("checkout", "-q", "-b", "proposal")+ self.write("fixture.txt", CANARY + "\n")+ self.commit("Introduced on proposal")+ self.write("fixture.txt", "original\n")+ self.commit("Removed on proposal")+ self.git("checkout", "-q", "main")+ self.git("merge", "-q", "--no-ff", "-m", "Integrated proposal", "proposal")+ old_stream = self.git("log", "-p", "-U0", "-1")+ self.assertNotIn("diff --git", old_stream)+ record = self.admitted(self.scan())+ self.assertTrue(record["marker_in_patch"])+ self.assertNotIn("--first-parent", record["log_opts"])++ def test_merge_only_resolution_is_part_of_the_actual_patch_stream(self):+ self.git("checkout", "-q", "-b", "proposal")+ self.write("feature.txt", "proposal\n")+ self.commit("Proposal")+ self.git("checkout", "-q", "main")+ self.write("main.txt", "main\n")+ base = self.commit("Advanced main")+ self.git("merge", "-q", "--no-ff", "--no-commit", "proposal")+ self.write("resolution.txt", CANARY + "\n")+ head = self.commit("Integrated with a merge-only change")+ self.assertNotIn(CANARY, self.git("log", "-p", "-U0", "-1"))+ record = self.admitted(self.scan(base=base))+ self.assertTrue(record["marker_in_patch"])+ self.assertIn("--diff-merges=first-parent", record["log_opts"])+ self.assertIn(f"{base}..{head}", record["log_opts"])++ def test_full_history_fallback_for_absent_zero_missing_malformed_or_equal_base(self):+ self.write("fixture.txt", CANARY + "\n")+ marker = self.commit("Historic marker")+ self.write("fixture.txt", "clean final tree\n")+ head = self.commit("Removed marker")+ for base in ("", "0" * 40, "f" * 40, "HEAD^", head, "--all; unexpected"):+ with self.subTest(base=base):+ result = self.scan(base=base)+ record = self.admitted(result)+ self.assertTrue(record["marker_in_patch"])+ self.assertIn("complete checkout history", result.stdout)+ self.assertIn(f" {head} --", record["log_opts"])+ self.assertNotIn("..", record["log_opts"])+ self.assertNotIn(marker + "..", record["log_opts"])++ def test_nonancestor_base_falls_back_to_complete_checkout_history(self):+ self.git("checkout", "-q", "-b", "unrelated")+ self.write("other.txt", "other\n")+ other = self.commit("Other branch")+ self.git("checkout", "-q", "main")+ self.write("fixture.txt", CANARY + "\n")+ self.commit("Main marker")+ result = self.scan(base=other)+ record = self.admitted(result)+ self.assertTrue(record["marker_in_patch"])+ self.assertNotIn("..", record["log_opts"])+ self.assertIn("complete checkout history", result.stdout)++ def test_shallow_history_is_a_failure_not_a_narrow_scan(self):+ self.write("new.txt", "new\n")+ self.commit("Second commit")+ shallow = self.root / "shallow"+ self.git("clone", "-q", "--depth", "1", self.repo.as_uri(), str(shallow))+ self.repo = shallow+ result = self.scan()+ self.assertEqual(result.returncode, 1)+ self.assertIn("complete checkout history", result.stderr)+ self.assertFalse(self.record.exists())++ def test_invalid_or_mismatched_expected_head_never_invokes_detector(self):+ for head in ("f" * 40, "HEAD", "0" * 40, "--all; unexpected"):+ with self.subTest(head=head):+ result = self.scan(head=head)+ self.assertEqual(result.returncode, 1)+ self.assertFalse(self.record.exists())++ def test_detector_finding_and_operational_failure_exits_are_preserved(self):+ self.write("new.txt", "new\n")+ self.commit("Second commit")+ for code in (2, 42):+ with self.subTest(exit=code):+ result = self.scan(env={"DETECTOR_EXIT": str(code)})+ self.assertEqual(result.returncode, code)+ self.assertTrue(self.record.exists())++ def test_git_ancestry_error_cannot_be_treated_as_a_divergent_base(self):+ self.write("new.txt", "new\n")+ self.commit("Second commit")+ real_git = shutil.which("git")+ wrapper = self.bin / "git"+ wrapper.write_text(f'#!/bin/bash\nif [[ "$1" == merge-base ]]; then exit 3; fi\nexec "{real_git}" "$@"\n')+ wrapper.chmod(0o755)+ result = self.scan()+ self.assertEqual(result.returncode, 3)+ self.assertIn("checkout ancestry", result.stderr)+ self.assertFalse(self.record.exists())++ def test_actual_detector_command_preserves_source_redaction_and_fatal_verdict(self):+ self.write("new.txt", "new\n")+ self.commit("Second commit")+ record = self.admitted(self.scan())+ args = record["args"]+ self.assertEqual(args[:3], ["detect", "--source", "."])+ self.assertEqual(args[4:], ["--no-banner", "--redact", "--exit-code", "2",+ "--report-format", "sarif", "--report-path", "results.sarif"])+ for flag in ("--full-history", "--no-ext-diff", "--no-textconv", "--no-renames"):+ self.assertIn(flag, record["log_opts"])+ for forbidden in ("-1", "--all", "--first-parent", "--since", "--author"):+ self.assertNotIn(forbidden, record["log_opts"].split())+++class SecretScanWorkflowTest(unittest.TestCase):+ def test_unconditional_pinned_job_uses_complete_history_and_exact_event_sources(self):+ workflow = WORKFLOW.read_text()+ match = re.search(r"^ secret-scan:\n(.*?)(?=^ [\w-]+:)", workflow, re.M | re.S)+ self.assertIsNotNone(match)+ job = match[1]+ self.assertNotIn("if:", job)+ self.assertNotIn("continue-on-error:", job)+ self.assertIn('GITLEAKS_VERSION: "8.30.1"', job)+ self.assertIn('GITLEAKS_LINUX_X64_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb"', job)+ self.assertIn('"${GITLEAKS_LINUX_X64_SHA256} /tmp/${archive}" | sha256sum -c -', job)+ self.assertIn("persist-credentials: false", job)+ self.assertIn("fetch-depth: 0", job)+ self.assertIn("EXPECTED_HEAD: ${{ github.sha }}", job)+ self.assertIn("DIFF_BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}", job)+ self.assertIn('bash scripts/scan-ci-secrets.sh "$EXPECTED_HEAD" "$DIFF_BASE"', job)+ self.assertIn("python3 scripts/tests/test_ci_secret_scan.py", job)+ self.assertNotIn("--log-opts=-1", job)+++if __name__ == "__main__":+ unittest.main()