Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

11.5 KB
Raw Blame
"""#1125 — no tracked file may carry a committed Git conflict marker.

``docs/performance-and-quantization.md`` shipped literal ``<<<<<<< HEAD`` /
``=======`` / ``>>>>>>> <sha>`` lines in #1066 with all 15 CI contexts green.

Two reasons it stayed green:

* ``.pre-commit-config.yaml`` ships ``check-merge-conflict``, but **CI never runs
  pre-commit** — the ``lint`` job runs only ``ruff check`` and ``actionlint``.
* ``ruff`` does not read Markdown, so ``docs/`` is outside its paths anyway. A
  marker in a Markdown file is invisible twice over.

So this is a repo-wide scan over every tracked text file, run as a pytest test
rather than a workflow step: it lands on all nine matrix cells, is visible to
contributors running ``pytest`` locally, and cannot be skipped by a fork's
permissions. It deliberately does NOT add ``pre-commit run --all-files`` to CI —
that is the larger question the issue defers.

**The ``=======`` false positive.** A bare line of seven equals signs is legal
Markdown (a setext heading underline) and appears in ReST. Matching it on its
own would fire on legitimate documentation. So ``=======`` is reported only when
the *same file* also contains a ``<<<<<<<`` or ``>>>>>>>`` line — i.e. when it is
part of an actual conflict block, which is exactly how Git emits them. The two
angle-bracket markers are unambiguous and always reported.

Diff3/zdiff3 conflict blocks also contain a ``||||||| base`` ancestor marker,
which is detected alongside the standard Git conflict markers.

Binary files are detected by attempting a strict UTF-8 decode; a file that fails
is treated as binary and skipped rather than crashing the scan.
"""

from __future__ import annotations

import re
import subprocess
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parent.parent

#: Conflict-marker forms Git writes, anchored at column 0. The angle-bracket
#: markers carry a trailing space then a branch/sha label; the divider is seven
#: equals signs to end of line. Diff3 adds seven pipe characters for the
#: common ancestor marker.
_START = re.compile(r"^<{7}(?: .*)?$", re.M)
_END = re.compile(r"^>{7}(?: .*)?$", re.M)
_DIVIDER = re.compile(r"^={7}$", re.M)
_DIFF3_BASE = re.compile(r"^\|{7}(?: .*)?$", re.M)


def _tracked_files() -> list[Path]:
    try:
        out = subprocess.run(
            ["git", "ls-files", "-z"],
            cwd=REPO_ROOT,
            capture_output=True,
            check=True,
        )
    except (subprocess.CalledProcessError, FileNotFoundError):
        pytest.skip("guard requires a git checkout")

    return [REPO_ROOT / name for name in out.stdout.decode("utf-8").split("\0") if name]


def _read_text(path: Path) -> str | None:
    """Return the decoded text, or ``None`` for a file we treat as binary."""
    try:
        raw = path.read_bytes()
    except OSError:
        return None

    try:
        return raw.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n")
    except UnicodeDecodeError:
        return None


def find_conflict_markers(text: str) -> list[tuple[int, str]]:
    """Return ``(lineno, marker_kind)`` for each conflict-marker line.

    ``marker_kind`` is one of ``"<<<<<<<"``, ``"|||||||"``,
    ``"======="``, ``">>>>>>>"``.

    A standalone ``=======`` with no companion angle-bracket marker in the same
    text is NOT returned — that is the setext-heading / ReST case the guard must
    tolerate.
    """
    starts = [m.start() for m in _START.finditer(text)]
    bases = [m.start() for m in _DIFF3_BASE.finditer(text)]
    ends = [m.start() for m in _END.finditer(text)]

    has_block_context = bool(starts or ends)

    found: list[tuple[int, str]] = []

    for offset in starts:
        lineno = text.count("\n", 0, offset) + 1
        found.append((lineno, "<<<<<<<"))

    for offset in bases:
        lineno = text.count("\n", 0, offset) + 1
        found.append((lineno, "|||||||"))

    for offset in ends:
        lineno = text.count("\n", 0, offset) + 1
        found.append((lineno, ">>>>>>>"))

    if has_block_context:
        for match in _DIVIDER.finditer(text):
            lineno = text.count("\n", 0, match.start()) + 1
            found.append((lineno, "======="))

    found.sort()
    return found


def _scan_repo() -> list[str]:
    problems: list[str] = []

    for path in _tracked_files():
        text = _read_text(path)

        if text is None:
            continue

        rel = path.relative_to(REPO_ROOT).as_posix()

        for lineno, kind in find_conflict_markers(text):
            problems.append(f"{rel}:{lineno}: stray conflict marker {kind}")

    return problems


class TestNoCommittedConflictMarkers:
    def test_non_git_checkout_is_skipped(self, monkeypatch):
        def fail_git(*args, **kwargs):
            raise subprocess.CalledProcessError(128, "git")

        monkeypatch.setattr(subprocess, "run", fail_git)

        with pytest.raises(
            pytest.skip.Exception,
            match="guard requires a git checkout",
        ):
            _tracked_files()

    def test_no_tracked_file_carries_a_conflict_marker(self):
        problems = _scan_repo()

        assert not problems, (
            "These tracked files contain a committed Git conflict marker. "
            "Resolve the conflict and remove the marker lines before merging "
            "(see #1125):\n  " + "\n  ".join(problems)
        )

    def test_the_scan_actually_covers_the_regression_docs_file(
        self, monkeypatch
    ):
        """The repo scan must actually read the regression docs file."""
        target = REPO_ROOT / "docs" / "performance-and-quantization.md"
        original_read_text = _read_text

        def fake_read_text(path):
            text = original_read_text(path)
            if path == target and text is not None:
                return "<<<<<<< injected\n" + text
            return text

        monkeypatch.setattr(
            "tests.test_no_committed_conflict_markers._read_text",
            fake_read_text,
        )

        problems = _scan_repo()

        assert any(
            problem.startswith("docs/performance-and-quantization.md:")
            for problem in problems
        ), problems[:5]
        assert (
            "docs/performance-and-quantization.md:1: stray conflict marker <<<<<<<"
            in problems
        ), problems[:5]

    def test_the_repo_scan_checks_every_tracked_text_file(self, monkeypatch):
        """Plant a marker in every file the scan reads; every text file must be reported."""
        original_read_text = _read_text

        def planted(path):
            text = original_read_text(path)
            return None if text is None else "<<<<<<< planted\n" + text

        monkeypatch.setattr(
            "tests.test_no_committed_conflict_markers._read_text",
            planted,
        )

        reported = {problem.rsplit(":", 2)[0] for problem in _scan_repo()}
        # The oracle must not come from the code under test: list the index
        # directly and decide text-ness here, so narrowing _tracked_files()
        # or _read_text() cannot narrow both sides of the comparison.
        listed = subprocess.run(
            ["git", "ls-files", "-z"],
            cwd=REPO_ROOT,
            capture_output=True,
            check=True,
        ).stdout.decode("utf-8")

        def is_utf8_text(name):
            try:
                (REPO_ROOT / name).read_bytes().decode("utf-8")
            except (OSError, UnicodeDecodeError):
                return False
            return True

        expected = {name for name in listed.split("\0") if name and is_utf8_text(name)}

        assert reported == expected, sorted(expected ^ reported)[:10]


class TestTheScannerCanActuallyFail:
    """The repo is clean, so these are the only proof the scanner works."""

    def test_it_catches_a_start_marker(self):
        text = "# header\n<<<<<<< HEAD\nours\n"
        found = find_conflict_markers(text)
        assert (2, "<<<<<<<") in found, found

    def test_it_catches_a_diff3_base_marker(self):
        text = (
            "<<<<<<< HEAD\n"
            "ours\n"
            "||||||| base\n"
            "base\n"
            "=======\n"
            "theirs\n"
            ">>>>>>> topic\n"
        )

        found = find_conflict_markers(text)
        assert (3, "|||||||") in found, found

    def test_it_catches_unlabelled_markers(self):
        text = "<<<<<<<\nours\n|||||||\nbase\n=======\ntheirs\n>>>>>>>\n"
        assert find_conflict_markers(text) == [
            (1, "<<<<<<<"),
            (3, "|||||||"),
            (5, "======="),
            (7, ">>>>>>>"),
        ]

    def test_it_catches_an_end_marker(self):
        text = "# header\ntheir\n>>>>>>> abc1234\n"
        found = find_conflict_markers(text)
        assert (3, ">>>>>>>") in found, found

    def test_it_catches_a_complete_conflict_block(self):
        text = "before\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> topic\nafter\n"
        kinds = {kind for _, kind in find_conflict_markers(text)}
        assert kinds == {"<<<<<<<", "=======", ">>>>>>>"}

    def test_crlf_conflict_block_reports_divider(self, tmp_path: Path):
        target = tmp_path / "crlf.txt"
        target.write_bytes(
            b"before\r\n<<<<<<< HEAD\r\nours\r\n=======\r\ntheirs\r\n>>>>>>> topic\r\n"
        )

        text = _read_text(target)
        assert text is not None

        found = find_conflict_markers(text)

        assert found == [
            (2, "<<<<<<<"),
            (4, "======="),
            (6, ">>>>>>>"),
        ], found

    def test_a_standalone_divider_is_not_flagged(self):
        """CONTROL. Seven equals signs alone is a setext underline / ReST rule."""
        text = "Title\n=======\nsome body prose\n"
        assert find_conflict_markers(text) == []

    def test_many_equals_signes_without_angle_markers_are_clean(self):
        text = "\n".join(["===="] * 5) + "\n=======\n"
        assert find_conflict_markers(text) == []

    def test_the_divider_only_counts_when_a_block_marker_shares_the_file(self):
        ours = "=======\n"
        theirs = "<<<<<<< HEAD\n=======\n"

        assert find_conflict_markers(ours) == []

        divider_lines = [
            line
            for line, kind in find_conflict_markers(theirs)
            if kind == "======="
        ]

        assert divider_lines == [2], theirs

    def test_reported_line_numbers_are_one_indexed_and_correct(self):
        text = "line1\nline2\n<<<<<<< HEAD\nline4\n"
        found = find_conflict_markers(text)
        assert found == [(3, "<<<<<<<")], found


class TestReadTextTreatsUndecodableAsBinary:
    def test_undecodable_bytes_return_none(self, tmp_path: Path):
        target = tmp_path / "blob.bin"
        target.write_bytes(b"\xff\xfe\x00\x01\x80\x81")
        assert _read_text(target) is None

    def test_valid_utf8_returns_text(self, tmp_path: Path):
        target = tmp_path / "ok.txt"
        target.write_bytes("héllo ✓\n".encode("utf-8"))
        assert _read_text(target) == "héllo ✓\n"


class TestFailureNamesFileAndLine:
    def test_a_planted_marker_produces_a_diagnostic_with_path_and_line(
        self, tmp_path: Path
    ):
        offender = tmp_path / "doc.md"
        offender.write_text(
            "intro\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> x\n",
            encoding="utf-8",
        )

        rel = "doc.md"
        problems = [
            f"{rel}:{lineno}: stray conflict marker {kind}"
            for lineno, kind in find_conflict_markers(
                offender.read_text(encoding="utf-8")
            )
        ]

        joined = "\n".join(problems)

        assert "doc.md:" in joined
        assert ":2:" in joined
        assert "<<<<<<<" in joined