Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1125 — no tracked file may carry a committed Git conflict marker.
``docs/performance-and-quantization.md`` shipped literal ``<<<<<<< HEAD`` /
``=======`` / ``>>>>>>> <sha>`` lines in #1066 with all 15 CI contexts green.
Two reasons it stayed green:
* ``.pre-commit-config.yaml`` ships ``check-merge-conflict``, but **CI never runs
pre-commit** — the ``lint`` job runs only ``ruff check`` and ``actionlint``.
* ``ruff`` does not read Markdown, so ``docs/`` is outside its paths anyway. A
marker in a Markdown file is invisible twice over.
So this is a repo-wide scan over every tracked text file, run as a pytest test
rather than a workflow step: it lands on all nine matrix cells, is visible to
contributors running ``pytest`` locally, and cannot be skipped by a fork's
permissions. It deliberately does NOT add ``pre-commit run --all-files`` to CI —
that is the larger question the issue defers.
**The ``=======`` false positive.** A bare line of seven equals signs is legal
Markdown (a setext heading underline) and appears in ReST. Matching it on its
own would fire on legitimate documentation. So ``=======`` is reported only when
the *same file* also contains a ``<<<<<<<`` or ``>>>>>>>`` line — i.e. when it is
part of an actual conflict block, which is exactly how Git emits them. The two
angle-bracket markers are unambiguous and always reported.
Diff3/zdiff3 conflict blocks also contain a ``||||||| base`` ancestor marker,
which is detected alongside the standard Git conflict markers.
Binary files are detected by attempting a strict UTF-8 decode; a file that fails
is treated as binary and skipped rather than crashing the scan.
"""
from __future__ import annotations
import re
import subprocess
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
#: Conflict-marker forms Git writes, anchored at column 0. The angle-bracket
#: markers carry a trailing space then a branch/sha label; the divider is seven
#: equals signs to end of line. Diff3 adds seven pipe characters for the
#: common ancestor marker.
_START = re.compile(r"^<{7}(?: .*)?$", re.M)
_END = re.compile(r"^>{7}(?: .*)?$", re.M)
_DIVIDER = re.compile(r"^={7}$", re.M)
_DIFF3_BASE = re.compile(r"^\|{7}(?: .*)?$", re.M)
def _tracked_files() -> list[Path]:
try:
out = subprocess.run(
["git", "ls-files", "-z"],
cwd=REPO_ROOT,
capture_output=True,
check=True,
)
except (subprocess.CalledProcessError, FileNotFoundError):
pytest.skip("guard requires a git checkout")
return [REPO_ROOT / name for name in out.stdout.decode("utf-8").split("\0") if name]
def _read_text(path: Path) -> str | None:
"""Return the decoded text, or ``None`` for a file we treat as binary."""
try:
raw = path.read_bytes()
except OSError:
return None
try:
return raw.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n")
except UnicodeDecodeError:
return None
def find_conflict_markers(text: str) -> list[tuple[int, str]]:
"""Return ``(lineno, marker_kind)`` for each conflict-marker line.
``marker_kind`` is one of ``"<<<<<<<"``, ``"|||||||"``,
``"======="``, ``">>>>>>>"``.
A standalone ``=======`` with no companion angle-bracket marker in the same
text is NOT returned — that is the setext-heading / ReST case the guard must
tolerate.
"""
starts = [m.start() for m in _START.finditer(text)]
bases = [m.start() for m in _DIFF3_BASE.finditer(text)]
ends = [m.start() for m in _END.finditer(text)]
has_block_context = bool(starts or ends)
found: list[tuple[int, str]] = []
for offset in starts:
lineno = text.count("\n", 0, offset) + 1
found.append((lineno, "<<<<<<<"))
for offset in bases:
lineno = text.count("\n", 0, offset) + 1
found.append((lineno, "|||||||"))
for offset in ends:
lineno = text.count("\n", 0, offset) + 1
found.append((lineno, ">>>>>>>"))
if has_block_context:
for match in _DIVIDER.finditer(text):
lineno = text.count("\n", 0, match.start()) + 1
found.append((lineno, "======="))
found.sort()
return found
def _scan_repo() -> list[str]:
problems: list[str] = []
for path in _tracked_files():
text = _read_text(path)
if text is None:
continue
rel = path.relative_to(REPO_ROOT).as_posix()
for lineno, kind in find_conflict_markers(text):
problems.append(f"{rel}:{lineno}: stray conflict marker {kind}")
return problems
class TestNoCommittedConflictMarkers:
def test_non_git_checkout_is_skipped(self, monkeypatch):
def fail_git(*args, **kwargs):
raise subprocess.CalledProcessError(128, "git")
monkeypatch.setattr(subprocess, "run", fail_git)
with pytest.raises(
pytest.skip.Exception,
match="guard requires a git checkout",
):
_tracked_files()
def test_no_tracked_file_carries_a_conflict_marker(self):
problems = _scan_repo()
assert not problems, (
"These tracked files contain a committed Git conflict marker. "
"Resolve the conflict and remove the marker lines before merging "
"(see #1125):\n " + "\n ".join(problems)
)
def test_the_scan_actually_covers_the_regression_docs_file(
self, monkeypatch
):
"""The repo scan must actually read the regression docs file."""
target = REPO_ROOT / "docs" / "performance-and-quantization.md"
original_read_text = _read_text
def fake_read_text(path):
text = original_read_text(path)
if path == target and text is not None:
return "<<<<<<< injected\n" + text
return text
monkeypatch.setattr(
"tests.test_no_committed_conflict_markers._read_text",
fake_read_text,
)
problems = _scan_repo()
assert any(
problem.startswith("docs/performance-and-quantization.md:")
for problem in problems
), problems[:5]
assert (
"docs/performance-and-quantization.md:1: stray conflict marker <<<<<<<"
in problems
), problems[:5]
def test_the_repo_scan_checks_every_tracked_text_file(self, monkeypatch):
"""Plant a marker in every file the scan reads; every text file must be reported."""
original_read_text = _read_text
def planted(path):
text = original_read_text(path)
return None if text is None else "<<<<<<< planted\n" + text
monkeypatch.setattr(
"tests.test_no_committed_conflict_markers._read_text",
planted,
)
reported = {problem.rsplit(":", 2)[0] for problem in _scan_repo()}
# The oracle must not come from the code under test: list the index
# directly and decide text-ness here, so narrowing _tracked_files()
# or _read_text() cannot narrow both sides of the comparison.
listed = subprocess.run(
["git", "ls-files", "-z"],
cwd=REPO_ROOT,
capture_output=True,
check=True,
).stdout.decode("utf-8")
def is_utf8_text(name):
try:
(REPO_ROOT / name).read_bytes().decode("utf-8")
except (OSError, UnicodeDecodeError):
return False
return True
expected = {name for name in listed.split("\0") if name and is_utf8_text(name)}
assert reported == expected, sorted(expected ^ reported)[:10]
class TestTheScannerCanActuallyFail:
"""The repo is clean, so these are the only proof the scanner works."""
def test_it_catches_a_start_marker(self):
text = "# header\n<<<<<<< HEAD\nours\n"
found = find_conflict_markers(text)
assert (2, "<<<<<<<") in found, found
def test_it_catches_a_diff3_base_marker(self):
text = (
"<<<<<<< HEAD\n"
"ours\n"
"||||||| base\n"
"base\n"
"=======\n"
"theirs\n"
">>>>>>> topic\n"
)
found = find_conflict_markers(text)
assert (3, "|||||||") in found, found
def test_it_catches_unlabelled_markers(self):
text = "<<<<<<<\nours\n|||||||\nbase\n=======\ntheirs\n>>>>>>>\n"
assert find_conflict_markers(text) == [
(1, "<<<<<<<"),
(3, "|||||||"),
(5, "======="),
(7, ">>>>>>>"),
]
def test_it_catches_an_end_marker(self):
text = "# header\ntheir\n>>>>>>> abc1234\n"
found = find_conflict_markers(text)
assert (3, ">>>>>>>") in found, found
def test_it_catches_a_complete_conflict_block(self):
text = "before\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> topic\nafter\n"
kinds = {kind for _, kind in find_conflict_markers(text)}
assert kinds == {"<<<<<<<", "=======", ">>>>>>>"}
def test_crlf_conflict_block_reports_divider(self, tmp_path: Path):
target = tmp_path / "crlf.txt"
target.write_bytes(
b"before\r\n<<<<<<< HEAD\r\nours\r\n=======\r\ntheirs\r\n>>>>>>> topic\r\n"
)
text = _read_text(target)
assert text is not None
found = find_conflict_markers(text)
assert found == [
(2, "<<<<<<<"),
(4, "======="),
(6, ">>>>>>>"),
], found
def test_a_standalone_divider_is_not_flagged(self):
"""CONTROL. Seven equals signs alone is a setext underline / ReST rule."""
text = "Title\n=======\nsome body prose\n"
assert find_conflict_markers(text) == []
def test_many_equals_signes_without_angle_markers_are_clean(self):
text = "\n".join(["===="] * 5) + "\n=======\n"
assert find_conflict_markers(text) == []
def test_the_divider_only_counts_when_a_block_marker_shares_the_file(self):
ours = "=======\n"
theirs = "<<<<<<< HEAD\n=======\n"
assert find_conflict_markers(ours) == []
divider_lines = [
line
for line, kind in find_conflict_markers(theirs)
if kind == "======="
]
assert divider_lines == [2], theirs
def test_reported_line_numbers_are_one_indexed_and_correct(self):
text = "line1\nline2\n<<<<<<< HEAD\nline4\n"
found = find_conflict_markers(text)
assert found == [(3, "<<<<<<<")], found
class TestReadTextTreatsUndecodableAsBinary:
def test_undecodable_bytes_return_none(self, tmp_path: Path):
target = tmp_path / "blob.bin"
target.write_bytes(b"\xff\xfe\x00\x01\x80\x81")
assert _read_text(target) is None
def test_valid_utf8_returns_text(self, tmp_path: Path):
target = tmp_path / "ok.txt"
target.write_bytes("héllo ✓\n".encode("utf-8"))
assert _read_text(target) == "héllo ✓\n"
class TestFailureNamesFileAndLine:
def test_a_planted_marker_produces_a_diagnostic_with_path_and_line(
self, tmp_path: Path
):
offender = tmp_path / "doc.md"
offender.write_text(
"intro\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> x\n",
encoding="utf-8",
)
rel = "doc.md"
problems = [
f"{rel}:{lineno}: stray conflict marker {kind}"
for lineno, kind in find_conflict_markers(
offender.read_text(encoding="utf-8")
)
]
joined = "\n".join(problems)
assert "doc.md:" in joined
assert ":2:" in joined
assert "<<<<<<<" in joined