Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Terminal output strips the C1 control range, U+0080-U+009F.
``soup_cli.utils.terminal`` holds the one table in the tree of characters that
must never reach the terminal. It covered C0 (minus TAB / LF / CR) and DEL, but
not C1: U+0080-U+009F passed through ``strip_control`` and ``for_terminal``
unchanged, and some terminals act on a C1 character in UTF-8 output the way
they act on ESC (U+009B is a one-character CSI). Every printer built on the two
helpers inherited that, so the range is pinned here at both edges and through
two real printers.
The ``.can`` manifest ``author`` field now refuses control characters outright
(``unicodedata.category == "Cc"``: C0, DEL and C1) instead of only NUL / CR /
LF -- an author handle has no use for any of them.
Every non-ASCII character below is built with ``chr()`` so this file stays
ASCII.
"""
from __future__ import annotations
import io
import tarfile
from io import StringIO
import pytest
from rich.console import Console
from typer.testing import CliRunner
runner = CliRunner()
BACKSLASH = chr(92)
C1_CODES = tuple(range(0x80, 0xA0))
def _c1_found(text: str) -> list[str]:
return [hex(ord(ch)) for ch in text if 0x80 <= ord(ch) <= 0x9F]
def _plain_console(buffer: StringIO) -> Console:
return Console(file=buffer, color_system=None, width=400)
class TestTheHelpersStripC1:
@pytest.mark.parametrize("code", C1_CODES, ids=[f"{code:#04x}" for code in C1_CODES])
def test_every_c1_code_point_is_stripped(self, code: int) -> None:
from soup_cli.utils.terminal import for_terminal, strip_control
assert strip_control(f"a{chr(code)}b") == "ab"
assert for_terminal(f"a{chr(code)}b") == "ab"
@pytest.mark.parametrize(
"code, kept",
[(0x7E, True), (0x7F, False), (0x80, False), (0x9F, False), (0xA0, True), (0xA1, True)],
ids=["0x7e-tilde", "0x7f-del", "0x80-first-c1", "0x9f-last-c1", "0xa0-nbsp", "0xa1"],
)
def test_the_range_edges(self, code: int, kept: bool) -> None:
from soup_cli.utils.terminal import for_terminal, strip_control
expected = f"a{chr(code)}b" if kept else "ab"
assert strip_control(f"a{chr(code)}b") == expected
assert for_terminal(f"a{chr(code)}b") == expected
def test_printable_text_outside_ascii_is_left_alone(self) -> None:
from soup_cli.utils.terminal import for_terminal, strip_control
text = "caf" + chr(0xE9) + " " + chr(0x65E5) + chr(0x672C) + " " + chr(0x2014) + " x"
assert strip_control(text) == text
assert for_terminal(text) == text
def test_tab_newline_and_carriage_return_are_still_kept(self) -> None:
from soup_cli.utils.terminal import strip_control
assert strip_control("a\tb\nc\rd") == "a\tb\nc\rd"
def test_the_strip_runs_before_the_escape(self) -> None:
"""A C1 character between a backslash and a tag must not end up live markup.
Escaping first would see ``backslash, C1, [bold]`` (no backslash
directly before the bracket), add one backslash, and the later strip
would leave two -- which Rich reads as a literal backslash followed by
a LIVE tag. Stripping first lets the escape see the real neighbours.
"""
from soup_cli.utils.terminal import for_terminal
value = BACKSLASH + chr(0x9B) + "[bold]x[/bold]"
buffer = StringIO()
_plain_console(buffer).print(for_terminal(value))
assert buffer.getvalue().rstrip("\n") == BACKSLASH + "[bold]x[/bold]"
def _write_can(path: str, manifest: str, config: str = "base: m\n") -> str:
with tarfile.open(path, "w:gz") as archive:
for name, body in (("manifest.yaml", manifest), ("config.yaml", config)):
payload = body.encode("utf-8")
info = tarfile.TarInfo(name)
info.size = len(payload)
archive.addfile(info, io.BytesIO(payload))
return path
def _yaml_escape(code: int) -> str:
"""The YAML double-quoted spelling of one code point, e.g. ``\\x9b``."""
return f"{BACKSLASH}x{code:02x}"
class TestCanInspectPrintsNoC1:
MANIFEST = (
"can_format_version: 1\n"
'name: "n"\n'
'author: "a"\n'
'created_at: "2026-01-01"\n'
'base_hash: "h"\n'
f'tags: ["t{_yaml_escape(0x9B)}31m"]\n'
f'description: "{_yaml_escape(0x9D)}0;TITLE{_yaml_escape(0x9C)}d"\n'
)
@pytest.mark.parametrize("force_terminal", [False, True], ids=["plain", "terminal"])
def test_description_and_tags(self, tmp_path, monkeypatch, force_terminal: bool) -> None:
from soup_cli.cans.unpack import inspect_can
from soup_cli.cli import app
from soup_cli.commands import can as can_cmd
monkeypatch.chdir(tmp_path)
_write_can("c.can", self.MANIFEST)
manifest = inspect_can("c.can")
assert _c1_found(manifest.description), "fixture must carry C1 characters"
assert _c1_found(manifest.tags[0]), "fixture must carry C1 characters"
buffer = StringIO()
console = (
Console(file=buffer, force_terminal=True, width=400)
if force_terminal
else _plain_console(buffer)
)
monkeypatch.setattr(can_cmd, "console", console)
result = runner.invoke(app, ["can", "inspect", "c.can"])
assert result.exit_code == 0, (result.output, repr(result.exception))
out = buffer.getvalue()
assert "TITLE" in out and "31m" in out, out
assert _c1_found(out) == [], repr(out)
class TestTheConfigLoaderPrintsNoC1:
def test_an_unknown_key_carrying_c1_is_named_without_it(self, tmp_path, monkeypatch) -> None:
from soup_cli.config import loader
buffer = StringIO()
monkeypatch.setattr(loader, "console", _plain_console(buffer))
path = tmp_path / "soup.yaml"
path.write_text(
"base: test-model\n"
"data:\n"
" train: ./data.jsonl\n"
"training:\n"
f' "{_yaml_escape(0x9B)}2Jquantizaton": 4bit\n',
encoding="utf-8",
)
with pytest.raises(SystemExit):
loader.load_config(path)
out = buffer.getvalue()
assert "2Jquantizaton" in out, out
assert _c1_found(out) == [], repr(out)
class TestManifestAuthorRefusesControlCharacters:
@staticmethod
def _manifest(author: str):
from soup_cli.cans.schema import Manifest
return Manifest(
can_format_version=1,
name="x",
author=author,
created_at="2026-04-20",
base_hash="abc",
)
@pytest.mark.parametrize(
"code",
[0x00, 0x09, 0x0A, 0x0D, 0x1B, 0x7F, 0x80, 0x85, 0x9B, 0x9F],
ids=["nul", "tab", "lf", "cr", "esc", "del", "0x80", "nel", "csi", "0x9f"],
)
def test_a_control_character_is_refused_naming_the_field(self, code: int) -> None:
with pytest.raises(ValueError, match="author") as excinfo:
self._manifest(f"alice{chr(code)}x")
assert "control" in str(excinfo.value)
@pytest.mark.parametrize(
"author",
["alice", "J" + chr(0xF6) + "rg", "[team] alice", "a" + chr(0xA0) + "b"],
ids=["ascii", "latin1-letter", "brackets", "nbsp"],
)
def test_printable_authors_are_still_accepted(self, author: str) -> None:
assert self._manifest(author).author == author
def test_can_inspect_names_the_refused_field(self, tmp_path, monkeypatch) -> None:
from soup_cli.cli import app
from soup_cli.commands import can as can_cmd
monkeypatch.chdir(tmp_path)
_write_can(
"c.can",
"can_format_version: 1\n"
'name: "n"\n'
f'author: "a{_yaml_escape(0x9B)}2K"\n'
'created_at: "2026-01-01"\n'
'base_hash: "h"\n',
)
buffer = StringIO()
monkeypatch.setattr(can_cmd, "console", _plain_console(buffer))
result = runner.invoke(app, ["can", "inspect", "c.can"])
assert result.exit_code == 1, (result.output, repr(result.exception))
out = buffer.getvalue()
assert "Cannot inspect can" in out and "author" in out, out
assert _c1_found(out) == [], repr(out)
BIDI_AND_INVISIBLE_CODES = (
0x061C,
0x200B,
0x200E,
0x200F,
0x2028,
0x2029,
0x202A,
0x202B,
0x202C,
0x202D,
0x202E,
0x2066,
0x2067,
0x2068,
0x2069,
0xFEFF,
)
class TestTheHelpersStripBidiAndInvisibleFormatCharacters:
"""Bidi controls and invisible separators reorder or hide what a preview shows."""
@pytest.mark.parametrize(
"code", BIDI_AND_INVISIBLE_CODES, ids=[f"{code:#06x}" for code in BIDI_AND_INVISIBLE_CODES]
)
def test_every_listed_code_point_is_stripped(self, code: int) -> None:
from soup_cli.utils.terminal import for_terminal, strip_control
assert strip_control(f"a{chr(code)}b") == "ab"
assert for_terminal(f"a{chr(code)}b") == "ab"
def test_a_right_to_left_override_cannot_reorder_the_text(self) -> None:
from soup_cli.utils.terminal import for_terminal
assert for_terminal("say " + chr(0x202E) + "evil" + chr(0x202C)) == "say evil"
@pytest.mark.parametrize("code", [0x200C, 0x200D], ids=["zwnj", "zwj"])
def test_zero_width_joiners_are_kept(self, code: int) -> None:
from soup_cli.utils.terminal import for_terminal, strip_control
assert strip_control(f"a{chr(code)}b") == f"a{chr(code)}b"
assert for_terminal(f"a{chr(code)}b") == f"a{chr(code)}b"
def test_a_zwj_emoji_sequence_is_left_intact(self) -> None:
from soup_cli.utils.terminal import for_terminal
family = chr(0x1F468) + chr(0x200D) + chr(0x1F469) + chr(0x200D) + chr(0x1F467)
assert for_terminal(family) == family
def test_a_persian_word_with_zwnj_is_left_intact(self) -> None:
from soup_cli.utils.terminal import for_terminal
codes = (0x645, 0x6CC, 0x200C, 0x62E, 0x648, 0x627, 0x647, 0x645)
word = "".join(chr(code) for code in codes)
assert for_terminal(word) == word
@pytest.mark.parametrize("code", [0x200A, 0x2030, 0x2060, 0x2065, 0x206A], ids=hex)
def test_neighbouring_code_points_are_not_swept_up(self, code: int) -> None:
from soup_cli.utils.terminal import strip_control
assert strip_control(f"a{chr(code)}b") == f"a{chr(code)}b"