Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1422: an output path that aliases an input must be refused before any write.
Four commands (``ingest``, ``prune-prompt``, ``ra-dit`` and ``data best-of-n``)
opened their output for writing without checking that it was not one of their
inputs, so the write destroyed the input and the command still exited 0.
``ensure_distinct_paths`` is the one shared check. It compares real paths
(``normcase(realpath(...))``) and, for files that exist, ``os.path.samefile``,
so a spelling of the input (``./x``, a case variant on Windows) and a hard link
are refused as well as the identical string. It opens and creates nothing.
This file covers the shared helper and soup ingest.
Other commands have dedicated output-safety test modules.
"""
from __future__ import annotations
import os
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.utils.paths import ensure_distinct_paths
from tests.conftest import strip_ansi
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _in_tmp(tmp_path, monkeypatch):
# Soup refuses paths outside the working directory, so every test runs
# inside its own temporary directory and never touches the repository.
monkeypatch.chdir(tmp_path)
def _write(name: str, text: str = "x\n") -> str:
# newline="" keeps the bytes exactly as given on Windows, so the
# byte-identical assertions below compare what was really written.
with open(name, "w", encoding="utf-8", newline="") as fh:
fh.write(text)
return name
def _read(name: str) -> str:
with open(name, encoding="utf-8", newline="") as fh:
return fh.read()
def _flat(output: str) -> str:
# Rich wraps long messages at the terminal width; flatten the lines so an
# assertion on a phrase does not depend on where the wrap fell.
return " ".join(strip_ansi(output).split())
# ---------------------------------------------------------------------------
# The shared helper
# ---------------------------------------------------------------------------
class TestEnsureDistinctPaths:
def test_same_string_is_refused_and_names_both_options(self):
# The plain case from the issue: --logs is also the default --output.
_write("a.jsonl")
with pytest.raises(ValueError) as info:
ensure_distinct_paths([("--logs", "a.jsonl")], [("--output", "a.jsonl")])
message = str(info.value)
# The operator must be told which two options collide.
assert "--logs" in message
assert "--output" in message
def test_dot_slash_spelling_is_refused(self):
# A different spelling of the same file must not slip past a string compare.
_write("a.jsonl")
with pytest.raises(ValueError):
ensure_distinct_paths([("--logs", "./a.jsonl")], [("--output", "a.jsonl")])
def test_a_missing_file_spelled_the_same_is_refused(self):
# Nothing exists yet, so samefile cannot help; the real-path compare must.
with pytest.raises(ValueError):
ensure_distinct_paths([("--logs", "no.jsonl")], [("--output", "no.jsonl")])
@pytest.mark.skipif(os.name != "nt", reason="case-insensitive file system")
def test_case_variant_is_refused_on_windows(self):
# On Windows Traces.jsonl and traces.jsonl are one file.
_write("Traces.jsonl")
with pytest.raises(ValueError):
ensure_distinct_paths(
[("--logs", "Traces.jsonl")], [("--output", "traces.jsonl")]
)
@pytest.mark.skipif(os.name != "nt", reason="case-insensitive file system")
def test_case_variant_of_a_path_that_does_not_exist_yet_is_refused_on_windows(self):
with pytest.raises(ValueError):
ensure_distinct_paths([("--logs", "Missing.jsonl")], [("--output", "missing.jsonl")])
@pytest.mark.skipif(os.name != "nt", reason="case-insensitive file system")
def test_nested_case_variant_of_a_missing_directory_is_refused_on_windows(self):
with pytest.raises(ValueError):
ensure_distinct_paths(
[("--retriever-config output", "out/R1/stage")],
[("--generator-config output", "out/r1")],
reject_nested=True,
)
@pytest.mark.requires_symlink
def test_symlinked_directory_alias_of_a_missing_output_is_refused(self):
os.mkdir("real")
os.symlink("real", "link", target_is_directory=True)
with pytest.raises(ValueError):
ensure_distinct_paths(
[("--retriever-config output", "real/r1")],
[("--generator-config output", "link/r1")],
reject_nested=True,
)
with pytest.raises(ValueError):
ensure_distinct_paths([("--logs", "real/in.jsonl")], [("--output", "link/in.jsonl")])
with pytest.raises(ValueError):
ensure_distinct_paths(
[("--retriever-config output", "link/r1")],
[("--generator-config output", "real/r1")],
reject_nested=True,
)
with pytest.raises(ValueError):
ensure_distinct_paths([("--logs", "link/in.jsonl")], [("--output", "real/in.jsonl")])
@pytest.mark.requires_hardlink
def test_hard_link_is_refused(self):
# A hard link has a different path but is the same file on disk, so only
# os.path.samefile catches it.
_write("a.jsonl")
os.link("a.jsonl", "b.jsonl")
with pytest.raises(ValueError):
ensure_distinct_paths([("--input", "a.jsonl")], [("--output", "b.jsonl")])
def test_distinct_paths_pass(self):
# The control: the check is about aliasing, not about every pair of paths.
_write("a.jsonl")
ensure_distinct_paths([("--logs", "a.jsonl")], [("--output", "b.jsonl")])
def test_the_check_writes_nothing(self):
# It is called before any open-for-write, so it must not create or
# change a file itself.
_write("a.jsonl", "keep\n")
ensure_distinct_paths([("--logs", "a.jsonl")], [("--output", "b.jsonl")])
assert _read("a.jsonl") == "keep\n"
assert not os.path.exists("b.jsonl")
# ---------------------------------------------------------------------------
# soup ingest
# ---------------------------------------------------------------------------
class TestIngestRefusesItsOwnInput:
def test_default_output_equal_to_logs_is_refused(self):
# The issue's first reproduction: without --output, the default
# traces.jsonl is also --logs. On main this left a 0-byte file and exit 0.
original = '{"id": "t1", "input": "hi", "output": "hello"}\n'
_write("traces.jsonl", original)
result = CliRunner().invoke(
app, ["ingest", "--source", "langfuse", "--logs", "traces.jsonl"]
)
assert result.exit_code != 0, result.output
out = _flat(result.output)
assert "--logs" in out
assert "--output" in out
# The input survives byte for byte.
assert _read("traces.jsonl") == original
def test_a_different_output_still_works(self):
# The control: a normal run is unchanged by the new check.
_write("in.jsonl", '{"id": "t1", "input": "hi", "output": "hello"}\n')
result = CliRunner().invoke(
app,
["ingest", "--source", "langfuse", "--logs", "in.jsonl", "-o", "out.jsonl"],
)
assert result.exit_code == 0, result.output
assert os.path.getsize("out.jsonl") > 0
def test_dot_slash_spelling_of_logs_is_refused(self):
# The same file spelled ./traces.jsonl must be refused like traces.jsonl.
original = '{"id": "t1", "input": "hi", "output": "hello"}\n'
_write("traces.jsonl", original)
result = CliRunner().invoke(
app, ["ingest", "--source", "langfuse", "--logs", "./traces.jsonl"]
)
assert result.exit_code != 0, result.output
assert _read("traces.jsonl") == original
def test_explicit_output_equal_to_logs_is_refused(self):
# Not only the default output: -o pointing at --logs is the same bug.
original = '{"id": "t1", "input": "hi", "output": "hello"}\n'
_write("e.jsonl", original)
result = CliRunner().invoke(
app,
["ingest", "--source", "langfuse", "--logs", "e.jsonl", "-o", "e.jsonl"],
)
assert result.exit_code != 0, result.output
assert _read("e.jsonl") == original
def test_a_logs_directory_leaves_an_existing_output_untouched(self):
# A read failure must never leave a truncated output behind: on main this
# emptied a good earlier traces.jsonl before the read failed.
good = '{"id": "t1", "input": "hi", "output": "hello"}\n'
_write("traces.jsonl", good)
os.mkdir("some_dir")
result = CliRunner().invoke(
app, ["ingest", "--source", "langfuse", "--logs", "some_dir"]
)
assert result.exit_code != 0
assert _read("traces.jsonl") == good
def test_a_non_utf8_export_leaves_an_existing_output_untouched(self):
# Same property for an export the reader cannot decode.
good = '{"id": "t1", "input": "hi", "output": "hello"}\n'
_write("traces.jsonl", good)
with open("bad.jsonl", "wb") as fh:
fh.write(b"\xff\xfe\x00 not utf-8\n")
result = CliRunner().invoke(
app, ["ingest", "--source", "langfuse", "--logs", "bad.jsonl"]
)
assert result.exit_code != 0
assert _read("traces.jsonl") == good
# The staging file of the atomic write must be cleaned up too.
assert [n for n in os.listdir(".") if n.startswith(".soup.")] == []