Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for soup infer — batch inference command."""
import json
from unittest.mock import MagicMock, PropertyMock
import pytest
from tests.conftest import strip_ansi
class _BatchTokenizer:
def __init__(self, *, templated: bool):
self.chat_template = "dummy" if templated else None
self.pad_token_id = 0
self.padding_side = "right"
self.calls = []
self.padding_sides = []
self.template_calls = []
def apply_chat_template(self, messages, **kwargs):
self.template_calls.append((messages, kwargs))
return f"templated:{messages[0]['content']}"
def __call__(
self,
texts,
*,
add_special_tokens=True,
padding=False,
return_tensors=None,
):
import torch
is_batch = isinstance(texts, list)
text_rows = texts if is_batch else [texts]
self.calls.append(
{
"is_batch": is_batch,
"add_special_tokens": add_special_tokens,
"padding": padding,
"return_tensors": return_tensors,
}
)
self.padding_sides.append(self.padding_side)
rows = [list(range(1, len(text) % 3 + 2)) for text in text_rows]
width = max(len(row) for row in rows)
padded = []
masks = []
for row in rows:
pad_count = width - len(row)
if self.padding_side == "left":
padded.append([self.pad_token_id] * pad_count + row)
masks.append([0] * pad_count + [1] * len(row))
else:
padded.append(row + [self.pad_token_id] * pad_count)
masks.append([1] * len(row) + [0] * pad_count)
return {
"input_ids": torch.tensor(padded),
"attention_mask": torch.tensor(masks),
}
def decode(self, token_ids, skip_special_tokens=True):
return f"response-{int(token_ids[-1])}"
class _DeterministicBatchModel:
def __init__(self):
import torch
self.device = torch.device("cpu")
self.generate_calls = []
def generate(self, input_ids, attention_mask, **kwargs):
import torch
self.generate_calls.append(
{
"input_ids": input_ids.clone(),
"attention_mask": attention_mask.clone(),
"kwargs": kwargs,
}
)
generated = []
for row, mask in zip(input_ids, attention_mask, strict=True):
signature = int(row[mask.bool()].sum())
generated.append(torch.tensor([[100 + signature]]))
return torch.cat([input_ids, torch.cat(generated)], dim=1)
class _CharTokenizer:
"""Small tokenizer whose generated answer can depend on prompt content."""
chat_template = "chars"
pad_token_id = 0
eos_token_id = 2
padding_side = "right"
def apply_chat_template(self, messages, **kwargs):
return messages[0]["content"]
def __call__(
self,
texts,
*,
add_special_tokens=True,
padding=False,
return_tensors=None,
):
import torch
rows = texts if isinstance(texts, list) else [texts]
encoded = [[ord(char) % 251 + 3 for char in text] for text in rows]
width = max(len(row) for row in encoded)
input_ids = []
attention_mask = []
for row in encoded:
pad_count = width - len(row)
if self.padding_side == "left":
input_ids.append([self.pad_token_id] * pad_count + row)
attention_mask.append([0] * pad_count + [1] * len(row))
else:
input_ids.append(row + [self.pad_token_id] * pad_count)
attention_mask.append([1] * len(row) + [0] * pad_count)
return {
"input_ids": torch.tensor(input_ids),
"attention_mask": torch.tensor(attention_mask),
}
def decode(self, token_ids, skip_special_tokens=True):
tokens = [int(token) for token in token_ids]
return "answer-" + str(tokens[0] - 1000) if tokens else ""
class _EchoModel:
"""Return a token derived from each row, with EOS for selected prompts."""
def __init__(self):
import torch
self.device = torch.device("cpu")
def generate(self, input_ids, attention_mask, **kwargs):
import torch
generated = []
for row, mask in zip(input_ids, attention_mask, strict=True):
prompt_sum = int(row[mask.bool()].sum())
response = 1000 + prompt_sum
if prompt_sum % 2:
generated.append(torch.tensor([response, 2]))
else:
generated.append(torch.tensor([response]))
width = max(len(row) for row in generated)
padded = [
torch.cat([row, torch.zeros(width - len(row), dtype=torch.long)])
for row in generated
]
return torch.cat([input_ids, torch.stack(padded)], dim=1)
def _expected_pairs(prompts):
tokenizer = _CharTokenizer()
expected = []
for prompt in prompts:
encoded = tokenizer(prompt, return_tensors="pt")
prompt_sum = int(encoded["input_ids"].sum())
expected.append((prompt, f"answer-{prompt_sum}", 2 if prompt_sum % 2 else 1))
return expected
# ─── Prompt Reading Tests ───────────────────────────────────────────────────
class TestReadPrompts:
"""Test prompt reading from files."""
def test_read_jsonl_prompts(self, tmp_path):
"""Should read prompts from JSONL with 'prompt' field."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "prompts.jsonl"
lines = [
json.dumps({"prompt": "What is AI?"}),
json.dumps({"prompt": "Explain gravity."}),
json.dumps({"prompt": "Hello world."}),
]
path.write_text("\n".join(lines))
result = _read_prompts(path)
assert len(result) == 3
assert result[0] == "What is AI?"
assert result[1] == "Explain gravity."
assert result[2] == "Hello world."
class TestPromptDependentBatchPairing:
MAX_TOKENS = 12
PAIRING_PROMPTS = [
"a",
"hello world",
"x!",
"the quick brown fox",
"hi",
"zz top!",
"0123456789",
]
def test_batch_matches_one_by_one_prompt_dependent_results(self):
from soup_cli.commands.infer import _generate, _generate_batch
tokenizer = _CharTokenizer()
model = _EchoModel()
batched = _generate_batch(
model,
tokenizer,
self.PAIRING_PROMPTS,
max_tokens=self.MAX_TOKENS,
temperature=0,
)
one_by_one = [
_generate(
model,
tokenizer,
[{"role": "user", "content": prompt}],
max_tokens=self.MAX_TOKENS,
temperature=0,
)
for prompt in self.PAIRING_PROMPTS
]
assert [
(prompt, response, token_count)
for prompt, (response, token_count) in zip(
self.PAIRING_PROMPTS, batched, strict=True
)
] == [
(prompt, response, token_count)
for prompt, (response, token_count) in zip(
self.PAIRING_PROMPTS, one_by_one, strict=True
)
] == _expected_pairs(self.PAIRING_PROMPTS)
@pytest.mark.parametrize("batch_size", [2, 3, 8])
def test_cli_batch_sizes_match_batch_one(self, tmp_path, monkeypatch, batch_size):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_path = tmp_path / "prompts.jsonl"
prompts_path.write_text(
"".join(
json.dumps({"prompt": prompt}) + "\n"
for prompt in self.PAIRING_PROMPTS
)
)
monkeypatch.chdir(tmp_path)
outputs = []
for size in (1, batch_size):
model = _EchoModel()
tokenizer = _CharTokenizer()
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, model=model, tokenizer=tokenizer, **kwargs: (
model,
tokenizer,
),
)
output_path = tmp_path / f"output-{size}.jsonl"
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_path),
"--output", str(output_path),
"--device", "cpu",
"--batch-size", str(size),
"--max-tokens", str(self.MAX_TOKENS),
"--temperature", "0",
],
)
assert result.exit_code == 0, result.output
outputs.append(
[json.loads(line) for line in output_path.read_text().splitlines()]
)
assert outputs[0] == outputs[1]
assert [
(row["prompt"], row["response"], row["tokens_generated"])
for row in outputs[1]
] == _expected_pairs(self.PAIRING_PROMPTS)
def test_read_plain_text_prompts(self, tmp_path):
"""Should read plain text lines as prompts."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "prompts.txt"
path.write_text("What is AI?\nExplain gravity.\nHello world.\n")
result = _read_prompts(path)
assert len(result) == 3
assert result[0] == "What is AI?"
def test_read_skips_empty_lines(self, tmp_path):
"""Should skip empty lines."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "prompts.txt"
path.write_text("Line one\n\n\nLine two\n\n")
result = _read_prompts(path)
assert len(result) == 2
def test_read_empty_file(self, tmp_path):
"""Should return empty list for empty file."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "empty.jsonl"
path.write_text("")
result = _read_prompts(path)
assert result == []
def test_read_mixed_jsonl_and_text(self, tmp_path):
"""Should handle mixed JSONL and plain text lines."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "mixed.txt"
lines = [
json.dumps({"prompt": "From JSONL"}),
"Plain text prompt",
]
path.write_text("\n".join(lines))
result = _read_prompts(path)
assert len(result) == 2
assert result[0] == "From JSONL"
assert result[1] == "Plain text prompt"
def test_read_jsonl_without_prompt_field(self, tmp_path):
"""JSONL without 'prompt' field should be treated as plain text."""
from soup_cli.commands.infer import _read_prompts
path = tmp_path / "noprompt.jsonl"
lines = [
json.dumps({"text": "This has no prompt field"}),
]
path.write_text("\n".join(lines))
result = _read_prompts(path)
assert len(result) == 1
# Falls through to plain text since 'prompt' key missing
assert "text" in result[0]
# ─── CLI Validation Tests ──────────────────────────────────────────────────
class TestInferCLI:
"""Test infer CLI command validation."""
def test_input_file_not_found(self, tmp_path):
"""Should fail if input file doesn't exist."""
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
result = runner.invoke(app, [
"infer",
"--model", str(tmp_path),
"--input", "/nonexistent/prompts.jsonl",
"--output", str(tmp_path / "out.jsonl"),
])
assert result.exit_code != 0
assert "not found" in result.output.lower()
def test_model_not_found(self, tmp_path):
"""Should fail if model path doesn't exist."""
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
runner = CliRunner()
result = runner.invoke(app, [
"infer",
"--model", "/nonexistent/model",
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
])
assert result.exit_code != 0
assert "not found" in result.output.lower()
def test_empty_input_file(self, tmp_path):
"""Should fail if input file has no prompts."""
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "empty.jsonl"
prompts_file.write_text("")
runner = CliRunner()
result = runner.invoke(app, [
"infer",
"--model", str(tmp_path),
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
])
assert result.exit_code != 0
assert "no prompts" in result.output.lower()
def test_infer_command_registered(self):
"""Infer command should be registered in the app."""
from soup_cli.cli import app
command_names = [
cmd.name or (cmd.callback.__name__ if cmd.callback else None)
for cmd in app.registered_commands
]
assert "infer" in command_names
def test_infer_help(self):
"""Infer command should show help text."""
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
result = runner.invoke(app, ["infer", "--help"])
assert result.exit_code == 0
assert "batch inference" in result.output.lower()
assert "--batch-size" in strip_ansi(result.output)
@pytest.mark.parametrize("batch_size", [0, -1])
def test_batch_size_must_be_positive(self, tmp_path, batch_size, monkeypatch):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, **kwargs: (object(), object()),
)
result = CliRunner().invoke(
app,
[
"infer",
"--model", str(tmp_path),
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
"--batch-size", str(batch_size),
],
)
assert result.exit_code == 2
assert "x>=1" in result.output
def test_cuda_graphs_refuses_batched_generation_before_model_load(
self, tmp_path, monkeypatch
):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
def fail_model_load(*args, **kwargs):
raise AssertionError("model loading must not run for an invalid option pair")
monkeypatch.setattr("soup_cli.commands.infer._load_model", fail_model_load)
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
"--cuda-graphs",
"--batch-size", "2",
],
)
assert result.exit_code == 2
assert "cannot be combined" in strip_ansi(result.output)
assert "--batch-size above 1" in strip_ansi(result.output)
def test_required_options_error(self):
"""Should fail if required options are missing."""
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
result = runner.invoke(app, ["infer"])
assert result.exit_code != 0
def test_max_tokens_too_large_rejected(self, tmp_path):
"""--max-tokens above 16384 should be rejected by CLI."""
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
runner = CliRunner()
result = runner.invoke(app, [
"infer",
"--model", str(tmp_path),
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
"--max-tokens", "99999",
])
assert result.exit_code != 0
def test_max_tokens_zero_rejected(self, tmp_path):
"""--max-tokens 0 should be rejected by CLI."""
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
runner = CliRunner()
result = runner.invoke(app, [
"infer",
"--model", str(tmp_path),
"--input", str(prompts_file),
"--output", str(tmp_path / "out.jsonl"),
"--max-tokens", "0",
])
assert result.exit_code != 0
# ─── Load Model Tests ────────────────────────────────────────────────────
class TestLoadModel:
"""Test _load_model adapter detection and error paths."""
def test_adapter_without_base_model_exits(self, tmp_path):
"""Should exit if adapter found but no base model detectable."""
from soup_cli.commands.infer import _load_model
# Create adapter_config.json without base_model_name_or_path
adapter_config = tmp_path / "adapter_config.json"
adapter_config.write_text(json.dumps({"peft_type": "LORA"}))
from typer import Exit
with pytest.raises(Exit):
_load_model(str(tmp_path), None, "cpu")
def test_adapter_with_corrupt_json_exits(self, tmp_path):
"""Should exit if adapter_config.json is corrupt and no --base given."""
from soup_cli.commands.infer import _load_model
adapter_config = tmp_path / "adapter_config.json"
adapter_config.write_text("not valid json {{{")
from typer import Exit
with pytest.raises(Exit):
_load_model(str(tmp_path), None, "cpu")
def test_adapter_config_reads_base_model(self, tmp_path):
"""Should read base_model_name_or_path from adapter_config.json."""
adapter_config = tmp_path / "adapter_config.json"
adapter_config.write_text(json.dumps({
"base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct"
}))
config = json.loads(adapter_config.read_text())
assert config["base_model_name_or_path"] == "meta-llama/Llama-3.1-8B-Instruct"
def test_no_adapter_config_is_full_model(self, tmp_path):
"""Without adapter_config.json, model is treated as full model."""
assert not (tmp_path / "adapter_config.json").exists()
# ─── Generate Function Tests ─────────────────────────────────────────────
class TestGenerate:
"""Test _generate function branches."""
def _make_mock_model_and_tokenizer(self, has_chat_template=True):
"""Create mock model and tokenizer for testing _generate."""
import torch
mock_tokenizer = MagicMock()
mock_tokenizer.pad_token_id = 0
if has_chat_template:
mock_tokenizer.chat_template = "dummy_template"
mock_tokenizer.apply_chat_template = MagicMock(
return_value="<|user|>Hello<|assistant|>"
)
else:
mock_tokenizer.chat_template = None
# Mock tokenizer call (tokenizer(text, return_tensors="pt"))
mock_inputs = {
"input_ids": torch.tensor([[1, 2, 3]]),
"attention_mask": torch.tensor([[1, 1, 1]]),
}
mock_tokenizer.return_value = mock_inputs
mock_tokenizer.decode = MagicMock(return_value="Hello there!")
mock_model = MagicMock()
type(mock_model).device = PropertyMock(return_value=torch.device("cpu"))
mock_model.generate = MagicMock(
return_value=torch.tensor([[1, 2, 3, 4, 5, 6]]) # 3 input + 3 new tokens
)
return mock_model, mock_tokenizer
def test_generate_with_chat_template(self):
"""Should use apply_chat_template when available."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer(has_chat_template=True)
messages = [{"role": "user", "content": "Hello"}]
response, token_count = _generate(model, tokenizer, messages)
tokenizer.apply_chat_template.assert_called_once()
assert isinstance(response, str)
assert token_count == 3 # 6 total - 3 input
def test_generate_without_chat_template(self):
"""Should use fallback formatter when no chat_template."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer(has_chat_template=False)
messages = [
{"role": "system", "content": "You are helpful"},
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi"},
]
response, token_count = _generate(model, tokenizer, messages)
# Should NOT call apply_chat_template
assert not hasattr(tokenizer.apply_chat_template, 'called') or \
not tokenizer.apply_chat_template.called
assert isinstance(response, str)
assert token_count == 3
def test_generate_greedy_temperature_zero(self):
"""Should set do_sample=False when temperature=0."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer()
messages = [{"role": "user", "content": "Hello"}]
_generate(model, tokenizer, messages, temperature=0.0)
# Check gen_kwargs passed to model.generate
call_kwargs = model.generate.call_args[1]
assert call_kwargs["do_sample"] is False
assert "temperature" not in call_kwargs
assert "top_p" not in call_kwargs
def test_generate_sampling_temperature_positive(self):
"""Should set do_sample=True and include temperature when > 0."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer()
messages = [{"role": "user", "content": "Hello"}]
_generate(model, tokenizer, messages, temperature=0.7)
call_kwargs = model.generate.call_args[1]
assert call_kwargs["do_sample"] is True
assert call_kwargs["temperature"] == pytest.approx(0.7)
assert "top_p" in call_kwargs
def test_generate_returns_tuple(self):
"""_generate should return (text, token_count) tuple."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer()
messages = [{"role": "user", "content": "Hello"}]
result = _generate(model, tokenizer, messages)
assert isinstance(result, tuple)
assert len(result) == 2
assert isinstance(result[0], str)
assert isinstance(result[1], int)
def test_generate_token_count_from_tensor_shape(self):
"""Token count should come from tensor shape, not re-encoding."""
import torch
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer()
# Return 7 tokens total, 3 are input => 4 new tokens
model.generate.return_value = torch.tensor([[1, 2, 3, 10, 20, 30, 40]])
messages = [{"role": "user", "content": "Hello"}]
_, token_count = _generate(model, tokenizer, messages)
assert token_count == 4 # 7 - 3 input tokens
def test_generate_fallback_formats_roles(self):
"""Fallback formatter should include all role types."""
from soup_cli.commands.infer import _generate
model, tokenizer = self._make_mock_model_and_tokenizer(has_chat_template=False)
messages = [
{"role": "system", "content": "Be helpful"},
{"role": "user", "content": "What is 2+2?"},
]
_generate(model, tokenizer, messages)
# Check that the tokenizer was called with formatted text
tokenizer_call_args = tokenizer.call_args[0][0]
assert "System: Be helpful" in tokenizer_call_args
assert "User: What is 2+2?" in tokenizer_call_args
assert "Assistant:" in tokenizer_call_args
class TestGenerateBatch:
def test_batch_template_errors_are_not_silently_fallback(self):
from soup_cli.commands.infer import _generate_batch
class BrokenTemplateTokenizer(_BatchTokenizer):
def apply_chat_template(self, messages, **kwargs):
raise ValueError("broken chat template")
with pytest.raises(ValueError, match="broken chat template"):
_generate_batch(
_DeterministicBatchModel(),
BrokenTemplateTokenizer(templated=True),
["first", "second"],
max_tokens=1,
temperature=0,
)
def test_batch_generation_uses_left_padding_and_one_call(self):
from soup_cli.commands.infer import _generate_batch
model = _DeterministicBatchModel()
tokenizer = _BatchTokenizer(templated=True)
results = _generate_batch(
model,
tokenizer,
["xx", "xxxx"],
max_tokens=4,
temperature=0,
)
assert len(results) == 2
assert len(model.generate_calls) == 1
assert tokenizer.padding_side == "right"
assert tokenizer.padding_sides[-1] == "left"
assert tokenizer.calls[-1] == {
"is_batch": True,
"add_special_tokens": False,
"padding": True,
"return_tensors": "pt",
}
assert model.generate_calls[0]["input_ids"][0, 0].item() == 0
def test_batch_generation_counts_tokens_before_padding(self):
import torch
from soup_cli.commands.infer import _generate_batch
class VariableLengthBatchModel:
device = torch.device("cpu")
def generate(self, input_ids, attention_mask, **kwargs):
generated = []
lengths = [2, 5]
for index, (row, mask) in enumerate(
zip(input_ids, attention_mask, strict=True)
):
signature = int(row[mask.bool()].sum())
tokens = [
100 + signature + offset
for offset in range(lengths[index])
]
generated.append(torch.tensor([tokens], dtype=torch.long))
max_length = max(lengths)
padded = [
torch.cat(
[
tokens,
torch.zeros(
(1, max_length - tokens.shape[1]),
dtype=torch.long,
),
],
dim=1,
)
for tokens in generated
]
return torch.cat([input_ids, torch.cat(padded)], dim=1)
model = VariableLengthBatchModel()
tokenizer = _BatchTokenizer(templated=True)
results = _generate_batch(
model,
tokenizer,
["first", "second"],
max_tokens=5,
temperature=0,
)
assert [token_count for _, token_count in results] == [2, 5]
def test_batch_generation_stops_counting_at_eos(self):
import torch
from soup_cli.commands.infer import _generate_batch
class EosBatchModel:
device = torch.device("cpu")
def generate(self, input_ids, attention_mask, **kwargs):
generated = []
for index, (row, mask) in enumerate(
zip(input_ids, attention_mask, strict=True)
):
signature = int(row[mask.bool()].sum())
if index == 0:
tokens = [
100 + signature,
99,
101 + signature,
102 + signature,
103 + signature,
]
else:
tokens = [
110 + signature,
111 + signature,
112 + signature,
113 + signature,
114 + signature,
]
generated.append(torch.tensor([tokens], dtype=torch.long))
return torch.cat([input_ids, torch.cat(generated)], dim=1)
model = EosBatchModel()
tokenizer = _BatchTokenizer(templated=True)
tokenizer.eos_token_id = 99
results = _generate_batch(
model,
tokenizer,
["first", "second"],
max_tokens=5,
temperature=0,
)
assert [token_count for _, token_count in results] == [2, 5]
def test_batch_generation_uses_attention_mask_for_padding(self):
import torch
from soup_cli.commands.infer import _generate_batch
class MaskCheckingModel:
device = torch.device("cpu")
def generate(self, input_ids, attention_mask, **kwargs):
assert torch.equal(
attention_mask == 0,
input_ids == 99,
)
generated = torch.tensor(
[[101], [102]],
dtype=torch.long,
)
return torch.cat([input_ids, generated], dim=1)
model = MaskCheckingModel()
tokenizer = _BatchTokenizer(templated=True)
tokenizer.pad_token_id = 99
results = _generate_batch(
model,
tokenizer,
["x", "xx"],
max_tokens=1,
temperature=0,
)
assert [response for response, _ in results] == [
"response-101",
"response-102",
]
def test_batch_generation_restores_original_padding_side(self):
from soup_cli.commands.infer import _generate_batch
model = _DeterministicBatchModel()
tokenizer = _BatchTokenizer(templated=True)
_generate_batch(model, tokenizer, ["first", "second"], temperature=0)
assert tokenizer.padding_sides[-1] == "left"
assert tokenizer.padding_side == "right"
def test_templated_and_untemplated_batch_encoding(self):
from soup_cli.commands.infer import _generate_batch
for templated, expected_special_tokens in ((True, False), (False, True)):
model = _DeterministicBatchModel()
tokenizer = _BatchTokenizer(templated=templated)
_generate_batch(model, tokenizer, ["one", "two"], temperature=0)
assert tokenizer.calls[-1]["add_special_tokens"] is expected_special_tokens
def test_batch_generation_preserves_prompt_response_order(self):
import torch
from soup_cli.commands.infer import _generate_batch
class OrderedBatchModel:
device = torch.device("cpu")
def generate(self, input_ids, attention_mask, **kwargs):
generated = [
torch.tensor([[101 + index]], dtype=torch.long)
for index in range(input_ids.shape[0])
]
return torch.cat([input_ids, torch.cat(generated)], dim=1)
model = OrderedBatchModel()
tokenizer = _BatchTokenizer(templated=True)
results = _generate_batch(
model,
tokenizer,
["first", "second"],
max_tokens=1,
temperature=0,
)
assert [response for response, _ in results] == [
"response-101",
"response-102",
]
def test_batch_size_one_uses_legacy_generate_path(self, tmp_path, monkeypatch):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_path = tmp_path / "prompts.jsonl"
prompts_path.write_text(json.dumps({"prompt": "hello"}) + "\n")
output_path = tmp_path / "output.jsonl"
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, **kwargs: (object(), object()),
)
calls = []
def fake_generate(model, tokenizer, messages, *, max_tokens, temperature):
calls.append(messages)
return "legacy-response", 3
def fail_generate_batch(*args, **kwargs):
raise AssertionError("_generate_batch must not be used for batch_size=1")
monkeypatch.setattr("soup_cli.commands.infer._generate", fake_generate)
monkeypatch.setattr(
"soup_cli.commands.infer._generate_batch",
fail_generate_batch,
)
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_path),
"--output", str(output_path),
"--device", "cpu",
"--batch-size", "1",
],
)
assert result.exit_code == 0, result.output
assert len(calls) == 1
assert json.loads(output_path.read_text()) == {
"prompt": "hello",
"response": "legacy-response",
"tokens_generated": 3,
}
def test_default_batch_size_uses_legacy_generate_path(self, tmp_path, monkeypatch):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_path = tmp_path / "prompts.jsonl"
prompts_path.write_text(json.dumps({"prompt": "hello"}) + "\n")
output_path = tmp_path / "output.jsonl"
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, **kwargs: (object(), object()),
)
calls = []
def fake_generate(model, tokenizer, messages, *, max_tokens, temperature):
calls.append(messages)
return "legacy-response", 3
def fail_generate_batch(*args, **kwargs):
raise AssertionError("_generate_batch must not be used for default batch_size=1")
monkeypatch.setattr("soup_cli.commands.infer._generate", fake_generate)
monkeypatch.setattr(
"soup_cli.commands.infer._generate_batch",
fail_generate_batch,
)
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_path),
"--output", str(output_path),
"--device", "cpu",
],
)
assert result.exit_code == 0, result.output
assert len(calls) == 1
assert json.loads(output_path.read_text()) == {
"prompt": "hello",
"response": "legacy-response",
"tokens_generated": 3,
}
def test_cli_batches_keep_prompt_response_pairs_in_input_order(self, tmp_path, monkeypatch):
import torch
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_path = tmp_path / "prompts.jsonl"
prompts = ["please say a", "say b", "say c", "now say d", "say e"]
prompts_path.write_text(
"".join(json.dumps({"prompt": prompt}) + "\n" for prompt in prompts)
)
output_path = tmp_path / "output.jsonl"
class UniqueBatchModel:
device = torch.device("cpu")
def __init__(self):
self.next_token = 101
self.generate_calls = []
def generate(self, input_ids, attention_mask, **kwargs):
self.generate_calls.append(input_ids.clone())
rows = []
for _ in range(input_ids.shape[0]):
rows.append(torch.tensor([[self.next_token]]))
self.next_token += 1
return torch.cat([input_ids, torch.cat(rows)], dim=1)
model = UniqueBatchModel()
tokenizer = _BatchTokenizer(templated=True)
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, **kwargs: (model, tokenizer),
)
monkeypatch.chdir(tmp_path)
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_path),
"--output", str(output_path),
"--device", "cpu",
"--batch-size", "2",
"--temperature", "0",
],
)
assert result.exit_code == 0, result.output
rows = [json.loads(line) for line in output_path.read_text().splitlines()]
assert [(row["prompt"], row["response"]) for row in rows] == [
(prompt, f"response-{101 + index}")
for index, prompt in enumerate(prompts)
]
assert len(model.generate_calls) == 3
def test_batch_size_one_and_eight_have_identical_outputs(
self, tmp_path, monkeypatch
):
from typer.testing import CliRunner
from soup_cli.cli import app
prompts = [f"prompt-{index}" for index in range(5)]
prompts_path = tmp_path / "prompts.jsonl"
prompts_path.write_text(
"".join(json.dumps({"prompt": prompt}) + "\n" for prompt in prompts)
)
monkeypatch.chdir(tmp_path)
outputs = []
models = []
for batch_size in (1, 8):
model = _DeterministicBatchModel()
tokenizer = _BatchTokenizer(templated=True)
monkeypatch.setattr(
"soup_cli.commands.infer._load_model",
lambda *args, model=model, tokenizer=tokenizer, **kwargs: (
model,
tokenizer,
),
)
output_path = tmp_path / f"output-{batch_size}.jsonl"
result = CliRunner().invoke(
app,
[
"infer",
"--model", "fake/model",
"--input", str(prompts_path),
"--output", str(output_path),
"--device", "cpu",
"--batch-size", str(batch_size),
"--temperature", "0",
],
)
assert result.exit_code == 0, result.output
outputs.append([json.loads(line) for line in output_path.read_text().splitlines()])
models.append(model)
assert outputs[0] == outputs[1]
assert len(models[0].generate_calls) == 5
assert len(models[1].generate_calls) == 1
# ─── Import Tests ─────────────────────────────────────────────────────────
class TestInferImports:
"""Test that infer module is importable."""
def test_import_infer_module(self):
from soup_cli.commands.infer import infer
assert infer is not None
def test_import_read_prompts(self):
from soup_cli.commands.infer import _read_prompts
assert callable(_read_prompts)
def test_import_load_model(self):
from soup_cli.commands.infer import _load_model
assert callable(_load_model)
def test_import_generate(self):
from soup_cli.commands.infer import _generate
assert callable(_generate)
# ─── Issue #1097: HF Repo ID Preservation Tests ─────────────────────────────
class TestInferHFRepoId:
"""Ensure HuggingFace repo IDs are not corrupted with OS-specific path separators (#1097)."""
def test_cli_passes_hf_repo_id_byte_identical_with_slashes(self, tmp_path, monkeypatch):
"""soup infer --model owner/repo must pass repo ID with forward slashes (#1097)."""
from unittest.mock import MagicMock, patch
from typer.testing import CliRunner
from soup_cli.cli import app
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
mock_load = MagicMock()
mock_model = MagicMock()
mock_tok = MagicMock()
mock_load.return_value = (mock_model, mock_tok)
monkeypatch.chdir(tmp_path)
runner = CliRunner()
with patch("soup_cli.commands.infer._load_model", mock_load), \
patch("soup_cli.commands.infer._generate", return_value=("answer", 5)):
result = runner.invoke(app, [
"infer",
"--model", "HuggingFaceTB/SmolLM2-135M-Instruct",
"--input", str(prompts_file),
"--output", "out.jsonl",
])
assert result.exit_code == 0, result.output
mock_load.assert_called_once()
passed_model_path = mock_load.call_args[0][0]
assert passed_model_path == "HuggingFaceTB/SmolLM2-135M-Instruct"
assert "\\" not in passed_model_path
assert "/" in passed_model_path
assert mock_load.call_args.kwargs.get("is_local") is False
def test_cli_passes_is_local_true_for_local_dir(self, tmp_path, monkeypatch):
"""soup infer --model <local_dir> passes is_local=True to _load_model (#1097 / #1118)."""
from unittest.mock import MagicMock, patch
from typer.testing import CliRunner
from soup_cli.cli import app
model_dir = tmp_path / "my_model"
model_dir.mkdir()
prompts_file = tmp_path / "prompts.jsonl"
prompts_file.write_text(json.dumps({"prompt": "test"}) + "\n")
mock_load = MagicMock()
mock_model = MagicMock()
mock_tok = MagicMock()
mock_load.return_value = (mock_model, mock_tok)
monkeypatch.chdir(tmp_path)
runner = CliRunner()
with patch("soup_cli.commands.infer._load_model", mock_load), \
patch("soup_cli.commands.infer._generate", return_value=("answer", 5)):
result = runner.invoke(app, [
"infer",
"--model", str(model_dir),
"--input", str(prompts_file),
"--output", "out.jsonl",
])
assert result.exit_code == 0, result.output
mock_load.assert_called_once()
assert mock_load.call_args.kwargs.get("is_local") is True
def test_load_model_hf_repo_id_passes_slash_to_transformers(self):
"""_load_model must pass the HF repo id unmodified to transformers (#1097)."""
from unittest.mock import MagicMock, patch
from soup_cli.commands.infer import _load_model
repo_id = "HuggingFaceTB/SmolLM2-135M-Instruct"
mock_tok_inst = MagicMock()
mock_tok_inst.pad_token = None
mock_tok_inst.eos_token = "<eos>"
mock_model_inst = MagicMock()
with patch(
"transformers.AutoTokenizer.from_pretrained", return_value=mock_tok_inst
) as mock_tok, patch(
"transformers.AutoModelForCausalLM.from_pretrained", return_value=mock_model_inst
) as mock_model, patch(
"soup_cli.utils.trust_remote.resolve_trust_remote_code", return_value=False
):
_load_model(repo_id, None, "cpu", trust_remote_code=False)
mock_tok.assert_called_once()
called_tok_arg = mock_tok.call_args[0][0]
assert called_tok_arg == repo_id
assert "\\" not in called_tok_arg
assert "/" in called_tok_arg
mock_model.assert_called_once()
called_model_arg = mock_model.call_args[0][0]
assert called_model_arg == repo_id
assert "\\" not in called_model_arg
assert "/" in called_model_arg