Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Decoded response limits exercise httpx streaming, then each production caller."""
from __future__ import annotations
import gzip
import json
import logging
from contextlib import contextmanager
from typing import Any, Iterator
import httpx
import pytest
class _Body(httpx.SyncByteStream):
def __init__(self, chunks: list[bytes]) -> None:
self.chunks = chunks
self.reads = 0
self.closed = False
def __iter__(self) -> Iterator[bytes]:
for chunk in self.chunks:
self.reads += 1
yield chunk
def close(self) -> None:
self.closed = True
def _stream_reply(
monkeypatch: pytest.MonkeyPatch,
chunks: list[bytes],
*,
headers: dict[str, str] | None = None,
status: int = 200,
) -> tuple[_Body, list[tuple[str, str, dict[str, Any]]]]:
body = _Body(chunks)
calls: list[tuple[str, str, dict[str, Any]]] = []
@contextmanager
def stream(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
calls.append((method, url, kwargs))
response = httpx.Response(
status,
headers=headers,
stream=body,
request=httpx.Request(method, url),
)
try:
yield response
finally:
response.close()
monkeypatch.setattr(httpx, "stream", stream)
monkeypatch.setattr(httpx, "post", lambda *a, **k: pytest.fail("uncapped httpx.post"))
return body, calls
@pytest.mark.parametrize("header", [None, "8", "0", "nonsense", "-1"])
def test_exact_decoded_boundary_is_buffered(
monkeypatch: pytest.MonkeyPatch,
header: str | None,
) -> None:
from soup_cli.utils.http_response import post_capped
headers = {"Retry-After": "2", "Content-Type": "application/json"}
if header is not None:
headers["Content-Length"] = header
body, calls = _stream_reply(monkeypatch, [b'{"a":', b"12}"], headers=headers, status=429)
response = post_capped(
"https://example.com/api",
max_bytes=8,
endpoint_label="--api-base",
timeout=3.0,
)
assert response.status_code == 429
assert response.json() == {"a": 12}
assert response.headers["Retry-After"] == "2"
assert response.headers["Content-Length"] == "8"
assert response.request.url == "https://example.com/api"
assert body.closed
assert calls == [("POST", "https://example.com/api", {"timeout": 3.0})]
def test_content_length_refuses_without_reading_and_closes(monkeypatch: pytest.MonkeyPatch) -> None:
from soup_cli.utils.http_response import ResponseTooLargeError, post_capped
body, _ = _stream_reply(monkeypatch, [b"secret body"], headers={"Content-Length": "9"})
with pytest.raises(ResponseTooLargeError, match=r"--api-base.*8 bytes"):
post_capped("https://example.com", max_bytes=8, endpoint_label="--api-base")
assert body.reads == 0
assert body.closed
def test_unknown_length_stops_before_remaining_chunks(monkeypatch: pytest.MonkeyPatch) -> None:
from soup_cli.utils.http_response import ResponseTooLargeError, post_capped
body, _ = _stream_reply(monkeypatch, [b"a" * 65536, b"b" * 65536, b"unread"])
with pytest.raises(ResponseTooLargeError, match="65536 bytes"):
post_capped("https://example.com", max_bytes=65536, endpoint_label="--base-url")
assert body.reads == 2
assert body.closed
@pytest.mark.parametrize("size", [64, 65], ids=["at-cap", "above-cap"])
def test_gzip_limit_counts_decoded_bytes(monkeypatch: pytest.MonkeyPatch, size: int) -> None:
from soup_cli.utils.http_response import ResponseTooLargeError, post_capped
payload = b"a" * size
compressed = gzip.compress(payload)
assert len(compressed) < 64
body, _ = _stream_reply(
monkeypatch,
[compressed],
headers={
"Content-Encoding": "gzip",
"Content-Length": str(len(compressed)),
},
)
if size > 64:
with pytest.raises(ResponseTooLargeError, match="64 bytes"):
post_capped("https://example.com", max_bytes=64, endpoint_label="--api-base")
else:
response = post_capped("https://example.com", max_bytes=64, endpoint_label="--api-base")
assert response.content == payload
assert "Content-Encoding" not in response.headers
assert response.headers["Content-Length"] == "64"
assert body.closed
def test_stream_error_closes(monkeypatch: pytest.MonkeyPatch) -> None:
from soup_cli.utils.http_response import post_capped
class BrokenBody(_Body):
def __iter__(self) -> Iterator[bytes]:
yield b"first"
raise httpx.ReadError("connection ended")
body = BrokenBody([])
@contextmanager
def stream(*args: Any, **kwargs: Any) -> Iterator[httpx.Response]:
response = httpx.Response(200, stream=body)
try:
yield response
finally:
response.close()
monkeypatch.setattr(httpx, "stream", stream)
with pytest.raises(httpx.ReadError, match="connection ended"):
post_capped("https://example.com", max_bytes=64, endpoint_label="--api-base")
assert body.closed
TARGETS = [
"judge",
"ollama",
"vllm",
"anthropic",
"openai",
"server",
"magpie-ollama",
"magpie-vllm",
]
def _call_target(target: str) -> Any:
if target == "judge":
from soup_cli.eval.judge import JudgeEvaluator
return JudgeEvaluator(
provider="server", model="m", api_base="http://localhost:8000"
)._call_llm("p")
if target.startswith("magpie-"):
from soup_cli.utils.magpie import make_magpie_generate_fn
return make_magpie_generate_fn(target.removeprefix("magpie-"), model="m")("p")
common = dict(prompt="p", count=1, fmt="alpaca", model_name="m", temperature=0.0)
if target == "ollama":
from soup_cli.data.providers.ollama import generate_ollama
return generate_ollama(**common, base_url="http://localhost:11434", generation_prompt="p")
if target == "vllm":
from soup_cli.data.providers.vllm import generate_vllm
return generate_vllm(**common, base_url="http://localhost:8000", generation_prompt="p")
if target == "anthropic":
from soup_cli.data.providers.anthropic import generate_anthropic
return generate_anthropic(**common, generation_prompt="p")
from soup_cli.commands.generate import _generate_openai, _generate_server
if target == "openai":
return _generate_openai(**common, api_key="synthetic-key", api_base=None, seed_examples=[])
return _generate_server(**common, api_base="http://localhost:8000", seed_examples=[])
@pytest.mark.parametrize("target", TARGETS)
@pytest.mark.parametrize("compressed", [False, True], ids=["plain", "gzip"])
def test_every_target_limits_oversize_before_parsing(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
target: str,
compressed: bool,
) -> None:
from soup_cli.eval.judge import JudgeUnavailableError
from soup_cli.utils import http_response, magpie
monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-secret-key")
monkeypatch.setattr(http_response, "JUDGE_RESPONSE_LIMIT_BYTES", 64)
monkeypatch.setattr(http_response, "GENERATION_RESPONSE_LIMIT_BYTES", 64)
monkeypatch.setattr(magpie, "_MAX_RESPONSE_BYTES", 64)
payload = b"secret-body-" * 20
headers = {"Content-Encoding": "gzip"} if compressed else {}
body, calls = _stream_reply(
monkeypatch, [gzip.compress(payload) if compressed else payload], headers=headers
)
expected_error = (
JudgeUnavailableError if target == "judge" else http_response.ResponseTooLargeError
)
with caplog.at_level(logging.DEBUG):
if target.startswith("magpie-"):
assert _call_target(target) == ""
error_text = ""
else:
with pytest.raises(expected_error, match=r"64 bytes") as caught:
_call_target(target)
error_text = str(caught.value)
assert (
"--api-base" in error_text
or "--base-url" in error_text
or "--provider anthropic" in error_text
)
assert "secret-body" not in error_text + caplog.text
assert "synthetic-secret-key" not in error_text + caplog.text
assert len(calls) == 1 # An oversized judge response is not a transient outage.
assert body.closed
@pytest.mark.parametrize("provider", ["ollama", "vllm"])
@pytest.mark.parametrize("compressed", [False, True], ids=["plain", "gzip"])
@pytest.mark.parametrize("oversized_stage", ["instruction", "response", "all"])
def test_magpie_drops_oversized_sample_and_respects_attempt_bound(
monkeypatch: pytest.MonkeyPatch, tmp_path, caplog: pytest.LogCaptureFixture,
provider: str, compressed: bool, oversized_stage: str,
) -> None:
from soup_cli.utils import magpie
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(magpie, "_MAX_RESPONSE_BYTES", 64)
def reply(text: str) -> bytes:
payload = {"response": text} if provider == "ollama" else {"choices": [{"text": text}]}
return json.dumps(payload).encode()
oversized = b"secret-body-" * 20
payloads = {
"instruction": [oversized, reply("q2"), reply("a2")],
"response": [reply("q1"), oversized, reply("q2"), reply("a2")],
"all": [oversized, oversized],
}[oversized_stage]
bodies = [_Body([gzip.compress(p) if compressed else p]) for p in payloads]
calls: list[dict[str, Any]] = []
@contextmanager
def stream(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
body = bodies[len(calls)]
calls.append(kwargs)
response = httpx.Response(
200, stream=body, request=httpx.Request(method, url),
headers={"Content-Encoding": "gzip"} if compressed else {},
)
try:
yield response
finally:
response.close()
monkeypatch.setattr(httpx, "stream", stream)
monkeypatch.setattr(httpx, "post", lambda *a, **kw: pytest.fail("uncapped httpx.post"))
config = magpie.MagpieConfig("Qwen/Qwen2.5-7B-Instruct", provider, 1, False)
with caplog.at_level(logging.DEBUG):
result = magpie.run_magpie(config, output_path="rows.jsonl", max_attempts=2)
assert result.attempts == 2
assert result.rows_kept == (0 if oversized_stage == "all" else 1)
assert len(calls) == len(bodies)
assert all(body.closed for body in bodies)
assert "secret-body" not in caplog.text
rows = [json.loads(line) for line in (tmp_path / "rows.jsonl").read_text().splitlines()]
assert rows == ([] if oversized_stage == "all" else [{"messages": [
{"role": "user", "content": "q2"}, {"role": "assistant", "content": "a2"},
]}])
@pytest.mark.parametrize("target", TARGETS)
def test_every_target_preserves_valid_reply(monkeypatch: pytest.MonkeyPatch, target: str) -> None:
monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-secret-key")
generated = '[{"instruction": "q", "output": "a"}]'
payload = {
"choices": [{"message": {"content": generated}, "text": generated}],
"response": generated,
"content": [{"type": "text", "text": generated}],
}
body, calls = _stream_reply(monkeypatch, [json.dumps(payload).encode()])
result = _call_target(target)
assert result == (
generated
if target == "judge" or target.startswith("magpie-")
else [{"instruction": "q", "output": "a"}]
)
assert len(calls) == 1
assert body.closed
def test_limits_and_judge_url_redaction(monkeypatch: pytest.MonkeyPatch) -> None:
from soup_cli.eval.judge import JudgeUnavailableError, _judge_request
from soup_cli.utils.http_response import (
GENERATION_RESPONSE_LIMIT_BYTES,
JUDGE_RESPONSE_LIMIT_BYTES,
)
assert JUDGE_RESPONSE_LIMIT_BYTES == 1024 * 1024
assert GENERATION_RESPONSE_LIMIT_BYTES == 8 * 1024 * 1024
body, calls = _stream_reply(
monkeypatch, [], headers={"Content-Length": str(JUDGE_RESPONSE_LIMIT_BYTES + 1)}
)
with pytest.raises(JudgeUnavailableError) as caught:
_judge_request(
"https://user:[email protected]/api?api_key=query-secret#fragment", {}, {}
)
text = str(caught.value)
assert "1048576 bytes" in text
assert "--api-base" in text
assert caught.value.retries_spent is False
assert "password" not in text
assert "query-secret" not in text
assert "fragment" not in text
assert body.reads == 0
assert body.closed
assert len(calls) == 1
@pytest.mark.parametrize("target", ["ollama", "vllm", "anthropic", "openai", "server"])
def test_provider_error_does_not_log_response_body(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, target: str,
) -> None:
monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-secret-key")
body, _ = _stream_reply(monkeypatch, [b"secret-response-body"], status=500)
with caplog.at_level(logging.DEBUG), pytest.raises(ValueError, match="500") as caught:
_call_target(target)
assert "secret-response-body" not in str(caught.value) + caplog.text
assert "synthetic-secret-key" not in str(caught.value) + caplog.text
assert body.closed
def test_judge_retry_preserves_streamed_status_and_headers(monkeypatch: pytest.MonkeyPatch) -> None:
from soup_cli.eval import judge
delays: list[float] = []
monkeypatch.setattr(judge, "_sleep", delays.append)
bodies = [
_Body([b'{"error": "busy"}']),
_Body([b'{"choices": [{"message": {"content": "ok"}}]}']),
]
attempts: list[str] = []
@contextmanager
def stream(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
index = len(attempts)
attempts.append(url)
response = httpx.Response(
429 if index == 0 else 200,
headers={"Retry-After": "2"}, stream=bodies[index],
request=httpx.Request(method, url),
)
try:
yield response
finally:
response.close()
monkeypatch.setattr(httpx, "stream", stream)
assert judge._judge_request("https://example.com/api", {}, {}) == "ok"
assert delays == [2.0]
assert len(attempts) == 2
assert all(body.closed for body in bodies)