Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: walk record offsets and batch member writes in log export (#16358)
Daniel Han0900b157b22e
diff --git a/studio/backend/tests/test_debug_log_export_batching.py b/studio/backend/tests/test_debug_log_export_batching.pynew file mode 100644index 00000000000..596a82b5589--- /dev/null+++ b/studio/backend/tests/test_debug_log_export_batching.py@@ -0,0 +1,159 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Offset walking and batched member writes must leave the export archive byte for byte unchanged."""++from __future__ import annotations++import io+import random+import sys+import zipfile+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from utils import debug_log_export, debug_log_sources+from utils.log_redaction import redact_log_text+++class _Clock:+ """One tick per call: the deadline cuts at the same record every run."""++ def __init__(self) -> None:+ self.now = 0.0++ def monotonic(self) -> float:+ self.now += 1.0+ return self.now+++def _write_logs(tmp_path: Path, bodies: list[bytes]) -> list[debug_log_sources.LogSource]:+ sources = []+ for index, body in enumerate(bodies):+ path = tmp_path / f"f{index}.log"+ path.write_bytes(body)+ sources.append(+ debug_log_sources.LogSource(+ id = f"s{index}",+ family = f"fam{index}",+ label = path.name,+ realpath = str(path),+ size_bytes = len(body),+ modified_at = 0,+ is_current = index == 0,+ )+ )+ return sources+++def _mixed_body(seed: int, size: int) -> bytes:+ rng = random.Random(seed)+ pieces = [+ b"plain line",+ b"HF_TOKEN=hf_abcdefghijklmnopqrstuvwxyz12",+ b"caf\xc3\xa9",+ b"\xff\xfe invalid",+ b"\r",+ b"x" * 700,+ ]+ parts = []+ total = 0+ while total < size:+ record = b"".join(rng.choice(pieces) for _ in range(rng.randint(0, 6)))+ record += rng.choice([b"\n", b"\r\n", b"\n\n"])+ parts.append(record)+ total += len(record)+ return b"".join(parts) + b"tail without newline"+++def _reference_lines(body: bytes) -> list[str]:+ lines = body.split(b"\n")+ out = []+ for raw in lines:+ if len(raw) > debug_log_export.MAX_RECORD_BYTES:+ out.append(debug_log_export.OVERSIZED_MARKER)+ continue+ try:+ text = raw.decode("utf-8")+ except UnicodeDecodeError:+ out.append(debug_log_export.UNREADABLE_MARKER)+ continue+ out.append(redact_log_text(text.rstrip("\r")))+ if lines[-1] == b"":+ out.pop()+ return out+++def _build(monkeypatch, sources) -> bytes:+ monkeypatch.setattr(debug_log_sources, "list_sources", lambda: sources)+ handle = debug_log_export.build_log_archive()+ try:+ return handle.read()+ finally:+ handle.close()++[email protected]("chunk", [7, 1000, 64 * 1024, 64 * 1024 + 1])+def test_records_split_across_read_chunks_match_a_whole_file_split(monkeypatch, tmp_path, chunk):+ monkeypatch.setattr(debug_log_export, "_READ_CHUNK_BYTES", chunk)+ body = _mixed_body(chunk, 200_000)+ data = _build(monkeypatch, _write_logs(tmp_path, [body]))+ with zipfile.ZipFile(io.BytesIO(data)) as archive:+ text = archive.read("fam0/f0.log").decode("utf-8")+ assert text == "\n".join(_reference_lines(body)) + "\n"++[email protected]("seed", range(6))+def test_write_batch_size_does_not_change_archive_bytes(monkeypatch, tmp_path, seed):+ rng = random.Random(seed)+ monkeypatch.setattr(debug_log_export, "_READ_CHUNK_BYTES", rng.choice([1000, 64 * 1024]))+ monkeypatch.setattr(debug_log_export, "MAX_TOTAL_SOURCE_BYTES", rng.choice([10**9, 150_000]))+ bodies = [_mixed_body(seed * 10 + index, rng.randint(1, 120_000)) for index in range(3)]+ sources = _write_logs(tmp_path, bodies)+ archives = []+ for batch in (1, debug_log_export.STREAM_CHUNK_BYTES, 10**12):+ monkeypatch.setattr(debug_log_export, "STREAM_CHUNK_BYTES", batch)+ monkeypatch.setattr(debug_log_export, "time", _Clock())+ monkeypatch.setattr(debug_log_export, "MAX_BUILD_SECONDS", rng.choice([10**9, 900.0]))+ archives.append(_build(monkeypatch, sources))+ assert archives[0] == archives[1] == archives[2]+++def test_deadline_cuts_after_the_same_record_regardless_of_batching(monkeypatch, tmp_path):+ body = b"".join(b"line %d\n" % index for index in range(5000))+ sources = _write_logs(tmp_path, [body])+ monkeypatch.setattr(debug_log_export, "MAX_BUILD_SECONDS", 2000.0)+ monkeypatch.setattr(debug_log_export, "time", _Clock())+ data = _build(monkeypatch, sources)+ with zipfile.ZipFile(io.BytesIO(data)) as archive:+ lines = archive.read("fam0/f0.log").decode().splitlines()+ assert lines[-1] == debug_log_export.TRUNCATED_MARKER+ assert lines[:-1] == [f"line {index}" for index in range(len(lines) - 1)]+ assert 1000 < len(lines) < 5000+++def test_records_yielded_before_a_read_error_are_all_written(monkeypatch, tmp_path):+ body = b"".join(b"record %d\n" % index for index in range(3000))+ sources = _write_logs(tmp_path, [body])+ original = debug_log_export._redact_record+ calls = []++ def failing(raw: bytes) -> str:+ calls.append(raw)+ if len(calls) == 2500:+ raise OSError(5, "boom")+ return original(raw)++ monkeypatch.setattr(debug_log_export, "_redact_record", failing)+ data = _build(monkeypatch, sources)+ with zipfile.ZipFile(io.BytesIO(data)) as archive:+ assert archive.testzip() is None+ lines = archive.read("fam0/f0.log").decode().splitlines()+ warnings = archive.read(debug_log_export.WARNINGS_MEMBER).decode()+ assert lines == [f"record {index}" for index in range(2499)]+ assert "fam0/f0.log: OSError (errno 5)" in warningsdiff --git a/studio/backend/utils/debug_log_export.py b/studio/backend/utils/debug_log_export.pyindex 078f302d3de..06980a927ba 100644--- a/studio/backend/utils/debug_log_export.py+++ b/studio/backend/utils/debug_log_export.py@@ -162,14 +162,17 @@ def _redacted_records(handle: IO[bytes], fd: int, limit: int, deadline: float) -breakconsumed += len(chunk)buffer += chunk+ # An offset, not a re-slice per record: that copied the rest of the chunk once per line.+ position = 0while True:if time.monotonic() > deadline:yield TRUNCATED_MARKERreturn- newline = buffer.find(b"\n")+ newline = buffer.find(b"\n", position)if newline == -1:break- record, buffer = buffer[:newline], buffer[newline + 1 :]+ record = buffer[position:newline]+ position = newline + 1if dropping:# The rest of a record already given up on.dropping = False@@ -179,6 +182,8 @@ def _redacted_records(handle: IO[bytes], fd: int, limit: int, deadline: float) -yield OVERSIZED_MARKERelse:yield _redact_record(record)+ if position:+ buffer = buffer[position:]if len(buffer) > MAX_RECORD_BYTES:# A log with no newline at all must not be held in memory whole.dropping = True@@ -251,7 +256,6 @@ def build_log_archive() -> tempfile.SpooledTemporaryFile:try:with handle:skipped, size = _seek_to_tail(handle, fd, allowance)- # Check gid 0 before requiring a name: if lookup fails, unnamed gid 0 would let repair run groupadd -g 0 and usermod into root, the grant to refuse.if size > 0 and skipped >= size:# remaining is not spent: this is one file's property and the# next source may still fit.@@ -273,10 +277,25 @@ def build_log_archive() -> tempfile.SpooledTemporaryFile:f"[skipped the first {skipped} bytes of this log]\n".encode())before = handle.tell()+ # Batched: per-record writes cost more in per-call overhead than deflate.+ pending: list[bytes] = []+ pending_bytes = 0try:for record in _redacted_records(handle, fd, allowance, deadline):- destination.write((record + "\n").encode("utf-8"))+ encoded = (record + "\n").encode("utf-8")+ pending.append(encoded)+ pending_bytes += len(encoded)+ if pending_bytes >= STREAM_CHUNK_BYTES:+ block = b"".join(pending)+ pending.clear()+ pending_bytes = 0+ destination.write(block)finally:+ # Also on a reader error, so a partial log keeps every record yielded.+ if pending:+ block = b"".join(pending)+ pending.clear()+ destination.write(block)# Charged even on a partial read; inside the with because# tell() needs an open handle.remaining -= max(0, handle.tell() - before)