Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: walk record offsets and batch member writes in log export (#16358)

Daniel Han0900b157b22e
diff --git a/studio/backend/tests/test_debug_log_export_batching.py b/studio/backend/tests/test_debug_log_export_batching.py
new file mode 100644
index 00000000000..596a82b5589
--- /dev/null
+++ b/studio/backend/tests/test_debug_log_export_batching.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Offset walking and batched member writes must leave the export archive byte for byte unchanged."""
+
+from __future__ import annotations
+
+import io
+import random
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from utils import debug_log_export, debug_log_sources
+from utils.log_redaction import redact_log_text
+
+
+class _Clock:
+ """One tick per call: the deadline cuts at the same record every run."""
+
+ def __init__(self) -> None:
+ self.now = 0.0
+
+ def monotonic(self) -> float:
+ self.now += 1.0
+ return self.now
+
+
+def _write_logs(tmp_path: Path, bodies: list[bytes]) -> list[debug_log_sources.LogSource]:
+ sources = []
+ for index, body in enumerate(bodies):
+ path = tmp_path / f"f{index}.log"
+ path.write_bytes(body)
+ sources.append(
+ debug_log_sources.LogSource(
+ id = f"s{index}",
+ family = f"fam{index}",
+ label = path.name,
+ realpath = str(path),
+ size_bytes = len(body),
+ modified_at = 0,
+ is_current = index == 0,
+ )
+ )
+ return sources
+
+
+def _mixed_body(seed: int, size: int) -> bytes:
+ rng = random.Random(seed)
+ pieces = [
+ b"plain line",
+ b"HF_TOKEN=hf_abcdefghijklmnopqrstuvwxyz12",
+ b"caf\xc3\xa9",
+ b"\xff\xfe invalid",
+ b"\r",
+ b"x" * 700,
+ ]
+ parts = []
+ total = 0
+ while total < size:
+ record = b"".join(rng.choice(pieces) for _ in range(rng.randint(0, 6)))
+ record += rng.choice([b"\n", b"\r\n", b"\n\n"])
+ parts.append(record)
+ total += len(record)
+ return b"".join(parts) + b"tail without newline"
+
+
+def _reference_lines(body: bytes) -> list[str]:
+ lines = body.split(b"\n")
+ out = []
+ for raw in lines:
+ if len(raw) > debug_log_export.MAX_RECORD_BYTES:
+ out.append(debug_log_export.OVERSIZED_MARKER)
+ continue
+ try:
+ text = raw.decode("utf-8")
+ except UnicodeDecodeError:
+ out.append(debug_log_export.UNREADABLE_MARKER)
+ continue
+ out.append(redact_log_text(text.rstrip("\r")))
+ if lines[-1] == b"":
+ out.pop()
+ return out
+
+
+def _build(monkeypatch, sources) -> bytes:
+ monkeypatch.setattr(debug_log_sources, "list_sources", lambda: sources)
+ handle = debug_log_export.build_log_archive()
+ try:
+ return handle.read()
+ finally:
+ handle.close()
+
+
[email protected]("chunk", [7, 1000, 64 * 1024, 64 * 1024 + 1])
+def test_records_split_across_read_chunks_match_a_whole_file_split(monkeypatch, tmp_path, chunk):
+ monkeypatch.setattr(debug_log_export, "_READ_CHUNK_BYTES", chunk)
+ body = _mixed_body(chunk, 200_000)
+ data = _build(monkeypatch, _write_logs(tmp_path, [body]))
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
+ text = archive.read("fam0/f0.log").decode("utf-8")
+ assert text == "\n".join(_reference_lines(body)) + "\n"
+
+
[email protected]("seed", range(6))
+def test_write_batch_size_does_not_change_archive_bytes(monkeypatch, tmp_path, seed):
+ rng = random.Random(seed)
+ monkeypatch.setattr(debug_log_export, "_READ_CHUNK_BYTES", rng.choice([1000, 64 * 1024]))
+ monkeypatch.setattr(debug_log_export, "MAX_TOTAL_SOURCE_BYTES", rng.choice([10**9, 150_000]))
+ bodies = [_mixed_body(seed * 10 + index, rng.randint(1, 120_000)) for index in range(3)]
+ sources = _write_logs(tmp_path, bodies)
+ archives = []
+ for batch in (1, debug_log_export.STREAM_CHUNK_BYTES, 10**12):
+ monkeypatch.setattr(debug_log_export, "STREAM_CHUNK_BYTES", batch)
+ monkeypatch.setattr(debug_log_export, "time", _Clock())
+ monkeypatch.setattr(debug_log_export, "MAX_BUILD_SECONDS", rng.choice([10**9, 900.0]))
+ archives.append(_build(monkeypatch, sources))
+ assert archives[0] == archives[1] == archives[2]
+
+
+def test_deadline_cuts_after_the_same_record_regardless_of_batching(monkeypatch, tmp_path):
+ body = b"".join(b"line %d\n" % index for index in range(5000))
+ sources = _write_logs(tmp_path, [body])
+ monkeypatch.setattr(debug_log_export, "MAX_BUILD_SECONDS", 2000.0)
+ monkeypatch.setattr(debug_log_export, "time", _Clock())
+ data = _build(monkeypatch, sources)
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
+ lines = archive.read("fam0/f0.log").decode().splitlines()
+ assert lines[-1] == debug_log_export.TRUNCATED_MARKER
+ assert lines[:-1] == [f"line {index}" for index in range(len(lines) - 1)]
+ assert 1000 < len(lines) < 5000
+
+
+def test_records_yielded_before_a_read_error_are_all_written(monkeypatch, tmp_path):
+ body = b"".join(b"record %d\n" % index for index in range(3000))
+ sources = _write_logs(tmp_path, [body])
+ original = debug_log_export._redact_record
+ calls = []
+
+ def failing(raw: bytes) -> str:
+ calls.append(raw)
+ if len(calls) == 2500:
+ raise OSError(5, "boom")
+ return original(raw)
+
+ monkeypatch.setattr(debug_log_export, "_redact_record", failing)
+ data = _build(monkeypatch, sources)
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
+ assert archive.testzip() is None
+ lines = archive.read("fam0/f0.log").decode().splitlines()
+ warnings = archive.read(debug_log_export.WARNINGS_MEMBER).decode()
+ assert lines == [f"record {index}" for index in range(2499)]
+ assert "fam0/f0.log: OSError (errno 5)" in warnings
diff --git a/studio/backend/utils/debug_log_export.py b/studio/backend/utils/debug_log_export.py
index 078f302d3de..06980a927ba 100644
--- a/studio/backend/utils/debug_log_export.py
+++ b/studio/backend/utils/debug_log_export.py
@@ -162,14 +162,17 @@ def _redacted_records(handle: IO[bytes], fd: int, limit: int, deadline: float) -
break
consumed += len(chunk)
buffer += chunk
+ # An offset, not a re-slice per record: that copied the rest of the chunk once per line.
+ position = 0
while True:
if time.monotonic() > deadline:
yield TRUNCATED_MARKER
return
- newline = buffer.find(b"\n")
+ newline = buffer.find(b"\n", position)
if newline == -1:
break
- record, buffer = buffer[:newline], buffer[newline + 1 :]
+ record = buffer[position:newline]
+ position = newline + 1
if dropping:
# The rest of a record already given up on.
dropping = False
@@ -179,6 +182,8 @@ def _redacted_records(handle: IO[bytes], fd: int, limit: int, deadline: float) -
yield OVERSIZED_MARKER
else:
yield _redact_record(record)
+ if position:
+ buffer = buffer[position:]
if len(buffer) > MAX_RECORD_BYTES:
# A log with no newline at all must not be held in memory whole.
dropping = True
@@ -251,7 +256,6 @@ def build_log_archive() -> tempfile.SpooledTemporaryFile:
try:
with handle:
skipped, size = _seek_to_tail(handle, fd, allowance)
- # Check gid 0 before requiring a name: if lookup fails, unnamed gid 0 would let repair run groupadd -g 0 and usermod into root, the grant to refuse.
if size > 0 and skipped >= size:
# remaining is not spent: this is one file's property and the
# next source may still fit.
@@ -273,10 +277,25 @@ def build_log_archive() -> tempfile.SpooledTemporaryFile:
f"[skipped the first {skipped} bytes of this log]\n".encode()
)
before = handle.tell()
+ # Batched: per-record writes cost more in per-call overhead than deflate.
+ pending: list[bytes] = []
+ pending_bytes = 0
try:
for record in _redacted_records(handle, fd, allowance, deadline):
- destination.write((record + "\n").encode("utf-8"))
+ encoded = (record + "\n").encode("utf-8")
+ pending.append(encoded)
+ pending_bytes += len(encoded)
+ if pending_bytes >= STREAM_CHUNK_BYTES:
+ block = b"".join(pending)
+ pending.clear()
+ pending_bytes = 0
+ destination.write(block)
finally:
+ # Also on a reader error, so a partial log keeps every record yielded.
+ if pending:
+ block = b"".join(pending)
+ pending.clear()
+ destination.write(block)
# Charged even on a partial read; inside the with because
# tell() needs an open handle.
remaining -= max(0, handle.tell() - before)