Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(dataprep): don't append EOS to full-size mid-stride chunks (#10734)

Caio Theodoro91f85a45a394
diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py
index ac86aba0a0a..385f6960331 100644
--- a/tests/test_raw_text.py
+++ b/tests/test_raw_text.py
@@ -292,6 +292,62 @@ def test_smart_chunk_text_single_chunk_no_eos_returns_plain_list():
return True
+def test_smart_chunk_text_no_eos_on_intermediate_full_chunks():
+ """Only the final chunk gets EOS; mid-stride chunks stay exactly chunk_size long."""
+
+ class WordTokenizer:
+ def __init__(self):
+ self.eos_token = "</s>"
+ self.eos_token_id = -1
+
+ def __call__(
+ self,
+ text,
+ return_tensors = None,
+ add_special_tokens = False,
+ ):
+ token_ids = list(range(len(text.split())))
+ if return_tensors == "pt":
+ return {"input_ids": [token_ids]}
+ return {"input_ids": token_ids}
+
+ def decode(
+ self,
+ token_ids,
+ skip_special_tokens = False,
+ ):
+ return " ".join(f"word_{i}" for i in token_ids)
+
+ text = " ".join(f"w{i}" for i in range(37)) # 37 tokens: several full chunks + a short tail
+ loader = RawTextDataLoader(WordTokenizer(), chunk_size = 10, stride = 3)
+
+ tokenized_chunks = loader.chunk_text(text, return_tokenized = True)
+ assert len(tokenized_chunks) > 2, "test needs several chunks to cover the intermediate case"
+ for i, chunk in enumerate(tokenized_chunks):
+ ids = chunk["input_ids"]
+ is_last = i == len(tokenized_chunks) - 1
+ if is_last:
+ assert ids[-1] == -1, f"last chunk should end with eos_token_id, got {ids}"
+ else:
+ assert (
+ len(ids) == 10
+ ), f"chunk {i} should stay exactly chunk_size (10), got {len(ids)}: {ids}"
+ assert (
+ ids[-1] != -1
+ ), f"chunk {i} is not the last chunk but ends with eos_token_id: {ids}"
+
+ text_chunks = loader.chunk_text(text, return_tokenized = False)
+ assert len(text_chunks) > 2
+ for i, chunk in enumerate(text_chunks):
+ is_last = i == len(text_chunks) - 1
+ assert (
+ chunk.endswith("</s>") == is_last
+ ), f"chunk {i} (last={is_last}) eos suffix mismatch: {chunk!r}"
+
+ print("✅ test_smart_chunk_text_no_eos_on_intermediate_full_chunks passed!")
+ return True
+
+
def test_load_from_file_skips_non_object_json_lines():
"""Non-object .jsonl lines (valid JSON, not dicts) are skipped, not fatal."""
# "context" contains "text", ["text"] holds it, 42 isn't iterable -- each
@@ -670,6 +726,7 @@ def test_validate_dataset_reports_zero_min_length_when_nothing_has_content():
if __name__ == "__main__":
success = test_raw_text_loader()
success = test_smart_chunk_text_single_chunk_no_eos_returns_plain_list() and success
+ success = test_smart_chunk_text_no_eos_on_intermediate_full_chunks() and success
success = test_load_from_file_skips_non_object_json_lines() and success
success = test_smart_chunk_text_empty_input_returns_no_chunks() and success
success = test_load_from_files_all_empty_raises() and success
diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py
index d6d92312b8e..8e744da7229 100644
--- a/unsloth/dataprep/raw_text.py
+++ b/unsloth/dataprep/raw_text.py
@@ -188,8 +188,8 @@ class RawTextDataLoader:
chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens)
)
- # Append EOS on the last or a full chunk.
- if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size:
+ # EOS only at the true end: a full chunk mid-stride continues in the next chunk.
+ if end_idx == len(tokens):
eos_token_id = getattr(self.tokenizer, "eos_token_id", None)
if eos_token_id is not None:
chunk_tokens_list.append(eos_token_id)
@@ -200,8 +200,7 @@ class RawTextDataLoader:
else:
chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)
- # Append EOS on the last or a full chunk.
- if end_idx == len(tokens) or len(chunk_tokens) == chunk_size:
+ if end_idx == len(tokens):
eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""
chunk_text += eos_token