Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(dataprep): don't append EOS to full-size mid-stride chunks (#10734)
Caio Theodoro91f85a45a394
diff --git a/tests/test_raw_text.py b/tests/test_raw_text.pyindex ac86aba0a0a..385f6960331 100644--- a/tests/test_raw_text.py+++ b/tests/test_raw_text.py@@ -292,6 +292,62 @@ def test_smart_chunk_text_single_chunk_no_eos_returns_plain_list():return True+def test_smart_chunk_text_no_eos_on_intermediate_full_chunks():+ """Only the final chunk gets EOS; mid-stride chunks stay exactly chunk_size long."""++ class WordTokenizer:+ def __init__(self):+ self.eos_token = "</s>"+ self.eos_token_id = -1++ def __call__(+ self,+ text,+ return_tensors = None,+ add_special_tokens = False,+ ):+ token_ids = list(range(len(text.split())))+ if return_tensors == "pt":+ return {"input_ids": [token_ids]}+ return {"input_ids": token_ids}++ def decode(+ self,+ token_ids,+ skip_special_tokens = False,+ ):+ return " ".join(f"word_{i}" for i in token_ids)++ text = " ".join(f"w{i}" for i in range(37)) # 37 tokens: several full chunks + a short tail+ loader = RawTextDataLoader(WordTokenizer(), chunk_size = 10, stride = 3)++ tokenized_chunks = loader.chunk_text(text, return_tokenized = True)+ assert len(tokenized_chunks) > 2, "test needs several chunks to cover the intermediate case"+ for i, chunk in enumerate(tokenized_chunks):+ ids = chunk["input_ids"]+ is_last = i == len(tokenized_chunks) - 1+ if is_last:+ assert ids[-1] == -1, f"last chunk should end with eos_token_id, got {ids}"+ else:+ assert (+ len(ids) == 10+ ), f"chunk {i} should stay exactly chunk_size (10), got {len(ids)}: {ids}"+ assert (+ ids[-1] != -1+ ), f"chunk {i} is not the last chunk but ends with eos_token_id: {ids}"++ text_chunks = loader.chunk_text(text, return_tokenized = False)+ assert len(text_chunks) > 2+ for i, chunk in enumerate(text_chunks):+ is_last = i == len(text_chunks) - 1+ assert (+ chunk.endswith("</s>") == is_last+ ), f"chunk {i} (last={is_last}) eos suffix mismatch: {chunk!r}"++ print("✅ test_smart_chunk_text_no_eos_on_intermediate_full_chunks passed!")+ return True++def test_load_from_file_skips_non_object_json_lines():"""Non-object .jsonl lines (valid JSON, not dicts) are skipped, not fatal."""# "context" contains "text", ["text"] holds it, 42 isn't iterable -- each@@ -670,6 +726,7 @@ def test_validate_dataset_reports_zero_min_length_when_nothing_has_content():if __name__ == "__main__":success = test_raw_text_loader()success = test_smart_chunk_text_single_chunk_no_eos_returns_plain_list() and success+ success = test_smart_chunk_text_no_eos_on_intermediate_full_chunks() and successsuccess = test_load_from_file_skips_non_object_json_lines() and successsuccess = test_smart_chunk_text_empty_input_returns_no_chunks() and successsuccess = test_load_from_files_all_empty_raises() and successdiff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.pyindex d6d92312b8e..8e744da7229 100644--- a/unsloth/dataprep/raw_text.py+++ b/unsloth/dataprep/raw_text.py@@ -188,8 +188,8 @@ class RawTextDataLoader:chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens))- # Append EOS on the last or a full chunk.- if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size:+ # EOS only at the true end: a full chunk mid-stride continues in the next chunk.+ if end_idx == len(tokens):eos_token_id = getattr(self.tokenizer, "eos_token_id", None)if eos_token_id is not None:chunk_tokens_list.append(eos_token_id)@@ -200,8 +200,7 @@ class RawTextDataLoader:else:chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)- # Append EOS on the last or a full chunk.- if end_idx == len(tokens) or len(chunk_tokens) == chunk_size:+ if end_idx == len(tokens):eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""chunk_text += eos_token