Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Count the final EOS token within the raw-text chunk budget (#11040)
Vineeth Sai Varikuntlaa809a51d512a
diff --git a/tests/test_raw_text.py b/tests/test_raw_text.pyindex c63c0e15bf6..e38c6e0f9c6 100644--- a/tests/test_raw_text.py+++ b/tests/test_raw_text.py@@ -800,6 +800,81 @@ def test_validate_dataset_reports_zero_min_length_when_nothing_has_content():return True+def _eos_tokenizer():+ class Tokenizer:+ eos_token_id = 99+ eos_token = "</s>"++ def __call__(self, text, **kwargs):+ return {"input_ids": [[int(word) for word in text.split()]]}++ def decode(self, ids, **kwargs):+ return "".join(f" {i}" for i in ids)++ return Tokenizer()+++def test_tokenized_chunks_reserve_space_for_the_final_eos():+ for size in (2, 4):+ for stride in range(size):+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = size, stride = stride)+ for count in (size - 1, size, 2 * size - stride, 2 * size, 3 * size):+ if count == 0:+ continue+ chunks = loader.chunk_text(" ".join(map(str, range(count))))+ assert all(len(chunk["input_ids"]) <= size for chunk in chunks)+ assert all(chunk["input_ids"] != [99] for chunk in chunks)+ assert all(+ len(chunk["attention_mask"]) == len(chunk["input_ids"]) for chunk in chunks+ )+ assert chunks[-1]["input_ids"][-1] == 99+ restored = list(chunks[0]["input_ids"])+ for chunk in chunks[1:]:+ restored.extend(t for t in chunk["input_ids"] if t not in restored)+ assert restored == list(range(count)) + [99]+ # A chunk's first token is never a label: the last content token must still be one.+ labelled = {t for chunk in chunks for t in chunk["input_ids"][1:]}+ assert count < 2 or count - 1 in labelled+ if stride == 0 and count % size == 0:+ # The EOS gets a full window of context, not a [token, EOS] row.+ assert len(chunks[-1]["input_ids"]) == size+ dataset = loader.create_causal_dataset(chunks)+ assert dataset["labels"] == dataset["input_ids"]+++def test_text_chunks_reserve_space_for_the_final_eos():+ # MLX trains on text chunks and truncates at max_seq_length, which would drop an overflowing EOS.+ for size in (2, 4):+ for stride in range(size):+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = size, stride = stride)+ for count in (size - 1, size, 2 * size - stride, 2 * size, 3 * size):+ if count == 0:+ continue+ chunks = loader.chunk_text(" ".join(map(str, range(count))), return_tokenized = False)+ pieces = [chunk.replace("</s>", " </s>").split() for chunk in chunks]+ assert all(len(piece) <= size for piece in pieces)+ assert all(piece != ["</s>"] for piece in pieces)+ assert pieces[-1][-1] == "</s>"+ restored = list(pieces[0])+ for piece in pieces[1:]:+ restored.extend(t for t in piece if t not in restored)+ assert restored == [str(i) for i in range(count)] + ["</s>"]+ labelled = {t for piece in pieces for t in piece[1:]}+ assert count < 2 or str(count - 1) in labelled+++def test_chunk_size_one_keeps_the_final_eos_overflow():+ # Reserving here would only produce a lone-EOS chunk.+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = 1, stride = 0)+ for count in (1, 2, 3):+ chunks = loader.chunk_text(" ".join(map(str, range(count))))+ assert all(chunk["input_ids"] != [99] for chunk in chunks)+ assert all(len(chunk["attention_mask"]) == len(chunk["input_ids"]) for chunk in chunks)+ assert chunks[-1]["input_ids"] == [count - 1, 99]+ restored = [token for chunk in chunks for token in chunk["input_ids"]]+ assert restored == list(range(count)) + [99]++if __name__ == "__main__":success = test_raw_text_loader()test_clean_text_keeps_text_in_any_script()@@ -815,4 +890,7 @@ if __name__ == "__main__":success = test_validate_dataset_accepts_objects_without_column_names() and successsuccess = test_validate_dataset_streams_instead_of_materialising_columns() and successsuccess = test_validate_dataset_reports_zero_min_length_when_nothing_has_content() and success+ test_tokenized_chunks_reserve_space_for_the_final_eos()+ test_text_chunks_reserve_space_for_the_final_eos()+ test_chunk_size_one_keeps_the_final_eos_overflow()sys.exit(0 if success else 1)diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.pyindex 9a06ad09da7..24cb9a9e7eb 100644--- a/unsloth/dataprep/raw_text.py+++ b/unsloth/dataprep/raw_text.py@@ -163,55 +163,62 @@ class RawTextDataLoader:# Tokenizer returned a count; build a rangetokens = list(range(tokens))- if len(tokens) <= chunk_size:+ eos_token_id = getattr(self.tokenizer, "eos_token_id", None)+ eos_token = getattr(self.tokenizer, "eos_token", None) or ""+ has_eos = eos_token_id is not None if return_tokenized else bool(eos_token)+ # The final EOS counts toward chunk_size; chunk_size 1 cannot hold a token plus EOS.+ reserve_eos = has_eos and chunk_size > 1+ num_tokens = len(tokens) + int(reserve_eos)++ if num_tokens <= chunk_size:# Fits in a single chunkif return_tokenized:tokens = tokens.tolist() if hasattr(tokens, "tolist") else list(tokens)- eos_token_id = getattr(self.tokenizer, "eos_token_id", None)if eos_token_id is not None:tokens.append(eos_token_id)attention_mask = [1] * len(tokens)return [{"input_ids": tokens, "attention_mask": attention_mask}]else:- eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""return [text + eos_token]- chunks = []+ bounds = []start_idx = 0+ while True:+ end_idx = min(start_idx + chunk_size, num_tokens)+ bounds.append([start_idx, end_idx])+ if end_idx == num_tokens:+ break+ start_idx += chunk_size - stride- while start_idx < len(tokens):- end_idx = min(start_idx + chunk_size, len(tokens))+ # Stride 0 + exact multiple leaves a lone-EOS chunk: end a full window at the EOS instead, so the+ # EOS keeps its context (a [token, EOS] row spikes loss) and no content token loses its label.+ if reserve_eos and len(bounds) > 1 and bounds[-1][0] == len(tokens):+ bounds[-1][0] = num_tokens - chunk_size++ chunks = []+ for i, (start_idx, end_idx) in enumerate(bounds):chunk_tokens = tokens[start_idx:end_idx]+ # EOS only at the true end: a full chunk mid-stride continues in the next chunk.+ is_last = i == len(bounds) - 1if return_tokenized:chunk_tokens_list = (chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens))-- # EOS only at the true end: a full chunk mid-stride continues in the next chunk.- if end_idx == len(tokens):- eos_token_id = getattr(self.tokenizer, "eos_token_id", None)- if eos_token_id is not None:- chunk_tokens_list.append(eos_token_id)+ if is_last and eos_token_id is not None:+ chunk_tokens_list.append(eos_token_id)attention_mask = [1] * len(chunk_tokens_list)chunks.append({"input_ids": chunk_tokens_list, "attention_mask": attention_mask})else:chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)-- if end_idx == len(tokens):- eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""+ if is_last:chunk_text += eos_tokenchunks.append(chunk_text)- # Advance with stride overlap- if end_idx == len(tokens):- break- start_idx += chunk_size - stride-return chunksdef _read_file_by_format(self, file_path, file_format):