Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Count the final EOS token within the raw-text chunk budget (#11040)

Vineeth Sai Varikuntlaa809a51d512a
diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py
index c63c0e15bf6..e38c6e0f9c6 100644
--- a/tests/test_raw_text.py
+++ b/tests/test_raw_text.py
@@ -800,6 +800,81 @@ def test_validate_dataset_reports_zero_min_length_when_nothing_has_content():
return True
+def _eos_tokenizer():
+ class Tokenizer:
+ eos_token_id = 99
+ eos_token = "</s>"
+
+ def __call__(self, text, **kwargs):
+ return {"input_ids": [[int(word) for word in text.split()]]}
+
+ def decode(self, ids, **kwargs):
+ return "".join(f" {i}" for i in ids)
+
+ return Tokenizer()
+
+
+def test_tokenized_chunks_reserve_space_for_the_final_eos():
+ for size in (2, 4):
+ for stride in range(size):
+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = size, stride = stride)
+ for count in (size - 1, size, 2 * size - stride, 2 * size, 3 * size):
+ if count == 0:
+ continue
+ chunks = loader.chunk_text(" ".join(map(str, range(count))))
+ assert all(len(chunk["input_ids"]) <= size for chunk in chunks)
+ assert all(chunk["input_ids"] != [99] for chunk in chunks)
+ assert all(
+ len(chunk["attention_mask"]) == len(chunk["input_ids"]) for chunk in chunks
+ )
+ assert chunks[-1]["input_ids"][-1] == 99
+ restored = list(chunks[0]["input_ids"])
+ for chunk in chunks[1:]:
+ restored.extend(t for t in chunk["input_ids"] if t not in restored)
+ assert restored == list(range(count)) + [99]
+ # A chunk's first token is never a label: the last content token must still be one.
+ labelled = {t for chunk in chunks for t in chunk["input_ids"][1:]}
+ assert count < 2 or count - 1 in labelled
+ if stride == 0 and count % size == 0:
+ # The EOS gets a full window of context, not a [token, EOS] row.
+ assert len(chunks[-1]["input_ids"]) == size
+ dataset = loader.create_causal_dataset(chunks)
+ assert dataset["labels"] == dataset["input_ids"]
+
+
+def test_text_chunks_reserve_space_for_the_final_eos():
+ # MLX trains on text chunks and truncates at max_seq_length, which would drop an overflowing EOS.
+ for size in (2, 4):
+ for stride in range(size):
+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = size, stride = stride)
+ for count in (size - 1, size, 2 * size - stride, 2 * size, 3 * size):
+ if count == 0:
+ continue
+ chunks = loader.chunk_text(" ".join(map(str, range(count))), return_tokenized = False)
+ pieces = [chunk.replace("</s>", " </s>").split() for chunk in chunks]
+ assert all(len(piece) <= size for piece in pieces)
+ assert all(piece != ["</s>"] for piece in pieces)
+ assert pieces[-1][-1] == "</s>"
+ restored = list(pieces[0])
+ for piece in pieces[1:]:
+ restored.extend(t for t in piece if t not in restored)
+ assert restored == [str(i) for i in range(count)] + ["</s>"]
+ labelled = {t for piece in pieces for t in piece[1:]}
+ assert count < 2 or str(count - 1) in labelled
+
+
+def test_chunk_size_one_keeps_the_final_eos_overflow():
+ # Reserving here would only produce a lone-EOS chunk.
+ loader = RawTextDataLoader(_eos_tokenizer(), chunk_size = 1, stride = 0)
+ for count in (1, 2, 3):
+ chunks = loader.chunk_text(" ".join(map(str, range(count))))
+ assert all(chunk["input_ids"] != [99] for chunk in chunks)
+ assert all(len(chunk["attention_mask"]) == len(chunk["input_ids"]) for chunk in chunks)
+ assert chunks[-1]["input_ids"] == [count - 1, 99]
+ restored = [token for chunk in chunks for token in chunk["input_ids"]]
+ assert restored == list(range(count)) + [99]
+
+
if __name__ == "__main__":
success = test_raw_text_loader()
test_clean_text_keeps_text_in_any_script()
@@ -815,4 +890,7 @@ if __name__ == "__main__":
success = test_validate_dataset_accepts_objects_without_column_names() and success
success = test_validate_dataset_streams_instead_of_materialising_columns() and success
success = test_validate_dataset_reports_zero_min_length_when_nothing_has_content() and success
+ test_tokenized_chunks_reserve_space_for_the_final_eos()
+ test_text_chunks_reserve_space_for_the_final_eos()
+ test_chunk_size_one_keeps_the_final_eos_overflow()
sys.exit(0 if success else 1)
diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py
index 9a06ad09da7..24cb9a9e7eb 100644
--- a/unsloth/dataprep/raw_text.py
+++ b/unsloth/dataprep/raw_text.py
@@ -163,55 +163,62 @@ class RawTextDataLoader:
# Tokenizer returned a count; build a range
tokens = list(range(tokens))
- if len(tokens) <= chunk_size:
+ eos_token_id = getattr(self.tokenizer, "eos_token_id", None)
+ eos_token = getattr(self.tokenizer, "eos_token", None) or ""
+ has_eos = eos_token_id is not None if return_tokenized else bool(eos_token)
+ # The final EOS counts toward chunk_size; chunk_size 1 cannot hold a token plus EOS.
+ reserve_eos = has_eos and chunk_size > 1
+ num_tokens = len(tokens) + int(reserve_eos)
+
+ if num_tokens <= chunk_size:
# Fits in a single chunk
if return_tokenized:
tokens = tokens.tolist() if hasattr(tokens, "tolist") else list(tokens)
- eos_token_id = getattr(self.tokenizer, "eos_token_id", None)
if eos_token_id is not None:
tokens.append(eos_token_id)
attention_mask = [1] * len(tokens)
return [{"input_ids": tokens, "attention_mask": attention_mask}]
else:
- eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""
return [text + eos_token]
- chunks = []
+ bounds = []
start_idx = 0
+ while True:
+ end_idx = min(start_idx + chunk_size, num_tokens)
+ bounds.append([start_idx, end_idx])
+ if end_idx == num_tokens:
+ break
+ start_idx += chunk_size - stride
- while start_idx < len(tokens):
- end_idx = min(start_idx + chunk_size, len(tokens))
+ # Stride 0 + exact multiple leaves a lone-EOS chunk: end a full window at the EOS instead, so the
+ # EOS keeps its context (a [token, EOS] row spikes loss) and no content token loses its label.
+ if reserve_eos and len(bounds) > 1 and bounds[-1][0] == len(tokens):
+ bounds[-1][0] = num_tokens - chunk_size
+
+ chunks = []
+ for i, (start_idx, end_idx) in enumerate(bounds):
chunk_tokens = tokens[start_idx:end_idx]
+ # EOS only at the true end: a full chunk mid-stride continues in the next chunk.
+ is_last = i == len(bounds) - 1
if return_tokenized:
chunk_tokens_list = (
chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens)
)
-
- # EOS only at the true end: a full chunk mid-stride continues in the next chunk.
- if end_idx == len(tokens):
- eos_token_id = getattr(self.tokenizer, "eos_token_id", None)
- if eos_token_id is not None:
- chunk_tokens_list.append(eos_token_id)
+ if is_last and eos_token_id is not None:
+ chunk_tokens_list.append(eos_token_id)
attention_mask = [1] * len(chunk_tokens_list)
chunks.append({"input_ids": chunk_tokens_list, "attention_mask": attention_mask})
else:
chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)
-
- if end_idx == len(tokens):
- eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""
+ if is_last:
chunk_text += eos_token
chunks.append(chunk_text)
- # Advance with stride overlap
- if end_idx == len(tokens):
- break
- start_idx += chunk_size - stride
-
return chunks
def _read_file_by_format(self, file_path, file_format):