Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add tokenbatch size to HFTrainer pipeline, closes #1133

davidmezzettic3735dd5d46b
diff --git a/src/python/txtai/data/base.py b/src/python/txtai/data/base.py
index 86012e26..5a98c6d2 100644
--- a/src/python/txtai/data/base.py
+++ b/src/python/txtai/data/base.py
@@ -24,7 +24,7 @@ class Data:
self.columns = columns
self.maxlength = maxlength
- def __call__(self, train, validation, workers):
+ def __call__(self, train, validation, workers, batch):
"""
Tokenizes training and validation data and returns processed datasets.
@@ -32,14 +32,15 @@ class Data:
train: training data
validation: validation data
workers: number of concurrent tokenizers when processing datasets, only main process used when set to None
+ batch: tokenize batch size
Returns:
(train, validation)
"""
- return (self.prepare(train, self.process, workers), self.prepare(validation, self.process, workers) if validation else None)
+ return (self.prepare(train, self.process, workers, batch), self.prepare(validation, self.process, workers, batch) if validation else None)
- def prepare(self, data, fn, workers):
+ def prepare(self, data, fn, workers, batch):
"""
Prepares and tokenizes data for training.
@@ -47,6 +48,7 @@ class Data:
data: input data
fn: tokenize processing function to apply
workers: number of concurrent tokenizers when processing datasets, only main process used when set to None
+ batch: tokenize batch size
Returns:
tokens
@@ -54,7 +56,7 @@ class Data:
if hasattr(data, "map"):
# Hugging Face dataset
- tokens = data.map(fn, batched=True, num_proc=workers, remove_columns=data.column_names)
+ tokens = data.map(fn, batched=True, batch_size=batch, num_proc=workers, remove_columns=data.column_names)
else:
# Re-orient data into columns for efficient batch tokenization
columns = {}
diff --git a/src/python/txtai/pipeline/train/hftrainer.py b/src/python/txtai/pipeline/train/hftrainer.py
index 32676848..4be49af1 100644
--- a/src/python/txtai/pipeline/train/hftrainer.py
+++ b/src/python/txtai/pipeline/train/hftrainer.py
@@ -54,6 +54,7 @@ class HFTrainer(Tensors):
merge="concat",
teacher=None,
distillation=(1.0, 0.5),
+ tokenizebatch=1000,
**args
):
"""
@@ -76,6 +77,7 @@ class HFTrainer(Tensors):
merge: determines how chunks are combined for language modeling tasks - "concat" (default), "pack" or None
teacher: path to teacher model for distillation, supports same options as base
distillation: distillation parameters for (temperature, alpha), defaults to (1.0, 0.5)
+ tokenizebatch: batch size for tokenization process
args: training arguments
Returns:
@@ -102,7 +104,7 @@ class HFTrainer(Tensors):
process, collator, labels = self.prepare(task, train, tokenizer, columns, maxlength, stride, prefix, merge, args)
# Tokenize training and validation data
- train, validation = process(train, validation, os.cpu_count() if tokenizers and isinstance(tokenizers, bool) else tokenizers)
+ train, validation = process(train, validation, os.cpu_count() if tokenizers and isinstance(tokenizers, bool) else tokenizers, tokenizebatch)
# Create model to train
model = self.model(task, base, config, labels, tokenizer, quantize)