💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add tokenbatch size to HFTrainer pipeline, closes #1133
davidmezzettic3735dd5d46b
diff --git a/src/python/txtai/data/base.py b/src/python/txtai/data/base.pyindex 86012e26..5a98c6d2 100644--- a/src/python/txtai/data/base.py+++ b/src/python/txtai/data/base.py@@ -24,7 +24,7 @@ class Data:self.columns = columnsself.maxlength = maxlength- def __call__(self, train, validation, workers):+ def __call__(self, train, validation, workers, batch):"""Tokenizes training and validation data and returns processed datasets.@@ -32,14 +32,15 @@ class Data:train: training datavalidation: validation dataworkers: number of concurrent tokenizers when processing datasets, only main process used when set to None+ batch: tokenize batch sizeReturns:(train, validation)"""- return (self.prepare(train, self.process, workers), self.prepare(validation, self.process, workers) if validation else None)+ return (self.prepare(train, self.process, workers, batch), self.prepare(validation, self.process, workers, batch) if validation else None)- def prepare(self, data, fn, workers):+ def prepare(self, data, fn, workers, batch):"""Prepares and tokenizes data for training.@@ -47,6 +48,7 @@ class Data:data: input datafn: tokenize processing function to applyworkers: number of concurrent tokenizers when processing datasets, only main process used when set to None+ batch: tokenize batch sizeReturns:tokens@@ -54,7 +56,7 @@ class Data:if hasattr(data, "map"):# Hugging Face dataset- tokens = data.map(fn, batched=True, num_proc=workers, remove_columns=data.column_names)+ tokens = data.map(fn, batched=True, batch_size=batch, num_proc=workers, remove_columns=data.column_names)else:# Re-orient data into columns for efficient batch tokenizationcolumns = {}diff --git a/src/python/txtai/pipeline/train/hftrainer.py b/src/python/txtai/pipeline/train/hftrainer.pyindex 32676848..4be49af1 100644--- a/src/python/txtai/pipeline/train/hftrainer.py+++ b/src/python/txtai/pipeline/train/hftrainer.py@@ -54,6 +54,7 @@ class HFTrainer(Tensors):merge="concat",teacher=None,distillation=(1.0, 0.5),+ tokenizebatch=1000,**args):"""@@ -76,6 +77,7 @@ class HFTrainer(Tensors):merge: determines how chunks are combined for language modeling tasks - "concat" (default), "pack" or Noneteacher: path to teacher model for distillation, supports same options as basedistillation: distillation parameters for (temperature, alpha), defaults to (1.0, 0.5)+ tokenizebatch: batch size for tokenization processargs: training argumentsReturns:@@ -102,7 +104,7 @@ class HFTrainer(Tensors):process, collator, labels = self.prepare(task, train, tokenizer, columns, maxlength, stride, prefix, merge, args)# Tokenize training and validation data- train, validation = process(train, validation, os.cpu_count() if tokenizers and isinstance(tokenizers, bool) else tokenizers)+ train, validation = process(train, validation, os.cpu_count() if tokenizers and isinstance(tokenizers, bool) else tokenizers, tokenizebatch)# Create model to trainmodel = self.model(task, base, config, labels, tokenizer, quantize)