💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Improve logic for handling overflows with text tokenization
davidmezzetti4fd3fc48e7e6
diff --git a/src/python/txtai/pipeline/hfmodel.py b/src/python/txtai/pipeline/hfmodel.pyindex b7eacb3c..9613208a 100644--- a/src/python/txtai/pipeline/hfmodel.py+++ b/src/python/txtai/pipeline/hfmodel.py@@ -65,8 +65,15 @@ class HFModel(Tensors):(tokenization result, indices)"""+ # Pre-process and split on newlines+ batch, positions = [], []+ for x, text in enumerate(texts):+ elements = [t + " " for t in text.split("\n") if t]+ batch.extend(elements)+ positions.extend([x] * len(elements))+# Run tokenizer- tokens = tokenizer(texts, padding=True)+ tokens = tokenizer(batch, padding=True)inputids, attention, indices = [], [], []for x, ids in enumerate(tokens["input_ids"]):@@ -91,11 +98,11 @@ class HFModel(Tensors):inputids.append(chunk)attention.append(mask)- indices.append(x)+ indices.append(positions[x])else:inputids.append(ids)attention.append(tokens["attention_mask"][x])- indices.append(x)+ indices.append(positions[x])tokens = {"input_ids": inputids, "attention_mask": attention}