💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep uppercase tokens in alphanumeric tokenization with lowercase=False (#1287)
Mohammad Hijjawie313248ff25e
diff --git a/src/python/txtai/pipeline/data/tokenizer.py b/src/python/txtai/pipeline/data/tokenizer.pyindex 6dc575b8..2ede126a 100644--- a/src/python/txtai/pipeline/data/tokenizer.py+++ b/src/python/txtai/pipeline/data/tokenizer.py@@ -82,7 +82,7 @@ class Tokenizer(Pipeline):# - Strings to be at least 2 characters long AND# - At least 1 non-trailing alpha character in string# Note: The standard Python re module is much faster than regex for this expression- self.alphanum = re.compile(r"^\d*[a-z][\-.0-9:_a-z]{1,}$")+ self.alphanum = re.compile(r"^\d*[a-zA-Z][\-.0-9:_a-zA-Z]{1,}$")elif regexp:# Regular expression for tokenizationself.regexp = regex.compile(regexp)diff --git a/test/python/testpipeline/testdata/testtokenizer.py b/test/python/testpipeline/testdata/testtokenizer.pyindex b73f727a..cd8fdfb7 100644--- a/test/python/testpipeline/testdata/testtokenizer.py+++ b/test/python/testpipeline/testdata/testtokenizer.py@@ -21,6 +21,9 @@ class TestTokenizer(unittest.TestCase):self.assertEqual(Tokenizer.tokenize("Y this is a test!"), ["test"])self.assertEqual(Tokenizer.tokenize("abc123 ABC 123"), ["abc123", "abc"])+ # Alphanumeric tokenization keeps uppercase tokens when lowercasing is disabled+ self.assertEqual(Tokenizer.tokenize("Abc123 ABC 123 Y", lowercase=False), ["Abc123", "ABC"])+def testEmptyTokenize(self):"""Test handling empty and None inputs