Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep uppercase tokens in alphanumeric tokenization with lowercase=False (#1287)

Mohammad Hijjawie313248ff25e
diff --git a/src/python/txtai/pipeline/data/tokenizer.py b/src/python/txtai/pipeline/data/tokenizer.py
index 6dc575b8..2ede126a 100644
--- a/src/python/txtai/pipeline/data/tokenizer.py
+++ b/src/python/txtai/pipeline/data/tokenizer.py
@@ -82,7 +82,7 @@ class Tokenizer(Pipeline):
# - Strings to be at least 2 characters long AND
# - At least 1 non-trailing alpha character in string
# Note: The standard Python re module is much faster than regex for this expression
- self.alphanum = re.compile(r"^\d*[a-z][\-.0-9:_a-z]{1,}$")
+ self.alphanum = re.compile(r"^\d*[a-zA-Z][\-.0-9:_a-zA-Z]{1,}$")
elif regexp:
# Regular expression for tokenization
self.regexp = regex.compile(regexp)
diff --git a/test/python/testpipeline/testdata/testtokenizer.py b/test/python/testpipeline/testdata/testtokenizer.py
index b73f727a..cd8fdfb7 100644
--- a/test/python/testpipeline/testdata/testtokenizer.py
+++ b/test/python/testpipeline/testdata/testtokenizer.py
@@ -21,6 +21,9 @@ class TestTokenizer(unittest.TestCase):
self.assertEqual(Tokenizer.tokenize("Y this is a test!"), ["test"])
self.assertEqual(Tokenizer.tokenize("abc123 ABC 123"), ["abc123", "abc"])
+ # Alphanumeric tokenization keeps uppercase tokens when lowercasing is disabled
+ self.assertEqual(Tokenizer.tokenize("Abc123 ABC 123 Y", lowercase=False), ["Abc123", "ABC"])
+
def testEmptyTokenize(self):
"""
Test handling empty and None inputs