Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add multilingual graph topic modeling, closes #511

davidmezzetti26634f8cbc90
diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.py
index 9770d6c5..3bb4dc2a 100644
--- a/src/python/txtai/graph/topics.py
+++ b/src/python/txtai/graph/topics.py
@@ -2,8 +2,6 @@
Topics module
"""
-import re
-
from ..pipeline import Tokenizer
from ..scoring import ScoringFactory
@@ -22,8 +20,7 @@ class Topics:
"""
self.config = config
- self.pattern = re.compile(r"(?u)\b\w\w+\b")
- self.tokenizer = Tokenizer()
+ self.tokenizer = Tokenizer(stopwords=True)
# Additional stopwords to ignore when building topic names
self.stopwords = set()
@@ -109,7 +106,7 @@ class Topics:
"""
text = graph.attribute(node, "text")
- return self.pattern.findall(text.lower()) if text else []
+ return self.tokenizer(text) if text else []
def topn(self, terms, n):
"""