💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add multilingual graph topic modeling, closes #511
davidmezzetti26634f8cbc90
diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.pyindex 9770d6c5..3bb4dc2a 100644--- a/src/python/txtai/graph/topics.py+++ b/src/python/txtai/graph/topics.py@@ -2,8 +2,6 @@Topics module"""-import re-from ..pipeline import Tokenizerfrom ..scoring import ScoringFactory@@ -22,8 +20,7 @@ class Topics:"""self.config = config- self.pattern = re.compile(r"(?u)\b\w\w+\b")- self.tokenizer = Tokenizer()+ self.tokenizer = Tokenizer(stopwords=True)# Additional stopwords to ignore when building topic namesself.stopwords = set()@@ -109,7 +106,7 @@ class Topics:"""text = graph.attribute(node, "text")- return self.pattern.findall(text.lower()) if text else []+ return self.tokenizer(text) if text else []def topn(self, terms, n):"""