💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep community nodes that match none of the topic terms (#1243)
hylindcdded892d3c
diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.pyindex 8c0dfde0..9cf3073f 100644--- a/src/python/txtai/graph/topics.py+++ b/src/python/txtai/graph/topics.py@@ -82,8 +82,13 @@ class Topics:# Get topn termsterms = self.topn(idf, topn)- # Sort community by score descending- community = [uid for uid, _ in scoring.search(terms, len(community))]+ # Sort community by score descending. Term search only returns nodes with a+ # positive score, so nodes matching none of the topic terms come back empty.+ # Keep them, ranked last by centrality, otherwise they are dropped from the+ # topic and lose their topic/topicrank attributes entirely.+ ranked = [uid for uid, _ in scoring.search(terms, len(community))]+ unranked = sorted(set(community) - set(ranked), key=lambda x: centrality[x], reverse=True)+ community = ranked + unrankedelse:# No text found for topic, generate topic nameterms = ["topic", str(index)]diff --git a/test/python/testgraph.py b/test/python/testgraph.pyindex 0523dd4f..d4613198 100644--- a/test/python/testgraph.py+++ b/test/python/testgraph.py@@ -12,6 +12,7 @@ from unittest.mock import patchfrom txtai.archive import ArchiveFactoryfrom txtai.embeddings import Embeddingsfrom txtai.graph import Graph, GraphFactory+from txtai.graph.topics import Topicsfrom txtai.serialize import SerializeFactory@@ -83,14 +84,45 @@ class TestGraph(unittest.TestCase):# Get graph referencegraph = self.embeddings.graph- # Rebuild topics with updated graph settings+ # Rebuild topics with updated graph settings. Every node must land in a topic,+ # including nodes matching none of their community's topic terms.graph.config = {"topics": {"algorithm": "greedy"}}graph.addtopics()self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)graph.config = {"topics": {"algorithm": "lpa"}}graph.addtopics()- self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 4)+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)++ def testCommunityUnmatchedNodes(self):+ """+ Test that nodes matching none of their community's topic terms are still kept+ """++ graph = GraphFactory.create({})+ graph.initialize()++ # Three nodes share vocabulary, one shares none of it+ texts = {+ "n1": "machine learning models train on data",+ "n2": "machine learning training data pipelines",+ "n3": "learning models and training data",+ "unmatched": "zzz",+ }+ for uid, text in texts.items():+ graph.addnode(uid, text=text)++ community = list(texts)+ centrality = {"n1": 0.9, "n2": 0.8, "n3": 0.7, "unmatched": 0.1}++ topics = Topics({"labels": "bm25", "terms": 4})+ _, members = topics.score(graph, 0, community, centrality)++ # No node may be dropped - term search only scores nodes matching a topic term+ self.assertEqual(sorted(members), sorted(community))++ # Scored nodes still rank ahead of unmatched ones+ self.assertEqual(members[-1], "unmatched")def testCustomBackend(self):"""