Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep community nodes that match none of the topic terms (#1243)

hylindcdded892d3c
diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.py
index 8c0dfde0..9cf3073f 100644
--- a/src/python/txtai/graph/topics.py
+++ b/src/python/txtai/graph/topics.py
@@ -82,8 +82,13 @@ class Topics:
# Get topn terms
terms = self.topn(idf, topn)
- # Sort community by score descending
- community = [uid for uid, _ in scoring.search(terms, len(community))]
+ # Sort community by score descending. Term search only returns nodes with a
+ # positive score, so nodes matching none of the topic terms come back empty.
+ # Keep them, ranked last by centrality, otherwise they are dropped from the
+ # topic and lose their topic/topicrank attributes entirely.
+ ranked = [uid for uid, _ in scoring.search(terms, len(community))]
+ unranked = sorted(set(community) - set(ranked), key=lambda x: centrality[x], reverse=True)
+ community = ranked + unranked
else:
# No text found for topic, generate topic name
terms = ["topic", str(index)]
diff --git a/test/python/testgraph.py b/test/python/testgraph.py
index 0523dd4f..d4613198 100644
--- a/test/python/testgraph.py
+++ b/test/python/testgraph.py
@@ -12,6 +12,7 @@ from unittest.mock import patch
from txtai.archive import ArchiveFactory
from txtai.embeddings import Embeddings
from txtai.graph import Graph, GraphFactory
+from txtai.graph.topics import Topics
from txtai.serialize import SerializeFactory
@@ -83,14 +84,45 @@ class TestGraph(unittest.TestCase):
# Get graph reference
graph = self.embeddings.graph
- # Rebuild topics with updated graph settings
+ # Rebuild topics with updated graph settings. Every node must land in a topic,
+ # including nodes matching none of their community's topic terms.
graph.config = {"topics": {"algorithm": "greedy"}}
graph.addtopics()
self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)
graph.config = {"topics": {"algorithm": "lpa"}}
graph.addtopics()
- self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 4)
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)
+
+ def testCommunityUnmatchedNodes(self):
+ """
+ Test that nodes matching none of their community's topic terms are still kept
+ """
+
+ graph = GraphFactory.create({})
+ graph.initialize()
+
+ # Three nodes share vocabulary, one shares none of it
+ texts = {
+ "n1": "machine learning models train on data",
+ "n2": "machine learning training data pipelines",
+ "n3": "learning models and training data",
+ "unmatched": "zzz",
+ }
+ for uid, text in texts.items():
+ graph.addnode(uid, text=text)
+
+ community = list(texts)
+ centrality = {"n1": 0.9, "n2": 0.8, "n3": 0.7, "unmatched": 0.1}
+
+ topics = Topics({"labels": "bm25", "terms": 4})
+ _, members = topics.score(graph, 0, community, centrality)
+
+ # No node may be dropped - term search only scores nodes matching a topic term
+ self.assertEqual(sorted(members), sorted(community))
+
+ # Scored nodes still rank ahead of unmatched ones
+ self.assertEqual(members[-1], "unmatched")
def testCustomBackend(self):
"""