💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
"""
Similarity module tests
"""
import unittest
from txtai.pipeline import Similarity
class TestSimilarity(unittest.TestCase):
"""
Similarity tests.
"""
@classmethod
def setUpClass(cls):
"""
Create single labels instance.
"""
cls.data = [
"US tops 5 million confirmed virus cases",
"Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
"Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
"The National Park Service warns against sacrificing slower friends in a bear attack",
"Maine man wins $1M from $25 lottery ticket",
"Make huge profits without work, earn up to $100,000 a day",
]
cls.similarity = Similarity("prajjwal1/bert-medium-mnli")
def testCrossEncoder(self):
"""
Test cross-encoder similarity model
"""
similarity = Similarity("cross-encoder/ms-marco-MiniLM-L-2-v2", crossencode=True)
uid = similarity("Who won the lottery?", self.data)[0][0]
self.assertEqual(self.data[uid], self.data[4])
def testCrossEncoderBatch(self):
"""
Test cross-encoder similarity model with multiple inputs
"""
similarity = Similarity("cross-encoder/ms-marco-MiniLM-L-2-v2", crossencode=True)
results = [r[0][0] for r in similarity(["Who won the lottery?", "Where did an iceberg collapse?"], self.data)]
self.assertEqual(results, [4, 1])
def testCrossEncoderLabels(self):
"""
Test cross-encoder ranking with a multi-class model, restricted to a single label via labels
"""
similarity = Similarity("prajjwal1/bert-medium-mnli", crossencode=True)
premise = "A dog is running in the park."
entailment = "An animal is outside."
contradiction = "The park is empty and there are no animals."
# Without labels, each candidate is scored by whichever label it individually scored highest
# on - here the contradiction's own top score outscores the entailment's own top score, so
# the less relevant candidate wins
uid = similarity(premise, [entailment, contradiction])[0][0]
self.assertEqual(uid, 1)
# Restricting to a single label (LABEL_1 = entailment for this model) compares every
# candidate on that same label instead, fixing the ranking
uid = similarity(premise, [entailment, contradiction], labels=["1"])[0][0]
self.assertEqual(uid, 0)
def testCrossEncoderProbability(self):
"""
Test cross-encoder labels rank candidates by the requested label's probability, not its raw logit
"""
similarity = Similarity("prajjwal1/bert-medium-mnli", crossencode=True)
premise = "A dog is running in the park."
texts = ["The dog is chasing a ball in the park.", "It is raining.", "Nobody is running."]
# "Nobody is running." has the highest entailment logit but the lowest entailment probability
for multilabel in [True, None, False]:
result = similarity(premise, texts, multilabel=multilabel, labels=["1"])
self.assertEqual([uid for uid, _ in result], [0, 1, 2])
# Raw scores are the entailment probabilities
result = similarity(premise, texts, multilabel=None, labels=["1"])
self.assertAlmostEqual(result[0][1], 0.052, places=3)
def testCrossEncoderLabelsOrder(self):
"""
Test cross-encoder labels resolve to label ids when the model config's label order differs from id order
"""
similarity = Similarity("prajjwal1/bert-medium-mnli", crossencode=True)
premise = "A dog is running in the park."
texts = ["An animal is outside.", "The park is empty and there are no animals."]
# Scores restricted to LABEL_1 with the model config as loaded
expected = similarity(premise, texts, labels=["1"])
# Rotate the config label mappings so insertion order no longer matches label ids
config = similarity.pipeline.model.config
ids = list(config.id2label)
config.id2label = {x: config.id2label[x] for x in ids[1:] + ids[:1]}
config.label2id = {label: x for x, label in config.id2label.items()}
# Verify the same label is scored when requested by name and by id
self.assertEqual(similarity(premise, texts, labels=["LABEL_1"]), expected)
self.assertEqual(similarity(premise, texts, labels=["1"]), expected)
def testCrossEncoderLabelsInvalid(self):
"""
Test cross-encoder raises an error when no requested label matches the model
"""
similarity = Similarity("prajjwal1/bert-medium-mnli", crossencode=True)
with self.assertRaises(ValueError):
similarity("A dog is running in the park.", ["An animal is outside."], labels=["not-a-real-label"])
def testLateEncoder(self):
"""
Test late-encoder similarity model
"""
similarity = Similarity("neuml/pylate-bert-tiny", lateencode=True)
uid = similarity("Who won the lottery?", self.data)[0][0]
self.assertEqual(self.data[uid], self.data[4])
# Test encode method
# pylint: disable=E1101
self.assertEqual(similarity.encode(["Who won the lottery?"], "data").shape, (1, 8, 128))
def testLateEncoderBatch(self):
"""
Test late-encoder similarity model with multiple inputs
"""
similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True)
queries = ["Who won the lottery?", "Where did an iceberg collapse?"]
results = similarity(queries, self.data)
self.assertEqual([r[0][0] for r in results], [4, 1])
# Test scores don't change with query batch padding
for query, scores in zip(queries, results):
alone = dict(similarity(query, self.data))
for uid, score in scores:
self.assertAlmostEqual(score, alone[uid], places=4)
def testLateEncoderLimit(self):
"""
Test late-encoder similarity model with a limit returns the top scoring results
"""
similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True)
queries = ["Who won the lottery?", "Where did an iceberg collapse?"]
for query in queries:
results = similarity(query, self.data, limit=3)
scores = [score for _, score in results]
# Limited results must match the top results of the unlimited call, sorted by score descending
self.assertEqual([uid for uid, _ in results], [uid for uid, _ in similarity(query, self.data)[:3]])
self.assertEqual(scores, sorted(scores, reverse=True))
def testLateEncoderPadding(self):
"""
Test late-encoder scores don't change with data batch padding
"""
texts = self.data + ["Short text."]
queries = ["Who won the lottery?", "Where did an iceberg collapse?"]
# Center token vectors on a fixed collection mean, this is batch independent and makes negative token similarities common
similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True)
vectors = similarity.encode(texts, "data")
mean = vectors[vectors.abs().sum(dim=-1) > 0].mean(dim=0).cpu().numpy()
similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": {"scope": "collection", "mean": mean}})
# Each text encoded alone has no padding, scores must match the exact MaxSim over true tokens
for query in queries:
scores, vectors = dict(similarity(query, texts)), similarity.encode([query], "query")[0]
for uid, text in enumerate(texts):
exact = (vectors @ similarity.encode([text], "data")[0].T).max(dim=1).values.mean().item()
self.assertAlmostEqual(scores[uid], exact, places=4)
def testSimilarity(self):
"""
Test similarity with single query
"""
uid = self.similarity("feel good story", self.data)[0][0]
self.assertEqual(self.data[uid], self.data[4])
def testSimilarityBatch(self):
"""
Test similarity with multiple queries
"""
results = [r[0][0] for r in self.similarity(["feel good story", "climate change"], self.data)]
self.assertEqual(results, [4, 1])
def testSimilarityFixed(self):
"""
Test similarity with a fixed label text classification model
"""
similarity = Similarity(dynamic=False)
# Test with query as label text and label id
self.assertLessEqual(similarity("negative", ["This is the best sentence ever"])[0][1], 0.1)
self.assertLessEqual(similarity("0", ["This is the best sentence ever"])[0][1], 0.1)
def testSimilarityLong(self):
"""
Test similarity with long text
"""
uid = self.similarity("other", ["Very long text " * 1000, "other text"])[0][0]
self.assertEqual(uid, 1)