💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix Cluster combining avg() as an unweighted mean of shard averages (#1237)
Amir Fathi72f9ca7605ed
diff --git a/src/python/txtai/database/sql/aggregate.py b/src/python/txtai/database/sql/aggregate.pyindex e21d5ae8..6a511fb8 100644--- a/src/python/txtai/database/sql/aggregate.py+++ b/src/python/txtai/database/sql/aggregate.py@@ -6,6 +6,7 @@ import itertoolsimport operatorfrom .base import SQL+from .error import SQLErrorclass Aggregate(SQL):@@ -70,10 +71,32 @@ class Aggregate(SQL):elif column.startswith("min("):aggregates[column] = minelif column.startswith("avg("):- aggregates[column] = lambda x: sum(x) / len(x)+ aggregates[column] = self.avgreturn aggregates+ def avg(self, values, counts):+ """+ Combines per-shard/per-group average values into a single average, weighted by a+ matching count(*) column when one was selected. Without it, there is no way to+ recombine the averages correctly, so this raises instead of returning a wrong number.++ Args:+ values: list of per-shard/per-group average values+ counts: list of matching count(*) values, or None++ Returns:+ combined average+ """++ if len(values) == 1:+ return values[0]++ if not counts or sum(counts) == 0:+ raise SQLError("avg() requires a count(*) column to combine results from multiple shards")++ return sum(value * count for value, count in zip(values, counts)) / sum(counts)+def aggregate(self, query, results, columns, aggcolumns):"""Merges aggregate columns in results.@@ -94,16 +117,22 @@ class Aggregate(SQL):else:results = [results]+ # Column providing the row count each group's values were computed over, used to weight avg() columns+ countcolumn = next((column for column in columns if column.lower() == "count(*)"), None)+# Compute column valuesrows = []for result in results:+ # Row counts for this group, if a count(*) column was selected+ counts = [r[countcolumn] for r in result] if countcolumn else None+# Calculate/copy column valuesrow = {}for column in columns:if column in aggcolumns:# Calculate aggregate value- function = aggcolumns[column]- row[column] = function([r[column] for r in result])+ values = [r[column] for r in result]+ row[column] = self.avg(values, counts) if column.lower().startswith("avg(") else aggcolumns[column](values)else:# Non aggregate column value repeat, use first valuerow[column] = result[0][column]diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.pyindex 2b30cb4a..b72a3097 100644--- a/test/python/testapi/testcluster.py+++ b/test/python/testapi/testcluster.py@@ -241,7 +241,8 @@ class TestCluster(unittest.TestCase):query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text='This is a test'")self.assertEqual(- self.client.get(f"search?query={query}").json(), [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 6.5}]+ self.client.get(f"search?query={query}").json(),+ [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 182.8 / 28}],)query = urllib.parse.quote("select count(*), text txt from txtai group by txt order by count(*) desc")diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.pyindex e4b3b752..5ba63b78 100644--- a/test/python/testdatabase/testsql.py+++ b/test/python/testdatabase/testsql.py@@ -8,6 +8,7 @@ from txtai.database import DatabaseFactory, SQL, SQLErrorfrom txtai.database.sql import Aggregate+# pylint: disable=R0904class TestSQL(unittest.TestCase):"""Test SQL parsing and generation.@@ -41,6 +42,27 @@ class TestSQL(unittest.TestCase):# SQL query with results still aggregates as beforeself.assertEqual(aggregate("select count(*) from txtai", [{"count(*)": 1}, {"count(*)": 2}]), [{"count(*)": 3}])+ def testAggregateAvg(self):+ """+ Test Aggregate combines avg() results as a count(*)-weighted mean, not an unweighted+ average of averages+ """++ aggregate = Aggregate()++ # 2 rows averaging 100 and 8 rows averaging 10 must combine to the true weighted mean, 28+ query = "select count(*), avg(price) from txtai"+ results = [{"count(*)": 2, "avg(price)": 100.0}, {"count(*)": 8, "avg(price)": 10.0}]+ self.assertEqual(aggregate(query, results), [{"count(*)": 10, "avg(price)": 28.0}])++ # A single shard's average passes through unchanged+ self.assertEqual(aggregate("select avg(price) from txtai", [{"avg(price)": 42.0}]), [{"avg(price)": 42.0}])++ # Combining averages from multiple shards with no count(*) to weight by cannot be done+ # correctly, so it must raise rather than return an unweighted, silently wrong result+ with self.assertRaises(SQLError):+ aggregate("select avg(price) from txtai", [{"avg(price)": 100.0}, {"avg(price)": 10.0}])+def testAlias(self):"""Test alias clauses