Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix Cluster combining avg() as an unweighted mean of shard averages (#1237)

Amir Fathi72f9ca7605ed
diff --git a/src/python/txtai/database/sql/aggregate.py b/src/python/txtai/database/sql/aggregate.py
index e21d5ae8..6a511fb8 100644
--- a/src/python/txtai/database/sql/aggregate.py
+++ b/src/python/txtai/database/sql/aggregate.py
@@ -6,6 +6,7 @@ import itertools
import operator
from .base import SQL
+from .error import SQLError
class Aggregate(SQL):
@@ -70,10 +71,32 @@ class Aggregate(SQL):
elif column.startswith("min("):
aggregates[column] = min
elif column.startswith("avg("):
- aggregates[column] = lambda x: sum(x) / len(x)
+ aggregates[column] = self.avg
return aggregates
+ def avg(self, values, counts):
+ """
+ Combines per-shard/per-group average values into a single average, weighted by a
+ matching count(*) column when one was selected. Without it, there is no way to
+ recombine the averages correctly, so this raises instead of returning a wrong number.
+
+ Args:
+ values: list of per-shard/per-group average values
+ counts: list of matching count(*) values, or None
+
+ Returns:
+ combined average
+ """
+
+ if len(values) == 1:
+ return values[0]
+
+ if not counts or sum(counts) == 0:
+ raise SQLError("avg() requires a count(*) column to combine results from multiple shards")
+
+ return sum(value * count for value, count in zip(values, counts)) / sum(counts)
+
def aggregate(self, query, results, columns, aggcolumns):
"""
Merges aggregate columns in results.
@@ -94,16 +117,22 @@ class Aggregate(SQL):
else:
results = [results]
+ # Column providing the row count each group's values were computed over, used to weight avg() columns
+ countcolumn = next((column for column in columns if column.lower() == "count(*)"), None)
+
# Compute column values
rows = []
for result in results:
+ # Row counts for this group, if a count(*) column was selected
+ counts = [r[countcolumn] for r in result] if countcolumn else None
+
# Calculate/copy column values
row = {}
for column in columns:
if column in aggcolumns:
# Calculate aggregate value
- function = aggcolumns[column]
- row[column] = function([r[column] for r in result])
+ values = [r[column] for r in result]
+ row[column] = self.avg(values, counts) if column.lower().startswith("avg(") else aggcolumns[column](values)
else:
# Non aggregate column value repeat, use first value
row[column] = result[0][column]
diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.py
index 2b30cb4a..b72a3097 100644
--- a/test/python/testapi/testcluster.py
+++ b/test/python/testapi/testcluster.py
@@ -241,7 +241,8 @@ class TestCluster(unittest.TestCase):
query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text='This is a test'")
self.assertEqual(
- self.client.get(f"search?query={query}").json(), [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 6.5}]
+ self.client.get(f"search?query={query}").json(),
+ [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 182.8 / 28}],
)
query = urllib.parse.quote("select count(*), text txt from txtai group by txt order by count(*) desc")
diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.py
index e4b3b752..5ba63b78 100644
--- a/test/python/testdatabase/testsql.py
+++ b/test/python/testdatabase/testsql.py
@@ -8,6 +8,7 @@ from txtai.database import DatabaseFactory, SQL, SQLError
from txtai.database.sql import Aggregate
+# pylint: disable=R0904
class TestSQL(unittest.TestCase):
"""
Test SQL parsing and generation.
@@ -41,6 +42,27 @@ class TestSQL(unittest.TestCase):
# SQL query with results still aggregates as before
self.assertEqual(aggregate("select count(*) from txtai", [{"count(*)": 1}, {"count(*)": 2}]), [{"count(*)": 3}])
+ def testAggregateAvg(self):
+ """
+ Test Aggregate combines avg() results as a count(*)-weighted mean, not an unweighted
+ average of averages
+ """
+
+ aggregate = Aggregate()
+
+ # 2 rows averaging 100 and 8 rows averaging 10 must combine to the true weighted mean, 28
+ query = "select count(*), avg(price) from txtai"
+ results = [{"count(*)": 2, "avg(price)": 100.0}, {"count(*)": 8, "avg(price)": 10.0}]
+ self.assertEqual(aggregate(query, results), [{"count(*)": 10, "avg(price)": 28.0}])
+
+ # A single shard's average passes through unchanged
+ self.assertEqual(aggregate("select avg(price) from txtai", [{"avg(price)": 42.0}]), [{"avg(price)": 42.0}])
+
+ # Combining averages from multiple shards with no count(*) to weight by cannot be done
+ # correctly, so it must raise rather than return an unweighted, silently wrong result
+ with self.assertRaises(SQLError):
+ aggregate("select avg(price) from txtai", [{"avg(price)": 100.0}, {"avg(price)": 10.0}])
+
def testAlias(self):
"""
Test alias clauses