💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add sql offset clause support, closes #293
davidmezzettie444d771eeba
diff --git a/src/python/txtai/database/sql/base.py b/src/python/txtai/database/sql/base.pyindex 0fd8e75e..7b399a28 100644--- a/src/python/txtai/database/sql/base.py+++ b/src/python/txtai/database/sql/base.py@@ -14,7 +14,7 @@ class SQL:"""# List of clauses to parse- CLAUSES = ["select", "from", "where", "group", "having", "order", "limit"]+ CLAUSES = ["select", "from", "where", "group", "having", "order", "limit", "offset"]def __init__(self, database=None, tolist=False):"""@@ -59,6 +59,7 @@ class SQL:"having": self.parse(tokens, positions, "having", aliases=aliases),"orderby": self.parse(tokens, positions, "order", offset=2, aliases=aliases),"limit": self.parse(tokens, positions, "limit", aliases=aliases),+ "offset": self.parse(tokens, positions, "offset", aliases=aliases),}# Add parsed similar queries, if anydiff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.pyindex 1e83efea..ebb5cabe 100644--- a/src/python/txtai/database/sqlite.py+++ b/src/python/txtai/database/sqlite.py@@ -270,12 +270,13 @@ class SQLite(Database):# Return ids clause placeholderreturn SQLite.IDS_CLAUSE % batch+ # pylint: disable=R0912def query(self, query, limit):# Extract query componentsselect = query.get("select", self.defaults())where = query.get("where")groupby, having = query.get("groupby"), query.get("having")- orderby, qlimit = query.get("orderby"), query.get("limit")+ orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")similarity = query.get("similar")# Build query text@@ -297,6 +298,10 @@ class SQLite(Database):if qlimit is not None or limit:query += f" LIMIT {qlimit if qlimit else limit}"+ # Apply offset+ if offset is not None:+ query += f" OFFSET {offset}"+# Clear scores when no similar clauses presentif not similarity:self.scores(None)diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.pyindex a7d8d1c0..e83fef45 100644--- a/test/python/testdatabase/testembeddings.py+++ b/test/python/testdatabase/testembeddings.py@@ -405,6 +405,10 @@ class TestEmbeddings(unittest.TestCase):result = self.embeddings.search("select * from txtai where similar('feel good story', 1) limit 1")[0]self.assertEqual(result["text"], self.data[4])+ # Test similar with offset+ result = self.embeddings.search("select * from txtai where similar('feel good story') offset 1")[0]+ self.assertEqual(result["text"], self.data[5])+# Test whereresult = self.embeddings.search("select * from txtai where text like '%iceberg%'", 1)[0]self.assertEqual(result["text"], self.data[1])diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.pyindex 06033ccd..342be47e 100644--- a/test/python/testdatabase/testsql.py+++ b/test/python/testdatabase/testsql.py@@ -119,6 +119,16 @@ class TestSQL(unittest.TestCase):self.assertSql("limit", prefix + "limit 100", "100")+ def testOffset(self):+ """+ Test offset clauses+ """++ prefix = "select count(*) from txtai "++ self.assertSql("offset", prefix + "limit 100 offset 50", "50")+ self.assertSql("offset", prefix + "offset 50", "50")+def testOrderby(self):"""Test order by clauses