Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add sql offset clause support, closes #293

davidmezzettie444d771eeba
diff --git a/src/python/txtai/database/sql/base.py b/src/python/txtai/database/sql/base.py
index 0fd8e75e..7b399a28 100644
--- a/src/python/txtai/database/sql/base.py
+++ b/src/python/txtai/database/sql/base.py
@@ -14,7 +14,7 @@ class SQL:
"""
# List of clauses to parse
- CLAUSES = ["select", "from", "where", "group", "having", "order", "limit"]
+ CLAUSES = ["select", "from", "where", "group", "having", "order", "limit", "offset"]
def __init__(self, database=None, tolist=False):
"""
@@ -59,6 +59,7 @@ class SQL:
"having": self.parse(tokens, positions, "having", aliases=aliases),
"orderby": self.parse(tokens, positions, "order", offset=2, aliases=aliases),
"limit": self.parse(tokens, positions, "limit", aliases=aliases),
+ "offset": self.parse(tokens, positions, "offset", aliases=aliases),
}
# Add parsed similar queries, if any
diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.py
index 1e83efea..ebb5cabe 100644
--- a/src/python/txtai/database/sqlite.py
+++ b/src/python/txtai/database/sqlite.py
@@ -270,12 +270,13 @@ class SQLite(Database):
# Return ids clause placeholder
return SQLite.IDS_CLAUSE % batch
+ # pylint: disable=R0912
def query(self, query, limit):
# Extract query components
select = query.get("select", self.defaults())
where = query.get("where")
groupby, having = query.get("groupby"), query.get("having")
- orderby, qlimit = query.get("orderby"), query.get("limit")
+ orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")
similarity = query.get("similar")
# Build query text
@@ -297,6 +298,10 @@ class SQLite(Database):
if qlimit is not None or limit:
query += f" LIMIT {qlimit if qlimit else limit}"
+ # Apply offset
+ if offset is not None:
+ query += f" OFFSET {offset}"
+
# Clear scores when no similar clauses present
if not similarity:
self.scores(None)
diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.py
index a7d8d1c0..e83fef45 100644
--- a/test/python/testdatabase/testembeddings.py
+++ b/test/python/testdatabase/testembeddings.py
@@ -405,6 +405,10 @@ class TestEmbeddings(unittest.TestCase):
result = self.embeddings.search("select * from txtai where similar('feel good story', 1) limit 1")[0]
self.assertEqual(result["text"], self.data[4])
+ # Test similar with offset
+ result = self.embeddings.search("select * from txtai where similar('feel good story') offset 1")[0]
+ self.assertEqual(result["text"], self.data[5])
+
# Test where
result = self.embeddings.search("select * from txtai where text like '%iceberg%'", 1)[0]
self.assertEqual(result["text"], self.data[1])
diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.py
index 06033ccd..342be47e 100644
--- a/test/python/testdatabase/testsql.py
+++ b/test/python/testdatabase/testsql.py
@@ -119,6 +119,16 @@ class TestSQL(unittest.TestCase):
self.assertSql("limit", prefix + "limit 100", "100")
+ def testOffset(self):
+ """
+ Test offset clauses
+ """
+
+ prefix = "select count(*) from txtai "
+
+ self.assertSql("offset", prefix + "limit 100 offset 50", "50")
+ self.assertSql("offset", prefix + "offset 50", "50")
+
def testOrderby(self):
"""
Test order by clauses