Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add test for tabular pipeline

davidmezzetti4fa2d2da45b8
diff --git a/src/python/txtai/pipeline/data/tabular.py b/src/python/txtai/pipeline/data/tabular.py
index b462f337..1b0cffbd 100644
--- a/src/python/txtai/pipeline/data/tabular.py
+++ b/src/python/txtai/pipeline/data/tabular.py
@@ -27,8 +27,8 @@ class Tabular(Pipeline):
Args:
idcolumn: column name to use for row id
textcolumns: list of columns to combine as a text field
- content: if True, a dict per row is generated contained all fields. If content is a list, a subset of fields
- is included in the generated content records.
+ content: if True, a dict per row is generated with all fields. If content is a list, a subset of fields
+ is included in the generated rows.
"""
if not PANDAS:
diff --git a/test/python/testtabular.py b/test/python/testtabular.py
index 7dc087b7..0d9d88e8 100644
--- a/test/python/testtabular.py
+++ b/test/python/testtabular.py
@@ -23,6 +23,33 @@ class TestTabular(unittest.TestCase):
cls.tabular = Tabular("id", ["text"])
+ def testContent(self):
+ """
+ Tests parsing additional content
+ """
+
+ tabular = Tabular("id", ["text"], True)
+
+ row = {"id": 0, "text": "This is a test", "flag": 1}
+
+ # When content is enabled, both (uid, text, tags) and (uid, data, tags) rows will be generated
+ rows = tabular([row])
+ uid, data, _ = rows[1]
+
+ # Data should contain the entire input row
+ self.assertEqual(uid, 0)
+ self.assertEqual(data, row)
+
+ # Only select flag field
+ tabular.content = ["flag"]
+ rows = tabular([row])
+ uid, data, _ = rows[1]
+
+ # Data should only contain a single field, flag
+ self.assertEqual(uid, 0)
+ self.assertTrue(list(data.keys()) == ["flag"])
+ self.assertEqual(data["flag"], 1)
+
def testCSV(self):
"""
Tests parsing a CSV file