💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add test for tabular pipeline
davidmezzetti4fa2d2da45b8
diff --git a/src/python/txtai/pipeline/data/tabular.py b/src/python/txtai/pipeline/data/tabular.pyindex b462f337..1b0cffbd 100644--- a/src/python/txtai/pipeline/data/tabular.py+++ b/src/python/txtai/pipeline/data/tabular.py@@ -27,8 +27,8 @@ class Tabular(Pipeline):Args:idcolumn: column name to use for row idtextcolumns: list of columns to combine as a text field- content: if True, a dict per row is generated contained all fields. If content is a list, a subset of fields- is included in the generated content records.+ content: if True, a dict per row is generated with all fields. If content is a list, a subset of fields+ is included in the generated rows."""if not PANDAS:diff --git a/test/python/testtabular.py b/test/python/testtabular.pyindex 7dc087b7..0d9d88e8 100644--- a/test/python/testtabular.py+++ b/test/python/testtabular.py@@ -23,6 +23,33 @@ class TestTabular(unittest.TestCase):cls.tabular = Tabular("id", ["text"])+ def testContent(self):+ """+ Tests parsing additional content+ """++ tabular = Tabular("id", ["text"], True)++ row = {"id": 0, "text": "This is a test", "flag": 1}++ # When content is enabled, both (uid, text, tags) and (uid, data, tags) rows will be generated+ rows = tabular([row])+ uid, data, _ = rows[1]++ # Data should contain the entire input row+ self.assertEqual(uid, 0)+ self.assertEqual(data, row)++ # Only select flag field+ tabular.content = ["flag"]+ rows = tabular([row])+ uid, data, _ = rows[1]++ # Data should only contain a single field, flag+ self.assertEqual(uid, 0)+ self.assertTrue(list(data.keys()) == ["flag"])+ self.assertEqual(data["flag"], 1)+def testCSV(self):"""Tests parsing a CSV file