💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add flatten parameter to entity pipeline #203
davidmezzetti8a46872365fc
diff --git a/src/python/txtai/pipeline/text/entity.py b/src/python/txtai/pipeline/text/entity.pyindex 0ae7f70a..60f75d07 100644--- a/src/python/txtai/pipeline/text/entity.py+++ b/src/python/txtai/pipeline/text/entity.py@@ -13,17 +13,18 @@ class Entity(HFPipeline):def __init__(self, path=None, quantize=False, gpu=True, model=None):super().__init__("token-classification", path, quantize, gpu, model)- def __call__(self, text, aggregate="simple", workers=0):+ def __call__(self, text, aggregate="simple", flatten=None, workers=0):"""Applies a token classifier to text.Args:text: text|listaggregate: method to combine multi token entities - options are "simple" (default), "first", "average" or "max"+ flatten: flatten output to a list of labels if present. Accepts a boolean or float value to only keep scores greater than that number.workers: number of concurrent workers to use for processing data, defaults to NoneReturns:- list of (entity, entity type, score)+ list of (entity, entity type, score) or list of entities depending on flatten parameter"""# Run token classification pipeline@@ -33,9 +34,15 @@ class Entity(HFPipeline):if isinstance(text, str):results = [results]- # Extract (entity, entity type, score) tuples+ # Score threshold when flatten is set+ threshold = 0.0 if isinstance(flatten, bool) else flatten++ # Extract entities if flatten set, otherwise extract (entity, entity type, score) tuplesoutputs = []for result in results:- outputs.append([(r["word"], r["entity_group"], float(r["score"])) for r in result])+ if flatten:+ outputs.append([r["word"] for r in result if r["score"] >= threshold])+ else:+ outputs.append([(r["word"], r["entity_group"], float(r["score"])) for r in result])return outputs[0] if isinstance(text, str) else outputsdiff --git a/test/python/testpipeline/testentity.py b/test/python/testpipeline/testentity.pyindex 5e34a114..0adcf0fc 100644--- a/test/python/testpipeline/testentity.py+++ b/test/python/testpipeline/testentity.py@@ -12,13 +12,24 @@ class TestEntity(unittest.TestCase):Entity tests."""+ @classmethod+ def setUpClass(cls):+ cls.entity = Entity("dslim/bert-base-NER")+def testEntity(self):"""Test entity"""# Run entity extraction- entity = Entity("dslim/bert-base-NER")- entities = entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg")-+ entities = self.entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg")self.assertEqual([e[0] for e in entities], ["Canada", "Manhattan"])++ def testEntityFlatten(self):+ """+ Test entity with flattened output+ """++ # Run entity extraction+ entities = self.entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg", flatten=True)+ self.assertEqual(entities, ["Canada", "Manhattan"])