Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add flatten parameter to entity pipeline #203

davidmezzetti8a46872365fc
diff --git a/src/python/txtai/pipeline/text/entity.py b/src/python/txtai/pipeline/text/entity.py
index 0ae7f70a..60f75d07 100644
--- a/src/python/txtai/pipeline/text/entity.py
+++ b/src/python/txtai/pipeline/text/entity.py
@@ -13,17 +13,18 @@ class Entity(HFPipeline):
def __init__(self, path=None, quantize=False, gpu=True, model=None):
super().__init__("token-classification", path, quantize, gpu, model)
- def __call__(self, text, aggregate="simple", workers=0):
+ def __call__(self, text, aggregate="simple", flatten=None, workers=0):
"""
Applies a token classifier to text.
Args:
text: text|list
aggregate: method to combine multi token entities - options are "simple" (default), "first", "average" or "max"
+ flatten: flatten output to a list of labels if present. Accepts a boolean or float value to only keep scores greater than that number.
workers: number of concurrent workers to use for processing data, defaults to None
Returns:
- list of (entity, entity type, score)
+ list of (entity, entity type, score) or list of entities depending on flatten parameter
"""
# Run token classification pipeline
@@ -33,9 +34,15 @@ class Entity(HFPipeline):
if isinstance(text, str):
results = [results]
- # Extract (entity, entity type, score) tuples
+ # Score threshold when flatten is set
+ threshold = 0.0 if isinstance(flatten, bool) else flatten
+
+ # Extract entities if flatten set, otherwise extract (entity, entity type, score) tuples
outputs = []
for result in results:
- outputs.append([(r["word"], r["entity_group"], float(r["score"])) for r in result])
+ if flatten:
+ outputs.append([r["word"] for r in result if r["score"] >= threshold])
+ else:
+ outputs.append([(r["word"], r["entity_group"], float(r["score"])) for r in result])
return outputs[0] if isinstance(text, str) else outputs
diff --git a/test/python/testpipeline/testentity.py b/test/python/testpipeline/testentity.py
index 5e34a114..0adcf0fc 100644
--- a/test/python/testpipeline/testentity.py
+++ b/test/python/testpipeline/testentity.py
@@ -12,13 +12,24 @@ class TestEntity(unittest.TestCase):
Entity tests.
"""
+ @classmethod
+ def setUpClass(cls):
+ cls.entity = Entity("dslim/bert-base-NER")
+
def testEntity(self):
"""
Test entity
"""
# Run entity extraction
- entity = Entity("dslim/bert-base-NER")
- entities = entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg")
-
+ entities = self.entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg")
self.assertEqual([e[0] for e in entities], ["Canada", "Manhattan"])
+
+ def testEntityFlatten(self):
+ """
+ Test entity with flattened output
+ """
+
+ # Run entity extraction
+ entities = self.entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg", flatten=True)
+ self.assertEqual(entities, ["Canada", "Manhattan"])