💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add entity extraction pipeline, closes #203
davidmezzettie9f34cd8506f
diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.pyindex 3a8830d1..5e1a3fab 100644--- a/src/python/txtai/api/routers/__init__.py+++ b/src/python/txtai/api/routers/__init__.py@@ -4,6 +4,7 @@ Router importsfrom . import captionfrom . import embeddings+from . import entityfrom . import extractorfrom . import labelsfrom . import objectsdiff --git a/src/python/txtai/api/routers/entity.py b/src/python/txtai/api/routers/entity.pynew file mode 100644index 00000000..88da19cf--- /dev/null+++ b/src/python/txtai/api/routers/entity.py@@ -0,0 +1,41 @@+"""+Defines API paths for entity endpoints.+"""++from typing import List++from fastapi import APIRouter, Body++from .. import application++router = APIRouter()++[email protected]("/entity")+def entity(text: str):+ """+ Applies a token classifier to text.++ Args:+ text: input text++ Returns:+ list of (entity, entity type, score) per text element+ """++ return application.get().pipeline("entity", (text,))++[email protected]("/batchentity")+def batchentity(texts: List[str] = Body(...)):+ """+ Applies a token classifier to text.++ Args:+ texts: list of text++ Returns:+ list of (entity, entity type, score) per text element+ """++ return application.get().pipeline("entity", (texts,))diff --git a/src/python/txtai/pipeline/text/__init__.py b/src/python/txtai/pipeline/text/__init__.pyindex e0c96e07..19603640 100644--- a/src/python/txtai/pipeline/text/__init__.py+++ b/src/python/txtai/pipeline/text/__init__.py@@ -2,6 +2,7 @@Text imports"""+from .entity import Entityfrom .extractor import Extractorfrom .labels import Labelsfrom .questions import Questionsdiff --git a/src/python/txtai/pipeline/text/entity.py b/src/python/txtai/pipeline/text/entity.pynew file mode 100644index 00000000..0ae7f70a--- /dev/null+++ b/src/python/txtai/pipeline/text/entity.py@@ -0,0 +1,41 @@+"""+Entity module+"""++from ..hfpipeline import HFPipeline+++class Entity(HFPipeline):+ """+ Applies a token classifier to text. Extracts known entity/label combinations.+ """++ def __init__(self, path=None, quantize=False, gpu=True, model=None):+ super().__init__("token-classification", path, quantize, gpu, model)++ def __call__(self, text, aggregate="simple", workers=0):+ """+ Applies a token classifier to text.++ Args:+ text: text|list+ aggregate: method to combine multi token entities - options are "simple" (default), "first", "average" or "max"+ workers: number of concurrent workers to use for processing data, defaults to None++ Returns:+ list of (entity, entity type, score)+ """++ # Run token classification pipeline+ results = self.pipeline(text, aggregation_strategy=aggregate, num_workers=workers)++ # Convert results to a list if necessary+ if isinstance(text, str):+ results = [results]++ # Extract (entity, entity type, score) tuples+ outputs = []+ for result in results:+ outputs.append([(r["word"], r["entity_group"], float(r["score"])) for r in result])++ return outputs[0] if isinstance(text, str) else outputsdiff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.pyindex 01bfc6fe..91d58175 100644--- a/test/python/testapi/testpipelines.py+++ b/test/python/testapi/testpipelines.py@@ -21,6 +21,10 @@ PIPELINES = """# Image captionscaption:+# Entity extraction+entity:+ path: dslim/bert-base-NER+# Label settingslabels:path: prajjwal1/bert-medium-mnli@@ -121,6 +125,22 @@ class TestPipelines(unittest.TestCase):captions = self.client.post("batchcaption", json=[path, path]).json()self.assertEqual(captions, ["a book shelf filled with many books"] * 2)+ def testEntity(self):+ """+ Test entity extraction via API+ """++ entities = self.client.get(f"entity?text={self.data[1]}").json()+ self.assertEqual([e[0] for e in entities], ["Canada", "Manhattan"])++ def testEntityBatch(self):+ """+ Test batch entity via API+ """++ entities = self.client.post("batchentity", json=[self.data[1]]).json()+ self.assertEqual([e[0] for e in entities[0]], ["Canada", "Manhattan"])+def testEmpty(self):"""Test empty API configurationdiff --git a/test/python/testpipeline/testentity.py b/test/python/testpipeline/testentity.pynew file mode 100644index 00000000..5e34a114--- /dev/null+++ b/test/python/testpipeline/testentity.py@@ -0,0 +1,24 @@+"""+Entity module tests+"""++import unittest++from txtai.pipeline import Entity+++class TestEntity(unittest.TestCase):+ """+ Entity tests.+ """++ def testEntity(self):+ """+ Test entity+ """++ # Run entity extraction+ entity = Entity("dslim/bert-base-NER")+ entities = entity("Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg")++ self.assertEqual([e[0] for e in entities], ["Canada", "Manhattan"])