The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community[minor]: GLiNER link extraction (#24314)
Ben Chambers83f3d95ffaaf
diff --git a/libs/community/extended_testing_deps.txt b/libs/community/extended_testing_deps.txtindex eca081b0a5..eb3ad5b0d0 100644--- a/libs/community/extended_testing_deps.txt+++ b/libs/community/extended_testing_deps.txt@@ -25,6 +25,7 @@ fireworks-ai>=0.9.0,<0.10friendli-client>=1.2.4,<2geopandas>=0.13.1gitpython>=3.1.32,<4+gliner>=0.2.7google-cloud-documentai>=2.20.1,<3gql>=3.4.1,<4gradientai>=1.4.0,<2diff --git a/libs/community/langchain_community/graph_vectorstores/extractors/__init__.py b/libs/community/langchain_community/graph_vectorstores/extractors/__init__.pyindex 4644869929..4f7d4036f9 100644--- a/libs/community/langchain_community/graph_vectorstores/extractors/__init__.py+++ b/libs/community/langchain_community/graph_vectorstores/extractors/__init__.py@@ -1,3 +1,7 @@+from langchain_community.graph_vectorstores.extractors.gliner_link_extractor import (+ GLiNERInput,+ GLiNERLinkExtractor,+)from langchain_community.graph_vectorstores.extractors.hierarchy_link_extractor import (HierarchyInput,HierarchyLinkExtractor,@@ -14,10 +18,12 @@ from langchain_community.graph_vectorstores.extractors.link_extractor_adapter im)__all__ = [- "LinkExtractor",- "LinkExtractorAdapter",+ "GLiNERInput",+ "GLiNERLinkExtractor","HierarchyInput","HierarchyLinkExtractor","HtmlInput","HtmlLinkExtractor",+ "LinkExtractor",+ "LinkExtractorAdapter",]diff --git a/libs/community/langchain_community/graph_vectorstores/extractors/gliner_link_extractor.py b/libs/community/langchain_community/graph_vectorstores/extractors/gliner_link_extractor.pynew file mode 100644index 0000000000..f5aa2eb27a--- /dev/null+++ b/libs/community/langchain_community/graph_vectorstores/extractors/gliner_link_extractor.py@@ -0,0 +1,71 @@+from typing import Any, Dict, Iterable, List, Optional, Set, Union++from langchain_core.documents import Document+from langchain_core.graph_vectorstores.links import Link++from langchain_community.graph_vectorstores.extractors.link_extractor import (+ LinkExtractor,+)++# TypeAlias is not available in Python 3.9, we can't use that or the newer `type`.+GLiNERInput = Union[str, Document]+++class GLiNERLinkExtractor(LinkExtractor[GLiNERInput]):+ """Link documents with common named entities using GLiNER <https://github.com/urchade/GLiNER>."""++ def __init__(+ self,+ labels: List[str],+ *,+ kind: str = "entity",+ model: str = "urchade/gliner_mediumv2.1",+ extract_kwargs: Optional[Dict[str, Any]] = None,+ ):+ """Extract keywords using GLiNER.++ Example:++ .. code-block:: python++ extractor = GLiNERLinkExtractor(+ labels=["Person", "Award", "Date", "Competitions", "Teams"]+ )+ results = extractor.extract_one("some long text...")++ Args:+ labels: List of kinds of entities to extract.+ kind: Kind of links to produce with this extractor.+ model: GLiNER model to use.+ extract_kwargs: Keyword arguments to pass to GLiNER.+ """+ try:+ from gliner import GLiNER++ self._model = GLiNER.from_pretrained(model)++ except ImportError:+ raise ImportError(+ "gliner is required for GLiNERLinkExtractor. "+ "Please install it with `pip install gliner`."+ ) from None++ self._labels = labels+ self._kind = kind+ self._extract_kwargs = extract_kwargs or {}++ def extract_one(self, input: GLiNERInput) -> Set[Link]: # noqa: A002+ return next(iter(self.extract_many([input])))++ def extract_many(+ self,+ inputs: Iterable[GLiNERInput],+ ) -> Iterable[Set[Link]]:+ strs = [i if isinstance(i, str) else i.page_content for i in inputs]+ for entities in self._model.batch_predict_entities(+ strs, self._labels, **self._extract_kwargs+ ):+ yield {+ Link.bidir(kind=f"{self._kind}:{e['label']}", tag=e["text"])+ for e in entities+ }diff --git a/libs/community/langchain_community/graph_vectorstores/extractors/hierarchy_link_extractor.py b/libs/community/langchain_community/graph_vectorstores/extractors/hierarchy_link_extractor.pyindex f3449effe4..7a324df261 100644--- a/libs/community/langchain_community/graph_vectorstores/extractors/hierarchy_link_extractor.py+++ b/libs/community/langchain_community/graph_vectorstores/extractors/hierarchy_link_extractor.py@@ -30,7 +30,9 @@ class HierarchyLinkExtractor(LinkExtractor[HierarchyInput]):"""Extract links from a document hierarchy.Example:- .. code_block: python++ .. code-block:: python+# Given three paths (in this case, within the "Root" document):h1 = ["Root", "H1"]h1a = ["Root", "H1", "a"]diff --git a/libs/community/tests/integration_tests/graph_vectorstores/extractors/test_gliner_link_extractor.py b/libs/community/tests/integration_tests/graph_vectorstores/extractors/test_gliner_link_extractor.pynew file mode 100644index 0000000000..b6c09db202--- /dev/null+++ b/libs/community/tests/integration_tests/graph_vectorstores/extractors/test_gliner_link_extractor.py@@ -0,0 +1,43 @@+import pytest+from langchain_core.graph_vectorstores.links import Link++from langchain_community.graph_vectorstores.extractors import GLiNERLinkExtractor++PAGE_1 = """+Cristiano Ronaldo dos Santos Aveiro (Portuguese pronunciation: [kɾiʃ'tjɐnu+ʁɔ'naldu]; born 5 February 1985) is a Portuguese professional footballer who+plays as a forward for and captains both Saudi Pro League club Al Nassr and the+Portugal national team. Widely regarded as one of the greatest players of all+time, Ronaldo has won five Ballon d'Or awards,[note 3] a record three UEFA Men's+Player of the Year Awards, and four European Golden Shoes, the most by a+European player. He has won 33 trophies in his career, including seven league+titles, five UEFA Champions Leagues, the UEFA European Championship and the UEFA+Nations League. Ronaldo holds the records for most appearances (183), goals+(140) and assists (42) in the Champions League, goals in the European+Championship (14), international goals (128) and international appearances+(205). He is one of the few players to have made over 1,200 professional career+appearances, the most by an outfield player, and has scored over 850 official+senior career goals for club and country, making him the top goalscorer of all+time.+"""++[email protected]("gliner")+def test_one_from_keywords() -> None:+ extractor = GLiNERLinkExtractor(+ labels=["Person", "Award", "Date", "Competitions", "Teams"]+ )++ results = extractor.extract_one(PAGE_1)+ assert results == {+ Link.bidir(kind="entity:Person", tag="Cristiano Ronaldo dos Santos Aveiro"),+ Link.bidir(kind="entity:Award", tag="European Golden Shoes"),+ Link.bidir(kind="entity:Competitions", tag="European\nChampionship"),+ Link.bidir(kind="entity:Award", tag="UEFA Men's\nPlayer of the Year Awards"),+ Link.bidir(kind="entity:Date", tag="5 February 1985"),+ Link.bidir(kind="entity:Competitions", tag="UEFA Champions Leagues"),+ Link.bidir(kind="entity:Teams", tag="Portugal national team"),+ Link.bidir(kind="entity:Competitions", tag="UEFA European Championship"),+ Link.bidir(kind="entity:Competitions", tag="UEFA\nNations League"),+ Link.bidir(kind="entity:Award", tag="Ballon d'Or"),+ }