The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: add include_labels option to ConfluenceLoader (#28259)
Naka Masatoce3b69aa05f2
diff --git a/libs/community/langchain_community/document_loaders/confluence.py b/libs/community/langchain_community/document_loaders/confluence.pyindex 70c86e7dce..263c0c8d31 100644--- a/libs/community/langchain_community/document_loaders/confluence.py+++ b/libs/community/langchain_community/document_loaders/confluence.py@@ -166,6 +166,7 @@ class ConfluenceLoader(BaseLoader):include_archived_content: bool = False,include_attachments: bool = False,include_comments: bool = False,+ include_labels: bool = False,content_format: ContentFormat = ContentFormat.STORAGE,limit: Optional[int] = 50,max_pages: Optional[int] = 1000,@@ -181,6 +182,7 @@ class ConfluenceLoader(BaseLoader):self.include_archived_content = include_archived_contentself.include_attachments = include_attachmentsself.include_comments = include_comments+ self.include_labels = include_labelsself.content_format = content_formatself.limit = limitself.max_pages = max_pages@@ -327,12 +329,20 @@ class ConfluenceLoader(BaseLoader):)include_attachments = self._resolve_param("include_attachments", kwargs)include_comments = self._resolve_param("include_comments", kwargs)+ include_labels = self._resolve_param("include_labels", kwargs)content_format = self._resolve_param("content_format", kwargs)limit = self._resolve_param("limit", kwargs)max_pages = self._resolve_param("max_pages", kwargs)ocr_languages = self._resolve_param("ocr_languages", kwargs)keep_markdown_format = self._resolve_param("keep_markdown_format", kwargs)keep_newlines = self._resolve_param("keep_newlines", kwargs)+ expand = ",".join(+ [+ content_format.value,+ "version",+ *(["metadata.labels"] if include_labels else []),+ ]+ )if not space_key and not page_ids and not label and not cql:raise ValueError(@@ -347,13 +357,14 @@ class ConfluenceLoader(BaseLoader):limit=limit,max_pages=max_pages,status="any" if include_archived_content else "current",- expand=f"{content_format.value},version",+ expand=expand,)yield from self.process_pages(pages,include_restricted_content,include_attachments,include_comments,+ include_labels,content_format,ocr_languages=ocr_languages,keep_markdown_format=keep_markdown_format,@@ -380,13 +391,14 @@ class ConfluenceLoader(BaseLoader):limit=limit,max_pages=max_pages,include_archived_spaces=include_archived_content,- expand=f"{content_format.value},version",+ expand=expand,)yield from self.process_pages(pages,include_restricted_content,include_attachments,include_comments,+ False, # labels are not included in the search resultscontent_format,ocr_languages,keep_markdown_format,@@ -408,7 +420,8 @@ class ConfluenceLoader(BaseLoader):before_sleep=before_sleep_log(logger, logging.WARNING),)(self.confluence.get_page_by_id)page = get_page(- page_id=page_id, expand=f"{content_format.value},version"+ page_id=page_id,+ expand=expand,)if not include_restricted_content and not self.is_public_page(page):continue@@ -416,6 +429,7 @@ class ConfluenceLoader(BaseLoader):page,include_attachments,include_comments,+ include_labels,content_format,ocr_languages,keep_markdown_format,@@ -498,6 +512,7 @@ class ConfluenceLoader(BaseLoader):include_restricted_content: bool,include_attachments: bool,include_comments: bool,+ include_labels: bool,content_format: ContentFormat,ocr_languages: Optional[str] = None,keep_markdown_format: Optional[bool] = False,@@ -511,6 +526,7 @@ class ConfluenceLoader(BaseLoader):page,include_attachments,include_comments,+ include_labels,content_format,ocr_languages=ocr_languages,keep_markdown_format=keep_markdown_format,@@ -522,6 +538,7 @@ class ConfluenceLoader(BaseLoader):page: dict,include_attachments: bool,include_comments: bool,+ include_labels: bool,content_format: ContentFormat,ocr_languages: Optional[str] = None,keep_markdown_format: Optional[bool] = False,@@ -575,10 +592,19 @@ class ConfluenceLoader(BaseLoader):]text = text + "".join(comment_texts)+ if include_labels:+ labels = [+ label["name"]+ for label in page.get("metadata", {})+ .get("labels", {})+ .get("results", [])+ ]+metadata = {"title": page["title"],"id": page["id"],"source": self.base_url.strip("/") + page["_links"]["webui"],+ **({"labels": labels} if include_labels else {}),}if "version" in page and "when" in page["version"]:diff --git a/libs/community/tests/unit_tests/document_loaders/test_confluence.py b/libs/community/tests/unit_tests/document_loaders/test_confluence.pyindex feecb1588b..abb47326be 100644--- a/libs/community/tests/unit_tests/document_loaders/test_confluence.py+++ b/libs/community/tests/unit_tests/document_loaders/test_confluence.py@@ -195,6 +195,36 @@ class TestConfluenceLoader:assert mock_confluence.cql.call_count == 0assert mock_confluence.get_page_child_by_type.call_count == 0+ @pytest.mark.requires("markdownify")+ def test_confluence_loader_when_include_lables_set_to_true(+ self, mock_confluence: MagicMock+ ) -> None:+ # one response with two pages+ mock_confluence.get_all_pages_from_space.return_value = [+ self._get_mock_page("123", include_labels=True),+ self._get_mock_page("456", include_labels=False),+ ]+ mock_confluence.get_all_restrictions_for_content.side_effect = [+ self._get_mock_page_restrictions("123"),+ self._get_mock_page_restrictions("456"),+ ]++ conflence_loader = self._get_mock_confluence_loader(+ mock_confluence,+ space_key=self.MOCK_SPACE_KEY,+ include_labels=True,+ max_pages=2,+ )++ documents = conflence_loader.load()++ assert mock_confluence.get_all_pages_from_space.call_count == 1++ assert len(documents) == 2+ assert all(isinstance(doc, Document) for doc in documents)+ assert documents[0].metadata["labels"] == ["l1", "l2"]+ assert documents[1].metadata["labels"] == []+def _get_mock_confluence_loader(self, mock_confluence: MagicMock, **kwargs: Any) -> ConfluenceLoader:@@ -208,7 +238,10 @@ class TestConfluenceLoader:return confluence_loaderdef _get_mock_page(- self, page_id: str, content_format: ContentFormat = ContentFormat.STORAGE+ self,+ page_id: str,+ content_format: ContentFormat = ContentFormat.STORAGE,+ include_labels: bool = False,) -> Dict:return {"id": f"{page_id}",@@ -216,6 +249,20 @@ class TestConfluenceLoader:"body": {f"{content_format.name.lower()}": {"value": f"<p>Content {page_id}</p>"}},+ **(+ {+ "metadata": {+ "labels": {+ "results": [+ {"prefix": "global", "name": "l1", "id": "111"},+ {"prefix": "global", "name": "l2", "id": "222"},+ ]+ }+ }+ if include_labels+ else {},+ }+ ),"status": "current","type": "page","_links": {