Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): surface Notion database HTTP errors (#44893)

Winoren7cfaaa7da793
diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py
index 3ba8b741b2b..82273fe1142 100644
--- a/api/core/rag/extractor/notion_extractor.py
+++ b/api/core/rag/extractor/notion_extractor.py
@@ -83,7 +83,7 @@ class NotionExtractor(BaseExtractor):
def _get_notion_database_data(
self, database_id: str, query_dict: Mapping[str, Any] = MappingProxyType({})
) -> list[Document]:
- """Get all the pages from a Notion database."""
+ """Get all database pages, failing on HTTP errors instead of returning partial content."""
assert self._notion_access_token is not None, "Notion access token is required"
database_content = []
@@ -106,6 +106,7 @@ class NotionExtractor(BaseExtractor):
timeout=_REQUEST_TIMEOUT,
)
+ res.raise_for_status()
response_data = res.json()
if "results" not in response_data or response_data["results"] is None:
diff --git a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
index 1365e0be1ae..8ca82216deb 100644
--- a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
+++ b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
@@ -31,6 +31,15 @@ from models.enums import DataSourceType, DocumentCreatedFrom
from tests.helpers.unit.core.datasource.factories import datasource_entity, provider_entity
+def _notion_database_response(data: Mapping[str, object]) -> httpx.Response:
+ """Associate database fixtures with the request required for HTTP status checks."""
+ return httpx.Response(
+ 200,
+ json=data,
+ request=httpx.Request("POST", notion_extractor_module.DATABASE_URL_TMPL.format(database_id="database-789")),
+ )
+
+
class TestOnlineDocumentProviderCases:
class TestOnlineDocumentDatasourcePluginProviderController:
def test_init(self) -> None:
@@ -562,7 +571,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -590,7 +599,7 @@ class TestNotionProviderCases:
"has_more": True,
"next_cursor": "cursor-xyz",
}
- first_response = httpx.Response(200, json=first_response_data)
+ first_response = _notion_database_response(first_response_data)
second_response_data = {
"object": "list",
"results": [
@@ -601,7 +610,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- second_response = httpx.Response(200, json=second_response_data)
+ second_response = _notion_database_response(second_response_data)
mock_post.side_effect = [first_response, second_response]
# Act
@@ -635,7 +644,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -665,7 +674,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -687,7 +696,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -701,7 +710,7 @@ class TestNotionProviderCases:
"""Test handling of malformed API response."""
# Arrange
mock_response_data = {"object": "list"}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -1104,7 +1113,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- database_response = httpx.Response(200, json=database_response_data)
+ database_response = _notion_database_response(database_response_data)
mock_post.return_value = database_response
# Act
@@ -1553,7 +1562,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -1590,7 +1599,7 @@ class TestNotionProviderCases:
"has_more": True,
"next_cursor": "cursor-1",
}
- page1_response = httpx.Response(200, json=page1_response_data)
+ page1_response = _notion_database_response(page1_response_data)
page2_response_data = {
"object": "list",
@@ -1603,7 +1612,7 @@ class TestNotionProviderCases:
"has_more": True,
"next_cursor": "cursor-2",
}
- page2_response = httpx.Response(200, json=page2_response_data)
+ page2_response = _notion_database_response(page2_response_data)
page3_response_data = {
"object": "list",
@@ -1616,7 +1625,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- page3_response = httpx.Response(200, json=page3_response_data)
+ page3_response = _notion_database_response(page3_response_data)
mock_post.side_effect = [page1_response, page2_response, page3_response]
@@ -1658,7 +1667,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Act
@@ -1801,7 +1810,7 @@ class TestNotionProviderCases:
"has_more": False,
"next_cursor": None,
}
- mock_response = httpx.Response(200, json=mock_response_data)
+ mock_response = _notion_database_response(mock_response_data)
mock_post.return_value = mock_response
# Create a custom query filter
diff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
index a88afe56e42..0b43494b2a0 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
@@ -839,3 +839,99 @@ class TestNotionPageRichTextCases:
content = extractor._read_block("child-block-id", num_tabs=1)
assert content == "\tSee \n\tProject Alpha\n\tE = mc^2\n\n"
+
+
+def _database_extractor() -> notion_extractor.NotionExtractor:
+ return notion_extractor.NotionExtractor(
+ notion_workspace_id="workspace",
+ notion_obj_id="database",
+ notion_page_type="database",
+ tenant_id="tenant",
+ notion_token_loader=lambda: "synthetic-token",
+ )
+
+
+def _database_page(title: str, next_cursor: str | None = None) -> dict[str, object]:
+ return {
+ "results": [{"properties": {"Title": {"type": "title", "title": [{"plain_text": title}]}}}],
+ "has_more": next_cursor is not None,
+ "next_cursor": next_cursor,
+ }
+
+
[email protected]("status", [401, 403, 404, 429, 500, 503])
[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])
+def test_database_http_errors_do_not_return_empty_or_partial_documents(
+ monkeypatch: pytest.MonkeyPatch, status: int, after_first_page: bool
+) -> None:
+ cursors: list[str | None] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ assert request.method == "POST"
+ assert request.url.path == "/v1/databases/database/query"
+ assert request.headers["Authorization"] == "Bearer synthetic-token"
+ assert request.headers["Notion-Version"] == "2022-06-28"
+ cursors.append(json.loads(request.content).get("start_cursor"))
+ if after_first_page and len(cursors) == 1:
+ return httpx.Response(200, json=_database_page("first page", "next-page"))
+ return httpx.Response(
+ status,
+ json={"object": "error", "status": status, "code": "synthetic_error", "message": "Synthetic failure"},
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)
+ with pytest.raises(httpx.HTTPStatusError) as caught:
+ _database_extractor().extract()
+
+ assert caught.value.response.status_code == status
+ assert cursors == ([None, "next-page"] if after_first_page else [None])
+
+
[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])
+def test_database_http_errors_are_checked_before_decoding_json(
+ monkeypatch: pytest.MonkeyPatch, after_first_page: bool
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ if after_first_page and len(requests) == 1:
+ return httpx.Response(200, json=_database_page("first page", "next-page"))
+ return httpx.Response(503, text="<html>Service unavailable</html>")
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)
+ with pytest.raises(httpx.HTTPStatusError) as caught:
+ _database_extractor().extract()
+
+ assert caught.value.response.status_code == 503
+ assert len(requests) == (2 if after_first_page else 1)
+
+
+ "pages", [[], ["first page"], ["first page", "second page"]], ids=["empty", "single", "multiple"]
+)
+def test_database_success_preserves_empty_results_and_complete_pagination(
+ monkeypatch: pytest.MonkeyPatch, pages: list[str]
+) -> None:
+ cursors: list[str | None] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ cursor = json.loads(request.content).get("start_cursor")
+ cursors.append(cursor)
+ if not pages:
+ return httpx.Response(200, json={"results": [], "has_more": False, "next_cursor": None})
+ index = 0 if cursor is None else 1
+ assert index < len(pages)
+ next_cursor = "next-page" if index == 0 and len(pages) > 1 else None
+ return httpx.Response(200, json=_database_page(pages[index], next_cursor))
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)
+ documents = _database_extractor().extract()
+
+ assert [document.page_content for document in documents] == (
+ ["\n".join(f"Title:{title}\n" for title in pages)] if pages else []
+ )
+ assert cursors == ([None, "next-page"] if len(pages) > 1 else [None])