Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(rag): surface Notion database HTTP errors (#44893)
Winoren7cfaaa7da793
diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.pyindex 3ba8b741b2b..82273fe1142 100644--- a/api/core/rag/extractor/notion_extractor.py+++ b/api/core/rag/extractor/notion_extractor.py@@ -83,7 +83,7 @@ class NotionExtractor(BaseExtractor):def _get_notion_database_data(self, database_id: str, query_dict: Mapping[str, Any] = MappingProxyType({})) -> list[Document]:- """Get all the pages from a Notion database."""+ """Get all database pages, failing on HTTP errors instead of returning partial content."""assert self._notion_access_token is not None, "Notion access token is required"database_content = []@@ -106,6 +106,7 @@ class NotionExtractor(BaseExtractor):timeout=_REQUEST_TIMEOUT,)+ res.raise_for_status()response_data = res.json()if "results" not in response_data or response_data["results"] is None:diff --git a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.pyindex 1365e0be1ae..8ca82216deb 100644--- a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py+++ b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py@@ -31,6 +31,15 @@ from models.enums import DataSourceType, DocumentCreatedFromfrom tests.helpers.unit.core.datasource.factories import datasource_entity, provider_entity+def _notion_database_response(data: Mapping[str, object]) -> httpx.Response:+ """Associate database fixtures with the request required for HTTP status checks."""+ return httpx.Response(+ 200,+ json=data,+ request=httpx.Request("POST", notion_extractor_module.DATABASE_URL_TMPL.format(database_id="database-789")),+ )++class TestOnlineDocumentProviderCases:class TestOnlineDocumentDatasourcePluginProviderController:def test_init(self) -> None:@@ -562,7 +571,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -590,7 +599,7 @@ class TestNotionProviderCases:"has_more": True,"next_cursor": "cursor-xyz",}- first_response = httpx.Response(200, json=first_response_data)+ first_response = _notion_database_response(first_response_data)second_response_data = {"object": "list","results": [@@ -601,7 +610,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- second_response = httpx.Response(200, json=second_response_data)+ second_response = _notion_database_response(second_response_data)mock_post.side_effect = [first_response, second_response]# Act@@ -635,7 +644,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -665,7 +674,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -687,7 +696,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -701,7 +710,7 @@ class TestNotionProviderCases:"""Test handling of malformed API response."""# Arrangemock_response_data = {"object": "list"}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -1104,7 +1113,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- database_response = httpx.Response(200, json=database_response_data)+ database_response = _notion_database_response(database_response_data)mock_post.return_value = database_response# Act@@ -1553,7 +1562,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -1590,7 +1599,7 @@ class TestNotionProviderCases:"has_more": True,"next_cursor": "cursor-1",}- page1_response = httpx.Response(200, json=page1_response_data)+ page1_response = _notion_database_response(page1_response_data)page2_response_data = {"object": "list",@@ -1603,7 +1612,7 @@ class TestNotionProviderCases:"has_more": True,"next_cursor": "cursor-2",}- page2_response = httpx.Response(200, json=page2_response_data)+ page2_response = _notion_database_response(page2_response_data)page3_response_data = {"object": "list",@@ -1616,7 +1625,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- page3_response = httpx.Response(200, json=page3_response_data)+ page3_response = _notion_database_response(page3_response_data)mock_post.side_effect = [page1_response, page2_response, page3_response]@@ -1658,7 +1667,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Act@@ -1801,7 +1810,7 @@ class TestNotionProviderCases:"has_more": False,"next_cursor": None,}- mock_response = httpx.Response(200, json=mock_response_data)+ mock_response = _notion_database_response(mock_response_data)mock_post.return_value = mock_response# Create a custom query filterdiff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.pyindex a88afe56e42..0b43494b2a0 100644--- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py@@ -839,3 +839,99 @@ class TestNotionPageRichTextCases:content = extractor._read_block("child-block-id", num_tabs=1)assert content == "\tSee \n\tProject Alpha\n\tE = mc^2\n\n"+++def _database_extractor() -> notion_extractor.NotionExtractor:+ return notion_extractor.NotionExtractor(+ notion_workspace_id="workspace",+ notion_obj_id="database",+ notion_page_type="database",+ tenant_id="tenant",+ notion_token_loader=lambda: "synthetic-token",+ )+++def _database_page(title: str, next_cursor: str | None = None) -> dict[str, object]:+ return {+ "results": [{"properties": {"Title": {"type": "title", "title": [{"plain_text": title}]}}}],+ "has_more": next_cursor is not None,+ "next_cursor": next_cursor,+ }++[email protected]("status", [401, 403, 404, 429, 500, 503])[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])+def test_database_http_errors_do_not_return_empty_or_partial_documents(+ monkeypatch: pytest.MonkeyPatch, status: int, after_first_page: bool+) -> None:+ cursors: list[str | None] = []++ def respond(request: httpx.Request) -> httpx.Response:+ assert request.method == "POST"+ assert request.url.path == "/v1/databases/database/query"+ assert request.headers["Authorization"] == "Bearer synthetic-token"+ assert request.headers["Notion-Version"] == "2022-06-28"+ cursors.append(json.loads(request.content).get("start_cursor"))+ if after_first_page and len(cursors) == 1:+ return httpx.Response(200, json=_database_page("first page", "next-page"))+ return httpx.Response(+ status,+ json={"object": "error", "status": status, "code": "synthetic_error", "message": "Synthetic failure"},+ )++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)+ with pytest.raises(httpx.HTTPStatusError) as caught:+ _database_extractor().extract()++ assert caught.value.response.status_code == status+ assert cursors == ([None, "next-page"] if after_first_page else [None])++[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])+def test_database_http_errors_are_checked_before_decoding_json(+ monkeypatch: pytest.MonkeyPatch, after_first_page: bool+) -> None:+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ if after_first_page and len(requests) == 1:+ return httpx.Response(200, json=_database_page("first page", "next-page"))+ return httpx.Response(503, text="<html>Service unavailable</html>")++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)+ with pytest.raises(httpx.HTTPStatusError) as caught:+ _database_extractor().extract()++ assert caught.value.response.status_code == 503+ assert len(requests) == (2 if after_first_page else 1)+++ "pages", [[], ["first page"], ["first page", "second page"]], ids=["empty", "single", "multiple"]+)+def test_database_success_preserves_empty_results_and_complete_pagination(+ monkeypatch: pytest.MonkeyPatch, pages: list[str]+) -> None:+ cursors: list[str | None] = []++ def respond(request: httpx.Request) -> httpx.Response:+ cursor = json.loads(request.content).get("start_cursor")+ cursors.append(cursor)+ if not pages:+ return httpx.Response(200, json={"results": [], "has_more": False, "next_cursor": None})+ index = 0 if cursor is None else 1+ assert index < len(pages)+ next_cursor = "next-page" if index == 0 and len(pages) > 1 else None+ return httpx.Response(200, json=_database_page(pages[index], next_cursor))++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "post", client.post)+ documents = _database_extractor().extract()++ assert [document.page_content for document in documents] == (+ ["\n".join(f"Title:{title}\n" for title in pages)] if pages else []+ )+ assert cursors == ([None, "next-page"] if len(pages) > 1 else [None])