Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(notion-extractor): guard against empty table row results (#37836)
citizen204cd5c89a8af49
diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.pyindex 82273fe1142..bbfe0dbaa9d 100644--- a/api/core/rag/extractor/notion_extractor.py+++ b/api/core/rag/extractor/notion_extractor.py@@ -6,6 +6,7 @@ from types import MappingProxyTypefrom typing import Any, cast, overrideimport httpx+from pydantic import BaseModel, ConfigDictfrom sqlalchemy import updatefrom core.rag.extractor.entity.extract_setting import StoredDocumentExtractionInput@@ -35,6 +36,16 @@ HEADING_SPLITTER = {}+class _NotionTablePage(BaseModel):+ """A successful table page has rows and an explicit pagination cursor."""++ model_config = ConfigDict(strict=True, hide_input_in_errors=True)++ results: list[dict[str, Any]]+ next_cursor: str | None+ """The next page's cursor, or ``None`` for the final page."""++class NotionExtractor(BaseExtractor):def __init__(self,@@ -281,13 +292,18 @@ class NotionExtractor(BaseExtractor):return "".join(segment.get("plain_text") or segment.get("text", {}).get("content", "") for segment in cell)def _read_table_rows(self, block_id: str) -> str:- """Read table rows."""+ """Fetch all rows for a table block using the configured access token.++ Return Markdown only after every page succeeds, skipping empty pages+ while following their cursors. HTTP, network, JSON, and response+ validation errors propagate instead of returning empty or partial data.+ """assert self._notion_access_token is not None, "Notion access token is required"- done = False+ has_more = Trueresult_lines_arr = []start_cursor = Noneblock_url = BLOCK_CHILD_URL_TMPL.format(block_id=block_id)- while not done:+ while has_more:query_dict: Mapping[str, Any] = MappingProxyType({}) if not start_cursor else {"start_cursor": start_cursor}res = httpx.request(@@ -301,31 +317,30 @@ class NotionExtractor(BaseExtractor):params=query_dict,timeout=_REQUEST_TIMEOUT,)- data = res.json()- # get table headers text- table_header_cell_texts = []- table_header_cells = data["results"][0]["table_row"]["cells"]- for table_header_cell in table_header_cells:- table_header_cell_texts.append(self._get_cell_text(table_header_cell))- # Initialize Markdown table with headers- markdown_table = "| " + " | ".join(table_header_cell_texts) + " |\n"- markdown_table += "| " + " | ".join(["---"] * len(table_header_cell_texts)) + " |\n"-- # Process data to format each row in Markdown table format- results = data["results"]- for i in range(len(results) - 1):- column_texts = []- table_column_cells = data["results"][i + 1]["table_row"]["cells"]- for table_column_cell in table_column_cells:- column_texts.append(self._get_cell_text(table_column_cell))- # Add row to Markdown table- markdown_table += "| " + " | ".join(column_texts) + " |\n"- result_lines_arr.append(markdown_table)- if data["next_cursor"] is None:- done = True- break- else:- start_cursor = data["next_cursor"]+ res.raise_for_status()+ data = _NotionTablePage.model_validate(res.json())+ results = data.results+ if results:+ # get table headers text+ table_header_cell_texts = []+ table_header_cells = results[0]["table_row"]["cells"]+ for table_header_cell in table_header_cells:+ table_header_cell_texts.append(self._get_cell_text(table_header_cell))+ # Initialize Markdown table with headers+ markdown_table = "| " + " | ".join(table_header_cell_texts) + " |\n"+ markdown_table += "| " + " | ".join(["---"] * len(table_header_cell_texts)) + " |\n"++ # Process data to format each row in Markdown table format+ for result in results[1:]:+ column_texts = []+ table_column_cells = result["table_row"]["cells"]+ for table_column_cell in table_column_cells:+ column_texts.append(self._get_cell_text(table_column_cell))+ # Add row to Markdown table+ markdown_table += "| " + " | ".join(column_texts) + " |\n"+ result_lines_arr.append(markdown_table)+ start_cursor = data.next_cursor+ has_more = start_cursor is not Noneresult_lines = "\n".join(result_lines_arr)return result_linesdiff --git a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.pyindex 8ca82216deb..4b13fb84d11 100644--- a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py+++ b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py@@ -40,6 +40,15 @@ def _notion_database_response(data: Mapping[str, object]) -> httpx.Response:)+def _notion_table_response(data: Mapping[str, object]) -> httpx.Response:+ """Attach table fixtures to their GET request so HTTP status checks run normally."""+ return httpx.Response(+ 200,+ json=data,+ request=httpx.Request("GET", notion_extractor_module.BLOCK_CHILD_URL_TMPL.format(block_id="table-block-123")),+ )++class TestOnlineDocumentProviderCases:class TestOnlineDocumentDatasourcePluginProviderController:def test_init(self) -> None:@@ -782,7 +791,7 @@ class TestNotionProviderCases:"next_cursor": None,"has_more": False,}- mock_request.return_value = httpx.Response(200, json=mock_data)+ mock_request.return_value = _notion_table_response(mock_data)# Actresult = extractor._read_table_rows("table-block-123")@@ -814,7 +823,7 @@ class TestNotionProviderCases:"next_cursor": None,"has_more": False,}- mock_request.return_value = httpx.Response(200, json=mock_data)+ mock_request.return_value = _notion_table_response(mock_data)# Actresult = extractor._read_table_rows("table-block-123")@@ -853,7 +862,7 @@ class TestNotionProviderCases:"next_cursor": None,"has_more": False,}- mock_request.side_effect = [httpx.Response(200, json=first_page), httpx.Response(200, json=second_page)]+ mock_request.side_effect = [_notion_table_response(first_page), _notion_table_response(second_page)]# Actresult = extractor._read_table_rows("table-block-123")@@ -1881,7 +1890,7 @@ class TestNotionProviderCases:"next_cursor": None,"has_more": False,}- mock_request.return_value = httpx.Response(200, json=mock_data)+ mock_request.return_value = _notion_table_response(mock_data)# Actresult = extractor._read_table_rows("table-block-123")diff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.pyindex 0b43494b2a0..792fb2c07a7 100644--- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py@@ -6,6 +6,7 @@ from unittest import mockimport httpximport pytest+from pydantic import ValidationErrorfrom pytest_mock import MockerFixturefrom sqlalchemy import Enginefrom sqlalchemy.orm import Session, sessionmaker@@ -841,6 +842,240 @@ class TestNotionPageRichTextCases:assert content == "\tSee \n\tProject Alpha\n\tE = mc^2\n\n"+def _table_extractor() -> notion_extractor.NotionExtractor:+ """Return a page extractor with a synthetic access token and no external state."""+ return notion_extractor.NotionExtractor(+ notion_workspace_id="workspace",+ notion_obj_id="page",+ notion_page_type="page",+ tenant_id="tenant",+ notion_token_loader=lambda: "synthetic-token",+ )+++def _table_page(header: str, next_cursor: str | None = None) -> dict[str, object]:+ """Build one valid table page whose cursor controls whether another request is required."""+ return {+ "results": [{"table_row": {"cells": [[{"plain_text": header}]]}}],+ "next_cursor": next_cursor,+ }+++def test_table_rows_empty_response_returns_empty_content(monkeypatch: pytest.MonkeyPatch) -> None:+ """A valid terminal page without rows produces empty content after one request."""+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Record the synthetic table request and return a valid empty terminal page."""+ requests.append(request)+ return httpx.Response(200, json={"results": [], "next_cursor": None})++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ content = _table_extractor()._read_table_rows("table")++ assert content == ""+ assert len(requests) == 1+ assert requests[0].url.params.get("start_cursor") is None+++def test_table_rows_empty_page_advances_to_the_next_cursor(monkeypatch: pytest.MonkeyPatch) -> None:+ """An empty nonterminal page advances its cursor and returns the later page's Markdown."""+ cursors: list[str | None] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Return an empty first page followed by one table page for the supplied cursor."""+ cursor = request.url.params.get("start_cursor")+ cursors.append(cursor)+ if cursor is None:+ return httpx.Response(200, json={"results": [], "next_cursor": "next-page"})+ return httpx.Response(200, json=_table_page("Header"))++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ content = _table_extractor()._read_table_rows("table")++ assert content == "| Header |\n| --- |\n"+ assert cursors == [None, "next-page"]+++def test_table_rows_empty_final_page_preserves_prior_content(monkeypatch: pytest.MonkeyPatch) -> None:+ """A terminal empty page leaves successfully rendered prior pages intact."""+ cursors: list[str | None] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Return one rendered page followed by an empty terminal page."""+ cursor = request.url.params.get("start_cursor")+ cursors.append(cursor)+ if cursor is None:+ return httpx.Response(200, json=_table_page("Header", "next-page"))+ return httpx.Response(200, json={"results": [], "next_cursor": None})++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ content = _table_extractor()._read_table_rows("table")++ assert content == "| Header |\n| --- |\n"+ assert cursors == [None, "next-page"]++[email protected]("status", [401, 403, 429, 500, 503])[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])+def test_table_rows_http_errors_do_not_return_empty_or_partial_content(+ monkeypatch: pytest.MonkeyPatch, status: int, after_first_page: bool+) -> None:+ """Any failed table page raises HTTPStatusError before empty or partial content can return."""+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Optionally return one valid page, then a synthetic Notion HTTP error."""+ requests.append(request)+ if after_first_page and len(requests) == 1:+ return httpx.Response(200, json=_table_page("Header", "next-page"))+ return httpx.Response(+ status,+ json={"object": "error", "status": status, "code": "synthetic_error", "message": "Synthetic failure"},+ )++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(httpx.HTTPStatusError) as caught:+ _table_extractor()._read_table_rows("table")++ assert caught.value.response.status_code == status+ assert len(requests) == (2 if after_first_page else 1)+++def test_table_rows_http_errors_are_checked_before_decoding_json(monkeypatch: pytest.MonkeyPatch) -> None:+ """A failed non-JSON response raises for its HTTP status before body decoding."""++ def respond(_request: httpx.Request) -> httpx.Response:+ """Return a synthetic HTML service failure for any table request."""+ return httpx.Response(503, text="<html>Service unavailable</html>")++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(httpx.HTTPStatusError) as caught:+ _table_extractor()._read_table_rows("table")++ assert caught.value.response.status_code == 503++[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])+def test_table_rows_success_with_invalid_json_does_not_return_partial_content(+ monkeypatch: pytest.MonkeyPatch, after_first_page: bool+) -> None:+ """A 200 response with invalid JSON propagates decoding failure, including after prior pages."""+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Optionally return one valid page, then a successful response with invalid JSON."""+ requests.append(request)+ if after_first_page and len(requests) == 1:+ return httpx.Response(200, json=_table_page("Header", "next-page"))+ return httpx.Response(200, text="not-json")++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(json.JSONDecodeError):+ _table_extractor()._read_table_rows("table")++ assert len(requests) == (2 if after_first_page else 1)+++ "invalid_page",+ [+ {"next_cursor": None},+ {"results": []},+ {"results": None, "next_cursor": None},+ {"results": "not-a-list", "next_cursor": None},+ {"results": ["not-an-object"], "next_cursor": None},+ [],+ {"results": [], "next_cursor": 42},+ ],+ ids=[+ "missing-results",+ "missing-next-cursor",+ "null-results",+ "non-list-results",+ "non-object-result",+ "non-object-page",+ "invalid-cursor",+ ],+)[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])+def test_table_rows_reject_malformed_success_responses(+ monkeypatch: pytest.MonkeyPatch, invalid_page: object, after_first_page: bool+) -> None:+ """Malformed successful pages fail validation before empty or partial content can return."""+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ """Optionally return one valid page, then the supplied malformed success payload."""+ requests.append(request)+ if after_first_page and len(requests) == 1:+ return httpx.Response(200, json=_table_page("Header", "next-page"))+ return httpx.Response(200, json=invalid_page)++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(ValidationError):+ _table_extractor()._read_table_rows("table")++ assert len(requests) == (2 if after_first_page else 1)+++def test_table_rows_network_errors_propagate(monkeypatch: pytest.MonkeyPatch) -> None:+ """A table transport failure propagates without producing content."""++ def respond(request: httpx.Request) -> httpx.Response:+ """Raise a synthetic connection failure for the accepted table request."""+ raise httpx.ConnectError("Synthetic connection failure", request=request)++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(httpx.ConnectError, match="Synthetic connection failure"):+ _table_extractor()._read_table_rows("table")+++def test_page_import_does_not_return_a_partial_document_when_table_loading_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Page extraction fails when one table request fails, even after other blocks were read."""++ def respond(request: httpx.Request) -> httpx.Response:+ """Return a mixed page root, then rate-limit its nested table request."""+ if request.url.path.endswith("/page/children"):+ return httpx.Response(+ 200,+ json={+ "results": [+ {+ "type": "paragraph",+ "id": "paragraph",+ "has_children": False,+ "paragraph": {"rich_text": [{"plain_text": "Retained paragraph"}]},+ },+ {"type": "table", "id": "table", "has_children": True, "table": {}},+ ],+ "next_cursor": None,+ },+ )+ assert request.url.path.endswith("/table/children")+ return httpx.Response(+ 429,+ json={"object": "error", "status": 429, "code": "rate_limited", "message": "Synthetic failure"},+ )++ with httpx.Client(transport=httpx.MockTransport(respond)) as client:+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)+ with pytest.raises(httpx.HTTPStatusError) as caught:+ _table_extractor()._load_data_as_documents("page", "page")++ assert caught.value.response.status_code == 429++def _database_extractor() -> notion_extractor.NotionExtractor:return notion_extractor.NotionExtractor(notion_workspace_id="workspace",