Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(notion-extractor): guard against empty table row results (#37836)

citizen204cd5c89a8af49
diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py
index 82273fe1142..bbfe0dbaa9d 100644
--- a/api/core/rag/extractor/notion_extractor.py
+++ b/api/core/rag/extractor/notion_extractor.py
@@ -6,6 +6,7 @@ from types import MappingProxyType
from typing import Any, cast, override
import httpx
+from pydantic import BaseModel, ConfigDict
from sqlalchemy import update
from core.rag.extractor.entity.extract_setting import StoredDocumentExtractionInput
@@ -35,6 +36,16 @@ HEADING_SPLITTER = {
}
+class _NotionTablePage(BaseModel):
+ """A successful table page has rows and an explicit pagination cursor."""
+
+ model_config = ConfigDict(strict=True, hide_input_in_errors=True)
+
+ results: list[dict[str, Any]]
+ next_cursor: str | None
+ """The next page's cursor, or ``None`` for the final page."""
+
+
class NotionExtractor(BaseExtractor):
def __init__(
self,
@@ -281,13 +292,18 @@ class NotionExtractor(BaseExtractor):
return "".join(segment.get("plain_text") or segment.get("text", {}).get("content", "") for segment in cell)
def _read_table_rows(self, block_id: str) -> str:
- """Read table rows."""
+ """Fetch all rows for a table block using the configured access token.
+
+ Return Markdown only after every page succeeds, skipping empty pages
+ while following their cursors. HTTP, network, JSON, and response
+ validation errors propagate instead of returning empty or partial data.
+ """
assert self._notion_access_token is not None, "Notion access token is required"
- done = False
+ has_more = True
result_lines_arr = []
start_cursor = None
block_url = BLOCK_CHILD_URL_TMPL.format(block_id=block_id)
- while not done:
+ while has_more:
query_dict: Mapping[str, Any] = MappingProxyType({}) if not start_cursor else {"start_cursor": start_cursor}
res = httpx.request(
@@ -301,31 +317,30 @@ class NotionExtractor(BaseExtractor):
params=query_dict,
timeout=_REQUEST_TIMEOUT,
)
- data = res.json()
- # get table headers text
- table_header_cell_texts = []
- table_header_cells = data["results"][0]["table_row"]["cells"]
- for table_header_cell in table_header_cells:
- table_header_cell_texts.append(self._get_cell_text(table_header_cell))
- # Initialize Markdown table with headers
- markdown_table = "| " + " | ".join(table_header_cell_texts) + " |\n"
- markdown_table += "| " + " | ".join(["---"] * len(table_header_cell_texts)) + " |\n"
-
- # Process data to format each row in Markdown table format
- results = data["results"]
- for i in range(len(results) - 1):
- column_texts = []
- table_column_cells = data["results"][i + 1]["table_row"]["cells"]
- for table_column_cell in table_column_cells:
- column_texts.append(self._get_cell_text(table_column_cell))
- # Add row to Markdown table
- markdown_table += "| " + " | ".join(column_texts) + " |\n"
- result_lines_arr.append(markdown_table)
- if data["next_cursor"] is None:
- done = True
- break
- else:
- start_cursor = data["next_cursor"]
+ res.raise_for_status()
+ data = _NotionTablePage.model_validate(res.json())
+ results = data.results
+ if results:
+ # get table headers text
+ table_header_cell_texts = []
+ table_header_cells = results[0]["table_row"]["cells"]
+ for table_header_cell in table_header_cells:
+ table_header_cell_texts.append(self._get_cell_text(table_header_cell))
+ # Initialize Markdown table with headers
+ markdown_table = "| " + " | ".join(table_header_cell_texts) + " |\n"
+ markdown_table += "| " + " | ".join(["---"] * len(table_header_cell_texts)) + " |\n"
+
+ # Process data to format each row in Markdown table format
+ for result in results[1:]:
+ column_texts = []
+ table_column_cells = result["table_row"]["cells"]
+ for table_column_cell in table_column_cells:
+ column_texts.append(self._get_cell_text(table_column_cell))
+ # Add row to Markdown table
+ markdown_table += "| " + " | ".join(column_texts) + " |\n"
+ result_lines_arr.append(markdown_table)
+ start_cursor = data.next_cursor
+ has_more = start_cursor is not None
result_lines = "\n".join(result_lines_arr)
return result_lines
diff --git a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
index 8ca82216deb..4b13fb84d11 100644
--- a/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
+++ b/api/tests/unit_tests/core/datasource/online_document/test_online_document_provider.py
@@ -40,6 +40,15 @@ def _notion_database_response(data: Mapping[str, object]) -> httpx.Response:
)
+def _notion_table_response(data: Mapping[str, object]) -> httpx.Response:
+ """Attach table fixtures to their GET request so HTTP status checks run normally."""
+ return httpx.Response(
+ 200,
+ json=data,
+ request=httpx.Request("GET", notion_extractor_module.BLOCK_CHILD_URL_TMPL.format(block_id="table-block-123")),
+ )
+
+
class TestOnlineDocumentProviderCases:
class TestOnlineDocumentDatasourcePluginProviderController:
def test_init(self) -> None:
@@ -782,7 +791,7 @@ class TestNotionProviderCases:
"next_cursor": None,
"has_more": False,
}
- mock_request.return_value = httpx.Response(200, json=mock_data)
+ mock_request.return_value = _notion_table_response(mock_data)
# Act
result = extractor._read_table_rows("table-block-123")
@@ -814,7 +823,7 @@ class TestNotionProviderCases:
"next_cursor": None,
"has_more": False,
}
- mock_request.return_value = httpx.Response(200, json=mock_data)
+ mock_request.return_value = _notion_table_response(mock_data)
# Act
result = extractor._read_table_rows("table-block-123")
@@ -853,7 +862,7 @@ class TestNotionProviderCases:
"next_cursor": None,
"has_more": False,
}
- mock_request.side_effect = [httpx.Response(200, json=first_page), httpx.Response(200, json=second_page)]
+ mock_request.side_effect = [_notion_table_response(first_page), _notion_table_response(second_page)]
# Act
result = extractor._read_table_rows("table-block-123")
@@ -1881,7 +1890,7 @@ class TestNotionProviderCases:
"next_cursor": None,
"has_more": False,
}
- mock_request.return_value = httpx.Response(200, json=mock_data)
+ mock_request.return_value = _notion_table_response(mock_data)
# Act
result = extractor._read_table_rows("table-block-123")
diff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
index 0b43494b2a0..792fb2c07a7 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py
@@ -6,6 +6,7 @@ from unittest import mock
import httpx
import pytest
+from pydantic import ValidationError
from pytest_mock import MockerFixture
from sqlalchemy import Engine
from sqlalchemy.orm import Session, sessionmaker
@@ -841,6 +842,240 @@ class TestNotionPageRichTextCases:
assert content == "\tSee \n\tProject Alpha\n\tE = mc^2\n\n"
+def _table_extractor() -> notion_extractor.NotionExtractor:
+ """Return a page extractor with a synthetic access token and no external state."""
+ return notion_extractor.NotionExtractor(
+ notion_workspace_id="workspace",
+ notion_obj_id="page",
+ notion_page_type="page",
+ tenant_id="tenant",
+ notion_token_loader=lambda: "synthetic-token",
+ )
+
+
+def _table_page(header: str, next_cursor: str | None = None) -> dict[str, object]:
+ """Build one valid table page whose cursor controls whether another request is required."""
+ return {
+ "results": [{"table_row": {"cells": [[{"plain_text": header}]]}}],
+ "next_cursor": next_cursor,
+ }
+
+
+def test_table_rows_empty_response_returns_empty_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A valid terminal page without rows produces empty content after one request."""
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Record the synthetic table request and return a valid empty terminal page."""
+ requests.append(request)
+ return httpx.Response(200, json={"results": [], "next_cursor": None})
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ content = _table_extractor()._read_table_rows("table")
+
+ assert content == ""
+ assert len(requests) == 1
+ assert requests[0].url.params.get("start_cursor") is None
+
+
+def test_table_rows_empty_page_advances_to_the_next_cursor(monkeypatch: pytest.MonkeyPatch) -> None:
+ """An empty nonterminal page advances its cursor and returns the later page's Markdown."""
+ cursors: list[str | None] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Return an empty first page followed by one table page for the supplied cursor."""
+ cursor = request.url.params.get("start_cursor")
+ cursors.append(cursor)
+ if cursor is None:
+ return httpx.Response(200, json={"results": [], "next_cursor": "next-page"})
+ return httpx.Response(200, json=_table_page("Header"))
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ content = _table_extractor()._read_table_rows("table")
+
+ assert content == "| Header |\n| --- |\n"
+ assert cursors == [None, "next-page"]
+
+
+def test_table_rows_empty_final_page_preserves_prior_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A terminal empty page leaves successfully rendered prior pages intact."""
+ cursors: list[str | None] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Return one rendered page followed by an empty terminal page."""
+ cursor = request.url.params.get("start_cursor")
+ cursors.append(cursor)
+ if cursor is None:
+ return httpx.Response(200, json=_table_page("Header", "next-page"))
+ return httpx.Response(200, json={"results": [], "next_cursor": None})
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ content = _table_extractor()._read_table_rows("table")
+
+ assert content == "| Header |\n| --- |\n"
+ assert cursors == [None, "next-page"]
+
+
[email protected]("status", [401, 403, 429, 500, 503])
[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])
+def test_table_rows_http_errors_do_not_return_empty_or_partial_content(
+ monkeypatch: pytest.MonkeyPatch, status: int, after_first_page: bool
+) -> None:
+ """Any failed table page raises HTTPStatusError before empty or partial content can return."""
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Optionally return one valid page, then a synthetic Notion HTTP error."""
+ requests.append(request)
+ if after_first_page and len(requests) == 1:
+ return httpx.Response(200, json=_table_page("Header", "next-page"))
+ return httpx.Response(
+ status,
+ json={"object": "error", "status": status, "code": "synthetic_error", "message": "Synthetic failure"},
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(httpx.HTTPStatusError) as caught:
+ _table_extractor()._read_table_rows("table")
+
+ assert caught.value.response.status_code == status
+ assert len(requests) == (2 if after_first_page else 1)
+
+
+def test_table_rows_http_errors_are_checked_before_decoding_json(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A failed non-JSON response raises for its HTTP status before body decoding."""
+
+ def respond(_request: httpx.Request) -> httpx.Response:
+ """Return a synthetic HTML service failure for any table request."""
+ return httpx.Response(503, text="<html>Service unavailable</html>")
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(httpx.HTTPStatusError) as caught:
+ _table_extractor()._read_table_rows("table")
+
+ assert caught.value.response.status_code == 503
+
+
[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])
+def test_table_rows_success_with_invalid_json_does_not_return_partial_content(
+ monkeypatch: pytest.MonkeyPatch, after_first_page: bool
+) -> None:
+ """A 200 response with invalid JSON propagates decoding failure, including after prior pages."""
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Optionally return one valid page, then a successful response with invalid JSON."""
+ requests.append(request)
+ if after_first_page and len(requests) == 1:
+ return httpx.Response(200, json=_table_page("Header", "next-page"))
+ return httpx.Response(200, text="not-json")
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(json.JSONDecodeError):
+ _table_extractor()._read_table_rows("table")
+
+ assert len(requests) == (2 if after_first_page else 1)
+
+
+ "invalid_page",
+ [
+ {"next_cursor": None},
+ {"results": []},
+ {"results": None, "next_cursor": None},
+ {"results": "not-a-list", "next_cursor": None},
+ {"results": ["not-an-object"], "next_cursor": None},
+ [],
+ {"results": [], "next_cursor": 42},
+ ],
+ ids=[
+ "missing-results",
+ "missing-next-cursor",
+ "null-results",
+ "non-list-results",
+ "non-object-result",
+ "non-object-page",
+ "invalid-cursor",
+ ],
+)
[email protected]("after_first_page", [False, True], ids=["first-page", "later-page"])
+def test_table_rows_reject_malformed_success_responses(
+ monkeypatch: pytest.MonkeyPatch, invalid_page: object, after_first_page: bool
+) -> None:
+ """Malformed successful pages fail validation before empty or partial content can return."""
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Optionally return one valid page, then the supplied malformed success payload."""
+ requests.append(request)
+ if after_first_page and len(requests) == 1:
+ return httpx.Response(200, json=_table_page("Header", "next-page"))
+ return httpx.Response(200, json=invalid_page)
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(ValidationError):
+ _table_extractor()._read_table_rows("table")
+
+ assert len(requests) == (2 if after_first_page else 1)
+
+
+def test_table_rows_network_errors_propagate(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A table transport failure propagates without producing content."""
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Raise a synthetic connection failure for the accepted table request."""
+ raise httpx.ConnectError("Synthetic connection failure", request=request)
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(httpx.ConnectError, match="Synthetic connection failure"):
+ _table_extractor()._read_table_rows("table")
+
+
+def test_page_import_does_not_return_a_partial_document_when_table_loading_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Page extraction fails when one table request fails, even after other blocks were read."""
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ """Return a mixed page root, then rate-limit its nested table request."""
+ if request.url.path.endswith("/page/children"):
+ return httpx.Response(
+ 200,
+ json={
+ "results": [
+ {
+ "type": "paragraph",
+ "id": "paragraph",
+ "has_children": False,
+ "paragraph": {"rich_text": [{"plain_text": "Retained paragraph"}]},
+ },
+ {"type": "table", "id": "table", "has_children": True, "table": {}},
+ ],
+ "next_cursor": None,
+ },
+ )
+ assert request.url.path.endswith("/table/children")
+ return httpx.Response(
+ 429,
+ json={"object": "error", "status": 429, "code": "rate_limited", "message": "Synthetic failure"},
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as client:
+ monkeypatch.setattr(notion_extractor.httpx, "request", client.request)
+ with pytest.raises(httpx.HTTPStatusError) as caught:
+ _table_extractor()._load_data_as_documents("page", "page")
+
+ assert caught.value.response.status_code == 429
+
+
def _database_extractor() -> notion_extractor.NotionExtractor:
return notion_extractor.NotionExtractor(
notion_workspace_id="workspace",