Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): flatten readabilipy plain_text items in web reader tool (#41954)

Harsh Kashyap09ddcbe01e11
diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py
index 2e630b803eb..056703413af 100644
--- a/api/core/tools/utils/web_reader_tool.py
+++ b/api/core/tools/utils/web_reader_tool.py
@@ -1,6 +1,5 @@
import mimetypes
import re
-from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from urllib.parse import unquote
@@ -101,15 +100,24 @@ def get_url(url: str, user_agent: str | None = None) -> str:
class Article:
title: str
author: str
- text: Sequence[dict]
+ text: str
def extract_using_readabilipy(html: str):
json_article: dict[str, Any] = simple_json_from_html_string(html, use_readability=False)
+ # readabilipy returns plain_text as a list of dicts whose "text" values
+ # contain the (possibly HTML-tagged) paragraph content; flatten them into
+ # a single clean text block.
+ plain_text = json_article.get("plain_text") or []
+ text = "\n".join(
+ stripped
+ for item in plain_text
+ if isinstance(item, dict) and (stripped := re.sub(r"<[^>]+>", "", str(item.get("text") or "")).strip())
+ )
article = Article(
title=json_article.get("title") or "",
author=json_article.get("byline") or "",
- text=json_article.get("plain_text") or [],
+ text=text,
)
return article
diff --git a/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py b/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py
index f66a2cc7067..80ab51b73c5 100644
--- a/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py
+++ b/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py
@@ -286,9 +286,31 @@ def test_extract_using_readabilipy_field_mapping_and_defaults(monkeypatch: pytes
article = extract_using_readabilipy("<html>...</html>")
assert article.title == "Hello"
assert article.author == "Alice"
- assert isinstance(article.text, list)
- assert article.text
- assert article.text[0]["text"] == "world"
+ assert article.text == "world"
+
+
+def test_extract_using_readabilipy_flattens_plain_text_items(monkeypatch: pytest.MonkeyPatch):
+ """readabilipy returns plain_text as a list of dicts; the article text must be clean joined text,
+ not the Python repr of that list, and HTML tags inside items must be stripped."""
+
+ def fake_simple_json_from_html_string(html, use_readability=True):
+ return {
+ "title": "T",
+ "byline": "A",
+ "plain_text": [
+ {"type": "text", "text": "<p>First paragraph.</p>"},
+ {"type": "text", "text": "Second paragraph."},
+ {"type": "text", "text": ""},
+ "not-a-dict",
+ ],
+ }
+
+ import core.tools.utils.web_reader_tool as mod
+
+ monkeypatch.setattr(mod, "simple_json_from_html_string", fake_simple_json_from_html_string)
+
+ article = extract_using_readabilipy("<html>...</html>")
+ assert article.text == "First paragraph.\nSecond paragraph."
def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.MonkeyPatch):
@@ -302,7 +324,7 @@ def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.Mon
article = extract_using_readabilipy("<html>...</html>")
assert article.title == ""
assert article.author == ""
- assert article.text == []
+ assert article.text == ""
# ---------------------------