Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test(api): add unit tests for remove_leading_symbols (#45254)

TINGyu123644b8a241f06052
diff --git a/api/tests/unit_tests/core/tools/utils/test_remove_leading_symbols.py b/api/tests/unit_tests/core/tools/utils/test_remove_leading_symbols.py
new file mode 100644
index 00000000000..d811765ba4b
--- /dev/null
+++ b/api/tests/unit_tests/core/tools/utils/test_remove_leading_symbols.py
@@ -0,0 +1,59 @@
+import pytest
+
+from core.tools.utils.text_processing_utils import remove_leading_symbols
+
+
+ ("text", "expected"),
+ [
+ pytest.param("~hello world", "hello world", id="ascii-tilde"),
+ pytest.param("\"#$%&'()*+,./:;<=>?@^_`~tail", "tail", id="ascii-punctuation-run"),
+ pytest.param("。中文标题", "中文标题", id="cjk-period"),
+ pytest.param("、。ItemList", "ItemList", id="cjk-comma-and-period"),
+ pytest.param("\u3000full-width-space", "full-width-space", id="ideographic-space"),
+ pytest.param("\u201cquoted\u201d", "quoted\u201d", id="left-double-quote-only"),
+ pytest.param("—–dash", "dash", id="em-and-en-dash"),
+ ],
+)
+def test_remove_leading_symbols_strips_supported_punctuation(text: str, expected: str) -> None:
+ assert remove_leading_symbols(text) == expected
+
+
+ "text",
+ [
+ pytest.param("plain text", id="no-leading-symbols"),
+ pytest.param("hello, world", id="internal-punctuation-kept"),
+ pytest.param("!exclamation-kept", id="ascii-exclamation-not-in-class"),
+ pytest.param("-hyphen-kept", id="ascii-hyphen-not-in-class"),
+ pytest.param("[bracket-kept", id="ascii-bracket-excluded-by-design"),
+ pytest.param("【lenticular-kept", id="cjk-lenticular-bracket-excluded-by-design"),
+ pytest.param("…ellipsis-kept", id="horizontal-ellipsis-outside-ranges"),
+ pytest.param("", id="empty-string"),
+ ],
+)
+def test_remove_leading_symbols_preserves_unmatched_inputs(text: str) -> None:
+ assert remove_leading_symbols(text) == text
+
+
+ ("text", "expected"),
+ [
+ pytest.param("[label](https://example.com)", "[label](https://example.com)", id="bare-markdown-link"),
+ pytest.param(
+ "[label](https://example.com) trailing text",
+ "[label](https://example.com) trailing text",
+ id="markdown-link-with-trailing-text",
+ ),
+ pytest.param("[label](https://example.com/a(b))", "[label](https://example.com/a(b))", id="url-with-parens"),
+ ],
+)
+def test_remove_leading_symbols_preserves_leading_markdown_links(text: str, expected: str) -> None:
+ assert remove_leading_symbols(text) == expected
+
+
+def test_remove_leading_symbols_non_http_link_is_kept() -> None:
+ # Only http(s) markdown links match the preserve-pattern; other schemes fall through to
+ # normal stripping, which leaves the text unchanged because brackets are excluded
+ # from the strip class.
+ assert remove_leading_symbols("[label](ftp://example.com)") == "[label](ftp://example.com)"