Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

fenced_code: Add a function for closing a message's unclosed fences.

Aman Agrawal232fea9c9dc6
diff --git a/zerver/lib/markdown/fenced_code.py b/zerver/lib/markdown/fenced_code.py
index beb11cd4744..52c85b1be97 100644
--- a/zerver/lib/markdown/fenced_code.py
+++ b/zerver/lib/markdown/fenced_code.py
@@ -86,7 +86,7 @@ from django.utils.html import escape
from markdown import Markdown
from markdown.extensions import Extension, codehilite
from markdown.extensions.codehilite import CodeHiliteExtension, parse_hl_lines
-from markdown.preprocessors import Preprocessor
+from markdown.preprocessors import NormalizeWhitespace, Preprocessor
from pygments.lexers import find_lexer_class_by_name
from pygments.util import ClassNotFound
from typing_extensions import override
@@ -590,6 +590,38 @@ class FencedBlockPreprocessor(Preprocessor):
return txt
+class UnclosedFenceFinder(FencedBlockPreprocessor):
+ """Finds the fenced blocks that content leaves open, using the same
+ handlers that parse fenced blocks when rendering a message. Blocks
+ that content closes aren't used, so this skips formatting code and
+ math blocks, which is slow.
+ """
+
+ @override
+ def format_code(self, lang: str | None, text: str) -> str:
+ return text
+
+ @override
+ def format_tex(self, text: str) -> str:
+ return text
+
+ def get_unclosed_fences(self, content: str, default_language: str | None) -> list[str]:
+ # Markdown normalizes whitespace before parsing fenced blocks.
+ lines = NormalizeWhitespace(self.md).run(content.split("\n"))
+ self.handle_lines(lines, [], default_language)
+ return [handler.fence for handler in reversed(self.handlers) if handler.fence is not None]
+
+
+def close_unclosed_fences(content: str, default_language: str | None = None) -> str:
+ """Appends a closing fence for each fenced block that content leaves
+ open, so that the blocks can't swallow text placed after content.
+ default_language is the realm's default code block language, used
+ for fences that don't name a language.
+ """
+ unclosed_fences = UnclosedFenceFinder(Markdown()).get_unclosed_fences(content, default_language)
+ return "\n".join([content, *unclosed_fences])
+
+
def makeExtension(*args: Any, **kwargs: Any) -> FencedCodeExtension:
return FencedCodeExtension(kwargs)
diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py
index 90f36d049c0..10007c268c0 100644
--- a/zerver/tests/test_markdown.py
+++ b/zerver/tests/test_markdown.py
@@ -50,7 +50,7 @@ from zerver.lib.markdown import (
url_embed_preview_enabled,
url_to_a,
)
-from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor
+from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor, close_unclosed_fences
from zerver.lib.markdown.from_html import convert_html_to_markdown
from zerver.lib.mdiff import diff_strings
from zerver.lib.mention import (
@@ -210,6 +210,43 @@ class FencedBlockPreprocessorTest(ZulipTestCase):
lines = processor.run(markdown_input)
self.assertEqual(lines, expected)
+ def test_close_unclosed_fences(self) -> None:
+ cases = [
+ ("no fences", "hello", ""),
+ ("closed code block", "```\ncode\n```", ""),
+ ("unclosed code block", "```\ncode", "\n```"),
+ ("fence on the last code line", "```\ncode```", "\n```"),
+ ("unclosed tilde code block", "~~~ py\ncode", "\n~~~"),
+ ("code blocks don't nest", "```py\n```quote\ncode", "\n```"),
+ ("longer code block fence", "````\n```\ncode", "\n````"),
+ ("math blocks don't nest", "```math\n```quote\nx^2\n```", ""),
+ ("unclosed spoiler", "```spoiler Header\nsecret", "\n```"),
+ ("unclosed quote in quote", "````quote\n```quote\nhi", "\n```\n````"),
+ ("closed code block in closed quote", "````quote\n```\ncode\n```\n````", ""),
+ ("unclosed code block in quote", "````quote\n```\ncode\n````", "\n```\n````"),
+ ("windows line endings", "```quote\r\nhi\r\n", "\n```"),
+ ("tab after the fence", "```\tpy\ncode", "\n```"),
+ ("control characters in the fence", "\x02```\x03\ncode", "\n```"),
+ ]
+ for case, content, closing_fences in cases:
+ with self.subTest(case=case):
+ closed_content = close_unclosed_fences(content)
+ self.assertEqual(closed_content, content + closing_fences)
+ rendered_content = markdown_convert_wrapper(closed_content + "\n\nafter")
+ self.assertTrue(rendered_content.endswith("\n<p>after</p>"))
+
+ def test_close_unclosed_fences_with_default_code_block_language(self) -> None:
+ realm = get_realm("zulip")
+ do_set_realm_property(realm, "default_code_block_language", "quote", acting_user=None)
+
+ # With quote as the default language, the first fence opens a quote
+ # block, and the second opens a code block inside it.
+ content = "```\n```py\ncode"
+ closed_content = close_unclosed_fences(content, realm.default_code_block_language)
+ self.assertEqual(closed_content, content + "\n```\n```")
+ rendered_content = markdown_convert_wrapper(closed_content + "\n\nafter")
+ self.assertTrue(rendered_content.endswith("\n<p>after</p>"))
+
def markdown_convert_wrapper(content: str) -> str:
return markdown_convert(