Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
fenced_code: Add a function for closing a message's unclosed fences.
Aman Agrawal232fea9c9dc6
diff --git a/zerver/lib/markdown/fenced_code.py b/zerver/lib/markdown/fenced_code.pyindex beb11cd4744..52c85b1be97 100644--- a/zerver/lib/markdown/fenced_code.py+++ b/zerver/lib/markdown/fenced_code.py@@ -86,7 +86,7 @@ from django.utils.html import escapefrom markdown import Markdownfrom markdown.extensions import Extension, codehilitefrom markdown.extensions.codehilite import CodeHiliteExtension, parse_hl_lines-from markdown.preprocessors import Preprocessor+from markdown.preprocessors import NormalizeWhitespace, Preprocessorfrom pygments.lexers import find_lexer_class_by_namefrom pygments.util import ClassNotFoundfrom typing_extensions import override@@ -590,6 +590,38 @@ class FencedBlockPreprocessor(Preprocessor):return txt+class UnclosedFenceFinder(FencedBlockPreprocessor):+ """Finds the fenced blocks that content leaves open, using the same+ handlers that parse fenced blocks when rendering a message. Blocks+ that content closes aren't used, so this skips formatting code and+ math blocks, which is slow.+ """++ @override+ def format_code(self, lang: str | None, text: str) -> str:+ return text++ @override+ def format_tex(self, text: str) -> str:+ return text++ def get_unclosed_fences(self, content: str, default_language: str | None) -> list[str]:+ # Markdown normalizes whitespace before parsing fenced blocks.+ lines = NormalizeWhitespace(self.md).run(content.split("\n"))+ self.handle_lines(lines, [], default_language)+ return [handler.fence for handler in reversed(self.handlers) if handler.fence is not None]+++def close_unclosed_fences(content: str, default_language: str | None = None) -> str:+ """Appends a closing fence for each fenced block that content leaves+ open, so that the blocks can't swallow text placed after content.+ default_language is the realm's default code block language, used+ for fences that don't name a language.+ """+ unclosed_fences = UnclosedFenceFinder(Markdown()).get_unclosed_fences(content, default_language)+ return "\n".join([content, *unclosed_fences])++def makeExtension(*args: Any, **kwargs: Any) -> FencedCodeExtension:return FencedCodeExtension(kwargs)diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.pyindex 90f36d049c0..10007c268c0 100644--- a/zerver/tests/test_markdown.py+++ b/zerver/tests/test_markdown.py@@ -50,7 +50,7 @@ from zerver.lib.markdown import (url_embed_preview_enabled,url_to_a,)-from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor+from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor, close_unclosed_fencesfrom zerver.lib.markdown.from_html import convert_html_to_markdownfrom zerver.lib.mdiff import diff_stringsfrom zerver.lib.mention import (@@ -210,6 +210,43 @@ class FencedBlockPreprocessorTest(ZulipTestCase):lines = processor.run(markdown_input)self.assertEqual(lines, expected)+ def test_close_unclosed_fences(self) -> None:+ cases = [+ ("no fences", "hello", ""),+ ("closed code block", "```\ncode\n```", ""),+ ("unclosed code block", "```\ncode", "\n```"),+ ("fence on the last code line", "```\ncode```", "\n```"),+ ("unclosed tilde code block", "~~~ py\ncode", "\n~~~"),+ ("code blocks don't nest", "```py\n```quote\ncode", "\n```"),+ ("longer code block fence", "````\n```\ncode", "\n````"),+ ("math blocks don't nest", "```math\n```quote\nx^2\n```", ""),+ ("unclosed spoiler", "```spoiler Header\nsecret", "\n```"),+ ("unclosed quote in quote", "````quote\n```quote\nhi", "\n```\n````"),+ ("closed code block in closed quote", "````quote\n```\ncode\n```\n````", ""),+ ("unclosed code block in quote", "````quote\n```\ncode\n````", "\n```\n````"),+ ("windows line endings", "```quote\r\nhi\r\n", "\n```"),+ ("tab after the fence", "```\tpy\ncode", "\n```"),+ ("control characters in the fence", "\x02```\x03\ncode", "\n```"),+ ]+ for case, content, closing_fences in cases:+ with self.subTest(case=case):+ closed_content = close_unclosed_fences(content)+ self.assertEqual(closed_content, content + closing_fences)+ rendered_content = markdown_convert_wrapper(closed_content + "\n\nafter")+ self.assertTrue(rendered_content.endswith("\n<p>after</p>"))++ def test_close_unclosed_fences_with_default_code_block_language(self) -> None:+ realm = get_realm("zulip")+ do_set_realm_property(realm, "default_code_block_language", "quote", acting_user=None)++ # With quote as the default language, the first fence opens a quote+ # block, and the second opens a code block inside it.+ content = "```\n```py\ncode"+ closed_content = close_unclosed_fences(content, realm.default_code_block_language)+ self.assertEqual(closed_content, content + "\n```\n```")+ rendered_content = markdown_convert_wrapper(closed_content + "\n\nafter")+ self.assertTrue(rendered_content.endswith("\n<p>after</p>"))+def markdown_convert_wrapper(content: str) -> str:return markdown_convert(