π« Toolkit to help you get started with SDD or any other process!
fix(toml): escape control characters so generated command files parse (#3341)
Quratulain-bilal94c7ec288fd3
diff --git a/src/specify_cli/_toml_string.py b/src/specify_cli/_toml_string.pynew file mode 100644index 000000000..c9b024245--- /dev/null+++ b/src/specify_cli/_toml_string.py@@ -0,0 +1,60 @@+"""Shared TOML string-escaping helpers.++Both TOML command renderers β ``TomlIntegration`` (gemini, tabnine) in+``specify_cli.integrations.base`` and ``CommandRegistrar.render_toml_command``+(extension/preset commands) in ``specify_cli.agents`` β need the same rules for+detecting characters TOML forbids literally and for emitting a fully-escaped+basic string. Keeping one implementation here avoids the two drifting apart if+the escaping rules change again.+"""+from __future__ import annotations+++def has_illegal_toml_control(value: str) -> bool:+ """True when *value* contains a character TOML forbids literally.++ TOML basic/literal strings (single- or multi-line) allow tab and, in the+ multiline forms, newlines β but every other control character+ (``U+0000``β``U+001F`` and ``U+007F``) must be ``\\u``-escaped, which only a+ basic string can do. A bare carriage return counts too: a multiline basic+ string treats ``\\r`` as a newline only when paired into ``\\r\\n``; a lone+ ``\\r`` is an illegal control character.+ """+ length = len(value)+ for i, ch in enumerate(value):+ code = ord(ch)+ if ch == "\r":+ # Only a CR that is part of a CRLF newline is allowed literally.+ if i + 1 < length and value[i + 1] == "\n":+ continue+ return True+ if (code < 0x20 and ch not in ("\t", "\n")) or code == 0x7F:+ return True+ return False+++def escape_toml_basic(value: str) -> str:+ """Render *value* as a single-line basic string, escaping everything.++ Always valid TOML: backslash/quote are escaped, the common control chars+ use their short escapes, and any remaining control character is emitted as+ a ``\\uXXXX`` sequence.+ """+ out: list[str] = []+ for ch in value:+ code = ord(ch)+ if ch == "\\":+ out.append("\\\\")+ elif ch == '"':+ out.append('\\"')+ elif ch == "\n":+ out.append("\\n")+ elif ch == "\r":+ out.append("\\r")+ elif ch == "\t":+ out.append("\\t")+ elif code < 0x20 or code == 0x7F:+ out.append(f"\\u{code:04x}")+ else:+ out.append(ch)+ return '"' + "".join(out) + '"'diff --git a/src/specify_cli/agents.py b/src/specify_cli/agents.pyindex c127407d4..391015394 100644--- a/src/specify_cli/agents.py+++ b/src/specify_cli/agents.py@@ -16,6 +16,8 @@ from typing import Any, Dict, List, Optionalimport yamlfrom ._init_options import is_ai_skills_enabled, load_init_options+from ._toml_string import escape_toml_basic as _escape_toml_basic+from ._toml_string import has_illegal_toml_control as _has_illegal_toml_controlfrom ._utils import relative_extension_path_violation@@ -258,7 +260,12 @@ class CommandRegistrar:# ``C:\\Users\\...`` whose ``\\U`` reads as an invalid unicode escape) would# produce unparseable TOML β route those to the *literal* form ('''...'''),# which does not process escapes, or to the escaped basic string.- if '"""' not in body and "\\" not in body:+ # Control characters (U+0000βU+001F except tab/newline, U+007F) and a bare+ # CR are illegal in every TOML string form, so a body containing them must+ # go to the escaped basic string regardless of which delimiters it uses.+ if self._has_illegal_toml_control(body):+ toml_lines.append(f"prompt = {self._render_basic_toml_string(body)}")+ elif '"""' not in body and "\\" not in body:toml_lines.append('prompt = """')toml_lines.append(body)toml_lines.append('"""')@@ -271,17 +278,11 @@ class CommandRegistrar:return "\n".join(toml_lines)- @staticmethod- def _render_basic_toml_string(value: str) -> str:- """Render *value* as a TOML basic string literal."""- escaped = (- value.replace("\\", "\\\\")- .replace('"', '\\"')- .replace("\n", "\\n")- .replace("\r", "\\r")- .replace("\t", "\\t")- )- return f'"{escaped}"'+ # Control-char detection and basic-string escaping are shared with the+ # gemini/tabnine renderer in ``specify_cli.integrations.base`` via+ # ``specify_cli._toml_string`` so the two never drift apart.+ _has_illegal_toml_control = staticmethod(_has_illegal_toml_control)+ _render_basic_toml_string = staticmethod(_escape_toml_basic)def render_yaml_command(self,diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.pyindex 706a3cb5d..4eb9bb19a 100644--- a/src/specify_cli/integrations/base.py+++ b/src/specify_cli/integrations/base.py@@ -25,6 +25,9 @@ from typing import TYPE_CHECKING, Anyimport yaml+from .._toml_string import escape_toml_basic as _escape_toml_basic+from .._toml_string import has_illegal_toml_control as _has_illegal_toml_control+if TYPE_CHECKING:from .manifest import IntegrationManifest@@ -953,6 +956,12 @@ class TomlIntegration(IntegrationBase):body = "".join(lines[frontmatter_end + 1 :])return frontmatter, body+ # Control-char detection and basic-string escaping are shared with the+ # extension/preset renderer in ``specify_cli.agents`` via+ # ``specify_cli._toml_string`` so the two never drift apart.+ _has_illegal_toml_control = staticmethod(_has_illegal_toml_control)+ _escape_toml_basic = staticmethod(_escape_toml_basic)+@staticmethoddef _render_toml_string(value: str) -> str:"""Render *value* as a TOML string literal.@@ -962,6 +971,12 @@ class TomlIntegration(IntegrationBase):literal string or escaped basic string when delimiters appear inthe content."""+ # Control characters other than tab/newline (and a bare CR) cannot+ # appear literally in any TOML string; route them to a fully-escaped+ # basic string so the generated file stays parseable.+ if TomlIntegration._has_illegal_toml_control(value):+ return TomlIntegration._escape_toml_basic(value)+if "\n" not in value and "\r" not in value:escaped = value.replace("\\", "\\\\").replace('"', '\\"')return f'"{escaped}"'@@ -974,17 +989,7 @@ class TomlIntegration(IntegrationBase):if "'''" not in value and not value.endswith("'"):return "'''\n" + value + "'''"- return (- '"'- + (- value.replace("\\", "\\\\")- .replace('"', '\\"')- .replace("\n", "\\n")- .replace("\r", "\\r")- .replace("\t", "\\t")- )- + '"'- )+ return TomlIntegration._escape_toml_basic(value)@staticmethoddef _render_toml(description: str, body: str) -> str:diff --git a/tests/integrations/test_integration_base_toml.py b/tests/integrations/test_integration_base_toml.pyindex 68f5fd075..37bad8a60 100644--- a/tests/integrations/test_integration_base_toml.py+++ b/tests/integrations/test_integration_base_toml.py@@ -291,6 +291,18 @@ class TomlIntegrationTests:"closing delimiter should be inline when body does not end with a quote")+ def test_toml_string_escapes_control_characters(self):+ """A value with control chars / a bare CR must render as parseable TOML.++ TOML forbids literal control characters (U+0000βU+001F except tab and+ newline, plus U+007F) in every string form, and a bare CR that is not+ part of a CRLF pair. The renderer used to emit these raw into a basic or+ ``\"\"\"`` multiline string, producing a config file that fails to parse."""+ value = "start\x00null\x01ctrl\x1besc\x7fdel\rlone-cr end"+ rendered = TomlIntegration._render_toml_string(value)+ parsed = tomllib.loads(f"prompt = {rendered}")+ assert parsed["prompt"] == value+def test_toml_is_valid(self, tmp_path):"""Every generated TOML file must parse without errors."""i = get_integration(self.KEY)diff --git a/tests/test_extensions.py b/tests/test_extensions.pyindex 3118ccdef..288518036 100644--- a/tests/test_extensions.py+++ b/tests/test_extensions.py@@ -1795,6 +1795,25 @@ $ARGUMENTSassert parsed["description"] == "first line\nsecond line\n"+ def test_render_toml_command_escapes_control_characters(self):+ """Control characters and a lone CR must be escaped so the TOML parses.++ TOML forbids literal control characters (U+0000βU+001F except tab and+ newline, plus U+007F) in any string, and treats a bare CR outside a+ CRLF pair as illegal. The renderer used to emit these raw β into a+ basic string (single-line) or a ``\"\"\"`` multiline string (for a lone+ CR) β producing a command file that fails to parse."""+ from specify_cli.agents import CommandRegistrar as AgentCommandRegistrar++ registrar = AgentCommandRegistrar()+ body = "start\x00null\x01ctrl\x1besc\x7fdel\rlone-cr end"+ output = registrar.render_toml_command(+ {"description": "d"}, body, "extension:test-ext"+ )++ parsed = tomllib.loads(output)+ assert parsed["prompt"] == body+def test_render_toml_command_preserves_backslashes_in_body(self):"""A backslash in the body (e.g. a Windows path) must not break TOML.