Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): normalise CRLF in stored signatures and docstrings (#3929)

daiyu8eda28538e8a
diff --git a/packages/core/src/repowise/core/ingestion/parser/__init__.py b/packages/core/src/repowise/core/ingestion/parser/__init__.py
index 7e611f089..372625da4 100644
--- a/packages/core/src/repowise/core/ingestion/parser/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/parser/__init__.py
@@ -124,6 +124,7 @@ from .symbols import (
_java_enum_constant,
_match_identity,
_nesting_parent,
+ _normalize_newlines,
_parent_type_name,
)
@@ -242,7 +243,7 @@ class ASTParser:
export_aliases=ts_export_aliases(src) if lang in _TS_JS_LANGUAGES else {},
calls=calls,
heritage=extract_heritage(matches, config, file_info, src),
- docstring=extract_module_docstring(root, src, lang),
+ docstring=_normalize_newlines(extract_module_docstring(root, src, lang)),
parse_errors=parse_errors,
content_hash=content_hash,
type_refs=_extract_type_refs(matches, src, lang),
diff --git a/packages/core/src/repowise/core/ingestion/parser/symbols.py b/packages/core/src/repowise/core/ingestion/parser/symbols.py
index 49f064aa1..cc95ba696 100644
--- a/packages/core/src/repowise/core/ingestion/parser/symbols.py
+++ b/packages/core/src/repowise/core/ingestion/parser/symbols.py
@@ -70,6 +70,28 @@ _node_text = node_text
_MODULE_ANCHORED_NODE_TYPES = frozenset({"assignment", "variable_declarator"})
+def _normalize_newlines(text: str | None) -> str | None:
+ """Return *text* with CRLF line endings rewritten as LF.
+
+ The signature and docstring strings are sliced straight out of the file
+ bytes, so a repository committed with CRLF — or a Windows checkout with
+ ``core.autocrlf=true`` — stores literal ``\r\n`` in them. The same
+ symbol then reads differently depending on the checkout's line-ending
+ setting, and the extra carriage returns cost tokens on every MCP reply.
+
+ Normalising the source before parsing is not an option: tree-sitter byte
+ offsets and line numbers have to keep matching the real file. So it
+ happens here, where the stored strings are produced. A lone ``\r`` is
+ not a line ending and is left alone.
+
+ *None* passes straight through: the docstring helpers return it when a
+ symbol has no docstring, and that absence is information.
+ """
+ if text is None:
+ return None
+ return text.replace("\r\n", "\n")
+
+
def _match_identity(
capture_dict: dict,
config: LanguageConfig,
@@ -330,10 +352,14 @@ def _symbol_from_match(capture_dict: dict, file: _SymbolPass) -> tuple[Symbol, N
name=name,
qualified_name=_build_qualified_name(file_info.path, scope, name),
kind=_method_or_kind(kind, parent_name, def_node, language), # type: ignore[arg-type]
- signature=build_signature(node_type, name, params_text, def_node, src),
+ signature=_normalize_newlines(
+ build_signature(node_type, name, params_text, def_node, src)
+ ) or "",
start_line=start_line,
end_line=end_line,
- docstring=extract_symbol_docstring(def_node, src, language),
+ docstring=_normalize_newlines(
+ extract_symbol_docstring(def_node, src, language)
+ ),
decorators=(
[m for m in modifier_texts if m.startswith("@")]
+ _attribute_decorators(def_node, language, src)
diff --git a/tests/unit/ingestion/parser/test_crlf_text.py b/tests/unit/ingestion/parser/test_crlf_text.py
new file mode 100644
index 000000000..b6c0c8d27
--- /dev/null
+++ b/tests/unit/ingestion/parser/test_crlf_text.py
@@ -0,0 +1,86 @@
+"""Stored symbol text must not depend on the checkout's line-ending setting.
+
+Covers #3912: a file committed with CRLF (or a Windows checkout running with
+``core.autocrlf=true``) used to keep the literal ``\\r\\n`` inside
+``Symbol.signature`` and ``Symbol.docstring``, so the same symbol read
+differently from one MCP tool to the next.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.parser import ASTParser
+from tests.unit.ingestion.parser._helpers import _make_file_info
+
+# The repro from the issue, byte for byte: a multi-line signature whose docstring
+# is also multi-line, with CRLF endings throughout.
+CRLF_SOURCE = (
+ b'"""Module docstring.\r\n'
+ b'\r\n'
+ b'Second line.\r\n'
+ b'"""\r\n'
+ b'\r\n'
+ b'\r\n'
+ b'def run_update(\r\n'
+ b' path: str | None,\r\n'
+ b' force: bool = False,\r\n'
+ b') -> int:\r\n'
+ b' """Compute a budget.\r\n'
+ b'\r\n'
+ b' Small changes get a small budget.\r\n'
+ b' """\r\n'
+ b' return 1\r\n'
+)
+
+LF_SOURCE = CRLF_SOURCE.replace(b'\r\n', b'\n')
+
+PATH = 'pkg/mod.py'
+LANGUAGE = 'python'
+
+
[email protected](scope='module')
+def crlf_result():
+ return ASTParser().parse_file(_make_file_info(PATH, LANGUAGE), CRLF_SOURCE)
+
+
[email protected](scope='module')
+def lf_result():
+ return ASTParser().parse_file(_make_file_info(PATH, LANGUAGE), LF_SOURCE)
+
+
+def _symbol(result):
+ return next(symbol for symbol in result.symbols if symbol.name == 'run_update')
+
+
+def test_signature_has_no_carriage_returns(crlf_result) -> None:
+ """A CRLF file's signature is stored with LF, like an LF file's."""
+ signature = _symbol(crlf_result).signature
+
+ assert '\r' not in signature
+ assert signature.startswith('def run_update(')
+
+
+def test_symbol_docstring_has_no_carriage_returns(crlf_result) -> None:
+ """A CRLF file's symbol docstring is stored with LF."""
+ docstring = _symbol(crlf_result).docstring
+
+ assert docstring is not None
+ assert '\r' not in docstring
+ assert docstring.startswith('Compute a budget.')
+
+
+def test_module_docstring_has_no_carriage_returns(crlf_result) -> None:
+ """A CRLF file's module docstring is stored with LF."""
+ docstring = crlf_result.docstring
+
+ assert docstring is not None
+ assert '\r' not in docstring
+ assert docstring.startswith('Module docstring.')
+
+
+def test_lf_source_is_unchanged(crlf_result, lf_result) -> None:
+ """A CRLF checkout and an LF checkout of one file store the same text."""
+ assert _symbol(crlf_result).signature == _symbol(lf_result).signature
+ assert _symbol(crlf_result).docstring == _symbol(lf_result).docstring
+ assert crlf_result.docstring == lf_result.docstring