Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): normalise CRLF in stored signatures and docstrings (#3929)
daiyu8eda28538e8a
diff --git a/packages/core/src/repowise/core/ingestion/parser/__init__.py b/packages/core/src/repowise/core/ingestion/parser/__init__.pyindex 7e611f089..372625da4 100644--- a/packages/core/src/repowise/core/ingestion/parser/__init__.py+++ b/packages/core/src/repowise/core/ingestion/parser/__init__.py@@ -124,6 +124,7 @@ from .symbols import (_java_enum_constant,_match_identity,_nesting_parent,+ _normalize_newlines,_parent_type_name,)@@ -242,7 +243,7 @@ class ASTParser:export_aliases=ts_export_aliases(src) if lang in _TS_JS_LANGUAGES else {},calls=calls,heritage=extract_heritage(matches, config, file_info, src),- docstring=extract_module_docstring(root, src, lang),+ docstring=_normalize_newlines(extract_module_docstring(root, src, lang)),parse_errors=parse_errors,content_hash=content_hash,type_refs=_extract_type_refs(matches, src, lang),diff --git a/packages/core/src/repowise/core/ingestion/parser/symbols.py b/packages/core/src/repowise/core/ingestion/parser/symbols.pyindex 49f064aa1..cc95ba696 100644--- a/packages/core/src/repowise/core/ingestion/parser/symbols.py+++ b/packages/core/src/repowise/core/ingestion/parser/symbols.py@@ -70,6 +70,28 @@ _node_text = node_text_MODULE_ANCHORED_NODE_TYPES = frozenset({"assignment", "variable_declarator"})+def _normalize_newlines(text: str | None) -> str | None:+ """Return *text* with CRLF line endings rewritten as LF.++ The signature and docstring strings are sliced straight out of the file+ bytes, so a repository committed with CRLF — or a Windows checkout with+ ``core.autocrlf=true`` — stores literal ``\r\n`` in them. The same+ symbol then reads differently depending on the checkout's line-ending+ setting, and the extra carriage returns cost tokens on every MCP reply.++ Normalising the source before parsing is not an option: tree-sitter byte+ offsets and line numbers have to keep matching the real file. So it+ happens here, where the stored strings are produced. A lone ``\r`` is+ not a line ending and is left alone.++ *None* passes straight through: the docstring helpers return it when a+ symbol has no docstring, and that absence is information.+ """+ if text is None:+ return None+ return text.replace("\r\n", "\n")++def _match_identity(capture_dict: dict,config: LanguageConfig,@@ -330,10 +352,14 @@ def _symbol_from_match(capture_dict: dict, file: _SymbolPass) -> tuple[Symbol, Nname=name,qualified_name=_build_qualified_name(file_info.path, scope, name),kind=_method_or_kind(kind, parent_name, def_node, language), # type: ignore[arg-type]- signature=build_signature(node_type, name, params_text, def_node, src),+ signature=_normalize_newlines(+ build_signature(node_type, name, params_text, def_node, src)+ ) or "",start_line=start_line,end_line=end_line,- docstring=extract_symbol_docstring(def_node, src, language),+ docstring=_normalize_newlines(+ extract_symbol_docstring(def_node, src, language)+ ),decorators=([m for m in modifier_texts if m.startswith("@")]+ _attribute_decorators(def_node, language, src)diff --git a/tests/unit/ingestion/parser/test_crlf_text.py b/tests/unit/ingestion/parser/test_crlf_text.pynew file mode 100644index 000000000..b6c0c8d27--- /dev/null+++ b/tests/unit/ingestion/parser/test_crlf_text.py@@ -0,0 +1,86 @@+"""Stored symbol text must not depend on the checkout's line-ending setting.++Covers #3912: a file committed with CRLF (or a Windows checkout running with+``core.autocrlf=true``) used to keep the literal ``\\r\\n`` inside+``Symbol.signature`` and ``Symbol.docstring``, so the same symbol read+differently from one MCP tool to the next.+"""++from __future__ import annotations++import pytest++from repowise.core.ingestion.parser import ASTParser+from tests.unit.ingestion.parser._helpers import _make_file_info++# The repro from the issue, byte for byte: a multi-line signature whose docstring+# is also multi-line, with CRLF endings throughout.+CRLF_SOURCE = (+ b'"""Module docstring.\r\n'+ b'\r\n'+ b'Second line.\r\n'+ b'"""\r\n'+ b'\r\n'+ b'\r\n'+ b'def run_update(\r\n'+ b' path: str | None,\r\n'+ b' force: bool = False,\r\n'+ b') -> int:\r\n'+ b' """Compute a budget.\r\n'+ b'\r\n'+ b' Small changes get a small budget.\r\n'+ b' """\r\n'+ b' return 1\r\n'+)++LF_SOURCE = CRLF_SOURCE.replace(b'\r\n', b'\n')++PATH = 'pkg/mod.py'+LANGUAGE = 'python'++[email protected](scope='module')+def crlf_result():+ return ASTParser().parse_file(_make_file_info(PATH, LANGUAGE), CRLF_SOURCE)++[email protected](scope='module')+def lf_result():+ return ASTParser().parse_file(_make_file_info(PATH, LANGUAGE), LF_SOURCE)+++def _symbol(result):+ return next(symbol for symbol in result.symbols if symbol.name == 'run_update')+++def test_signature_has_no_carriage_returns(crlf_result) -> None:+ """A CRLF file's signature is stored with LF, like an LF file's."""+ signature = _symbol(crlf_result).signature++ assert '\r' not in signature+ assert signature.startswith('def run_update(')+++def test_symbol_docstring_has_no_carriage_returns(crlf_result) -> None:+ """A CRLF file's symbol docstring is stored with LF."""+ docstring = _symbol(crlf_result).docstring++ assert docstring is not None+ assert '\r' not in docstring+ assert docstring.startswith('Compute a budget.')+++def test_module_docstring_has_no_carriage_returns(crlf_result) -> None:+ """A CRLF file's module docstring is stored with LF."""+ docstring = crlf_result.docstring++ assert docstring is not None+ assert '\r' not in docstring+ assert docstring.startswith('Module docstring.')+++def test_lf_source_is_unchanged(crlf_result, lf_result) -> None:+ """A CRLF checkout and an LF checkout of one file store the same text."""+ assert _symbol(crlf_result).signature == _symbol(lf_result).signature+ assert _symbol(crlf_result).docstring == _symbol(lf_result).docstring+ assert crlf_result.docstring == lf_result.docstring