Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(cobol): parse free-format COBOL sources by normalizing column layout (#4454) (#4722)
Rishabh Mishrad18c57a701ad
diff --git a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py b/packages/core/src/repowise/core/ingestion/cobol/reference_format.pyindex 390c4b974..f609e4fe0 100644--- a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py+++ b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py@@ -17,6 +17,7 @@ _COMPILER_DIRECTIVE = re.compile(r"\s*(PROCESS|CBL)\b", re.IGNORECASE)# or the end of the line, so a data name such as ``CBL-REC`` stays a data name._COMPILER_DIRECTIVE_COLUMN_1 = re.compile(r"\s*(PROCESS|CBL)(?=\s|$)", re.IGNORECASE)_FREE_FORMAT = re.compile(r">>\s*SOURCE\s+(FORMAT\s+(IS\s+)?)?FREE|SOURCEFORMAT\W+FREE", re.I)+_FREE_DIRECTIVE = re.compile(r"^\s*>>", re.IGNORECASE)def is_free_format(text: str) -> bool:@@ -24,6 +25,23 @@ def is_free_format(text: str) -> bool:return _FREE_FORMAT.search(text) is not None+def normalize_free_format(text: str) -> str:+ """Move free-format code into columns the grammar expects and blank directives.++ Line numbers are preserved. Compiler directive lines (such as+ ``>>SOURCE FORMAT FREE``) become blank, and code lines are shifted 7+ columns right so column 1 lands in Area A (column 8).+ """+ out: list[str] = []+ for raw in text.split("\n"):+ line, eol = (raw[:-1], "\r") if raw.endswith("\r") else (raw, "")+ if _FREE_DIRECTIVE.match(line) or _FREE_FORMAT.search(line) or not line.strip():+ out.append(eol)+ continue+ out.append(" " * _TEXT_START + line + eol)+ return "\n".join(out)++def normalize_fixed_format(text: str) -> str:"""Blank what the compiler ignores and close literals split by continuation lines.diff --git a/packages/core/src/repowise/core/ingestion/cobol/source.py b/packages/core/src/repowise/core/ingestion/cobol/source.pyindex 639268eed..9a6e27619 100644--- a/packages/core/src/repowise/core/ingestion/cobol/source.py+++ b/packages/core/src/repowise/core/ingestion/cobol/source.py@@ -6,7 +6,7 @@ import refrom .fragments import is_fragment, is_procedure_fragment, wrap_fragmentfrom .grammar_gaps import rewrite_gaps-from .reference_format import is_free_format, normalize_fixed_format+from .reference_format import is_free_format, normalize_fixed_format, normalize_free_formatfrom .statements import mask_statements# A lossless round trip for bytes that are not UTF-8 (EBCDIC-converted Latin-1).@@ -18,8 +18,7 @@ _INLINE_COMMENT = re.compile(r"('[^'\n]*'|\"[^\"\n]*\")|\*>[^\r\n]*")def code_view(source: bytes) -> str:"""*source* as program text only: comments, sequence area and columns 73+ blank."""text = source.decode(_ENCODING, _ERRORS)- # Free format keeps its text as written; only fixed format has columns.- code = text if is_free_format(text) else normalize_fixed_format(text)+ code = normalize_free_format(text) if is_free_format(text) else normalize_fixed_format(text)return _INLINE_COMMENT.sub(lambda m: m.group(1) or " " * len(m.group(0)), code)diff --git a/tests/unit/ingestion/parser/test_cobol.py b/tests/unit/ingestion/parser/test_cobol.pyindex 8aad04815..a69f6d052 100644--- a/tests/unit/ingestion/parser/test_cobol.py+++ b/tests/unit/ingestion/parser/test_cobol.py@@ -396,3 +396,31 @@ class TestCobolCallScope:EXEC CICS XCTL PROGRAM('COCRDLIC') END-EXEC."""assert self._resolve(parser, {"cocrdlic.cbl": caller}, {}) == set()+++class TestCobolFreeFormat:+ def test_free_format_program_parses_symbols_and_calls(self, parser: ASTParser) -> None:+ source = (+ b">>SOURCE FORMAT FREE\n"+ b"IDENTIFICATION DIVISION.\n"+ b"PROGRAM-ID. HELLO.\n"+ b"DATA DIVISION.\n"+ b"WORKING-STORAGE SECTION.\n"+ b"01 WS-A PIC 9. *> a counter\n"+ b"PROCEDURE DIVISION.\n"+ b"MAIN-PARA.\n"+ b" PERFORM SHOW-PARA\n"+ b" STOP RUN.\n"+ b"SHOW-PARA.\n"+ b" DISPLAY \"HI\". *> greet\n"+ )+ parsed = parser.parse_file(_cobol("HELLO.cbl"), source)+ assert parsed.parse_errors == []+ assert {s.name: s.kind for s in parsed.symbols} == {+ "HELLO": "module",+ "WS-A": "variable",+ "MAIN-PARA": "function",+ "SHOW-PARA": "function",+ }+ assert [c.target_name for c in parsed.calls] == ["SHOW-PARA"]+diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.pyindex 0a7693668..7c3e9b9db 100644--- a/tests/unit/ingestion/parser/test_cobol_source.py+++ b/tests/unit/ingestion/parser/test_cobol_source.py@@ -230,7 +230,7 @@ def test_floating_comments_hide_statements_and_keep_literals() -> None:assert copy_statements(code) == []assert [block.line for block in exec_blocks(code)] == [9]assert "'*> kept'" in code- assert len(code) == len(source)+ assert len(code.splitlines()) == len(source.splitlines())def test_copy_statements_are_found_in_code_and_not_in_comments() -> None: