Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(cobol): parse free-format COBOL sources by normalizing column layout (#4454) (#4722)

Rishabh Mishrad18c57a701ad
diff --git a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py
index 390c4b974..f609e4fe0 100644
--- a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py
+++ b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py
@@ -17,6 +17,7 @@ _COMPILER_DIRECTIVE = re.compile(r"\s*(PROCESS|CBL)\b", re.IGNORECASE)
# or the end of the line, so a data name such as ``CBL-REC`` stays a data name.
_COMPILER_DIRECTIVE_COLUMN_1 = re.compile(r"\s*(PROCESS|CBL)(?=\s|$)", re.IGNORECASE)
_FREE_FORMAT = re.compile(r">>\s*SOURCE\s+(FORMAT\s+(IS\s+)?)?FREE|SOURCEFORMAT\W+FREE", re.I)
+_FREE_DIRECTIVE = re.compile(r"^\s*>>", re.IGNORECASE)
def is_free_format(text: str) -> bool:
@@ -24,6 +25,23 @@ def is_free_format(text: str) -> bool:
return _FREE_FORMAT.search(text) is not None
+def normalize_free_format(text: str) -> str:
+ """Move free-format code into columns the grammar expects and blank directives.
+
+ Line numbers are preserved. Compiler directive lines (such as
+ ``>>SOURCE FORMAT FREE``) become blank, and code lines are shifted 7
+ columns right so column 1 lands in Area A (column 8).
+ """
+ out: list[str] = []
+ for raw in text.split("\n"):
+ line, eol = (raw[:-1], "\r") if raw.endswith("\r") else (raw, "")
+ if _FREE_DIRECTIVE.match(line) or _FREE_FORMAT.search(line) or not line.strip():
+ out.append(eol)
+ continue
+ out.append(" " * _TEXT_START + line + eol)
+ return "\n".join(out)
+
+
def normalize_fixed_format(text: str) -> str:
"""Blank what the compiler ignores and close literals split by continuation lines.
diff --git a/packages/core/src/repowise/core/ingestion/cobol/source.py b/packages/core/src/repowise/core/ingestion/cobol/source.py
index 639268eed..9a6e27619 100644
--- a/packages/core/src/repowise/core/ingestion/cobol/source.py
+++ b/packages/core/src/repowise/core/ingestion/cobol/source.py
@@ -6,7 +6,7 @@ import re
from .fragments import is_fragment, is_procedure_fragment, wrap_fragment
from .grammar_gaps import rewrite_gaps
-from .reference_format import is_free_format, normalize_fixed_format
+from .reference_format import is_free_format, normalize_fixed_format, normalize_free_format
from .statements import mask_statements
# A lossless round trip for bytes that are not UTF-8 (EBCDIC-converted Latin-1).
@@ -18,8 +18,7 @@ _INLINE_COMMENT = re.compile(r"('[^'\n]*'|\"[^\"\n]*\")|\*>[^\r\n]*")
def code_view(source: bytes) -> str:
"""*source* as program text only: comments, sequence area and columns 73+ blank."""
text = source.decode(_ENCODING, _ERRORS)
- # Free format keeps its text as written; only fixed format has columns.
- code = text if is_free_format(text) else normalize_fixed_format(text)
+ code = normalize_free_format(text) if is_free_format(text) else normalize_fixed_format(text)
return _INLINE_COMMENT.sub(lambda m: m.group(1) or " " * len(m.group(0)), code)
diff --git a/tests/unit/ingestion/parser/test_cobol.py b/tests/unit/ingestion/parser/test_cobol.py
index 8aad04815..a69f6d052 100644
--- a/tests/unit/ingestion/parser/test_cobol.py
+++ b/tests/unit/ingestion/parser/test_cobol.py
@@ -396,3 +396,31 @@ class TestCobolCallScope:
EXEC CICS XCTL PROGRAM('COCRDLIC') END-EXEC.
"""
assert self._resolve(parser, {"cocrdlic.cbl": caller}, {}) == set()
+
+
+class TestCobolFreeFormat:
+ def test_free_format_program_parses_symbols_and_calls(self, parser: ASTParser) -> None:
+ source = (
+ b">>SOURCE FORMAT FREE\n"
+ b"IDENTIFICATION DIVISION.\n"
+ b"PROGRAM-ID. HELLO.\n"
+ b"DATA DIVISION.\n"
+ b"WORKING-STORAGE SECTION.\n"
+ b"01 WS-A PIC 9. *> a counter\n"
+ b"PROCEDURE DIVISION.\n"
+ b"MAIN-PARA.\n"
+ b" PERFORM SHOW-PARA\n"
+ b" STOP RUN.\n"
+ b"SHOW-PARA.\n"
+ b" DISPLAY \"HI\". *> greet\n"
+ )
+ parsed = parser.parse_file(_cobol("HELLO.cbl"), source)
+ assert parsed.parse_errors == []
+ assert {s.name: s.kind for s in parsed.symbols} == {
+ "HELLO": "module",
+ "WS-A": "variable",
+ "MAIN-PARA": "function",
+ "SHOW-PARA": "function",
+ }
+ assert [c.target_name for c in parsed.calls] == ["SHOW-PARA"]
+
diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.py
index 0a7693668..7c3e9b9db 100644
--- a/tests/unit/ingestion/parser/test_cobol_source.py
+++ b/tests/unit/ingestion/parser/test_cobol_source.py
@@ -230,7 +230,7 @@ def test_floating_comments_hide_statements_and_keep_literals() -> None:
assert copy_statements(code) == []
assert [block.line for block in exec_blocks(code)] == [9]
assert "'*> kept'" in code
- assert len(code) == len(source)
+ assert len(code.splitlines()) == len(source.splitlines())
def test_copy_statements_are_found_in_code_and_not_in_comments() -> None: