Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(cobol): read the procedure of programs whose PROCEDURE DIVISION header comes from a copybook (#4472)
Raghav Chamadiya0259b4484cda
diff --git a/docs/layers/LANGUAGE_SUPPORT.md b/docs/layers/LANGUAGE_SUPPORT.mdindex 85cd96143..d2a3c0827 100644--- a/docs/layers/LANGUAGE_SUPPORT.md+++ b/docs/layers/LANGUAGE_SUPPORT.md@@ -85,6 +85,10 @@ COBOL is read in fixed reference format, with `EXEC SQL/CICS/DLI` blocks and`COPY` statements masked before parsing; a copybook parses as a data orprocedure fragment. `COPY` and `EXEC SQL INCLUDE` members resolve to thenearest copybook with that name; system copybooks such as `DFHAID` stay external.+A program with no `PROCEDURE DIVISION` text, whose header comes from a copybook+(the CICS `COPY CENTRY.` idiom), takes the first `COPY` followed by a statement+or paragraph header as that header, so its paragraphs, `PERFORM`s and `CALL`s+are read.A DB2 DCLGEN member (`.dcl`) is a copybook when it holds an`EXEC SQL DECLARE ... TABLE` and level-numbered data entries. PL/I DCLGENoutput and OpenVMS DCL procedures share the extension and are not indexed.diff --git a/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py b/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.pyindex 4fe4febe6..81b50909b 100644--- a/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py+++ b/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py@@ -15,7 +15,8 @@ a paragraph is reported when all of these hold:* the file holds one program, with no ``DECLARATIVES``, ``ENTRY``,``IGNORE CONDITION`` or ``REPLACE``, and every procedure-division ``COPY``(and every ``COPY`` inside those, transitively) resolved and free of- ``REPLACING``.+ ``REPLACING``; a copybook that supplies the division header (a CICS+ ``COPY CENTRY.``) must be read and hold ``PROCEDURE DIVISION``.Any doubt drops the paragraph (recall is the price). A finding is lowconfidence and never deletion-ready.@@ -27,7 +28,12 @@ import refrom pathlib import PurePosixPathfrom typing import TYPE_CHECKING-from repowise.core.ingestion.cobol import CopyStatement, code_view, copy_statements+from repowise.core.ingestion.cobol import (+ CopyStatement,+ code_view,+ copy_statements,+ procedure_division,+)from .constants import _is_fixture_pathfrom .models import DeadCodeFindingData, DeadCodeKind@@ -98,10 +104,13 @@ def unreachable_paragraphs(code: str, copied: dict[str, str]) -> list[tuple[str,def _procedure(code: str, copied: dict[str, str]) -> tuple[int, list[CopyStatement]] | None:"""Where the procedure division starts and its COPYs, or ``None`` if it is not judged."""- divisions = list(_PROCEDURE.finditer(code))- if len(divisions) != 1 or _UNSAFE_PROGRAM.search(code):+ start, header = procedure_division(code)+ divisions = len(_PROCEDURE.findall(code)) + (header is not None)+ if divisions != 1 or _UNSAFE_PROGRAM.search(code):+ return None+ # A copied header counts only when its member is read and is that header.+ if header is not None and not _PROCEDURE.match(copied.get(header, "").lstrip()):return None- start = divisions[0].end()line_of_start = code.count("\n", 0, start) + 1copies = [c for c in copy_statements(code) if c.line >= line_of_start]return (start, copies) if _closed(copies, copied) else Nonediff --git a/packages/core/src/repowise/core/ingestion/cobol/__init__.py b/packages/core/src/repowise/core/ingestion/cobol/__init__.pyindex 80f115803..a6b68e6e6 100644--- a/packages/core/src/repowise/core/ingestion/cobol/__init__.py+++ b/packages/core/src/repowise/core/ingestion/cobol/__init__.py@@ -1,7 +1,13 @@"""COBOL source handling: reference format, EXEC and COPY statements, copybooks."""from .source import code_view, prepare_cobol_source-from .statements import CopyStatement, ExecBlock, copy_statements, exec_blocks+from .statements import (+ CopyStatement,+ ExecBlock,+ copy_statements,+ exec_blocks,+ procedure_division,+)__all__ = ["CopyStatement",@@ -10,4 +16,5 @@ __all__ = ["copy_statements","exec_blocks","prepare_cobol_source",+ "procedure_division",]diff --git a/packages/core/src/repowise/core/ingestion/cobol/statements.py b/packages/core/src/repowise/core/ingestion/cobol/statements.pyindex 2b04c281d..2610fc867 100644--- a/packages/core/src/repowise/core/ingestion/cobol/statements.py+++ b/packages/core/src/repowise/core/ingestion/cobol/statements.py@@ -28,6 +28,29 @@ _COPY = re.compile(re.IGNORECASE | re.DOTALL,)_PROCEDURE_DIVISION = re.compile(r"\bPROCEDURE\s+DIVISION\b", re.IGNORECASE)+_DATA_DIVISION = re.compile(r"(?<![\w-])DATA\s+DIVISION\b", re.IGNORECASE)+# A division or section header of the environment or data division: none+# follows the procedure division's header.+_NOT_PROCEDURE_HEADER = re.compile(+ r"(?<![\w-])(?:(?:ENVIRONMENT|DATA)\s+DIVISION|(?:CONFIGURATION|INPUT-OUTPUT|FILE"+ r"|WORKING-STORAGE|LOCAL-STORAGE|LINKAGE|COMMUNICATION|REPORT|SCREEN)\s+SECTION)\b",+ re.IGNORECASE,+)+# Words alone on a line with a period that open no paragraph: environment+# paragraphs, listing directives, ``DECLARATIVES``.+_NOT_A_PARAGRAPH = (+ r"(?:FILE-CONTROL|I-O-CONTROL|SPECIAL-NAMES|SOURCE-COMPUTER|OBJECT-COMPUTER|REPOSITORY"+ r"|EJECT|SKIP[123]|TITLE|DECLARATIVES)[ \t]*\."+)+# What can follow a COPY only in the procedure division: a verb, ``EXEC CICS``,+# or a paragraph or section header alone on its line. A data COPY is followed+# by an entry, a section header, another COPY or nothing.+_STATEMENT_TEXT = re.compile(+ r"\s*(?:(?:ACCEPT|ADD|CALL|COMPUTE|CONTINUE|DISPLAY|EVALUATE|GO|GOBACK|IF|INITIALIZE"+ r"|INSPECT|MOVE|OPEN|PERFORM|READ|SET|STRING|WRITE)(?![\w-])|EXEC\s+CICS\b"+ rf"|(?!{_NOT_A_PARAGRAPH}){_NAME}(?:\s+SECTION)?[ \t]*\.[ \t]*\r?$)",+ re.IGNORECASE | re.MULTILINE,+)_CLOSING_PERIOD = re.compile(r"[ \t]*\.")_FILLER = "CONTINUE"_DATA_PLACEHOLDER = "01 FILLER PIC X."@@ -100,16 +123,42 @@ def copy_statements(code: str) -> list[CopyStatement]:return [statement for _, statement in sorted(found, key=lambda item: item[0])]+def procedure_division(code: str) -> tuple[int, str | None]:+ """Where *code*'s procedure division opens (``len(code)`` without one), and+ the member of the COPY that supplies its header, if a copybook does.++ A CICS program can copy its header (``COPY CENTRY.``, a member holding only+ ``PROCEDURE DIVISION.``). With no header text, the first COPY after the+ ``DATA DIVISION`` header that is followed by statement text, and by no+ environment or data division or section header, is taken as the header.+ Ceiling: the member's own text is not read, since a parse sees one file;+ reading it would thread the resolved copybooks into every parse. A header+ COPY followed by another COPY, an ``EXEC SQL`` or a verb not listed, or in+ a program with no ``DATA DIVISION`` or with a nested program, is missed.+ """+ division = _PROCEDURE_DIVISION.search(code)+ if division:+ return division.end(), None+ data = _DATA_DIVISION.search(code)+ for m in _COPY.finditer(code, data.end() if data else len(code)):+ if _STATEMENT_TEXT.match(code, m.end()) and not _NOT_PROCEDURE_HEADER.search(+ code, m.end()+ ):+ return m.end(), _member_name(m.group("q1") or m.group("q2") or m.group("word"))+ return len(code), None++def mask_statements(code: str, procedure_from: int | None = None) -> str:"""Blank COPY statements and EXEC blocks, keeping every line and offset.An EXEC block inside the procedure division becomes ``CONTINUE`` so thestatement it stood for still parses; elsewhere it is blanked. A procedure- copybook passes ``procedure_from=0``.+ copybook passes ``procedure_from=0``. A COPY supplying the division header+ becomes that header."""+ header = Noneif procedure_from is None:- division = _PROCEDURE_DIVISION.search(code)- procedure_from = division.end() if division else len(code)+ procedure_from, header = procedure_division(code)chars = list(code)for block in exec_blocks(code):end = block.end@@ -127,7 +176,10 @@ def mask_statements(code: str, procedure_from: int | None = None) -> str:for m in reversed(list(_COPY.finditer(masked))):newlines = "".join(c for c in m.group(0) if c in "\r\n")# In data, an FD or group header needs the record the copybook supplied.- text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""+ if header and m.end() == procedure_from:+ text = "PROCEDURE DIVISION."+ else:+ text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""masked = masked[: m.start()] + text + newlines + masked[m.end() :]return maskeddiff --git a/scripts/cobol_acceptance.py b/scripts/cobol_acceptance.pyindex 58cb92db0..0e634d134 100644--- a/scripts/cobol_acceptance.py+++ b/scripts/cobol_acceptance.py@@ -29,6 +29,10 @@ _CSD_EXTS = {".csd"}_CSD_COMMAND = re.compile(r"^\s*(?:DEFINE|ADD|ALTER|APPEND|COPY|DELETE|LIST|REMOVE)\s", re.I | re.M)_NOT_A_PARAGRAPH = {"UNTIL", "VARYING", "WITH", "TEST", "FOREVER", "THRU", "THROUGH"}_STRUCTURAL = ("calls", "imports", "framework")+_NOT_PROCEDURE = (+ r"^\s*(?:(?:IDENTIFICATION|ID|ENVIRONMENT|DATA)\s+DIVISION|PROGRAM-ID"+ r"|(?:FILE|WORKING-STORAGE|LOCAL-STORAGE|LINKAGE)\s+SECTION)\s*\."+)def _walk(root: str) -> dict[str, dict[str, list[str]]]:@@ -78,6 +82,17 @@ def truth(root: str) -> dict:transactions = _csd_truth(root, files, progs, pairs := collections.defaultdict(set))paragraphs: set[tuple[str, str]] = set()performs: set[tuple[str, str]] = set()+ # A copybook whose code is the division header (a CICS ``COPY CENTRY.``) opens it too.+ headers = {+ stem+ for stem, rels in cpys.items()+ for rel in rels+ if re.match(+ r"\s*PROCEDURE\s+DIVISION\b",+ "\n".join(t for _, _, t in _code_lines(os.path.join(root, rel))),+ re.I,+ )+ }for rels in progs.values():for rel in rels:@@ -86,13 +101,19 @@ def truth(root: str) -> dict:own: set[str] = set()in_proc = Falsefor _, area_a, body in lines:- if re.search(r"PROCEDURE\s+DIVISION", body, re.I):+ copied = re.match(r"\s*COPY\s+'?([A-Z0-9#@$-]+)", body, re.I)+ if re.search(r"PROCEDURE\s+DIVISION", body, re.I) or (+ copied and copied.group(1).upper() in headers+ ):in_proc = Truecontinue+ # A nested program's header and data division close the procedure.+ if re.search(_NOT_PROCEDURE, body, re.I):+ in_proc = Falseif not in_proc:continuem = re.match(r"([A-Z0-9][A-Z0-9-]*)(\s+SECTION)?\s*\.\s*$", body.rstrip(), re.I)- if m and area_a != " ":+ if m and area_a != " " and m.group(1).upper() != "DECLARATIVES":own.add(m.group(1).upper())paragraphs |= {(rel, p) for p in own}for m in re.finditer(r"\bPERFORM\s+([A-Z0-9][A-Z0-9-]*)", text, re.I):@@ -230,7 +251,10 @@ def index_view(db: str) -> dict:edges = con.execute("select source_node_id, target_node_id, edge_type from graph_edges").fetchall()- symbols = [r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol"]+ # The per-file ``__module__`` node is graph plumbing, not a symbol ``wiki_symbols`` holds.+ symbols = [+ r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol" and r[4] != "__module__"+ ]procs = {(r[3], r[4].upper()) for r in symbols if r[5] == "function"}same_file_calls: set[tuple[str, str]] = set()cross_perform: list[tuple[str, str]] = []diff --git a/tests/unit/dead_code/test_cobol_external_entry.py b/tests/unit/dead_code/test_cobol_external_entry.pyindex 5a80a4acc..edcff0605 100644--- a/tests/unit/dead_code/test_cobol_external_entry.py+++ b/tests/unit/dead_code/test_cobol_external_entry.py@@ -209,3 +209,28 @@ def test_end_program_is_a_paragraph_name_not_a_scope_terminator() -> None:" EXEC CICS RETURN END-EXEC.\n MAINLINE-EXIT.\n EXIT.\n")assert _unreachable(body) == ["MAINLINE-EXIT"]+++def test_a_program_whose_procedure_header_is_copied_is_judged() -> None:+ from repowise.core.analysis.dead_code.cobol_paragraphs import unreachable_paragraphs++ code = (+ " IDENTIFICATION DIVISION.\n"+ " PROGRAM-ID. CICSPGM.\n"+ " DATA DIVISION.\n"+ " LINKAGE SECTION.\n"+ " 01 DFHCOMMAREA PIC X.\n"+ " COPY CENTRY.\n"+ " PERFORM WORK-PARA.\n"+ " GOBACK.\n"+ " WORK-PARA.\n"+ " GOBACK.\n"+ " LOST-PARA.\n"+ " EXIT.\n"+ )+ header = {"CENTRY": " PROCEDURE DIVISION.\n"}++ assert unreachable_paragraphs(code, header) == [("LOST-PARA", 11, 12)]+ # Unresolved, or a member that is not the header: the program is not judged.+ assert unreachable_paragraphs(code, {}) == []+ assert unreachable_paragraphs(code, {"CENTRY": " MOVE 1 TO A.\n"}) == []diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.pyindex 1b10d947e..0a7693668 100644--- a/tests/unit/ingestion/parser/test_cobol_source.py+++ b/tests/unit/ingestion/parser/test_cobol_source.py@@ -261,3 +261,126 @@ def test_copy_statements_are_found_in_code_and_not_in_comments() -> None:("MEMBER1", "MYLIB", 10, False, False),("DCLTRTYP", None, 12, False, True),]+++def _centry_program(after_linkage: str) -> bytes:+ lines = [+ " IDENTIFICATION DIVISION.",+ " PROGRAM-ID. DEMO.",+ " DATA DIVISION.",+ " WORKING-STORAGE SECTION.",+ " 01 WS-AREA.",+ " COPY CAREA.",+ " 01 WS-A PIC X(10).",+ " LINKAGE SECTION.",+ " 01 DFHCOMMAREA PIC X(10).",+ after_linkage,+ ]+ return ("\n".join(lines) + "\n").encode()+++def test_a_copybook_supplying_the_procedure_header_opens_the_procedure() -> None:+ # The CICS idiom: ``CENTRY`` holds only ``PROCEDURE DIVISION.``.+ source = _centry_program(+ " COPY CENTRY.\n"+ " MOVE DFHCOMMAREA TO WS-A.\n"+ " CALL 'SSECURE' USING WS-A.\n"+ " PERFORM WORK-PARA.\n"+ " COPY CRETURN.\n"+ " WORK-PARA.\n"+ " MOVE SPACES TO WS-A.\n"+ )++ parsed = _parse(source)++ assert parsed.parse_errors == []+ assert _names(parsed)["WORK-PARA"] == 15+ assert {"SSECURE", "WORK-PARA"} <= {call.target_name for call in parsed.calls}+ assert [i.module_path for i in parsed.imports] == ["CAREA", "CENTRY", "CRETURN"]+++def test_a_header_copybook_followed_by_a_paragraph_opens_the_procedure() -> None:+ parsed = _parse(_centry_program(" COPY CENTRY.\n MAIN-PARA.\n GOBACK.\n"))++ assert parsed.parse_errors == []+ assert _names(parsed)["MAIN-PARA"] == 11+++ "after_linkage",+ [+ # A LINKAGE copybook, then more data.+ " COPY CLINK.\n 01 LK-B PIC X.\n",+ # A copybook that ends the data division with nothing after it.+ " COPY CLINK.\n",+ # A copybook followed by another data section.+ " COPY CLINK.\n LOCAL-STORAGE SECTION.\n 01 LS-A PIC X.\n",+ # Copybooks in a row: the next one is not statement text.+ " COPY CLINK.\n COPY CLINK2.\n",+ # Listing directives with a period are no paragraph headers.+ " COPY CLINK.\n EJECT.\n",+ " COPY CLINK.\n SKIP1.\n",+ ],+ ids=["more-data", "nothing-after", "data-section", "copy-after-copy", "eject", "skip1"],+)+def test_a_data_division_copybook_never_opens_the_procedure(after_linkage: str) -> None:+ from repowise.core.ingestion.cobol import prepare_cobol_source++ prepared = prepare_cobol_source(_centry_program(after_linkage)).decode()++ assert "PROCEDURE" not in prepared+ assert prepared.count("01 FILLER PIC X.") == after_linkage.count("COPY") + 1+++ "environment",+ [+ " INPUT-OUTPUT SECTION.\n COPY CFC.\n FILE-CONTROL.\n",+ " CONFIGURATION SECTION.\n COPY CSRC.\n SPECIAL-NAMES.\n",+ " CONFIGURATION SECTION.\n COPY CSRC.\n OBJECT-COMPUTER.\n",+ " INPUT-OUTPUT SECTION.\n COPY CSEL.\n I-O-CONTROL.\n",+ ],+ ids=["file-control", "special-names", "object-computer", "i-o-control"],+)[email protected]("data", [True, False], ids=["with-data", "no-data"])+def test_an_environment_division_copybook_never_opens_the_procedure(+ environment: str, data: bool+) -> None:+ from repowise.core.ingestion.cobol import prepare_cobol_source++ data_division = " DATA DIVISION.\n WORKING-STORAGE SECTION.\n 01 A PIC X.\n"+ source = (+ " IDENTIFICATION DIVISION.\n PROGRAM-ID. DEMO.\n"+ " ENVIRONMENT DIVISION.\n" + environment + (data_division if data else "")+ )++ assert "PROCEDURE" not in prepare_cobol_source(source.encode()).decode()+++def test_a_program_with_its_procedure_header_prepares_as_before() -> None:+ from repowise.core.ingestion.cobol import prepare_cobol_source++ source = (+ b" IDENTIFICATION DIVISION.\n"+ b" PROGRAM-ID. DEMO.\n"+ b" DATA DIVISION.\n"+ b" WORKING-STORAGE SECTION.\n"+ b" COPY CAREA.\n"+ b" PROCEDURE DIVISION.\n"+ b" COPY CENTRY.\n"+ b" MOVE 1 TO A.\n"+ b" EXEC CICS RETURN END-EXEC.\n"+ )++ # The data COPY becomes a placeholder entry; the procedure COPY is blanked.+ assert prepare_cobol_source(source) == (+ b" IDENTIFICATION DIVISION.\n"+ b" PROGRAM-ID. DEMO.\n"+ b" DATA DIVISION.\n"+ b" WORKING-STORAGE SECTION.\n"+ b" 01 FILLER PIC X.\n"+ b" PROCEDURE DIVISION.\n"+ b" \n"+ b" MOVE 1 TO A.\n"+ b" CONTINUE .\n"+ )