Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(cobol): read the procedure of programs whose PROCEDURE DIVISION header comes from a copybook (#4472)

Raghav Chamadiya0259b4484cda
diff --git a/docs/layers/LANGUAGE_SUPPORT.md b/docs/layers/LANGUAGE_SUPPORT.md
index 85cd96143..d2a3c0827 100644
--- a/docs/layers/LANGUAGE_SUPPORT.md
+++ b/docs/layers/LANGUAGE_SUPPORT.md
@@ -85,6 +85,10 @@ COBOL is read in fixed reference format, with `EXEC SQL/CICS/DLI` blocks and
`COPY` statements masked before parsing; a copybook parses as a data or
procedure fragment. `COPY` and `EXEC SQL INCLUDE` members resolve to the
nearest copybook with that name; system copybooks such as `DFHAID` stay external.
+A program with no `PROCEDURE DIVISION` text, whose header comes from a copybook
+(the CICS `COPY CENTRY.` idiom), takes the first `COPY` followed by a statement
+or paragraph header as that header, so its paragraphs, `PERFORM`s and `CALL`s
+are read.
A DB2 DCLGEN member (`.dcl`) is a copybook when it holds an
`EXEC SQL DECLARE ... TABLE` and level-numbered data entries. PL/I DCLGEN
output and OpenVMS DCL procedures share the extension and are not indexed.
diff --git a/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py b/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py
index 4fe4febe6..81b50909b 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/cobol_paragraphs.py
@@ -15,7 +15,8 @@ a paragraph is reported when all of these hold:
* the file holds one program, with no ``DECLARATIVES``, ``ENTRY``,
``IGNORE CONDITION`` or ``REPLACE``, and every procedure-division ``COPY``
(and every ``COPY`` inside those, transitively) resolved and free of
- ``REPLACING``.
+ ``REPLACING``; a copybook that supplies the division header (a CICS
+ ``COPY CENTRY.``) must be read and hold ``PROCEDURE DIVISION``.
Any doubt drops the paragraph (recall is the price). A finding is low
confidence and never deletion-ready.
@@ -27,7 +28,12 @@ import re
from pathlib import PurePosixPath
from typing import TYPE_CHECKING
-from repowise.core.ingestion.cobol import CopyStatement, code_view, copy_statements
+from repowise.core.ingestion.cobol import (
+ CopyStatement,
+ code_view,
+ copy_statements,
+ procedure_division,
+)
from .constants import _is_fixture_path
from .models import DeadCodeFindingData, DeadCodeKind
@@ -98,10 +104,13 @@ def unreachable_paragraphs(code: str, copied: dict[str, str]) -> list[tuple[str,
def _procedure(code: str, copied: dict[str, str]) -> tuple[int, list[CopyStatement]] | None:
"""Where the procedure division starts and its COPYs, or ``None`` if it is not judged."""
- divisions = list(_PROCEDURE.finditer(code))
- if len(divisions) != 1 or _UNSAFE_PROGRAM.search(code):
+ start, header = procedure_division(code)
+ divisions = len(_PROCEDURE.findall(code)) + (header is not None)
+ if divisions != 1 or _UNSAFE_PROGRAM.search(code):
+ return None
+ # A copied header counts only when its member is read and is that header.
+ if header is not None and not _PROCEDURE.match(copied.get(header, "").lstrip()):
return None
- start = divisions[0].end()
line_of_start = code.count("\n", 0, start) + 1
copies = [c for c in copy_statements(code) if c.line >= line_of_start]
return (start, copies) if _closed(copies, copied) else None
diff --git a/packages/core/src/repowise/core/ingestion/cobol/__init__.py b/packages/core/src/repowise/core/ingestion/cobol/__init__.py
index 80f115803..a6b68e6e6 100644
--- a/packages/core/src/repowise/core/ingestion/cobol/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/cobol/__init__.py
@@ -1,7 +1,13 @@
"""COBOL source handling: reference format, EXEC and COPY statements, copybooks."""
from .source import code_view, prepare_cobol_source
-from .statements import CopyStatement, ExecBlock, copy_statements, exec_blocks
+from .statements import (
+ CopyStatement,
+ ExecBlock,
+ copy_statements,
+ exec_blocks,
+ procedure_division,
+)
__all__ = [
"CopyStatement",
@@ -10,4 +16,5 @@ __all__ = [
"copy_statements",
"exec_blocks",
"prepare_cobol_source",
+ "procedure_division",
]
diff --git a/packages/core/src/repowise/core/ingestion/cobol/statements.py b/packages/core/src/repowise/core/ingestion/cobol/statements.py
index 2b04c281d..2610fc867 100644
--- a/packages/core/src/repowise/core/ingestion/cobol/statements.py
+++ b/packages/core/src/repowise/core/ingestion/cobol/statements.py
@@ -28,6 +28,29 @@ _COPY = re.compile(
re.IGNORECASE | re.DOTALL,
)
_PROCEDURE_DIVISION = re.compile(r"\bPROCEDURE\s+DIVISION\b", re.IGNORECASE)
+_DATA_DIVISION = re.compile(r"(?<![\w-])DATA\s+DIVISION\b", re.IGNORECASE)
+# A division or section header of the environment or data division: none
+# follows the procedure division's header.
+_NOT_PROCEDURE_HEADER = re.compile(
+ r"(?<![\w-])(?:(?:ENVIRONMENT|DATA)\s+DIVISION|(?:CONFIGURATION|INPUT-OUTPUT|FILE"
+ r"|WORKING-STORAGE|LOCAL-STORAGE|LINKAGE|COMMUNICATION|REPORT|SCREEN)\s+SECTION)\b",
+ re.IGNORECASE,
+)
+# Words alone on a line with a period that open no paragraph: environment
+# paragraphs, listing directives, ``DECLARATIVES``.
+_NOT_A_PARAGRAPH = (
+ r"(?:FILE-CONTROL|I-O-CONTROL|SPECIAL-NAMES|SOURCE-COMPUTER|OBJECT-COMPUTER|REPOSITORY"
+ r"|EJECT|SKIP[123]|TITLE|DECLARATIVES)[ \t]*\."
+)
+# What can follow a COPY only in the procedure division: a verb, ``EXEC CICS``,
+# or a paragraph or section header alone on its line. A data COPY is followed
+# by an entry, a section header, another COPY or nothing.
+_STATEMENT_TEXT = re.compile(
+ r"\s*(?:(?:ACCEPT|ADD|CALL|COMPUTE|CONTINUE|DISPLAY|EVALUATE|GO|GOBACK|IF|INITIALIZE"
+ r"|INSPECT|MOVE|OPEN|PERFORM|READ|SET|STRING|WRITE)(?![\w-])|EXEC\s+CICS\b"
+ rf"|(?!{_NOT_A_PARAGRAPH}){_NAME}(?:\s+SECTION)?[ \t]*\.[ \t]*\r?$)",
+ re.IGNORECASE | re.MULTILINE,
+)
_CLOSING_PERIOD = re.compile(r"[ \t]*\.")
_FILLER = "CONTINUE"
_DATA_PLACEHOLDER = "01 FILLER PIC X."
@@ -100,16 +123,42 @@ def copy_statements(code: str) -> list[CopyStatement]:
return [statement for _, statement in sorted(found, key=lambda item: item[0])]
+def procedure_division(code: str) -> tuple[int, str | None]:
+ """Where *code*'s procedure division opens (``len(code)`` without one), and
+ the member of the COPY that supplies its header, if a copybook does.
+
+ A CICS program can copy its header (``COPY CENTRY.``, a member holding only
+ ``PROCEDURE DIVISION.``). With no header text, the first COPY after the
+ ``DATA DIVISION`` header that is followed by statement text, and by no
+ environment or data division or section header, is taken as the header.
+ Ceiling: the member's own text is not read, since a parse sees one file;
+ reading it would thread the resolved copybooks into every parse. A header
+ COPY followed by another COPY, an ``EXEC SQL`` or a verb not listed, or in
+ a program with no ``DATA DIVISION`` or with a nested program, is missed.
+ """
+ division = _PROCEDURE_DIVISION.search(code)
+ if division:
+ return division.end(), None
+ data = _DATA_DIVISION.search(code)
+ for m in _COPY.finditer(code, data.end() if data else len(code)):
+ if _STATEMENT_TEXT.match(code, m.end()) and not _NOT_PROCEDURE_HEADER.search(
+ code, m.end()
+ ):
+ return m.end(), _member_name(m.group("q1") or m.group("q2") or m.group("word"))
+ return len(code), None
+
+
def mask_statements(code: str, procedure_from: int | None = None) -> str:
"""Blank COPY statements and EXEC blocks, keeping every line and offset.
An EXEC block inside the procedure division becomes ``CONTINUE`` so the
statement it stood for still parses; elsewhere it is blanked. A procedure
- copybook passes ``procedure_from=0``.
+ copybook passes ``procedure_from=0``. A COPY supplying the division header
+ becomes that header.
"""
+ header = None
if procedure_from is None:
- division = _PROCEDURE_DIVISION.search(code)
- procedure_from = division.end() if division else len(code)
+ procedure_from, header = procedure_division(code)
chars = list(code)
for block in exec_blocks(code):
end = block.end
@@ -127,7 +176,10 @@ def mask_statements(code: str, procedure_from: int | None = None) -> str:
for m in reversed(list(_COPY.finditer(masked))):
newlines = "".join(c for c in m.group(0) if c in "\r\n")
# In data, an FD or group header needs the record the copybook supplied.
- text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""
+ if header and m.end() == procedure_from:
+ text = "PROCEDURE DIVISION."
+ else:
+ text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""
masked = masked[: m.start()] + text + newlines + masked[m.end() :]
return masked
diff --git a/scripts/cobol_acceptance.py b/scripts/cobol_acceptance.py
index 58cb92db0..0e634d134 100644
--- a/scripts/cobol_acceptance.py
+++ b/scripts/cobol_acceptance.py
@@ -29,6 +29,10 @@ _CSD_EXTS = {".csd"}
_CSD_COMMAND = re.compile(r"^\s*(?:DEFINE|ADD|ALTER|APPEND|COPY|DELETE|LIST|REMOVE)\s", re.I | re.M)
_NOT_A_PARAGRAPH = {"UNTIL", "VARYING", "WITH", "TEST", "FOREVER", "THRU", "THROUGH"}
_STRUCTURAL = ("calls", "imports", "framework")
+_NOT_PROCEDURE = (
+ r"^\s*(?:(?:IDENTIFICATION|ID|ENVIRONMENT|DATA)\s+DIVISION|PROGRAM-ID"
+ r"|(?:FILE|WORKING-STORAGE|LOCAL-STORAGE|LINKAGE)\s+SECTION)\s*\."
+)
def _walk(root: str) -> dict[str, dict[str, list[str]]]:
@@ -78,6 +82,17 @@ def truth(root: str) -> dict:
transactions = _csd_truth(root, files, progs, pairs := collections.defaultdict(set))
paragraphs: set[tuple[str, str]] = set()
performs: set[tuple[str, str]] = set()
+ # A copybook whose code is the division header (a CICS ``COPY CENTRY.``) opens it too.
+ headers = {
+ stem
+ for stem, rels in cpys.items()
+ for rel in rels
+ if re.match(
+ r"\s*PROCEDURE\s+DIVISION\b",
+ "\n".join(t for _, _, t in _code_lines(os.path.join(root, rel))),
+ re.I,
+ )
+ }
for rels in progs.values():
for rel in rels:
@@ -86,13 +101,19 @@ def truth(root: str) -> dict:
own: set[str] = set()
in_proc = False
for _, area_a, body in lines:
- if re.search(r"PROCEDURE\s+DIVISION", body, re.I):
+ copied = re.match(r"\s*COPY\s+'?([A-Z0-9#@$-]+)", body, re.I)
+ if re.search(r"PROCEDURE\s+DIVISION", body, re.I) or (
+ copied and copied.group(1).upper() in headers
+ ):
in_proc = True
continue
+ # A nested program's header and data division close the procedure.
+ if re.search(_NOT_PROCEDURE, body, re.I):
+ in_proc = False
if not in_proc:
continue
m = re.match(r"([A-Z0-9][A-Z0-9-]*)(\s+SECTION)?\s*\.\s*$", body.rstrip(), re.I)
- if m and area_a != " ":
+ if m and area_a != " " and m.group(1).upper() != "DECLARATIVES":
own.add(m.group(1).upper())
paragraphs |= {(rel, p) for p in own}
for m in re.finditer(r"\bPERFORM\s+([A-Z0-9][A-Z0-9-]*)", text, re.I):
@@ -230,7 +251,10 @@ def index_view(db: str) -> dict:
edges = con.execute(
"select source_node_id, target_node_id, edge_type from graph_edges"
).fetchall()
- symbols = [r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol"]
+ # The per-file ``__module__`` node is graph plumbing, not a symbol ``wiki_symbols`` holds.
+ symbols = [
+ r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol" and r[4] != "__module__"
+ ]
procs = {(r[3], r[4].upper()) for r in symbols if r[5] == "function"}
same_file_calls: set[tuple[str, str]] = set()
cross_perform: list[tuple[str, str]] = []
diff --git a/tests/unit/dead_code/test_cobol_external_entry.py b/tests/unit/dead_code/test_cobol_external_entry.py
index 5a80a4acc..edcff0605 100644
--- a/tests/unit/dead_code/test_cobol_external_entry.py
+++ b/tests/unit/dead_code/test_cobol_external_entry.py
@@ -209,3 +209,28 @@ def test_end_program_is_a_paragraph_name_not_a_scope_terminator() -> None:
" EXEC CICS RETURN END-EXEC.\n MAINLINE-EXIT.\n EXIT.\n"
)
assert _unreachable(body) == ["MAINLINE-EXIT"]
+
+
+def test_a_program_whose_procedure_header_is_copied_is_judged() -> None:
+ from repowise.core.analysis.dead_code.cobol_paragraphs import unreachable_paragraphs
+
+ code = (
+ " IDENTIFICATION DIVISION.\n"
+ " PROGRAM-ID. CICSPGM.\n"
+ " DATA DIVISION.\n"
+ " LINKAGE SECTION.\n"
+ " 01 DFHCOMMAREA PIC X.\n"
+ " COPY CENTRY.\n"
+ " PERFORM WORK-PARA.\n"
+ " GOBACK.\n"
+ " WORK-PARA.\n"
+ " GOBACK.\n"
+ " LOST-PARA.\n"
+ " EXIT.\n"
+ )
+ header = {"CENTRY": " PROCEDURE DIVISION.\n"}
+
+ assert unreachable_paragraphs(code, header) == [("LOST-PARA", 11, 12)]
+ # Unresolved, or a member that is not the header: the program is not judged.
+ assert unreachable_paragraphs(code, {}) == []
+ assert unreachable_paragraphs(code, {"CENTRY": " MOVE 1 TO A.\n"}) == []
diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.py
index 1b10d947e..0a7693668 100644
--- a/tests/unit/ingestion/parser/test_cobol_source.py
+++ b/tests/unit/ingestion/parser/test_cobol_source.py
@@ -261,3 +261,126 @@ def test_copy_statements_are_found_in_code_and_not_in_comments() -> None:
("MEMBER1", "MYLIB", 10, False, False),
("DCLTRTYP", None, 12, False, True),
]
+
+
+def _centry_program(after_linkage: str) -> bytes:
+ lines = [
+ " IDENTIFICATION DIVISION.",
+ " PROGRAM-ID. DEMO.",
+ " DATA DIVISION.",
+ " WORKING-STORAGE SECTION.",
+ " 01 WS-AREA.",
+ " COPY CAREA.",
+ " 01 WS-A PIC X(10).",
+ " LINKAGE SECTION.",
+ " 01 DFHCOMMAREA PIC X(10).",
+ after_linkage,
+ ]
+ return ("\n".join(lines) + "\n").encode()
+
+
+def test_a_copybook_supplying_the_procedure_header_opens_the_procedure() -> None:
+ # The CICS idiom: ``CENTRY`` holds only ``PROCEDURE DIVISION.``.
+ source = _centry_program(
+ " COPY CENTRY.\n"
+ " MOVE DFHCOMMAREA TO WS-A.\n"
+ " CALL 'SSECURE' USING WS-A.\n"
+ " PERFORM WORK-PARA.\n"
+ " COPY CRETURN.\n"
+ " WORK-PARA.\n"
+ " MOVE SPACES TO WS-A.\n"
+ )
+
+ parsed = _parse(source)
+
+ assert parsed.parse_errors == []
+ assert _names(parsed)["WORK-PARA"] == 15
+ assert {"SSECURE", "WORK-PARA"} <= {call.target_name for call in parsed.calls}
+ assert [i.module_path for i in parsed.imports] == ["CAREA", "CENTRY", "CRETURN"]
+
+
+def test_a_header_copybook_followed_by_a_paragraph_opens_the_procedure() -> None:
+ parsed = _parse(_centry_program(" COPY CENTRY.\n MAIN-PARA.\n GOBACK.\n"))
+
+ assert parsed.parse_errors == []
+ assert _names(parsed)["MAIN-PARA"] == 11
+
+
+ "after_linkage",
+ [
+ # A LINKAGE copybook, then more data.
+ " COPY CLINK.\n 01 LK-B PIC X.\n",
+ # A copybook that ends the data division with nothing after it.
+ " COPY CLINK.\n",
+ # A copybook followed by another data section.
+ " COPY CLINK.\n LOCAL-STORAGE SECTION.\n 01 LS-A PIC X.\n",
+ # Copybooks in a row: the next one is not statement text.
+ " COPY CLINK.\n COPY CLINK2.\n",
+ # Listing directives with a period are no paragraph headers.
+ " COPY CLINK.\n EJECT.\n",
+ " COPY CLINK.\n SKIP1.\n",
+ ],
+ ids=["more-data", "nothing-after", "data-section", "copy-after-copy", "eject", "skip1"],
+)
+def test_a_data_division_copybook_never_opens_the_procedure(after_linkage: str) -> None:
+ from repowise.core.ingestion.cobol import prepare_cobol_source
+
+ prepared = prepare_cobol_source(_centry_program(after_linkage)).decode()
+
+ assert "PROCEDURE" not in prepared
+ assert prepared.count("01 FILLER PIC X.") == after_linkage.count("COPY") + 1
+
+
+ "environment",
+ [
+ " INPUT-OUTPUT SECTION.\n COPY CFC.\n FILE-CONTROL.\n",
+ " CONFIGURATION SECTION.\n COPY CSRC.\n SPECIAL-NAMES.\n",
+ " CONFIGURATION SECTION.\n COPY CSRC.\n OBJECT-COMPUTER.\n",
+ " INPUT-OUTPUT SECTION.\n COPY CSEL.\n I-O-CONTROL.\n",
+ ],
+ ids=["file-control", "special-names", "object-computer", "i-o-control"],
+)
[email protected]("data", [True, False], ids=["with-data", "no-data"])
+def test_an_environment_division_copybook_never_opens_the_procedure(
+ environment: str, data: bool
+) -> None:
+ from repowise.core.ingestion.cobol import prepare_cobol_source
+
+ data_division = " DATA DIVISION.\n WORKING-STORAGE SECTION.\n 01 A PIC X.\n"
+ source = (
+ " IDENTIFICATION DIVISION.\n PROGRAM-ID. DEMO.\n"
+ " ENVIRONMENT DIVISION.\n" + environment + (data_division if data else "")
+ )
+
+ assert "PROCEDURE" not in prepare_cobol_source(source.encode()).decode()
+
+
+def test_a_program_with_its_procedure_header_prepares_as_before() -> None:
+ from repowise.core.ingestion.cobol import prepare_cobol_source
+
+ source = (
+ b" IDENTIFICATION DIVISION.\n"
+ b" PROGRAM-ID. DEMO.\n"
+ b" DATA DIVISION.\n"
+ b" WORKING-STORAGE SECTION.\n"
+ b" COPY CAREA.\n"
+ b" PROCEDURE DIVISION.\n"
+ b" COPY CENTRY.\n"
+ b" MOVE 1 TO A.\n"
+ b" EXEC CICS RETURN END-EXEC.\n"
+ )
+
+ # The data COPY becomes a placeholder entry; the procedure COPY is blanked.
+ assert prepare_cobol_source(source) == (
+ b" IDENTIFICATION DIVISION.\n"
+ b" PROGRAM-ID. DEMO.\n"
+ b" DATA DIVISION.\n"
+ b" WORKING-STORAGE SECTION.\n"
+ b" 01 FILLER PIC X.\n"
+ b" PROCEDURE DIVISION.\n"
+ b" \n"
+ b" MOVE 1 TO A.\n"
+ b" CONTINUE .\n"
+ )