Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(cobol): parse fixed format, embedded EXEC blocks and copybooks without losing paragraphs (#4434)

Raghav Chamadiyacfdbf0dbffa1
diff --git a/docs/layers/LANGUAGE_SUPPORT.md b/docs/layers/LANGUAGE_SUPPORT.md
index d02a0a73b..b011e3708 100644
--- a/docs/layers/LANGUAGE_SUPPORT.md
+++ b/docs/layers/LANGUAGE_SUPPORT.md
@@ -77,6 +77,9 @@ calls are entry paths the repository graph cannot observe.
The COBOL and Apex grammars download on first use; offline, set
`REPOWISE_GRAMMAR_DIR` to a pre-staged grammar cache. `repowise doctor` reports
whether it is cached.
+COBOL is read in fixed reference format, with `EXEC SQL/CICS/DLI` blocks and
+`COPY` statements masked before parsing; a copybook parses as a data or
+procedure fragment.
Apex (`.cls`, `.trigger`, `.apex`) is parsed for symbols, heritage and calls,
resolved case-insensitively within each sfdx project's namespace; its dead-code
findings are review-only. Apex test coverage from the Salesforce CLI is read as a
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py
index 388c1d581..54409d40b 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py
@@ -239,7 +239,8 @@ def _runtime_grammar_rows(repo_path: _DoctorPath) -> list[DoctorCheck]:
}
try:
listed = subprocess.run(
- ["git", "ls-files", "-z"],
+ # Untracked, unignored files too: ingestion walks them as well.
+ ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
cwd=repo_path,
capture_output=True,
check=True,
diff --git a/packages/core/src/repowise/core/ingestion/cobol/__init__.py b/packages/core/src/repowise/core/ingestion/cobol/__init__.py
new file mode 100644
index 000000000..80f115803
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/__init__.py
@@ -0,0 +1,13 @@
+"""COBOL source handling: reference format, EXEC and COPY statements, copybooks."""
+
+from .source import code_view, prepare_cobol_source
+from .statements import CopyStatement, ExecBlock, copy_statements, exec_blocks
+
+__all__ = [
+ "CopyStatement",
+ "ExecBlock",
+ "code_view",
+ "copy_statements",
+ "exec_blocks",
+ "prepare_cobol_source",
+]
diff --git a/packages/core/src/repowise/core/ingestion/cobol/fragments.py b/packages/core/src/repowise/core/ingestion/cobol/fragments.py
new file mode 100644
index 000000000..dd359db78
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/fragments.py
@@ -0,0 +1,49 @@
+"""Copybooks parse as fragments inside a synthetic two-line program header.
+
+The header overwrites two blank lines (comments are already blank) ahead of
+the first code line, so every line keeps its number. A copybook without two
+such lines parses unwrapped. ``FRAGMENT_PROGRAM`` names the synthetic program,
+and the symbol extractor drops it.
+"""
+
+from __future__ import annotations
+
+import re
+
+FRAGMENT_PROGRAM = "REPOWISE-COPYBOOK"
+
+_PROGRAM_HEADER = re.compile(r"\b(IDENTIFICATION|ID)\s+DIVISION\b|\bPROGRAM-ID\b", re.I)
+_LEVEL_NUMBER = re.compile(r"\s*\d{1,2}\s")
+_HEADER = " IDENTIFICATION DIVISION. PROGRAM-ID. " + FRAGMENT_PROGRAM + "."
+_DATA = " DATA DIVISION. WORKING-STORAGE SECTION."
+_PROCEDURE = " PROCEDURE DIVISION."
+
+
+def is_fragment(code: str) -> bool:
+ """A member with no program header: a copybook, whatever its extension."""
+ return code.strip() != "" and _PROGRAM_HEADER.search(code) is None
+
+
+def is_procedure_fragment(code: str) -> bool:
+ """Whether the first code line is procedure text rather than a data entry."""
+ # A code view has no sequence area to skip; a ``>>`` directive is not code.
+ lines = (line.strip() for line in code.split("\n"))
+ first = next((line for line in lines if line and not line.startswith(">>")), "")
+ return _LEVEL_NUMBER.match(first) is None
+
+
+def wrap_fragment(code: str) -> str:
+ """*code* with the synthetic header written over its first two blank lines."""
+ lines = code.split("\n")
+ blanks = []
+ for index, line in enumerate(lines):
+ if line.strip():
+ break
+ blanks.append(index)
+ if len(blanks) < 2:
+ return code
+ second = _PROCEDURE if is_procedure_fragment(code) else _DATA
+ for index, text in zip(blanks[-2:], (_HEADER, second), strict=True):
+ eol = "\r" if lines[index].endswith("\r") else ""
+ lines[index] = text + eol
+ return "\n".join(lines)
diff --git a/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.py b/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.py
new file mode 100644
index 000000000..af2bd482d
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.py
@@ -0,0 +1,46 @@
+"""Rewrites for valid COBOL the grammar rejects; each keeps every line."""
+
+from __future__ import annotations
+
+import re
+
+_I = re.IGNORECASE
+# ``IF A NOT= B``: the grammar only tokenises ``NOT =``.
+_NOT_EQUAL = re.compile(r"(?<![\w-])NOT= ", _I)
+# ``x'00'``: the grammar reads only an upper-case X prefix.
+_HEX_PREFIX = re.compile(r"(?<![\w-])x(?=['\"])")
+# ``FUNCTION CURRENT-DATE(1:4)``: no reference modification on a function.
+_FUNCTION_REFMOD = re.compile(
+ r"(FUNCTION\s+[A-Z0-9-]+(?:\s*\([^()]*\))?)(\s*\([^()]*:[^()]*\))", _I
+)
+# ``ENTRY 'NAME' [USING ...].``, an alternate entry point (IMS ``DLITCBL``).
+_ENTRY = re.compile(r"(?<![\w-])ENTRY\s+'[^']*'(?:\s+USING(?:\s+[A-Z0-9-]+)+)?\s*\.", _I)
+# A data section with no entries before the next section or division.
+_EMPTY_SECTION = re.compile(
+ r"(?<![\w-])(?:WORKING-STORAGE|LOCAL-STORAGE|LINKAGE|FILE)\s+SECTION\s*\."
+ r"(?=\s*(?:[A-Z-]+\s+SECTION|PROCEDURE\s+DIVISION|\Z))",
+ _I,
+)
+# ``PROGRAM-ID. NAME`` with no closing period, common in IBM samples.
+_PROGRAM_ID_NO_PERIOD = re.compile(r"(PROGRAM-ID\.\s*[A-Z0-9][A-Z0-9-]*)(?=[ \t]*\r?\n)", _I)
+# Identification paragraphs hold free-form comment entries, possibly empty.
+_COMMENT_ENTRY = re.compile(
+ r"(?<![\w-])(AUTHOR|INSTALLATION|DATE-WRITTEN|DATE-COMPILED|SECURITY|REMARKS)\s*\..*?"
+ r"(?=(?<![\w-])(?:AUTHOR|INSTALLATION|DATE-WRITTEN|DATE-COMPILED|SECURITY|REMARKS)\s*\."
+ r"|(?<![\w-])(?:ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION|\Z)",
+ _I | re.DOTALL,
+)
+
+
+def rewrite_gaps(code: str) -> str:
+ code = _NOT_EQUAL.sub(lambda m: m.group(0)[:3] + " =", code)
+ code = _HEX_PREFIX.sub("X", code)
+ code = _FUNCTION_REFMOD.sub(lambda m: m.group(1) + _blank(m.group(2)), code)
+ code = _ENTRY.sub(lambda m: _blank(m.group(0)), code)
+ code = _EMPTY_SECTION.sub(lambda m: _blank(m.group(0)), code)
+ code = _PROGRAM_ID_NO_PERIOD.sub(r"\1.", code)
+ return _COMMENT_ENTRY.sub(lambda m: _blank(m.group(0)), code)
+
+
+def _blank(text: str) -> str:
+ return re.sub(r"[^\r\n]", " ", text)
diff --git a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py
new file mode 100644
index 000000000..ca4dffcb8
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py
@@ -0,0 +1,80 @@
+"""Fixed reference format: sequence area, indicator column and continuation lines.
+
+Every function keeps the line count, so a line in the result is the same line
+in the file.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Columns 1-6 are the sequence area, 7 the indicator, 8-72 the program text.
+_TEXT_START, _TEXT_END = 7, 72
+_COMMENT_INDICATORS = frozenset("*/$Dd")
+# Compiler options before IDENTIFICATION DIVISION (``PROCESS SQL``, ``CBL ...``).
+_COMPILER_DIRECTIVE = re.compile(r"\s*(PROCESS|CBL)\b", re.IGNORECASE)
+_FREE_FORMAT = re.compile(r">>\s*SOURCE\s+(FORMAT\s+(IS\s+)?)?FREE|SOURCEFORMAT\W+FREE", re.I)
+
+
+def is_free_format(text: str) -> bool:
+ """A ``>>SOURCE FORMAT FREE`` (or Micro Focus ``SOURCEFORMAT"FREE"``) directive."""
+ return _FREE_FORMAT.search(text) is not None
+
+
+def normalize_fixed_format(text: str) -> str:
+ """Blank what the compiler ignores and close literals split by continuation lines.
+
+ Comment and debug lines become blank, the sequence area and columns 73+
+ become spaces, and a literal continued on a ``-`` line is closed at column
+ 72 with its continuation blanked: the grammar has no continuation rule.
+ """
+ out: list[str] = []
+ open_quote = "" # the quote of a literal still open at the end of the previous code line
+ seen_division = False
+ for raw in text.split("\n"):
+ line, eol = (raw[:-1], "\r") if raw.endswith("\r") else (raw, "")
+ indicator = line[6:7]
+ body = line[_TEXT_START:_TEXT_END]
+ if indicator in _COMMENT_INDICATORS or not body.strip():
+ out.append(eol)
+ continue
+ if not seen_division and _COMPILER_DIRECTIVE.match(body):
+ out.append(eol)
+ continue
+ seen_division = True
+ if indicator == "-" and open_quote:
+ body, open_quote = _continued_literal(body, open_quote)
+ else:
+ open_quote = _open_quote_at_end(body)
+ if open_quote and indicator != "-":
+ # Close the literal at column 72 so this line parses on its own.
+ body = body.ljust(_TEXT_END - _TEXT_START)[:-1] + open_quote
+ out.append(" " * _TEXT_START + body + eol)
+ return "\n".join(out)
+
+
+def _open_quote_at_end(body: str) -> str:
+ """The quote character of a literal left open at the end of *body*, or ""."""
+ quote = ""
+ for ch in body:
+ if quote:
+ if ch == quote:
+ quote = ""
+ elif ch in "'\"":
+ quote = ch
+ return quote
+
+
+def _continued_literal(body: str, quote: str) -> tuple[str, str]:
+ """Blank a continuation line's share of a literal; return it and the quote still open."""
+ start = body.find(quote)
+ if start < 0:
+ return " " * len(body), quote
+ end = body.find(quote, start + 1)
+ if end < 0:
+ return " " * len(body), quote
+ rest = body[end + 1 :]
+ still_open = _open_quote_at_end(rest)
+ if still_open:
+ rest = rest.ljust(_TEXT_END - _TEXT_START - end - 1)[:-1] + still_open
+ return " " * (end + 1) + rest, still_open
diff --git a/packages/core/src/repowise/core/ingestion/cobol/source.py b/packages/core/src/repowise/core/ingestion/cobol/source.py
new file mode 100644
index 000000000..609106fa4
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/source.py
@@ -0,0 +1,34 @@
+"""The text the COBOL grammar parses, with every line kept at its number."""
+
+from __future__ import annotations
+
+import re
+
+from .fragments import is_fragment, is_procedure_fragment, wrap_fragment
+from .grammar_gaps import rewrite_gaps
+from .reference_format import is_free_format, normalize_fixed_format
+from .statements import mask_statements
+
+# A lossless round trip for bytes that are not UTF-8 (EBCDIC-converted Latin-1).
+_ENCODING, _ERRORS = "utf-8", "surrogateescape"
+# A ``*>`` comment to the end of its line; a literal matches first so its text is kept.
+_INLINE_COMMENT = re.compile(r"('[^'\n]*'|\"[^\"\n]*\")|\*>[^\r\n]*")
+
+
+def code_view(source: bytes) -> str:
+ """*source* as program text only: comments, sequence area and columns 73+ blank."""
+ text = source.decode(_ENCODING, _ERRORS)
+ # Free format keeps its text as written; only fixed format has columns.
+ code = text if is_free_format(text) else normalize_fixed_format(text)
+ return _INLINE_COMMENT.sub(lambda m: m.group(1) or " " * len(m.group(0)), code)
+
+
+def prepare_cobol_source(source: bytes) -> bytes:
+ """The parse input: a code view with EXEC/COPY masked, copybooks wrapped."""
+ code = code_view(source)
+ if is_fragment(code):
+ procedure = is_procedure_fragment(code)
+ code = wrap_fragment(mask_statements(code, procedure_from=0 if procedure else None))
+ else:
+ code = mask_statements(code)
+ return rewrite_gaps(code).encode(_ENCODING, _ERRORS)
diff --git a/packages/core/src/repowise/core/ingestion/cobol/statements.py b/packages/core/src/repowise/core/ingestion/cobol/statements.py
new file mode 100644
index 000000000..6bdc5e8bd
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/cobol/statements.py
@@ -0,0 +1,137 @@
+"""``EXEC ... END-EXEC`` blocks and ``COPY`` statements, found in a code view.
+
+Both are read from the code view (comments already blank) and masked before
+the grammar sees them: the COBOL-85 grammar has no embedded SQL/CICS/DLI and
+accepts ``COPY`` in only some positions.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+_NAME = r"[A-Z0-9#@$][A-Z0-9#@$_-]*"
+
+_EXEC = re.compile(
+ r"(?<![\w-])EXEC\s+(?P<kind>CICS|SQL|SQLIMS|DLI)\b(?P<body>.*?)\bEND-EXEC\b",
+ re.IGNORECASE | re.DOTALL,
+)
+_SQL_INCLUDE = re.compile(rf"^\s*INCLUDE\s+(?P<member>{_NAME})\s*$", re.IGNORECASE)
+# ``COPY member [OF|IN library] [SUPPRESS] [REPLACING ...].``; pseudo-text
+# (``==...==``) may hold periods, so it is matched as a unit.
+_COPY = re.compile(
+ rf"(?<![\w-])COPY\s+(?:'(?P<q1>[^']+)'|\"(?P<q2>[^\"]+)\"|(?P<word>{_NAME}))"
+ rf"(?:\s+(?:OF|IN)\s+(?:'(?P<lq>[^']+)'|(?P<lib>{_NAME})))?"
+ r"(?:\s+SUPPRESS)?"
+ r"(?P<replacing>\s+REPLACING(?:\s+(?:==.*?==|[^\s.=]+))+)?"
+ r"\s*\.",
+ re.IGNORECASE | re.DOTALL,
+)
+_PROCEDURE_DIVISION = re.compile(r"\bPROCEDURE\s+DIVISION\b", re.IGNORECASE)
+_FILLER = "CONTINUE"
+_DATA_PLACEHOLDER = "01 FILLER PIC X."
+
+
+@dataclass(frozen=True)
+class ExecBlock:
+ """One ``EXEC kind ... END-EXEC`` block; ``body`` is the text between them."""
+
+ kind: str
+ body: str
+ line: int
+ end_line: int
+ start: int
+ end: int
+
+
+@dataclass(frozen=True)
+class CopyStatement:
+ """A ``COPY`` (or ``EXEC SQL INCLUDE``) of a member, by its upper-cased name."""
+
+ member: str
+ library: str | None
+ line: int
+ replacing: bool = False
+ include: bool = False
+
+
+def exec_blocks(code: str) -> list[ExecBlock]:
+ out = []
+ for m in _EXEC.finditer(code):
+ line = code.count("\n", 0, m.start()) + 1
+ out.append(
+ ExecBlock(
+ kind=m.group("kind").upper(),
+ body=m.group("body"),
+ line=line,
+ end_line=line + m.group(0).count("\n"),
+ start=m.start(),
+ end=m.end(),
+ )
+ )
+ return out
+
+
+def copy_statements(code: str) -> list[CopyStatement]:
+ """Every ``COPY`` and ``EXEC SQL INCLUDE``, in source order."""
+ found: list[tuple[int, CopyStatement]] = []
+ for m in _COPY.finditer(code):
+ member = m.group("q1") or m.group("q2") or m.group("word")
+ library = m.group("lq") or m.group("lib")
+ found.append(
+ (
+ m.start(),
+ CopyStatement(
+ member=_member_name(member),
+ library=library.upper() if library else None,
+ line=code.count("\n", 0, m.start()) + 1,
+ replacing=bool(m.group("replacing")),
+ ),
+ )
+ )
+ for block in exec_blocks(code):
+ include = _SQL_INCLUDE.match(block.body) if block.kind == "SQL" else None
+ if include:
+ member = include.group("member").upper()
+ # SQLCA and SQLDA are the precompiler's own areas, never members.
+ if member not in {"SQLCA", "SQLDA"}:
+ found.append((block.start, CopyStatement(member, None, block.line, include=True)))
+ return [statement for _, statement in sorted(found, key=lambda item: item[0])]
+
+
+def mask_statements(code: str, procedure_from: int | None = None) -> str:
+ """Blank COPY statements and EXEC blocks, keeping every line and offset.
+
+ An EXEC block inside the procedure division becomes ``CONTINUE`` so the
+ statement it stood for still parses; elsewhere it is blanked. A procedure
+ copybook passes ``procedure_from=0``.
+ """
+ if procedure_from is None:
+ division = _PROCEDURE_DIVISION.search(code)
+ procedure_from = division.end() if division else len(code)
+ chars = list(code)
+ for block in exec_blocks(code):
+ _blank(chars, block.start, block.end)
+ if block.start >= procedure_from and not _SQL_INCLUDE.match(block.body):
+ chars[block.start : block.start + len(_FILLER)] = _FILLER
+ masked = "".join(chars)
+ # Right to left, so a replacement of another length leaves earlier offsets alone.
+ for m in reversed(list(_COPY.finditer(masked))):
+ newlines = "".join(c for c in m.group(0) if c in "\r\n")
+ # In data, an FD or group header needs the record the copybook supplied.
+ text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""
+ masked = masked[: m.start()] + text + newlines + masked[m.end() :]
+ return masked
+
+
+def _blank(chars: list[str], start: int, end: int) -> None:
+ for i in range(start, end):
+ if chars[i] not in "\r\n":
+ chars[i] = " "
+
+
+def _member_name(raw: str) -> str:
+ """``'lgpolicy.cpy'`` and ``LGPOLICY`` both name the member ``LGPOLICY``."""
+ name = raw.strip().replace("\\", "/").rsplit("/", 1)[-1]
+ stem = name.rsplit(".", 1)[0] if "." in name else name
+ return stem.upper()
diff --git a/packages/core/src/repowise/core/ingestion/extractors/cobol.py b/packages/core/src/repowise/core/ingestion/extractors/cobol.py
index 086898387..641496975 100644
--- a/packages/core/src/repowise/core/ingestion/extractors/cobol.py
+++ b/packages/core/src/repowise/core/ingestion/extractors/cobol.py
@@ -6,10 +6,13 @@ import re
from tree_sitter import Node
+from ..cobol.fragments import FRAGMENT_PROGRAM
+
_SECTION_SUFFIX = re.compile(r"\s+SECTION\s*\.\s*$", re.IGNORECASE)
_PARAGRAPH_SUFFIX = re.compile(r"\s*\.\s*$")
_PROGRAM_ID_IN_ERROR = re.compile(r"\bPROGRAM-ID\s*\.\s*([A-Z0-9][A-Z0-9-]*)", re.IGNORECASE)
_PROCEDURE_HEADERS = frozenset({"section_header", "paragraph_header"})
+_UNNAMED = frozenset({"FILLER", FRAGMENT_PROGRAM})
def normalize_cobol_symbol_name(raw_name: str, node_type: str) -> str:
@@ -23,7 +26,9 @@ def normalize_cobol_symbol_name(raw_name: str, node_type: str) -> str:
name = _SECTION_SUFFIX.sub("", name)
elif node_type == "paragraph_header":
name = _PARAGRAPH_SUFFIX.sub("", name)
- return name.upper()
+ name = name.upper()
+ # FILLER is an unnamed data item, and the wrapper program is synthetic; "" drops both.
+ return "" if name in _UNNAMED else name
def normalize_cobol_call_target(raw_name: str, _node_type: str) -> str:
diff --git a/packages/core/src/repowise/core/ingestion/sfc_source.py b/packages/core/src/repowise/core/ingestion/sfc_source.py
index 56c203253..da813651e 100644
--- a/packages/core/src/repowise/core/ingestion/sfc_source.py
+++ b/packages/core/src/repowise/core/ingestion/sfc_source.py
@@ -896,4 +896,8 @@ def prepare_source(language: str, source: bytes, *, path: str | None = None) ->
from .parser_helpers import prepare_apex_source
return prepare_apex_source(source)
+ if language == "cobol" and source:
+ from .cobol import prepare_cobol_source
+
+ return prepare_cobol_source(source)
return source
diff --git a/scripts/cobol_acceptance.py b/scripts/cobol_acceptance.py
index 6ef28d3eb..de5ce7824 100644
--- a/scripts/cobol_acceptance.py
+++ b/scripts/cobol_acceptance.py
@@ -96,6 +96,10 @@ def truth(root: str) -> dict:
for m in re.finditer(r"^\s*COPY\s+'?([A-Z0-9#@$-]+)'?", text, re.I | re.M):
for cpy in cpys.get(m.group(1).upper(), []):
pairs["copy"].add((rel, cpy))
+ pairs["copied"].add((rel, cpy))
+ for m in re.finditer(r"EXEC\s+SQL\s+INCLUDE\s+([A-Z0-9#@$-]+)", text, re.I):
+ for cpy in cpys.get(m.group(1).upper(), []):
+ pairs["copied"].add((rel, cpy))
for m in re.finditer(r"\bCALL\s+'([A-Z0-9#@$-]+)'", text, re.I):
for dst in progs.get(m.group(1).upper(), []):
pairs["call"].add((rel, dst))
@@ -168,7 +172,8 @@ def index_view(db: str) -> dict:
).fetchall()
symbols = [r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol"]
procs = {(r[3], r[4].upper()) for r in symbols if r[5] == "function"}
- same_file_calls, cross_perform = set(), 0
+ same_file_calls: set[tuple[str, str]] = set()
+ cross_perform: list[tuple[str, str]] = []
file_pairs: dict[str, set[tuple[str, str]]] = collections.defaultdict(set)
for src, dst, etype in edges:
fs, ft = file_of(src), file_of(dst)
@@ -177,7 +182,7 @@ def index_view(db: str) -> dict:
if fs == ft and target is not None:
same_file_calls.add((fs, (target[4] or "").upper()))
elif target is not None and target[5] == "function" and target[2] == "cobol":
- cross_perform += 1
+ cross_perform.append((fs, ft))
if etype in _STRUCTURAL and fs != ft and dst in nodes:
file_pairs[etype].add((fs, ft))
return {
@@ -201,7 +206,11 @@ def metrics(root: str, db: str) -> dict:
per_prog = collections.defaultdict(set)
for rel, name in t["paragraphs"]:
per_prog[rel].add(name)
- whole = sum(1 for rel, names in per_prog.items() if all((rel, n) in ix["procs"] for n in names))
+ whole = sum(
+ 1
+ for rel in t["files"]["program"]
+ if all((rel, n) in ix["procs"] for n in per_prog.get(rel, ()))
+ )
cobol_files = t["files"]["program"] + t["files"]["copybook"]
touched = {f for pair in any_pairs for f in pair}
pair_hits = {
@@ -217,7 +226,10 @@ def metrics(root: str, db: str) -> dict:
"paragraph_recall": ratio(len(t["paragraphs"] & ix["procs"]), len(t["paragraphs"])),
"perform_recall": ratio(len(t["performs"] & ix["same_file_calls"]), len(t["performs"])),
**{f"{k}_pairs": v for k, v in pair_hits.items()},
- "cross_program_perform_edges": ix["cross_perform"],
+ # A PERFORM into a copybook the program copies is right; any other is not.
+ "cross_program_perform_edges": sum(
+ 1 for pair in ix["cross_perform"] if pair not in t["pairs"]["copied"]
+ ),
"cobol_files_isolated": ratio(
sum(1 for f in cobol_files if f not in touched), len(cobol_files)
),
diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.py
new file mode 100644
index 000000000..61d38ddbc
--- /dev/null
+++ b/tests/unit/ingestion/parser/test_cobol_source.py
@@ -0,0 +1,195 @@
+"""COBOL source preparation: what the grammar rejects is masked, and every line keeps its number."""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.cobol import code_view, copy_statements, exec_blocks
+from repowise.core.ingestion.cobol.fragments import is_procedure_fragment
+from repowise.core.ingestion.parser import ASTParser
+
+from ._helpers import _make_file_info
+
+grammar_pack = pytest.importorskip("tree_sitter_language_pack")
+try:
+ grammar_pack.get_language("cobol")
+except Exception as exc: # pragma: no cover - depends on first-use grammar cache/network
+ pytest.skip(f"COBOL grammar unavailable: {exc}", allow_module_level=True)
+
+
+def _program(data: str = "", procedure: str = "", header: str = "") -> bytes:
+ lines = [
+ " IDENTIFICATION DIVISION.",
+ " PROGRAM-ID. DEMO.",
+ *([header] if header else []),
+ " DATA DIVISION.",
+ " WORKING-STORAGE SECTION.",
+ " 01 WS-A PIC X(10).",
+ " 01 WS-B PIC X(10).",
+ *([data] if data else []),
+ " PROCEDURE DIVISION.",
+ " MAIN-PARA.",
+ procedure or " CONTINUE.",
+ " PERFORM LAST-PARA.",
+ " LAST-PARA.",
+ " GOBACK.",
+ ]
+ return ("\n".join(lines) + "\n").encode()
+
+
+def _parse(source: bytes, path: str = "DEMO.cbl"):
+ return ASTParser().parse_file(_make_file_info(path, "cobol"), source)
+
+
+def _names(parsed) -> dict[str, int]:
+ return {s.name: s.start_line for s in parsed.symbols}
+
+
+ "case",
+ [
+ _program(procedure=" EXEC CICS RETURN TRANSID('X') END-EXEC."),
+ _program(
+ procedure=" EXEC SQL\n SELECT 1 INTO :WS-A FROM T\n END-EXEC."
+ ),
+ _program(data=" COPY CVACT01Y."),
+ _program(data=" 01 GROUP-A.\n COPY CVACT01Y."),
+ _program(procedure=" IF WS-A NOT= WS-B CONTINUE END-IF."),
+ _program(procedure=" INSPECT WS-A REPLACING ALL x'00' BY x'40'."),
+ _program(procedure=" MOVE FUNCTION CURRENT-DATE(1:4) TO WS-A."),
+ _program(procedure=" ENTRY 'DLITCBL' USING WS-A."),
+ _program(header=" AUTHOR. ME.\n DATE-COMPILED."),
+ _program(data=" LINKAGE SECTION."),
+ _program(
+ data=" 01 WS-L PIC X(70) VALUE 'ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTU\n"
+ " - 'VWXYZ'."
+ ),
+ ],
+ ids=[
+ "exec-cics",
+ "exec-sql",
+ "copy",
+ "copy-under-group",
+ "not-equal",
+ "hex-lowercase",
+ "function-refmod",
+ "entry",
+ "comment-entries",
+ "empty-linkage",
+ "continuation",
+ ],
+)
+def test_constructs_the_grammar_rejects_parse_with_every_paragraph(case: bytes) -> None:
+ parsed = _parse(case)
+
+ assert parsed.parse_errors == []
+ names = _names(parsed)
+ lines = case.decode().split("\n")
+ for paragraph in ("MAIN-PARA", "LAST-PARA"):
+ assert lines[names[paragraph] - 1].strip() == f"{paragraph}."
+
+
+def test_sequence_numbers_and_comment_lines_are_ignored() -> None:
+ source = (
+ b"000100 IDENTIFICATION DIVISION. 00010000\n"
+ b"000200 PROGRAM-ID. SEQ. 00020000\n"
+ b"000300* A COMMENT WITH PERFORM NOWHERE. 00030000\n"
+ b"000400 PROCEDURE DIVISION. 00040000\n"
+ b"000500 FIRST-PARA. 00050000\n"
+ b"000600 GOBACK. 00060000\n"
+ )
+
+ parsed = _parse(source)
+
+ assert parsed.parse_errors == []
+ assert _names(parsed) == {"SEQ": 1, "FIRST-PARA": 5}
+
+
+def test_a_data_copybook_parses_as_a_fragment_at_its_own_lines() -> None:
+ source = (
+ b" *****************************************************\n"
+ b" * ACCOUNT RECORD\n"
+ b" *****************************************************\n"
+ b" 01 ACCOUNT-RECORD.\n"
+ b" 05 ACCT-ID PIC 9(11).\n"
+ b" 05 FILLER PIC X(10).\n"
+ )
+
+ parsed = _parse(source, "CVACT01Y.cpy")
+
+ assert parsed.parse_errors == []
+ # No synthetic program and no FILLER: only the copybook's named items.
+ assert _names(parsed) == {"ACCOUNT-RECORD": 4, "ACCT-ID": 5}
+
+
+def test_a_procedure_copybook_yields_its_paragraphs() -> None:
+ source = (
+ b" * COMMON CODE\n"
+ b" *\n"
+ b" YYYY-STORE-PFKEY.\n"
+ b" EXEC CICS RETURN END-EXEC.\n"
+ b" YYYY-STORE-PFKEY-EXIT.\n"
+ b" EXIT.\n"
+ )
+
+ parsed = _parse(source, "CSSTRPFY.cpy")
+
+ assert parsed.parse_errors == []
+ assert _names(parsed) == {"YYYY-STORE-PFKEY": 3, "YYYY-STORE-PFKEY-EXIT": 5}
+
+
+def test_a_free_format_data_copybook_is_not_read_as_procedure_text() -> None:
+ source = b"\n\n>>SOURCE FORMAT FREE\n01 ACCOUNT-RECORD.\n 05 ACCT-ID PIC 9(11).\n"
+
+ assert not is_procedure_fragment(code_view(source))
+
+
+def test_floating_comments_hide_statements_and_keep_literals() -> None:
+ source = (
+ b">>SOURCE FORMAT FREE\n"
+ b"IDENTIFICATION DIVISION.\n"
+ b"PROGRAM-ID. DEMO.\n"
+ b"PROCEDURE DIVISION.\n"
+ b"MAIN-PARA. *> EXEC CICS is issued in LAST-PARA; COPY OLDMEM.\n"
+ b" DISPLAY '*> kept'.\n"
+ b" PERFORM LAST-PARA.\n"
+ b"LAST-PARA.\n"
+ b" EXEC CICS RETURN END-EXEC.\n"
+ )
+
+ code = code_view(source)
+
+ assert copy_statements(code) == []
+ assert [block.line for block in exec_blocks(code)] == [9]
+ assert "'*> kept'" in code
+ assert len(code) == len(source)
+
+
+def test_copy_statements_are_found_in_code_and_not_in_comments() -> None:
+ source = (
+ b" IDENTIFICATION DIVISION.\n"
+ b" PROGRAM-ID. DEMO.\n"
+ b" DATA DIVISION.\n"
+ b" WORKING-STORAGE SECTION.\n"
+ b" *COPY COMMENTED.\n"
+ b" COPY COCOM01Y.\n"
+ b" COPY 'lgpolicy.cpy'.\n"
+ b" COPY CSSETATY REPLACING\n"
+ b" ==(TESTVAR1)== BY ==ACCT.==.\n"
+ b" COPY MEMBER1 OF MYLIB.\n"
+ b" EXEC SQL INCLUDE SQLCA END-EXEC.\n"
+ b" EXEC SQL INCLUDE DCLTRTYP END-EXEC.\n"
+ )
+
+ found = [
+ (c.member, c.library, c.line, c.replacing, c.include)
+ for c in copy_statements(code_view(source))
+ ]
+
+ assert found == [
+ ("COCOM01Y", None, 6, False, False),
+ ("LGPOLICY", None, 7, False, False),
+ ("CSSETATY", None, 8, True, False),
+ ("MEMBER1", "MYLIB", 10, False, False),
+ ("DCLTRTYP", None, 12, False, True),
+ ]
diff --git a/tests/unit/ingestion/test_runtime_grammars.py b/tests/unit/ingestion/test_runtime_grammars.py
index c3e5d2e46..369fd42ce 100644
--- a/tests/unit/ingestion/test_runtime_grammars.py
+++ b/tests/unit/ingestion/test_runtime_grammars.py
@@ -115,3 +115,21 @@ def test_doctor_reports_an_uncached_grammar_without_downloading_it(
assert [(r.name, r.ok) for r in rows] == [("COBOL grammar", False)]
assert "1 files would index without symbols" in rows[0].detail
assert offline_pack == []
+
+
+def test_doctor_counts_untracked_files_ingestion_would_index(
+ tmp_path, monkeypatch: pytest.MonkeyPatch, offline_pack: list[str]
+) -> None:
+ from repowise.cli.commands.doctor_cmd._checks import core
+
+ subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
+ (tmp_path / ".gitignore").write_text("build/\n")
+ (tmp_path / "PAYROLL.cbl").write_text(" IDENTIFICATION DIVISION.\n")
+ (tmp_path / "build").mkdir()
+ (tmp_path / "build" / "GEN.cbl").write_text("")
+ monkeypatch.setattr(grammars, "runtime_grammar_cache", lambda tag: (False, "/cache"))
+
+ rows = core._runtime_grammar_rows(tmp_path)
+
+ assert [(r.name, r.ok) for r in rows] == [("COBOL grammar", False)]
+ assert "1 files would index without symbols" in rows[0].detail