Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(cobol): parse fixed format, embedded EXEC blocks and copybooks without losing paragraphs (#4434)
Raghav Chamadiyacfdbf0dbffa1
diff --git a/docs/layers/LANGUAGE_SUPPORT.md b/docs/layers/LANGUAGE_SUPPORT.mdindex d02a0a73b..b011e3708 100644--- a/docs/layers/LANGUAGE_SUPPORT.md+++ b/docs/layers/LANGUAGE_SUPPORT.md@@ -77,6 +77,9 @@ calls are entry paths the repository graph cannot observe.The COBOL and Apex grammars download on first use; offline, set`REPOWISE_GRAMMAR_DIR` to a pre-staged grammar cache. `repowise doctor` reportswhether it is cached.+COBOL is read in fixed reference format, with `EXEC SQL/CICS/DLI` blocks and+`COPY` statements masked before parsing; a copybook parses as a data or+procedure fragment.Apex (`.cls`, `.trigger`, `.apex`) is parsed for symbols, heritage and calls,resolved case-insensitively within each sfdx project's namespace; its dead-codefindings are review-only. Apex test coverage from the Salesforce CLI is read as adiff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.pyindex 388c1d581..54409d40b 100644--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/_checks/core.py@@ -239,7 +239,8 @@ def _runtime_grammar_rows(repo_path: _DoctorPath) -> list[DoctorCheck]:}try:listed = subprocess.run(- ["git", "ls-files", "-z"],+ # Untracked, unignored files too: ingestion walks them as well.+ ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"],cwd=repo_path,capture_output=True,check=True,diff --git a/packages/core/src/repowise/core/ingestion/cobol/__init__.py b/packages/core/src/repowise/core/ingestion/cobol/__init__.pynew file mode 100644index 000000000..80f115803--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/__init__.py@@ -0,0 +1,13 @@+"""COBOL source handling: reference format, EXEC and COPY statements, copybooks."""++from .source import code_view, prepare_cobol_source+from .statements import CopyStatement, ExecBlock, copy_statements, exec_blocks++__all__ = [+ "CopyStatement",+ "ExecBlock",+ "code_view",+ "copy_statements",+ "exec_blocks",+ "prepare_cobol_source",+]diff --git a/packages/core/src/repowise/core/ingestion/cobol/fragments.py b/packages/core/src/repowise/core/ingestion/cobol/fragments.pynew file mode 100644index 000000000..dd359db78--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/fragments.py@@ -0,0 +1,49 @@+"""Copybooks parse as fragments inside a synthetic two-line program header.++The header overwrites two blank lines (comments are already blank) ahead of+the first code line, so every line keeps its number. A copybook without two+such lines parses unwrapped. ``FRAGMENT_PROGRAM`` names the synthetic program,+and the symbol extractor drops it.+"""++from __future__ import annotations++import re++FRAGMENT_PROGRAM = "REPOWISE-COPYBOOK"++_PROGRAM_HEADER = re.compile(r"\b(IDENTIFICATION|ID)\s+DIVISION\b|\bPROGRAM-ID\b", re.I)+_LEVEL_NUMBER = re.compile(r"\s*\d{1,2}\s")+_HEADER = " IDENTIFICATION DIVISION. PROGRAM-ID. " + FRAGMENT_PROGRAM + "."+_DATA = " DATA DIVISION. WORKING-STORAGE SECTION."+_PROCEDURE = " PROCEDURE DIVISION."+++def is_fragment(code: str) -> bool:+ """A member with no program header: a copybook, whatever its extension."""+ return code.strip() != "" and _PROGRAM_HEADER.search(code) is None+++def is_procedure_fragment(code: str) -> bool:+ """Whether the first code line is procedure text rather than a data entry."""+ # A code view has no sequence area to skip; a ``>>`` directive is not code.+ lines = (line.strip() for line in code.split("\n"))+ first = next((line for line in lines if line and not line.startswith(">>")), "")+ return _LEVEL_NUMBER.match(first) is None+++def wrap_fragment(code: str) -> str:+ """*code* with the synthetic header written over its first two blank lines."""+ lines = code.split("\n")+ blanks = []+ for index, line in enumerate(lines):+ if line.strip():+ break+ blanks.append(index)+ if len(blanks) < 2:+ return code+ second = _PROCEDURE if is_procedure_fragment(code) else _DATA+ for index, text in zip(blanks[-2:], (_HEADER, second), strict=True):+ eol = "\r" if lines[index].endswith("\r") else ""+ lines[index] = text + eol+ return "\n".join(lines)diff --git a/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.py b/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.pynew file mode 100644index 000000000..af2bd482d--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/grammar_gaps.py@@ -0,0 +1,46 @@+"""Rewrites for valid COBOL the grammar rejects; each keeps every line."""++from __future__ import annotations++import re++_I = re.IGNORECASE+# ``IF A NOT= B``: the grammar only tokenises ``NOT =``.+_NOT_EQUAL = re.compile(r"(?<![\w-])NOT= ", _I)+# ``x'00'``: the grammar reads only an upper-case X prefix.+_HEX_PREFIX = re.compile(r"(?<![\w-])x(?=['\"])")+# ``FUNCTION CURRENT-DATE(1:4)``: no reference modification on a function.+_FUNCTION_REFMOD = re.compile(+ r"(FUNCTION\s+[A-Z0-9-]+(?:\s*\([^()]*\))?)(\s*\([^()]*:[^()]*\))", _I+)+# ``ENTRY 'NAME' [USING ...].``, an alternate entry point (IMS ``DLITCBL``).+_ENTRY = re.compile(r"(?<![\w-])ENTRY\s+'[^']*'(?:\s+USING(?:\s+[A-Z0-9-]+)+)?\s*\.", _I)+# A data section with no entries before the next section or division.+_EMPTY_SECTION = re.compile(+ r"(?<![\w-])(?:WORKING-STORAGE|LOCAL-STORAGE|LINKAGE|FILE)\s+SECTION\s*\."+ r"(?=\s*(?:[A-Z-]+\s+SECTION|PROCEDURE\s+DIVISION|\Z))",+ _I,+)+# ``PROGRAM-ID. NAME`` with no closing period, common in IBM samples.+_PROGRAM_ID_NO_PERIOD = re.compile(r"(PROGRAM-ID\.\s*[A-Z0-9][A-Z0-9-]*)(?=[ \t]*\r?\n)", _I)+# Identification paragraphs hold free-form comment entries, possibly empty.+_COMMENT_ENTRY = re.compile(+ r"(?<![\w-])(AUTHOR|INSTALLATION|DATE-WRITTEN|DATE-COMPILED|SECURITY|REMARKS)\s*\..*?"+ r"(?=(?<![\w-])(?:AUTHOR|INSTALLATION|DATE-WRITTEN|DATE-COMPILED|SECURITY|REMARKS)\s*\."+ r"|(?<![\w-])(?:ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION|\Z)",+ _I | re.DOTALL,+)+++def rewrite_gaps(code: str) -> str:+ code = _NOT_EQUAL.sub(lambda m: m.group(0)[:3] + " =", code)+ code = _HEX_PREFIX.sub("X", code)+ code = _FUNCTION_REFMOD.sub(lambda m: m.group(1) + _blank(m.group(2)), code)+ code = _ENTRY.sub(lambda m: _blank(m.group(0)), code)+ code = _EMPTY_SECTION.sub(lambda m: _blank(m.group(0)), code)+ code = _PROGRAM_ID_NO_PERIOD.sub(r"\1.", code)+ return _COMMENT_ENTRY.sub(lambda m: _blank(m.group(0)), code)+++def _blank(text: str) -> str:+ return re.sub(r"[^\r\n]", " ", text)diff --git a/packages/core/src/repowise/core/ingestion/cobol/reference_format.py b/packages/core/src/repowise/core/ingestion/cobol/reference_format.pynew file mode 100644index 000000000..ca4dffcb8--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/reference_format.py@@ -0,0 +1,80 @@+"""Fixed reference format: sequence area, indicator column and continuation lines.++Every function keeps the line count, so a line in the result is the same line+in the file.+"""++from __future__ import annotations++import re++# Columns 1-6 are the sequence area, 7 the indicator, 8-72 the program text.+_TEXT_START, _TEXT_END = 7, 72+_COMMENT_INDICATORS = frozenset("*/$Dd")+# Compiler options before IDENTIFICATION DIVISION (``PROCESS SQL``, ``CBL ...``).+_COMPILER_DIRECTIVE = re.compile(r"\s*(PROCESS|CBL)\b", re.IGNORECASE)+_FREE_FORMAT = re.compile(r">>\s*SOURCE\s+(FORMAT\s+(IS\s+)?)?FREE|SOURCEFORMAT\W+FREE", re.I)+++def is_free_format(text: str) -> bool:+ """A ``>>SOURCE FORMAT FREE`` (or Micro Focus ``SOURCEFORMAT"FREE"``) directive."""+ return _FREE_FORMAT.search(text) is not None+++def normalize_fixed_format(text: str) -> str:+ """Blank what the compiler ignores and close literals split by continuation lines.++ Comment and debug lines become blank, the sequence area and columns 73++ become spaces, and a literal continued on a ``-`` line is closed at column+ 72 with its continuation blanked: the grammar has no continuation rule.+ """+ out: list[str] = []+ open_quote = "" # the quote of a literal still open at the end of the previous code line+ seen_division = False+ for raw in text.split("\n"):+ line, eol = (raw[:-1], "\r") if raw.endswith("\r") else (raw, "")+ indicator = line[6:7]+ body = line[_TEXT_START:_TEXT_END]+ if indicator in _COMMENT_INDICATORS or not body.strip():+ out.append(eol)+ continue+ if not seen_division and _COMPILER_DIRECTIVE.match(body):+ out.append(eol)+ continue+ seen_division = True+ if indicator == "-" and open_quote:+ body, open_quote = _continued_literal(body, open_quote)+ else:+ open_quote = _open_quote_at_end(body)+ if open_quote and indicator != "-":+ # Close the literal at column 72 so this line parses on its own.+ body = body.ljust(_TEXT_END - _TEXT_START)[:-1] + open_quote+ out.append(" " * _TEXT_START + body + eol)+ return "\n".join(out)+++def _open_quote_at_end(body: str) -> str:+ """The quote character of a literal left open at the end of *body*, or ""."""+ quote = ""+ for ch in body:+ if quote:+ if ch == quote:+ quote = ""+ elif ch in "'\"":+ quote = ch+ return quote+++def _continued_literal(body: str, quote: str) -> tuple[str, str]:+ """Blank a continuation line's share of a literal; return it and the quote still open."""+ start = body.find(quote)+ if start < 0:+ return " " * len(body), quote+ end = body.find(quote, start + 1)+ if end < 0:+ return " " * len(body), quote+ rest = body[end + 1 :]+ still_open = _open_quote_at_end(rest)+ if still_open:+ rest = rest.ljust(_TEXT_END - _TEXT_START - end - 1)[:-1] + still_open+ return " " * (end + 1) + rest, still_opendiff --git a/packages/core/src/repowise/core/ingestion/cobol/source.py b/packages/core/src/repowise/core/ingestion/cobol/source.pynew file mode 100644index 000000000..609106fa4--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/source.py@@ -0,0 +1,34 @@+"""The text the COBOL grammar parses, with every line kept at its number."""++from __future__ import annotations++import re++from .fragments import is_fragment, is_procedure_fragment, wrap_fragment+from .grammar_gaps import rewrite_gaps+from .reference_format import is_free_format, normalize_fixed_format+from .statements import mask_statements++# A lossless round trip for bytes that are not UTF-8 (EBCDIC-converted Latin-1).+_ENCODING, _ERRORS = "utf-8", "surrogateescape"+# A ``*>`` comment to the end of its line; a literal matches first so its text is kept.+_INLINE_COMMENT = re.compile(r"('[^'\n]*'|\"[^\"\n]*\")|\*>[^\r\n]*")+++def code_view(source: bytes) -> str:+ """*source* as program text only: comments, sequence area and columns 73+ blank."""+ text = source.decode(_ENCODING, _ERRORS)+ # Free format keeps its text as written; only fixed format has columns.+ code = text if is_free_format(text) else normalize_fixed_format(text)+ return _INLINE_COMMENT.sub(lambda m: m.group(1) or " " * len(m.group(0)), code)+++def prepare_cobol_source(source: bytes) -> bytes:+ """The parse input: a code view with EXEC/COPY masked, copybooks wrapped."""+ code = code_view(source)+ if is_fragment(code):+ procedure = is_procedure_fragment(code)+ code = wrap_fragment(mask_statements(code, procedure_from=0 if procedure else None))+ else:+ code = mask_statements(code)+ return rewrite_gaps(code).encode(_ENCODING, _ERRORS)diff --git a/packages/core/src/repowise/core/ingestion/cobol/statements.py b/packages/core/src/repowise/core/ingestion/cobol/statements.pynew file mode 100644index 000000000..6bdc5e8bd--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/cobol/statements.py@@ -0,0 +1,137 @@+"""``EXEC ... END-EXEC`` blocks and ``COPY`` statements, found in a code view.++Both are read from the code view (comments already blank) and masked before+the grammar sees them: the COBOL-85 grammar has no embedded SQL/CICS/DLI and+accepts ``COPY`` in only some positions.+"""++from __future__ import annotations++import re+from dataclasses import dataclass++_NAME = r"[A-Z0-9#@$][A-Z0-9#@$_-]*"++_EXEC = re.compile(+ r"(?<![\w-])EXEC\s+(?P<kind>CICS|SQL|SQLIMS|DLI)\b(?P<body>.*?)\bEND-EXEC\b",+ re.IGNORECASE | re.DOTALL,+)+_SQL_INCLUDE = re.compile(rf"^\s*INCLUDE\s+(?P<member>{_NAME})\s*$", re.IGNORECASE)+# ``COPY member [OF|IN library] [SUPPRESS] [REPLACING ...].``; pseudo-text+# (``==...==``) may hold periods, so it is matched as a unit.+_COPY = re.compile(+ rf"(?<![\w-])COPY\s+(?:'(?P<q1>[^']+)'|\"(?P<q2>[^\"]+)\"|(?P<word>{_NAME}))"+ rf"(?:\s+(?:OF|IN)\s+(?:'(?P<lq>[^']+)'|(?P<lib>{_NAME})))?"+ r"(?:\s+SUPPRESS)?"+ r"(?P<replacing>\s+REPLACING(?:\s+(?:==.*?==|[^\s.=]+))+)?"+ r"\s*\.",+ re.IGNORECASE | re.DOTALL,+)+_PROCEDURE_DIVISION = re.compile(r"\bPROCEDURE\s+DIVISION\b", re.IGNORECASE)+_FILLER = "CONTINUE"+_DATA_PLACEHOLDER = "01 FILLER PIC X."+++@dataclass(frozen=True)+class ExecBlock:+ """One ``EXEC kind ... END-EXEC`` block; ``body`` is the text between them."""++ kind: str+ body: str+ line: int+ end_line: int+ start: int+ end: int+++@dataclass(frozen=True)+class CopyStatement:+ """A ``COPY`` (or ``EXEC SQL INCLUDE``) of a member, by its upper-cased name."""++ member: str+ library: str | None+ line: int+ replacing: bool = False+ include: bool = False+++def exec_blocks(code: str) -> list[ExecBlock]:+ out = []+ for m in _EXEC.finditer(code):+ line = code.count("\n", 0, m.start()) + 1+ out.append(+ ExecBlock(+ kind=m.group("kind").upper(),+ body=m.group("body"),+ line=line,+ end_line=line + m.group(0).count("\n"),+ start=m.start(),+ end=m.end(),+ )+ )+ return out+++def copy_statements(code: str) -> list[CopyStatement]:+ """Every ``COPY`` and ``EXEC SQL INCLUDE``, in source order."""+ found: list[tuple[int, CopyStatement]] = []+ for m in _COPY.finditer(code):+ member = m.group("q1") or m.group("q2") or m.group("word")+ library = m.group("lq") or m.group("lib")+ found.append(+ (+ m.start(),+ CopyStatement(+ member=_member_name(member),+ library=library.upper() if library else None,+ line=code.count("\n", 0, m.start()) + 1,+ replacing=bool(m.group("replacing")),+ ),+ )+ )+ for block in exec_blocks(code):+ include = _SQL_INCLUDE.match(block.body) if block.kind == "SQL" else None+ if include:+ member = include.group("member").upper()+ # SQLCA and SQLDA are the precompiler's own areas, never members.+ if member not in {"SQLCA", "SQLDA"}:+ found.append((block.start, CopyStatement(member, None, block.line, include=True)))+ return [statement for _, statement in sorted(found, key=lambda item: item[0])]+++def mask_statements(code: str, procedure_from: int | None = None) -> str:+ """Blank COPY statements and EXEC blocks, keeping every line and offset.++ An EXEC block inside the procedure division becomes ``CONTINUE`` so the+ statement it stood for still parses; elsewhere it is blanked. A procedure+ copybook passes ``procedure_from=0``.+ """+ if procedure_from is None:+ division = _PROCEDURE_DIVISION.search(code)+ procedure_from = division.end() if division else len(code)+ chars = list(code)+ for block in exec_blocks(code):+ _blank(chars, block.start, block.end)+ if block.start >= procedure_from and not _SQL_INCLUDE.match(block.body):+ chars[block.start : block.start + len(_FILLER)] = _FILLER+ masked = "".join(chars)+ # Right to left, so a replacement of another length leaves earlier offsets alone.+ for m in reversed(list(_COPY.finditer(masked))):+ newlines = "".join(c for c in m.group(0) if c in "\r\n")+ # In data, an FD or group header needs the record the copybook supplied.+ text = _DATA_PLACEHOLDER if m.start() < procedure_from else ""+ masked = masked[: m.start()] + text + newlines + masked[m.end() :]+ return masked+++def _blank(chars: list[str], start: int, end: int) -> None:+ for i in range(start, end):+ if chars[i] not in "\r\n":+ chars[i] = " "+++def _member_name(raw: str) -> str:+ """``'lgpolicy.cpy'`` and ``LGPOLICY`` both name the member ``LGPOLICY``."""+ name = raw.strip().replace("\\", "/").rsplit("/", 1)[-1]+ stem = name.rsplit(".", 1)[0] if "." in name else name+ return stem.upper()diff --git a/packages/core/src/repowise/core/ingestion/extractors/cobol.py b/packages/core/src/repowise/core/ingestion/extractors/cobol.pyindex 086898387..641496975 100644--- a/packages/core/src/repowise/core/ingestion/extractors/cobol.py+++ b/packages/core/src/repowise/core/ingestion/extractors/cobol.py@@ -6,10 +6,13 @@ import refrom tree_sitter import Node+from ..cobol.fragments import FRAGMENT_PROGRAM+_SECTION_SUFFIX = re.compile(r"\s+SECTION\s*\.\s*$", re.IGNORECASE)_PARAGRAPH_SUFFIX = re.compile(r"\s*\.\s*$")_PROGRAM_ID_IN_ERROR = re.compile(r"\bPROGRAM-ID\s*\.\s*([A-Z0-9][A-Z0-9-]*)", re.IGNORECASE)_PROCEDURE_HEADERS = frozenset({"section_header", "paragraph_header"})+_UNNAMED = frozenset({"FILLER", FRAGMENT_PROGRAM})def normalize_cobol_symbol_name(raw_name: str, node_type: str) -> str:@@ -23,7 +26,9 @@ def normalize_cobol_symbol_name(raw_name: str, node_type: str) -> str:name = _SECTION_SUFFIX.sub("", name)elif node_type == "paragraph_header":name = _PARAGRAPH_SUFFIX.sub("", name)- return name.upper()+ name = name.upper()+ # FILLER is an unnamed data item, and the wrapper program is synthetic; "" drops both.+ return "" if name in _UNNAMED else namedef normalize_cobol_call_target(raw_name: str, _node_type: str) -> str:diff --git a/packages/core/src/repowise/core/ingestion/sfc_source.py b/packages/core/src/repowise/core/ingestion/sfc_source.pyindex 56c203253..da813651e 100644--- a/packages/core/src/repowise/core/ingestion/sfc_source.py+++ b/packages/core/src/repowise/core/ingestion/sfc_source.py@@ -896,4 +896,8 @@ def prepare_source(language: str, source: bytes, *, path: str | None = None) ->from .parser_helpers import prepare_apex_sourcereturn prepare_apex_source(source)+ if language == "cobol" and source:+ from .cobol import prepare_cobol_source++ return prepare_cobol_source(source)return sourcediff --git a/scripts/cobol_acceptance.py b/scripts/cobol_acceptance.pyindex 6ef28d3eb..de5ce7824 100644--- a/scripts/cobol_acceptance.py+++ b/scripts/cobol_acceptance.py@@ -96,6 +96,10 @@ def truth(root: str) -> dict:for m in re.finditer(r"^\s*COPY\s+'?([A-Z0-9#@$-]+)'?", text, re.I | re.M):for cpy in cpys.get(m.group(1).upper(), []):pairs["copy"].add((rel, cpy))+ pairs["copied"].add((rel, cpy))+ for m in re.finditer(r"EXEC\s+SQL\s+INCLUDE\s+([A-Z0-9#@$-]+)", text, re.I):+ for cpy in cpys.get(m.group(1).upper(), []):+ pairs["copied"].add((rel, cpy))for m in re.finditer(r"\bCALL\s+'([A-Z0-9#@$-]+)'", text, re.I):for dst in progs.get(m.group(1).upper(), []):pairs["call"].add((rel, dst))@@ -168,7 +172,8 @@ def index_view(db: str) -> dict:).fetchall()symbols = [r for r in nodes.values() if r[1] == "symbol" and r[2] == "cobol"]procs = {(r[3], r[4].upper()) for r in symbols if r[5] == "function"}- same_file_calls, cross_perform = set(), 0+ same_file_calls: set[tuple[str, str]] = set()+ cross_perform: list[tuple[str, str]] = []file_pairs: dict[str, set[tuple[str, str]]] = collections.defaultdict(set)for src, dst, etype in edges:fs, ft = file_of(src), file_of(dst)@@ -177,7 +182,7 @@ def index_view(db: str) -> dict:if fs == ft and target is not None:same_file_calls.add((fs, (target[4] or "").upper()))elif target is not None and target[5] == "function" and target[2] == "cobol":- cross_perform += 1+ cross_perform.append((fs, ft))if etype in _STRUCTURAL and fs != ft and dst in nodes:file_pairs[etype].add((fs, ft))return {@@ -201,7 +206,11 @@ def metrics(root: str, db: str) -> dict:per_prog = collections.defaultdict(set)for rel, name in t["paragraphs"]:per_prog[rel].add(name)- whole = sum(1 for rel, names in per_prog.items() if all((rel, n) in ix["procs"] for n in names))+ whole = sum(+ 1+ for rel in t["files"]["program"]+ if all((rel, n) in ix["procs"] for n in per_prog.get(rel, ()))+ )cobol_files = t["files"]["program"] + t["files"]["copybook"]touched = {f for pair in any_pairs for f in pair}pair_hits = {@@ -217,7 +226,10 @@ def metrics(root: str, db: str) -> dict:"paragraph_recall": ratio(len(t["paragraphs"] & ix["procs"]), len(t["paragraphs"])),"perform_recall": ratio(len(t["performs"] & ix["same_file_calls"]), len(t["performs"])),**{f"{k}_pairs": v for k, v in pair_hits.items()},- "cross_program_perform_edges": ix["cross_perform"],+ # A PERFORM into a copybook the program copies is right; any other is not.+ "cross_program_perform_edges": sum(+ 1 for pair in ix["cross_perform"] if pair not in t["pairs"]["copied"]+ ),"cobol_files_isolated": ratio(sum(1 for f in cobol_files if f not in touched), len(cobol_files)),diff --git a/tests/unit/ingestion/parser/test_cobol_source.py b/tests/unit/ingestion/parser/test_cobol_source.pynew file mode 100644index 000000000..61d38ddbc--- /dev/null+++ b/tests/unit/ingestion/parser/test_cobol_source.py@@ -0,0 +1,195 @@+"""COBOL source preparation: what the grammar rejects is masked, and every line keeps its number."""++from __future__ import annotations++import pytest++from repowise.core.ingestion.cobol import code_view, copy_statements, exec_blocks+from repowise.core.ingestion.cobol.fragments import is_procedure_fragment+from repowise.core.ingestion.parser import ASTParser++from ._helpers import _make_file_info++grammar_pack = pytest.importorskip("tree_sitter_language_pack")+try:+ grammar_pack.get_language("cobol")+except Exception as exc: # pragma: no cover - depends on first-use grammar cache/network+ pytest.skip(f"COBOL grammar unavailable: {exc}", allow_module_level=True)+++def _program(data: str = "", procedure: str = "", header: str = "") -> bytes:+ lines = [+ " IDENTIFICATION DIVISION.",+ " PROGRAM-ID. DEMO.",+ *([header] if header else []),+ " DATA DIVISION.",+ " WORKING-STORAGE SECTION.",+ " 01 WS-A PIC X(10).",+ " 01 WS-B PIC X(10).",+ *([data] if data else []),+ " PROCEDURE DIVISION.",+ " MAIN-PARA.",+ procedure or " CONTINUE.",+ " PERFORM LAST-PARA.",+ " LAST-PARA.",+ " GOBACK.",+ ]+ return ("\n".join(lines) + "\n").encode()+++def _parse(source: bytes, path: str = "DEMO.cbl"):+ return ASTParser().parse_file(_make_file_info(path, "cobol"), source)+++def _names(parsed) -> dict[str, int]:+ return {s.name: s.start_line for s in parsed.symbols}+++ "case",+ [+ _program(procedure=" EXEC CICS RETURN TRANSID('X') END-EXEC."),+ _program(+ procedure=" EXEC SQL\n SELECT 1 INTO :WS-A FROM T\n END-EXEC."+ ),+ _program(data=" COPY CVACT01Y."),+ _program(data=" 01 GROUP-A.\n COPY CVACT01Y."),+ _program(procedure=" IF WS-A NOT= WS-B CONTINUE END-IF."),+ _program(procedure=" INSPECT WS-A REPLACING ALL x'00' BY x'40'."),+ _program(procedure=" MOVE FUNCTION CURRENT-DATE(1:4) TO WS-A."),+ _program(procedure=" ENTRY 'DLITCBL' USING WS-A."),+ _program(header=" AUTHOR. ME.\n DATE-COMPILED."),+ _program(data=" LINKAGE SECTION."),+ _program(+ data=" 01 WS-L PIC X(70) VALUE 'ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTU\n"+ " - 'VWXYZ'."+ ),+ ],+ ids=[+ "exec-cics",+ "exec-sql",+ "copy",+ "copy-under-group",+ "not-equal",+ "hex-lowercase",+ "function-refmod",+ "entry",+ "comment-entries",+ "empty-linkage",+ "continuation",+ ],+)+def test_constructs_the_grammar_rejects_parse_with_every_paragraph(case: bytes) -> None:+ parsed = _parse(case)++ assert parsed.parse_errors == []+ names = _names(parsed)+ lines = case.decode().split("\n")+ for paragraph in ("MAIN-PARA", "LAST-PARA"):+ assert lines[names[paragraph] - 1].strip() == f"{paragraph}."+++def test_sequence_numbers_and_comment_lines_are_ignored() -> None:+ source = (+ b"000100 IDENTIFICATION DIVISION. 00010000\n"+ b"000200 PROGRAM-ID. SEQ. 00020000\n"+ b"000300* A COMMENT WITH PERFORM NOWHERE. 00030000\n"+ b"000400 PROCEDURE DIVISION. 00040000\n"+ b"000500 FIRST-PARA. 00050000\n"+ b"000600 GOBACK. 00060000\n"+ )++ parsed = _parse(source)++ assert parsed.parse_errors == []+ assert _names(parsed) == {"SEQ": 1, "FIRST-PARA": 5}+++def test_a_data_copybook_parses_as_a_fragment_at_its_own_lines() -> None:+ source = (+ b" *****************************************************\n"+ b" * ACCOUNT RECORD\n"+ b" *****************************************************\n"+ b" 01 ACCOUNT-RECORD.\n"+ b" 05 ACCT-ID PIC 9(11).\n"+ b" 05 FILLER PIC X(10).\n"+ )++ parsed = _parse(source, "CVACT01Y.cpy")++ assert parsed.parse_errors == []+ # No synthetic program and no FILLER: only the copybook's named items.+ assert _names(parsed) == {"ACCOUNT-RECORD": 4, "ACCT-ID": 5}+++def test_a_procedure_copybook_yields_its_paragraphs() -> None:+ source = (+ b" * COMMON CODE\n"+ b" *\n"+ b" YYYY-STORE-PFKEY.\n"+ b" EXEC CICS RETURN END-EXEC.\n"+ b" YYYY-STORE-PFKEY-EXIT.\n"+ b" EXIT.\n"+ )++ parsed = _parse(source, "CSSTRPFY.cpy")++ assert parsed.parse_errors == []+ assert _names(parsed) == {"YYYY-STORE-PFKEY": 3, "YYYY-STORE-PFKEY-EXIT": 5}+++def test_a_free_format_data_copybook_is_not_read_as_procedure_text() -> None:+ source = b"\n\n>>SOURCE FORMAT FREE\n01 ACCOUNT-RECORD.\n 05 ACCT-ID PIC 9(11).\n"++ assert not is_procedure_fragment(code_view(source))+++def test_floating_comments_hide_statements_and_keep_literals() -> None:+ source = (+ b">>SOURCE FORMAT FREE\n"+ b"IDENTIFICATION DIVISION.\n"+ b"PROGRAM-ID. DEMO.\n"+ b"PROCEDURE DIVISION.\n"+ b"MAIN-PARA. *> EXEC CICS is issued in LAST-PARA; COPY OLDMEM.\n"+ b" DISPLAY '*> kept'.\n"+ b" PERFORM LAST-PARA.\n"+ b"LAST-PARA.\n"+ b" EXEC CICS RETURN END-EXEC.\n"+ )++ code = code_view(source)++ assert copy_statements(code) == []+ assert [block.line for block in exec_blocks(code)] == [9]+ assert "'*> kept'" in code+ assert len(code) == len(source)+++def test_copy_statements_are_found_in_code_and_not_in_comments() -> None:+ source = (+ b" IDENTIFICATION DIVISION.\n"+ b" PROGRAM-ID. DEMO.\n"+ b" DATA DIVISION.\n"+ b" WORKING-STORAGE SECTION.\n"+ b" *COPY COMMENTED.\n"+ b" COPY COCOM01Y.\n"+ b" COPY 'lgpolicy.cpy'.\n"+ b" COPY CSSETATY REPLACING\n"+ b" ==(TESTVAR1)== BY ==ACCT.==.\n"+ b" COPY MEMBER1 OF MYLIB.\n"+ b" EXEC SQL INCLUDE SQLCA END-EXEC.\n"+ b" EXEC SQL INCLUDE DCLTRTYP END-EXEC.\n"+ )++ found = [+ (c.member, c.library, c.line, c.replacing, c.include)+ for c in copy_statements(code_view(source))+ ]++ assert found == [+ ("COCOM01Y", None, 6, False, False),+ ("LGPOLICY", None, 7, False, False),+ ("CSSETATY", None, 8, True, False),+ ("MEMBER1", "MYLIB", 10, False, False),+ ("DCLTRTYP", None, 12, False, True),+ ]diff --git a/tests/unit/ingestion/test_runtime_grammars.py b/tests/unit/ingestion/test_runtime_grammars.pyindex c3e5d2e46..369fd42ce 100644--- a/tests/unit/ingestion/test_runtime_grammars.py+++ b/tests/unit/ingestion/test_runtime_grammars.py@@ -115,3 +115,21 @@ def test_doctor_reports_an_uncached_grammar_without_downloading_it(assert [(r.name, r.ok) for r in rows] == [("COBOL grammar", False)]assert "1 files would index without symbols" in rows[0].detailassert offline_pack == []+++def test_doctor_counts_untracked_files_ingestion_would_index(+ tmp_path, monkeypatch: pytest.MonkeyPatch, offline_pack: list[str]+) -> None:+ from repowise.cli.commands.doctor_cmd._checks import core++ subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)+ (tmp_path / ".gitignore").write_text("build/\n")+ (tmp_path / "PAYROLL.cbl").write_text(" IDENTIFICATION DIVISION.\n")+ (tmp_path / "build").mkdir()+ (tmp_path / "build" / "GEN.cbl").write_text("")+ monkeypatch.setattr(grammars, "runtime_grammar_cache", lambda tag: (False, "/cache"))++ rows = core._runtime_grammar_rows(tmp_path)++ assert [(r.name, r.ok) for r in rows] == [("COBOL grammar", False)]+ assert "1 files would index without symbols" in rows[0].detail