Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(code-origin): class parser-generator output as generated and copied single-file libraries as vendored (#4784)
Raghav Chamadiya811c81de3c57
diff --git a/packages/core/src/repowise/core/code_origin.py b/packages/core/src/repowise/core/code_origin.pyindex 2fc2eab5c..1116c7e67 100644--- a/packages/core/src/repowise/core/code_origin.py+++ b/packages/core/src/repowise/core/code_origin.py@@ -121,6 +121,15 @@ _GENERATED_SENTENCE_RE = re.compile(_GENERATED_BANNER_LINE_RE = re.compile(r"^GENERATED CODE - DO NOT MODIFY BY HAND\s*$")# Visual Studio rewrites resource headers, but ingestion must keep their include edges._VISUAL_CPP_BANNER_LINE_RE = re.compile(r"^MICROSOFT VISUAL C\+\+ GENERATED INCLUDE FILE\.?\s*$")+# Bison and flex open their C output with the tool's own sentence.+_PARSER_TOOL_BANNER_RE = re.compile(+ r"^A (?:BISON PARSER, MADE BY GNU BISON|LEXICAL SCANNER GENERATED BY FLEX)\b"+)+# Ragel, yacc and lex point their C output back at the grammar with a+# ``#line N "grammar.rl"`` directive near the top. A ``#line`` to a C file is+# not one: hand-written code can carry that.+_GRAMMAR_LINE_RE = re.compile(r'^#\s*line\s+\d+\s+"[^"]*\.(?:rl|y|yy|l|ll)"')+_GRAMMAR_LINE_LINES = 10_GENERATED_SENTENCE_LINES = 40_PREAMBLE_PREFIXES = ("package ", "import ", "from ", "using ")@@ -196,9 +205,18 @@ def _says_generated(text: str) -> bool:_GENERATED_SENTENCE_RE.search(text)or _GENERATED_BANNER_LINE_RE.search(text)or _VISUAL_CPP_BANNER_LINE_RE.search(text)+ or _PARSER_TOOL_BANNER_RE.search(text))+def _from_grammar(name: str, header: str) -> bool:+ """A C or C++ file whose head points back at the grammar it was made from."""+ if not name.lower().endswith(_deps_suffixes()):+ return False+ lines = header.splitlines()[:_GRAMMAR_LINE_LINES]+ return any(_GRAMMAR_LINE_RE.match(line.strip()) for line in lines)++def _above_import(lines: list[str], i: int) -> bool:"""A comment right above an import is about that import ("this file isgenerated at build time" over the import of a build artifact)."""@@ -302,6 +320,8 @@ _VENDORED_DIR_TOKENS = frozenset("third_party","thirdparty","third-party",+ "3rdparty",+ "3rd_party","node_modules","bower_components","site-packages",@@ -326,6 +346,11 @@ _NATIVE_TREE = "native"# sub-packages (``internal/deps/resolver/``) is usually first-party._DEPS_DIR = "deps"_DEPS_LANGUAGES = ("c", "cpp")+# C and C++ trees also keep copies in a root ``extern/`` (matplotlib's+# ``extern/agg24-svn/``) and in ``bundled/`` (spdlog's ``fmt/bundled/``). Only+# there: javalin's Kotlin ``plugin/bundled/`` is its own code.+_NATIVE_VENDORED_ROOT = "extern"+_NATIVE_VENDORED_DIR = "bundled"_MINIFIED_SUFFIXES = (".min.js", ".min.css", ".min.mjs")@@ -339,7 +364,11 @@ def _is_vendored_path(lowered_name: str, dirs: list[str]) -> bool:d == _NATIVE_TREE and dirs[i + 1] in _VENDORED_ROOT_TOKENS for i, d in enumerate(dirs[:-2])):return True- if _DEPS_DIR in dirs[:-1] and lowered_name.endswith(_deps_suffixes()):+ if lowered_name.endswith(_deps_suffixes()) and (+ _DEPS_DIR in dirs[:-1]+ or _NATIVE_VENDORED_DIR in dirs+ or dirs[:1] == [_NATIVE_VENDORED_ROOT]+ ):return Truereturn lowered_name.endswith(_MINIFIED_SUFFIXES) or _in_static_release_dir(dirs)@@ -379,14 +408,58 @@ _LICENSE_RE = re.compile(_BANNER_RE = re.compile(r"/\*!|@license|@preserve|\b[A-Za-z][\w.\-]*\s+v?\d+\.\d+(?:\.\d+)?\b")+# A license's own version ("Apache License, Version 2.0") is not a release.+_LICENSE_VERSION_RE = re.compile(+ r"(?:licen[cs]e|\b[al]?gpl|\bmpl|\bepl)\W*(?:v(?:ersion)?\.?\s*)?\d+(?:\.\d+)*"+ r"|\bversion\s+\d+(?:\.\d+)*\s+of\s+the\s+licen[cs]e",+ re.IGNORECASE,+)+_URL_RE = re.compile(r"https?://\S+")+# A single-header library copied into the source tree (``src/json.hpp``), or+# a browser library dropped in the root ``scripts/`` folder. Outside asset+# directories the header rule runs only on these, only when the project is+# known, and only when the banner names the file itself: a repository's own+# header names its owner, not each file.+_LIBRARY_HEADER_SUFFIXES = (".h", ".hh", ".hpp", ".hxx")+_ROOT_SCRIPTS_DIR = "scripts"+_SCRIPT_SUFFIXES = (".js", ".mjs", ".cjs")+_MIN_SELF_NAME = 4+# Links into a repository's source tree name a file, not a library's home page.+_SOURCE_LINK_SEGMENTS = frozenset({"blob", "tree", "src"})+++def _copied_library_scope(name: str, dirs: list[str], project: str | None) -> str | None:+ """``"asset"`` under an asset directory, ``"library"`` for a single-file+ library shape outside one, None where the header rule does not run."""+ if any(d in _ASSET_DIR_TOKENS for d in dirs):+ return "asset"+ lowered = name.lower()+ shaped = lowered.endswith(_LIBRARY_HEADER_SUFFIXES) or (+ dirs[:1] == [_ROOT_SCRIPTS_DIR] and lowered.endswith(_SCRIPT_SUFFIXES)+ )+ return "library" if project and shaped else None+++def _homepage_names(stem: str, comments: str) -> bool:+ """Whether a URL in the comments is the file's home page (exprtk.hpp's+ ``partow.net/programming/exprtk/``)."""+ for url in _URL_RE.findall(comments):+ segments = {_squash(s) for s in url.split("/")[3:]}+ if stem in segments and _SOURCE_LINK_SEGMENTS.isdisjoint(segments):+ return True+ return Falsedef _comment_lines(header: str) -> list[str]:lines: list[str] = []+ in_block = Falsefor raw in header.splitlines()[:_HEADER_LINES]:- text = _comment_text(raw.strip(), False)+ line = raw.strip()+ # A block comment's body lines need no leading ``*`` (json.hpp 3.10).+ text = line if in_block else _comment_text(line, False)if text is not None:lines.append(text)+ in_block = (in_block or line.startswith("/*")) and "*/" not in linereturn lines@@ -395,23 +468,40 @@ def _squash(text: str) -> str:return re.sub(r"[^0-9a-z]", "", text.lower())-def _third_party_header(header: str, project: str | None) -> bool:+def _third_party_header(+ name: str, header: str, project: str | None, *, names_file: bool = False+) -> bool:"""Whether the leading comments carry another project's license banner.Conservative on purpose: it needs a license or copyright line **and** aversioned or preserved banner, both in comments, and the banner must notname *project* when the caller knows it. A repository's own``Copyright (c) 2024 Acme`` header has no release banner, so it stays.+ With *names_file*, the banner must be preserved (``/*!``) or a URL to the+ file's own home page, and the comments must name the file's stem."""comments = "\n".join(_comment_lines(header))- # The comment prefix is stripped from *comments*, so a preserved ``/*!``- # opener is read from the raw head.- banner = _BANNER_RE.search(comments) or header.lstrip().startswith("/*!")- if not comments or not _LICENSE_RE.search(comments) or not banner:+ stem = _squash(PurePosixPath(name).stem)+ if names_file and (len(stem) < _MIN_SELF_NAME or stem not in _squash(comments)):+ return False+ if not comments or not _LICENSE_RE.search(comments):+ return False+ if not _has_banner(header, comments, stem, names_file):return Falsereturn not (project and _squash(project) in _squash(comments))+def _has_banner(header: str, comments: str, stem: str, names_file: bool) -> bool:+ # The comment prefix is stripped from *comments*, so a preserved ``/*!``+ # opener is read from the raw head.+ if header.lstrip().startswith("/*!"):+ return True+ if names_file:+ # "protocol v1.2" in a first-party header is not a release banner.+ return _homepage_names(stem, comments)+ return bool(_BANNER_RE.search(_LICENSE_VERSION_RE.sub(" ", comments)))++# --------------------------------------------------------------------------# Build# --------------------------------------------------------------------------@@ -862,20 +952,24 @@ def code_origin(if _is_generated_path(name, dirs) or (header and _is_generated_content(name, dirs, header)):return "generated"- if _is_vendored_path(name.lower(), dirs) or (- header- and any(d in _ASSET_DIR_TOKENS for d in dirs)- and _third_party_header(header, project)- ):+ if _is_vendored_path(name.lower(), dirs) or _copied_library(name, dirs, header, project):return "vendored"return _maintained_origin(normalized, name, dirs, is_test, repo_root, header)+def _copied_library(name: str, dirs: list[str], header: str, project: str | None) -> bool:+ scope = _copied_library_scope(name, dirs, project) if header else None+ return scope is not None and _third_party_header(+ name, header, project, names_file=scope == "library"+ )++def _is_generated_content(name: str, dirs: list[str], header: str) -> bool:return (_declares_generated(name, header)or _is_reference_stub(name, dirs, header)or _is_bundler_output(header)+ or _from_grammar(name, header))diff --git a/tests/unit/test_code_origin.py b/tests/unit/test_code_origin.pyindex 219588800..bb45a2bb9 100644--- a/tests/unit/test_code_origin.py+++ b/tests/unit/test_code_origin.py@@ -345,6 +345,165 @@ def test_third_party_header_under_asset_dirs_is_vendored(header: str, folder: stdef test_third_party_header_outside_asset_dirs_stays() -> None:assert code_origin("src/lib.js", _JQUERY) == "production"+ assert code_origin("src/lib.js", _JQUERY, project="app") == "production"+++_JSON_HPP = (+ "/*\n"+ " __ _____ _____ _____\n"+ " __| | __| | | | JSON for Modern C++\n"+ "| | |__ | | | | | | version 3.11.2\n"+ "|_____|_____|_____|_|___| https://github.com/nlohmann/json\n"+ "*/\n"+ "// Copyright (c) 2013-2022 Niels Lohmann <https://nlohmann.me>\n"+ "// SPDX-License-Identifier: MIT\n"+ "#pragma once\n"+)+_JSON_HPP_311 = (+ "// __ _____ _____ _____\n"+ "// __| | __| | | | JSON for Modern C++\n"+ "// | | |__ | | | | | | version 3.11.2\n"+ "// |_____|_____|_____|_|___| https://github.com/nlohmann/json\n"+ "//\n"+ "// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n"+ "// SPDX-License-Identifier: MIT\n"+ "#pragma once\n"+)+_EXPRTK = (+ "/*\n"+ " ******************************************************************\n"+ " * C++ Mathematical Expression Toolkit Library *\n"+ " * Author: Arash Partow (1999-2024) *\n"+ " * URL: https://www.partow.net/programming/exprtk/index.html *\n"+ " * Copyright notice: *\n"+ " * Free use of the C++ Mathematical Expression Toolkit Library is *\n"+ " * permitted under the guidelines and in accordance with the most *\n"+ " * current version of the MIT License. *\n"+ " ******************************************************************\n"+ "*/\n"+ "#ifndef INCLUDE_EXPRTK_HPP\n"+)+++ ("path", "header"),+ [+ ("src/json.hpp", _JSON_HPP),+ ("src/json.hpp", _JSON_HPP_311),+ ("src/modules/calc/exprtk.hpp", _EXPRTK),+ ("scripts/jquery.js", _JQUERY),+ ],+)+def test_single_file_library_outside_asset_dirs_is_vendored(path: str, header: str) -> None:+ assert code_origin(path, header, project="PowerToys") == "vendored"+ # Without the project name the repository's own banner cannot be told apart.+ assert code_origin(path, header) != "vendored"+++def test_own_single_header_banner_is_not_vendored() -> None:+ for header in (_JSON_HPP, _JSON_HPP_311):+ assert code_origin("include/nlohmann/json.hpp", header, project="json") == "production"+ own = (+ "// Acme Widgets v2.1\n"+ "// Copyright (c) 2024 Acme Corp. Licensed under the MIT License.\n"+ "#pragma once\n"+ )+ assert code_origin("include/acme/widgets.hpp", own, project="acme-widgets") == "production"+++ "comment",+ [+ "// Implements widgets protocol v1.2, see RFC 7230 section 3.1",+ "// https://github.com/acme/widgets/blob/main/src/widgets.h",+ "// Widgets for the engine. https://acme.dev/engine/",+ ],+)+def test_own_header_without_its_project_name_is_not_vendored(comment: str) -> None:+ head = f"// Copyright 2020 Acme Corp.\n// MIT License\n{comment}\n#pragma once\n"+ assert code_origin("src/widgets.h", head, project="engine-core") == "production"+++def test_license_version_is_not_a_release_banner() -> None:+ apache = (+ "// Copyright 2017 The Abseil Authors.\n"+ '// Licensed under the Apache License, Version 2.0 (the "License");\n'+ "#pragma once\n"+ )+ assert code_origin("absl/base/macros.h", apache, project="abseil-cpp") == "production"+ assert code_origin("static/js/chat.js", apache) == "production"+ lgpl = "/* Copyright Acme. Under either version 2.1 of the License, or later. */\n"+ assert code_origin("static/app.js", lgpl) == "production"+++ "path",+ [+ "include/spdlog/fmt/bundled/format.h",+ "extern/agg24-svn/include/agg_basics.h",+ "src/3rdparty/zlib/inflate.c",+ "lib/3rd_party/lua/lapi.c",+ ],+)+def test_native_library_shelves_are_vendored(path: str) -> None:+ assert code_origin(path) == "vendored"+++def test_bundled_and_extern_count_only_for_c_and_cpp() -> None:+ assert code_origin("src/main/java/io/javalin/plugin/bundled/CorsPlugin.kt") == "production"+ assert code_origin("extern/client.go") == "production"+ # Below the root, ``extern/`` is as often a library's own C API.+ assert code_origin("include/foo/extern/api.h") == "production"+++_RAGEL_C = (+ "\n"+ '#line 1 "ext/puma_http11/http11_parser.rl"\n'+ "/**\n * Copyright (c) 2005 Zed A. Shaw\n */\n"+ '#include "http11_parser.h"\n'+)+_BISON_C = (+ "/* A Bison parser, made by GNU Bison 3.8.2. */\n\n"+ "/* Bison implementation for Yacc-like parsers in C */\n"+ "#define YYBISON 30802\n"+)+_FLEX_C = (+ '#line 2 "lex.yy.c"\n\n'+ "#define YY_INT_ALIGNED short int\n\n"+ "/* A lexical scanner generated by flex */\n\n"+ "#define FLEX_SCANNER\n"+)+_YACC_C = '#include <stdio.h>\n#line 2 "gram.y"\nstatic int yylex(void);\n'+++ ("path", "text"),+ [+ ("ext/puma_http11/http11_parser.c", _RAGEL_C),+ ("src/parse.c", _BISON_C),+ ("src/parse.h", _BISON_C),+ ("src/scan.c", _FLEX_C),+ ("src/gram.cc", _YACC_C),+ ],+)+def test_parser_generator_output_is_generated(path: str, text: str) -> None:+ assert code_origin(path, text) == "generated"+++ "text",+ [+ '#line 1 "src/parser.c"\nint parse(void) { return 0; }\n',+ '#line 40 "util.h"\nint helper(void);\n',+ "int a;\n" * 12 + '#line 1 "late.rl"\n',+ ],+)+def test_line_directive_to_c_source_is_not_generated(text: str) -> None:+ assert code_origin("src/parser.c", text) == "production"+++def test_line_directive_rule_is_c_and_cpp_only() -> None:+ assert code_origin("src/parser.py", '#line 1 "x.rl"\nx = 1\n') == "production"def test_own_license_header_is_not_vendored() -> None: