Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(code-origin): class parser-generator output as generated and copied single-file libraries as vendored (#4784)

Raghav Chamadiya811c81de3c57
diff --git a/packages/core/src/repowise/core/code_origin.py b/packages/core/src/repowise/core/code_origin.py
index 2fc2eab5c..1116c7e67 100644
--- a/packages/core/src/repowise/core/code_origin.py
+++ b/packages/core/src/repowise/core/code_origin.py
@@ -121,6 +121,15 @@ _GENERATED_SENTENCE_RE = re.compile(
_GENERATED_BANNER_LINE_RE = re.compile(r"^GENERATED CODE - DO NOT MODIFY BY HAND\s*$")
# Visual Studio rewrites resource headers, but ingestion must keep their include edges.
_VISUAL_CPP_BANNER_LINE_RE = re.compile(r"^MICROSOFT VISUAL C\+\+ GENERATED INCLUDE FILE\.?\s*$")
+# Bison and flex open their C output with the tool's own sentence.
+_PARSER_TOOL_BANNER_RE = re.compile(
+ r"^A (?:BISON PARSER, MADE BY GNU BISON|LEXICAL SCANNER GENERATED BY FLEX)\b"
+)
+# Ragel, yacc and lex point their C output back at the grammar with a
+# ``#line N "grammar.rl"`` directive near the top. A ``#line`` to a C file is
+# not one: hand-written code can carry that.
+_GRAMMAR_LINE_RE = re.compile(r'^#\s*line\s+\d+\s+"[^"]*\.(?:rl|y|yy|l|ll)"')
+_GRAMMAR_LINE_LINES = 10
_GENERATED_SENTENCE_LINES = 40
_PREAMBLE_PREFIXES = ("package ", "import ", "from ", "using ")
@@ -196,9 +205,18 @@ def _says_generated(text: str) -> bool:
_GENERATED_SENTENCE_RE.search(text)
or _GENERATED_BANNER_LINE_RE.search(text)
or _VISUAL_CPP_BANNER_LINE_RE.search(text)
+ or _PARSER_TOOL_BANNER_RE.search(text)
)
+def _from_grammar(name: str, header: str) -> bool:
+ """A C or C++ file whose head points back at the grammar it was made from."""
+ if not name.lower().endswith(_deps_suffixes()):
+ return False
+ lines = header.splitlines()[:_GRAMMAR_LINE_LINES]
+ return any(_GRAMMAR_LINE_RE.match(line.strip()) for line in lines)
+
+
def _above_import(lines: list[str], i: int) -> bool:
"""A comment right above an import is about that import ("this file is
generated at build time" over the import of a build artifact)."""
@@ -302,6 +320,8 @@ _VENDORED_DIR_TOKENS = frozenset(
"third_party",
"thirdparty",
"third-party",
+ "3rdparty",
+ "3rd_party",
"node_modules",
"bower_components",
"site-packages",
@@ -326,6 +346,11 @@ _NATIVE_TREE = "native"
# sub-packages (``internal/deps/resolver/``) is usually first-party.
_DEPS_DIR = "deps"
_DEPS_LANGUAGES = ("c", "cpp")
+# C and C++ trees also keep copies in a root ``extern/`` (matplotlib's
+# ``extern/agg24-svn/``) and in ``bundled/`` (spdlog's ``fmt/bundled/``). Only
+# there: javalin's Kotlin ``plugin/bundled/`` is its own code.
+_NATIVE_VENDORED_ROOT = "extern"
+_NATIVE_VENDORED_DIR = "bundled"
_MINIFIED_SUFFIXES = (".min.js", ".min.css", ".min.mjs")
@@ -339,7 +364,11 @@ def _is_vendored_path(lowered_name: str, dirs: list[str]) -> bool:
d == _NATIVE_TREE and dirs[i + 1] in _VENDORED_ROOT_TOKENS for i, d in enumerate(dirs[:-2])
):
return True
- if _DEPS_DIR in dirs[:-1] and lowered_name.endswith(_deps_suffixes()):
+ if lowered_name.endswith(_deps_suffixes()) and (
+ _DEPS_DIR in dirs[:-1]
+ or _NATIVE_VENDORED_DIR in dirs
+ or dirs[:1] == [_NATIVE_VENDORED_ROOT]
+ ):
return True
return lowered_name.endswith(_MINIFIED_SUFFIXES) or _in_static_release_dir(dirs)
@@ -379,14 +408,58 @@ _LICENSE_RE = re.compile(
_BANNER_RE = re.compile(
r"/\*!|@license|@preserve|\b[A-Za-z][\w.\-]*\s+v?\d+\.\d+(?:\.\d+)?\b"
)
+# A license's own version ("Apache License, Version 2.0") is not a release.
+_LICENSE_VERSION_RE = re.compile(
+ r"(?:licen[cs]e|\b[al]?gpl|\bmpl|\bepl)\W*(?:v(?:ersion)?\.?\s*)?\d+(?:\.\d+)*"
+ r"|\bversion\s+\d+(?:\.\d+)*\s+of\s+the\s+licen[cs]e",
+ re.IGNORECASE,
+)
+_URL_RE = re.compile(r"https?://\S+")
+# A single-header library copied into the source tree (``src/json.hpp``), or
+# a browser library dropped in the root ``scripts/`` folder. Outside asset
+# directories the header rule runs only on these, only when the project is
+# known, and only when the banner names the file itself: a repository's own
+# header names its owner, not each file.
+_LIBRARY_HEADER_SUFFIXES = (".h", ".hh", ".hpp", ".hxx")
+_ROOT_SCRIPTS_DIR = "scripts"
+_SCRIPT_SUFFIXES = (".js", ".mjs", ".cjs")
+_MIN_SELF_NAME = 4
+# Links into a repository's source tree name a file, not a library's home page.
+_SOURCE_LINK_SEGMENTS = frozenset({"blob", "tree", "src"})
+
+
+def _copied_library_scope(name: str, dirs: list[str], project: str | None) -> str | None:
+ """``"asset"`` under an asset directory, ``"library"`` for a single-file
+ library shape outside one, None where the header rule does not run."""
+ if any(d in _ASSET_DIR_TOKENS for d in dirs):
+ return "asset"
+ lowered = name.lower()
+ shaped = lowered.endswith(_LIBRARY_HEADER_SUFFIXES) or (
+ dirs[:1] == [_ROOT_SCRIPTS_DIR] and lowered.endswith(_SCRIPT_SUFFIXES)
+ )
+ return "library" if project and shaped else None
+
+
+def _homepage_names(stem: str, comments: str) -> bool:
+ """Whether a URL in the comments is the file's home page (exprtk.hpp's
+ ``partow.net/programming/exprtk/``)."""
+ for url in _URL_RE.findall(comments):
+ segments = {_squash(s) for s in url.split("/")[3:]}
+ if stem in segments and _SOURCE_LINK_SEGMENTS.isdisjoint(segments):
+ return True
+ return False
def _comment_lines(header: str) -> list[str]:
lines: list[str] = []
+ in_block = False
for raw in header.splitlines()[:_HEADER_LINES]:
- text = _comment_text(raw.strip(), False)
+ line = raw.strip()
+ # A block comment's body lines need no leading ``*`` (json.hpp 3.10).
+ text = line if in_block else _comment_text(line, False)
if text is not None:
lines.append(text)
+ in_block = (in_block or line.startswith("/*")) and "*/" not in line
return lines
@@ -395,23 +468,40 @@ def _squash(text: str) -> str:
return re.sub(r"[^0-9a-z]", "", text.lower())
-def _third_party_header(header: str, project: str | None) -> bool:
+def _third_party_header(
+ name: str, header: str, project: str | None, *, names_file: bool = False
+) -> bool:
"""Whether the leading comments carry another project's license banner.
Conservative on purpose: it needs a license or copyright line **and** a
versioned or preserved banner, both in comments, and the banner must not
name *project* when the caller knows it. A repository's own
``Copyright (c) 2024 Acme`` header has no release banner, so it stays.
+ With *names_file*, the banner must be preserved (``/*!``) or a URL to the
+ file's own home page, and the comments must name the file's stem.
"""
comments = "\n".join(_comment_lines(header))
- # The comment prefix is stripped from *comments*, so a preserved ``/*!``
- # opener is read from the raw head.
- banner = _BANNER_RE.search(comments) or header.lstrip().startswith("/*!")
- if not comments or not _LICENSE_RE.search(comments) or not banner:
+ stem = _squash(PurePosixPath(name).stem)
+ if names_file and (len(stem) < _MIN_SELF_NAME or stem not in _squash(comments)):
+ return False
+ if not comments or not _LICENSE_RE.search(comments):
+ return False
+ if not _has_banner(header, comments, stem, names_file):
return False
return not (project and _squash(project) in _squash(comments))
+def _has_banner(header: str, comments: str, stem: str, names_file: bool) -> bool:
+ # The comment prefix is stripped from *comments*, so a preserved ``/*!``
+ # opener is read from the raw head.
+ if header.lstrip().startswith("/*!"):
+ return True
+ if names_file:
+ # "protocol v1.2" in a first-party header is not a release banner.
+ return _homepage_names(stem, comments)
+ return bool(_BANNER_RE.search(_LICENSE_VERSION_RE.sub(" ", comments)))
+
+
# --------------------------------------------------------------------------
# Build
# --------------------------------------------------------------------------
@@ -862,20 +952,24 @@ def code_origin(
if _is_generated_path(name, dirs) or (header and _is_generated_content(name, dirs, header)):
return "generated"
- if _is_vendored_path(name.lower(), dirs) or (
- header
- and any(d in _ASSET_DIR_TOKENS for d in dirs)
- and _third_party_header(header, project)
- ):
+ if _is_vendored_path(name.lower(), dirs) or _copied_library(name, dirs, header, project):
return "vendored"
return _maintained_origin(normalized, name, dirs, is_test, repo_root, header)
+def _copied_library(name: str, dirs: list[str], header: str, project: str | None) -> bool:
+ scope = _copied_library_scope(name, dirs, project) if header else None
+ return scope is not None and _third_party_header(
+ name, header, project, names_file=scope == "library"
+ )
+
+
def _is_generated_content(name: str, dirs: list[str], header: str) -> bool:
return (
_declares_generated(name, header)
or _is_reference_stub(name, dirs, header)
or _is_bundler_output(header)
+ or _from_grammar(name, header)
)
diff --git a/tests/unit/test_code_origin.py b/tests/unit/test_code_origin.py
index 219588800..bb45a2bb9 100644
--- a/tests/unit/test_code_origin.py
+++ b/tests/unit/test_code_origin.py
@@ -345,6 +345,165 @@ def test_third_party_header_under_asset_dirs_is_vendored(header: str, folder: st
def test_third_party_header_outside_asset_dirs_stays() -> None:
assert code_origin("src/lib.js", _JQUERY) == "production"
+ assert code_origin("src/lib.js", _JQUERY, project="app") == "production"
+
+
+_JSON_HPP = (
+ "/*\n"
+ " __ _____ _____ _____\n"
+ " __| | __| | | | JSON for Modern C++\n"
+ "| | |__ | | | | | | version 3.11.2\n"
+ "|_____|_____|_____|_|___| https://github.com/nlohmann/json\n"
+ "*/\n"
+ "// Copyright (c) 2013-2022 Niels Lohmann <https://nlohmann.me>\n"
+ "// SPDX-License-Identifier: MIT\n"
+ "#pragma once\n"
+)
+_JSON_HPP_311 = (
+ "// __ _____ _____ _____\n"
+ "// __| | __| | | | JSON for Modern C++\n"
+ "// | | |__ | | | | | | version 3.11.2\n"
+ "// |_____|_____|_____|_|___| https://github.com/nlohmann/json\n"
+ "//\n"
+ "// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n"
+ "// SPDX-License-Identifier: MIT\n"
+ "#pragma once\n"
+)
+_EXPRTK = (
+ "/*\n"
+ " ******************************************************************\n"
+ " * C++ Mathematical Expression Toolkit Library *\n"
+ " * Author: Arash Partow (1999-2024) *\n"
+ " * URL: https://www.partow.net/programming/exprtk/index.html *\n"
+ " * Copyright notice: *\n"
+ " * Free use of the C++ Mathematical Expression Toolkit Library is *\n"
+ " * permitted under the guidelines and in accordance with the most *\n"
+ " * current version of the MIT License. *\n"
+ " ******************************************************************\n"
+ "*/\n"
+ "#ifndef INCLUDE_EXPRTK_HPP\n"
+)
+
+
+ ("path", "header"),
+ [
+ ("src/json.hpp", _JSON_HPP),
+ ("src/json.hpp", _JSON_HPP_311),
+ ("src/modules/calc/exprtk.hpp", _EXPRTK),
+ ("scripts/jquery.js", _JQUERY),
+ ],
+)
+def test_single_file_library_outside_asset_dirs_is_vendored(path: str, header: str) -> None:
+ assert code_origin(path, header, project="PowerToys") == "vendored"
+ # Without the project name the repository's own banner cannot be told apart.
+ assert code_origin(path, header) != "vendored"
+
+
+def test_own_single_header_banner_is_not_vendored() -> None:
+ for header in (_JSON_HPP, _JSON_HPP_311):
+ assert code_origin("include/nlohmann/json.hpp", header, project="json") == "production"
+ own = (
+ "// Acme Widgets v2.1\n"
+ "// Copyright (c) 2024 Acme Corp. Licensed under the MIT License.\n"
+ "#pragma once\n"
+ )
+ assert code_origin("include/acme/widgets.hpp", own, project="acme-widgets") == "production"
+
+
+ "comment",
+ [
+ "// Implements widgets protocol v1.2, see RFC 7230 section 3.1",
+ "// https://github.com/acme/widgets/blob/main/src/widgets.h",
+ "// Widgets for the engine. https://acme.dev/engine/",
+ ],
+)
+def test_own_header_without_its_project_name_is_not_vendored(comment: str) -> None:
+ head = f"// Copyright 2020 Acme Corp.\n// MIT License\n{comment}\n#pragma once\n"
+ assert code_origin("src/widgets.h", head, project="engine-core") == "production"
+
+
+def test_license_version_is_not_a_release_banner() -> None:
+ apache = (
+ "// Copyright 2017 The Abseil Authors.\n"
+ '// Licensed under the Apache License, Version 2.0 (the "License");\n'
+ "#pragma once\n"
+ )
+ assert code_origin("absl/base/macros.h", apache, project="abseil-cpp") == "production"
+ assert code_origin("static/js/chat.js", apache) == "production"
+ lgpl = "/* Copyright Acme. Under either version 2.1 of the License, or later. */\n"
+ assert code_origin("static/app.js", lgpl) == "production"
+
+
+ "path",
+ [
+ "include/spdlog/fmt/bundled/format.h",
+ "extern/agg24-svn/include/agg_basics.h",
+ "src/3rdparty/zlib/inflate.c",
+ "lib/3rd_party/lua/lapi.c",
+ ],
+)
+def test_native_library_shelves_are_vendored(path: str) -> None:
+ assert code_origin(path) == "vendored"
+
+
+def test_bundled_and_extern_count_only_for_c_and_cpp() -> None:
+ assert code_origin("src/main/java/io/javalin/plugin/bundled/CorsPlugin.kt") == "production"
+ assert code_origin("extern/client.go") == "production"
+ # Below the root, ``extern/`` is as often a library's own C API.
+ assert code_origin("include/foo/extern/api.h") == "production"
+
+
+_RAGEL_C = (
+ "\n"
+ '#line 1 "ext/puma_http11/http11_parser.rl"\n'
+ "/**\n * Copyright (c) 2005 Zed A. Shaw\n */\n"
+ '#include "http11_parser.h"\n'
+)
+_BISON_C = (
+ "/* A Bison parser, made by GNU Bison 3.8.2. */\n\n"
+ "/* Bison implementation for Yacc-like parsers in C */\n"
+ "#define YYBISON 30802\n"
+)
+_FLEX_C = (
+ '#line 2 "lex.yy.c"\n\n'
+ "#define YY_INT_ALIGNED short int\n\n"
+ "/* A lexical scanner generated by flex */\n\n"
+ "#define FLEX_SCANNER\n"
+)
+_YACC_C = '#include <stdio.h>\n#line 2 "gram.y"\nstatic int yylex(void);\n'
+
+
+ ("path", "text"),
+ [
+ ("ext/puma_http11/http11_parser.c", _RAGEL_C),
+ ("src/parse.c", _BISON_C),
+ ("src/parse.h", _BISON_C),
+ ("src/scan.c", _FLEX_C),
+ ("src/gram.cc", _YACC_C),
+ ],
+)
+def test_parser_generator_output_is_generated(path: str, text: str) -> None:
+ assert code_origin(path, text) == "generated"
+
+
+ "text",
+ [
+ '#line 1 "src/parser.c"\nint parse(void) { return 0; }\n',
+ '#line 40 "util.h"\nint helper(void);\n',
+ "int a;\n" * 12 + '#line 1 "late.rl"\n',
+ ],
+)
+def test_line_directive_to_c_source_is_not_generated(text: str) -> None:
+ assert code_origin("src/parser.c", text) == "production"
+
+
+def test_line_directive_rule_is_c_and_cpp_only() -> None:
+ assert code_origin("src/parser.py", '#line 1 "x.rl"\nx = 1\n') == "production"
def test_own_license_header_is_not_vendored() -> None: