Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.2 KB
Raw Blame
"""Regression tests for bounded Dependabot PR generation."""

import re
from fnmatch import fnmatchcase
from pathlib import Path

from packaging.requirements import Requirement
import pytest
import yaml


REPO_ROOT = Path(__file__).resolve().parents[2]
DEPENDABOT_CONFIG = REPO_ROOT / ".github" / "dependabot.yml"
SEMVER_LEVELS = {"major", "minor", "patch"}
TEST_NPM_DIRECTORIES = {
    "/tests",
    "/tests/accessibility_tests",
    "/tests/api_tests_with_login",
    "/tests/infrastructure_tests",
    "/tests/puppeteer",
    "/tests/ui_tests",
    "/tests/ui_tests/playwright",
}


def _npm_entries():
    config = yaml.safe_load(DEPENDABOT_CONFIG.read_text(encoding="utf-8"))
    return [
        update
        for update in config["updates"]
        if update["package-ecosystem"] == "npm"
    ]


def _test_lock_directories():
    directories = set()
    for lockfile in (REPO_ROOT / "tests").glob("**/package-lock.json"):
        relative = lockfile.relative_to(REPO_ROOT)
        if "node_modules" in relative.parts or any(
            part.startswith(".") for part in relative.parts
        ):
            continue
        directories.add(f"/{relative.parent.as_posix()}")
    return directories


def _assert_group(entry, name, *, applies_to, update_types=None):
    group = entry["groups"][name]
    assert group["applies-to"] == applies_to
    assert group["patterns"] == ["*"]
    if update_types is not None:
        assert set(group["update-types"]) == set(update_types)


def test_npm_updates_have_separate_grouped_runtime_and_test_streams():
    entries = _npm_entries()

    assert len(entries) == 2
    root = next(entry for entry in entries if entry.get("directory") == "/")
    tests = next(entry for entry in entries if "directories" in entry)

    assert set(tests["directories"]) == TEST_NPM_DIRECTORIES
    assert set(tests["directories"]) == _test_lock_directories()
    assert tests["schedule"] == {
        "interval": "weekly",
        "day": "tuesday",
        "time": "04:00",
    }
    assert root["schedule"]["interval"] == "weekly"


def test_npm_minor_patch_and_security_updates_are_grouped():
    entries = _npm_entries()
    root = next(entry for entry in entries if entry.get("directory") == "/")
    tests = next(entry for entry in entries if "directories" in entry)

    _assert_group(
        root,
        "frontend-minor-and-patch",
        applies_to="version-updates",
        update_types={"minor", "patch"},
    )
    _assert_group(
        root,
        "frontend-security",
        applies_to="security-updates",
    )
    _assert_group(
        tests,
        "test-minor-and-patch",
        applies_to="version-updates",
        update_types={"minor", "patch"},
    )
    _assert_group(
        tests,
        "test-security",
        applies_to="security-updates",
    )


def _entries(ecosystem):
    config = yaml.safe_load(DEPENDABOT_CONFIG.read_text(encoding="utf-8"))
    return [
        update
        for update in config["updates"]
        if update["package-ecosystem"] == ecosystem
    ]


@pytest.mark.parametrize(
    ("directory", "prefix"),
    [("/", "python"), ("/requirements", "requirements")],
)
def test_pip_minor_patch_and_security_updates_are_grouped(directory, prefix):
    """Pip updates retain catch-all groups after any coupled family.

    Majors are deliberately outside every version-updates group, so each
    breaking upgrade still arrives as its own PR.
    """
    entry = next(
        entry for entry in _entries("pip") if entry["directory"] == directory
    )
    expected = [
        f"{prefix}-minor-and-patch",
        f"{prefix}-security",
    ]
    if directory == "/requirements":
        expected[:0] = [
            "mcp-runtime-minor-and-patch",
            "mcp-runtime-security",
        ]
    assert list(entry["groups"]) == expected
    _assert_group(
        entry,
        f"{prefix}-minor-and-patch",
        applies_to="version-updates",
        update_types={"minor", "patch"},
    )
    _assert_group(entry, f"{prefix}-security", applies_to="security-updates")


def test_pip_grouping_keeps_existing_filters():
    root, requirements = (
        next(e for e in _entries("pip") if e["directory"] == d)
        for d in ("/", "/requirements")
    )
    assert {i["dependency-name"] for i in root["ignore"]} >= {
        "faiss-cpu",
        "pyee",
    }
    assert "requirements/pip-vendor-refresh.txt" in root["exclude-paths"]
    assert "pip-vendor-refresh.txt" in requirements["exclude-paths"]
    assert requirements["allow"] == [{"dependency-type": "all"}]


def test_requirements_files_have_one_update_owner():
    entries = _entries("pip")
    for path in sorted((REPO_ROOT / "requirements").iterdir()):
        if path.suffix not in {".in", ".txt"}:
            continue
        owners = []
        for entry in entries:
            directory = entry["directory"]
            base = REPO_ROOT / directory.lstrip("/")
            if not path.is_relative_to(base):
                continue
            relative = path.relative_to(base).as_posix()
            if not any(
                fnmatchcase(relative, pattern)
                for pattern in entry.get("exclude-paths", [])
            ):
                owners.append(directory)
        if path.name == "pip-vendor-refresh.txt":
            expected = []
        elif path.name.startswith("bootstrap-"):
            expected = ["/"]
        else:
            expected = ["/requirements"]
        assert owners == expected, (path.name, owners)


def test_mcp_groups_follow_the_extra_and_application_bounds():
    requirements = next(
        entry
        for entry in _entries("pip")
        if entry["directory"] == "/requirements"
    )
    names = {
        Requirement(line).name.casefold().replace("_", "-")
        for line in (REPO_ROOT / "requirements/mcp-extra.in")
        .read_text()
        .splitlines()
        if line.strip() and not line.lstrip().startswith("#")
    }
    for name, kind in (
        ("mcp-runtime-minor-and-patch", "version-updates"),
        ("mcp-runtime-security", "security-updates"),
    ):
        group = requirements["groups"][name]
        assert group["applies-to"] == kind
        assert set(group["patterns"]) == names
        for dependency in names:
            matching = [
                key
                for key, candidate in requirements["groups"].items()
                if candidate.get("applies-to", "version-updates") == kind
                and any(
                    _glob_matches(pattern, dependency)
                    for pattern in candidate["patterns"]
                )
            ]
            assert matching[0] == name, (dependency, matching)
    assert requirements["groups"]["mcp-runtime-minor-and-patch"][
        "update-types"
    ] == ["minor", "patch"]


def test_math_rendering_peers_are_grouped_before_other_frontend_updates():
    root = next(
        entry for entry in _npm_entries() if entry.get("directory") == "/"
    )
    peers = {"marked", "marked-katex-extension", "katex"}
    for name, kind in (
        ("math-rendering", "version-updates"),
        ("math-rendering-security", "security-updates"),
    ):
        group = root["groups"][name]
        assert group.get("applies-to", "version-updates") == kind
        assert set(group["patterns"]) == peers
        assert "update-types" not in group
        for dependency in peers:
            matching = [
                key
                for key, candidate in root["groups"].items()
                if candidate.get("applies-to", "version-updates") == kind
                and any(
                    _glob_matches(pattern, dependency)
                    for pattern in candidate["patterns"]
                )
            ]
            assert matching[0] == name, (dependency, matching)


def test_github_actions_are_grouped_behind_codeql_lockstep_groups():
    """Every action update lands in one PR; codeql-action keeps its own.

    The codeql-action groups must come first: first match wins, so a
    catch-all above them would capture codeql-action and its dedicated
    init/analyze lockstep group (#5400) would never form.
    """
    (entry,) = _entries("github-actions")
    assert list(entry["groups"]) == [
        "codeql-action",
        "codeql-action-security",
        "github-actions",
        "github-actions-security",
    ]
    # Action bumps are mostly major-tag moves, so the catch-all takes every
    # update type rather than only minor and patch.
    _assert_group(entry, "github-actions", applies_to="version-updates")
    assert "update-types" not in entry["groups"]["github-actions"]
    _assert_group(
        entry, "github-actions-security", applies_to="security-updates"
    )
    assert _hidden_groups(entry) == []


def _glob_matches(glob, text):
    """Match like Dependabot: ``*`` is the only wildcard, case is ignored.

    ``text`` may be a pattern itself. Its ``*`` is then an ordinary
    character, and a match means ``glob`` takes every name ``text`` takes.
    """
    regex = ".*".join(re.escape(part) for part in glob.lower().split("*"))
    return re.fullmatch(regex, text.lower()) is not None


def _covers_updates(first, second):
    """True when ``first`` takes every update ``second`` could take.

    Only ``update-types`` and ``dependency-type`` are read. An absent
    ``update-types`` means every semver level and an absent
    ``dependency-type`` both kinds, so groups split by them (minor and patch
    vs major, production vs development) do not cover each other. By
    ``update-types``, a group that lists only some semver levels does not
    cover one without them, but one that lists all three does. By
    ``dependency-type``, a group that has one covers only a group with the
    same one.
    """
    first_types = set(first.get("update-types") or SEMVER_LEVELS)
    second_types = set(second.get("update-types") or SEMVER_LEVELS)
    first_kind = first.get("dependency-type")
    second_kind = second.get("dependency-type")
    return second_types <= first_types and first_kind in (None, second_kind)


def _hidden_groups(entry):
    """List ``(earlier, later, pattern)`` for patterns an earlier group takes.

    Dependabot gives a dependency to the FIRST group, in file order, that
    takes it, and groups of different ``applies-to`` never compete. Within
    one ``applies-to`` pool, a later group's pattern is reported when an
    earlier group has a glob that

    * is strictly broader: ``*`` and ``vite*`` hide ``vitest``, and ``@*``
      hides ``@vitest/*``. Neither group's ``update-types``,
      ``dependency-type`` or ``exclude-patterns`` is read here, so this can
      over-report (a ``*`` group limited to major updates still counts);
      moving the specific group above it is the fix; or
    * is equally broad and the earlier group's ``update-types`` and
      ``dependency-type`` cover the later group's (an absent key covers
      everything): the later group is a duplicate and never forms. Groups
      split by those keys (minor and patch vs major, production vs
      development) may share a glob, and so may a later group without them
      after a group they limit.

    A group without ``patterns`` is ``*``, and ``dependency-type`` is read as
    written even where the ecosystem ignores it. Not compared: groups that
    only partly overlap, by glob (``*test`` above ``vite*``) or by
    update-types (minor and patch above patch and major), since neither is
    the specific one; a later group that only several earlier groups
    together take; and ``exclude-patterns``, which are never read and so can
    only add reports.
    """
    groups = list((entry.get("groups") or {}).items())
    return [
        (earlier, later, pattern)
        for index, (earlier, first) in enumerate(groups)
        for later, second in groups[index + 1 :]
        if first.get("applies-to", "version-updates")
        == second.get("applies-to", "version-updates")
        for pattern in second.get("patterns") or ["*"]
        if any(
            _glob_matches(glob, pattern)
            and (
                not _glob_matches(pattern, glob)
                or _covers_updates(first, second)
            )
            for glob in first.get("patterns") or ["*"]
        )
    ]


def test_specific_groups_precede_broader_groups_in_every_entry():
    """No dependabot group may sit below a group that already takes it.

    First match wins (#6871): a broader group defined above a narrower one
    silently captures the narrower group's dependencies, so the narrower
    grouping (the vitest lockstep pair is the concrete case) never forms
    and nothing reports an error. A repeated group never forms either.
    YAML maps load in document order, which is the file's order.
    """
    config = yaml.safe_load(DEPENDABOT_CONFIG.read_text(encoding="utf-8"))
    assert any(
        len(entry.get("groups") or {}) > 1 for entry in config["updates"]
    ), "no entry with several groups: the ordering check would be vacuous"
    for entry in config["updates"]:
        hidden = _hidden_groups(entry)
        assert not hidden, (
            f"dependabot entry "
            f"{entry.get('directory') or entry.get('directories')}: "
            f"{hidden} (earlier group, later group, hidden pattern) - "
            f"first-match-wins means the earlier group captures them. Keep "
            f"specific groups above broader ones, and give groups that "
            f"share a glob distinct update-types, dependency-type or "
            f"applies-to."
        )


VITEST_GROUP = {"patterns": ["vitest", "@vitest/*"]}
MINOR_AND_PATCH = ["minor", "patch"]


@pytest.mark.parametrize(
    ("broader", "hidden"),
    [
        pytest.param(
            {"patterns": ["*"]}, ["vitest", "@vitest/*"], id="exact-wildcard"
        ),
        pytest.param({"patterns": ["vite*"]}, ["vitest"], id="prefix-glob"),
        pytest.param({"patterns": ["@*"]}, ["@vitest/*"], id="scope-glob"),
        pytest.param(
            {"patterns": ["Vite*"]}, ["vitest"], id="glob-in-other-case"
        ),
        pytest.param(
            {"patterns": ["*", "x"]},
            ["vitest", "@vitest/*"],
            id="wildcard-plus-name",
        ),
        pytest.param(
            {"dependency-type": "production"},
            ["vitest", "@vitest/*"],
            id="dependency-type-only",
        ),
        pytest.param(
            {"patterns": ["*"], "update-types": MINOR_AND_PATCH},
            ["vitest", "@vitest/*"],
            id="limited-to-update-types",
        ),
        pytest.param(
            {"patterns": ["*"], "exclude-patterns": ["vitest", "@vitest/*"]},
            ["vitest", "@vitest/*"],
            id="excluding-the-patterns",
        ),
    ],
)
def test_guard_flags_a_broader_group_above_a_specific_one(broader, hidden):
    entry = {"groups": {"broad": broader, "vitest": VITEST_GROUP}}

    assert _hidden_groups(entry) == [
        ("broad", "vitest", pattern) for pattern in hidden
    ]


@pytest.mark.parametrize(
    ("first", "second", "hidden"),
    [
        pytest.param(
            {"patterns": ["*"]},
            {"dependency-type": "production"},
            ["*"],
            id="dependency-type-only-below-wildcard",
        ),
        pytest.param(
            {"patterns": ["*"]},
            {"patterns": ["*"], "update-types": ["major"]},
            ["*"],
            id="wildcard-for-major-below-wildcard",
        ),
        pytest.param(
            {"patterns": ["*"]},
            {"patterns": ["*"], "exclude-patterns": ["x"]},
            ["*"],
            id="wildcard-excluding-below-wildcard",
        ),
        pytest.param(
            {"patterns": ["*"]},
            {"patterns": ["*"]},
            ["*"],
            id="wildcard-twice",
        ),
        pytest.param(
            VITEST_GROUP,
            VITEST_GROUP,
            ["vitest", "@vitest/*"],
            id="lockstep-pair-repeated-without-applies-to",
        ),
        pytest.param(
            {"patterns": ["Vitest"]},
            {"patterns": ["vitest"]},
            ["vitest"],
            id="same-name-in-other-case",
        ),
        pytest.param(
            {"patterns": ["*"], "update-types": MINOR_AND_PATCH},
            {"patterns": ["*"], "update-types": ["patch"]},
            ["*"],
            id="update-types-inside-the-earlier-group",
        ),
        pytest.param(
            {"patterns": ["*"], "update-types": ["major", "minor", "patch"]},
            {"patterns": ["*"]},
            ["*"],
            id="every-update-type-spelled-out",
        ),
        pytest.param(
            {"patterns": ["*"], "dependency-type": "production"},
            {"dependency-type": "production", "update-types": ["major"]},
            ["*"],
            id="same-dependency-type",
        ),
    ],
)
def test_guard_flags_a_group_the_earlier_one_takes_whole(first, second, hidden):
    entry = {"groups": {"first": first, "second": second}}

    assert _hidden_groups(entry) == [
        ("first", "second", pattern) for pattern in hidden
    ]


@pytest.mark.parametrize(
    "groups",
    [
        pytest.param(
            {"vitest": VITEST_GROUP, "broad": {"patterns": ["*"]}},
            id="specific-first",
        ),
        pytest.param(
            {
                "broad": {"patterns": ["*"], "applies-to": "security-updates"},
                "vitest": VITEST_GROUP,
            },
            id="separate-pool",
        ),
        pytest.param(
            {
                "unscoped": {"patterns": ["vite*"]},
                "scoped": {"patterns": ["@vitest/*"]},
            },
            id="disjoint-globs",
        ),
        pytest.param(
            {
                "minor-and-patch": {
                    "patterns": ["*"],
                    "update-types": MINOR_AND_PATCH,
                },
                "major": {"patterns": ["*"], "update-types": ["major"]},
            },
            id="split-by-update-types",
        ),
        pytest.param(
            {
                "production": {"dependency-type": "production"},
                "development": {"dependency-type": "development"},
            },
            id="split-by-dependency-type",
        ),
        pytest.param(
            {
                "minor-and-patch": {
                    "patterns": ["*"],
                    "update-types": MINOR_AND_PATCH,
                },
                "rest": {"patterns": ["*"]},
            },
            id="catch-all-after-update-types",
        ),
        pytest.param(
            {
                "production": {"dependency-type": "production"},
                "rest": {"patterns": ["*"]},
            },
            id="catch-all-after-dependency-type",
        ),
        pytest.param(
            {
                "suffix": {"patterns": ["*test"]},
                "prefix": {"patterns": ["vite*"]},
            },
            id="partly-overlapping-globs",
        ),
    ],
)
def test_guard_accepts_orderings_that_hide_nothing(groups):
    assert _hidden_groups({"groups": groups}) == []


def test_guard_flags_real_specific_groups_moved_below_the_wildcards():
    """The guard bites on this repository's own groups, not only stand-ins."""
    root = next(
        entry for entry in _npm_entries() if entry.get("directory") == "/"
    )
    reversed_groups = dict(reversed(list(root["groups"].items())))

    assert _hidden_groups(root) == []
    expected = {
        name
        for name, group in root["groups"].items()
        if group["patterns"] != ["*"]
    }
    assert {
        later for _, later, _ in _hidden_groups({"groups": reversed_groups})
    } == expected