Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Do not scan a dep that only an extra installs, platform marker or not (#11582)

Daniel Han3b87ee8b53eb
diff --git a/scripts/scan_packages.py b/scripts/scan_packages.py
index 6efdc2290ee..af59247a625 100644
--- a/scripts/scan_packages.py
+++ b/scripts/scan_packages.py
@@ -1772,13 +1772,56 @@ _MARKER_ENV_VARS = (
)
+def _marker_can_hold_without_extras(parsed) -> bool:
+ """Can a parsed PEP 508 marker be true on SOME target with no extra requested?
+
+ Markers have no negation, only ``and``/``or`` over comparisons, so the formula is monotone in
+ its atoms: it can be true somewhere iff it is true with every non-``extra`` comparison set to
+ True and every ``extra`` comparison evaluated against the empty extra. That keeps
+ ``sys_platform == 'win32'`` and ``python_version >= '3.8' or extra == 'dev'`` (true on some
+ target) and drops ``extra == 'dev' and python_version >= '3.9'``, which no target installs
+ without the extra. Raises on a shape it does not know, so the caller keeps the dep.
+ """
+ groups: list[list[bool]] = [[]]
+ for item in parsed:
+ if isinstance(item, list):
+ groups[-1].append(_marker_can_hold_without_extras(item))
+ elif isinstance(item, tuple) and len(item) == 3:
+ lhs, op, rhs = item
+ names = {type(lhs).__name__, type(rhs).__name__}
+ if names != {"Variable", "Value"}:
+ raise ValueError(f"unexpected marker atom {item!r}")
+ variable = lhs if type(lhs).__name__ == "Variable" else rhs
+ if variable.value != "extra":
+ groups[-1].append(True)
+ continue
+ from packaging.markers import Marker
+
+ env = {"extra": ""}
+ text = f"{lhs.serialize()} {op.serialize()} {rhs.serialize()}"
+ groups[-1].append(bool(Marker(text).evaluate(env)))
+ elif item == "or":
+ groups.append([])
+ elif item == "and":
+ continue
+ else:
+ raise ValueError(f"unexpected marker token {item!r}")
+ return any(all(group) for group in groups)
+
+
def _marker_holds_by_default(marker: str) -> bool:
- """Keep (scan) a dep unless its marker is purely ``extra``-gated. The scanner runs on one OS/Python but a package may be installed on another, so a marker that can be true on a different target is always kept; only a marker depending solely on ``extra`` and false with no extra requested is dropped. Conservative: on any uncertainty, keep."""
+ """Keep (scan) a dep unless no install reaches it without an extra. The scanner runs on one OS/Python but a package may be installed on another, so a marker that can be true on a different target is always kept; a marker false on every target once no extra is requested (``extra == 'dev'``, ``extra == 'dev' and python_version >= '3.9'``) is dropped. Conservative: on any uncertainty, keep."""
m = marker.strip()
if not m or "extra" not in m:
return True # no extra gate: installed by default on some target -> scan
if any(v in m for v in _MARKER_ENV_VARS):
- return True # also platform/python gated: true on some target -> scan
+ # Also platform/python gated. Those atoms can each be true on some target, but an extra
+ # still has to be requested when it is AND-ed with them.
+ try:
+ from packaging.markers import Marker
+ return _marker_can_hold_without_extras(Marker(m)._markers)
+ except Exception:
+ return True # an unknown shape: keep, and scan it
# Pure extra marker: decide by evaluating with no extra requested.
try:
from packaging.markers import Marker, default_environment
diff --git a/tests/security/test_scan_packages.py b/tests/security/test_scan_packages.py
index 340dae3f386..2b008d4beaa 100644
--- a/tests/security/test_scan_packages.py
+++ b/tests/security/test_scan_packages.py
@@ -1740,7 +1740,41 @@ def test_marker_holds_by_default():
# package may install on another, so these deps must still be scanned.
assert sp._marker_holds_by_default("sys_platform == 'win32'") is True
assert sp._marker_holds_by_default("python_version == '3.13'") is True
- assert sp._marker_holds_by_default("sys_platform == 'win32' and extra == 'gpu'") is True
+ # ...but AND-ed with an extra they still need the extra: no target installs these by default.
+ assert sp._marker_holds_by_default("sys_platform == 'win32' and extra == 'gpu'") is False
+ assert sp._marker_holds_by_default('extra == "dev" and python_version >= "3.9"') is False
+ assert (
+ sp._marker_holds_by_default(
+ 'extra == "dev" and (python_version >= "3.8" and python_version < "3.9")'
+ )
+ is False
+ )
+ # An OR with a platform branch, or a default-true extra comparison, is reachable somewhere.
+ assert sp._marker_holds_by_default("extra == 'dev' or sys_platform == 'win32'") is True
+ assert (
+ sp._marker_holds_by_default(
+ "(extra == 'dev' and python_version >= '3.9') or sys_platform == 'win32'"
+ )
+ is True
+ )
+ assert sp._marker_holds_by_default("extra != 'dev' and sys_platform == 'win32'") is True
+
+
+def test_requires_dist_recovery_does_not_pull_a_dev_extra():
+ """loguru 0.7.3 gates its whole dev toolchain on `extra == "dev" and python_version ...`.
+ When its tree failed to co-resolve, the recovery path scanned pytest, tox, sphinx, mypy and
+ pre-commit as if a default install pulled them, and their findings failed the extras shard."""
+ meta = _meta(
+ [],
+ requires = [
+ 'colorama>=0.3.4; sys_platform == "win32"',
+ 'pre-commit==4.0.1; extra == "dev" and python_version >= "3.9"',
+ 'tox==4.23.2; extra == "dev" and python_version >= "3.8"',
+ 'pytest==8.3.2; extra == "dev" and python_version >= "3.8"',
+ 'sphinx==8.1.3; extra == "dev" and python_version >= "3.11"',
+ ],
+ )
+ assert sp._requires_dist_names(meta) == ["colorama>=0.3.4"]
def test_requires_dist_for_fails_closed_on_missing_pin_metadata(monkeypatch):