Skip to content

Personal AI, On Personal Devices

Python

fix(ci): gate live HuggingFace Hub download tests behind `hub` marker (#507)

Jon Saad-Falcon4523715bff11
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cdc09d78..de995b2c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,7 +64,7 @@ jobs:
- name: Run tests
run: |
- uv run pytest tests/ -v --tb=short -m "not live and not cloud" \
+ uv run pytest tests/ -v --tb=short -m "not live and not cloud and not hub" \
--cov=openjarvis \
--cov-report=term-missing \
--cov-report=xml \
diff --git a/pyproject.toml b/pyproject.toml
index 7c718d9b..5b5a9120 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -174,6 +174,7 @@ markers = [
"live_channel: requires real channel credentials (env vars)",
"nvidia: requires NVIDIA GPU",
"slow: long-running test",
+ "hub: downloads real datasets from the HuggingFace Hub at runtime; excluded from the default CI lane (run with -m hub)",
"live_external: requires HERMES_AGENT_PATH and OPENCLAW_PATH; spawns real foreign-framework subprocesses",
"modal: requires Modal token + network; runs real swebench harness on Modal",
]
diff --git a/tests/evals/datasets/test_external_agent_datasets.py b/tests/evals/datasets/test_external_agent_datasets.py
index bbfdc2df..c5b8ca1c 100644
--- a/tests/evals/datasets/test_external_agent_datasets.py
+++ b/tests/evals/datasets/test_external_agent_datasets.py
@@ -10,6 +10,14 @@ These tests download each dataset once to ~/.cache/huggingface and
verify the provider loads, iterates, and honours the split kwarg.
The ModuleNotFoundError skip branch is defensive — currently
unreachable since all three provider modules exist.
+
+Marked ``hub``: they hit the live HuggingFace Hub, so they are excluded
+from the default CI lane (which runs ``-m "not live and not cloud and not
+hub"``) — a transient Hub outage or rate-limit must not redden ``main``.
+Run them on demand with ``pytest -m hub``. The ADP provider swallows
+per-config download errors and returns 0 records on a network failure
+(see ``adp.py``), so a Hub outage surfaces here as ``assert 1 <= 0``
+rather than an exception — another reason these can't run unguarded in CI.
"""
from __future__ import annotations
@@ -18,6 +26,8 @@ import importlib
import pytest
+pytestmark = pytest.mark.hub
+
PROVIDERS = [
("openjarvis.evals.datasets.adp", "ADPDataset"),
("openjarvis.evals.datasets.toolorchestra", "ToolOrchestraDataset"),
diff --git a/tests/evals/test_dataset_splits_integration.py b/tests/evals/test_dataset_splits_integration.py
index e312ec21..b0955f17 100644
--- a/tests/evals/test_dataset_splits_integration.py
+++ b/tests/evals/test_dataset_splits_integration.py
@@ -1,4 +1,13 @@
-"""Integration test: each provider's split kwarg produces disjoint train/test slices."""
+"""Integration test: each provider's split kwarg produces disjoint train/test slices.
+
+Marked ``hub``: every provider here downloads a real corpus from the
+HuggingFace Hub, so the module is excluded from the default CI lane (which
+runs ``-m "not live and not cloud and not hub"``). A transient Hub outage
+or rate-limit raises connectivity errors (``LocalEntryNotFoundError``,
+``HfHubHTTPError``) that the gated/not-found skip branch below does not
+catch — so running these unguarded made ``main`` flaky-red. Run on demand
+with ``pytest -m hub``.
+"""
from __future__ import annotations
@@ -6,6 +15,8 @@ import importlib
import pytest
+pytestmark = pytest.mark.hub
+
PROVIDERS = [
("openjarvis.evals.datasets.pinchbench", "PinchBenchDataset"),
("openjarvis.evals.datasets.liveresearch", "LiveResearchBenchDataset"),