Personal AI, On Personal Devices
Python
fix(ci): gate live HuggingFace Hub download tests behind `hub` marker (#507)
Jon Saad-Falcon4523715bff11
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex cdc09d78..de995b2c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -64,7 +64,7 @@ jobs:- name: Run testsrun: |- uv run pytest tests/ -v --tb=short -m "not live and not cloud" \+ uv run pytest tests/ -v --tb=short -m "not live and not cloud and not hub" \--cov=openjarvis \--cov-report=term-missing \--cov-report=xml \diff --git a/pyproject.toml b/pyproject.tomlindex 7c718d9b..5b5a9120 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -174,6 +174,7 @@ markers = ["live_channel: requires real channel credentials (env vars)","nvidia: requires NVIDIA GPU","slow: long-running test",+ "hub: downloads real datasets from the HuggingFace Hub at runtime; excluded from the default CI lane (run with -m hub)","live_external: requires HERMES_AGENT_PATH and OPENCLAW_PATH; spawns real foreign-framework subprocesses","modal: requires Modal token + network; runs real swebench harness on Modal",]diff --git a/tests/evals/datasets/test_external_agent_datasets.py b/tests/evals/datasets/test_external_agent_datasets.pyindex bbfdc2df..c5b8ca1c 100644--- a/tests/evals/datasets/test_external_agent_datasets.py+++ b/tests/evals/datasets/test_external_agent_datasets.py@@ -10,6 +10,14 @@ These tests download each dataset once to ~/.cache/huggingface andverify the provider loads, iterates, and honours the split kwarg.The ModuleNotFoundError skip branch is defensive — currentlyunreachable since all three provider modules exist.++Marked ``hub``: they hit the live HuggingFace Hub, so they are excluded+from the default CI lane (which runs ``-m "not live and not cloud and not+hub"``) — a transient Hub outage or rate-limit must not redden ``main``.+Run them on demand with ``pytest -m hub``. The ADP provider swallows+per-config download errors and returns 0 records on a network failure+(see ``adp.py``), so a Hub outage surfaces here as ``assert 1 <= 0``+rather than an exception — another reason these can't run unguarded in CI."""from __future__ import annotations@@ -18,6 +26,8 @@ import importlibimport pytest+pytestmark = pytest.mark.hub+PROVIDERS = [("openjarvis.evals.datasets.adp", "ADPDataset"),("openjarvis.evals.datasets.toolorchestra", "ToolOrchestraDataset"),diff --git a/tests/evals/test_dataset_splits_integration.py b/tests/evals/test_dataset_splits_integration.pyindex e312ec21..b0955f17 100644--- a/tests/evals/test_dataset_splits_integration.py+++ b/tests/evals/test_dataset_splits_integration.py@@ -1,4 +1,13 @@-"""Integration test: each provider's split kwarg produces disjoint train/test slices."""+"""Integration test: each provider's split kwarg produces disjoint train/test slices.++Marked ``hub``: every provider here downloads a real corpus from the+HuggingFace Hub, so the module is excluded from the default CI lane (which+runs ``-m "not live and not cloud and not hub"``). A transient Hub outage+or rate-limit raises connectivity errors (``LocalEntryNotFoundError``,+``HfHubHTTPError``) that the gated/not-found skip branch below does not+catch — so running these unguarded made ``main`` flaky-red. Run on demand+with ``pytest -m hub``.+"""from __future__ import annotations@@ -6,6 +15,8 @@ import importlibimport pytest+pytestmark = pytest.mark.hub+PROVIDERS = [("openjarvis.evals.datasets.pinchbench", "PinchBenchDataset"),("openjarvis.evals.datasets.liveresearch", "LiveResearchBenchDataset"),