Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

ci: run the MLX parity tests in a nightly job

John Costab1848d3f240d
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 3206f65..8e68613 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,6 +43,24 @@ jobs:
- run: pip install -e ".[test,train]"
- run: pytest -q -m "not slow"
+ # MLX parity against JAX (marked slow, run on the CPU device via mlx[cpu]).
+ # Kept out of `release`'s needs so a parity regression shows up nightly
+ # without blocking the publish.
+ test-mlx:
+ needs: gate
+ if: needs.gate.outputs.run == 'true'
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - run: pip install -e ".[test,train,mlx]"
+ # NEEDLE_REQUIRE_MLX turns a missing mlx into an error instead of a silent skip.
+ - run: pytest -q -m slow tests/test_mlx.py tests/test_quantize_mlx.py
+ env:
+ NEEDLE_REQUIRE_MLX: "1"
+
release:
needs: [gate, test]
if: needs.gate.outputs.run == 'true'
diff --git a/pyproject.toml b/pyproject.toml
index 9ab23f8..b62705c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,6 +28,8 @@ metal = [
mic = ["numpy", "soxr", "sounddevice"]
compare = ["openai-whisper", "moonshine-voice"]
test = ["pytest", "pydantic"]
+# mlx-cpu (the CPU backend Linux needs) is gated to Linux by mlx itself; macOS gets mlx + Metal.
+mlx = ["mlx[cpu]"]
[project.scripts]
needle = "needle.cli:main"
diff --git a/tests/conftest.py b/tests/conftest.py
index 67165e7..48f847d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,6 +38,15 @@ requires_engine = pytest.mark.skipif(
reason="needle C++ engine not installed (auto-fetched from HF on first real use)")
+def import_mlx():
+ """``mlx.core``, or skip the module; NEEDLE_REQUIRE_MLX=1 (CI) turns a missing mlx into an error."""
+ if os.environ.get("NEEDLE_REQUIRE_MLX") == "1":
+ import mlx.core
+
+ return mlx.core
+ return pytest.importorskip("mlx.core")
+
+
@pytest.fixture(scope="session")
def tiny_checkpoint(tmp_path_factory):
import numpy as np
diff --git a/tests/test_mlx.py b/tests/test_mlx.py
index f185787..37023fd 100644
--- a/tests/test_mlx.py
+++ b/tests/test_mlx.py
@@ -1,7 +1,8 @@
import numpy as np
import pytest
+from conftest import import_mlx
-mx = pytest.importorskip("mlx.core")
+mx = import_mlx()
jax = pytest.importorskip("jax")
pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]
diff --git a/tests/test_quantize_mlx.py b/tests/test_quantize_mlx.py
index 674c22b..06e9c44 100644
--- a/tests/test_quantize_mlx.py
+++ b/tests/test_quantize_mlx.py
@@ -1,7 +1,8 @@
import numpy as np
import pytest
+from conftest import import_mlx
-mx = pytest.importorskip("mlx.core")
+mx = import_mlx()
jax = pytest.importorskip("jax")
pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]