Skip to content

Personal AI, On Personal Devices

Python
21.4 KB
Raw Blame
"""Tests for the in-process Apple Foundation Models engine.

The real ``apple-fm-sdk`` builds Swift bindings and only runs on macOS 26+
with Apple Intelligence, and there is no macOS CI runner, so these tests
inject a stub SDK into ``sys.modules`` before importing the engine. The stub
mirrors the shapes the engine actually depends on:

* ``stream_response`` yields **cumulative** snapshots, not deltas
* ``token_count`` is async and takes either a value or a transcript
* ``is_available`` is an *instance* method returning ``(bool, reason)``

The pure helpers in ``_apple_fm_support`` need no stub at all and are tested
directly.
"""

from __future__ import annotations

import asyncio
import importlib
import sys
import types
from typing import Any

import pytest

from openjarvis.core.types import Message, Role

# ---------------------------------------------------------------------------
# Pure helpers — no SDK needed
# ---------------------------------------------------------------------------


class TestSnapshotAccumulator:
    def test_cumulative_snapshots_become_deltas(self):
        from openjarvis.engine._apple_fm_support import snapshot_deltas

        # The SDK re-sends everything generated so far on each iteration.
        snapshots = ["The", "The capital", "The capital is Paris."]
        assert snapshot_deltas(snapshots) == [
            "The",
            " capital",
            " is Paris.",
        ]

    def test_naive_concatenation_would_have_duplicated(self):
        from openjarvis.engine._apple_fm_support import SnapshotAccumulator

        acc = SnapshotAccumulator()
        for snap in ["a", "ab", "abc"]:
            acc.add(snap)
        # Appending each chunk (what every delta-based backend does) would
        # give "aababc"; the accumulator tracks the snapshot instead.
        assert acc.text == "abc"

    def test_repeated_snapshot_yields_nothing(self):
        from openjarvis.engine._apple_fm_support import snapshot_deltas

        assert snapshot_deltas(["ab", "ab", "abc"]) == ["ab", "c"]

    def test_revised_snapshot_is_reported_whole(self):
        """Guided generation can revise rather than extend."""
        from openjarvis.engine._apple_fm_support import snapshot_deltas

        assert snapshot_deltas(["hello wor", "goodbye"]) == ["hello wor", "goodbye"]


class TestModelLabels:
    def test_known_labels_normalize(self):
        from openjarvis.engine._apple_fm_support import validate_model_label

        assert validate_model_label("  AFM-3-Core  ") == "afm-3-core"

    def test_unknown_label_names_the_alternatives(self):
        from openjarvis.engine._apple_fm_support import validate_model_label

        with pytest.raises(ValueError, match="afm-3-core-advanced"):
            validate_model_label("afm-4")


class TestOptionsAndSampling:
    def test_max_tokens_maps_to_sdk_name(self):
        from openjarvis.engine._apple_fm_support import build_options_kwargs

        kwargs = build_options_kwargs({"max_tokens": "128", "temperature": "0.5"})
        assert kwargs == {"maximum_response_tokens": 128, "temperature": 0.5}

    def test_unset_values_defer_to_the_sdk(self):
        from openjarvis.engine._apple_fm_support import build_options_kwargs

        assert build_options_kwargs({}) == {}

    def test_sampling_defaults_to_greedy_for_reproducibility(self):
        from openjarvis.engine._apple_fm_support import parse_sampling_spec

        assert parse_sampling_spec({}) == ("greedy", {})

    def test_random_sampling_carries_its_fields(self):
        from openjarvis.engine._apple_fm_support import parse_sampling_spec

        mode, kwargs = parse_sampling_spec(
            {"sampling": "random", "seed": "7", "top": 3}
        )
        assert mode == "random"
        assert kwargs == {"seed": 7, "top": 3}

    def test_unknown_sampling_mode_rejected(self):
        from openjarvis.engine._apple_fm_support import parse_sampling_spec

        with pytest.raises(ValueError, match="greedy"):
            parse_sampling_spec({"sampling": "beam"})


# ---------------------------------------------------------------------------
# Stub SDK
# ---------------------------------------------------------------------------


class _Recorder(dict):
    pass


def _install_stub_sdk(
    *,
    available: tuple[bool, Any] = (True, None),
    snapshots: list[str] | None = None,
    token_counts: list[int] | None = None,
    raise_on_stream: str | None = None,
) -> _Recorder:
    """Inject a fake ``apple_fm_sdk`` and return a call recorder."""
    rec = _Recorder(
        options=[],
        sessions=[],
        stream_calls=[],
        token_count_calls=[],
        models=[],
    )
    sdk = types.ModuleType("apple_fm_sdk")

    class SystemLanguageModelUseCase:
        GENERAL = "general"
        CONTENT_TAGGING = "content_tagging"

    class SystemLanguageModelGuardrails:
        DEFAULT = "default"
        PERMISSIVE_CONTENT_TRANSFORMATIONS = "permissive"

    class SamplingMode:
        @staticmethod
        def greedy(**kw):
            return ("greedy", kw)

        @staticmethod
        def random(**kw):
            return ("random", kw)

    class GenerationOptions:
        def __init__(
            self, temperature=None, maximum_response_tokens=None, sampling=None
        ):
            self.temperature = temperature
            self.maximum_response_tokens = maximum_response_tokens
            self.sampling = sampling
            rec["options"].append(self)

    class ExceededContextWindowSizeError(Exception):
        pass

    class GuardrailViolationError(Exception):
        pass

    class RefusalError(Exception):
        pass

    class UnsupportedLanguageOrLocaleError(Exception):
        pass

    class SystemLanguageModel:
        context_size = 4096

        def __init__(self, use_case=None, guardrails=None):
            self.use_case = use_case
            self.guardrails = guardrails
            rec["models"].append(self)

        def is_available(self):
            return available

        async def token_count(self, value=None, *, instructions=None):
            rec["token_count_calls"].append(
                {
                    "value": value,
                    "instructions": instructions,
                }
            )
            counts = token_counts or []
            idx = len(rec["token_count_calls"]) - 1
            return counts[idx] if idx < len(counts) else 0

    class _Transcript:
        pass

    class LanguageModelSession:
        def __init__(self, instructions=None, model=None, tools=None):
            self.instructions = instructions
            self.model = model
            self.transcript = _Transcript()
            rec["sessions"].append(self)

        async def stream_response(self, prompt, options=None):
            rec["stream_calls"].append({"prompt": prompt, "options": options})
            if raise_on_stream is not None:
                # Named rather than passed as an instance so callers don't
                # need a stub SDK in hand before installing one.
                raise getattr(sdk, raise_on_stream)("stub failure")
            for snap in snapshots or []:
                yield snap

    sdk.SystemLanguageModelUseCase = SystemLanguageModelUseCase
    sdk.SystemLanguageModelGuardrails = SystemLanguageModelGuardrails
    sdk.SamplingMode = SamplingMode
    sdk.GenerationOptions = GenerationOptions
    sdk.SystemLanguageModel = SystemLanguageModel
    sdk.LanguageModelSession = LanguageModelSession
    sdk.ExceededContextWindowSizeError = ExceededContextWindowSizeError
    sdk.GuardrailViolationError = GuardrailViolationError
    sdk.RefusalError = RefusalError
    sdk.UnsupportedLanguageOrLocaleError = UnsupportedLanguageOrLocaleError

    sys.modules["apple_fm_sdk"] = sdk
    return rec


def _load_engine_module():
    """Import a fresh engine module against whatever stub is installed.

    ``tests/conftest.py`` wipes ``EngineRegistry`` between tests, so the
    module must be re-imported (not just fetched from the registry) for the
    ``@EngineRegistry.register`` decorator to run again.
    """
    sys.modules.pop("openjarvis.engine.apple_fm", None)
    return importlib.import_module("openjarvis.engine.apple_fm")


@pytest.fixture
def engine_mod():
    created = []

    def _make(**stub_kwargs):
        rec = _install_stub_sdk(**stub_kwargs)
        mod = _load_engine_module()
        created.append(mod)
        return mod, rec

    yield _make
    sys.modules.pop("openjarvis.engine.apple_fm", None)
    sys.modules.pop("apple_fm_sdk", None)


def _engine(mod, **kwargs):
    eng = mod.AppleFMEngine(**kwargs)
    eng._prepared = True
    eng._context_size = 4096
    return eng


# ---------------------------------------------------------------------------
# Engine behaviour
# ---------------------------------------------------------------------------


class TestRegistration:
    def test_registers_under_afm(self, engine_mod):
        from openjarvis.core.registry import EngineRegistry

        mod, _ = engine_mod()
        assert EngineRegistry.get("afm") is mod.AppleFMEngine
        assert mod.AppleFMEngine.engine_id == "afm"

    def test_rejects_unknown_use_case(self, engine_mod):
        mod, _ = engine_mod()
        with pytest.raises(ValueError, match="content_tagging"):
            mod.AppleFMEngine(use_case="nonsense")

    def test_rejects_unknown_sampling_at_construction(self, engine_mod):
        """A typo must fail now, not midway through a long run."""
        mod, _ = engine_mod()
        with pytest.raises(ValueError, match="greedy"):
            mod.AppleFMEngine(sampling="beam")


class TestServeability:
    def test_only_serves_afm_labels(self, engine_mod):
        """Unlike other local engines, this one must not accept any model id.

        There is exactly one on-device model, so accepting arbitrary ids would
        let engine selection route a Llama request here and answer with a
        different model entirely.
        """
        mod, _ = engine_mod()
        eng = _engine(mod)
        assert eng.can_serve("afm-3-core") is True
        assert eng.can_serve("llama3.2:3b") is False


class TestHealth:
    def test_reports_unavailable_reason(self, engine_mod, caplog):
        mod, _ = engine_mod(available=(False, types.SimpleNamespace(name="ASSETS")))
        eng = mod.AppleFMEngine()
        with caplog.at_level("WARNING"):
            assert eng.health() is False
        assert "ASSETS" in caplog.text


class TestGenerate:
    def test_returns_content_and_real_token_counts(self, engine_mod):
        # token_count call order: transcript-before, prompt, transcript-after
        mod, rec = engine_mod(
            snapshots=["Paris", "Paris is", "Paris is the capital."],
            token_counts=[12, 8, 27],
        )
        eng = _engine(mod)

        result = eng.generate(
            [Message(role=Role.USER, content="Capital of France?")],
            model="afm-3-core",
            max_tokens=64,
        )
        eng.close()

        assert result["content"] == "Paris is the capital."
        # prompt = transcript-before (12) + prompt alone (8)
        assert result["usage"]["prompt_tokens"] == 20
        # completion = transcript-after (27) - prompt (20)
        assert result["usage"]["completion_tokens"] == 7
        assert result["usage"]["total_tokens"] == 27
        assert result["finish_reason"] == "stop"

    def test_system_message_becomes_session_instructions(self, engine_mod):
        """Not folded into the prompt text.

        The shim historically prefixed system content onto the prompt, which
        both loses the distinction the SDK draws and inflates prompt tokens.
        """
        mod, rec = engine_mod(snapshots=["ok"], token_counts=[1, 1, 2])
        eng = _engine(mod)
        eng.generate(
            [
                Message(role=Role.SYSTEM, content="Be terse."),
                Message(role=Role.USER, content="Hi"),
            ],
            model="afm-3",
        )
        eng.close()

        assert rec["sessions"][-1].instructions == "Be terse."
        assert rec["stream_calls"][-1]["prompt"] == "Hi"

    def test_configured_instructions_precede_request_system_message(self, engine_mod):
        mod, rec = engine_mod(snapshots=["ok"], token_counts=[1, 1, 2])
        eng = _engine(mod, instructions="Always answer in English.")
        eng.generate(
            [
                Message(role=Role.SYSTEM, content="Be terse."),
                Message(role=Role.USER, content="Hi"),
            ],
            model="afm-3",
        )
        eng.close()
        assert (
            rec["sessions"][-1].instructions == "Always answer in English.\nBe terse."
        )

    def test_generation_options_carry_sampling_and_limits(self, engine_mod):
        mod, rec = engine_mod(snapshots=["x"], token_counts=[0, 0, 1])
        eng = _engine(mod, sampling="random")
        eng.generate(
            [Message(role=Role.USER, content="Hi")],
            model="afm-3",
            temperature=0.25,
            max_tokens=99,
        )
        eng.close()

        opts = rec["stream_calls"][-1]["options"]
        assert opts.temperature == 0.25
        assert opts.maximum_response_tokens == 99
        assert opts.sampling == ("random", {})

    def test_each_request_gets_a_fresh_session(self, engine_mod):
        """Reused sessions accumulate a transcript that overflows the 4096-token
        context after a handful of requests, and would make per-request energy
        depend on the requests before it."""
        mod, rec = engine_mod(snapshots=["x"], token_counts=[0, 0, 1] * 3)
        eng = _engine(mod)
        msgs = [Message(role=Role.USER, content="Hi")]
        for _ in range(3):
            eng.generate(msgs, model="afm-3")
        eng.close()

        assert len(rec["sessions"]) == 3
        assert len({id(s) for s in rec["sessions"]}) == 3

    def test_token_count_failure_degrades_to_zero_not_an_error(self, engine_mod):
        mod, _ = engine_mod(snapshots=["hi"])
        eng = _engine(mod)

        async def _boom(*a, **k):
            raise RuntimeError("counter unavailable")

        eng._ensure_model().token_count = _boom

        result = eng.generate([Message(role=Role.USER, content="Hi")], model="afm-3")
        eng.close()
        assert result["content"] == "hi"
        assert result["usage"]["completion_tokens"] == 0


class TestErrorMapping:
    def test_context_overflow_becomes_a_context_length_error(self, engine_mod):
        from openjarvis.engine._base import (
            EngineContextLengthError,
            looks_like_context_length_error,
        )

        mod, _ = engine_mod(raise_on_stream="ExceededContextWindowSizeError")
        eng = _engine(mod)
        with pytest.raises(EngineContextLengthError) as exc:
            eng.generate([Message(role=Role.USER, content="x" * 100)], model="afm-3")
        eng.close()
        # The shared heuristic must recognise the phrasing too, so callers
        # branching on text rather than type still get "conversation too long".
        assert looks_like_context_length_error(str(exc.value))
        # The SDK formats its messages as "<reason>: <detail>" and passes
        # None as the detail, so the raw text ends in ": None". That must not
        # reach the user -- it reads as a broken error path rather than a
        # plain overflow, and it lands in every affected eval row.
        assert not str(exc.value).rstrip().endswith("None")
        assert "4096" in str(exc.value)

    def test_context_error_does_not_repeat_the_sdk_reason(self, engine_mod):
        from openjarvis.engine._base import EngineContextLengthError

        mod, _ = engine_mod(raise_on_stream="ExceededContextWindowSizeError")
        eng = _engine(mod)
        with pytest.raises(EngineContextLengthError) as exc:
            eng.generate([Message(role=Role.USER, content="x")], model="afm-3")
        eng.close()
        # "context window" should appear once, from our own sentence.
        assert str(exc.value).lower().count("context window") == 1

    def test_refusal_returns_empty_instead_of_killing_the_run(self, engine_mod):
        """A declined prompt is a per-query outcome, not an engine fault.

        A long eval run must not die because one prompt tripped a guardrail.
        """
        mod, _ = engine_mod(
            raise_on_stream="GuardrailViolationError",
            token_counts=[3, 5],
        )
        eng = _engine(mod)
        result = eng.generate([Message(role=Role.USER, content="...")], model="afm-3")

        assert result["content"] == ""
        assert result["finish_reason"] == "content_filter"
        assert eng._skipped == {"GuardrailViolationError": 1}
        eng.close()


class TestStreaming:
    def test_stream_yields_deltas_that_reassemble(self, engine_mod):
        mod, _ = engine_mod(
            snapshots=["The", "The cap", "The capital is Paris."],
            token_counts=[4, 6, 20],
        )
        eng = _engine(mod)

        async def _collect():
            return [
                c
                async for c in eng.stream(
                    [Message(role=Role.USER, content="Hi")], model="afm-3"
                )
            ]

        chunks = asyncio.run(_collect())
        eng.close()
        assert "".join(chunks) == "The capital is Paris."

    def test_stream_full_reports_usage_at_end(self, engine_mod):
        mod, _ = engine_mod(snapshots=["ab", "abc"], token_counts=[4, 6, 15])
        eng = _engine(mod)

        async def _collect():
            return [
                c
                async for c in eng.stream_full(
                    [Message(role=Role.USER, content="Hi")], model="afm-3"
                )
            ]

        chunks = asyncio.run(_collect())
        eng.close()
        final = chunks[-1]
        assert final.finish_reason == "stop"
        assert final.usage["prompt_tokens"] == 10
        assert final.usage["completion_tokens"] == 5


class TestPrepare:
    def test_warms_up_and_records_context_size(self, engine_mod):
        """Warmup runs inside prepare() so the ~10s cold start lands outside
        the first request's energy window."""
        mod, rec = engine_mod(snapshots=["w"])
        eng = mod.AppleFMEngine()
        eng.prepare("afm-3-core")
        eng.close()

        assert eng._context_size == 4096
        assert rec["stream_calls"][0]["prompt"] == "warmup"
        assert rec["stream_calls"][0]["options"].maximum_response_tokens == 4

    def test_unavailable_model_raises_connection_error(self, engine_mod):
        from openjarvis.engine._base import EngineConnectionError

        mod, _ = engine_mod(
            available=(False, types.SimpleNamespace(name="APPLE_INTELLIGENCE_OFF"))
        )
        eng = mod.AppleFMEngine()
        with pytest.raises(EngineConnectionError, match="APPLE_INTELLIGENCE_OFF"):
            eng.prepare("afm-3-core")

    def test_bad_label_rejected_before_touching_the_model(self, engine_mod):
        mod, _ = engine_mod()
        eng = mod.AppleFMEngine()
        with pytest.raises(ValueError, match="afm-3-core-advanced"):
            eng.prepare("gpt-4")


class TestDescribe:
    def test_records_what_the_variant_cannot_tell_us(self, engine_mod):
        mod, _ = engine_mod()
        eng = _engine(mod, use_case="content_tagging")
        info = eng.describe()
        eng.close()

        assert info["engine_id"] == "afm"
        assert info["afm_use_case"] == "content_tagging"
        assert info["afm_context_size"] == 4096
        # The executing variant is not observable, so the host chip is how a
        # run gets attributed after the fact.
        assert info["host_chip"]
        assert "dynamic profile" in info["variant_selection"]


# ---------------------------------------------------------------------------
# On-hardware smoke test
# ---------------------------------------------------------------------------


@pytest.mark.apple
@pytest.mark.macos15
class TestOnRealHardware:
    def test_generates_with_real_token_counts(self):
        sys.modules.pop("apple_fm_sdk", None)
        sys.modules.pop("openjarvis.engine.apple_fm", None)
        pytest.importorskip("apple_fm_sdk")
        mod = importlib.import_module("openjarvis.engine.apple_fm")

        eng = mod.AppleFMEngine(instructions="Answer in one word.")
        if not eng.health():
            pytest.skip("Apple Intelligence unavailable on this host")

        result = eng.generate(
            [Message(role=Role.USER, content="Name one planet.")],
            model="afm-3-core",
            max_tokens=16,
        )
        eng.close()

        assert result["content"].strip()
        # The whole point of the 0.2.1 SDK floor: the shim reported 0 here,
        # which zeroed throughput and every per-token energy figure.
        assert result["usage"]["completion_tokens"] > 0
        assert result["usage"]["prompt_tokens"] > 0


class TestAfmIsTreatedAsLocal:
    """`afm` must be recognised as an on-device engine everywhere it matters.

    The Python SDK has no Private Cloud Compute path, so AFM inference never
    leaves the machine. An engine key missing from these sets produces a false
    "your data is leaving this machine" signal.
    """

    def test_data_boundary_audit_classifies_afm_as_local(self):
        from openjarvis.security.data_boundary_audit import (
            LOCAL_ENGINE_KEYS,
            _target_is_cloud,
        )

        assert "afm" in LOCAL_ENGINE_KEYS
        assert _target_is_cloud("afm", "afm-3-core") is False

    def test_image_privacy_guard_does_not_warn_for_afm(self):
        """`jarvis ask -i photo.png --engine afm` must not claim the image
        leaves the machine."""
        from openjarvis.cli.ask import LOCAL_ENGINES

        assert "afm" in LOCAL_ENGINES
        assert "openai" not in LOCAL_ENGINES