Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.0 KB
Raw Blame
#!/usr/bin/env python3
"""Multi-GPU selection, memory estimate and device_map logic, tested with all hardware mocked."""

import os
import sys
import unittest
from pathlib import Path
from unittest.mock import patch, MagicMock

_backend_root = Path(__file__).resolve().parent.parent
if str(_backend_root) not in sys.path:
    sys.path.insert(0, str(_backend_root))


def _make_fake_config(
    vocab_size = 32000,
    hidden_size = 4096,
    intermediate_size = 11008,
    num_hidden_layers = 32,
    num_attention_heads = 32,
    num_key_value_heads = 8,
    tie_word_embeddings = False,
):
    """Fake HF config-like object for estimation tests."""
    from types import SimpleNamespace
    return SimpleNamespace(
        vocab_size = vocab_size,
        hidden_size = hidden_size,
        intermediate_size = intermediate_size,
        num_hidden_layers = num_hidden_layers,
        num_attention_heads = num_attention_heads,
        num_key_value_heads = num_key_value_heads,
        tie_word_embeddings = tie_word_embeddings,
    )


class TestEstimateFP16ModelSizeFromConfig(unittest.TestCase):
    """Config-based model size estimation."""

    def test_llama_8b_size_reasonable(self):
        from utils.hardware.hardware import _estimate_fp16_model_size_bytes_from_config

        config = _make_fake_config(
            vocab_size = 128256,
            hidden_size = 4096,
            intermediate_size = 14336,
            num_hidden_layers = 32,
            num_attention_heads = 32,
            num_key_value_heads = 8,
            tie_word_embeddings = False,
        )
        size = _estimate_fp16_model_size_bytes_from_config(config)
        self.assertIsNotNone(size)
        size_gb = size / (1024**3)
        self.assertGreater(size_gb, 12)
        self.assertLess(size_gb, 20)

    def test_small_model(self):
        from utils.hardware.hardware import _estimate_fp16_model_size_bytes_from_config

        config = _make_fake_config(
            vocab_size = 32000,
            hidden_size = 2048,
            intermediate_size = 5504,
            num_hidden_layers = 22,
            num_attention_heads = 32,
            num_key_value_heads = 4,
        )
        size = _estimate_fp16_model_size_bytes_from_config(config)
        self.assertIsNotNone(size)
        size_gb = size / (1024**3)
        self.assertGreater(size_gb, 1)
        self.assertLess(size_gb, 5)

    def test_returns_none_for_incomplete_config(self):
        from utils.hardware.hardware import _estimate_fp16_model_size_bytes_from_config
        from types import SimpleNamespace

        config = SimpleNamespace(vocab_size = 32000)
        size = _estimate_fp16_model_size_bytes_from_config(config)
        self.assertIsNone(size)

    def test_moe_model(self):
        from utils.hardware.hardware import _estimate_fp16_model_size_bytes_from_config
        from types import SimpleNamespace

        config = SimpleNamespace(
            vocab_size = 152064,
            hidden_size = 3584,
            intermediate_size = 18944,
            num_hidden_layers = 28,
            num_attention_heads = 28,
            num_key_value_heads = 4,
            tie_word_embeddings = False,
            num_local_experts = 64,
            moe_intermediate_size = 2560,
        )
        size = _estimate_fp16_model_size_bytes_from_config(config)
        self.assertIsNotNone(size)
        size_gb = size / (1024**3)
        self.assertGreater(size_gb, 50)


class TestEstimateRequiredModelMemory(unittest.TestCase):
    """Memory requirement estimation."""

    def test_inference_fp16_uses_1_3x(self):
        from utils.hardware.hardware import estimate_required_model_memory_gb
        with patch(
            "utils.hardware.hardware.estimate_fp16_model_size_bytes",
            return_value = (10 * (1024**3), "config"),
        ):
            required, meta = estimate_required_model_memory_gb(
                "test/model",
                training_type = None,
                load_in_4bit = False,
            )
            self.assertIsNotNone(required)
            self.assertAlmostEqual(required, 13.0, places = 0)
            self.assertEqual(meta["mode"], "inference")

    def test_inference_4bit_uses_reduced_estimate(self):
        from utils.hardware.hardware import estimate_required_model_memory_gb
        with patch(
            "utils.hardware.hardware.estimate_fp16_model_size_bytes",
            return_value = (30 * (1024**3), "config"),
        ):
            required, meta = estimate_required_model_memory_gb(
                "test/model",
                training_type = None,
                load_in_4bit = True,
            )
            self.assertIsNotNone(required)
            # 30/3.2 + max(9.375*0.3, 2) = 12.19GB
            self.assertAlmostEqual(required, 12.2, places = 0)

    def test_4bit_training_reduces_base(self):
        from utils.hardware.hardware import estimate_required_model_memory_gb
        with patch(
            "utils.hardware.hardware.estimate_fp16_model_size_bytes",
            return_value = (30 * (1024**3), "config"),
        ):
            required, meta = estimate_required_model_memory_gb(
                "test/model",
                training_type = "LoRA/QLoRA",
                load_in_4bit = True,
            )
            self.assertIsNotNone(required)
            # base=9.375, lora=1.2, act=4.5, cuda=1.4
            self.assertAlmostEqual(required, 16.5, places = 0)

    def test_full_finetune_uses_3_5x(self):
        from utils.hardware.hardware import estimate_required_model_memory_gb
        with patch(
            "utils.hardware.hardware.estimate_fp16_model_size_bytes",
            return_value = (10 * (1024**3), "config"),
        ):
            required, meta = estimate_required_model_memory_gb(
                "test/model",
                training_type = "Full Finetuning",
            )
            self.assertIsNotNone(required)
            # 10 * 3.5 + 1.4 cuda overhead
            self.assertAlmostEqual(required, 36.4, places = 0)

    def test_returns_none_when_unavailable(self):
        from utils.hardware.hardware import estimate_required_model_memory_gb
        with patch(
            "utils.hardware.hardware.estimate_fp16_model_size_bytes",
            return_value = (None, "unavailable"),
        ):
            required, meta = estimate_required_model_memory_gb("test/model")
            self.assertIsNone(required)


class TestAutoSelectGpuIds(unittest.TestCase):
    """Automatic GPU selection by model size and free memory."""

    def _make_utilization(self, devices):
        """Fake utilization response."""
        return {
            "available": True,
            "devices": [
                {
                    "index": idx,
                    "vram_total_gb": total,
                    "vram_used_gb": total - free,
                }
                for idx, total, free in devices
            ],
        }

    def test_single_gpu_sufficient(self):
        from utils.hardware.hardware import auto_select_gpu_ids
        import utils.hardware.hardware as hw
        with (
            patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA),
            patch.object(
                hw,
                "estimate_required_model_memory_gb",
                return_value = (
                    10.0,
                    {
                        "mode": "inference",
                        "required_gb": 10.0,
                        "model_size_source": "config",
                        "model_size_gb": 7.7,
                    },
                ),
            ),
            patch.object(
                hw,
                "_get_parent_visible_gpu_spec",
                return_value = {
                    "raw": "0,1,2,3",
                    "numeric_ids": [0, 1, 2, 3],
                    "supports_explicit_gpu_ids": True,
                },
            ),
            patch.object(hw, "get_parent_visible_gpu_ids", return_value = [0, 1, 2, 3]),
            patch.object(
                hw,
                "get_visible_gpu_utilization",
                return_value = self._make_utilization(
                    [
                        (0, 80.0, 75.0),
                        (1, 80.0, 78.0),
                        (2, 80.0, 70.0),
                        (3, 80.0, 72.0),
                    ]
                ),
            ),
        ):
            selected, meta = auto_select_gpu_ids("test/model")
            self.assertEqual(len(selected), 1)
            self.assertEqual(selected[0], 1)

    def test_two_gpus_needed(self):
        from utils.hardware.hardware import auto_select_gpu_ids
        import utils.hardware.hardware as hw
        with (
            patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA),
            patch.object(
                hw,
                "estimate_required_model_memory_gb",
                return_value = (
                    50.0,
                    {
                        "mode": "inference",
                        "required_gb": 50.0,
                        "model_size_source": "config",
                        "model_size_gb": 38.0,
                    },
                ),
            ),
            patch.object(
                hw,
                "_get_parent_visible_gpu_spec",
                return_value = {
                    "raw": "0,1",
                    "numeric_ids": [0, 1],
                    "supports_explicit_gpu_ids": True,
                },
            ),
            patch.object(hw, "get_parent_visible_gpu_ids", return_value = [0, 1]),
            patch.object(
                hw,
                "get_visible_gpu_utilization",
                return_value = self._make_utilization(
                    [
                        (0, 40.0, 30.0),
                        (1, 40.0, 35.0),
                    ]
                ),
            ),
        ):
            selected, meta = auto_select_gpu_ids("test/model")
            self.assertEqual(len(selected), 2)

    def test_non_accelerator_returns_none(self):
        from utils.hardware.hardware import auto_select_gpu_ids
        import utils.hardware.hardware as hw
        with patch.object(hw, "get_device", return_value = hw.DeviceType.CPU):
            selected, meta = auto_select_gpu_ids("test/model")
            self.assertIsNone(selected)
            self.assertEqual(meta["selection_mode"], "non_accelerator")


class TestGetDeviceMap(unittest.TestCase):
    """device_map string generation."""

    def test_single_gpu_returns_sequential(self):
        from utils.hardware.hardware import get_device_map
        import utils.hardware.hardware as hw
        with (
            patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA),
            patch.object(
                hw,
                "_get_parent_visible_gpu_spec",
                return_value = {
                    "raw": "0",
                    "numeric_ids": [0],
                    "supports_explicit_gpu_ids": True,
                },
            ),
            patch.object(hw, "get_visible_gpu_count", return_value = 1),
        ):
            dm = get_device_map(gpu_ids = [0])
            self.assertEqual(dm, "sequential")

    def test_multi_gpu_returns_the_unsloth_planner(self):
        from utils.hardware.hardware import get_device_map
        import utils.hardware.hardware as hw
        with patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA):
            dm = get_device_map(gpu_ids = [0, 1])
            self.assertEqual(dm, "unsloth_balanced")

    def test_cpu_returns_sequential(self):
        from utils.hardware.hardware import get_device_map
        import utils.hardware.hardware as hw
        with patch.object(hw, "get_device", return_value = hw.DeviceType.CPU):
            dm = get_device_map(gpu_ids = None)
            self.assertEqual(dm, "sequential")


class TestResolveRequestedGpuIds(unittest.TestCase):
    """GPU ID validation."""

    def test_none_returns_parent_visible(self):
        from utils.hardware.hardware import resolve_requested_gpu_ids
        with (
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2,3"}, clear = False),
            patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 8),
        ):
            result = resolve_requested_gpu_ids(None)
            self.assertEqual(result, [2, 3])

    def test_empty_list_returns_parent_visible(self):
        from utils.hardware.hardware import resolve_requested_gpu_ids
        with (
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2,3"}, clear = False),
            patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 8),
        ):
            result = resolve_requested_gpu_ids([])
            self.assertEqual(result, [2, 3])

    def test_duplicates_rejected(self):
        from utils.hardware.hardware import resolve_requested_gpu_ids
        with (
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0,1,2"}, clear = False),
            patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 8),
        ):
            with self.assertRaises(ValueError):
                resolve_requested_gpu_ids([1, 1])

    def test_out_of_range_rejected(self):
        from utils.hardware.hardware import resolve_requested_gpu_ids
        with (
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0,1"}, clear = False),
            patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 4),
        ):
            with self.assertRaises(ValueError):
                resolve_requested_gpu_ids([5])

    def test_uuid_env_var_rejects_explicit_ids(self):
        from utils.hardware.hardware import resolve_requested_gpu_ids
        with (
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "GPU-abc,GPU-def"}, clear = False),
            patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 8),
        ):
            with self.assertRaises(ValueError):
                resolve_requested_gpu_ids([0])


class TestApplyGpuIds(unittest.TestCase):
    """CUDA_VISIBLE_DEVICES environment variable setting."""

    def test_apply_list(self):
        from utils.hardware.hardware import apply_gpu_ids
        with patch.dict(os.environ, {}, clear = False):
            apply_gpu_ids([3, 5])
            self.assertEqual(os.environ.get("CUDA_VISIBLE_DEVICES"), "3,5")

    def test_apply_none_does_nothing(self):
        from utils.hardware.hardware import apply_gpu_ids

        original = os.environ.get("CUDA_VISIBLE_DEVICES")
        apply_gpu_ids(None)
        self.assertEqual(os.environ.get("CUDA_VISIBLE_DEVICES"), original)


class TestMultiGpuOverheadAccounting(unittest.TestCase):
    """The first GPU keeps its full free memory; the overhead factor applies only to additional GPUs."""

    def _make_utilization(self, devices):
        return {
            "available": True,
            "devices": [
                {
                    "index": idx,
                    "vram_total_gb": total,
                    "vram_used_gb": total - free,
                }
                for idx, total, free in devices
            ],
        }

    def test_first_gpu_not_penalized(self):
        """A model that just fits on 1 GPU should not require 2 GPUs."""
        from utils.hardware.hardware import auto_select_gpu_ids
        import utils.hardware.hardware as hw

        with (
            patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA),
            patch.object(
                hw,
                "estimate_required_model_memory_gb",
                return_value = (
                    79.0,
                    {
                        "mode": "inference",
                        "required_gb": 79.0,
                        "model_size_source": "config",
                        "model_size_gb": 60.0,
                    },
                ),
            ),
            patch.object(
                hw,
                "_get_parent_visible_gpu_spec",
                return_value = {
                    "raw": "0,1",
                    "numeric_ids": [0, 1],
                    "supports_explicit_gpu_ids": True,
                },
            ),
            patch.object(hw, "get_parent_visible_gpu_ids", return_value = [0, 1]),
            patch.object(
                hw,
                "get_visible_gpu_utilization",
                return_value = self._make_utilization(
                    [
                        (0, 80.0, 80.0),
                        (1, 80.0, 80.0),
                    ]
                ),
            ),
        ):
            selected, meta = auto_select_gpu_ids("test/model")
            self.assertEqual(len(selected), 1)

    def test_second_gpu_has_overhead(self):
        """When 2 GPUs are needed, the second one's contribution is reduced."""
        from utils.hardware.hardware import auto_select_gpu_ids
        import utils.hardware.hardware as hw

        # 80 + 40*0.85 = 114GB >= 110GB required
        with (
            patch.object(hw, "get_device", return_value = hw.DeviceType.CUDA),
            patch.object(
                hw,
                "estimate_required_model_memory_gb",
                return_value = (
                    110.0,
                    {
                        "mode": "inference",
                        "required_gb": 110.0,
                        "model_size_source": "config",
                        "model_size_gb": 85.0,
                    },
                ),
            ),
            patch.object(
                hw,
                "_get_parent_visible_gpu_spec",
                return_value = {
                    "raw": "0,1",
                    "numeric_ids": [0, 1],
                    "supports_explicit_gpu_ids": True,
                },
            ),
            patch.object(hw, "get_parent_visible_gpu_ids", return_value = [0, 1]),
            patch.object(
                hw,
                "get_visible_gpu_utilization",
                return_value = self._make_utilization(
                    [
                        (0, 80.0, 80.0),
                        (1, 80.0, 40.0),
                    ]
                ),
            ),
        ):
            selected, meta = auto_select_gpu_ids("test/model")
            self.assertEqual(len(selected), 2)


if __name__ == "__main__":
    unittest.main()