Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tensor-parallel to layer-split fallback for GGUF loads; kept free of FastAPI/httpx for unit tests."""
from __future__ import annotations
import logging
from typing import Awaitable, Callable, Optional
from core.inference.llama_server_args import (
_effective_tensor_parallel,
strip_split_mode_only,
)
logger = logging.getLogger(__name__)
async def load_with_tensor_fallback(
attempt_load: Callable[[bool, Optional[list[str]]], Awaitable[bool]],
*,
requested_tensor: bool,
extra_args: Optional[list[str]],
label: str = "",
cancelled: Optional[Callable[[], bool]] = None,
) -> bool:
"""Retry failed tensor-split load as --split-mode layer; a hard crash counts as failure, a user cancel is not retried, LLAMA_ARG_SPLIT_MODE=tensor is covered."""
tensor_requested = _effective_tensor_parallel(extra_args, requested_tensor)
try:
success = await attempt_load(requested_tensor, extra_args)
except Exception as exc:
if not tensor_requested:
raise
logger.warning("Tensor-parallel load raised for '%s': %s", label, exc)
success = False
if success or not tensor_requested:
return success
# Cancelled, not unsupported: do not relaunch.
if cancelled is not None and cancelled():
return success
logger.warning(
"Tensor-parallel load failed for '%s'; retrying with layer split "
"(this model may not support tensor parallelism)",
label,
)
# Force --split-mode layer (CLI beats env) so LLAMA_ARG_SPLIT_MODE=tensor cannot re-crash it.
layer_extras = strip_split_mode_only(extra_args) or []
return await attempt_load(False, [*layer_extras, "--split-mode", "layer"])