Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Clear the duplicate-definition backlog and gate it at zero (#9769)

Rajarshi Dattab349936f09fa
diff --git a/.github/workflows/lint-ci.yml b/.github/workflows/lint-ci.yml
index 269886ee65f..6c8886aab09 100644
--- a/.github/workflows/lint-ci.yml
+++ b/.github/workflows/lint-ci.yml
@@ -292,11 +292,8 @@ jobs:
# duplicate that was already in the file is printed and ignored while one
# this PR introduced fails. Comparing added LINES instead would miss the
# case where the new copy is inserted ABOVE the old one, since the finding
- # is reported at the second, unchanged definition. There are 37 pre-existing
- # findings on main today; they are a separate cleanup, not a reason to leave
- # the gate off. Four of those are only visible once each control-flow branch is
- # scanned on its own: a repeated `from unsloth import is_bfloat16_supported`
- # inside one `if __name__ == "__main__":` block in four of the saving tests.
+ # is reported at the second, unchanged definition. The whole-tree step below
+ # catches pre-existing ones.
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
@@ -338,6 +335,13 @@ jobs:
python scripts/lint_duplicate_definitions.py \
--before "$MERGE_BASE" --after "$HEAD_SHA" "${CHANGED[@]}"
+ - name: Duplicate definitions (whole tree)
+ # The diff-scoped step passes pre-existing findings, so the backlog regrew
+ # to 38 while it was the only gate. Runs on push too.
+ run: |
+ python scripts/lint_duplicate_definitions.py \
+ unsloth unsloth_cli studio scripts tests cli.py unsloth-cli.py
+
- name: No silent llama-server parallel-slot downgrades
# #7717 clamped --parallel to 1 whenever MTP resolved, costing batched
# callers 4x throughput. Launch fewer slots only for a real capability
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 3a14f6edf2e..e7847441e52 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -5204,8 +5204,6 @@ _MAX_PATH_SCAN_CHARS = 2048
_MAX_TERMINAL_SCAN_CHARS = 4096
# Each pass doubles resolved alias hops; leftover work after the cap fails closed.
_MAX_SHELL_ASSIGN_EXPAND_PASSES = 16
-# A glob needs one of these to expand into anything but itself; used to skip the glob scans outright.
-_GLOB_META_RE = re.compile(r"[?*\[]")
# Where the memoised node list is parked on a parsed tree (see _tree_nodes).
_TREE_NODES_ATTR = "_unsloth_walk_nodes"
@@ -7448,9 +7446,6 @@ _OPENSSL_NETWORK_RE = re.compile(
# -c/eval it runs an unscreened payload. Paired with the var-executed-as-command test so `echo "${a[@]}"` is left
# alone.
_ARRAY_EXPANSION_RE = re.compile(r"\$\{\w+\[[@*]\]\}")
-# A wrapper's bare duration/count argument (timeout 5 rm, timeout 1.5s rm) that precedes the real command, so it is
-# not mistaken for the command itself.
-_WRAPPER_DURATION_RE = re.compile(r"\d+(?:\.\d+)?[smhd]?$")
# Non-shell interpreters running an inline program (python -c, node -e, php -r): the terminal path never screens that
# program the way the python tool does. sh/bash -c are omitted, the hard-block already recursing into their payloads.
_INLINE_CODE_INTERPRETERS = frozenset(
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 62f55d0a481..34586d1916a 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -2613,7 +2613,7 @@ def test_load_unknown_family_raises():
# load_progress state machine (no threads / network / real cache)
-from core.inference.diffusion import _LoadingState, _LoadState # noqa: E402
+from core.inference.diffusion import _LoadingState # noqa: E402
def test_load_progress_idle_and_ready():
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 3df8c045aab..075b4b3128e 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -3794,12 +3794,6 @@ class TestFriendlyErrorHttpx:
assert _friendly_error(RuntimeError("unrelated")) == "An internal error occurred"
-from routes.inference import ( # noqa: E402
- _drop_empty_assistant_sentinels,
- _openai_messages_for_gguf_chat,
-)
-
-
def _image_question_messages():
"""One user turn holding a question and a tiny png."""
return [
diff --git a/tests/kaggle/test_launch_cleanup.py b/tests/kaggle/test_launch_cleanup.py
index 33568b11d36..6994e5c8055 100644
--- a/tests/kaggle/test_launch_cleanup.py
+++ b/tests/kaggle/test_launch_cleanup.py
@@ -894,24 +894,6 @@ def test_the_leaked_kernel_warning_does_not_strand_the_handler(tmp_path):
assert _deletions(tmp_path), "no delete was attempted, so nothing could leak"
-def _failing_kaggle(bin_dir: Path, record: Path) -> None:
- """A `kaggle` that records the call and then refuses it, so nothing is ever
- released and release() reaches its leaked-kernel warning."""
- bin_dir.mkdir(parents = True, exist_ok = True)
- shim = bin_dir / "kaggle"
- shim.write_text(
- textwrap.dedent(f"""\
- #!{sys.executable}
- import sys, pathlib
- pathlib.Path({str(record)!r}).open("a").write(" ".join(sys.argv[1:]) + "\\n")
- sys.stderr.write("500 Server Error\\n")
- sys.exit(1)
- """),
- encoding = "utf-8",
- )
- shim.chmod(0o755)
-
-
def test_the_leaked_warning_is_still_a_github_annotation(tmp_path, monkeypatch, capsys):
"""Routing it through the safe writer must not add the [launch] prefix.
diff --git a/tests/python/test_compiler_watch_temp_scan.py b/tests/python/test_compiler_watch_temp_scan.py
index c7424c8eaf1..6e42c994018 100644
--- a/tests/python/test_compiler_watch_temp_scan.py
+++ b/tests/python/test_compiler_watch_temp_scan.py
@@ -33,8 +33,6 @@ import subprocess
import pytest
from unsloth_pwsh_runner import run_pwsh
-from unsloth_pwsh_runner import run_pwsh
-
REPO = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = REPO / ".github" / "scripts" / "Watch-ForCompiler.ps1"
diff --git a/tests/saving/language_models/test_merge_model_perplexity_llama_3_2.py b/tests/saving/language_models/test_merge_model_perplexity_llama_3_2.py
index 9500141f4a3..47a48641cb7 100644
--- a/tests/saving/language_models/test_merge_model_perplexity_llama_3_2.py
+++ b/tests/saving/language_models/test_merge_model_perplexity_llama_3_2.py
@@ -159,8 +159,6 @@ if __name__ == "__main__":
loftq_config = None,
)
- from unsloth import is_bfloat16_supported
-
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
diff --git a/tests/saving/language_models/test_merge_model_perplexity_mistral.py b/tests/saving/language_models/test_merge_model_perplexity_mistral.py
index d8a6c7e4260..72ff725b6db 100644
--- a/tests/saving/language_models/test_merge_model_perplexity_mistral.py
+++ b/tests/saving/language_models/test_merge_model_perplexity_mistral.py
@@ -217,8 +217,6 @@ if __name__ == "__main__":
loftq_config = None,
)
- from unsloth import is_bfloat16_supported
-
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
diff --git a/tests/saving/language_models/test_merge_model_perplexity_phi_4.py b/tests/saving/language_models/test_merge_model_perplexity_phi_4.py
index 26a99f46ec4..778974e41d1 100644
--- a/tests/saving/language_models/test_merge_model_perplexity_phi_4.py
+++ b/tests/saving/language_models/test_merge_model_perplexity_phi_4.py
@@ -159,8 +159,6 @@ if __name__ == "__main__":
loftq_config = None,
)
- from unsloth import is_bfloat16_supported
-
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
diff --git a/tests/saving/language_models/test_merged_model_perplexity_llama_3_1_8b.py b/tests/saving/language_models/test_merged_model_perplexity_llama_3_1_8b.py
index a3474444404..9d8f7763ffa 100644
--- a/tests/saving/language_models/test_merged_model_perplexity_llama_3_1_8b.py
+++ b/tests/saving/language_models/test_merged_model_perplexity_llama_3_1_8b.py
@@ -161,8 +161,6 @@ if __name__ == "__main__":
loftq_config = None,
)
- from unsloth import is_bfloat16_supported
-
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
diff --git a/tests/saving/text_to_speech_models/test_csm.py b/tests/saving/text_to_speech_models/test_csm.py
index 146346450a3..15e16c4afc8 100644
--- a/tests/saving/text_to_speech_models/test_csm.py
+++ b/tests/saving/text_to_speech_models/test_csm.py
@@ -141,7 +141,6 @@ print(f"{'='*80}")
from transformers import pipeline
-import torch
output_audio_path = "csm_audio.wav"
try:
diff --git a/tests/saving/text_to_speech_models/test_lasa.py b/tests/saving/text_to_speech_models/test_lasa.py
index e1df3a2a469..118fbfef101 100644
--- a/tests/saving/text_to_speech_models/test_lasa.py
+++ b/tests/saving/text_to_speech_models/test_lasa.py
@@ -143,7 +143,6 @@ print(f"{'='*80}")
from transformers import pipeline
-import torch
output_audio_path = "lasa_audio.wav"
input_text = "Hey there my name is Elise, <giggles> and I'm a speech generation model that can sound like a person."
diff --git a/tests/saving/vision_models/test_save_merge_qwen2_5vl32B_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_qwen2_5vl32B_model_ocr_benchmark.py
index 79e9c9cc897..f6458a1999d 100644
--- a/tests/saving/vision_models/test_save_merge_qwen2_5vl32B_model_ocr_benchmark.py
+++ b/tests/saving/vision_models/test_save_merge_qwen2_5vl32B_model_ocr_benchmark.py
@@ -31,8 +31,6 @@ from tests.utils.cleanup_utils import safe_remove_directory
from tests.utils.ocr_eval import OCRModelEvaluator
-from datasets import load_dataset
-
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
train_dataset = dataset.select(range(2000))
eval_dataset = dataset.select(range(2000, 2200))
@@ -71,12 +69,9 @@ system_message = "You are an expert french ocr system."
train_dataset = [format_data(sample) for sample in train_dataset]
eval_dataset = [format_data(sample) for sample in eval_dataset]
-import os
-import torch
from tqdm import tqdm
import pandas as pd
from jiwer import wer, cer
-from qwen_vl_utils import process_vision_info
ocr_evaluator = OCRModelEvaluator()
model_comparison_results = {}
diff --git a/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py
index d905a3df997..da7c455fa3c 100644
--- a/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py
+++ b/tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py
@@ -31,8 +31,6 @@ from tests.utils.cleanup_utils import safe_remove_directory
from tests.utils.ocr_eval import OCRModelEvaluator
-from datasets import load_dataset
-
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
train_dataset = dataset.select(range(2000))
@@ -72,12 +70,9 @@ system_message = "You are an expert french ocr system."
train_dataset = [format_data(sample) for sample in train_dataset]
eval_dataset = [format_data(sample) for sample in eval_dataset]
-import os
-import torch
from tqdm import tqdm
import pandas as pd
from jiwer import wer, cer
-from qwen_vl_utils import process_vision_info
ocr_evaluator = OCRModelEvaluator()
model_comparison_results = {}
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 05bed762418..dc5b76fc255 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -7447,8 +7447,6 @@ class TestHipSdkInstalledButDeviceInaccessible:
# TEST: --rocm-gfx forwarding -- setup.sh/setup.ps1 forward their resolved gfx
# arch to install_llama_prebuilt.py so the per-gfx prebuilt is picked.
-_SETUP_SH_PATH = PACKAGE_ROOT / "studio" / "setup.sh"
-
class TestNormalizeForwardedGfx:
"""A forwarded gfx string is reduced to a single clean gfx token."""
@@ -7798,7 +7796,6 @@ def test_pick_rocm_gfx_target_same_arch_multi_gpu(monkeypatch):
# TEST: WSL ROCDXG fixes -- drop-in persistence + system-HIP-before-bundle
-_INSTALL_SH_PATH = PACKAGE_ROOT / "install.sh"
_LLAMA_CPP_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py"
diff --git a/tests/studio/studiobench/instruments/coverage.py b/tests/studio/studiobench/instruments/coverage.py
index 1eea1d12366..5df233aceab 100644
--- a/tests/studio/studiobench/instruments/coverage.py
+++ b/tests/studio/studiobench/instruments/coverage.py
@@ -296,7 +296,6 @@ def ambiguity(snapshot: CoverageSnapshot, name: str) -> int:
# boundary, enforced by `assert_integers_only`.
import time # noqa: E402
-from typing import Any # noqa: E402
from ..analysis import assert_no_bare_zero, measured, merge, unmeasured # noqa: E402
from . import register_instrument # noqa: E402
diff --git a/tests/test_studio_install_workspace_guard.py b/tests/test_studio_install_workspace_guard.py
index 48243845630..e89a114ef8c 100644
--- a/tests/test_studio_install_workspace_guard.py
+++ b/tests/test_studio_install_workspace_guard.py
@@ -12,8 +12,6 @@ from pathlib import Path
import pytest
from unsloth_pwsh_runner import run_pwsh
-from unsloth_pwsh_runner import run_pwsh
-
REPO_ROOT = Path(__file__).resolve().parents[1]
INSTALL_SH = REPO_ROOT / "install.sh"
INSTALL_PS1 = REPO_ROOT / "install.ps1"
diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py
index ffb2e9a6229..96f713bb6ef 100644
--- a/unsloth/dataprep/synthetic.py
+++ b/unsloth/dataprep/synthetic.py
@@ -44,7 +44,6 @@ if (
import requests
import torch
import gc
-import time
import re
from unsloth_zoo.log import logger
import numpy as np
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 7dde169e53b..433b3dcf129 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -30,7 +30,6 @@ from ..device_type import (
from ..bnb_availability import native_kernels_ready
from .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemv
from .nvfp4 import NVFP4QuantState, nvfp4_dequantize, nvfp4_linear
-import functools
# torch.cuda.amp.custom_fwd is deprecated from 2.4.
import torch
@@ -65,7 +64,6 @@ elif DEVICE_TYPE == "npu":
# tl.math.tanh is now libdevice.tanh.
-import triton
import triton.language as tl
if Version(triton.__version__) >= Version("3.0.0"):
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 1c703c5c4f0..a48f8a48f98 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -3866,9 +3866,6 @@ def has_internet(
return False
-import psutil
-
-
def _get_statistics(statistics = None, force_download = True):
# Basic stats on which environment is in use: a README.md is downloaded from HF, all data public, so broken envs can be detected. Disable with UNSLOTH_DISABLE_STATISTICS.
n_cpus = psutil.cpu_count(logical = False)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index c121af9bd9a..07c24bfaa1c 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -169,7 +169,7 @@ def patch_saving_functions(*args, **kwargs):
patch_saving_functions._unsloth_deferred_shim = True
-import re, os, inspect, math, sys
+import re, os, inspect, sys
import types
try:
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index b4609619831..a80a34bdea7 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -55,7 +55,7 @@ try:
except:
MistralSdpaAttention = MistralAttention
MistralFlashAttention2 = MistralAttention
-from unsloth_zoo.utils import Version, _get_dtype
+from unsloth_zoo.utils import Version
def MistralAttention_fast_forward(
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index a6cfffce87d..1f3e9a772fd 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -201,7 +201,6 @@ from unsloth_zoo.patching_utils import patch_model_and_tokenizer
from unsloth_zoo.training_utils import prepare_model_for_training
from unsloth_zoo.utils import Version
-from transformers import __version__ as transformers_version
import types
import functools
diff --git a/unsloth/save.py b/unsloth/save.py
index 97367224498..4e656255344 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -60,7 +60,6 @@ import subprocess
import traceback
import psutil
import re
-from transformers.models.llama.modeling_llama import logger
from .models.loader_utils import (
get_model_name,
_resolve_hub_repo_cached_file,
@@ -5155,16 +5154,8 @@ def unsloth_convert_lora_to_ggml_and_save_locally(
return _unsloth_save_lora_gguf(self, tokenizer, save_directory, outtype = outtype)
-from .models.loader_utils import (
- get_model_name,
- _resolve_hub_repo_cached_file,
- _tokenizer_cache_dir,
- _tokenizer_wants_local_only,
-)
-
# Imported lazily at the two call sites: an older zoo, before its bitsandbytes import became optional, would otherwise break `import unsloth` on a host without bnb.
from unsloth_zoo.llama_cpp import (
- install_llama_cpp,
convert_to_gguf as _convert_to_gguf,
)