Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
test(train): check the step trains, and cover asr with asr_lora (#797) (#2175)
Yang Bai297d5f4041e8
diff --git a/tests/test_distill_classifier_asr_unlearn_checkpointing.py b/tests/test_distill_classifier_asr_unlearn_checkpointing.pyindex e80e4073..efa2da8e 100644--- a/tests/test_distill_classifier_asr_unlearn_checkpointing.py+++ b/tests/test_distill_classifier_asr_unlearn_checkpointing.py@@ -36,6 +36,8 @@ _WRAPPERS = {"distill": ("soup_cli.trainer.distill", "DistillTrainerWrapper"),"classifier": ("soup_cli.trainer.classifier", "ClassifierTrainerWrapper"),"asr": ("soup_cli.trainer.asr", "AsrTrainerWrapper"),+ # The shape of the three Whisper recipes: the same wrapper with ``asr_lora: true``.+ "asr_lora": ("soup_cli.trainer.asr", "AsrTrainerWrapper"),}@@ -183,12 +185,17 @@ def _build(return (0.1 * np.sin(2 * np.pi * 220.0 * seconds)).astype(np.float32)monkeypatch.setattr(tts_codec, "load_audio_mono", sine)- cfg = _config(task, _tiny_whisper_dir(tmp_path), tmp_path / "out", checkpointing)+ extra = {"asr_lora": True} if task == "asr_lora" else {}+ cfg = _config(+ "asr", _tiny_whisper_dir(tmp_path), tmp_path / "out", checkpointing, **extra+ )rows = [{"audio": f"clip{index}.wav", "text": "hello world"} for index in range(4)]module_name, class_name = _WRAPPERS[task]wrapper = getattr(importlib.import_module(module_name), class_name)(cfg, device="cpu")wrapper.setup({"train": rows})trainer = wrapper.trainer+ if task == "asr_lora":+ assert any("lora_" in name for name, _ in trainer.model.named_parameters())assert trainer.args.device.type == "cpu"assert next(trainer.model.parameters()).device.type == "cpu"return wrapper, trainer@@ -226,12 +233,18 @@ def test_checkpointing_state_after_one_cpu_training_step(trainer.args.max_steps = 1trainer.args.save_strategy = "no"trainer.args.disable_tqdm = True+ import torch++ trainable = [p for p in trainer.model.parameters() if p.requires_grad]+ before = [p.detach().clone() for p in trainable]calls = _count_layer_calls(trainer.model)trainer.train()assert trainer.state.global_step == 1assert trainer.model.is_gradient_checkpointing is checkpointing, task# One forward per step, plus the backward recompute when checkpointing is on.assert calls[0] == (2 if checkpointing else 1), task+ # The recompute count cannot show this: the step still trained the model.+ assert any(not torch.equal(old, new.detach()) for old, new in zip(before, trainable)), task@pytest.mark.parametrize("task", _WRAPPERS)