Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
# Soup template: Full RLHF Pipeline (SFT + Reward Model + PPO)
# Three-stage training: 1) SFT warmup, 2) Reward model, 3) PPO alignment
#
# Usage:
# Step 1: soup train --config soup_sft.yaml # SFT warmup
# Step 2: soup train --config soup_rm.yaml # Train reward model
# Step 3: soup train --config soup_ppo.yaml # PPO with reward model
#
# This template generates the PPO config (step 3).
# For steps 1-2, use: soup init --template chat (SFT) and edit task to reward_model.
base: meta-llama/Llama-3.1-8B-Instruct
task: ppo
# backend: unsloth # 2-5x faster, pip install "soup-cli[fast]"
data:
train: ./data/prompts.jsonl
format: chatml
val_split: 0.1
max_length: 2048
training:
epochs: 1
lr: 1e-6
batch_size: auto
gradient_accumulation_steps: 4
lora:
r: 64
alpha: 16
target_modules: auto
quantization: 4bit
reward_model: ./output_rm
ppo_epochs: 4
ppo_clip_ratio: 0.2
ppo_kl_penalty: 0.05
output: ./output_ppo