Skip to content

Official Codebase of 3DCodeVerse

Python

scripts: the GPU selector only ever looked at cards 0-3

gaoypeng26d46576b64a
diff --git a/finetune/scripts/free_gpus.sh b/finetune/scripts/free_gpus.sh
index 0d5f9882..b61c9376 100755
--- a/finetune/scripts/free_gpus.sh
+++ b/finetune/scripts/free_gpus.sh
@@ -1,11 +1,15 @@
#!/bin/bash
-# Print the GPUs (of 0-3) with at least <need_free_mib> free — other users share this box, so "completely empty"
-# is the wrong test; what matters is whether OUR job fits.
-# scripts/free_gpus.sh [need_free_mib] default 60000 (a 9B LoRA at cutoff 8192)
+# Print the GPUs with at least <need_free_mib> free — other users share this box, so "completely empty" is the
+# wrong test; what matters is whether OUR job fits.
+# scripts/free_gpus.sh [need_free_mib] [gpu_list] default 60000 (a 9B LoRA at cutoff 8192), all GPUs
+# The list used to be hardcoded to 0-3 from when this box exposed four cards, which silently hid 4-7 from every
+# script that auto-selects a GPU.
NEED=${1:-60000}
+LIST=${2:-$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ' ')}
OUT=""
-for i in 0 1 2 3; do
+for i in $LIST; do
read -r TOT USED <<< "$(nvidia-smi --query-gpu=memory.total,memory.used --format=csv,noheader,nounits -i $i | tr ',' ' ')"
+ [ -z "${TOT:-}" ] && continue
FREE=$((TOT - USED))
[ "$FREE" -ge "$NEED" ] && OUT="$OUT,$i"
done