Official Codebase of 3DCodeVerse
Python
scripts: the GPU selector only ever looked at cards 0-3
gaoypeng26d46576b64a
diff --git a/finetune/scripts/free_gpus.sh b/finetune/scripts/free_gpus.shindex 0d5f9882..b61c9376 100755--- a/finetune/scripts/free_gpus.sh+++ b/finetune/scripts/free_gpus.sh@@ -1,11 +1,15 @@#!/bin/bash-# Print the GPUs (of 0-3) with at least <need_free_mib> free — other users share this box, so "completely empty"-# is the wrong test; what matters is whether OUR job fits.-# scripts/free_gpus.sh [need_free_mib] default 60000 (a 9B LoRA at cutoff 8192)+# Print the GPUs with at least <need_free_mib> free — other users share this box, so "completely empty" is the+# wrong test; what matters is whether OUR job fits.+# scripts/free_gpus.sh [need_free_mib] [gpu_list] default 60000 (a 9B LoRA at cutoff 8192), all GPUs+# The list used to be hardcoded to 0-3 from when this box exposed four cards, which silently hid 4-7 from every+# script that auto-selects a GPU.NEED=${1:-60000}+LIST=${2:-$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ' ')}OUT=""-for i in 0 1 2 3; do+for i in $LIST; doread -r TOT USED <<< "$(nvidia-smi --query-gpu=memory.total,memory.used --format=csv,noheader,nounits -i $i | tr ',' ' ')"+ [ -z "${TOT:-}" ] && continueFREE=$((TOT - USED))[ "$FREE" -ge "$NEED" ] && OUT="$OUT,$i"done