ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env bash
#
# llm-cold-storage.sh — Archive idle HuggingFace models to cold storage
#
# Part of ODS tooling.
#
# Models not accessed in 7+ days are moved to cold storage on a backup drive.
# A symlink replaces the original so HuggingFace cache resolution still works.
# Models can be restored manually or are auto-detected if a process loads them.
#
# Usage:
# ./llm-cold-storage.sh # Archive idle models (dry-run)
# ./llm-cold-storage.sh --execute # Archive idle models (for real)
# ./llm-cold-storage.sh --restore <name> # Restore a specific model
# ./llm-cold-storage.sh --restore-all # Restore all archived models
# ./llm-cold-storage.sh --status # Show archive status
#
set -uo pipefail
HF_CACHE="${HF_CACHE:-$HOME/.cache/huggingface/hub}"
COLD_DIR="${COLD_DIR:-$HOME/llm-cold-storage}"
# Relative archive locations are relative to the invocation directory, not
# the Hugging Face cache directory where the replacement symlink is stored.
case "$COLD_DIR" in
/*) ;;
*) COLD_DIR="$(pwd -P)/$COLD_DIR" ;;
esac
LOG_FILE="${LOG_FILE:-$HOME/.local/log/llm-cold-storage.log}"
MAX_IDLE_DAYS=7
# Ensure the log directory exists
mkdir -p "$(dirname "$LOG_FILE")"
# Models to never archive (currently serving or critical)
PROTECTED_MODELS=(
"models--BAAI--bge-base-en-v1.5"
"models--Systran--faster-whisper-base"
"models--sentence-transformers--all-MiniLM-L6-v2"
)
log() {
local msg
msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
echo "$msg" | tee -a "$LOG_FILE"
}
is_protected() {
local name="$1"
for p in "${PROTECTED_MODELS[@]}"; do
[[ "$name" == "$p" ]] && return 0
done
return 1
}
is_model_in_use() {
local name="$1"
# Extract model identifier: models--Org--Name -> Org/Name
local model_id
model_id="$(echo "$name" | sed 's/^models--//; s/--/\//g')"
# Check if any running process references this model
if pgrep -f "$model_id" > /dev/null 2>&1; then
return 0
fi
return 1
}
get_last_access_days() {
local dir="$1"
# Find most recent atime across all files in $dir using portable stat
# (BSD `find -printf` is unavailable on macOS).
local newest_atime=""
if [[ "$(uname -s)" == "Darwin" ]]; then
# BSD: stat -f %a (atime as epoch seconds)
newest_atime="$(find "$dir" -type f -exec stat -f %a {} + 2>/dev/null | sort -rn | sed -n '1p')"
else
# GNU: stat -c %X (atime as epoch seconds)
newest_atime="$(find "$dir" -type f -exec stat -c %X {} + 2>/dev/null | sort -rn | sed -n '1p')"
fi
if [[ -z "$newest_atime" || ! "${newest_atime%.*}" =~ ^[0-9]+$ ]]; then
echo "9999"
return
fi
local now
now="$(date +%s)"
local age_secs=$(( now - ${newest_atime%.*} ))
echo "$(( age_secs / 86400 ))"
}
path_exists() {
[[ -e "$1" || -L "$1" ]]
}
link_targets() {
local target
[[ -L "$1" ]] || return 1
target="$(readlink "$1")" || return 1
[[ "$target" == "$2" ]]
}
# These guards preserve conflicting paths. Run with no external cache writers:
# shell checks cannot serialize arbitrary programs creating a destination.
do_archive() {
local dry_run="${1:-true}"
local archived=0
local skipped=0
local failed=0
log "========== LLM cold storage scan started (dry_run=$dry_run) =========="
for model_dir in "$HF_CACHE"/models--*/; do
[[ -d "$model_dir" ]] || continue
# Skip if already a symlink (already archived)
[[ -L "${model_dir%/}" ]] && continue
local name
name="$(basename "$model_dir")"
# Skip protected models
if is_protected "$name"; then
log "SKIP (protected): $name"
((skipped++))
continue
fi
# Skip if actively in use by a process
if is_model_in_use "$name"; then
log "SKIP (in use): $name"
((skipped++))
continue
fi
local idle_days
idle_days="$(get_last_access_days "$model_dir")"
local size
size="$(du -sh "$model_dir" 2>/dev/null | cut -f1)"
if (( idle_days >= MAX_IDLE_DAYS )); then
if [[ "$dry_run" == "true" ]]; then
log "WOULD ARCHIVE: $name ($size, idle ${idle_days}d)"
((archived++))
else
log "ARCHIVING: $name ($size, idle ${idle_days}d)"
local cache_path="${model_dir%/}" cold_path="$COLD_DIR/$name"
if ! mkdir -p "$COLD_DIR"; then
log "ERROR: cannot create cold directory: $COLD_DIR"
((failed++))
continue
fi
if path_exists "$cold_path"; then
log "ERROR: cold destination already exists: $cold_path"
((failed++))
continue
fi
# Move to cold storage
if mv "$cache_path" "$cold_path"; then
# Create symlink so HF cache still resolves
if ln -s "$cold_path" "$cache_path" && link_targets "$cache_path" "$cold_path"; then
log "ARCHIVED: $name -> $cold_path"
((archived++))
else
log "ERROR: could not publish the cache link for $name"
if ! path_exists "$cache_path"; then
if mv "$cold_path" "$cache_path"; then
log "ROLLBACK: restored $name to the hot cache"
else
log "ERROR: rollback failed; inspect $cold_path and $cache_path"
fi
else
log "ERROR: cache path exists; preserving it and the cold copy at $cold_path"
fi
((failed++))
fi
else
log "ERROR: failed to move $name to cold storage"
((failed++))
fi
fi
else
log "SKIP (recent, ${idle_days}d): $name ($size)"
((skipped++))
fi
done
log "========== Scan complete: $archived archived, $skipped skipped, $failed failed =========="
(( failed == 0 ))
}
do_restore() {
local name="$1"
# Normalize: accept "Qwen/Qwen2.5-7B" or "models--Qwen--Qwen2.5-7B"
if [[ "$name" != models--* ]]; then
name="models--$(echo "$name" | sed 's/\//--/g')"
fi
if [[ "$name" == */* || "$name" == models-- ]]; then
log "ERROR: invalid model name"
return 1
fi
local cold_path="$COLD_DIR/$name"
local cache_path="$HF_CACHE/$name"
if [[ ! -d "$cold_path" || -L "$cold_path" ]]; then
echo "ERROR: Model not found in cold storage: $cold_path"
return 1
fi
if ! mkdir -p "$HF_CACHE"; then
log "ERROR: cannot create cache directory: $HF_CACHE"
return 1
fi
local removed_link=false
if path_exists "$cache_path"; then
if ! link_targets "$cache_path" "$cold_path"; then
log "ERROR: refusing to replace existing cache path: $cache_path"
return 1
fi
if ! rm "$cache_path"; then
log "ERROR: could not remove owned cache link: $cache_path"
return 1
fi
removed_link=true
fi
log "RESTORING: $name to $cache_path"
if ! mv "$cold_path" "$cache_path"; then
log "ERROR: restore failed; inspect $cold_path and $cache_path"
if [[ "$removed_link" == true && -d "$cold_path" ]] && ! path_exists "$cache_path"; then
if ln -s "$cold_path" "$cache_path" && link_targets "$cache_path" "$cold_path"; then
log "ROLLBACK: reinstated the cold-storage link for $name"
else
log "ERROR: could not reinstate the cache link for $name"
fi
fi
return 1
fi
log "RESTORED: $name"
echo "Restored: $name"
}
do_restore_all() {
log "========== Restoring all archived models =========="
local restored=0 failed=0
for cold_model in "$COLD_DIR"/models--*/; do
[[ -d "$cold_model" ]] || continue
local name
name="$(basename "$cold_model")"
if do_restore "$name"; then
((restored++))
else
((failed++))
fi
done
log "========== Restore complete: $restored restored, $failed failed =========="
(( failed == 0 ))
}
show_status() {
echo "=== LLM Cold Storage Status ==="
echo ""
echo "Active models (on NVMe):"
for model_dir in "$HF_CACHE"/models--*/; do
[[ -d "$model_dir" ]] || continue
local name
name="$(basename "$model_dir")"
if [[ -L "${model_dir%/}" ]]; then
local size
size="$(du -sh "$model_dir" 2>/dev/null | cut -f1)"
echo " [SYMLINK -> cold] $name ($size)"
else
local size idle_days status=""
size="$(du -sh "$model_dir" 2>/dev/null | cut -f1)"
idle_days="$(get_last_access_days "$model_dir")"
is_protected "$name" && status=" [protected]"
is_model_in_use "$name" && status=" [in use]"
echo " [HOT] $name ($size, idle ${idle_days}d)${status}"
fi
done
echo ""
echo "Archived models (on backup SSD):"
local has_archived=false
for cold_model in "$COLD_DIR"/models--*/; do
[[ -d "$cold_model" ]] || continue
has_archived=true
local name size
name="$(basename "$cold_model")"
size="$(du -sh "$cold_model" 2>/dev/null | cut -f1)"
echo " [COLD] $name ($size)"
done
$has_archived || echo " (none)"
echo ""
echo "NVMe cache total: $(du -sh "$HF_CACHE" 2>/dev/null | cut -f1)"
echo "Cold storage total: $(du -sh "$COLD_DIR" 2>/dev/null | cut -f1)"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
case "${1:-}" in
--execute)
do_archive false
;;
--restore)
[[ -n "${2:-}" ]] || { echo "Usage: $0 --restore <model-name>"; exit 1; }
do_restore "$2"
;;
--restore-all)
do_restore_all
;;
--status)
show_status
;;
--help|-h)
echo "Usage: $0 [--execute|--restore <name>|--restore-all|--status|--help]"
echo ""
echo " (no args) Dry-run: show what would be archived"
echo " --execute Archive idle models (>$MAX_IDLE_DAYS days)"
echo " --restore <name> Restore model from cold storage"
echo " --restore-all Restore all archived models"
echo " --status Show current hot/cold status"
;;
*)
do_archive true
;;
esac
fi