2 GPUs
This commit is contained in:
user committed
1 parent
a91d4c50d4
commit
342c0ec9ea
3 files changed
+284
-85
No files matched your search
Regular → Executable
+261
-78
@@ -1,113 +1,296 @@
|
||||
#!/bin/bash
|
||||
# GPU Fan Controller — Delta QFR1212GHE (1 или 2 Tesla V100)
|
||||
# Режимы: 2=STANDBY(<25), 3=LOW(<30), 4=MED(<35), 5=HIGH(<40), 6=FULL(>45 или burst)
|
||||
# Режим 1 (STOP) — не используется, фан всегда крутится минимум на standby.
|
||||
# Если 2 карты: берём max(temp0, temp1) для логики. Burst проверяется по каждой карте отдельно.
|
||||
# GPU Fan Control — Delta QFR1212GHE (Tesla V100 / motherboard fallback)
|
||||
# Version: 3.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Description:
|
||||
# Controls the Delta QFR1212GHE cooling fan based on GPU temperatures.
|
||||
#
|
||||
# Mode selection logic:
|
||||
# 1 NVIDIA GPU detected → monitor that single GPU, use its temp.
|
||||
# 2 NVIDIA GPUs detected → monitor both GPUs, use max(temp1, temp2).
|
||||
# 0 NVIDIA GPUs detected → fall back to motherboard thermal zones,
|
||||
# use max(temp) across all zones.
|
||||
#
|
||||
# When the fan is controlled, its mode applies to ALL GPUs simultaneously
|
||||
# (one physical fan blows over both cards).
|
||||
#
|
||||
# Thresholds (raw, with 2°C hysteresis dead-band):
|
||||
# Mode 6 - FULL speed → TEMP >= 45°C or rise >= 3°C/2s → 120s burst
|
||||
# Mode 5 - HIGH speed → 33-39°C (hysteresis: 35-2 to 40-2)
|
||||
# Mode 4 - MED speed → 28-32°C (hysteresis: 30-2 to 35-2)
|
||||
# Mode 3 - LOW speed → 23-27°C (hysteresis: 25-2 to 30-2)
|
||||
# Mode 2 - STANDBY → < 23°C (hysteresis: 25-2)
|
||||
# Mode 1 - STOP → not used
|
||||
#
|
||||
# Hysteresis: 2°C dead-band prevents rapid mode switching.
|
||||
# Burst: if any GPU temperature rises by >=3°C within 2 seconds → FULL for 2 min.
|
||||
|
||||
LOG=/home/romanown/gpu_fan.log
|
||||
LOG=/var/log/gpu_fan.log
|
||||
DEV=/dev/ttyUSB0
|
||||
stty -F "$DEV" 115200
|
||||
|
||||
# Timestamp helper
|
||||
ts() { date '+%Y-%m-%dT%H:%M:%S'; }
|
||||
|
||||
# Open the serial device for writing to the fan controller.
|
||||
exec 3>"$DEV"
|
||||
stty -F "$DEV" 115200
|
||||
|
||||
# Save original stderr (→ systemd journal) as fd 4
|
||||
exec 4>&2
|
||||
|
||||
# stdout + stderr → log file
|
||||
exec 1>>"$LOG" 2>&1
|
||||
|
||||
# Пороги
|
||||
# Thresholds
|
||||
TH_COLD=25
|
||||
TH_LOW=30
|
||||
TH_MED=35
|
||||
TH_HIGH=40
|
||||
TH_BURST=45
|
||||
TH_BURST_ROISE=3
|
||||
TH_BURST_RISE=3
|
||||
BURST_WINDOW=2
|
||||
BURST_DURATION=120
|
||||
|
||||
# Хистерезис (мёртвая зона 2°)
|
||||
# Hysteresis (2°C dead-band)
|
||||
HYST=2
|
||||
|
||||
# Initialize variables
|
||||
prevTemp=0
|
||||
prevTime=0
|
||||
burstTimer=0
|
||||
lastMode=0
|
||||
noDataCount=0
|
||||
gpuCount=0
|
||||
|
||||
echo "[START] $(date) GPU Fan Ctrl PID=$$"
|
||||
# ---- GPU detection -------------------------------------------------------
|
||||
|
||||
# Collect GPU indices into an array (temps re-read each loop iteration).
|
||||
declare -a gpu_idx=()
|
||||
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
while IFS=',' read -r gpu_idx_val gpu_temp_val; do
|
||||
# Trim whitespace from temp
|
||||
gpu_temp_val="${gpu_temp_val// /}"
|
||||
[[ -z "$gpu_temp_val" ]] && continue
|
||||
gpu_idx+=("${gpu_idx_val// /}")
|
||||
done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,')
|
||||
fi
|
||||
|
||||
gpuCount=${#gpu_idx[@]}
|
||||
|
||||
if [ "$gpuCount" -eq 0 ]; then
|
||||
echo "[$(ts)] [INFO] No NVIDIA GPUs detected. Falling back to motherboard thermal zones."
|
||||
fi
|
||||
|
||||
# ---- Motherboard thermal zones (fallback) --------------------------------
|
||||
|
||||
# Store motherboard zone paths and types; temps re-read each iteration.
|
||||
declare -a board_paths=()
|
||||
declare -a board_types=()
|
||||
|
||||
for zone in /sys/class/thermal/thermal_zone*; do
|
||||
[ -f "$zone/temp" ] || continue
|
||||
zoneType=$(cat "$zone/type" 2>/dev/null)
|
||||
zoneTemp=$(cat "$zone/temp" 2>/dev/null)
|
||||
|
||||
# Convert from millidegrees if needed
|
||||
if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then
|
||||
zoneTemp=$((zoneTemp / 1000))
|
||||
fi
|
||||
|
||||
# Sanity check — skip zones with empty type or invalid temp
|
||||
if [[ -n "$zoneType" ]] && [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then
|
||||
board_paths+=("$zone")
|
||||
board_types+=("$zoneType")
|
||||
fi
|
||||
done
|
||||
|
||||
boardCount=${#board_paths[@]}
|
||||
|
||||
# ---- Helper: compute max temperature and build a display string ----------
|
||||
|
||||
# Read all GPU temperatures and find max.
|
||||
# Sets: maxTemp, maxIdx, allTempsStr
|
||||
read_max_gpu_temps() {
|
||||
maxTemp=0
|
||||
maxIdx=""
|
||||
allTempsStr=""
|
||||
local gpu_idx_val gpu_temp_val
|
||||
local sep=""
|
||||
while IFS=',' read -r gpu_idx_val gpu_temp_val; do
|
||||
# Trim whitespace from temp
|
||||
gpu_temp_val="${gpu_temp_val// /}"
|
||||
[[ -z "$gpu_temp_val" ]] && continue
|
||||
allTempsStr+="${sep}GPU-${gpu_idx_val// /}=$gpu_temp_val°C"
|
||||
sep=", "
|
||||
if [ "$gpu_temp_val" -gt "$maxTemp" ]; then
|
||||
maxTemp=$gpu_temp_val
|
||||
maxIdx="${gpu_idx_val// /}"
|
||||
fi
|
||||
done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,')
|
||||
}
|
||||
|
||||
# Read all motherboard thermal zone temperatures and find max.
|
||||
# Sets: maxTemp, maxIdx, allTempsStr
|
||||
read_max_board_temps() {
|
||||
maxTemp=0
|
||||
maxIdx=""
|
||||
allTempsStr=""
|
||||
local zoneTemp
|
||||
local sep=""
|
||||
for ((i=0; i<boardCount; i++)); do
|
||||
zoneTemp=$(cat "${board_paths[$i]}/temp" 2>/dev/null)
|
||||
# Convert from millidegrees if needed
|
||||
if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then
|
||||
zoneTemp=$((zoneTemp / 1000))
|
||||
fi
|
||||
if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then
|
||||
allTempsStr+="${sep}${board_types[$i]}=$zoneTemp°C"
|
||||
sep=", "
|
||||
if [ "$zoneTemp" -gt "$maxTemp" ]; then
|
||||
maxTemp=$zoneTemp
|
||||
maxIdx=${board_types[$i]}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ---- Initial read --------------------------------------------------------
|
||||
|
||||
if [ "$gpuCount" -gt 0 ]; then
|
||||
read_max_gpu_temps
|
||||
TEMP=$maxTemp
|
||||
SOURCE="gpu-$maxIdx"
|
||||
else
|
||||
read_max_board_temps
|
||||
if [ -n "$maxIdx" ]; then
|
||||
TEMP=$maxTemp
|
||||
SOURCE="board-$maxIdx"
|
||||
else
|
||||
# No sensors at all — full speed to be safe
|
||||
TEMP=50
|
||||
SOURCE="none (full speed)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Initialize tracking variables with current values
|
||||
prevTemp=$TEMP
|
||||
prevTime=$(date +%s)
|
||||
burstTimer=0
|
||||
lastMode=0
|
||||
|
||||
# Print detected GPUs (if any)
|
||||
if [ "$gpuCount" -gt 0 ]; then
|
||||
echo "[$(ts)] [INFO] Detected $gpuCount NVIDIA GPU(s):"
|
||||
for ((i=0; i<gpuCount; i++)); do
|
||||
echo "[$(ts)] GPU $i: index=${gpu_idx[$i]}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Print all motherboard zones
|
||||
if [ "$boardCount" -gt 0 ]; then
|
||||
echo "[$(ts)] [INFO] Detected $boardCount motherboard thermal zone(s):"
|
||||
for ((i=0; i<boardCount; i++)); do
|
||||
echo "[$(ts)] Zone $i: type=${board_types[$i]}"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "[$(ts)] [INFO] Starting fan control. Monitoring $SOURCE. Temperature: ${TEMP}°C"
|
||||
|
||||
# ---- Main loop -----------------------------------------------------------
|
||||
|
||||
while true; do
|
||||
# Определяем количество GPU и читаем все температуры
|
||||
ALLTEMP=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader | tr -cd '0-9\n')
|
||||
now=$(date +%s)
|
||||
|
||||
gpuCount=0
|
||||
maxTemp=0
|
||||
burstTriggered=0
|
||||
|
||||
while IFS= read -r t; do
|
||||
[ -z "$t" ] && continue
|
||||
[ "$t" -lt 0 ] 2>/dev/null && continue
|
||||
gpuCount=$((gpuCount + 1))
|
||||
if [ "$t" -gt "$maxTemp" ]; then maxTemp=$t; fi
|
||||
|
||||
# Burst по каждой карте отдельно
|
||||
if [ $prevTime -gt 0 ] && [ $((prevTime != 0 ? now - prevTime : 99999)) -le $BURST_WINDOW ]; then
|
||||
rise=$((t - prevTemp))
|
||||
if [ $rise -ge $TH_BURST_ROISE ]; then
|
||||
burstTriggered=1
|
||||
echo "[BURST-rise] GPU${gpuCount}: rise=$rise°C in $((now-prevTime))s → mode 6"
|
||||
fi
|
||||
now=$(date +%s)
|
||||
|
||||
# Re-read current temperatures
|
||||
if [ "$gpuCount" -gt 0 ]; then
|
||||
read_max_gpu_temps
|
||||
SOURCE="gpu-$maxIdx"
|
||||
else
|
||||
read_max_board_temps
|
||||
if [ -n "$maxIdx" ]; then
|
||||
SOURCE="board-$maxIdx"
|
||||
fi
|
||||
fi
|
||||
prevTemp=$t # сохраняем для следующей итерации
|
||||
done <<< "$ALLTEMP"
|
||||
TEMP=$maxTemp
|
||||
|
||||
if [ $gpuCount -eq 0 ]; then
|
||||
noDataCount=$((noDataCount + 1))
|
||||
if [ $noDataCount -ge 10 ]; then
|
||||
echo "[ERROR] nvidia-smi failed 10+ times, aborting"
|
||||
exit 1
|
||||
echo "[$(ts)] [STATUS] All temps: ${allTempsStr:-none}, max=$TEMP°C from $SOURCE" >&4
|
||||
|
||||
# Check if the temperature source is still alive
|
||||
if [ "$TEMP" -le 0 ] 2>/dev/null; then
|
||||
noDataCount=$((noDataCount + 1))
|
||||
if [ $noDataCount -ge 10 ]; then
|
||||
echo "[$(ts)] [ERROR] No temperature data for 10+ seconds. Running fan at full speed."
|
||||
echo "6" >&3
|
||||
exit 1
|
||||
fi
|
||||
sleep 1; continue
|
||||
fi
|
||||
sleep 1; continue
|
||||
fi
|
||||
noDataCount=0
|
||||
|
||||
TEMP=$maxTemp # берём max для логики
|
||||
|
||||
# Если в burst-режиме — держим режим 6
|
||||
if [ $burstTimer -gt 0 ]; then
|
||||
burstTimer=$((burstTimer - 1))
|
||||
if [ $lastMode -ne 6 ]; then
|
||||
echo "[MODE 6] full speed (burst, ${burstTimer}s remain)"
|
||||
lastMode=6
|
||||
echo "6" >&3
|
||||
noDataCount=0
|
||||
|
||||
# --- 1. Если burst-режим активен — держим режим 6 ---
|
||||
if [ $burstTimer -gt 0 ]; then
|
||||
burstTimer=$((burstTimer - 1))
|
||||
if [ $lastMode -ne 6 ]; then
|
||||
echo "[$(ts)] [MODE 6] Full speed (burst, ${burstTimer}s remaining)"
|
||||
lastMode=6
|
||||
echo "6" >&3
|
||||
fi
|
||||
prevTemp=$TEMP; prevTime=$now
|
||||
sleep 1; continue
|
||||
fi
|
||||
sleep 1; continue
|
||||
fi
|
||||
|
||||
# Если burst истёк И temp >= TH_BURST — новый burst
|
||||
if [ $TEMP -ge $TH_BURST ] || [ $burstTriggered -eq 1 ]; then
|
||||
burstTimer=$BURST_DURATION
|
||||
echo "[MODE 6] full speed (re-burst: $TEMP°C)"
|
||||
lastMode=6; prevTemp=$TEMP; prevTime=$now
|
||||
echo "6" >&3
|
||||
sleep 1; continue
|
||||
fi
|
||||
# --- 2. Если temp >= TH_BURST (45) и burst не активен — новый burst ---
|
||||
if [ "$TEMP" -ge $TH_BURST ]; then
|
||||
burstTimer=$BURST_DURATION
|
||||
echo "[$(ts)] [MODE 6] Full speed (re-burst: $TEMP°C)"
|
||||
lastMode=6; prevTemp=$TEMP; prevTime=$now
|
||||
echo "6" >&3
|
||||
sleep 1; continue
|
||||
fi
|
||||
|
||||
# Обычная логика с хистерезисом
|
||||
newMode=5 # default: HIGH (35-39)
|
||||
# --- 3. Burst detection: быстрый рост температуры ---
|
||||
if [ $((now - prevTime)) -le $BURST_WINDOW ] && [ $prevTime -gt 0 ]; then
|
||||
rise=$((TEMP - prevTemp))
|
||||
if [ $rise -ge $TH_BURST_RISE ]; then
|
||||
burstTimer=$BURST_DURATION
|
||||
echo "[$(ts)] [BURST] $TEMP°C (up from $prevTemp°C in $((now - prevTime))s, window=$BURST_WINDOW)"
|
||||
lastMode=6
|
||||
prevTemp=$TEMP; prevTime=$now
|
||||
echo "6" >&3
|
||||
sleep 1; continue
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $TEMP -lt $((TH_COLD - HYST)) ]; then
|
||||
newMode=2 # STANDBY (< 23)
|
||||
elif [ $TEMP -lt $((TH_LOW - HYST)) ]; then
|
||||
newMode=3 # LOW (< 28)
|
||||
elif [ $TEMP -lt $((TH_MED - HYST)) ]; then
|
||||
newMode=4 # MED (< 33)
|
||||
fi
|
||||
# --- 4. Обычная логика с гистерезисом ---
|
||||
newMode=5 # default: HIGH (35-39)
|
||||
if [ $TEMP -lt $((TH_COLD - HYST)) ]; then
|
||||
newMode=2 # STANDBY (< 23)
|
||||
elif [ $TEMP -lt $((TH_LOW - HYST)) ]; then
|
||||
newMode=3 # LOW (< 28)
|
||||
elif [ $TEMP -lt $((TH_MED - HYST)) ]; then
|
||||
newMode=4 # MED (< 33)
|
||||
fi
|
||||
|
||||
# Отправляем только если режим изменился
|
||||
if [ $newMode -ne $lastMode ]; then
|
||||
echo "[MODE $newMode] max_temp=$TEMP°C (gpuCount=$gpuCount) (hysteresis: no-change zone)"
|
||||
lastMode=$newMode
|
||||
echo "$newMode" >&3
|
||||
fi
|
||||
if [ $newMode -ne $lastMode ]; then
|
||||
echo "[$(ts)] [MODE $newMode] Temperature: $TEMP°C, source: $SOURCE"
|
||||
lastMode=$newMode
|
||||
echo "$newMode" >&3
|
||||
fi
|
||||
|
||||
prevTime=$now
|
||||
sleep 1
|
||||
prevTemp=$TEMP
|
||||
prevTime=$now
|
||||
sleep 1
|
||||
done
|
||||
Reference in new issue
Block a user