diff --git a/README.md b/README.md index ba9200a..8b581ee 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # GPU Fan Controller — Delta QFR1212GHE Автоматическое управление вентилятором на Tesla V100-PCIE-32GB (1 или 2 карты) по температуре GPU. -Вентилятор обдувает обе карты. Скрипт автоматически определяет количество карт и берёт max(temp) для логики. Burst по каждой карте отдельно. +Вентилятор обдувает обе карты. Скрипт определяет количество карт и берёт `max(temp)` для логики режимов. +Burst-детекция срабатывает если `max(temp1, temp2)` растёт на ≥3°C за 2 секунды. ## Подключение @@ -17,14 +18,14 @@ ```bash git clone ... -sudo cp scripts/gpu_fan_ctrl.sh /home/romanown/ -sudo chmod +x /home/romanown/gpu_fan_ctrl.sh +sudo cp scripts/gpu_fan_ctrl.sh /root/ +sudo chmod +x /root/gpu_fan_ctrl.sh sudo cp systemd/gpu-fan-ctrl.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable --now gpu-fan-ctrl.service -tail -f /home/romanown/gpu_fan.log +tail -f /var/log/gpu_fan.log ``` ## Ручное управление @@ -39,14 +40,29 @@ echo "RPM" > /dev/ttyUSB0 # обороты ## Режимы -| Режим | PWM | Температура | +| Режим | PWM | Температура (эффективная, с учётом гистерезиса) | |---|---|---| | STOP | 10000 | — | | STANDBY | 8000 | < 23°C | | LOW | 5000 | 23–28°C | | MED | 3000 | 28–33°C | | HIGH | 2500 | 33–39°C | -| FULL | 0 (1 мин) | > 45°C ИЛИ рост 3°C/2с | +| FULL | 0 (2 мин) | ≥ 45°C ИЛИ рост ≥ 3°C/2с | + +> Пороги в коде: TH_COLD=25, TH_LOW=30, TH_MED=35, TH_HIGH=40, TH_BURST=45. +> Гистерезис 2°C оттягивает переключение вниз для стабильности. +> FULL режим удерживается 2 минуты даже при снижении температуры. + +## Логика выбора источника температуры + +1. **1+ NVIDIA GPU** — мониторинг всех GPU, берётся `max(temp)` по всем картам. +2. **0 NVIDIA GPU** — фоллбэк на материнские `thermal_zone`, `max(temp)` по всем зонам. + +## Логика режимов + +- **FULL (6)** — при `max(temp) ≥ 45°C` или быстрый рост ≥3°C за 2 сек → удерживается 2 мин. +- **HIGH/MED/LOW/STANDBY (5/4/3/2)** — обычные пороги с 2°C гистерезисом. +- Режим вентилятора применяется ко всем картам одновременно (один физический вентилятор). ## Подпись diff --git a/scripts/gpu_fan_ctrl.sh b/scripts/gpu_fan_ctrl.sh old mode 100644 new mode 100755 index 985aace..6c120fc --- a/scripts/gpu_fan_ctrl.sh +++ b/scripts/gpu_fan_ctrl.sh @@ -1,113 +1,296 @@ #!/bin/bash -# GPU Fan Controller — Delta QFR1212GHE (1 или 2 Tesla V100) -# Режимы: 2=STANDBY(<25), 3=LOW(<30), 4=MED(<35), 5=HIGH(<40), 6=FULL(>45 или burst) -# Режим 1 (STOP) — не используется, фан всегда крутится минимум на standby. -# Если 2 карты: берём max(temp0, temp1) для логики. Burst проверяется по каждой карте отдельно. +# GPU Fan Control — Delta QFR1212GHE (Tesla V100 / motherboard fallback) +# Version: 3.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Description: +# Controls the Delta QFR1212GHE cooling fan based on GPU temperatures. +# +# Mode selection logic: +# 1 NVIDIA GPU detected → monitor that single GPU, use its temp. +# 2 NVIDIA GPUs detected → monitor both GPUs, use max(temp1, temp2). +# 0 NVIDIA GPUs detected → fall back to motherboard thermal zones, +# use max(temp) across all zones. +# +# When the fan is controlled, its mode applies to ALL GPUs simultaneously +# (one physical fan blows over both cards). +# +# Thresholds (raw, with 2°C hysteresis dead-band): +# Mode 6 - FULL speed → TEMP >= 45°C or rise >= 3°C/2s → 120s burst +# Mode 5 - HIGH speed → 33-39°C (hysteresis: 35-2 to 40-2) +# Mode 4 - MED speed → 28-32°C (hysteresis: 30-2 to 35-2) +# Mode 3 - LOW speed → 23-27°C (hysteresis: 25-2 to 30-2) +# Mode 2 - STANDBY → < 23°C (hysteresis: 25-2) +# Mode 1 - STOP → not used +# +# Hysteresis: 2°C dead-band prevents rapid mode switching. +# Burst: if any GPU temperature rises by >=3°C within 2 seconds → FULL for 2 min. -LOG=/home/romanown/gpu_fan.log +LOG=/var/log/gpu_fan.log DEV=/dev/ttyUSB0 -stty -F "$DEV" 115200 + +# Timestamp helper +ts() { date '+%Y-%m-%dT%H:%M:%S'; } + +# Open the serial device for writing to the fan controller. exec 3>"$DEV" +stty -F "$DEV" 115200 + +# Save original stderr (→ systemd journal) as fd 4 +exec 4>&2 + +# stdout + stderr → log file exec 1>>"$LOG" 2>&1 -# Пороги +# Thresholds TH_COLD=25 TH_LOW=30 TH_MED=35 TH_HIGH=40 TH_BURST=45 -TH_BURST_ROISE=3 +TH_BURST_RISE=3 BURST_WINDOW=2 BURST_DURATION=120 -# Хистерезис (мёртвая зона 2°) +# Hysteresis (2°C dead-band) HYST=2 +# Initialize variables prevTemp=0 prevTime=0 burstTimer=0 lastMode=0 noDataCount=0 -gpuCount=0 -echo "[START] $(date) GPU Fan Ctrl PID=$$" +# ---- GPU detection ------------------------------------------------------- + +# Collect GPU indices into an array (temps re-read each loop iteration). +declare -a gpu_idx=() + +if command -v nvidia-smi &>/dev/null; then + while IFS=',' read -r gpu_idx_val gpu_temp_val; do + # Trim whitespace from temp + gpu_temp_val="${gpu_temp_val// /}" + [[ -z "$gpu_temp_val" ]] && continue + gpu_idx+=("${gpu_idx_val// /}") + done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,') +fi + +gpuCount=${#gpu_idx[@]} + +if [ "$gpuCount" -eq 0 ]; then + echo "[$(ts)] [INFO] No NVIDIA GPUs detected. Falling back to motherboard thermal zones." +fi + +# ---- Motherboard thermal zones (fallback) -------------------------------- + +# Store motherboard zone paths and types; temps re-read each iteration. +declare -a board_paths=() +declare -a board_types=() + +for zone in /sys/class/thermal/thermal_zone*; do + [ -f "$zone/temp" ] || continue + zoneType=$(cat "$zone/type" 2>/dev/null) + zoneTemp=$(cat "$zone/temp" 2>/dev/null) + + # Convert from millidegrees if needed + if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then + zoneTemp=$((zoneTemp / 1000)) + fi + + # Sanity check — skip zones with empty type or invalid temp + if [[ -n "$zoneType" ]] && [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then + board_paths+=("$zone") + board_types+=("$zoneType") + fi +done + +boardCount=${#board_paths[@]} + +# ---- Helper: compute max temperature and build a display string ---------- + +# Read all GPU temperatures and find max. +# Sets: maxTemp, maxIdx, allTempsStr +read_max_gpu_temps() { + maxTemp=0 + maxIdx="" + allTempsStr="" + local gpu_idx_val gpu_temp_val + local sep="" + while IFS=',' read -r gpu_idx_val gpu_temp_val; do + # Trim whitespace from temp + gpu_temp_val="${gpu_temp_val// /}" + [[ -z "$gpu_temp_val" ]] && continue + allTempsStr+="${sep}GPU-${gpu_idx_val// /}=$gpu_temp_val°C" + sep=", " + if [ "$gpu_temp_val" -gt "$maxTemp" ]; then + maxTemp=$gpu_temp_val + maxIdx="${gpu_idx_val// /}" + fi + done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,') +} + +# Read all motherboard thermal zone temperatures and find max. +# Sets: maxTemp, maxIdx, allTempsStr +read_max_board_temps() { + maxTemp=0 + maxIdx="" + allTempsStr="" + local zoneTemp + local sep="" + for ((i=0; i/dev/null) + # Convert from millidegrees if needed + if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then + zoneTemp=$((zoneTemp / 1000)) + fi + if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then + allTempsStr+="${sep}${board_types[$i]}=$zoneTemp°C" + sep=", " + if [ "$zoneTemp" -gt "$maxTemp" ]; then + maxTemp=$zoneTemp + maxIdx=${board_types[$i]} + fi + fi + done +} + +# ---- Initial read -------------------------------------------------------- + +if [ "$gpuCount" -gt 0 ]; then + read_max_gpu_temps + TEMP=$maxTemp + SOURCE="gpu-$maxIdx" +else + read_max_board_temps + if [ -n "$maxIdx" ]; then + TEMP=$maxTemp + SOURCE="board-$maxIdx" + else + # No sensors at all — full speed to be safe + TEMP=50 + SOURCE="none (full speed)" + fi +fi + +# Initialize tracking variables with current values +prevTemp=$TEMP +prevTime=$(date +%s) +burstTimer=0 +lastMode=0 + +# Print detected GPUs (if any) +if [ "$gpuCount" -gt 0 ]; then + echo "[$(ts)] [INFO] Detected $gpuCount NVIDIA GPU(s):" + for ((i=0; i/dev/null && continue - gpuCount=$((gpuCount + 1)) - if [ "$t" -gt "$maxTemp" ]; then maxTemp=$t; fi - - # Burst по каждой карте отдельно - if [ $prevTime -gt 0 ] && [ $((prevTime != 0 ? now - prevTime : 99999)) -le $BURST_WINDOW ]; then - rise=$((t - prevTemp)) - if [ $rise -ge $TH_BURST_ROISE ]; then - burstTriggered=1 - echo "[BURST-rise] GPU${gpuCount}: rise=$rise°C in $((now-prevTime))s → mode 6" - fi + now=$(date +%s) + + # Re-read current temperatures + if [ "$gpuCount" -gt 0 ]; then + read_max_gpu_temps + SOURCE="gpu-$maxIdx" + else + read_max_board_temps + if [ -n "$maxIdx" ]; then + SOURCE="board-$maxIdx" + fi fi - prevTemp=$t # сохраняем для следующей итерации - done <<< "$ALLTEMP" + TEMP=$maxTemp - if [ $gpuCount -eq 0 ]; then - noDataCount=$((noDataCount + 1)) - if [ $noDataCount -ge 10 ]; then - echo "[ERROR] nvidia-smi failed 10+ times, aborting" - exit 1 + echo "[$(ts)] [STATUS] All temps: ${allTempsStr:-none}, max=$TEMP°C from $SOURCE" >&4 + + # Check if the temperature source is still alive + if [ "$TEMP" -le 0 ] 2>/dev/null; then + noDataCount=$((noDataCount + 1)) + if [ $noDataCount -ge 10 ]; then + echo "[$(ts)] [ERROR] No temperature data for 10+ seconds. Running fan at full speed." + echo "6" >&3 + exit 1 + fi + sleep 1; continue fi - sleep 1; continue - fi - noDataCount=0 - - TEMP=$maxTemp # берём max для логики - # Если в burst-режиме — держим режим 6 - if [ $burstTimer -gt 0 ]; then - burstTimer=$((burstTimer - 1)) - if [ $lastMode -ne 6 ]; then - echo "[MODE 6] full speed (burst, ${burstTimer}s remain)" - lastMode=6 - echo "6" >&3 + noDataCount=0 + + # --- 1. Если burst-режим активен — держим режим 6 --- + if [ $burstTimer -gt 0 ]; then + burstTimer=$((burstTimer - 1)) + if [ $lastMode -ne 6 ]; then + echo "[$(ts)] [MODE 6] Full speed (burst, ${burstTimer}s remaining)" + lastMode=6 + echo "6" >&3 + fi + prevTemp=$TEMP; prevTime=$now + sleep 1; continue fi - sleep 1; continue - fi - # Если burst истёк И temp >= TH_BURST — новый burst - if [ $TEMP -ge $TH_BURST ] || [ $burstTriggered -eq 1 ]; then - burstTimer=$BURST_DURATION - echo "[MODE 6] full speed (re-burst: $TEMP°C)" - lastMode=6; prevTemp=$TEMP; prevTime=$now - echo "6" >&3 - sleep 1; continue - fi + # --- 2. Если temp >= TH_BURST (45) и burst не активен — новый burst --- + if [ "$TEMP" -ge $TH_BURST ]; then + burstTimer=$BURST_DURATION + echo "[$(ts)] [MODE 6] Full speed (re-burst: $TEMP°C)" + lastMode=6; prevTemp=$TEMP; prevTime=$now + echo "6" >&3 + sleep 1; continue + fi - # Обычная логика с хистерезисом - newMode=5 # default: HIGH (35-39) + # --- 3. Burst detection: быстрый рост температуры --- + if [ $((now - prevTime)) -le $BURST_WINDOW ] && [ $prevTime -gt 0 ]; then + rise=$((TEMP - prevTemp)) + if [ $rise -ge $TH_BURST_RISE ]; then + burstTimer=$BURST_DURATION + echo "[$(ts)] [BURST] $TEMP°C (up from $prevTemp°C in $((now - prevTime))s, window=$BURST_WINDOW)" + lastMode=6 + prevTemp=$TEMP; prevTime=$now + echo "6" >&3 + sleep 1; continue + fi + fi - if [ $TEMP -lt $((TH_COLD - HYST)) ]; then - newMode=2 # STANDBY (< 23) - elif [ $TEMP -lt $((TH_LOW - HYST)) ]; then - newMode=3 # LOW (< 28) - elif [ $TEMP -lt $((TH_MED - HYST)) ]; then - newMode=4 # MED (< 33) - fi + # --- 4. Обычная логика с гистерезисом --- + newMode=5 # default: HIGH (35-39) + if [ $TEMP -lt $((TH_COLD - HYST)) ]; then + newMode=2 # STANDBY (< 23) + elif [ $TEMP -lt $((TH_LOW - HYST)) ]; then + newMode=3 # LOW (< 28) + elif [ $TEMP -lt $((TH_MED - HYST)) ]; then + newMode=4 # MED (< 33) + fi - # Отправляем только если режим изменился - if [ $newMode -ne $lastMode ]; then - echo "[MODE $newMode] max_temp=$TEMP°C (gpuCount=$gpuCount) (hysteresis: no-change zone)" - lastMode=$newMode - echo "$newMode" >&3 - fi + if [ $newMode -ne $lastMode ]; then + echo "[$(ts)] [MODE $newMode] Temperature: $TEMP°C, source: $SOURCE" + lastMode=$newMode + echo "$newMode" >&3 + fi - prevTime=$now - sleep 1 + prevTemp=$TEMP + prevTime=$now + sleep 1 done diff --git a/systemd/gpu-fan-ctrl.service b/systemd/gpu-fan-ctrl.service index 848aabf..aed3f26 100644 --- a/systemd/gpu-fan-ctrl.service +++ b/systemd/gpu-fan-ctrl.service @@ -5,7 +5,7 @@ Wants=network-online.target [Service] Type=simple -ExecStart=/home/romanown/gpu-fan-ctrl/scripts/gpu_fan_ctrl.sh +ExecStart=/root/gpu_fan_ctrl.sh Restart=on-failure RestartSec=5 StandardOutput=journal