2 GPUs

README.md

@@ -1,7 +1,8 @@
 # GPU Fan Controller — Delta QFR1212GHE
 
 Автоматическое управление вентилятором на Tesla V100-PCIE-32GB (1 или 2 карты) по температуре GPU.
-Вентилятор обдувает обе карты. Скрипт автоматически определяет количество карт и берёт max(temp) для логики. Burst по каждой карте отдельно.
+Вентилятор обдувает обе карты. Скрипт определяет количество карт и берёт `max(temp)` для логики режимов.
+Burst-детекция срабатывает если `max(temp1, temp2)` растёт на ≥3°C за 2 секунды.
 
 ## Подключение
 
@@ -17,14 +18,14 @@
 
 ```bash
 git clone ...
-sudo cp scripts/gpu_fan_ctrl.sh /home/romanown/
+sudo cp scripts/gpu_fan_ctrl.sh /root/
-sudo chmod +x /home/romanown/gpu_fan_ctrl.sh
+sudo chmod +x /root/gpu_fan_ctrl.sh
 
 sudo cp systemd/gpu-fan-ctrl.service /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemctl enable --now gpu-fan-ctrl.service
 
-tail -f /home/romanown/gpu_fan.log
+tail -f /var/log/gpu_fan.log
 ```
 
 ## Ручное управление
@@ -39,14 +40,29 @@ echo "RPM" > /dev/ttyUSB0 # обороты
 
 ## Режимы
 
-| Режим | PWM | Температура |
+| Режим | PWM | Температура (эффективная, с учётом гистерезиса) |
 |---|---|---|
 | STOP | 10000 | — |
 | STANDBY | 8000 | < 23°C |
 | LOW | 5000 | 23–28°C |
 | MED | 3000 | 28–33°C |
 | HIGH | 2500 | 33–39°C |
-| FULL | 0 (1 мин) | > 45°C ИЛИ рост 3°C/2с |
+| FULL | 0 (2 мин) | ≥ 45°C ИЛИ рост ≥ 3°C/2с |
+
+> Пороги в коде: TH_COLD=25, TH_LOW=30, TH_MED=35, TH_HIGH=40, TH_BURST=45.
+> Гистерезис 2°C оттягивает переключение вниз для стабильности.
+> FULL режим удерживается 2 минуты даже при снижении температуры.
+
+## Логика выбора источника температуры
+
+1. **1+ NVIDIA GPU** — мониторинг всех GPU, берётся `max(temp)` по всем картам.
+2. **0 NVIDIA GPU** — фоллбэк на материнские `thermal_zone`, `max(temp)` по всем зонам.
+
+## Логика режимов
+
+- **FULL (6)** — при `max(temp) ≥ 45°C` или быстрый рост ≥3°C за 2 сек → удерживается 2 мин.
+- **HIGH/MED/LOW/STANDBY (5/4/3/2)** — обычные пороги с 2°C гистерезисом.
+- Режим вентилятора применяется ко всем картам одновременно (один физический вентилятор).
 
 ## Подпись
 

scripts/gpu_fan_ctrl.sh

@@ -1,98 +1,281 @@
 #!/bin/bash
-# GPU Fan Controller — Delta QFR1212GHE (1 или 2 Tesla V100)
+# GPU Fan Control — Delta QFR1212GHE (Tesla V100 / motherboard fallback)
-# Режимы: 2=STANDBY(<25), 3=LOW(<30), 4=MED(<35), 5=HIGH(<40), 6=FULL(>45 или burst)
+# Version: 3.0
-# Режим 1 (STOP) — не используется, фан всегда крутится минимум на standby.
+#
-# Если 2 карты: берём max(temp0, temp1) для логики. Burst проверяется по каждой карте отдельно.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Description:
+#   Controls the Delta QFR1212GHE cooling fan based on GPU temperatures.
+#
+#   Mode selection logic:
+#     1 NVIDIA GPU detected  → monitor that single GPU, use its temp.
+#     2 NVIDIA GPUs detected → monitor both GPUs, use max(temp1, temp2).
+#     0 NVIDIA GPUs detected → fall back to motherboard thermal zones,
+#                               use max(temp) across all zones.
+#
+#   When the fan is controlled, its mode applies to ALL GPUs simultaneously
+#   (one physical fan blows over both cards).
+#
+#   Thresholds (raw, with 2°C hysteresis dead-band):
+#     Mode 6 - FULL speed    → TEMP >= 45°C or rise >= 3°C/2s → 120s burst
+#     Mode 5 - HIGH speed    → 33-39°C  (hysteresis: 35-2 to 40-2)
+#     Mode 4 - MED speed     → 28-32°C  (hysteresis: 30-2 to 35-2)
+#     Mode 3 - LOW speed     → 23-27°C  (hysteresis: 25-2 to 30-2)
+#     Mode 2 - STANDBY       → < 23°C   (hysteresis: 25-2)
+#     Mode 1 - STOP          → not used
+#
+#   Hysteresis: 2°C dead-band prevents rapid mode switching.
+#   Burst: if any GPU temperature rises by >=3°C within 2 seconds → FULL for 2 min.
 
-LOG=/home/romanown/gpu_fan.log
+LOG=/var/log/gpu_fan.log
 DEV=/dev/ttyUSB0
-stty -F "$DEV" 115200
+
+# Timestamp helper
+ts() { date '+%Y-%m-%dT%H:%M:%S'; }
+
+# Open the serial device for writing to the fan controller.
 exec 3>"$DEV"
+stty -F "$DEV" 115200
+
+# Save original stderr (→ systemd journal) as fd 4
+exec 4>&2
+
+# stdout + stderr → log file
 exec 1>>"$LOG" 2>&1
 
-# Пороги
+# Thresholds
 TH_COLD=25
 TH_LOW=30
 TH_MED=35
 TH_HIGH=40
 TH_BURST=45
-TH_BURST_ROISE=3
+TH_BURST_RISE=3
 BURST_WINDOW=2
 BURST_DURATION=120
 
-# Хистерезис (мёртвая зона 2°)
+# Hysteresis (2°C dead-band)
 HYST=2
 
+# Initialize variables
 prevTemp=0
 prevTime=0
 burstTimer=0
 lastMode=0
 noDataCount=0
-gpuCount=0
 
-echo "[START] $(date) GPU Fan Ctrl PID=$$"
+# ---- GPU detection -------------------------------------------------------
+
+# Collect GPU indices into an array (temps re-read each loop iteration).
+declare -a gpu_idx=()
+
+if command -v nvidia-smi &>/dev/null; then
+    while IFS=',' read -r gpu_idx_val gpu_temp_val; do
+        # Trim whitespace from temp
+        gpu_temp_val="${gpu_temp_val// /}"
+        [[ -z "$gpu_temp_val" ]] && continue
+        gpu_idx+=("${gpu_idx_val// /}")
+    done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,')
+fi
+
+gpuCount=${#gpu_idx[@]}
+
+if [ "$gpuCount" -eq 0 ]; then
+    echo "[$(ts)] [INFO] No NVIDIA GPUs detected. Falling back to motherboard thermal zones."
+fi
+
+# ---- Motherboard thermal zones (fallback) --------------------------------
+
+# Store motherboard zone paths and types; temps re-read each iteration.
+declare -a board_paths=()
+declare -a board_types=()
+
+for zone in /sys/class/thermal/thermal_zone*; do
+    [ -f "$zone/temp" ] || continue
+    zoneType=$(cat "$zone/type" 2>/dev/null)
+    zoneTemp=$(cat "$zone/temp" 2>/dev/null)
+
+    # Convert from millidegrees if needed
+    if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then
+        zoneTemp=$((zoneTemp / 1000))
+    fi
+
+    # Sanity check — skip zones with empty type or invalid temp
+    if [[ -n "$zoneType" ]] && [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then
+        board_paths+=("$zone")
+        board_types+=("$zoneType")
+    fi
+done
+
+boardCount=${#board_paths[@]}
+
+# ---- Helper: compute max temperature and build a display string ----------
+
+# Read all GPU temperatures and find max.
+# Sets: maxTemp, maxIdx, allTempsStr
+read_max_gpu_temps() {
+    maxTemp=0
+    maxIdx=""
+    allTempsStr=""
+    local gpu_idx_val gpu_temp_val
+    local sep=""
+    while IFS=',' read -r gpu_idx_val gpu_temp_val; do
+        # Trim whitespace from temp
+        gpu_temp_val="${gpu_temp_val// /}"
+        [[ -z "$gpu_temp_val" ]] && continue
+        allTempsStr+="${sep}GPU-${gpu_idx_val// /}=$gpu_temp_val°C"
+        sep=", "
+        if [ "$gpu_temp_val" -gt "$maxTemp" ]; then
+            maxTemp=$gpu_temp_val
+            maxIdx="${gpu_idx_val// /}"
+        fi
+    done < <(nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader 2>&1 | grep -E '^[0-9]+,')
+}
+
+# Read all motherboard thermal zone temperatures and find max.
+# Sets: maxTemp, maxIdx, allTempsStr
+read_max_board_temps() {
+    maxTemp=0
+    maxIdx=""
+    allTempsStr=""
+    local zoneTemp
+    local sep=""
+    for ((i=0; i<boardCount; i++)); do
+        zoneTemp=$(cat "${board_paths[$i]}/temp" 2>/dev/null)
+        # Convert from millidegrees if needed
+        if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -gt 1000 ]; then
+            zoneTemp=$((zoneTemp / 1000))
+        fi
+        if [[ "$zoneTemp" =~ ^[0-9]+$ ]] && [ "$zoneTemp" -ge 0 ] && [ "$zoneTemp" -le 255 ]; then
+            allTempsStr+="${sep}${board_types[$i]}=$zoneTemp°C"
+            sep=", "
+            if [ "$zoneTemp" -gt "$maxTemp" ]; then
+                maxTemp=$zoneTemp
+                maxIdx=${board_types[$i]}
+            fi
+        fi
+    done
+}
+
+# ---- Initial read --------------------------------------------------------
+
+if [ "$gpuCount" -gt 0 ]; then
+    read_max_gpu_temps
+    TEMP=$maxTemp
+    SOURCE="gpu-$maxIdx"
+else
+    read_max_board_temps
+    if [ -n "$maxIdx" ]; then
+        TEMP=$maxTemp
+        SOURCE="board-$maxIdx"
+    else
+        # No sensors at all — full speed to be safe
+        TEMP=50
+        SOURCE="none (full speed)"
+    fi
+fi
+
+# Initialize tracking variables with current values
+prevTemp=$TEMP
+prevTime=$(date +%s)
+burstTimer=0
+lastMode=0
+
+# Print detected GPUs (if any)
+if [ "$gpuCount" -gt 0 ]; then
+    echo "[$(ts)] [INFO] Detected $gpuCount NVIDIA GPU(s):"
+    for ((i=0; i<gpuCount; i++)); do
+        echo "[$(ts)]   GPU $i: index=${gpu_idx[$i]}"
+    done
+fi
+
+# Print all motherboard zones
+if [ "$boardCount" -gt 0 ]; then
+    echo "[$(ts)] [INFO] Detected $boardCount motherboard thermal zone(s):"
+    for ((i=0; i<boardCount; i++)); do
+        echo "[$(ts)]   Zone $i: type=${board_types[$i]}"
+    done
+fi
+
+echo "[$(ts)] [INFO] Starting fan control. Monitoring $SOURCE. Temperature: ${TEMP}°C"
+
+# ---- Main loop -----------------------------------------------------------
 
 while true; do
-  # Определяем количество GPU и читаем все температуры
-  ALLTEMP=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader | tr -cd '0-9\n')
     now=$(date +%s)
 
-  gpuCount=0
+    # Re-read current temperatures
-  maxTemp=0
+    if [ "$gpuCount" -gt 0 ]; then
-  burstTriggered=0
+        read_max_gpu_temps
-  
+        SOURCE="gpu-$maxIdx"
-  while IFS= read -r t; do
+    else
-    [ -z "$t" ] && continue
+        read_max_board_temps
-    [ "$t" -lt 0 ] 2>/dev/null && continue
+        if [ -n "$maxIdx" ]; then
-    gpuCount=$((gpuCount + 1))
+            SOURCE="board-$maxIdx"
-    if [ "$t" -gt "$maxTemp" ]; then maxTemp=$t; fi
-    
-    # Burst по каждой карте отдельно
-    if [ $prevTime -gt 0 ] && [ $((prevTime != 0 ? now - prevTime : 99999)) -le $BURST_WINDOW ]; then
-      rise=$((t - prevTemp))
-      if [ $rise -ge $TH_BURST_ROISE ]; then
-        burstTriggered=1
-        echo "[BURST-rise] GPU${gpuCount}: rise=$rise°C in $((now-prevTime))s → mode 6"
         fi
     fi
-    prevTemp=$t  # сохраняем для следующей итерации
+    TEMP=$maxTemp
-  done <<< "$ALLTEMP"
 
-  if [ $gpuCount -eq 0 ]; then
+    echo "[$(ts)] [STATUS] All temps: ${allTempsStr:-none}, max=$TEMP°C from $SOURCE" >&4
+
+    # Check if the temperature source is still alive
+    if [ "$TEMP" -le 0 ] 2>/dev/null; then
         noDataCount=$((noDataCount + 1))
         if [ $noDataCount -ge 10 ]; then
-      echo "[ERROR] nvidia-smi failed 10+ times, aborting"
+            echo "[$(ts)] [ERROR] No temperature data for 10+ seconds. Running fan at full speed."
+            echo "6" >&3
             exit 1
         fi
         sleep 1; continue
     fi
+
     noDataCount=0
 
-  TEMP=$maxTemp  # берём max для логики
+    # --- 1. Если burst-режим активен — держим режим 6 ---
-
-  # Если в burst-режиме — держим режим 6
     if [ $burstTimer -gt 0 ]; then
         burstTimer=$((burstTimer - 1))
         if [ $lastMode -ne 6 ]; then
-      echo "[MODE 6] full speed (burst, ${burstTimer}s remain)"
+            echo "[$(ts)] [MODE 6] Full speed (burst, ${burstTimer}s remaining)"
             lastMode=6
             echo "6" >&3
         fi
+        prevTemp=$TEMP; prevTime=$now
         sleep 1; continue
     fi
 
-  # Если burst истёк И temp >= TH_BURST — новый burst
+    # --- 2. Если temp >= TH_BURST (45) и burst не активен — новый burst ---
-  if [ $TEMP -ge $TH_BURST ] || [ $burstTriggered -eq 1 ]; then
+    if [ "$TEMP" -ge $TH_BURST ]; then
         burstTimer=$BURST_DURATION
-    echo "[MODE 6] full speed (re-burst: $TEMP°C)"
+        echo "[$(ts)] [MODE 6] Full speed (re-burst: $TEMP°C)"
         lastMode=6; prevTemp=$TEMP; prevTime=$now
         echo "6" >&3
         sleep 1; continue
     fi
 
-  # Обычная логика с хистерезисом
+    # --- 3. Burst detection: быстрый рост температуры ---
-  newMode=5  # default: HIGH (35-39)
+    if [ $((now - prevTime)) -le $BURST_WINDOW ] && [ $prevTime -gt 0 ]; then
+        rise=$((TEMP - prevTemp))
+        if [ $rise -ge $TH_BURST_RISE ]; then
+            burstTimer=$BURST_DURATION
+            echo "[$(ts)] [BURST] $TEMP°C (up from $prevTemp°C in $((now - prevTime))s, window=$BURST_WINDOW)"
+            lastMode=6
+            prevTemp=$TEMP; prevTime=$now
+            echo "6" >&3
+            sleep 1; continue
+        fi
+    fi
 
+    # --- 4. Обычная логика с гистерезисом ---
+    newMode=5  # default: HIGH (35-39)
     if [ $TEMP -lt $((TH_COLD - HYST)) ]; then
         newMode=2  # STANDBY (< 23)
     elif [ $TEMP -lt $((TH_LOW - HYST)) ]; then
@@ -101,13 +284,13 @@ while true; do
         newMode=4  # MED (< 33)
     fi
 
-  # Отправляем только если режим изменился
     if [ $newMode -ne $lastMode ]; then
-    echo "[MODE $newMode] max_temp=$TEMP°C (gpuCount=$gpuCount) (hysteresis: no-change zone)"
+        echo "[$(ts)] [MODE $newMode] Temperature: $TEMP°C, source: $SOURCE"
         lastMode=$newMode
         echo "$newMode" >&3
     fi
 
+    prevTemp=$TEMP
     prevTime=$now
     sleep 1
 done

systemd/gpu-fan-ctrl.service

@@ -5,7 +5,7 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-ExecStart=/home/romanown/gpu-fan-ctrl/scripts/gpu_fan_ctrl.sh
+ExecStart=/root/gpu_fan_ctrl.sh
 Restart=on-failure
 RestartSec=5
 StandardOutput=journal