#!/bin/bash
# usage_alerter.sh — 機器用量告警 v1（admin_machine_spec.php Phase 2）
#
# 跑在 Tools host 上，cron 每 10 min（偏 collector 5 min）：
#   1. 拉過去 1 小時內有 sample 的 target
#   2. 拉所有 enabled rules（machine_alert_rules）
#   3. 拉現有 alert_state（machine_alert_state）
#   4. 對每個 (target_ip, metric) ∈ {cpu_pct, ram_pct, disk_pct}：
#        a) 找最具體的規則（target_pattern ILIKE + metric 完全比對 > '*'）
#        b) ignore → 跳過
#        c) override → 用規則的門檻；否則用 DEFAULT
#        d) 連續 samples_required 筆都 ≥ crit → critical；都 ≥ warn → warning；否則 ok
#        e) 跟舊 state 比，狀態變化才推 Google Chat（state-change-only 不洗版）
#        f) UPSERT machine_alert_state
#
# webhook URL 從 config.php $GLOBALS['gchat_webhook_alerter'] 抽；空字串就 silent skip 推送（仍會評估+寫 state）
# log 寫 files/machine_usage_alerter.log

set -u

TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
CONFIG_PHP="$TOOLS_DIR/config.php"
LOG_FILE="$TOOLS_DIR/files/machine_usage_alerter.log"

_log() { printf '[%s] %s\n' "$(date '+%F %T')" "$*" | tee -a "$LOG_FILE" >&2; }

_cfg_get() {
    local key="$1"
    sed -n "s/.*\$GLOBALS\['${key}'\][[:space:]]*=[[:space:]]*['\"]\([^'\"]*\).*/\1/p" "$CONFIG_PHP" 2>/dev/null | head -1
}

[[ -f "$CONFIG_PHP" ]] || { _log "❌ config.php 找不到: $CONFIG_PHP"; exit 1; }

DB_IP="$(_cfg_get tools_db_ip)"
DB_NAME="$(_cfg_get tools_db_name)"
DB_USER="$(_cfg_get tools_db_user)"
DB_PASS="$(_cfg_get tools_db_pass)"
MACHINE_KIND="$(_cfg_get machine)"
TOOLS_URL="$(_cfg_get tools_url)"
WEBHOOK="$(_cfg_get gchat_webhook_alerter)"

ENV_CODE="$(_cfg_get project_code | tr 'a-z' 'A-Z')"
[[ "$MACHINE_KIND" == *DEV* || "$MACHINE_KIND" == *dev* ]] && ENV_CODE="${ENV_CODE}-DEV"

if [[ -z "$DB_IP" || -z "$DB_PASS" || -z "$DB_NAME" || -z "$DB_USER" ]]; then
    _log "❌ config.php 抽不出 tools_db_*（ip='$DB_IP' name='$DB_NAME' user='$DB_USER' pass='${DB_PASS:+***}'）"
    exit 1
fi

if [[ -z "$WEBHOOK" ]]; then
    _log "⚠️  gchat_webhook_alerter 未設定，仍會評估+寫 state，但不推 Google Chat"
fi

_psql() {
    PGPASSWORD="$DB_PASS" psql -h "$DB_IP" -U "$DB_USER" -d "$DB_NAME" \
        -AtX -F '|' -v ON_ERROR_STOP=1 "$@"
}

# ── Defaults（沒 rule override 時用這組）─────────────────────────────────
declare -A DEFAULT_WARN=( [cpu_pct]=70 [ram_pct]=75 [disk_pct]=80 )
declare -A DEFAULT_CRIT=( [cpu_pct]=80 [ram_pct]=85 [disk_pct]=90 )
declare -A DEFAULT_SAMPLES=( [cpu_pct]=3 [ram_pct]=3 [disk_pct]=1 )
METRICS=(cpu_pct ram_pct disk_pct)

# 顯示用 metric 中文（推 Google Chat 時用）
declare -A METRIC_LABEL=( [cpu_pct]=CPU [ram_pct]=RAM [disk_pct]=Disk )

# ── 1. 拉過去 1 小時 active targets ──────────────────────────────────────
mapfile -t TARGET_ROWS < <(_psql -c "
    SELECT DISTINCT ON (target_ip) target_ip, target
    FROM public.machine_usage_sample
    WHERE sampled_at >= NOW() - INTERVAL '1 hour'
    ORDER BY target_ip, sampled_at DESC;
")

if [[ ${#TARGET_ROWS[@]} -eq 0 ]]; then
    _log "⚠️  過去 1 小時無 sample（collector 掛了？）；alerter exit"
    exit 0
fi

# target_ip → target_name
declare -A TARGET_NAME
for r in "${TARGET_ROWS[@]}"; do
    [[ -z "$r" ]] && continue
    IFS='|' read -r ip name <<< "$r"
    TARGET_NAME["$ip"]="$name"
done

# ── 2. 拉所有 enabled rules（最具體的排前面，find_rule 取第一條命中）───
mapfile -t RULE_ROWS < <(_psql -c "
    SELECT id, target_pattern, metric, action,
           COALESCE(threshold_warn::text, ''),
           COALESCE(threshold_crit::text, ''),
           COALESCE(samples_required::text, '')
    FROM public.machine_alert_rules
    WHERE enabled = TRUE
    ORDER BY length(target_pattern) DESC, (metric = '*') ASC, id ASC;
")

# ── 3. 拉現有 alert_state（key = target_ip|metric → status|last_value）──
mapfile -t STATE_ROWS < <(_psql -c "
    SELECT target_ip, metric, status, COALESCE(last_value::text, '')
    FROM public.machine_alert_state;
")
declare -A CUR_STATUS CUR_VALUE
for r in "${STATE_ROWS[@]}"; do
    [[ -z "$r" ]] && continue
    IFS='|' read -r ip mt st val <<< "$r"
    CUR_STATUS["$ip|$mt"]="$st"
    CUR_VALUE["$ip|$mt"]="$val"
done

# ── 4. 拉最近 5 筆 sample per target（取最新 N 用）───────────────────────
mapfile -t SAMPLE_ROWS < <(_psql -c "
    SELECT target_ip,
           CASE WHEN ssh_ok THEN cpu_pct::text ELSE '' END,
           CASE WHEN ssh_ok AND ram_total_mb > 0 THEN ROUND(ram_used_mb::numeric / ram_total_mb * 100, 2)::text ELSE '' END,
           CASE WHEN ssh_ok AND disk_total_mb > 0 THEN ROUND(disk_used_mb::numeric / disk_total_mb * 100, 2)::text ELSE '' END,
           rn
    FROM (
        SELECT *, ROW_NUMBER() OVER (PARTITION BY target_ip ORDER BY sampled_at DESC) AS rn
        FROM public.machine_usage_sample
        WHERE sampled_at >= NOW() - INTERVAL '1 hour'
    ) s
    WHERE rn <= 5
    ORDER BY target_ip, rn;
")

# 把 sample 整理成 per-target per-metric 陣列（rn=1 是最新）
# SAMPLES_BY_KEY[ip|metric] = "v1 v2 v3 v4 v5"（空字串代表 ssh_ok=false 或 NULL）
declare -A SAMPLES_BY_KEY
for r in "${SAMPLE_ROWS[@]}"; do
    [[ -z "$r" ]] && continue
    IFS='|' read -r ip cpu ram disk rn <<< "$r"
    SAMPLES_BY_KEY["$ip|cpu_pct"]+="$cpu "
    SAMPLES_BY_KEY["$ip|ram_pct"]+="$ram "
    SAMPLES_BY_KEY["$ip|disk_pct"]+="$disk "
done

# ── 規則查詢：給定 (target_name, metric)，回 ignore | override:warn:crit:n | default:warn:crit:n
find_rule() {
    local target="$1" metric="$2"
    local r _id pat m action twarn tcrit tsamples
    for r in "${RULE_ROWS[@]}"; do
        [[ -z "$r" ]] && continue
        IFS='|' read -r _id pat m action twarn tcrit tsamples <<< "$r"
        # ILIKE 模擬：% → bash *；shopt nocasematch 開大小寫無關
        local glob="${pat//%/*}"
        local matched=0
        shopt -s nocasematch
        # shellcheck disable=SC2053  # 故意不 quote：$glob 要做 glob match（ILIKE 模擬核心）
        if [[ "$target" == $glob && ( "$m" == "$metric" || "$m" == "*" ) ]]; then
            matched=1
        fi
        shopt -u nocasematch
        if [[ "$matched" == "1" ]]; then
            if [[ "$action" == "ignore" ]]; then echo "ignore"; return; fi
            local w="${twarn:-${DEFAULT_WARN[$metric]}}"
            local c="${tcrit:-${DEFAULT_CRIT[$metric]}}"
            local s="${tsamples:-${DEFAULT_SAMPLES[$metric]}}"
            echo "override:$w:$c:$s"; return
        fi
    done
    echo "default:${DEFAULT_WARN[$metric]}:${DEFAULT_CRIT[$metric]}:${DEFAULT_SAMPLES[$metric]}"
}

# 算 new status：ALL of last N 都 ≥ crit → critical；都 ≥ warn → warning；否則 ok
# usage: eval_status <"v1 v2 v3 ..."> <warn> <crit> <n>
# echo: status|latest_value（latest 是 rn=1 的值；若 N 筆不足 / 空字串多 → echo "insufficient"）
eval_status() {
    local samples_str="$1" warn="$2" crit="$3" n="$4"
    read -r -a arr <<< "$samples_str"
    if [[ ${#arr[@]} -lt $n ]]; then echo "insufficient|"; return; fi
    local latest="${arr[0]}"
    local i v
    local all_ge_warn=1 all_ge_crit=1 valid=0
    for ((i=0; i<n; i++)); do
        v="${arr[i]}"
        if [[ -z "$v" ]]; then echo "insufficient|"; return; fi
        valid=$((valid+1))
        # 數值比較（NUMERIC，可能帶小數）
        if (( $(awk -v a="$v" -v b="$warn" 'BEGIN{print (a>=b)?1:0}') == 0 )); then all_ge_warn=0; fi
        if (( $(awk -v a="$v" -v b="$crit" 'BEGIN{print (a>=b)?1:0}') == 0 )); then all_ge_crit=0; fi
    done
    if [[ "$all_ge_crit" == "1" ]]; then echo "critical|$latest"; return; fi
    if [[ "$all_ge_warn" == "1" ]]; then echo "warning|$latest"; return; fi
    echo "ok|$latest"
}

# ── Google Chat webhook 推送 ─────────────────────────────────────────────
push_chat() {
    local title="$1" body="$2"
    [[ -z "$WEBHOOK" ]] && return 0
    local payload
    # Google Chat 接受 simple text，也支援 cards；先 simple text MVP
    payload=$(printf '{"text": "*%s*\\n%s"}' "$title" "$body" | sed 's/[[:cntrl:]]//g')
    curl -sS -m 8 -H 'Content-Type: application/json' -X POST -d "$payload" "$WEBHOOK" >/dev/null 2>&1 || \
        _log "⚠️  webhook 推送失敗 title='$title'"
}

# 把 last N 個 sample 字串化（log + chat body 用）
fmt_samples() {
    local samples_str="$1" n="$2"
    read -r -a arr <<< "$samples_str"
    local out i
    for ((i=0; i<n && i<${#arr[@]}; i++)); do
        out+="${arr[i]:-(no_data)} "
    done
    echo "${out% }"
}

# ── 5. 主迴圈 ────────────────────────────────────────────────────────────
ts_start=$(date '+%F %T')
declare -a UPSERT_VALUES
fired_count=0
ignored_count=0
unchanged_count=0
insufficient_count=0

for ip in "${!TARGET_NAME[@]}"; do
    target="${TARGET_NAME[$ip]}"
    for metric in "${METRICS[@]}"; do
        rule_info=$(find_rule "$target" "$metric")
        if [[ "$rule_info" == "ignore" ]]; then
            ignored_count=$((ignored_count+1))
            continue
        fi
        IFS=':' read -r _kind warn crit n <<< "$rule_info"

        samples_str="${SAMPLES_BY_KEY["$ip|$metric"]:-}"
        eval_out=$(eval_status "$samples_str" "$warn" "$crit" "$n")
        IFS='|' read -r new_status latest <<< "$eval_out"

        if [[ "$new_status" == "insufficient" ]]; then
            insufficient_count=$((insufficient_count+1))
            continue
        fi

        old_status="${CUR_STATUS["$ip|$metric"]:-ok}"

        if [[ "$new_status" != "$old_status" ]]; then
            # 推訊息
            label="${METRIC_LABEL[$metric]}"
            sample_brief=$(fmt_samples "$samples_str" "$n")
            link=""
            [[ -n "$TOOLS_URL" ]] && link=$'\n'"📊 ${TOOLS_URL%/}/admin_machine_spec.php"

            case "${old_status}:${new_status}" in
                ok:warning)
                    title="[${ENV_CODE}][WARNING] ${target} ${label} ≥ ${warn}% × ${n}"
                    body="現值 ${latest}% (samples: ${sample_brief})${link}"
                    ;;
                ok:critical)
                    title="[${ENV_CODE}][CRITICAL] ${target} ${label} ≥ ${crit}% × ${n}"
                    body="現值 ${latest}% (samples: ${sample_brief})${link}"
                    ;;
                warning:critical)
                    title="[${ENV_CODE}][CRITICAL] ${target} ${label} 升級到 ≥ ${crit}%"
                    body="現值 ${latest}% (samples: ${sample_brief})${link}"
                    ;;
                critical:warning)
                    title="[${ENV_CODE}][RECOVERING] ${target} ${label} 降到 warning"
                    body="現值 ${latest}% (samples: ${sample_brief})${link}"
                    ;;
                warning:ok|critical:ok)
                    title="[${ENV_CODE}][OK] ${target} ${label} 回復正常"
                    body="現值 ${latest}% (samples: ${sample_brief})${link}"
                    ;;
                *)
                    title="[${ENV_CODE}][${new_status^^}] ${target} ${label}"
                    body="state ${old_status}→${new_status} 現值 ${latest}%"
                    ;;
            esac

            # ram_pct + DB-class 補刀：建議去 admin_pg_analyzer.php 重評（resize 後 buffer 基準會失效）
            # 只在進入 warning/critical 時加（recovery 不加，避免 OK 訊息也帶建議）
            if [[ "$metric" == "ram_pct" && "$new_status" =~ ^(warning|critical)$ ]] \
               && [[ "$target" =~ ^(WORLDDB|GAMEDB|ACCOUNTDB|SOCIETYDB)[0-9]*$ ]]; then
                if [[ -n "$TOOLS_URL" ]]; then
                    body+=$'\n'"↳ 建議去 ${TOOLS_URL%/}/admin_pg_analyzer.php#target-${target} 看是否該調 shared_buffers / effective_cache_size"
                else
                    body+=$'\n'"↳ 建議去 admin_pg_analyzer.php 看是否該調 shared_buffers / effective_cache_size"
                fi
            fi

            push_chat "$title" "$body"
            _log "🔔 fire ${target}/${metric} ${old_status}→${new_status} latest=${latest}% threshold(warn=${warn} crit=${crit}) n=${n}"
            fired_count=$((fired_count+1))
        else
            unchanged_count=$((unchanged_count+1))
        fi

        # UPSERT 累積
        # NOTE: latest 可能空白（理論不會走到這，已被 insufficient 擋）
        latest_sql=$([[ -z "$latest" ]] && echo "NULL" || echo "$latest")
        # 安全跳脫 single-quote
        ip_e=${ip//\'/\'\'}
        metric_e=${metric//\'/\'\'}
        status_e=${new_status//\'/\'\'}
        target_e=${target//\'/\'\'}
        UPSERT_VALUES+=("('$ip_e','$metric_e','$status_e',${latest_sql},'$target_e')")
    done
done

# ── 6. UPSERT alert_state（state 沒變也更新 last_evaluated_at；變了也更新 last_changed_at）──
if [[ ${#UPSERT_VALUES[@]} -gt 0 ]]; then
    # 用 jq-like join：bash 沒有就用 printf 自接
    values_csv=""
    for v in "${UPSERT_VALUES[@]}"; do values_csv+="$v,"; done
    values_csv="${values_csv%,}"
    {
        echo "INSERT INTO public.machine_alert_state"
        echo "  (target_ip, metric, status, last_value, last_target_name)"
        echo "VALUES"
        echo "  $values_csv"
        echo "ON CONFLICT (target_ip, metric) DO UPDATE SET"
        echo "  status            = EXCLUDED.status,"
        echo "  last_value        = EXCLUDED.last_value,"
        echo "  last_target_name  = EXCLUDED.last_target_name,"
        echo "  last_changed_at   = CASE WHEN public.machine_alert_state.status <> EXCLUDED.status"
        echo "                           THEN CURRENT_TIMESTAMP"
        echo "                           ELSE public.machine_alert_state.last_changed_at"
        echo "                      END,"
        echo "  last_evaluated_at = CURRENT_TIMESTAMP;"
    } | _psql >/dev/null 2>>"$LOG_FILE"
    rc=$?
    if [[ $rc -ne 0 ]]; then
        _log "❌ UPSERT machine_alert_state 失敗 rc=$rc"
        exit 1
    fi
fi

_log "✓ env=${ENV_CODE} targets=${#TARGET_NAME[@]} fired=${fired_count} unchanged=${unchanged_count} ignored=${ignored_count} insufficient=${insufficient_count} ($ts_start → $(date '+%T'))"
