#!/bin/bash
# spec_orchestrator.sh — admin_machine_spec.php 觸發的協調腳本
#
# 用法（由 ajax_tools_backend.php trigger_job 透過 setsid nohup 觸發）：
#   spec_orchestrator.sh \
#       --batch-id BID \
#       --target n1-jp-deb12-live-worlddb1110 \
#       --action resize \
#       --new-ocpu 2 --new-ram-gb 16 \
#       [--applied-by user]
#
# 流程：
#   1. SSH center 跑 spec_runner.py，stdout/stderr 直出（PHP poll_job 看 log 即可）
#   2. 解析 spec_runner.py 輸出最後一行 STEP，更新 spec_apply_audit 進度
#   3. ssh exit code 決定 status='success' or 'fail'
#   4. action=grow-disk 成功後 / action=grow-fs-only → 再 SSH target 跑 OS fs grow
#   5. 釋放 spec_apply_lock

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./_lib_local.sh
source "${SCRIPT_DIR}/_lib_local.sh"

BATCH_ID=""
TARGET=""
ACTION=""
NEW_OCPU=""
NEW_RAM_GB=""
NEW_DISK_GB=""
TARGET_IP=""
APPLIED_BY="${USER:-unknown}"

while [[ $# -gt 0 ]]; do
    case "$1" in
        --batch-id)     BATCH_ID="$2"; shift 2 ;;
        --target)       TARGET="$2"; shift 2 ;;
        --action)       ACTION="$2"; shift 2 ;;
        --new-ocpu)     NEW_OCPU="$2"; shift 2 ;;
        --new-ram-gb)   NEW_RAM_GB="$2"; shift 2 ;;
        --new-disk-gb)  NEW_DISK_GB="$2"; shift 2 ;;
        --ip)           TARGET_IP="$2"; shift 2 ;;
        --applied-by)   APPLIED_BY="$2"; shift 2 ;;
        *) echo "未知參數: $1" >&2; exit 1 ;;
    esac
done

[[ -z "$BATCH_ID" || -z "$TARGET" || -z "$ACTION" ]] && {
    echo "缺少必要參數：--batch-id --target --action" >&2; exit 1; }

ENV_CODE=$(spec_target_to_env "$TARGET")
[[ -z "$ENV_CODE" ]] && {
    spec_audit_update "$BATCH_ID" "status='fail', error_msg='不認識的 target prefix: ${TARGET}', completed_at=NOW()"
    echo "STEP=fail|reason=unknown env from target prefix: $TARGET"
    exit 2
}

echo "=== machine_spec orchestrator ==="
echo "batch_id    : ${BATCH_ID}"
echo "target      : ${TARGET}"
echo "env_code    : ${ENV_CODE}"
echo "action      : ${ACTION}"
echo "new_ocpu    : ${NEW_OCPU}"
echo "new_ram_gb  : ${NEW_RAM_GB}"
echo "new_disk_gb : ${NEW_DISK_GB}"
echo "target_ip   : ${TARGET_IP}"
echo "applied_by  : ${APPLIED_BY}"

# ── OS filesystem grow（SSH target 跑 inline HEREDOC）─────────────────────────
# 用於 grow-disk 成功後 / grow-fs-only。
# 回傳 0 = fs 擴展成功，非 0 = 失敗（API 已成功時不影響整體 status）。
_ssh_grow_fs() {
    local ip="$1"
    [[ -z "$ip" ]] && { echo "[grow_fs] 缺 target IP，跳過 fs grow"; return 1; }

    echo "###### SSH target ${ip} 擴 OS filesystem ######"
    ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
        "n1admin@${ip}" 'bash -s' <<'GROWFS_EOF'
set -uo pipefail
ROOT_DEV=$(findmnt -no SOURCE /)            # e.g. /dev/sda1
ROOT_PKNAME=$(lsblk -no PKNAME "$ROOT_DEV") # e.g. sda
ROOT_DISK="/dev/${ROOT_PKNAME}"
PART_NUM=$(echo "$ROOT_DEV" | grep -oE '[0-9]+$')
FS_TYPE=$(findmnt -no FSTYPE /)

echo "[growfs] ROOT_DEV=$ROOT_DEV  ROOT_DISK=$ROOT_DISK  PART_NUM=$PART_NUM  FS_TYPE=$FS_TYPE"
echo "[growfs] before:"; df -hT /
SIZE_BEFORE=$(df -B1 --output=size / | tail -1 | tr -d ' ')
DISK_BEFORE=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
echo "[growfs] kernel 看到 ${ROOT_DISK} = ${DISK_BEFORE} bytes（rescan 前）"

# 0. SCSI rescan — Debian 12 OCI 上 partprobe 不會戳 sysfs rescan node，
#    kernel 還是看舊 size，導致 growpart 永遠 NOOP。要手動 echo 1 強制重讀。
RESCAN_NODE="/sys/class/block/${ROOT_PKNAME}/device/rescan"
if [ -e "$RESCAN_NODE" ]; then
    if echo 1 | sudo tee "$RESCAN_NODE" >/dev/null; then
        echo "[growfs] SCSI rescan: OK ($RESCAN_NODE)"
    else
        echo "[growfs] WARN: SCSI rescan tee 失敗" >&2
    fi
    sleep 2
else
    echo "[growfs] WARN: 沒有 $RESCAN_NODE，跳過 SCSI rescan"
fi
DISK_AFTER_RESCAN=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
echo "[growfs] kernel 看到 ${ROOT_DISK} = ${DISK_AFTER_RESCAN} bytes（rescan 後）"

# 1. partprobe — 重掃 partition table
sudo partprobe "$ROOT_DISK" 2>/dev/null || true

# 2. growpart — 擴 partition
#    rc=0 成功；rc=1 已是最大（NOOP）；rc>=2 失敗
GP_RC=0
sudo growpart "$ROOT_DISK" "$PART_NUM" || GP_RC=$?
case "$GP_RC" in
    0) echo "[growfs] growpart: OK" ;;
    1) echo "[growfs] growpart: NOOP（partition 已是最大）" ;;
    *) echo "[growfs] growpart FAILED rc=$GP_RC" >&2; exit 11 ;;
esac

# 3. 擴 filesystem
case "$FS_TYPE" in
    ext2|ext3|ext4) sudo resize2fs "$ROOT_DEV" || exit 12 ;;
    xfs)            sudo xfs_growfs / || exit 12 ;;
    *) echo "[growfs] 不支援的 fs type: $FS_TYPE" >&2; exit 13 ;;
esac

echo "[growfs] after:"; df -hT /
SIZE_AFTER=$(df -B1 --output=size / | tail -1 | tr -d ' ')

# 4. 真的長了嗎？分三種狀態判斷，不要 silent success 也不要假 fail：
#    a) fs 真的長了        → 成功（一般 grow path）
#    b) fs 沒長，但已對齊 disk → 已對齊 no-op，成功（UI 顯示舊值才會走到這，下面會寫 sample 修 UI）
#    c) fs 沒長，且 fs < disk → 真失敗（SCSI rescan 沒生效之類）
if [ "${SIZE_AFTER:-0}" -gt "${SIZE_BEFORE:-0}" ]; then
    echo "[growfs] DONE (從 ${SIZE_BEFORE} 長到 ${SIZE_AFTER} bytes)"
elif [ "${DISK_AFTER_RESCAN:-0}" -gt 0 ] \
     && [ "${SIZE_AFTER:-0}" -ge $((DISK_AFTER_RESCAN * 95 / 100)) ]; then
    # fs >= 95% disk size：扣掉 partition table / fs metadata overhead 後算對齊（ext4 200GB 約 98.3%）
    echo "[growfs] NO-OP: fs (${SIZE_AFTER} bytes) 已與 disk (${DISK_AFTER_RESCAN} bytes) 對齊，無需擴展"
    echo "[growfs] DONE (no-op)"
else
    echo "[growfs] FAIL: filesystem 沒有真的擴大 (before=${SIZE_BEFORE} after=${SIZE_AFTER} bytes; disk=${DISK_AFTER_RESCAN} bytes)" >&2
    echo "[growfs] 提示：kernel block size rescan 前=${DISK_BEFORE} 後=${DISK_AFTER_RESCAN}；若兩者相同代表 SCSI rescan 沒生效" >&2
    exit 14
fi
GROWFS_EOF
    return $?
}

# 把擴 fs 後的新 disk_total_mb 寫進 machine_usage_sample，
# 讓 admin_machine_spec.php 的 24h aggregate view (MAX) 立刻看到新值，
# 不必等下次 usage_collector cron。避免 grow-fs-only / grow-disk 成功後 UI 仍顯示舊值
# 而誤判「與 cloud 不一致」並要使用者再點一次。
_refresh_disk_usage_sample() {
    local ip="$1"
    [[ -z "$ip" ]] && return 0
    local dt_mb
    dt_mb=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
        "n1admin@${ip}" "df -m / | awk 'NR==2 {print \$2}'" 2>/dev/null | head -1 | tr -d ' \r\n')
    if [[ -n "$dt_mb" && "$dt_mb" =~ ^[0-9]+$ ]]; then
        _spec_psql "INSERT INTO public.machine_usage_sample (env_code, target, target_ip, ssh_ok, disk_total_mb, sampled_at) VALUES (\$txt\$${ENV_CODE}\$txt\$, \$txt\$${TARGET}\$txt\$, \$txt\$${ip}\$txt\$, true, ${dt_mb}, NOW())" >/dev/null 2>&1 \
            && echo "[refresh] machine_usage_sample.disk_total_mb=${dt_mb} 已寫入（UI 下次掃描即可看到）"
    else
        echo "[refresh] WARN: 無法取得 df 結果（dt_mb='${dt_mb}'），UI 要等下次 usage_collector cron"
    fi
}

# ── 分支：grow-fs-only 跳過 cloud API，只做 OS fs grow ─────────────────────────
if [[ "$ACTION" == "grow-fs-only" ]]; then
    spec_audit_update "$BATCH_ID" "status='running', progress_step='fs_grow_only'"

    set +e
    _ssh_grow_fs "$TARGET_IP"
    GROWFS_EXIT=$?
    set -e

    if [[ "$GROWFS_EXIT" -eq 0 ]]; then
        spec_audit_update "$BATCH_ID" "status='success', progress_step='fs_grown', completed_at=NOW()"
        echo "STEP=fs_grown|target=${TARGET}"
        _refresh_disk_usage_sample "$TARGET_IP"
        _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
        exit 0
    else
        spec_audit_update "$BATCH_ID" \
            "status='fail', progress_step='fs_grow_fail', error_msg=\$txt\$grow-fs-only failed (rc=${GROWFS_EXIT})\$txt\$, completed_at=NOW()"
        echo "STEP=fs_grow_fail|target=${TARGET}|rc=${GROWFS_EXIT}"
        _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
        exit "$GROWFS_EXIT"
    fi
fi

# ── 既有流程：呼叫 spec_runner.py（SSH center）─────────────────────────────────
echo "ssh: ${CENTER_USER}@${CENTER_HOST} ${SPEC_RUNNER}"
echo

# 組 spec_runner.py 參數
RUNNER_ARGS=(
    --env "$ENV_CODE"
    --action "$ACTION"
    --target "$TARGET"
)
[[ -n "$NEW_OCPU"    ]] && RUNNER_ARGS+=( --new-ocpu "$NEW_OCPU" )
[[ -n "$NEW_RAM_GB"  ]] && RUNNER_ARGS+=( --new-ram-gb "$NEW_RAM_GB" )
[[ -n "$NEW_DISK_GB" ]] && RUNNER_ARGS+=( --new-disk-gb "$NEW_DISK_GB" )

# 本地 log（給後端 progress 解析）
LOG_FILE=$(mktemp -t spec_orch_${BATCH_ID}_XXXX.log)

# 啟動執行（log 雙寫：stdout 給 PHP poll_job，檔案給後續 parse）
spec_audit_update "$BATCH_ID" "status='running', progress_step='start'"

set +e
spec_run_remote "${RUNNER_ARGS[@]}" 2>&1 | tee "$LOG_FILE"
RUNNER_EXIT=${PIPESTATUS[0]}
set -e

echo
echo "=== runner exit: ${RUNNER_EXIT} ==="

# 解析最後一個 STEP= 行作為 progress_step
LAST_STEP=$(grep -E '^STEP=' "$LOG_FILE" 2>/dev/null | tail -1 | sed -E 's/^STEP=([^|]+).*/\1/')

if [[ "$RUNNER_EXIT" -eq 0 ]]; then
    # ── grow-disk 成功 → 追加 SSH target 擴 OS fs ───────────────────────────
    # API resize 已成功，所以 status 一律 'success'；fs 失敗只降為 progress_step='fs_grow_warn' 警示
    if [[ "$ACTION" == "grow-disk" ]]; then
        # 等 cloud volume hydration 一下，避免 partprobe rescan 還看不到新 size
        sleep 5

        set +e
        _ssh_grow_fs "$TARGET_IP"
        GROWFS_EXIT=$?
        set -e

        if [[ "$GROWFS_EXIT" -eq 0 ]]; then
            spec_audit_update "$BATCH_ID" \
                "status='success', progress_step='fs_grown', completed_at=NOW()"
            echo "STEP=fs_grown|target=${TARGET}"
            _refresh_disk_usage_sample "$TARGET_IP"
        else
            spec_audit_update "$BATCH_ID" \
                "status='success', progress_step='fs_grow_warn', error_msg=\$txt\$cloud disk 擴展成功，但 OS fs 擴展失敗 (rc=${GROWFS_EXIT})；請手動執行 grow-fs-only 補做\$txt\$, completed_at=NOW()"
            echo "STEP=fs_grow_warn|target=${TARGET}|rc=${GROWFS_EXIT}"
        fi
    else
        spec_audit_update "$BATCH_ID" "status='success', progress_step='${LAST_STEP:-done}', completed_at=NOW()"
    fi
else
    # 取最後一個 fail 的 reason
    FAIL_REASON=$(grep -E '^STEP=fail' "$LOG_FILE" 2>/dev/null | tail -1 | sed -E 's/.*reason=([^|]*).*/\1/')
    [[ -z "$FAIL_REASON" ]] && FAIL_REASON="exit ${RUNNER_EXIT}"
    # 用 dollar-quote 避免 ' 跳脫地獄
    spec_audit_update "$BATCH_ID" \
        "status='fail', progress_step='${LAST_STEP:-fail}', error_msg=\$txt\$${FAIL_REASON}\$txt\$, completed_at=NOW()"
fi

# 釋放 lock
_spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"

# 清掉本地 log
rm -f "$LOG_FILE"

exit "$RUNNER_EXIT"
