#!/bin/bash
# spec_orchestrator.sh — admin_machine_spec.php 觸發的協調腳本
#
# 用法（由 ajax_tools_backend.php trigger_job 透過 setsid nohup 觸發）：
#   spec_orchestrator.sh \
#       --batch-id BID \
#       --target n1-jp-deb12-live-worlddb1110 \
#       --action resize \
#       --new-ocpu 2 --new-ram-gb 16 \
#       [--applied-by user] \
#       [--apply-pg 1 --pg-cpu 2 --pg-ram-gb 16]
#
# 當 action=resize 且 --apply-pg=1：
#   pre-stop  → SSH target sed postgresql.conf 寫 pgtune 新值（PG 不 restart）
#   stop → resize → start（spec_runner.py 走既有 chain）
#   post-start → SSH target SHOW 20 個 whitelist param 比對推薦值，記 audit
#   失敗 rollback：
#     stop 階段 fail（機器 RUNNING）→ SSH 跑 pg_rollback.sh
#     resize/start 階段 fail（機器 STOPPED）→ 不自動 rollback，alert user 手動處理
#
# 流程：
#   1. SSH center 跑 spec_runner.py，stdout/stderr 直出（PHP poll_job 看 log 即可）
#   2. 解析 spec_runner.py 輸出最後一行 STEP，更新 spec_apply_audit 進度
#   3. ssh exit code 決定 status='success' or 'fail'
#   4. action=grow-disk 成功後 / action=grow-fs-only → 再 SSH target 跑 OS fs grow
#   5. 釋放 spec_apply_lock

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./_lib_local.sh
source "${SCRIPT_DIR}/_lib_local.sh"

TOOLS_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
CONFIG_PHP="$TOOLS_DIR/config.php"
_cfg_get() {
    local key="$1"
    sed -n "s/.*\$GLOBALS\['${key}'\][[:space:]]*=[[:space:]]*['\"]\([^'\"]*\).*/\1/p" "$CONFIG_PHP" 2>/dev/null | head -1
}
REMOTE_USER="$(_cfg_get user)"
[[ -z "$REMOTE_USER" ]] && { echo "❌ 無法從 $CONFIG_PHP 讀取 user" >&2; exit 1; }

BATCH_ID=""
TARGET=""
ACTION=""
NEW_OCPU=""
NEW_RAM_GB=""
NEW_DISK_GB=""
TARGET_IP=""
APPLIED_BY="${USER:-unknown}"
APPLY_PG=0
PG_CPU=""
PG_RAM_GB=""
# PG SSH 用 PG primary alias（大寫，如 WORLDDB1001），與 instance name (TARGET) 不同
PG_TARGET=""

while [[ $# -gt 0 ]]; do
    case "$1" in
        --batch-id)     BATCH_ID="$2"; shift 2 ;;
        --target)       TARGET="$2"; shift 2 ;;
        --action)       ACTION="$2"; shift 2 ;;
        --new-ocpu)     NEW_OCPU="$2"; shift 2 ;;
        --new-ram-gb)   NEW_RAM_GB="$2"; shift 2 ;;
        --new-disk-gb)  NEW_DISK_GB="$2"; shift 2 ;;
        --ip)           TARGET_IP="$2"; shift 2 ;;
        --applied-by)   APPLIED_BY="$2"; shift 2 ;;
        --apply-pg)     APPLY_PG="$2"; shift 2 ;;
        --pg-cpu)       PG_CPU="$2"; shift 2 ;;
        --pg-ram-gb)    PG_RAM_GB="$2"; shift 2 ;;
        --pg-target)    PG_TARGET="$2"; shift 2 ;;
        *) echo "未知參數: $1" >&2; exit 1 ;;
    esac
done

# PG hooks fallback：未指定 --pg-target 時退回 TARGET（向下相容；通常 PG SSH 會失敗）
[[ -z "$PG_TARGET" ]] && PG_TARGET="$TARGET"

[[ -z "$BATCH_ID" || -z "$TARGET" || -z "$ACTION" ]] && {
    echo "缺少必要參數：--batch-id --target --action" >&2; exit 1; }

# trap EXIT 保證任何退出路徑（正常 / set -e / set -u trip / signal）都清 spec_apply_lock。
# 既有散在 6 處的同款 DELETE 不動，trap 只是 safety net；DELETE WHERE batch_id=X 是 idempotent，多跑一次無害。
_cleanup_lock() {
    _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'" >/dev/null 2>&1 || true
}
trap _cleanup_lock EXIT

ENV_CODE=$(spec_target_to_env "$TARGET")
[[ -z "$ENV_CODE" ]] && {
    spec_audit_update "$BATCH_ID" "status='fail', error_msg='不認識的 target prefix: ${TARGET}', completed_at=NOW()"
    echo "STEP=fail|reason=unknown env from target prefix: $TARGET"
    exit 2
}

echo "=== machine_spec orchestrator ==="
echo "batch_id    : ${BATCH_ID}"
echo "target      : ${TARGET}"
echo "env_code    : ${ENV_CODE}"
echo "action      : ${ACTION}"
echo "new_ocpu    : ${NEW_OCPU}"
echo "new_ram_gb  : ${NEW_RAM_GB}"
echo "new_disk_gb : ${NEW_DISK_GB}"
echo "target_ip   : ${TARGET_IP}"
echo "applied_by  : ${APPLIED_BY}"
echo "apply_pg    : ${APPLY_PG}"
[[ "$APPLY_PG" == "1" ]] && echo "pg_spec     : cpu=${PG_CPU} ram=${PG_RAM_GB}GB pg_target=${PG_TARGET}"

# ── OS filesystem grow（SSH target 跑 inline HEREDOC）─────────────────────────
# 用於 grow-disk 成功後 / grow-fs-only。
# 回傳 0 = fs 擴展成功，非 0 = 失敗（API 已成功時不影響整體 status）。
_ssh_grow_fs() {
    local ip="$1"
    local target_disk_gb="${2:-0}"
    [[ -z "$ip" ]] && { echo "[grow_fs] 缺 target IP，跳過 fs grow"; return 1; }

    echo "###### SSH target ${ip} 擴 OS filesystem (target_disk_gb=${target_disk_gb}) ######"
    ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
        "${REMOTE_USER}@${ip}" "TARGET_DISK_GB=${target_disk_gb} bash -s" <<'GROWFS_EOF'
set -uo pipefail
TARGET_DISK_GB="${TARGET_DISK_GB:-0}"
TARGET_BYTES=0
if [ "${TARGET_DISK_GB}" -gt 0 ] 2>/dev/null; then
    # cloud volume 以十進位 GB 配置（OCI 50GB ≈ 50.01e9 bytes），用 1024^3 會高估
    # 約 7.4%，讓全滿的碟卡在 ~93% < 95% gate → 誤報 fs_grow_warn。改十進位對齊真值。
    TARGET_BYTES=$((TARGET_DISK_GB * 1000 * 1000 * 1000))
fi

ROOT_DEV=$(findmnt -no SOURCE /)            # e.g. /dev/sda1
ROOT_PKNAME=$(lsblk -no PKNAME "$ROOT_DEV") # e.g. sda
ROOT_DISK="/dev/${ROOT_PKNAME}"
PART_NUM=$(echo "$ROOT_DEV" | grep -oE '[0-9]+$')
FS_TYPE=$(findmnt -no FSTYPE /)

echo "[growfs] ROOT_DEV=$ROOT_DEV  ROOT_DISK=$ROOT_DISK  PART_NUM=$PART_NUM  FS_TYPE=$FS_TYPE"
echo "[growfs] cloud target = ${TARGET_DISK_GB} GB (${TARGET_BYTES} bytes)"
echo "[growfs] before:"; df -hT /
SIZE_BEFORE=$(df -B1 --output=size / | tail -1 | tr -d ' ')
DISK_BEFORE=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
echo "[growfs] kernel 看到 ${ROOT_DISK} = ${DISK_BEFORE} bytes（rescan 前）"

# Pre-check：fs 已對齊 cloud target（>=95%）→ 直接 no-op 成功，不走 rescan/growpart
#  避免 UI 顯示舊值 / 重複觸發時誤跑流程。沒有 TARGET_BYTES 就跳過此 gate。
if [ "${TARGET_BYTES}" -gt 0 ] && [ "${SIZE_BEFORE:-0}" -ge $((TARGET_BYTES * 95 / 100)) ]; then
    echo "[growfs] PRE-CHECK: fs (${SIZE_BEFORE} bytes) 已對齊 cloud target (${TARGET_BYTES} bytes, >=95%)，跳過所有 grow step"
    echo "[growfs] DONE (no-op, pre-check)"
    exit 0
fi

# 0. SCSI rescan with retry — Debian 12 OCI 上 partprobe 不戳 sysfs rescan node，
#    kernel 看舊 size 會讓 growpart 永遠 NOOP。手動 echo 1 強制重讀；
#    第一次寫沒長到 target 就再試一次（rescan 偶發 timing race）。
RESCAN_NODE="/sys/class/block/${ROOT_PKNAME}/device/rescan"
DISK_AFTER_RESCAN=0
if [ -e "$RESCAN_NODE" ]; then
    if echo 1 | sudo tee "$RESCAN_NODE" >/dev/null; then
        echo "[growfs] SCSI rescan try1: tee OK ($RESCAN_NODE)"
    else
        echo "[growfs] WARN: SCSI rescan try1 tee 失敗" >&2
    fi
    sleep 2
    DISK_AFTER_RESCAN=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
    echo "[growfs] kernel 看到 ${ROOT_DISK} = ${DISK_AFTER_RESCAN} bytes（rescan try1 後）"

    # 還沒到 target？再試一次
    if [ "${TARGET_BYTES}" -gt 0 ] && [ "${DISK_AFTER_RESCAN:-0}" -lt $((TARGET_BYTES * 95 / 100)) ]; then
        echo "[growfs] try1 後 disk 還小於 target，重試 rescan..."
        echo 1 | sudo tee "$RESCAN_NODE" >/dev/null \
            && echo "[growfs] SCSI rescan try2: tee OK" \
            || echo "[growfs] WARN: SCSI rescan try2 tee 失敗" >&2
        sleep 5
        DISK_AFTER_RESCAN=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
        echo "[growfs] kernel 看到 ${ROOT_DISK} = ${DISK_AFTER_RESCAN} bytes（rescan try2 後）"
    fi
else
    echo "[growfs] WARN: 沒有 $RESCAN_NODE，跳過 SCSI rescan"
    DISK_AFTER_RESCAN=$(sudo blockdev --getsize64 "$ROOT_DISK" 2>/dev/null || echo 0)
fi

# Rescan 早期 gate：有 TARGET_BYTES 而 kernel 看到的 disk 還小於 95% target → 直接 fail，
# 不要硬跑 partprobe/growpart/resize2fs 然後最後再判失敗（log 比較好看而已，沒實質意義）。
if [ "${TARGET_BYTES}" -gt 0 ] && [ "${DISK_AFTER_RESCAN:-0}" -lt $((TARGET_BYTES * 95 / 100)) ]; then
    echo "[growfs] FAIL: SCSI rescan 沒生效 — kernel disk=${DISK_AFTER_RESCAN} bytes / cloud target=${TARGET_BYTES} bytes" >&2
    echo "[growfs] 修復：在 target 機跑 'echo 1 | sudo tee ${RESCAN_NODE}' 後再重試" >&2
    exit 15
fi

# 1. partprobe — 重掃 partition table
sudo partprobe "$ROOT_DISK" 2>/dev/null || true

# 2. growpart — 擴 partition
#    rc=0 成功；rc=1 已是最大（NOOP）；rc>=2 失敗
GP_RC=0
sudo growpart "$ROOT_DISK" "$PART_NUM" || GP_RC=$?
case "$GP_RC" in
    0) echo "[growfs] growpart: OK" ;;
    1) echo "[growfs] growpart: NOOP（partition 已是最大）" ;;
    *) echo "[growfs] growpart FAILED rc=$GP_RC" >&2; exit 11 ;;
esac

# 3. 擴 filesystem
case "$FS_TYPE" in
    ext2|ext3|ext4) sudo resize2fs "$ROOT_DEV" || exit 12 ;;
    xfs)            sudo xfs_growfs / || exit 12 ;;
    *) echo "[growfs] 不支援的 fs type: $FS_TYPE" >&2; exit 13 ;;
esac

echo "[growfs] after:"; df -hT /
SIZE_AFTER=$(df -B1 --output=size / | tail -1 | tr -d ' ')

# 4. 真的長了嗎？對齊判定優先用 cloud TARGET_BYTES（真值）；TARGET_BYTES=0（caller 沒傳）
#    時 fallback 用 DISK_AFTER_RESCAN — 退化但向下相容，不會比 patch 前嚴格。
#    a) fs 真的長了，且 >= 95% target（或 TARGET_BYTES=0）→ 成功
#    b) fs 沒長，但已 >= 95% target            → no-op 成功
#    c) TARGET_BYTES=0 fallback：fs >= 95% disk → no-op 成功
#    d) 其他                                   → 真失敗
ALIGN_REF=0
if [ "${TARGET_BYTES}" -gt 0 ]; then
    ALIGN_REF="$TARGET_BYTES"
    ALIGN_NAME="cloud target"
elif [ "${DISK_AFTER_RESCAN:-0}" -gt 0 ]; then
    ALIGN_REF="$DISK_AFTER_RESCAN"
    ALIGN_NAME="kernel disk"
fi

if [ "${SIZE_AFTER:-0}" -gt "${SIZE_BEFORE:-0}" ] \
   && { [ "${ALIGN_REF}" -eq 0 ] || [ "${SIZE_AFTER:-0}" -ge $((ALIGN_REF * 95 / 100)) ]; }; then
    echo "[growfs] DONE (從 ${SIZE_BEFORE} 長到 ${SIZE_AFTER} bytes)"
elif [ "${ALIGN_REF}" -gt 0 ] && [ "${SIZE_AFTER:-0}" -ge $((ALIGN_REF * 95 / 100)) ]; then
    echo "[growfs] DONE (no-op, fs ${SIZE_AFTER} bytes 已對齊 ${ALIGN_NAME} ${ALIGN_REF} bytes)"
else
    echo "[growfs] FAIL: fs 沒有真的擴大 (before=${SIZE_BEFORE} after=${SIZE_AFTER} disk=${DISK_AFTER_RESCAN} target=${TARGET_BYTES})" >&2
    exit 14
fi
GROWFS_EOF
    return $?
}

# 把擴 fs 後的新 disk_total_mb + disk_used_mb 寫進 machine_usage_sample，
# 讓 admin_machine_spec.php 的 24h aggregate view (MAX) 立刻看到新值，
# 不必等下次 usage_collector cron。避免 grow-fs-only / grow-disk 成功後 UI 仍顯示舊值
# 而誤判「與 cloud 不一致」並要使用者再點一次。
#
# 同時刪掉「disk_total_mb 顯著小於新值的 24h 內舊樣本」— disk 擴展是 step-function
# 事件，擴後的舊 disk_total 樣本對「現在的機器」沒參考價值，會把 disk_peak_pct 卡
# 在舊 100GB 比例 ~88% 直到 24h age out。CPU/RAM 部分損失只限這台 24h 內，可接受。
_refresh_disk_usage_sample() {
    local ip="$1"
    [[ -z "$ip" ]] && return 0
    local raw dt_mb du_mb
    raw=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
        "${REMOTE_USER}@${ip}" "df -m / | awk 'NR==2 {print \$2, \$3}'" 2>/dev/null | head -1 | tr -d '\r')
    read -r dt_mb du_mb <<< "$raw"
    if [[ -n "$dt_mb" && "$dt_mb" =~ ^[0-9]+$ ]]; then
        local du_val="NULL"
        [[ -n "$du_mb" && "$du_mb" =~ ^[0-9]+$ ]] && du_val="${du_mb}"
        _spec_psql "INSERT INTO public.machine_usage_sample (env_code, target, target_ip, ssh_ok, disk_used_mb, disk_total_mb, sampled_at) VALUES (\$txt\$${ENV_CODE}\$txt\$, \$txt\$${TARGET}\$txt\$, \$txt\$${ip}\$txt\$, true, ${du_val}, ${dt_mb}, NOW())" >/dev/null 2>&1 \
            && echo "[refresh] machine_usage_sample.disk_total_mb=${dt_mb} disk_used_mb=${du_val} 已寫入（UI 下次掃描即可看到）"

        # 順手把 24h 內 disk_total 明顯較小的舊樣本清掉，避免 disk_peak_pct 卡在舊比例
        # 閾值 90%：新 disk 擴大後，舊 disk_total < new*0.9 才視為 step-function 前的舊樣本
        local stale_th=$((dt_mb * 9 / 10))
        local del_n
        del_n=$(_spec_psql "DELETE FROM public.machine_usage_sample WHERE target=\$txt\$${TARGET}\$txt\$ AND env_code=\$txt\$${ENV_CODE}\$txt\$ AND disk_total_mb IS NOT NULL AND disk_total_mb < ${stale_th} AND sampled_at >= NOW() - INTERVAL '24 hours' AND sampled_at < NOW() - INTERVAL '1 second' RETURNING 1" 2>/dev/null | wc -l)
        if [[ "$del_n" -gt 0 ]]; then
            echo "[refresh] 已清掉 ${del_n} 筆 24h 內 disk_total < ${stale_th}MB 的 step-function 前舊樣本"
        fi
    else
        echo "[refresh] WARN: 無法取得 df 結果（raw='${raw}'），UI 要等下次 usage_collector cron"
    fi
}

# ── Shape ratio pre-flight（防 OCI/GCP API 在 stop 後才 reject）─────────────
# 從 STEP=spec 行抓 shape name；spec_runner.py --action get 內部 emit step("spec", ...)
_get_target_shape() {
    local out shape
    out=$(spec_run_remote --env "$ENV_CODE" --action get --target "$TARGET" 2>&1)
    shape=$(echo "$out" | grep -oE 'shape=[^|]+' | head -1 | sed 's/^shape=//')
    echo "$shape"
}

# 驗證 ocpu/ram 比例是否合法
# 回傳：0=合法、1=比例違規（reject）、2=不認得 shape（skip 不擋）
_validate_shape_ratio() {
    local shape="$1" ocpu="$2" ram="$3"
    [[ -z "$shape" || -z "$ocpu" || -z "$ram" ]] && return 2

    local min_per max_per max_ocpu
    case "$shape" in
        VM.Standard.E5.Flex)  min_per=1; max_per=16; max_ocpu=94 ;;
        VM.Standard.E4.Flex)  min_per=1; max_per=16; max_ocpu=64 ;;
        VM.Standard.A1.Flex)  min_per=6; max_per=64; max_ocpu=80 ;;
        # GCP custom name: e.g. n2-custom-4-16384 / n2d-custom-8-32768
        n2d-custom-*|n2-custom-*|e2-custom-*) min_per=1; max_per=8; max_ocpu=224 ;;
        *) return 2 ;;
    esac

    local pass
    pass=$(awk -v ram="$ram" -v ocpu="$ocpu" -v lo="$min_per" -v hi="$max_per" -v cap="$max_ocpu" \
        'BEGIN { r=ram/ocpu; if (r>=lo && r<=hi && ocpu<=cap) print 1; else print 0 }')
    if [[ "$pass" == "1" ]]; then
        return 0
    fi
    echo "[shape] 比例違規：${shape} 需 ${min_per}~${max_per} GB/OCPU 且 OCPU<=${max_ocpu}；輸入 OCPU=${ocpu} RAM=${ram}GB"
    return 1
}

# ── PG conf 整合 helpers（仅 action=resize + --apply-pg=1 時使用）─────────────
# 路徑：pg_analyzer 子目錄
PG_DIR="$(cd "${SCRIPT_DIR}/../pg_analyzer" 2>/dev/null && pwd)"

# 寫 conf 不重啟：pgtune 推薦 → JSON → pg_apply_batch.sh --no-restart
# 失敗 = pgtune 出不來 / SSH 失敗 / sed 失敗，呼叫端要 abort chain
_pg_apply_pre_stop() {
    local target="$1" cpu="$2" ram_gb="$3"
    [[ -z "$PG_DIR" ]] && { echo "[pg] pg_analyzer dir not found"; return 11; }

    echo "###### PG conf pre-stage (no-restart) ######"
    local params_json
    params_json=$("${PG_DIR}/pg_recommend_for_spec.sh" "$target" "$cpu" "$ram_gb" 2>&1)
    local rc=$?
    if [[ $rc -ne 0 || -z "$params_json" || "$params_json" == "[]" ]]; then
        echo "[pg] recommend failed (rc=${rc}): ${params_json}"
        return 11
    fi

    local result
    result=$("${PG_DIR}/pg_apply_batch.sh" "$target" "$params_json" --no-restart --batch-id "$BATCH_ID" 2>&1)
    rc=$?
    echo "$result"
    if [[ $rc -ne 0 || "$result" != *'"status":"ok_no_restart"'* ]]; then
        echo "[pg] pg_apply_batch --no-restart 失敗 (rc=${rc})"
        return 11
    fi
    return 0
}

# 開機後 SSH SHOW 20 個 whitelist param 與 pgtune 期望值比對
# 回傳 0 = PASS（全等），非 0 = PARTIAL / ERROR（chain 不算 fail，audit 警示用）
_pg_verify_post_start() {
    local target="$1" cpu="$2" ram_gb="$3"
    PG_VERIFY_DETAIL=""  # 全域：caller 會讀；early return 也得有預設值（set -u 才不會 trip）
    [[ -z "$PG_DIR" ]] && return 12

    echo "###### PG post-start verify ######"

    # source lib 拿 PGAPPLY_SSH_OPTS（也供後續 verify_show 用）
    # shellcheck source=/dev/null
    source "${PG_DIR}/_lib_pg_apply.sh"

    # 機器剛 boot 完，OCI/GCP RUNNING 不代表 sshd 已 ready：先 SSH poll 等 ready
    local ssh_ok=0 attempts=12 i
    echo "[pg] verify: 等 SSH ready (最多 $((attempts * 5)) 秒)..."
    for ((i=1; i<=attempts; i++)); do
        if ssh "${PGAPPLY_SSH_OPTS[@]}" "$target" "true" </dev/null >/dev/null 2>&1; then
            ssh_ok=1
            echo "[pg] verify: SSH ready (attempt ${i}/${attempts})"
            break
        fi
        sleep 5
    done
    if [[ $ssh_ok -eq 0 ]]; then
        echo "[pg] verify: SSH $((attempts * 5))s 內未 ready，skip"
        return 12
    fi

    local params_json recommend_err
    recommend_err=$(mktemp -t "pg_recommend_err_${BATCH_ID}_XXXX.txt")
    params_json=$("${PG_DIR}/pg_recommend_for_spec.sh" "$target" "$cpu" "$ram_gb" 2>"$recommend_err")
    if [[ -z "$params_json" || "$params_json" == "[]" ]]; then
        echo "[pg] verify: pg_recommend_for_spec.sh 未輸出有效 JSON（recommend 空 / SSH 失敗 / pgtune 失敗），skip"
        [[ -s "$recommend_err" ]] && { echo "[pg] recommend stderr:"; sed 's/^/    /' "$recommend_err"; }
        rm -f "$recommend_err"
        return 12
    fi
    rm -f "$recommend_err"

    local expected_file
    expected_file=$(mktemp -t "pg_verify_${BATCH_ID}_XXXX.txt")
    python3 - "$params_json" "$expected_file" <<'PY'
import json, sys
data = json.loads(sys.argv[1])
with open(sys.argv[2], "w") as f:
    for p in data:
        f.write(f"{p['name']}|{p['value']}\n")
PY

    # （_lib_pg_apply.sh 已在 SSH polling 階段 source 過，pgapply_verify_show 已可用）
    local verdict rc verify_err
    verify_err=$(mktemp -t "pg_verify_err_${BATCH_ID}_XXXX.txt")
    verdict=$(pgapply_verify_show "$target" "$expected_file" 2>"$verify_err")
    rc=$?
    echo "[pg] verify_show: ${verdict} (rc=${rc})"
    if [[ -s "$verify_err" ]]; then
        echo "[pg] verify_show stderr / mismatch detail:"
        sed 's/^/    /' "$verify_err"
        PG_VERIFY_DETAIL=$(awk 'NF{printf "%s%s", sep, $0; sep="; "}' "$verify_err")
    else
        PG_VERIFY_DETAIL=""
    fi
    echo "[pg] verify expected (pgtune output) vs actual (SHOW)：請在 stderr detail 中比對 expected=/actual="
    rm -f "$expected_file" "$verify_err"
    return $rc
}

# resize chain 失敗時的 PG rollback
# 回傳：0=rollback 成功；非 0=失敗 / skip
# 依「最後成功的 STEP」決定：
#   stopped/resizing/resized（shape 未變）→ 自動 start + inline rollback
#   starting（resize 成功 + start fail，shape 已變）→ 不自動 rollback
#   其他（仍 RUNNING）→ 跑 pg_rollback.sh
_pg_rollback_after_fail() {
    local target="$1" last_step="$2"
    [[ -z "$PG_DIR" ]] && return 13

    case "$last_step" in
        stopped|resizing|resized)
            echo "[pg] chain fail @ ${last_step} — 機器 STOPPED 但 shape 未變；自動啟動回原 SPEC + rollback PG conf"

            # Step 1/3: 啟動機器（shape 仍是原 SPEC）
            echo "[pg] 步驟 1/3: 啟動機器回原 SPEC..."
            set +e
            spec_run_remote --env "$ENV_CODE" --action start --target "$TARGET" 2>&1
            local sr_rc=$?
            set -e
            if [[ $sr_rc -ne 0 ]]; then
                echo "[pg] auto-start 失敗 (rc=${sr_rc})；機器仍 STOPPED；請手動：cloud console 啟動後跑 ${PG_DIR}/pg_rollback.sh ${target}"
                return 23
            fi

            # Step 2/3: 等 SSH ready（最多 3 分鐘，每 5 秒 retry）
            # 用 PG primary alias 走 ssh config，與 pg_rollback.sh 一致
            echo "[pg] 步驟 2/3: 等 SSH ready（最多 3 分鐘）..."
            local i=0
            while [[ $i -lt 36 ]]; do
                if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
                    "$target" "exit 0" 2>/dev/null; then
                    echo "[pg] SSH ready (${i} retries)"
                    break
                fi
                sleep 5
                i=$((i+1))
            done
            if [[ $i -ge 36 ]]; then
                echo "[pg] SSH timeout (3min)；機器 RUNNING 但 SSH 不通，請手動跑 ${PG_DIR}/pg_rollback.sh ${target}"
                return 24
            fi

            # Step 3/3: inline rollback（bypass pg_rollback.sh 的 preflight，
            #          因為 PG 可能因新 conf 在原 SPEC 上 OOM 起不來，preflight 會 FAIL_PG_DOWN）
            echo "[pg] 步驟 3/3: stop PG → restore conf → start PG..."
            set +e
            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
                "$target" bash 2>&1 <<'INLINE_RB'
set +e
CONF_PATH=$(sudo -n -u postgres psql -AtX -c "SHOW config_file;" 2>/dev/null)
if [[ -z "$CONF_PATH" ]]; then
    # PG 可能起不來，SHOW config_file 失敗。退一步從 systemd unit 推 PG 版本，hardcode path
    PG_VER=$(ls -d /etc/postgresql/[0-9]* 2>/dev/null | sed 's|.*/||' | sort -n | tail -1)
    [[ -z "$PG_VER" ]] && { echo "FAIL_NO_PG_VER"; exit 21; }
    CONF_PATH="/etc/postgresql/${PG_VER}/main/postgresql.conf"
    sudo test -f "$CONF_PATH" || { echo "FAIL_NO_CONF_PATH:${CONF_PATH}"; exit 21; }
fi

BACKUP_DIR="$(dirname "$CONF_PATH")/conf_backups"
LATEST_BAK=$(sudo find "$BACKUP_DIR" -maxdepth 1 -type f -name "$(basename "$CONF_PATH").bak.[0-9]*" 2>/dev/null | sort -r | head -1)
[[ -z "$LATEST_BAK" ]] && { echo "FAIL_NO_BACKUP:${BACKUP_DIR}"; exit 22; }

# Stop PG (可能已 crash; no-op 不報錯)
sudo systemctl stop postgresql 2>/dev/null

# Restore conf
if ! sudo cp -p "$LATEST_BAK" "$CONF_PATH"; then
    echo "FAIL_RESTORE_CP"
    exit 23
fi

# Start PG with old conf
if ! sudo systemctl start postgresql 2>&1; then
    echo "FAIL_PG_START"
    exit 24
fi

# Verify (sleep 讓 PG 完全 ready)
sleep 3
SB=$(sudo -n -u postgres psql -AtX -c "SHOW shared_buffers" 2>/dev/null)
[[ -z "$SB" ]] && { echo "FAIL_PG_VERIFY"; exit 25; }
echo "OK_ROLLED_BACK shared_buffers=${SB} restored_from=$(basename "$LATEST_BAK")"
INLINE_RB
            local rb_rc=$?
            set -e
            if [[ $rb_rc -eq 0 ]]; then
                echo "[pg] rollback 完成"
                return 0
            fi
            echo "[pg] inline rollback 失敗 (rc=${rb_rc})"
            return 25
            ;;
        starting)
            echo "[pg] chain fail @ starting — resize API 已成功（shape 已變）但 start 失敗；不自動 rollback"
            echo "[pg] 請從 cloud console 確認狀態，必要時手動跑 ${PG_DIR}/pg_rollback.sh ${target}"
            return 22
            ;;
        *)
            echo "[pg] chain fail @ ${last_step} — 假設機器仍 RUNNING，SSH 嘗試 rollback PG conf"
            "${PG_DIR}/pg_rollback.sh" "$target" 2>&1
            return $?
            ;;
    esac
}

# ── 分支：grow-fs-only 跳過 cloud API，只做 OS fs grow ─────────────────────────
if [[ "$ACTION" == "grow-fs-only" ]]; then
    spec_audit_update "$BATCH_ID" "status='running', progress_step='fs_grow_only'"

    set +e
    _ssh_grow_fs "$TARGET_IP" "${NEW_DISK_GB:-0}"
    GROWFS_EXIT=$?
    set -e

    if [[ "$GROWFS_EXIT" -eq 0 ]]; then
        spec_audit_update "$BATCH_ID" "status='success', progress_step='fs_grown', completed_at=NOW()"
        echo "STEP=fs_grown|target=${TARGET}"
        _refresh_disk_usage_sample "$TARGET_IP"
        _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
        exit 0
    else
        spec_audit_update "$BATCH_ID" \
            "status='fail', progress_step='fs_grow_fail', error_msg=\$txt\$grow-fs-only failed (rc=${GROWFS_EXIT})\$txt\$, completed_at=NOW()"
        echo "STEP=fs_grow_fail|target=${TARGET}|rc=${GROWFS_EXIT}"
        _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
        exit "$GROWFS_EXIT"
    fi
fi

# ── Pre-flight：shape ratio 合法性檢查（只對 resize 有意義）────────────────────
# 在 stop / sed conf 之前先驗，避免 OCI/GCP API 在 stop 後才 reject 留下不一致狀態
if [[ "$ACTION" == "resize" && -n "$NEW_OCPU" && -n "$NEW_RAM_GB" ]]; then
    SHAPE_NAME=$(_get_target_shape)
    if [[ -n "$SHAPE_NAME" ]]; then
        echo "###### pre-flight: shape ratio check (${SHAPE_NAME}) ######"
        set +e
        _validate_shape_ratio "$SHAPE_NAME" "$NEW_OCPU" "$NEW_RAM_GB"
        SHAPE_RC=$?
        set -e
        if [[ $SHAPE_RC -eq 1 ]]; then
            spec_audit_update "$BATCH_ID" \
                "status='fail', progress_step='shape_invalid', error_msg=\$txt\$Shape 比例違規（${SHAPE_NAME}, OCPU=${NEW_OCPU} RAM=${NEW_RAM_GB}GB）；機器與 PG conf 都未動\$txt\$, completed_at=NOW()"
            echo "STEP=shape_invalid|target=${TARGET}|shape=${SHAPE_NAME}|ocpu=${NEW_OCPU}|ram=${NEW_RAM_GB}"
            _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
            exit 30
        fi
        [[ $SHAPE_RC -eq 2 ]] && echo "[shape] 不認得 shape family，skip ratio check"
    else
        echo "[shape] 無法取得 shape name，skip ratio check"
    fi
fi

# ── 既有流程：呼叫 spec_runner.py（SSH center）─────────────────────────────────
echo "ssh: ${CENTER_USER}@${CENTER_HOST} ${SPEC_RUNNER}"
echo

# 組 spec_runner.py 參數
RUNNER_ARGS=(
    --env "$ENV_CODE"
    --action "$ACTION"
    --target "$TARGET"
)
[[ -n "$NEW_OCPU"    ]] && RUNNER_ARGS+=( --new-ocpu "$NEW_OCPU" )
[[ -n "$NEW_RAM_GB"  ]] && RUNNER_ARGS+=( --new-ram-gb "$NEW_RAM_GB" )
[[ -n "$NEW_DISK_GB" ]] && RUNNER_ARGS+=( --new-disk-gb "$NEW_DISK_GB" )

# 本地 log（給後端 progress 解析）
LOG_FILE=$(mktemp -t spec_orch_${BATCH_ID}_XXXX.log)

# 啟動執行（log 雙寫：stdout 給 PHP poll_job，檔案給後續 parse）
spec_audit_update "$BATCH_ID" "status='running', progress_step='start'"

# ── PG pre-stage：先寫新 conf（PG 不重啟，新值等下次 start 才生效）─────────────
if [[ "$APPLY_PG" == "1" && "$ACTION" == "resize" && -n "$PG_CPU" && -n "$PG_RAM_GB" ]]; then
    spec_audit_update "$BATCH_ID" "progress_step='pg_pre_stage'"
    set +e
    _pg_apply_pre_stop "$PG_TARGET" "$PG_CPU" "$PG_RAM_GB"
    PG_PRE_RC=$?
    set -e
    if [[ $PG_PRE_RC -ne 0 ]]; then
        spec_audit_update "$BATCH_ID" \
            "status='fail', progress_step='pg_write_fail', error_msg=\$txt\$PG conf pre-stage failed (rc=${PG_PRE_RC})；resize 未執行，機器未動\$txt\$, completed_at=NOW()"
        echo "STEP=pg_write_fail|target=${TARGET}|rc=${PG_PRE_RC}"
        _spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"
        exit 11
    fi
    echo "STEP=pg_staged|target=${TARGET}"
    spec_audit_update "$BATCH_ID" "progress_step='pg_staged'"
fi

set +e
spec_run_remote "${RUNNER_ARGS[@]}" 2>&1 | tee "$LOG_FILE"
RUNNER_EXIT=${PIPESTATUS[0]}
set -e

echo
echo "=== runner exit: ${RUNNER_EXIT} ==="

# 解析最後一個「非 STEP=fail」的 STEP= 行作為 chain 進度
# spec_runner.py 失敗時最後一行是 STEP=fail|reason=...；rollback 判斷要看 fail 之前的進度
LAST_STEP=$(grep -E '^STEP=' "$LOG_FILE" 2>/dev/null | grep -v '^STEP=fail' | tail -1 | sed -E 's/^STEP=([^|]+).*/\1/')

if [[ "$RUNNER_EXIT" -eq 0 ]]; then
    # ── grow-disk 成功 → 追加 SSH target 擴 OS fs ───────────────────────────
    # API resize 已成功，所以 status 一律 'success'；fs 失敗只降為 progress_step='fs_grow_warn' 警示
    if [[ "$ACTION" == "grow-disk" ]]; then
        # 等 cloud volume hydration 一下，避免 partprobe rescan 還看不到新 size
        sleep 5

        set +e
        _ssh_grow_fs "$TARGET_IP" "${NEW_DISK_GB:-0}"
        GROWFS_EXIT=$?
        set -e

        if [[ "$GROWFS_EXIT" -eq 0 ]]; then
            spec_audit_update "$BATCH_ID" \
                "status='success', progress_step='fs_grown', completed_at=NOW()"
            echo "STEP=fs_grown|target=${TARGET}"
            _refresh_disk_usage_sample "$TARGET_IP"
        else
            spec_audit_update "$BATCH_ID" \
                "status='success', progress_step='fs_grow_warn', error_msg=\$txt\$cloud disk 擴展成功，但 OS fs 擴展失敗 (rc=${GROWFS_EXIT})；請手動執行 grow-fs-only 補做\$txt\$, completed_at=NOW()"
            echo "STEP=fs_grow_warn|target=${TARGET}|rc=${GROWFS_EXIT}"
        fi
    else
        # action=resize + apply-pg=1：開機後驗 PG 20 個 param
        if [[ "$APPLY_PG" == "1" && "$ACTION" == "resize" ]]; then
            set +e
            _pg_verify_post_start "$PG_TARGET" "$PG_CPU" "$PG_RAM_GB"
            PG_VRC=$?
            set -e
            if [[ $PG_VRC -eq 0 ]]; then
                echo "STEP=pg_verified|target=${TARGET}"
                spec_audit_update "$BATCH_ID" "status='success', progress_step='pg_verified', completed_at=NOW()"
            else
                # verify 失敗只警示，不視為 chain fail（resize 已成功）
                pg_msg_detail="${PG_VERIFY_DETAIL:0:500}"
                [[ -n "$pg_msg_detail" ]] && pg_msg_detail=" | detail: ${pg_msg_detail}"
                spec_audit_update "$BATCH_ID" \
                    "status='success', progress_step='pg_verify_partial', error_msg=\$txt\$resize 成功；PG conf verify 部分 mismatch (rc=${PG_VRC})${pg_msg_detail}；請至 admin_pg_analyzer 補做\$txt\$, completed_at=NOW()"
                echo "STEP=pg_verify_partial|target=${TARGET}|rc=${PG_VRC}"
            fi
        else
            spec_audit_update "$BATCH_ID" "status='success', progress_step='${LAST_STEP:-done}', completed_at=NOW()"
        fi
    fi
else
    # 取最後一個 fail 的 reason
    FAIL_REASON=$(grep -E '^STEP=fail' "$LOG_FILE" 2>/dev/null | tail -1 | sed -E 's/.*reason=([^|]*).*/\1/')
    [[ -z "$FAIL_REASON" ]] && FAIL_REASON="exit ${RUNNER_EXIT}"

    # action=resize + apply-pg=1：依 chain 失敗階段做 PG rollback
    ROLLBACK_STEP="${LAST_STEP:-fail}"
    if [[ "$APPLY_PG" == "1" && "$ACTION" == "resize" ]]; then
        set +e
        _pg_rollback_after_fail "$PG_TARGET" "$LAST_STEP"
        PG_RB_RC=$?
        set -e
        FAIL_REASON="${FAIL_REASON} | pg_rollback rc=${PG_RB_RC}"
        if [[ $PG_RB_RC -eq 0 ]]; then
            ROLLBACK_STEP="pg_rolled_back"
            echo "STEP=pg_rolled_back|target=${TARGET}"
        elif [[ $PG_RB_RC -ge 23 && $PG_RB_RC -le 25 ]]; then
            # 23/24/25 = 自動 rollback 嘗試失敗（auto-start fail / SSH timeout / inline rollback fail）
            ROLLBACK_STEP="pg_rollback_fail"
            echo "STEP=pg_rollback_fail|target=${TARGET}|rc=${PG_RB_RC}"
        fi
        # rc=22 (starting / 不自動 rollback) / rc=13 (no PG_DIR) → 保留 LAST_STEP
    fi

    # 用 dollar-quote 避免 ' 跳脫地獄
    spec_audit_update "$BATCH_ID" \
        "status='fail', progress_step='${ROLLBACK_STEP}', error_msg=\$txt\$${FAIL_REASON}\$txt\$, completed_at=NOW()"
fi

# 釋放 lock
_spec_psql "DELETE FROM public.spec_apply_lock WHERE batch_id = '${BATCH_ID}'"

# 清掉本地 log
rm -f "$LOG_FILE"

exit "$RUNNER_EXIT"
