#!/usr/bin/env bash
# ============================================================
# Novalon CI/CD 健康检查
#
# 探针维度：
#   1. 容器存活与健康状态
#   2. HTTPS 端点可达性（Gitea / Jenkins / 官网）
#   3. TLS 证书剩余有效期
#   4. 磁盘与 Docker 卷增长（宿主 20G 已用 85%，这是头号风险）
#   5. 备份新鲜度（超过 36 小时无新备份 -> FAIL）
#   6. Jenkins 构建队列积压 / 执行器卡死
#
# 退出码：0 全通过；1 有 FAIL。便于 cron + Webhook 告警。
#
# crontab（每 10 分钟）:
#   */10 * * * * /home/novalon/docker-app/infra/cicd/monitoring/healthcheck-cicd.sh
#       >> /var/log/cicd-healthcheck.log 2>&1
#
# 用法:
#   ./healthcheck-cicd.sh             # 正常检查，FAIL 时推送告警
#   ./healthcheck-cicd.sh --quiet     # 只在 FAIL 时输出
#   ./healthcheck-cicd.sh --json      # 输出 JSON（供 Prometheus/自定义采集）
# ============================================================

set -uo pipefail   # 不用 -e，健康检查要跑完全部探针再汇总

BACKUP_ROOT="${BACKUP_ROOT:-/home/novalon/backups/cicd}"
ALERT_WEBHOOK_URL="${ALERT_WEBHOOK_URL:-}"
DISK_WARN_PCT="${DISK_WARN_PCT:-80}"
DISK_CRIT_PCT="${DISK_CRIT_PCT:-90}"
CERT_WARN_DAYS="${CERT_WARN_DAYS:-21}"
BACKUP_MAX_AGE_HOURS="${BACKUP_MAX_AGE_HOURS:-36}"

GITEA_URL="${GITEA_URL:-https://git.f.novalon.cn}"
JENKINS_URL="${JENKINS_URL:-https://ci.f.novalon.cn}"
SITE_URL="${SITE_URL:-https://novalon.cn}"

QUIET=0; JSON=0
for a in "$@"; do
  case "$a" in
    --quiet) QUIET=1 ;;
    --json)  JSON=1 ;;
  esac
done

PASS=0; FAIL=0; WARN=0
RESULTS=()

# ⚠️ 这三个函数必须显式 return 0：
#    曾踩坑 —— --quiet/--json 模式下 `[[ 条件 ]] && echo` 因条件为假而返回 1，
#    导致 `cond && ok ... || bad ...` 构造中 ok 已执行却"返回失败"，跌进 bad 分支，
#    造成 FAIL 计数随输出模式抖动（1/1/2）
ok()   { PASS=$((PASS+1)); RESULTS+=("PASS|$1|$2"); if [[ $QUIET == 0 && $JSON == 0 ]]; then echo "  ✅ $1 — $2"; fi; return 0; }
warn() { WARN=$((WARN+1)); RESULTS+=("WARN|$1|$2"); if [[ $JSON == 0 ]]; then echo "  ⚠️  $1 — $2"; fi; return 0; }
bad()  { FAIL=$((FAIL+1)); RESULTS+=("FAIL|$1|$2"); if [[ $JSON == 0 ]]; then echo "  ❌ $1 — $2"; fi; return 0; }

emit_json() {
  printf '{"ts":"%s","host":"%s","pass":%d,"warn":%d,"fail":%d,"checks":[' \
    "$(date -Is)" "$(hostname)" "$PASS" "$WARN" "$FAIL"
  local first=1
  for r in "${RESULTS[@]}"; do
    IFS='|' read -r lvl name msg <<< "$r"
    (( first )) || printf ','
    first=0
    printf '{"level":"%s","name":"%s","msg":"%s"}' "$lvl" "$name" "${msg//\"/\\\"}"
  done
  printf ']}\n'
}

alert() {
  [[ -z "$ALERT_WEBHOOK_URL" ]] && return 0
  local body="[CI/CD 健康检查] 主机 $(hostname)
$(printf '%s\n' "${RESULTS[@]}" | grep '^FAIL' | sed 's/^FAIL|/  ❌ /' | sed 's/|/ — /')
$(printf '%s\n' "${RESULTS[@]}" | grep '^WARN' | sed 's/^WARN|/  ⚠️  /' | sed 's/|/ — /')"
  curl -fsS -m 10 -X POST "$ALERT_WEBHOOK_URL" \
    -H 'Content-Type: application/json' \
    --data-binary "$(printf '{"msgtype":"text","text":{"content":%s}}' \
        "$(printf '%s' "$body" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"见日志\"')")" \
    >/dev/null 2>&1 || true
}

[[ $JSON == 0 ]] && echo "========== CI/CD 健康检查 $(date -Is) =========="

# ---------- 1. 容器状态 ----------
[[ $JSON == 0 ]] && echo "[1] 容器状态"
for c in gitea jenkins postgresql nginx; do
  name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -x "$c" || true)
  if [[ -z "$name" ]]; then
    # nginx 容器名带前缀，做模糊匹配
    name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -i "$c" | head -1 || true)
  fi
  if [[ -z "$name" ]]; then
    bad "container:$c" "容器未运行"
    continue
  fi
  # ⚠️ 未定义 healthcheck 的容器：inspect 返回空串（命令成功，|| 不会触发），
  #    必须显式把空串归一为 none，否则会误报"状态异常"
  st="$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)"
  st="${st:-none}"
  if [[ "$st" == "healthy" || "$st" == "none" ]]; then
    ok "container:$c" "运行中 ($st)"
  else
    bad "container:$c" "状态异常: $st"
  fi
done

# ---------- 2. HTTPS 端点 ----------
[[ $JSON == 0 ]] && echo "[2] HTTPS 端点"
check_url() {
  local label="$1" url="$2" expect="$3"
  # ⚠️ 不能用 `|| echo 000`：curl 偶发"已输出 http_code 但退出码非 0"，
  #    会把 200 与 000 拼接成 200000，造成假 FAIL。改用正则校验兜底。
  local code
  code="$(curl -sS -o /dev/null -w '%{http_code}' -m 12 --noproxy '*' "$url" 2>/dev/null || true)"
  [[ "$code" =~ ^[0-9]{3}$ ]] || code="000"
  if [[ "$code" =~ ^($expect)$ ]]; then ok "http:$label" "$code ($url)"
  else bad "http:$label" "期望 $expect 实得 $code ($url)"; fi
}
check_url gitea   "$GITEA_URL/"   "200|301|302|303"
check_url jenkins "$JENKINS_URL/" "200|403"
check_url site    "$SITE_URL/"    "200|301|302"

# ---------- 3. TLS 证书 ----------
[[ $JSON == 0 ]] && echo "[3] TLS 证书"
check_cert() {
  local host="$1"
  local end; end=$(echo | timeout 12 openssl s_client -servername "$host" -connect "$host:443" 2>/dev/null \
    | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
  if [[ -z "$end" ]]; then warn "cert:$host" "无法获取证书信息"; return; fi
  local end_epoch now days
  end_epoch=$(date -d "$end" +%s 2>/dev/null || date -j -f "%b %e %H:%M:%S %Y %Z" "$end" +%s 2>/dev/null || echo 0)
  now=$(date +%s)
  days=$(( (end_epoch - now) / 86400 ))
  if (( days <= 0 ));        then bad  "cert:$host" "证书已过期 ($days 天)"
  elif (( days < 7 ));       then bad  "cert:$host" "证书 $days 天后过期，立即续期"
  elif (( days < CERT_WARN_DAYS )); then warn "cert:$host" "证书 $days 天后过期"
  else ok "cert:$host" "剩余 $days 天"; fi
}
check_cert "${GITEA_URL#https://}"
check_cert "${JENKINS_URL#https://}"

# ---------- 4. 磁盘 ----------
[[ $JSON == 0 ]] && echo "[4] 磁盘与卷"
root_pct=$(df -P / | awk 'NR==2{gsub(/%/,"");print $5}')
root_free=$(df -h / | awk 'NR==2{print $4}')
if   (( root_pct >= DISK_CRIT_PCT )); then bad  "disk:root" "已用 ${root_pct}%（阈值 ${DISK_CRIT_PCT}%），可用 ${root_free}"
elif (( root_pct >= DISK_WARN_PCT )); then warn "disk:root" "已用 ${root_pct}%（阈值 ${DISK_WARN_PCT}%），可用 ${root_free}"
else ok "disk:root" "已用 ${root_pct}%，可用 ${root_free}"; fi

for v in jenkins_jenkins_home; do
  if docker volume inspect "$v" >/dev/null 2>&1; then
    sz=$(docker system df -v 2>/dev/null | grep -F "$v" | awk '{print $3}' | head -1)
    if [[ -n "$sz" ]]; then
      num=${sz%GB}
      if (( $(echo "$num > 5" | bc -l 2>/dev/null || echo 0) )); then
        warn "volume:$v" "体积 ${sz} 超过 5GB，检查构建历史清理策略"
      else ok "volume:$v" "体积 ${sz}"; fi
    fi
  fi
done

# 备份区体积
if [[ -d "$BACKUP_ROOT" ]]; then
  bsz=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1)
  ok "backup:size" "备份区 ${bsz}"
fi

# ---------- 5. 备份新鲜度 ----------
[[ $JSON == 0 ]] && echo "[5] 备份新鲜度"
if [[ ! -d "$BACKUP_ROOT" ]]; then
  bad "backup:freshness" "备份目录不存在: $BACKUP_ROOT（Jenkins 从未备份过）"
else
  latest=$(ls -1dt "$BACKUP_ROOT"/[0-9]*_[0-9]* 2>/dev/null | head -1 || true)
  if [[ -z "$latest" ]]; then
    bad "backup:freshness" "备份目录存在但无任何备份"
  else
    age_h=$(( ( $(date +%s) - $(stat -c %Y "$latest") ) / 3600 ))
    if (( age_h > BACKUP_MAX_AGE_HOURS )); then
      bad "backup:freshness" "最新备份 $(basename "$latest") 已 ${age_h}h（阈值 ${BACKUP_MAX_AGE_HOURS}h）"
    else
      ok "backup:freshness" "最新备份 $(basename "$latest")，${age_h}h 前"
    fi
    # 内容完整性
    cnt=$(find "$latest" -name '*.tar.gz' -o -name '*.sql.gz' 2>/dev/null | wc -l | tr -d ' ')
    # 注意：不要写成 `(( cnt >= 2 )) && ok || bad` —— 若 ok 返回非 0 会跌进 bad
    if (( cnt >= 2 )); then ok "backup:content" "含 $cnt 个产物"; else bad "backup:content" "产物仅 $cnt 个，备份不完整"; fi
  fi
fi

# ---------- 6. Jenkins 队列 ----------
[[ $JSON == 0 ]] && echo "[6] Jenkins 队列"
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx jenkins; then
  q=$(docker exec jenkins sh -c \
      'curl -fsS --noproxy "*" http://localhost:8080/queue/api/json?tree=items\[id\] 2>/dev/null | grep -o "\"id\"" | wc -l' 2>/dev/null || echo "-1")
  if [[ "$q" == "-1" ]]; then warn "jenkins:queue" "无法读取队列（需认证，属正常）"
  elif (( q > 10 )); then bad "jenkins:queue" "队列积压 $q 个任务，检查执行器是否卡死"
  else ok "jenkins:queue" "队列 $q"; fi
else
  bad "jenkins:queue" "Jenkins 容器未运行"
fi

# ---------- 汇总 ----------
if (( JSON == 1 )); then emit_json
else
  echo "----------------------------------------"
  echo "汇总: ✅ PASS=$PASS  ⚠️  WARN=$WARN  ❌ FAIL=$FAIL"
fi

if (( FAIL > 0 )); then alert; fi
exit $(( FAIL > 0 ? 1 : 0 ))