zhangxiang
4ab2f3cd8e
- infra/cicd：docker-compose（gitea/jenkins）、JCasC（凭据统一 ${ENV} 注入，无硬编码）、
备份/恢复、健康检查、凭据轮换与 git 历史清除脚本
- docs/deployment/cicd：安装、高可用备份监控、凭据事故复盘
- .env.example 仅为占位模板；真实 .env 由 .gitignore 排除
211 lines
8.9 KiB
Bash
Executable File
211 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================
|
||
# Novalon CI/CD 健康检查
|
||
#
|
||
# 探针维度：
|
||
# 1. 容器存活与健康状态
|
||
# 2. HTTPS 端点可达性（Gitea / Jenkins / 官网）
|
||
# 3. TLS 证书剩余有效期
|
||
# 4. 磁盘与 Docker 卷增长（宿主 20G 已用 85%，这是头号风险）
|
||
# 5. 备份新鲜度（超过 36 小时无新备份 -> FAIL）
|
||
# 6. Jenkins 构建队列积压 / 执行器卡死
|
||
#
|
||
# 退出码：0 全通过；1 有 FAIL。便于 cron + Webhook 告警。
|
||
#
|
||
# crontab（每 10 分钟）:
|
||
# */10 * * * * /home/novalon/docker-app/infra/cicd/monitoring/healthcheck-cicd.sh
|
||
# >> /var/log/cicd-healthcheck.log 2>&1
|
||
#
|
||
# 用法:
|
||
# ./healthcheck-cicd.sh # 正常检查，FAIL 时推送告警
|
||
# ./healthcheck-cicd.sh --quiet # 只在 FAIL 时输出
|
||
# ./healthcheck-cicd.sh --json # 输出 JSON（供 Prometheus/自定义采集）
|
||
# ============================================================
|
||
|
||
set -uo pipefail # 不用 -e，健康检查要跑完全部探针再汇总
|
||
|
||
BACKUP_ROOT="${BACKUP_ROOT:-/home/novalon/backups/cicd}"
|
||
ALERT_WEBHOOK_URL="${ALERT_WEBHOOK_URL:-}"
|
||
DISK_WARN_PCT="${DISK_WARN_PCT:-80}"
|
||
DISK_CRIT_PCT="${DISK_CRIT_PCT:-90}"
|
||
CERT_WARN_DAYS="${CERT_WARN_DAYS:-21}"
|
||
BACKUP_MAX_AGE_HOURS="${BACKUP_MAX_AGE_HOURS:-36}"
|
||
|
||
GITEA_URL="${GITEA_URL:-https://git.f.novalon.cn}"
|
||
JENKINS_URL="${JENKINS_URL:-https://ci.f.novalon.cn}"
|
||
SITE_URL="${SITE_URL:-https://novalon.cn}"
|
||
|
||
QUIET=0; JSON=0
|
||
for a in "$@"; do
|
||
case "$a" in
|
||
--quiet) QUIET=1 ;;
|
||
--json) JSON=1 ;;
|
||
esac
|
||
done
|
||
|
||
PASS=0; FAIL=0; WARN=0
|
||
RESULTS=()
|
||
|
||
# ⚠️ 这三个函数必须显式 return 0：
|
||
# 曾踩坑 —— --quiet/--json 模式下 `[[ 条件 ]] && echo` 因条件为假而返回 1，
|
||
# 导致 `cond && ok ... || bad ...` 构造中 ok 已执行却"返回失败"，跌进 bad 分支，
|
||
# 造成 FAIL 计数随输出模式抖动（1/1/2）
|
||
ok() { PASS=$((PASS+1)); RESULTS+=("PASS|$1|$2"); if [[ $QUIET == 0 && $JSON == 0 ]]; then echo " ✅ $1 — $2"; fi; return 0; }
|
||
warn() { WARN=$((WARN+1)); RESULTS+=("WARN|$1|$2"); if [[ $JSON == 0 ]]; then echo " ⚠️ $1 — $2"; fi; return 0; }
|
||
bad() { FAIL=$((FAIL+1)); RESULTS+=("FAIL|$1|$2"); if [[ $JSON == 0 ]]; then echo " ❌ $1 — $2"; fi; return 0; }
|
||
|
||
emit_json() {
|
||
printf '{"ts":"%s","host":"%s","pass":%d,"warn":%d,"fail":%d,"checks":[' \
|
||
"$(date -Is)" "$(hostname)" "$PASS" "$WARN" "$FAIL"
|
||
local first=1
|
||
for r in "${RESULTS[@]}"; do
|
||
IFS='|' read -r lvl name msg <<< "$r"
|
||
(( first )) || printf ','
|
||
first=0
|
||
printf '{"level":"%s","name":"%s","msg":"%s"}' "$lvl" "$name" "${msg//\"/\\\"}"
|
||
done
|
||
printf ']}\n'
|
||
}
|
||
|
||
alert() {
|
||
[[ -z "$ALERT_WEBHOOK_URL" ]] && return 0
|
||
local body="[CI/CD 健康检查] 主机 $(hostname)
|
||
$(printf '%s\n' "${RESULTS[@]}" | grep '^FAIL' | sed 's/^FAIL|/ ❌ /' | sed 's/|/ — /')
|
||
$(printf '%s\n' "${RESULTS[@]}" | grep '^WARN' | sed 's/^WARN|/ ⚠️ /' | sed 's/|/ — /')"
|
||
curl -fsS -m 10 -X POST "$ALERT_WEBHOOK_URL" \
|
||
-H 'Content-Type: application/json' \
|
||
--data-binary "$(printf '{"msgtype":"text","text":{"content":%s}}' \
|
||
"$(printf '%s' "$body" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"见日志\"')")" \
|
||
>/dev/null 2>&1 || true
|
||
}
|
||
|
||
[[ $JSON == 0 ]] && echo "========== CI/CD 健康检查 $(date -Is) =========="
|
||
|
||
# ---------- 1. 容器状态 ----------
|
||
[[ $JSON == 0 ]] && echo "[1] 容器状态"
|
||
for c in gitea jenkins postgresql nginx; do
|
||
name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -x "$c" || true)
|
||
if [[ -z "$name" ]]; then
|
||
# nginx 容器名带前缀，做模糊匹配
|
||
name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -i "$c" | head -1 || true)
|
||
fi
|
||
if [[ -z "$name" ]]; then
|
||
bad "container:$c" "容器未运行"
|
||
continue
|
||
fi
|
||
# ⚠️ 未定义 healthcheck 的容器：inspect 返回空串（命令成功，|| 不会触发），
|
||
# 必须显式把空串归一为 none，否则会误报"状态异常"
|
||
st="$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)"
|
||
st="${st:-none}"
|
||
if [[ "$st" == "healthy" || "$st" == "none" ]]; then
|
||
ok "container:$c" "运行中 ($st)"
|
||
else
|
||
bad "container:$c" "状态异常: $st"
|
||
fi
|
||
done
|
||
|
||
# ---------- 2. HTTPS 端点 ----------
|
||
[[ $JSON == 0 ]] && echo "[2] HTTPS 端点"
|
||
check_url() {
|
||
local label="$1" url="$2" expect="$3"
|
||
# ⚠️ 不能用 `|| echo 000`：curl 偶发"已输出 http_code 但退出码非 0"，
|
||
# 会把 200 与 000 拼接成 200000，造成假 FAIL。改用正则校验兜底。
|
||
local code
|
||
code="$(curl -sS -o /dev/null -w '%{http_code}' -m 12 --noproxy '*' "$url" 2>/dev/null || true)"
|
||
[[ "$code" =~ ^[0-9]{3}$ ]] || code="000"
|
||
if [[ "$code" =~ ^($expect)$ ]]; then ok "http:$label" "$code ($url)"
|
||
else bad "http:$label" "期望 $expect 实得 $code ($url)"; fi
|
||
}
|
||
check_url gitea "$GITEA_URL/" "200|301|302|303"
|
||
check_url jenkins "$JENKINS_URL/" "200|403"
|
||
check_url site "$SITE_URL/" "200|301|302"
|
||
|
||
# ---------- 3. TLS 证书 ----------
|
||
[[ $JSON == 0 ]] && echo "[3] TLS 证书"
|
||
check_cert() {
|
||
local host="$1"
|
||
local end; end=$(echo | timeout 12 openssl s_client -servername "$host" -connect "$host:443" 2>/dev/null \
|
||
| openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [[ -z "$end" ]]; then warn "cert:$host" "无法获取证书信息"; return; fi
|
||
local end_epoch now days
|
||
end_epoch=$(date -d "$end" +%s 2>/dev/null || date -j -f "%b %e %H:%M:%S %Y %Z" "$end" +%s 2>/dev/null || echo 0)
|
||
now=$(date +%s)
|
||
days=$(( (end_epoch - now) / 86400 ))
|
||
if (( days <= 0 )); then bad "cert:$host" "证书已过期 ($days 天)"
|
||
elif (( days < 7 )); then bad "cert:$host" "证书 $days 天后过期，立即续期"
|
||
elif (( days < CERT_WARN_DAYS )); then warn "cert:$host" "证书 $days 天后过期"
|
||
else ok "cert:$host" "剩余 $days 天"; fi
|
||
}
|
||
check_cert "${GITEA_URL#https://}"
|
||
check_cert "${JENKINS_URL#https://}"
|
||
|
||
# ---------- 4. 磁盘 ----------
|
||
[[ $JSON == 0 ]] && echo "[4] 磁盘与卷"
|
||
root_pct=$(df -P / | awk 'NR==2{gsub(/%/,"");print $5}')
|
||
root_free=$(df -h / | awk 'NR==2{print $4}')
|
||
if (( root_pct >= DISK_CRIT_PCT )); then bad "disk:root" "已用 ${root_pct}%（阈值 ${DISK_CRIT_PCT}%），可用 ${root_free}"
|
||
elif (( root_pct >= DISK_WARN_PCT )); then warn "disk:root" "已用 ${root_pct}%（阈值 ${DISK_WARN_PCT}%），可用 ${root_free}"
|
||
else ok "disk:root" "已用 ${root_pct}%，可用 ${root_free}"; fi
|
||
|
||
for v in jenkins_jenkins_home; do
|
||
if docker volume inspect "$v" >/dev/null 2>&1; then
|
||
sz=$(docker system df -v 2>/dev/null | grep -F "$v" | awk '{print $3}' | head -1)
|
||
if [[ -n "$sz" ]]; then
|
||
num=${sz%GB}
|
||
if (( $(echo "$num > 5" | bc -l 2>/dev/null || echo 0) )); then
|
||
warn "volume:$v" "体积 ${sz} 超过 5GB，检查构建历史清理策略"
|
||
else ok "volume:$v" "体积 ${sz}"; fi
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# 备份区体积
|
||
if [[ -d "$BACKUP_ROOT" ]]; then
|
||
bsz=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1)
|
||
ok "backup:size" "备份区 ${bsz}"
|
||
fi
|
||
|
||
# ---------- 5. 备份新鲜度 ----------
|
||
[[ $JSON == 0 ]] && echo "[5] 备份新鲜度"
|
||
if [[ ! -d "$BACKUP_ROOT" ]]; then
|
||
bad "backup:freshness" "备份目录不存在: $BACKUP_ROOT（Jenkins 从未备份过）"
|
||
else
|
||
latest=$(ls -1dt "$BACKUP_ROOT"/[0-9]*_[0-9]* 2>/dev/null | head -1 || true)
|
||
if [[ -z "$latest" ]]; then
|
||
bad "backup:freshness" "备份目录存在但无任何备份"
|
||
else
|
||
age_h=$(( ( $(date +%s) - $(stat -c %Y "$latest") ) / 3600 ))
|
||
if (( age_h > BACKUP_MAX_AGE_HOURS )); then
|
||
bad "backup:freshness" "最新备份 $(basename "$latest") 已 ${age_h}h（阈值 ${BACKUP_MAX_AGE_HOURS}h）"
|
||
else
|
||
ok "backup:freshness" "最新备份 $(basename "$latest")，${age_h}h 前"
|
||
fi
|
||
# 内容完整性
|
||
cnt=$(find "$latest" -name '*.tar.gz' -o -name '*.sql.gz' 2>/dev/null | wc -l | tr -d ' ')
|
||
# 注意：不要写成 `(( cnt >= 2 )) && ok || bad` —— 若 ok 返回非 0 会跌进 bad
|
||
if (( cnt >= 2 )); then ok "backup:content" "含 $cnt 个产物"; else bad "backup:content" "产物仅 $cnt 个，备份不完整"; fi
|
||
fi
|
||
fi
|
||
|
||
# ---------- 6. Jenkins 队列 ----------
|
||
[[ $JSON == 0 ]] && echo "[6] Jenkins 队列"
|
||
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx jenkins; then
|
||
q=$(docker exec jenkins sh -c \
|
||
'curl -fsS --noproxy "*" http://localhost:8080/queue/api/json?tree=items\[id\] 2>/dev/null | grep -o "\"id\"" | wc -l' 2>/dev/null || echo "-1")
|
||
if [[ "$q" == "-1" ]]; then warn "jenkins:queue" "无法读取队列（需认证，属正常）"
|
||
elif (( q > 10 )); then bad "jenkins:queue" "队列积压 $q 个任务，检查执行器是否卡死"
|
||
else ok "jenkins:queue" "队列 $q"; fi
|
||
else
|
||
bad "jenkins:queue" "Jenkins 容器未运行"
|
||
fi
|
||
|
||
# ---------- 汇总 ----------
|
||
if (( JSON == 1 )); then emit_json
|
||
else
|
||
echo "----------------------------------------"
|
||
echo "汇总: ✅ PASS=$PASS ⚠️ WARN=$WARN ❌ FAIL=$FAIL"
|
||
fi
|
||
|
||
if (( FAIL > 0 )); then alert; fi
|
||
exit $(( FAIL > 0 ? 1 : 0 ))
|