chore(infra): 新增 Gitea+Jenkins CI/CD 部署与凭据整改

- infra/cicd：docker-compose（gitea/jenkins）、JCasC（凭据统一 ${ENV} 注入，无硬编码）、备份/恢复、健康检查、凭据轮换与 git 历史清除脚本 - docs/deployment/cicd：安装、高可用备份监控、凭据事故复盘 - .env.example 仅为占位模板；真实 .env 由 .gitignore 排除
2026-09-20 10:37:57 +08:00
parent 7df2dde438
commit 4ab2f3cd8e
15 changed files with 2092 additions and 0 deletions
@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+# ============================================================
+# Novalon CI/CD 健康检查
+#
+# 探针维度：
+#   1. 容器存活与健康状态
+#   2. HTTPS 端点可达性（Gitea / Jenkins / 官网）
+#   3. TLS 证书剩余有效期
+#   4. 磁盘与 Docker 卷增长（宿主 20G 已用 85%，这是头号风险）
+#   5. 备份新鲜度（超过 36 小时无新备份 -> FAIL）
+#   6. Jenkins 构建队列积压 / 执行器卡死
+#
+# 退出码：0 全通过；1 有 FAIL。便于 cron + Webhook 告警。
+#
+# crontab（每 10 分钟）:
+#   */10 * * * * /home/novalon/docker-app/infra/cicd/monitoring/healthcheck-cicd.sh
+#       >> /var/log/cicd-healthcheck.log 2>&1
+#
+# 用法:
+#   ./healthcheck-cicd.sh             # 正常检查，FAIL 时推送告警
+#   ./healthcheck-cicd.sh --quiet     # 只在 FAIL 时输出
+#   ./healthcheck-cicd.sh --json      # 输出 JSON（供 Prometheus/自定义采集）
+# ============================================================
+
+set -uo pipefail   # 不用 -e，健康检查要跑完全部探针再汇总
+
+BACKUP_ROOT="${BACKUP_ROOT:-/home/novalon/backups/cicd}"
+ALERT_WEBHOOK_URL="${ALERT_WEBHOOK_URL:-}"
+DISK_WARN_PCT="${DISK_WARN_PCT:-80}"
+DISK_CRIT_PCT="${DISK_CRIT_PCT:-90}"
+CERT_WARN_DAYS="${CERT_WARN_DAYS:-21}"
+BACKUP_MAX_AGE_HOURS="${BACKUP_MAX_AGE_HOURS:-36}"
+
+GITEA_URL="${GITEA_URL:-https://git.f.novalon.cn}"
+JENKINS_URL="${JENKINS_URL:-https://ci.f.novalon.cn}"
+SITE_URL="${SITE_URL:-https://novalon.cn}"
+
+QUIET=0; JSON=0
+for a in "$@"; do
+  case "$a" in
+    --quiet) QUIET=1 ;;
+    --json)  JSON=1 ;;
+  esac
+done
+
+PASS=0; FAIL=0; WARN=0
+RESULTS=()
+
+# ⚠️ 这三个函数必须显式 return 0：
+#    曾踩坑 —— --quiet/--json 模式下 `[[ 条件 ]] && echo` 因条件为假而返回 1，
+#    导致 `cond && ok ... || bad ...` 构造中 ok 已执行却"返回失败"，跌进 bad 分支，
+#    造成 FAIL 计数随输出模式抖动（1/1/2）
+ok()   { PASS=$((PASS+1)); RESULTS+=("PASS|$1|$2"); if [[ $QUIET == 0 && $JSON == 0 ]]; then echo "  ✅ $1 — $2"; fi; return 0; }
+warn() { WARN=$((WARN+1)); RESULTS+=("WARN|$1|$2"); if [[ $JSON == 0 ]]; then echo "  ⚠️  $1 — $2"; fi; return 0; }
+bad()  { FAIL=$((FAIL+1)); RESULTS+=("FAIL|$1|$2"); if [[ $JSON == 0 ]]; then echo "  ❌ $1 — $2"; fi; return 0; }
+
+emit_json() {
+  printf '{"ts":"%s","host":"%s","pass":%d,"warn":%d,"fail":%d,"checks":[' \
+    "$(date -Is)" "$(hostname)" "$PASS" "$WARN" "$FAIL"
+  local first=1
+  for r in "${RESULTS[@]}"; do
+    IFS='|' read -r lvl name msg <<< "$r"
+    (( first )) || printf ','
+    first=0
+    printf '{"level":"%s","name":"%s","msg":"%s"}' "$lvl" "$name" "${msg//\"/\\\"}"
+  done
+  printf ']}\n'
+}
+
+alert() {
+  [[ -z "$ALERT_WEBHOOK_URL" ]] && return 0
+  local body="[CI/CD 健康检查] 主机 $(hostname)
+$(printf '%s\n' "${RESULTS[@]}" | grep '^FAIL' | sed 's/^FAIL|/  ❌ /' | sed 's/|/ — /')
+$(printf '%s\n' "${RESULTS[@]}" | grep '^WARN' | sed 's/^WARN|/  ⚠️  /' | sed 's/|/ — /')"
+  curl -fsS -m 10 -X POST "$ALERT_WEBHOOK_URL" \
+    -H 'Content-Type: application/json' \
+    --data-binary "$(printf '{"msgtype":"text","text":{"content":%s}}' \
+        "$(printf '%s' "$body" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"见日志\"')")" \
+    >/dev/null 2>&1 || true
+}
+
+[[ $JSON == 0 ]] && echo "========== CI/CD 健康检查 $(date -Is) =========="
+
+# ---------- 1. 容器状态 ----------
+[[ $JSON == 0 ]] && echo "[1] 容器状态"
+for c in gitea jenkins postgresql nginx; do
+  name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -x "$c" || true)
+  if [[ -z "$name" ]]; then
+    # nginx 容器名带前缀，做模糊匹配
+    name=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -i "$c" | head -1 || true)
+  fi
+  if [[ -z "$name" ]]; then
+    bad "container:$c" "容器未运行"
+    continue
+  fi
+  # ⚠️ 未定义 healthcheck 的容器：inspect 返回空串（命令成功，|| 不会触发），
+  #    必须显式把空串归一为 none，否则会误报"状态异常"
+  st="$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)"
+  st="${st:-none}"
+  if [[ "$st" == "healthy" || "$st" == "none" ]]; then
+    ok "container:$c" "运行中 ($st)"
+  else
+    bad "container:$c" "状态异常: $st"
+  fi
+done
+
+# ---------- 2. HTTPS 端点 ----------
+[[ $JSON == 0 ]] && echo "[2] HTTPS 端点"
+check_url() {
+  local label="$1" url="$2" expect="$3"
+  # ⚠️ 不能用 `|| echo 000`：curl 偶发"已输出 http_code 但退出码非 0"，
+  #    会把 200 与 000 拼接成 200000，造成假 FAIL。改用正则校验兜底。
+  local code
+  code="$(curl -sS -o /dev/null -w '%{http_code}' -m 12 --noproxy '*' "$url" 2>/dev/null || true)"
+  [[ "$code" =~ ^[0-9]{3}$ ]] || code="000"
+  if [[ "$code" =~ ^($expect)$ ]]; then ok "http:$label" "$code ($url)"
+  else bad "http:$label" "期望 $expect 实得 $code ($url)"; fi
+}
+check_url gitea   "$GITEA_URL/"   "200|301|302|303"
+check_url jenkins "$JENKINS_URL/" "200|403"
+check_url site    "$SITE_URL/"    "200|301|302"
+
+# ---------- 3. TLS 证书 ----------
+[[ $JSON == 0 ]] && echo "[3] TLS 证书"
+check_cert() {
+  local host="$1"
+  local end; end=$(echo | timeout 12 openssl s_client -servername "$host" -connect "$host:443" 2>/dev/null \
+    | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
+  if [[ -z "$end" ]]; then warn "cert:$host" "无法获取证书信息"; return; fi
+  local end_epoch now days
+  end_epoch=$(date -d "$end" +%s 2>/dev/null || date -j -f "%b %e %H:%M:%S %Y %Z" "$end" +%s 2>/dev/null || echo 0)
+  now=$(date +%s)
+  days=$(( (end_epoch - now) / 86400 ))
+  if (( days <= 0 ));        then bad  "cert:$host" "证书已过期 ($days 天)"
+  elif (( days < 7 ));       then bad  "cert:$host" "证书 $days 天后过期，立即续期"
+  elif (( days < CERT_WARN_DAYS )); then warn "cert:$host" "证书 $days 天后过期"
+  else ok "cert:$host" "剩余 $days 天"; fi
+}
+check_cert "${GITEA_URL#https://}"
+check_cert "${JENKINS_URL#https://}"
+
+# ---------- 4. 磁盘 ----------
+[[ $JSON == 0 ]] && echo "[4] 磁盘与卷"
+root_pct=$(df -P / | awk 'NR==2{gsub(/%/,"");print $5}')
+root_free=$(df -h / | awk 'NR==2{print $4}')
+if   (( root_pct >= DISK_CRIT_PCT )); then bad  "disk:root" "已用 ${root_pct}%（阈值 ${DISK_CRIT_PCT}%），可用 ${root_free}"
+elif (( root_pct >= DISK_WARN_PCT )); then warn "disk:root" "已用 ${root_pct}%（阈值 ${DISK_WARN_PCT}%），可用 ${root_free}"
+else ok "disk:root" "已用 ${root_pct}%，可用 ${root_free}"; fi
+
+for v in jenkins_jenkins_home; do
+  if docker volume inspect "$v" >/dev/null 2>&1; then
+    sz=$(docker system df -v 2>/dev/null | grep -F "$v" | awk '{print $3}' | head -1)
+    if [[ -n "$sz" ]]; then
+      num=${sz%GB}
+      if (( $(echo "$num > 5" | bc -l 2>/dev/null || echo 0) )); then
+        warn "volume:$v" "体积 ${sz} 超过 5GB，检查构建历史清理策略"
+      else ok "volume:$v" "体积 ${sz}"; fi
+    fi
+  fi
+done
+
+# 备份区体积
+if [[ -d "$BACKUP_ROOT" ]]; then
+  bsz=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1)
+  ok "backup:size" "备份区 ${bsz}"
+fi
+
+# ---------- 5. 备份新鲜度 ----------
+[[ $JSON == 0 ]] && echo "[5] 备份新鲜度"
+if [[ ! -d "$BACKUP_ROOT" ]]; then
+  bad "backup:freshness" "备份目录不存在: $BACKUP_ROOT（Jenkins 从未备份过）"
+else
+  latest=$(ls -1dt "$BACKUP_ROOT"/[0-9]*_[0-9]* 2>/dev/null | head -1 || true)
+  if [[ -z "$latest" ]]; then
+    bad "backup:freshness" "备份目录存在但无任何备份"
+  else
+    age_h=$(( ( $(date +%s) - $(stat -c %Y "$latest") ) / 3600 ))
+    if (( age_h > BACKUP_MAX_AGE_HOURS )); then
+      bad "backup:freshness" "最新备份 $(basename "$latest") 已 ${age_h}h（阈值 ${BACKUP_MAX_AGE_HOURS}h）"
+    else
+      ok "backup:freshness" "最新备份 $(basename "$latest")，${age_h}h 前"
+    fi
+    # 内容完整性
+    cnt=$(find "$latest" -name '*.tar.gz' -o -name '*.sql.gz' 2>/dev/null | wc -l | tr -d ' ')
+    # 注意：不要写成 `(( cnt >= 2 )) && ok || bad` —— 若 ok 返回非 0 会跌进 bad
+    if (( cnt >= 2 )); then ok "backup:content" "含 $cnt 个产物"; else bad "backup:content" "产物仅 $cnt 个，备份不完整"; fi
+  fi
+fi
+
+# ---------- 6. Jenkins 队列 ----------
+[[ $JSON == 0 ]] && echo "[6] Jenkins 队列"
+if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx jenkins; then
+  q=$(docker exec jenkins sh -c \
+      'curl -fsS --noproxy "*" http://localhost:8080/queue/api/json?tree=items\[id\] 2>/dev/null | grep -o "\"id\"" | wc -l' 2>/dev/null || echo "-1")
+  if [[ "$q" == "-1" ]]; then warn "jenkins:queue" "无法读取队列（需认证，属正常）"
+  elif (( q > 10 )); then bad "jenkins:queue" "队列积压 $q 个任务，检查执行器是否卡死"
+  else ok "jenkins:queue" "队列 $q"; fi
+else
+  bad "jenkins:queue" "Jenkins 容器未运行"
+fi
+
+# ---------- 汇总 ----------
+if (( JSON == 1 )); then emit_json
+else
+  echo "----------------------------------------"
+  echo "汇总: ✅ PASS=$PASS  ⚠️  WARN=$WARN  ❌ FAIL=$FAIL"
+fi
+
+if (( FAIL > 0 )); then alert; fi
+exit $(( FAIL > 0 ? 1 : 0 ))
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# ============================================================
+# 容器日志兜底截断
+#
+# 为什么需要它（2026-09-14 实测事故）：
+#   novalon-cicd/docker-compose.yml 里明明配了
+#     logging: { driver: json-file, options: { max-size: 10m, max-file: 3 } }
+#   但运行中容器的 HostConfig.LogConfig 实测为 {"Type":"json-file","Config":{}}
+#   —— 配置从未生效，Gitea 单份日志涨到 849MB，占掉整盘 4%。
+#
+# 根因：logging 驱动参数在**容器创建时**固化，事后改 compose 不重建则不生效。
+#
+# 本脚本是**临时兜底**，不是终态。
+# 终态：用 infra/cicd/docker-compose.gitea.yml 重建容器（含正确 logging 配置），
+#       之后本脚本可退化为纯监控用途。
+#
+# 用法:
+#   ./truncate-container-logs.sh              # 截断超过阈值的日志
+#   ./truncate-container-logs.sh --check      # 只报告，不截断
+#   ./truncate-container-logs.sh --threshold 50   # 自定义阈值(MB)
+# ============================================================
+
+set -uo pipefail
+
+THRESHOLD_MB=100
+CHECK_ONLY=0
+# 用 while 而非 for 解析：for 迭代的是展开后的固定列表，
+# 循环内 shift 不会改变已排定的迭代序列，导致 `--check --threshold 50` 取错值
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check)     CHECK_ONLY=1; shift ;;
+    --threshold) THRESHOLD_MB="${2:-100}"; shift 2 ;;
+    -h|--help)   sed -n '2,22p' "$0"; exit 0 ;;
+    *) echo "未知参数: $1" >&2; exit 2 ;;
+  esac
+done
+
+LOG_ROOT="${LOG_ROOT:-/var/lib/docker/containers}"
+ts() { date "+%Y-%m-%d %H:%M:%S"; }
+
+echo "[$(ts)] 容器日志检查（阈值 ${THRESHOLD_MB}MB，模式 $([[ $CHECK_ONLY == 1 ]] && echo 只读 || echo 截断)）"
+
+TOTAL_BEFORE=0
+TOTAL_AFTER=0
+HIT=0
+
+while IFS= read -r f; do
+  [[ -f "$f" ]] || continue
+  size_mb=$(( $(stat -c %s "$f") / 1048576 ))
+  [[ $size_mb -lt $THRESHOLD_MB ]] && continue
+
+  HIT=$((HIT + 1))
+  TOTAL_BEFORE=$((TOTAL_BEFORE + size_mb))
+
+  cid=$(basename "$(dirname "$f")")
+  cname=$(docker inspect -f '{{.Name}}' "$cid" 2>/dev/null | sed 's|^/||' || echo "$cid")
+  echo "  ⚠️  ${cname}: ${size_mb}MB  ($f)"
+
+  if (( CHECK_ONLY == 0 )); then
+    # truncate 对运行中的容器安全：Docker 以 O_APPEND 写，截断后继续追加
+    truncate -s 0 "$f" && echo "      → 已截断"
+    TOTAL_AFTER=$((TOTAL_AFTER + $(stat -c %s "$f") / 1048576))
+  fi
+done < <(find "$LOG_ROOT" -maxdepth 2 -name '*-json.log' 2>/dev/null)
+
+if (( HIT == 0 )); then
+  echo "  ✅ 无日志超过 ${THRESHOLD_MB}MB"
+else
+  echo "  命中 $HIT 个"
+  if (( CHECK_ONLY == 0 )); then
+    echo "  释放约 $((TOTAL_BEFORE - TOTAL_AFTER))MB"
+    df -h / | awk 'NR==2{print "  磁盘: 已用 "$5", 可用 "$4}'
+  fi
+fi
+
+# 提醒：若命中说明 logging 配置仍未生效，需要重建容器才是终态修复
+if (( HIT > 0 )); then
+  echo "  ℹ️  命中的容器说明其 logging 轮转未生效 —— 请用带 logging 配置的 compose 重建容器（终态修复）"
+fi