zhangxiang
4ab2f3cd8e
- infra/cicd：docker-compose（gitea/jenkins）、JCasC（凭据统一 ${ENV} 注入，无硬编码）、
备份/恢复、健康检查、凭据轮换与 git 历史清除脚本
- docs/deployment/cicd：安装、高可用备份监控、凭据事故复盘
- .env.example 仅为占位模板；真实 .env 由 .gitignore 排除
319 lines
14 KiB
Bash
Executable File
319 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================
|
||
# Novalon CI/CD 统一备份脚本
|
||
#
|
||
# 覆盖：
|
||
# 1. Gitea PostgreSQL 库（pg_dump，custom format）
|
||
# 2. Gitea 仓库裸库（/data/git/repositories）
|
||
# 3. Jenkins 配置薄备份（排除 builds/ workspace/ 缓存，保留作业定义与凭据）
|
||
#
|
||
# 设计要点：
|
||
# - 每份产物落盘后立刻校验（大小非零 + 归档可列出），坏备份当场失败而不是等到恢复时才发现
|
||
# - 磁盘预检：剩余空间不足直接中止并告警（宿主 20G 已用 85%，写满会连带拖垮 nginx）
|
||
# - 异地同步可选：配了 RCLONE_REMOTE 才同步，否则明确告警"仅本地"
|
||
# - 全链路日志，失败走 ALERT_WEBHOOK_URL
|
||
#
|
||
# 用法:
|
||
# ./backup-cicd.sh # 标准备份（库 + 仓库 + Jenkins 配置）
|
||
# ./backup-cicd.sh --full # 追加 Gitea LFS/附件/头像
|
||
# ./backup-cicd.sh --dry-run # 只预检，不产出
|
||
# ./backup-cicd.sh --jenkins-only # 只备份 Jenkins
|
||
#
|
||
# crontab（每天 03:30，见 docs/deployment/cicd/04-ha-backup-monitoring.md）:
|
||
# 30 3 * * * /home/novalon/docker-app/infra/cicd/backup/backup-cicd.sh >> /var/log/cicd-backup.log 2>&1
|
||
# ============================================================
|
||
|
||
set -Eeuo pipefail
|
||
|
||
# ---------- 配置 ----------
|
||
DEPLOY_ROOT="${DEPLOY_ROOT:-/home/novalon/docker-app}"
|
||
ENV_FILE="${ENV_FILE:-$DEPLOY_ROOT/infra/cicd/.env}"
|
||
BACKUP_ROOT="${BACKUP_ROOT:-/home/novalon/backups/cicd}"
|
||
RETENTION="${BACKUP_RETENTION:-3}" # 兼容旧变量
|
||
FULL_RETENTION="${BACKUP_FULL_RETENTION:-1}" # 全量保留份数（含 Gitea 仓库）
|
||
THIN_RETENTION="${BACKUP_THIN_RETENTION:-3}" # 薄保留份数（仅 DB + Jenkins 配置）
|
||
MIN_FREE_GB="${BACKUP_MIN_FREE_GB:-3}"
|
||
RCLONE_REMOTE="${RCLONE_REMOTE:-}"
|
||
RCLONE_PATH="${RCLONE_PATH:-novalon-cicd-backup}"
|
||
ALERT_WEBHOOK_URL="${ALERT_WEBHOOK_URL:-}"
|
||
|
||
GITEA_CONTAINER="${GITEA_CONTAINER:-gitea}"
|
||
PG_CONTAINER="${PG_CONTAINER:-postgresql}"
|
||
JENKINS_CONTAINER="${JENKINS_CONTAINER:-jenkins}"
|
||
GITEA_DB_NAME="${GITEA_DB_NAME:-gitea}"
|
||
GITEA_DB_USER="${GITEA_DB_USER:-gitea}"
|
||
|
||
FULL=0
|
||
DRY_RUN=0
|
||
JENKINS_ONLY=0
|
||
SKIP_REPOS=0
|
||
|
||
# ---------- 参数 ----------
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--full) FULL=1; shift ;;
|
||
--dry-run) DRY_RUN=1; shift ;;
|
||
--jenkins-only) JENKINS_ONLY=1; shift ;;
|
||
# 薄备份：跳过 Gitea 裸库（939MB，占单份 78%）。
|
||
# 依据：Git 仓库天然分布式，每份 clone 都是完整历史；
|
||
# 而 PostgreSQL 里的用户/权限/PR/Issue/Webhook 是唯一副本、不可重建。
|
||
# 故磁盘受限时优先保 DB，仓库交给「异地全量」或开发者本地副本。
|
||
--skip-repos) SKIP_REPOS=1; shift ;;
|
||
-h|--help) sed -n '2,34p' "$0"; exit 0 ;;
|
||
*) echo "未知参数: $1" >&2; exit 2 ;;
|
||
esac
|
||
done
|
||
|
||
# ---------- 加载 .env（若存在） ----------
|
||
if [[ -f "$ENV_FILE" ]]; then
|
||
# shellcheck disable=SC1090
|
||
set -a; source "$ENV_FILE"; set +a
|
||
BACKUP_ROOT="${BACKUP_ROOT:-/home/novalon/backups/cicd}"
|
||
RETENTION="${BACKUP_RETENTION:-3}"
|
||
FULL_RETENTION="${BACKUP_FULL_RETENTION:-1}"
|
||
THIN_RETENTION="${BACKUP_THIN_RETENTION:-3}"
|
||
MIN_FREE_GB="${BACKUP_MIN_FREE_GB:-3}"
|
||
RCLONE_REMOTE="${RCLONE_REMOTE:-}"
|
||
RCLONE_PATH="${RCLONE_PATH:-novalon-cicd-backup}"
|
||
ALERT_WEBHOOK_URL="${ALERT_WEBHOOK_URL:-}"
|
||
fi
|
||
|
||
TS="$(date +%Y%m%d_%H%M%S)"
|
||
STAMP_DIR="$BACKUP_ROOT/$TS"
|
||
LOG_PREFIX="[backup $TS]"
|
||
|
||
log() { echo "$LOG_PREFIX $*"; }
|
||
fail() { echo "$LOG_PREFIX ❌ $*" >&2; alert "备份失败: $*"; exit 1; }
|
||
|
||
alert() {
|
||
local msg="$1"
|
||
[[ -z "$ALERT_WEBHOOK_URL" ]] && return 0
|
||
curl -fsS -m 10 -X POST "$ALERT_WEBHOOK_URL" \
|
||
-H 'Content-Type: application/json' \
|
||
-d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[CI/CD备份告警] $msg ($(hostname))\"}}" \
|
||
>/dev/null 2>&1 || true
|
||
}
|
||
|
||
# 任何未捕获错误都触发告警
|
||
trap 'fail "第 $LINENO 行发生未捕获错误"' ERR
|
||
|
||
require_container() {
|
||
local name="$1"
|
||
docker inspect "$name" >/dev/null 2>&1 || fail "容器不存在: $name"
|
||
local st; st="$(docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false)"
|
||
[[ "$st" == "true" ]] || fail "容器未运行: $name"
|
||
}
|
||
|
||
# 归档完整性校验：非空 + tar/zip 可列出
|
||
verify_archive() {
|
||
local f="$1" kind="$2"
|
||
[[ -s "$f" ]] || fail "产物为空: $f"
|
||
local sz; sz=$(stat -c %s "$f")
|
||
(( sz > 1024 )) || fail "产物过小($sz B)，判定失败: $f"
|
||
case "$kind" in
|
||
tar) tar -tzf "$f" >/dev/null 2>&1 || fail "tar 校验失败（归档损坏）: $f" ;;
|
||
sql) gzip -t "$f" >/dev/null 2>&1 || fail "gzip 校验失败: $f" ;;
|
||
esac
|
||
log "✅ 校验通过 $(basename "$f") ($(numfmt --to=iec --suffix=B "$sz" 2>/dev/null || echo "${sz}B"))"
|
||
}
|
||
|
||
# ============================================================
|
||
log "========== CI/CD 备份开始 =========="
|
||
log "目标目录: $STAMP_DIR 保留份数: $RETENTION 全量: $FULL 跳过仓库: $SKIP_REPOS dry-run: $DRY_RUN"
|
||
|
||
# ---------- 预检 ----------
|
||
command -v docker >/dev/null || fail "docker 不可用"
|
||
|
||
FREE_GB=$(df -P "$BACKUP_ROOT" 2>/dev/null | awk 'NR==2{print int($4/1048576)}' \
|
||
|| df -P / | awk 'NR==2{print int($4/1048576)}')
|
||
log "备份卷可用空间: ${FREE_GB}GB（阈值 ${MIN_FREE_GB}GB）"
|
||
if (( FREE_GB < MIN_FREE_GB )); then
|
||
fail "可用空间 ${FREE_GB}GB < 阈值 ${MIN_FREE_GB}GB，中止备份以免写满磁盘。请清理或扩容。"
|
||
fi
|
||
|
||
if (( JENKINS_ONLY == 0 )); then
|
||
require_container "$PG_CONTAINER"
|
||
require_container "$GITEA_CONTAINER"
|
||
fi
|
||
require_container "$JENKINS_CONTAINER"
|
||
|
||
if (( DRY_RUN == 1 )); then
|
||
log "dry-run：预检全部通过，未产出备份。"
|
||
exit 0
|
||
fi
|
||
|
||
mkdir -p "$STAMP_DIR"
|
||
|
||
# ============================================================
|
||
# 1. PostgreSQL —— Gitea 元数据（用户/权限/PR/Issue/Webhook）
|
||
# ============================================================
|
||
if (( JENKINS_ONLY == 0 )); then
|
||
log "----- [1/3] PostgreSQL 导出 -----"
|
||
DB_DUMP="$STAMP_DIR/gitea-db-$TS.sql.gz"
|
||
if docker exec "$PG_CONTAINER" pg_dump -U "$GITEA_DB_USER" -d "$GITEA_DB_NAME" -Fc \
|
||
| gzip -9 > "$DB_DUMP"; then
|
||
verify_archive "$DB_DUMP" sql
|
||
else
|
||
rm -f "$DB_DUMP"; fail "pg_dump 执行失败"
|
||
fi
|
||
|
||
# ============================================================
|
||
# 2. Gitea 仓库裸库
|
||
# ============================================================
|
||
log "----- [2/3] Gitea 仓库导出 -----"
|
||
REPO_TAR="$STAMP_DIR/gitea-repos-$TS.tar.gz"
|
||
if (( SKIP_REPOS == 1 )); then
|
||
log "⏭ 已跳过仓库导出（--skip-repos）：本次仅保 DB 与 Jenkins 配置"
|
||
log "ℹ️ 仓库是 Git 分布式的，但**不要长期依赖**此项跳过 —— 需有异地全量或开发者副本兜底"
|
||
elif [[ "$FULL" == "1" ]]; then
|
||
# 全量：仓库 + LFS + 附件 + 头像
|
||
docker run --rm --volumes-from "$GITEA_CONTAINER" -v "$STAMP_DIR:/out" alpine \
|
||
tar czf "/out/$(basename "$REPO_TAR")" -C /data git 2>/dev/null \
|
||
|| fail "Gitea 全量归档失败"
|
||
else
|
||
# 标准：仅裸库（LFS/附件可从对象存储或后续补，磁盘优先）
|
||
docker run --rm --volumes-from "$GITEA_CONTAINER" -v "$STAMP_DIR:/out" alpine \
|
||
sh -c 'tar czf /out/'"$(basename "$REPO_TAR")"' -C /data git/repositories' \
|
||
|| fail "Gitea 仓库归档失败"
|
||
verify_archive "$REPO_TAR" tar
|
||
fi
|
||
fi
|
||
|
||
# ============================================================
|
||
# 3. Jenkins 薄备份
|
||
# ！关键：排除 builds/ workspace/ caches/ logs —— 这些占空间且可从作业定义重建
|
||
# ！但必须包含 credentials.xml 与 secrets/（否则恢复后凭据全丢）
|
||
# ============================================================
|
||
log "----- [3/3] Jenkins 配置薄备份 -----"
|
||
JENKINS_TAR="$STAMP_DIR/jenkins-config-$TS.tar.gz"
|
||
# 归档整个 JENKINS_HOME 并排除可重建项，不再写死具体路径
|
||
# （曾踩坑：nodes/ 在部分版本不存在，tar 直接以非 0 退出导致整段备份失败）
|
||
# 体积参考（2026-09-14 实测）：plugins 265M / war 111M / workspace 49M / updates 3.4M
|
||
# war、workspace、updates 均可从镜像或 SCM 重建，排除后单份约 80M
|
||
# ⚠️ ./plugins 也必须排除：实测它占薄备份 222MB 中的绝大部分，
|
||
# 而插件可由同目录导出的 jenkins-plugins-<ts>.txt 精确重装（jenkins-plugin-cli --plugin-file）
|
||
# 排除后薄备份从 222MB 降到个位数 MB，磁盘受限场景下这是决定性差异
|
||
set +e
|
||
docker run --rm --volumes-from "$JENKINS_CONTAINER" -v "$STAMP_DIR:/out" alpine \
|
||
sh -c '
|
||
cd /var/jenkins_home
|
||
tar czf /out/'"$(basename "$JENKINS_TAR")"' \
|
||
--exclude="./jobs/*/builds" \
|
||
--exclude="./jobs/*/workspace" \
|
||
--exclude="./jobs/*/workspace@tmp" \
|
||
--exclude="./jobs/*/last*" \
|
||
--exclude="./jobs/*/nextBuildNumber" \
|
||
--exclude="./workspace" \
|
||
--exclude="./workspace@tmp" \
|
||
--exclude="./plugins" \
|
||
--exclude="./war" \
|
||
--exclude="./caches" \
|
||
--exclude="./updates" \
|
||
--exclude="./logs" \
|
||
--exclude="./.cache" \
|
||
--exclude="./tools" \
|
||
--exclude="./.java" \
|
||
. 2>/dev/null
|
||
'
|
||
TAR_RC=$?
|
||
set -e
|
||
# GNU/busybox tar：0 成功；1 = "读取时文件发生变化"（Jenkins 在运行，属正常）；>=2 才是致命错误
|
||
if (( TAR_RC >= 2 )); then
|
||
fail "Jenkins 归档失败（tar 退出码 $TAR_RC）"
|
||
elif (( TAR_RC == 1 )); then
|
||
log "ℹ️ tar 退出码 1：归档过程中有文件变化（Jenkins 在运行），属可接受的警告"
|
||
fi
|
||
verify_archive "$JENKINS_TAR" tar
|
||
|
||
# 单独导出已安装插件清单，便于重建
|
||
docker exec "$JENKINS_CONTAINER" sh -c \
|
||
'ls /var/jenkins_home/plugins/*.jpi 2>/dev/null | xargs -n1 basename 2>/dev/null | sed "s/\.jpi$//"' \
|
||
> "$STAMP_DIR/jenkins-plugins-$TS.txt" 2>/dev/null || true
|
||
|
||
# ============================================================
|
||
# 校验：密钥文件必须存在，否则这份备份恢复不出来
|
||
# ============================================================
|
||
# Jenkins 2.5xx 起凭据解密依赖 secrets/master.key + secrets/hudson.util.Secret；
|
||
# 旧版本（<2.4xx）才是根目录的 secret.key。三者取其二即可恢复，必须至少有 master.key。
|
||
# （曾踩坑：只按旧版路径 secret.key 判定，在 2.568.1 上误报"无法解密"）
|
||
J_LIST="$(tar -tzf "$JENKINS_TAR" 2>/dev/null || true)"
|
||
# ⚠️ 必须用 herestring 而非管道：grep -q 命中即退出会关闭管道，
|
||
# 上游 printf 收到 SIGPIPE(141)，在 set -o pipefail 下整条管道被判失败 -> 误报"密钥缺失"
|
||
if grep -qE '(^|/)(secrets/master\.key|secret\.key)$' <<< "$J_LIST"; then
|
||
if grep -qE '(^|/)(secrets/hudson\.util\.Secret|secret\.key\.not-so-secret)$' <<< "$J_LIST"; then
|
||
log "✅ 凭据解密所需的 master key 与 hudson.util.Secret 均已包含"
|
||
else
|
||
log "⚠️ 找到 master key 但缺少 hudson.util.Secret —— 恢复后部分凭据可能不可解密"
|
||
fi
|
||
else
|
||
fail "归档缺少 master key（secrets/master.key 或 secret.key）—— 恢复后所有 Jenkins 凭据将不可解密"
|
||
fi
|
||
|
||
# 生成清单
|
||
cat > "$STAMP_DIR/MANIFEST.txt" <<EOF
|
||
备份时间: $(date -Is)
|
||
主机: $(hostname)
|
||
Gitea: $(docker inspect -f '{{.Config.Image}}' "$GITEA_CONTAINER" 2>/dev/null || echo n/a)
|
||
Jenkins: $(docker exec "$JENKINS_CONTAINER" sh -c 'java -jar /usr/share/jenkins/jenkins.war --version' 2>/dev/null || echo n/a)
|
||
Postgres: $(docker inspect -f '{{.Config.Image}}' "$PG_CONTAINER" 2>/dev/null || echo n/a)
|
||
模式: $( if (( SKIP_REPOS == 1 )); then echo thin-db-only; elif [[ "$FULL" == "1" ]]; then echo full; else echo standard; fi )
|
||
注: Jenkins 归档已排除 plugins/（可重建）。恢复后执行：
|
||
jenkins-plugin-cli --plugin-file jenkins-plugins-<ts>.txt
|
||
或在 Manage Jenkins -> Plugins 按清单重装。
|
||
产物:
|
||
$(ls -lh "$STAMP_DIR" | awk 'NR>1{print " " $9 " " $5}')
|
||
EOF
|
||
|
||
# ============================================================
|
||
# 异地同步
|
||
# ============================================================
|
||
if [[ -n "$RCLONE_REMOTE" ]]; then
|
||
if command -v rclone >/dev/null 2>&1; then
|
||
log "同步到异地: $RCLONE_REMOTE:$RCLONE_PATH/$TS"
|
||
rclone copy "$STAMP_DIR" "$RCLONE_REMOTE:$RCLONE_PATH/$TS" --transfers 2 \
|
||
|| log "⚠️ 异地同步失败（本地备份仍然有效）"
|
||
else
|
||
log "⚠️ 已配置 RCLONE_REMOTE 但未安装 rclone，跳过异地同步"
|
||
fi
|
||
else
|
||
log "⚠️ 未配置 RCLONE_REMOTE —— 备份仅存本地。同盘备份不抗主机故障，强烈建议配置异地。"
|
||
fi
|
||
|
||
# ============================================================
|
||
# 保留策略 —— 全量与薄分开计数
|
||
# （踩坑：若统一按「最近 N 份」裁剪，周中的薄备份会把周日全量挤出窗口，
|
||
# 导致 Gitea 仓库只受保护 3 天，等于没有全量）
|
||
# FULL_RETENTION 份全量（含仓库） + THIN_RETENTION 份薄（不含仓库）
|
||
# ============================================================
|
||
log "清理过期备份（全量保留 ${FULL_RETENTION} 份，薄保留 ${THIN_RETENTION} 份）..."
|
||
declare -A KEEP=()
|
||
|
||
# 取「最近的 N 份全量 / 薄」目录
|
||
# ⚠️ 循环体内用 if 而非 `cond && echo`：set -e 下条件为假会使子 shell 以非 0 退出，
|
||
# 导致进程替换提前终止（曾致第 302 行报"未捕获错误"）
|
||
keep_mark() {
|
||
local want="$1" mode="$2" d found
|
||
for d in "$BACKUP_ROOT"/[0-9]*_[0-9]*; do
|
||
if [[ ! -d "$d" ]]; then continue; fi
|
||
if [[ -n "$(ls "$d"/gitea-repos-*.tar.gz 2>/dev/null | head -1)" ]]; then found=1; else found=0; fi
|
||
if [[ "$mode" == "full" && "$found" == "1" ]] || [[ "$mode" == "thin" && "$found" == "0" ]]; then
|
||
echo "$d"
|
||
fi
|
||
done | sort -r | head -n "$want"
|
||
}
|
||
|
||
for d in $(keep_mark "$FULL_RETENTION" full); do KEEP["$d"]=1; done
|
||
for d in $(keep_mark "$THIN_RETENTION" thin); do KEEP["$d"]=1; done
|
||
|
||
for d in "$BACKUP_ROOT"/[0-9]*_[0-9]*; do
|
||
if [[ ! -d "$d" ]]; then continue; fi
|
||
if [[ -z "${KEEP["$d"]:-}" ]]; then
|
||
rm -rf "$d" && log " 已清理: $(basename "$d")"
|
||
fi
|
||
done
|
||
|
||
TOTAL=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1)
|
||
log "========== 备份完成 =========="
|
||
log "目录: $STAMP_DIR 备份区总计: ${TOTAL:-unknown}"
|
||
log "剩余磁盘: $(df -h / | awk 'NR==2{print $5" 已用, "$4" 可用"}')"
|