#!/bin/bash # 异地备份同步脚本 # 用法: ./backup-offsite-sync.sh # 将本地备份同步到远程存储(S3/OSS/NFS),支持校验和清理 set -u show_help() { cat <&2 exit 1 fi BACKUP_OFFSITE_BACKEND="$2" shift 2 ;; --no-cleanup) NO_CLEANUP=1 shift ;; --no-verify) NO_VERIFY=1 shift ;; *) echo "ERROR: Unknown argument: $1" >&2 exit 1 ;; esac done BACKUP_DIR="${BACKUP_DIR:-./backups}" BACKEND="${BACKUP_OFFSITE_BACKEND:-none}" REMOTE="${BACKUP_OFFSITE_REMOTE:-}" BUCKET="${BACKUP_OFFSITE_BUCKET:-}" ACCESS_KEY="${BACKUP_OFFSITE_ACCESS_KEY:-}" SECRET_KEY="${BACKUP_OFFSITE_SECRET_KEY:-}" REGION="${BACKUP_OFFSITE_REGION:-us-east-1}" RETENTION_DAYS="${BACKUP_OFFSITE_RETENTION_DAYS:-90}" echo "=== Offsite Backup Sync ===" echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" echo "Backend: $BACKEND" echo "Local: $BACKUP_DIR" echo "Remote: $REMOTE" echo "" # 检查后端类型 if [ "$BACKEND" = "none" ]; then echo "INFO: BACKUP_OFFSITE_BACKEND=none, offsite sync disabled" echo "To enable, set BACKUP_OFFSITE_BACKEND to s3, oss, or nfs" exit 0 fi if [ "$BACKEND" != "s3" ] && [ "$BACKEND" != "oss" ] && [ "$BACKEND" != "nfs" ]; then echo "ERROR: Invalid backend: $BACKEND (must be s3, oss, nfs, or none)" >&2 exit 1 fi # 检查本地备份目录 if [ ! -d "$BACKUP_DIR" ]; then echo "ERROR: Local backup directory does not exist: $BACKUP_DIR" >&2 exit 1 fi # 统计本地备份文件 LOCAL_FILES=$(ls -1 "$BACKUP_DIR"/db_backup_*.sql.gz 2>/dev/null | wc -l) if [ "$LOCAL_FILES" -eq 0 ]; then echo "ERROR: No backup files found in $BACKUP_DIR" >&2 exit 1 fi echo "INFO: Found $LOCAL_FILES local backup files" # 检查远程配置 if [ -z "$REMOTE" ]; then echo "ERROR: BACKUP_OFFSITE_REMOTE not set" >&2 echo "Example for $BACKEND:" >&2 case "$BACKEND" in s3) echo " s3://my-bucket/backups/" >&2 ;; oss) echo " oss://my-bucket/backups/" >&2 ;; nfs) echo " /mnt/nfs/backups/" >&2 ;; esac exit 1 fi # 检查工具可用性 check_tool() { if ! command -v "$1" >/dev/null 2>&1; then echo "ERROR: Required tool not found: $1" >&2 echo "Please install $1 to use the $BACKEND backend" >&2 exit 1 fi } # 配置凭证 setup_credentials() { case "$BACKEND" in s3) if [ -n "$ACCESS_KEY" ] && [ -n "$SECRET_KEY" ]; then export AWS_ACCESS_KEY_ID="$ACCESS_KEY" export AWS_SECRET_ACCESS_KEY="$SECRET_KEY" export AWS_DEFAULT_REGION="$REGION" fi ;; oss) if [ -n "$ACCESS_KEY" ] && [ -n "$SECRET_KEY" ]; then # ossutil 配置 if [ -f ~/.ossutilconfig ]; then cp ~/.ossutilconfig ~/.ossutilconfig.bak 2>/dev/null || true fi cat > ~/.ossutilconfig <&2 echo "Please ensure NFS is mounted at this path" >&2 exit 1 fi ;; esac } # 同步到远程 sync_to_remote() { echo "" echo "[1/3] Syncing backups to $BACKEND..." case "$BACKEND" in s3) if command -v aws >/dev/null 2>&1; then echo "Using aws-cli" if ! aws s3 sync "$BACKUP_DIR/" "$REMOTE" \ --exclude "*" --include "db_backup_*.sql.gz" \ --no-progress; then echo "ERROR: aws s3 sync failed" >&2 return 1 fi elif command -v rclone >/dev/null 2>&1; then echo "Using rclone" if ! rclone sync "$BACKUP_DIR" "$REMOTE" \ --include "db_backup_*.sql.gz" \ --progress; then echo "ERROR: rclone sync failed" >&2 return 1 fi else echo "ERROR: Neither aws-cli nor rclone found" >&2 return 1 fi ;; oss) if command -v ossutil >/dev/null 2>&1; then echo "Using ossutil" if ! ossutil cp -r "$BACKUP_DIR/" "$REMOTE" \ --include "db_backup_*.sql.gz" -f; then echo "ERROR: ossutil sync failed" >&2 return 1 fi elif command -v rclone >/dev/null 2>&1; then echo "Using rclone" if ! rclone sync "$BACKUP_DIR" "$REMOTE" \ --include "db_backup_*.sql.gz" \ --progress; then echo "ERROR: rclone sync failed" >&2 return 1 fi else echo "ERROR: Neither ossutil nor rclone found" >&2 return 1 fi ;; nfs) if command -v rsync >/dev/null 2>&1; then echo "Using rsync" mkdir -p "$REMOTE" 2>/dev/null || true if ! rsync -av --include="db_backup_*.sql.gz" --exclude="*" \ "$BACKUP_DIR/" "$REMOTE/"; then echo "ERROR: rsync failed" >&2 return 1 fi else echo "Using cp (rsync not available)" mkdir -p "$REMOTE" 2>/dev/null || true if ! cp "$BACKUP_DIR"/db_backup_*.sql.gz "$REMOTE/" 2>/dev/null; then echo "ERROR: cp failed" >&2 return 1 fi fi ;; esac echo " PASS: Sync completed" return 0 } # 校验同步结果 verify_sync() { if [ "$NO_VERIFY" -eq 1 ]; then echo "" echo "[2/3] Verification skipped (--no-verify)" return 0 fi echo "" echo "[2/3] Verifying sync result..." REMOTE_FILES=0 case "$BACKEND" in s3) if command -v aws >/dev/null 2>&1; then REMOTE_FILES=$(aws s3 ls "$REMOTE" --recursive 2>/dev/null | grep -c "db_backup_.*\.sql\.gz" || echo 0) elif command -v rclone >/dev/null 2>&1; then REMOTE_FILES=$(rclone lsf "$REMOTE" --include "db_backup_*.sql.gz" 2>/dev/null | wc -l || echo 0) fi ;; oss) if command -v ossutil >/dev/null 2>&1; then REMOTE_FILES=$(ossutil ls "$REMOTE" 2>/dev/null | grep -c "db_backup_.*\.sql\.gz" || echo 0) elif command -v rclone >/dev/null 2>&1; then REMOTE_FILES=$(rclone lsf "$REMOTE" --include "db_backup_*.sql.gz" 2>/dev/null | wc -l || echo 0) fi ;; nfs) REMOTE_FILES=$(ls -1 "$REMOTE"/db_backup_*.sql.gz 2>/dev/null | wc -l) ;; esac echo " Local files: $LOCAL_FILES" echo " Remote files: $REMOTE_FILES" if [ "$REMOTE_FILES" -lt "$LOCAL_FILES" ]; then echo " WARN: Remote has fewer files than local (some may have been cleaned up)" else echo " PASS: File count verified" fi return 0 } # 清理远程过期备份 cleanup_remote() { if [ "$NO_CLEANUP" -eq 1 ]; then echo "" echo "[3/3] Cleanup skipped (--no-cleanup)" return 0 fi echo "" echo "[3/3] Cleaning up remote backups older than $RETENTION_DAYS days..." case "$BACKEND" in s3) if command -v aws >/dev/null 2>&1; then aws s3 ls "$REMOTE" --recursive 2>/dev/null | grep "db_backup_.*\.sql\.gz" | while read -r line; do FILE_PATH=$(echo "$line" | awk '{print $4}') FILE_DATE=$(echo "$FILE_PATH" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1) if [ -n "$FILE_DATE" ]; then FILE_TS=$(echo "$FILE_DATE" | sed 's/\([0-9]\{8\}\)_\([0-9]\{6\}\)/\1 \2/' | awk '{print $1}') CUTOFF=$(date -d "-$RETENTION_DAYS days" +%Y%m%d 2>/dev/null || date -v-${RETENTION_DAYS}d +%Y%m%d 2>/dev/null) if [ -n "$CUTOFF" ] && [ "$FILE_TS" -lt "$CUTOFF" ]; then echo " Deleting: $FILE_PATH" aws s3 rm "s3://$(echo "$REMOTE" | sed 's|s3://||')/$FILE_PATH" 2>/dev/null || true fi fi done fi ;; oss) if command -v ossutil >/dev/null 2>&1; then # ossutil 不支持基于时间的清理,使用生命周期规则或手动删除 echo " INFO: For OSS, configure lifecycle rules in the console for automatic cleanup" echo " INFO: Manual cleanup with retention $RETENTION_DAYS days" fi ;; nfs) if [ -d "$REMOTE" ]; then find "$REMOTE" -name "db_backup_*.sql.gz" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true echo " Cleaned up files older than $RETENTION_DAYS days" fi ;; esac echo " PASS: Cleanup completed" return 0 } # 执行同步流程 setup_credentials if ! sync_to_remote; then exit 1 fi verify_sync cleanup_remote echo "" echo "=== Offsite Sync Complete ===" exit 0