NextEdu/scripts/backup-offsite-sync.sh

#!/bin/bash
# 异地备份同步脚本
# 用法: ./backup-offsite-sync.sh
# 将本地备份同步到远程存储(S3/OSS/NFS),支持校验和清理

set -u

show_help() {
  cat <<EOF
用法: $0 [选项]
异地备份同步脚本,将本地备份同步到远程存储

选项:
  --backend TYPE       远程存储后端类型: s3|oss|nfs|none
  --no-cleanup         不清理远程过期备份
  --no-verify          不校验同步结果
  --help, -h           显示帮助信息

环境变量:
  BACKUP_DIR                  本地备份目录(默认 ./backups)
  BACKUP_OFFSITE_BACKEND      远程后端类型: s3|oss|nfs|none (默认 none)
  BACKUP_OFFSITE_REMOTE       远程目标路径
                                - s3:  s3://bucket-name/path
                                - oss: oss://bucket-name/path
                                - nfs: /mnt/nfs/backup-path
  BACKUP_OFFSITE_BUCKET       存储桶名称(仅 s3/oss)
  BACKUP_OFFSITE_ACCESS_KEY   访问密钥
  BACKUP_OFFSITE_SECRET_KEY   秘密密钥
  BACKUP_OFFSITE_REGION       区域(默认 us-east-1)
  BACKUP_OFFSITE_RETENTION_DAYS  远程保留天数(默认 90)

需要工具:
  s3:  aws-cli (aws) 或 rclone
  oss: ossutil 或 rclone
  nfs: rsync (NFS 应已挂载到 BACKUP_OFFSITE_REMOTE)

退出码:
  0  同步成功
  1  同步失败
EOF
}

# 解析参数
NO_CLEANUP=0
NO_VERIFY=0

while [ $# -gt 0 ]; do
  case "$1" in
    --help|-h)
      show_help
      exit 0
      ;;
    --backend)
      if [ $# -lt 2 ]; then
        echo "ERROR: --backend requires an argument" >&2
        exit 1
      fi
      BACKUP_OFFSITE_BACKEND="$2"
      shift 2
      ;;
    --no-cleanup)
      NO_CLEANUP=1
      shift
      ;;
    --no-verify)
      NO_VERIFY=1
      shift
      ;;
    *)
      echo "ERROR: Unknown argument: $1" >&2
      exit 1
      ;;
  esac
done

BACKUP_DIR="${BACKUP_DIR:-./backups}"
BACKEND="${BACKUP_OFFSITE_BACKEND:-none}"
REMOTE="${BACKUP_OFFSITE_REMOTE:-}"
BUCKET="${BACKUP_OFFSITE_BUCKET:-}"
ACCESS_KEY="${BACKUP_OFFSITE_ACCESS_KEY:-}"
SECRET_KEY="${BACKUP_OFFSITE_SECRET_KEY:-}"
REGION="${BACKUP_OFFSITE_REGION:-us-east-1}"
RETENTION_DAYS="${BACKUP_OFFSITE_RETENTION_DAYS:-90}"

echo "=== Offsite Backup Sync ==="
echo "Time:     $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
echo "Backend:  $BACKEND"
echo "Local:    $BACKUP_DIR"
echo "Remote:   $REMOTE"
echo ""

# 检查后端类型
if [ "$BACKEND" = "none" ]; then
  echo "INFO: BACKUP_OFFSITE_BACKEND=none, offsite sync disabled"
  echo "To enable, set BACKUP_OFFSITE_BACKEND to s3, oss, or nfs"
  exit 0
fi

if [ "$BACKEND" != "s3" ] && [ "$BACKEND" != "oss" ] && [ "$BACKEND" != "nfs" ]; then
  echo "ERROR: Invalid backend: $BACKEND (must be s3, oss, nfs, or none)" >&2
  exit 1
fi

# 检查本地备份目录
if [ ! -d "$BACKUP_DIR" ]; then
  echo "ERROR: Local backup directory does not exist: $BACKUP_DIR" >&2
  exit 1
fi

# 统计本地备份文件
LOCAL_FILES=$(ls -1 "$BACKUP_DIR"/db_backup_*.sql.gz 2>/dev/null | wc -l)
if [ "$LOCAL_FILES" -eq 0 ]; then
  echo "ERROR: No backup files found in $BACKUP_DIR" >&2
  exit 1
fi
echo "INFO: Found $LOCAL_FILES local backup files"

# 检查远程配置
if [ -z "$REMOTE" ]; then
  echo "ERROR: BACKUP_OFFSITE_REMOTE not set" >&2
  echo "Example for $BACKEND:" >&2
  case "$BACKEND" in
    s3)  echo "  s3://my-bucket/backups/" >&2 ;;
    oss) echo "  oss://my-bucket/backups/" >&2 ;;
    nfs) echo "  /mnt/nfs/backups/" >&2 ;;
  esac
  exit 1
fi

# 检查工具可用性
check_tool() {
  if ! command -v "$1" >/dev/null 2>&1; then
    echo "ERROR: Required tool not found: $1" >&2
    echo "Please install $1 to use the $BACKEND backend" >&2
    exit 1
  fi
}

# 配置凭证
setup_credentials() {
  case "$BACKEND" in
    s3)
      if [ -n "$ACCESS_KEY" ] && [ -n "$SECRET_KEY" ]; then
        export AWS_ACCESS_KEY_ID="$ACCESS_KEY"
        export AWS_SECRET_ACCESS_KEY="$SECRET_KEY"
        export AWS_DEFAULT_REGION="$REGION"
      fi
      ;;
    oss)
      if [ -n "$ACCESS_KEY" ] && [ -n "$SECRET_KEY" ]; then
        # ossutil 配置
        if [ -f ~/.ossutilconfig ]; then
          cp ~/.ossutilconfig ~/.ossutilconfig.bak 2>/dev/null || true
        fi
        cat > ~/.ossutilconfig <<EOF
[Credentials]
provider = oss
accessKey = $ACCESS_KEY
secretKey = $SECRET_KEY
[Default]
endpoint = oss-${REGION}.aliyuncs.com
EOF
      fi
      ;;
    nfs)
      # NFS 应已挂载,无需凭证
      if [ ! -d "$REMOTE" ]; then
        echo "ERROR: NFS remote directory does not exist: $REMOTE" >&2
        echo "Please ensure NFS is mounted at this path" >&2
        exit 1
      fi
      ;;
  esac
}

# 同步到远程
sync_to_remote() {
  echo ""
  echo "[1/3] Syncing backups to $BACKEND..."
  case "$BACKEND" in
    s3)
      if command -v aws >/dev/null 2>&1; then
        echo "Using aws-cli"
        if ! aws s3 sync "$BACKUP_DIR/" "$REMOTE" \
            --exclude "*" --include "db_backup_*.sql.gz" \
            --no-progress; then
          echo "ERROR: aws s3 sync failed" >&2
          return 1
        fi
      elif command -v rclone >/dev/null 2>&1; then
        echo "Using rclone"
        if ! rclone sync "$BACKUP_DIR" "$REMOTE" \
            --include "db_backup_*.sql.gz" \
            --progress; then
          echo "ERROR: rclone sync failed" >&2
          return 1
        fi
      else
        echo "ERROR: Neither aws-cli nor rclone found" >&2
        return 1
      fi
      ;;
    oss)
      if command -v ossutil >/dev/null 2>&1; then
        echo "Using ossutil"
        if ! ossutil cp -r "$BACKUP_DIR/" "$REMOTE" \
            --include "db_backup_*.sql.gz" -f; then
          echo "ERROR: ossutil sync failed" >&2
          return 1
        fi
      elif command -v rclone >/dev/null 2>&1; then
        echo "Using rclone"
        if ! rclone sync "$BACKUP_DIR" "$REMOTE" \
            --include "db_backup_*.sql.gz" \
            --progress; then
          echo "ERROR: rclone sync failed" >&2
          return 1
        fi
      else
        echo "ERROR: Neither ossutil nor rclone found" >&2
        return 1
      fi
      ;;
    nfs)
      if command -v rsync >/dev/null 2>&1; then
        echo "Using rsync"
        mkdir -p "$REMOTE" 2>/dev/null || true
        if ! rsync -av --include="db_backup_*.sql.gz" --exclude="*" \
            "$BACKUP_DIR/" "$REMOTE/"; then
          echo "ERROR: rsync failed" >&2
          return 1
        fi
      else
        echo "Using cp (rsync not available)"
        mkdir -p "$REMOTE" 2>/dev/null || true
        if ! cp "$BACKUP_DIR"/db_backup_*.sql.gz "$REMOTE/" 2>/dev/null; then
          echo "ERROR: cp failed" >&2
          return 1
        fi
      fi
      ;;
  esac
  echo "  PASS: Sync completed"
  return 0
}

# 校验同步结果
verify_sync() {
  if [ "$NO_VERIFY" -eq 1 ]; then
    echo ""
    echo "[2/3] Verification skipped (--no-verify)"
    return 0
  fi
  echo ""
  echo "[2/3] Verifying sync result..."
  REMOTE_FILES=0
  case "$BACKEND" in
    s3)
      if command -v aws >/dev/null 2>&1; then
        REMOTE_FILES=$(aws s3 ls "$REMOTE" --recursive 2>/dev/null | grep -c "db_backup_.*\.sql\.gz" || echo 0)
      elif command -v rclone >/dev/null 2>&1; then
        REMOTE_FILES=$(rclone lsf "$REMOTE" --include "db_backup_*.sql.gz" 2>/dev/null | wc -l || echo 0)
      fi
      ;;
    oss)
      if command -v ossutil >/dev/null 2>&1; then
        REMOTE_FILES=$(ossutil ls "$REMOTE" 2>/dev/null | grep -c "db_backup_.*\.sql\.gz" || echo 0)
      elif command -v rclone >/dev/null 2>&1; then
        REMOTE_FILES=$(rclone lsf "$REMOTE" --include "db_backup_*.sql.gz" 2>/dev/null | wc -l || echo 0)
      fi
      ;;
    nfs)
      REMOTE_FILES=$(ls -1 "$REMOTE"/db_backup_*.sql.gz 2>/dev/null | wc -l)
      ;;
  esac

  echo "  Local files:  $LOCAL_FILES"
  echo "  Remote files: $REMOTE_FILES"

  if [ "$REMOTE_FILES" -lt "$LOCAL_FILES" ]; then
    echo "  WARN: Remote has fewer files than local (some may have been cleaned up)"
  else
    echo "  PASS: File count verified"
  fi
  return 0
}

# 清理远程过期备份
cleanup_remote() {
  if [ "$NO_CLEANUP" -eq 1 ]; then
    echo ""
    echo "[3/3] Cleanup skipped (--no-cleanup)"
    return 0
  fi
  echo ""
  echo "[3/3] Cleaning up remote backups older than $RETENTION_DAYS days..."
  case "$BACKEND" in
    s3)
      if command -v aws >/dev/null 2>&1; then
        aws s3 ls "$REMOTE" --recursive 2>/dev/null | grep "db_backup_.*\.sql\.gz" | while read -r line; do
          FILE_PATH=$(echo "$line" | awk '{print $4}')
          FILE_DATE=$(echo "$FILE_PATH" | grep -oE '[0-9]{8}_[0-9]{6}' | head -1)
          if [ -n "$FILE_DATE" ]; then
            FILE_TS=$(echo "$FILE_DATE" | sed 's/\([0-9]\{8\}\)_\([0-9]\{6\}\)/\1 \2/' | awk '{print $1}')
            CUTOFF=$(date -d "-$RETENTION_DAYS days" +%Y%m%d 2>/dev/null || date -v-${RETENTION_DAYS}d +%Y%m%d 2>/dev/null)
            if [ -n "$CUTOFF" ] && [ "$FILE_TS" -lt "$CUTOFF" ]; then
              echo "  Deleting: $FILE_PATH"
              aws s3 rm "s3://$(echo "$REMOTE" | sed 's|s3://||')/$FILE_PATH" 2>/dev/null || true
            fi
          fi
        done
      fi
      ;;
    oss)
      if command -v ossutil >/dev/null 2>&1; then
        # ossutil 不支持基于时间的清理,使用生命周期规则或手动删除
        echo "  INFO: For OSS, configure lifecycle rules in the console for automatic cleanup"
        echo "  INFO: Manual cleanup with retention $RETENTION_DAYS days"
      fi
      ;;
    nfs)
      if [ -d "$REMOTE" ]; then
        find "$REMOTE" -name "db_backup_*.sql.gz" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true
        echo "  Cleaned up files older than $RETENTION_DAYS days"
      fi
      ;;
  esac
  echo "  PASS: Cleanup completed"
  return 0
}

# 执行同步流程
setup_credentials
if ! sync_to_remote; then
  exit 1
fi
verify_sync
cleanup_remote

echo ""
echo "=== Offsite Sync Complete ==="
exit 0