e9ea34fe53
P6 生产硬化阶段交付物（46 文件）：
## 1. API Gateway 中间件链（services/api-gateway/internal/middleware/）
- circuit-breaker.go: gobreaker v2 熔断器（5s 窗口/50% 错误率/30s OPEN→HALF_OPEN）
- ratelimit.go: 令牌桶限流（sync.Map + cleanup goroutine，默认 100rps/20 burst）
- cors.go: CORS 中间件（CORS_ORIGINS 环境变量）
- recovery.go: panic 恢复 + uuid request_id
- security.go: 安全头 + 请求体 10MB 限制
- requestid.go: 请求 ID 注入
- health/health.go: /healthz + /readyz 健康检查
- main.go: 重写注册全部中间件链（Recovery→RequestID→CORS→Security→BodyLimit→RateLimit→CircuitBreaker→Auth）
## 2. 基础设施硬化（infra/）
- backup/backup-mysql.sh: MySQL 全量备份（mysqldump+gzip，按服务独立）
- backup/restore-mysql.sh: 恢复脚本
- backup/backup-cron.sh: cron 调度入口（5 服务批量备份）
- alertmanager/alertmanager.yml: 告警路由（webhook + 邮件示例）
- prometheus/rules.yml: 8 条告警规则（服务可用性/性能/资源 3 组）
- grafana/dashboards/microservices-overview.json: 4 panel 仪表盘
- grafana/provisioning/: 数据源和仪表盘 provisioning
- k8s/namespace.yaml: 4 命名空间（edu-system/services/monitoring/ingress）
- k8s/api-gateway-deployment.yaml: Deployment + Service 骨架
- chaos/experiments.yaml: 3 个 Litmus 混沌实验（pod-kill/network-latency/disk-fill）
- docker-compose.monitoring.yml: 监控栈 profile
- security/secrets.example.env: 8 项密钥占位符
- security/waf-rules.conf: ModSecurity WAF 规则骨架
## 3. 业务服务健康检查 + 优雅停机（5 个 NestJS 服务）
- services/{iam,core-edu,content,msg,classes}/src/shared/health/: /healthz + /readyz
- services/{iam,core-edu,content,msg,classes}/src/shared/lifecycle/: OnModuleInit + OnApplicationShutdown
## 4. Python 服务健康检查
- services/{ai,data-ana}/src/health/health.py: FastAPI APIRouter
## 5. 运维文档
- docs/architecture/runbooks/p6-hardening.md: P6 总览 Runbook（9 章节）
- docs/architecture/runbooks/incident-response.md: 事件响应手册（5 章节）
- docs/architecture/004-p6-addendum.md: 004 架构补记 P6 章节
- docs/troubleshooting/known-issues-p6-addendum.md: 15 条 P6 场景→技术映射
## 验收信号
- RPO ≤ 15min（MySQL 备份 + binlog PITR）
- RTO ≤ 30min（K8s 滚动更新 + DNS 切换）
- P99 ≤ 500ms（熔断 + 限流 + 缓存）
- 熔断器错误率 > 50% 触发 OPEN
- 限流 100rps/20 burst
- 备份保留 7 天
- 混沌实验每月 1 次
100 lines
2.5 KiB
Bash
100 lines
2.5 KiB
Bash
#!/bin/bash
|
||
# MySQL 全量备份脚本（Linux 容器内运行）
|
||
# 用法: backup-mysql.sh --service <name> [--keep <days>]
|
||
set -euo pipefail
|
||
|
||
# ---------- 参数解析 ----------
|
||
SERVICE=""
|
||
KEEP_DAYS=7
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--service)
|
||
SERVICE="${2:-}"
|
||
shift 2
|
||
;;
|
||
--keep)
|
||
KEEP_DAYS="${2:-7}"
|
||
shift 2
|
||
;;
|
||
-h|--help)
|
||
echo "Usage: $0 --service <name> [--keep <days>]" >&2
|
||
echo " --service 目标微服务数据库名（必填，如 iam/core-edu/content/msg/classes）" >&2
|
||
echo " --keep 保留天数（默认 7）" >&2
|
||
exit 0
|
||
;;
|
||
*)
|
||
echo "[ERROR] 未知参数: $1" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
done
|
||
|
||
# ---------- 参数校验 ----------
|
||
if [[ -z "$SERVICE" ]]; then
|
||
echo "[ERROR] --service 参数必填" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if ! [[ "$KEEP_DAYS" =~ ^[0-9]+$ ]] || [[ "$KEEP_DAYS" -lt 1 ]]; then
|
||
echo "[ERROR] --keep 必须为正整数" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# ---------- 环境变量 ----------
|
||
: "${MYSQL_HOST:=mysql}"
|
||
: "${MYSQL_PORT:=3306}"
|
||
: "${MYSQL_USER:=root}"
|
||
: "${MYSQL_PASSWORD:?MYSQL_PASSWORD 环境变量未设置}"
|
||
|
||
BACKUP_ROOT="/backups"
|
||
BACKUP_DIR="${BACKUP_ROOT}/${SERVICE}"
|
||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||
BACKUP_FILE="${BACKUP_DIR}/${TIMESTAMP}.sql.gz"
|
||
|
||
# ---------- 日志函数 ----------
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
|
||
}
|
||
|
||
# ---------- 主流程 ----------
|
||
log "[INFO] 开始备份服务: ${SERVICE} (保留 ${KEEP_DAYS} 天)"
|
||
|
||
mkdir -p "$BACKUP_DIR"
|
||
|
||
log "[INFO] 执行 mysqldump -> ${BACKUP_FILE}"
|
||
if ! mysqldump \
|
||
--host="$MYSQL_HOST" \
|
||
--port="$MYSQL_PORT" \
|
||
--user="$MYSQL_USER" \
|
||
--password="$MYSQL_PASSWORD" \
|
||
--single-transaction \
|
||
--routines \
|
||
--triggers \
|
||
--databases "$SERVICE" \
|
||
2>/dev/null \
|
||
| gzip -c > "$BACKUP_FILE"; then
|
||
log "[ERROR] mysqldump 失败 (service=${SERVICE})"
|
||
rm -f "$BACKUP_FILE"
|
||
exit 1
|
||
fi
|
||
|
||
# 校验产物非空
|
||
if [[ ! -s "$BACKUP_FILE" ]]; then
|
||
log "[ERROR] 备份文件为空: ${BACKUP_FILE}"
|
||
exit 1
|
||
fi
|
||
|
||
FILE_SIZE="$(stat -c %s "$BACKUP_FILE" 2>/dev/null || stat -f %z "$BACKUP_FILE")"
|
||
log "[INFO] 备份完成: ${BACKUP_FILE} (${FILE_SIZE} bytes)"
|
||
|
||
# ---------- 清理过期备份 ----------
|
||
log "[INFO] 清理超过 ${KEEP_DAYS} 天的旧备份"
|
||
find "$BACKUP_DIR" -type f -name '*.sql.gz' -mtime +${KEEP_DAYS} -print -delete \
|
||
| while read -r old_file; do
|
||
log "[INFO] 已删除: ${old_file}"
|
||
done
|
||
|
||
log "[INFO] 备份流程结束: ${SERVICE}"
|
||
exit 0
|