feat(p6): production hardening with circuit breaker, backup, monitoring and chaos engineering
P6 生产硬化阶段交付物(46 文件):
## 1. API Gateway 中间件链(services/api-gateway/internal/middleware/)
- circuit-breaker.go: gobreaker v2 熔断器(5s 窗口/50% 错误率/30s OPEN→HALF_OPEN)
- ratelimit.go: 令牌桶限流(sync.Map + cleanup goroutine,默认 100rps/20 burst)
- cors.go: CORS 中间件(CORS_ORIGINS 环境变量)
- recovery.go: panic 恢复 + uuid request_id
- security.go: 安全头 + 请求体 10MB 限制
- requestid.go: 请求 ID 注入
- health/health.go: /healthz + /readyz 健康检查
- main.go: 重写注册全部中间件链(Recovery→RequestID→CORS→Security→BodyLimit→RateLimit→CircuitBreaker→Auth)
## 2. 基础设施硬化(infra/)
- backup/backup-mysql.sh: MySQL 全量备份(mysqldump+gzip,按服务独立)
- backup/restore-mysql.sh: 恢复脚本
- backup/backup-cron.sh: cron 调度入口(5 服务批量备份)
- alertmanager/alertmanager.yml: 告警路由(webhook + 邮件示例)
- prometheus/rules.yml: 8 条告警规则(服务可用性/性能/资源 3 组)
- grafana/dashboards/microservices-overview.json: 4 panel 仪表盘
- grafana/provisioning/: 数据源和仪表盘 provisioning
- k8s/namespace.yaml: 4 命名空间(edu-system/services/monitoring/ingress)
- k8s/api-gateway-deployment.yaml: Deployment + Service 骨架
- chaos/experiments.yaml: 3 个 Litmus 混沌实验(pod-kill/network-latency/disk-fill)
- docker-compose.monitoring.yml: 监控栈 profile
- security/secrets.example.env: 8 项密钥占位符
- security/waf-rules.conf: ModSecurity WAF 规则骨架
## 3. 业务服务健康检查 + 优雅停机(5 个 NestJS 服务)
- services/{iam,core-edu,content,msg,classes}/src/shared/health/: /healthz + /readyz
- services/{iam,core-edu,content,msg,classes}/src/shared/lifecycle/: OnModuleInit + OnApplicationShutdown
## 4. Python 服务健康检查
- services/{ai,data-ana}/src/health/health.py: FastAPI APIRouter
## 5. 运维文档
- docs/architecture/runbooks/p6-hardening.md: P6 总览 Runbook(9 章节)
- docs/architecture/runbooks/incident-response.md: 事件响应手册(5 章节)
- docs/architecture/004-p6-addendum.md: 004 架构补记 P6 章节
- docs/troubleshooting/known-issues-p6-addendum.md: 15 条 P6 场景→技术映射
## 验收信号
- RPO ≤ 15min(MySQL 备份 + binlog PITR)
- RTO ≤ 30min(K8s 滚动更新 + DNS 切换)
- P99 ≤ 500ms(熔断 + 限流 + 缓存)
- 熔断器错误率 > 50% 触发 OPEN
- 限流 100rps/20 burst
- 备份保留 7 天
- 混沌实验每月 1 次
This commit is contained in:
69
infra/alertmanager/alertmanager.yml
Normal file
69
infra/alertmanager/alertmanager.yml
Normal file
@@ -0,0 +1,69 @@
|
||||
# Alertmanager 配置 - Edu 平台
|
||||
# 全局配置
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
# 告警模板(可扩展)
|
||||
templates:
|
||||
- /etc/alertmanager/templates/*.tmpl
|
||||
|
||||
# 路由树
|
||||
route:
|
||||
# 顶层默认接收器
|
||||
receiver: webhook-default
|
||||
# 按 alertname + service 分组
|
||||
group_by: ['alertname', 'service']
|
||||
# 首次告警等待时间(聚合相同组)
|
||||
group_wait: 30s
|
||||
# 同组新告警发送间隔
|
||||
group_interval: 5m
|
||||
# 重复告警发送间隔
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
# critical 告警走 webhook
|
||||
- matchers:
|
||||
- severity="critical"
|
||||
receiver: webhook-default
|
||||
continue: false
|
||||
# warning 告警走 webhook
|
||||
- matchers:
|
||||
- severity="warning"
|
||||
receiver: webhook-default
|
||||
continue: false
|
||||
|
||||
# 接收器列表
|
||||
receivers:
|
||||
# 默认 Webhook 接收器(指向内部告警路由服务)
|
||||
- name: webhook-default
|
||||
webhook_configs:
|
||||
- url: http://alert-router:5001/alert
|
||||
send_resolved: true
|
||||
max_alerts: 0
|
||||
|
||||
# 邮件接收器(注释示例,留作扩展)
|
||||
# 启用前需在 global.smtp_* 配置 SMTP 服务器
|
||||
# - name: mail-ops
|
||||
# email_configs:
|
||||
# - to: ops-team@example.com
|
||||
# from: alertmanager@example.com
|
||||
# smarthost: smtp.example.com:587
|
||||
# auth_username: alertmanager@example.com
|
||||
# auth_password: <SMTP_PASSWORD>
|
||||
# require_tls: true
|
||||
# headers:
|
||||
# Subject: '[Edu Alert] {{ .GroupLabels.alertname }}'
|
||||
|
||||
# 钉钉/企业微信接收器(注释示例,留作扩展)
|
||||
# - name: dingtalk-ops
|
||||
# webhook_configs:
|
||||
# - url: http://dingtalk-webhook:8060/dingtalk/ops/send
|
||||
# send_resolved: true
|
||||
|
||||
# 抑制规则:critical 告警抑制同服务同名的 warning 告警
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity="critical"
|
||||
target_matchers:
|
||||
- severity="warning"
|
||||
# 相同 alertname + service 才抑制
|
||||
equal: ['alertname', 'service']
|
||||
74
infra/backup/backup-cron.sh
Normal file
74
infra/backup/backup-cron.sh
Normal file
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# MySQL 备份 cron 调度入口
|
||||
# 调用 backup-mysql.sh 备份所有微服务数据库,单个失败不阻塞其他
|
||||
# 用法: backup-cron.sh [--keep <days>]
|
||||
set -euo pipefail
|
||||
|
||||
# ---------- 参数解析 ----------
|
||||
KEEP_DAYS=7
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--keep)
|
||||
KEEP_DAYS="${2:-7}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: $0 [--keep <days>]" >&2
|
||||
echo " --keep 保留天数(默认 7)" >&2
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "[ERROR] 未知参数: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------- 配置 ----------
|
||||
# 每个服务对应一个独立数据库
|
||||
SERVICES=(iam core-edu content msg classes)
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BACKUP_SCRIPT="${SCRIPT_DIR}/backup-mysql.sh"
|
||||
|
||||
# ---------- 日志函数 ----------
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
|
||||
}
|
||||
|
||||
if [[ ! -x "$BACKUP_SCRIPT" ]]; then
|
||||
log "[ERROR] 备份脚本不存在或不可执行: ${BACKUP_SCRIPT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------- 主流程 ----------
|
||||
log "[INFO] 开始批量备份,共 ${#SERVICES[@]} 个服务"
|
||||
|
||||
TOTAL=0
|
||||
SUCCESS=0
|
||||
FAILED=0
|
||||
FAILED_LIST=()
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
TOTAL=$((TOTAL + 1))
|
||||
log "[INFO] ---- 备份服务: ${svc} ----"
|
||||
if "$BACKUP_SCRIPT" --service "$svc" --keep "$KEEP_DAYS"; then
|
||||
SUCCESS=$((SUCCESS + 1))
|
||||
log "[INFO] 服务 ${svc} 备份成功"
|
||||
else
|
||||
FAILED=$((FAILED + 1))
|
||||
FAILED_LIST+=("$svc")
|
||||
log "[ERROR] 服务 ${svc} 备份失败,继续下一个"
|
||||
fi
|
||||
done
|
||||
|
||||
# ---------- 汇总 ----------
|
||||
log "[INFO] 批量备份结束: 总计=${TOTAL} 成功=${SUCCESS} 失败=${FAILED}"
|
||||
|
||||
if [[ ${FAILED} -gt 0 ]]; then
|
||||
log "[ERROR] 失败服务列表: ${FAILED_LIST[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
99
infra/backup/backup-mysql.sh
Normal file
99
infra/backup/backup-mysql.sh
Normal file
@@ -0,0 +1,99 @@
|
||||
#!/bin/bash
|
||||
# MySQL 全量备份脚本(Linux 容器内运行)
|
||||
# 用法: backup-mysql.sh --service <name> [--keep <days>]
|
||||
set -euo pipefail
|
||||
|
||||
# ---------- 参数解析 ----------
|
||||
SERVICE=""
|
||||
KEEP_DAYS=7
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--service)
|
||||
SERVICE="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep)
|
||||
KEEP_DAYS="${2:-7}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: $0 --service <name> [--keep <days>]" >&2
|
||||
echo " --service 目标微服务数据库名(必填,如 iam/core-edu/content/msg/classes)" >&2
|
||||
echo " --keep 保留天数(默认 7)" >&2
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "[ERROR] 未知参数: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------- 参数校验 ----------
|
||||
if [[ -z "$SERVICE" ]]; then
|
||||
echo "[ERROR] --service 参数必填" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$KEEP_DAYS" =~ ^[0-9]+$ ]] || [[ "$KEEP_DAYS" -lt 1 ]]; then
|
||||
echo "[ERROR] --keep 必须为正整数" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------- 环境变量 ----------
|
||||
: "${MYSQL_HOST:=mysql}"
|
||||
: "${MYSQL_PORT:=3306}"
|
||||
: "${MYSQL_USER:=root}"
|
||||
: "${MYSQL_PASSWORD:?MYSQL_PASSWORD 环境变量未设置}"
|
||||
|
||||
BACKUP_ROOT="/backups"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/${SERVICE}"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_FILE="${BACKUP_DIR}/${TIMESTAMP}.sql.gz"
|
||||
|
||||
# ---------- 日志函数 ----------
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
|
||||
}
|
||||
|
||||
# ---------- 主流程 ----------
|
||||
log "[INFO] 开始备份服务: ${SERVICE} (保留 ${KEEP_DAYS} 天)"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
log "[INFO] 执行 mysqldump -> ${BACKUP_FILE}"
|
||||
if ! mysqldump \
|
||||
--host="$MYSQL_HOST" \
|
||||
--port="$MYSQL_PORT" \
|
||||
--user="$MYSQL_USER" \
|
||||
--password="$MYSQL_PASSWORD" \
|
||||
--single-transaction \
|
||||
--routines \
|
||||
--triggers \
|
||||
--databases "$SERVICE" \
|
||||
2>/dev/null \
|
||||
| gzip -c > "$BACKUP_FILE"; then
|
||||
log "[ERROR] mysqldump 失败 (service=${SERVICE})"
|
||||
rm -f "$BACKUP_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 校验产物非空
|
||||
if [[ ! -s "$BACKUP_FILE" ]]; then
|
||||
log "[ERROR] 备份文件为空: ${BACKUP_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILE_SIZE="$(stat -c %s "$BACKUP_FILE" 2>/dev/null || stat -f %z "$BACKUP_FILE")"
|
||||
log "[INFO] 备份完成: ${BACKUP_FILE} (${FILE_SIZE} bytes)"
|
||||
|
||||
# ---------- 清理过期备份 ----------
|
||||
log "[INFO] 清理超过 ${KEEP_DAYS} 天的旧备份"
|
||||
find "$BACKUP_DIR" -type f -name '*.sql.gz' -mtime +${KEEP_DAYS} -print -delete \
|
||||
| while read -r old_file; do
|
||||
log "[INFO] 已删除: ${old_file}"
|
||||
done
|
||||
|
||||
log "[INFO] 备份流程结束: ${SERVICE}"
|
||||
exit 0
|
||||
94
infra/backup/restore-mysql.sh
Normal file
94
infra/backup/restore-mysql.sh
Normal file
@@ -0,0 +1,94 @@
|
||||
#!/bin/bash
|
||||
# MySQL 恢复脚本(Linux 容器内运行)
|
||||
# 用法: restore-mysql.sh --service <name> --file <path> [--yes]
|
||||
set -euo pipefail
|
||||
|
||||
# ---------- 参数解析 ----------
|
||||
SERVICE=""
|
||||
BACKUP_FILE=""
|
||||
ASSUME_YES=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--service)
|
||||
SERVICE="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--file)
|
||||
BACKUP_FILE="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--yes)
|
||||
ASSUME_YES=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: $0 --service <name> --file <path> [--yes]" >&2
|
||||
echo " --service 目标微服务数据库名(必填)" >&2
|
||||
echo " --file 备份文件路径 sql.gz(必填)" >&2
|
||||
echo " --yes 跳过确认提示" >&2
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "[ERROR] 未知参数: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------- 参数校验 ----------
|
||||
if [[ -z "$SERVICE" ]]; then
|
||||
echo "[ERROR] --service 参数必填" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$BACKUP_FILE" ]]; then
|
||||
echo "[ERROR] --file 参数必填" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$BACKUP_FILE" ]]; then
|
||||
echo "[ERROR] 备份文件不存在: ${BACKUP_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------- 环境变量 ----------
|
||||
: "${MYSQL_HOST:=mysql}"
|
||||
: "${MYSQL_PORT:=3306}"
|
||||
: "${MYSQL_USER:=root}"
|
||||
: "${MYSQL_PASSWORD:?MYSQL_PASSWORD 环境变量未设置}"
|
||||
|
||||
# ---------- 日志函数 ----------
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
|
||||
}
|
||||
|
||||
# ---------- 确认提示 ----------
|
||||
log "[WARN] 即将向数据库 [${SERVICE}] 导入备份文件: ${BACKUP_FILE}"
|
||||
log "[WARN] 此操作会覆盖目标数据库现有数据,不可撤销!"
|
||||
|
||||
if [[ "$ASSUME_YES" != "true" ]]; then
|
||||
read -r -p "确认继续? (输入 YES 继续): " confirm
|
||||
if [[ "$confirm" != "YES" ]]; then
|
||||
log "[INFO] 用户取消操作"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------- 主流程 ----------
|
||||
log "[INFO] 开始恢复服务: ${SERVICE}"
|
||||
|
||||
log "[INFO] 解压并导入: ${BACKUP_FILE}"
|
||||
if ! gunzip -c "$BACKUP_FILE" \
|
||||
| mysql \
|
||||
--host="$MYSQL_HOST" \
|
||||
--port="$MYSQL_PORT" \
|
||||
--user="$MYSQL_USER" \
|
||||
--password="$MYSQL_PASSWORD" \
|
||||
"$SERVICE"; then
|
||||
log "[ERROR] mysql 导入失败 (service=${SERVICE})"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "[INFO] 恢复完成: ${SERVICE} <- ${BACKUP_FILE}"
|
||||
exit 0
|
||||
45
infra/chaos/README.md
Normal file
45
infra/chaos/README.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# 混沌工程说明
|
||||
|
||||
## 适用范围
|
||||
|
||||
- **仅 Staging 环境**:禁止在生产环境运行混沌实验
|
||||
- **每月 1 次演练**:建议每月第一个工作周执行
|
||||
- **演练窗口**:业务低峰期(02:00 - 05:00)
|
||||
|
||||
## 实验清单
|
||||
|
||||
| 实验 | 注入故障 | 验证目标 |
|
||||
|------|----------|----------|
|
||||
| `pod-delete` | 杀死 50% Pod | 自愈能力、副本数冗余 |
|
||||
| `pod-network-latency` | 注入 200ms 延迟 | 超时与重试、降级策略 |
|
||||
| `disk-fill` | 填充磁盘至 80% | 日志写入、磁盘告警 |
|
||||
|
||||
## 前置条件
|
||||
|
||||
1. Staging 环境已部署完整监控(Prometheus + Alertmanager + Grafana)
|
||||
2. 已配置稳态探针(`steadyStateHypothesis`)
|
||||
3. 已配置告警通道并验证可达
|
||||
4. 演练参与者已就位(运维 + 研发 oncall)
|
||||
|
||||
## 演练流程
|
||||
|
||||
1. **演练前 30 分钟**:通知相关人员,确认稳态基线
|
||||
2. **启动实验**:按顺序执行,单实验单次注入
|
||||
3. **观察监控**:关注告警是否按预期触发、服务是否自愈
|
||||
4. **演练后**:归档实验结果,更新本目录 README
|
||||
|
||||
## 回滚预案
|
||||
|
||||
- **5 分钟内未恢复自动回滚**:通过 Litmus `steadyStateHypothesis` 失败触发实验终止
|
||||
- 手动回滚:
|
||||
```bash
|
||||
kubectl delete chaosengine edu-chaos-experiments -n edu-monitoring
|
||||
kubectl rollout restart deployment/api-gateway -n edu-services
|
||||
```
|
||||
- 极端情况:直接缩容 / 扩容受影响 Deployment
|
||||
|
||||
## 安全约束
|
||||
|
||||
- 每次实验**只注入一种故障**,避免叠加影响判断
|
||||
- 实验前快照数据库(参考 `infra/backup/backup-cron.sh`)
|
||||
- 实验期间禁止发布任何业务变更
|
||||
139
infra/chaos/experiments.yaml
Normal file
139
infra/chaos/experiments.yaml
Normal file
@@ -0,0 +1,139 @@
|
||||
# 混沌实验配置(Litmus Chaos 骨架)
|
||||
# 仅作 YAML 骨架,注释说明用途。实际运行需配合 Litmus Chaos Operator。
|
||||
# 文档:https://litmuschaos.github.io/litmus/
|
||||
apiVersion: litmuschaos.io/v1alpha1
|
||||
kind: ChaosEngine
|
||||
metadata:
|
||||
name: edu-chaos-experiments
|
||||
namespace: edu-monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: chaos
|
||||
spec:
|
||||
appinfo:
|
||||
appns: edu-services
|
||||
applabel: "app.kubernetes.io/name=api-gateway"
|
||||
appkind: deployment
|
||||
chaosServiceAccount: litmus-chaos-sa
|
||||
components:
|
||||
experiments:
|
||||
# ============================================================
|
||||
# 实验 1:服务 Pod 杀死(验证自愈与副本数)
|
||||
# ============================================================
|
||||
- name: pod-delete
|
||||
spec:
|
||||
components:
|
||||
env:
|
||||
# 杀死 1 个 Pod
|
||||
- name: TOTAL_CHAOS_DURATION
|
||||
value: "30"
|
||||
- name: CHAOS_INTERVAL
|
||||
value: "10"
|
||||
- name: FORCE
|
||||
value: "false"
|
||||
- name: PODS_AFFECTED_PERC
|
||||
value: "50"
|
||||
- name: TARGET_CONTAINER
|
||||
value: "api-gateway"
|
||||
# 假设:杀死 50% Pod 后,服务仍可对外可用(最少 1 个副本健康)
|
||||
# 稳态:up{job="api-gateway"} >= 1
|
||||
steadyStateHypothesis:
|
||||
steadyStateHypothesis:
|
||||
probe:
|
||||
- name: api-gateway-still-up
|
||||
type: httpProbe
|
||||
mode: Continuous
|
||||
runProperties:
|
||||
probeTimeout: 5
|
||||
httpProbeInputs:
|
||||
url: http://api-gateway.edu-services.svc:8080/healthz
|
||||
method:
|
||||
get: {}
|
||||
phases:
|
||||
- name: pre-chaos
|
||||
description: "混沌前稳态验证"
|
||||
- name: inject
|
||||
description: "注入 Pod 删除"
|
||||
- name: post-chaos
|
||||
description: "混沌后自愈验证"
|
||||
# ============================================================
|
||||
# 实验 2:网络延迟注入(验证超时与重试)
|
||||
# ============================================================
|
||||
- name: pod-network-latency
|
||||
spec:
|
||||
components:
|
||||
env:
|
||||
# 注入 200ms 网络延迟
|
||||
- name: NETWORK_LATENCY
|
||||
value: "200"
|
||||
- name: TOTAL_CHAOS_DURATION
|
||||
value: "60"
|
||||
- name: CHAOS_INTERVAL
|
||||
value: "10"
|
||||
- name: NETWORK_INTERFACE
|
||||
value: "eth0"
|
||||
- name: TARGET_CONTAINER
|
||||
value: "api-gateway"
|
||||
# 假设:200ms 延迟下 P99 < 2s,无 5xx 雪崩
|
||||
# 稳态:错误率 < 5%
|
||||
steadyStateHypothesis:
|
||||
steadyStateHypothesis:
|
||||
probe:
|
||||
- name: error-rate-below-threshold
|
||||
type: promProbe
|
||||
mode: Continuous
|
||||
runProperties:
|
||||
probeTimeout: 5
|
||||
promProbeInputs:
|
||||
source:
|
||||
url: http://prometheus.edu-monitoring.svc:9090
|
||||
query: |
|
||||
sum(rate(http_requests_total{service="api-gateway",status=~"5.."}[1m]))
|
||||
/ sum(rate(http_requests_total{service="api-gateway"}[1m]))
|
||||
comparator:
|
||||
criteria: "<="
|
||||
value: "0.05"
|
||||
phases:
|
||||
- name: pre-chaos
|
||||
description: "混沌前稳态验证"
|
||||
- name: inject
|
||||
description: "注入 200ms 网络延迟"
|
||||
- name: post-chaos
|
||||
description: "混沌后恢复验证"
|
||||
# ============================================================
|
||||
# 实验 3:磁盘填充(验证磁盘压力下日志写入与告警)
|
||||
# ============================================================
|
||||
- name: disk-fill
|
||||
spec:
|
||||
components:
|
||||
env:
|
||||
# 填充至 80% 磁盘使用率
|
||||
- name: FILL_PERCENTAGE
|
||||
value: "80"
|
||||
- name: TOTAL_CHAOS_DURATION
|
||||
value: "120"
|
||||
- name: CHAOS_INTERVAL
|
||||
value: "30"
|
||||
- name: TARGET_CONTAINER
|
||||
value: "api-gateway"
|
||||
# 假设:磁盘 80% 使用率下服务仍可写入日志,触发 DiskSpaceLow 告警
|
||||
# 稳态:服务 /healthz 可用
|
||||
steadyStateHypothesis:
|
||||
steadyStateHypothesis:
|
||||
probe:
|
||||
- name: api-gateway-healthz
|
||||
type: httpProbe
|
||||
mode: Continuous
|
||||
runProperties:
|
||||
probeTimeout: 5
|
||||
httpProbeInputs:
|
||||
url: http://api-gateway.edu-services.svc:8080/healthz
|
||||
method:
|
||||
get: {}
|
||||
phases:
|
||||
- name: pre-chaos
|
||||
description: "混沌前稳态验证"
|
||||
- name: inject
|
||||
description: "填充磁盘至 80%"
|
||||
- name: post-chaos
|
||||
description: "混沌后清理与恢复验证"
|
||||
119
infra/docker-compose.monitoring.yml
Normal file
119
infra/docker-compose.monitoring.yml
Normal file
@@ -0,0 +1,119 @@
|
||||
# 监控栈 docker-compose(独立 profile)
|
||||
# 使用方式: docker compose -f docker-compose.monitoring.yml --profile monitoring up -d
|
||||
version: "3.9"
|
||||
|
||||
networks:
|
||||
edu-network:
|
||||
name: edu-network
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
name: edu-prometheus-data
|
||||
alertmanager-data:
|
||||
name: edu-alertmanager-data
|
||||
grafana-data:
|
||||
name: edu-grafana-data
|
||||
|
||||
services:
|
||||
# ============================================================
|
||||
# Prometheus - 指标采集与存储
|
||||
# ============================================================
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.54.1
|
||||
container_name: edu-prometheus
|
||||
profiles: ["monitoring"]
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=15d"
|
||||
- "--web.enable-lifecycle"
|
||||
- "--web.enable-admin-api"
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/rules.yml:/etc/prometheus/rules.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
networks:
|
||||
- edu-network
|
||||
|
||||
# ============================================================
|
||||
# Alertmanager - 告警路由与抑制
|
||||
# ============================================================
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.27.0
|
||||
container_name: edu-alertmanager
|
||||
profiles: ["monitoring"]
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/alertmanager/alertmanager.yml"
|
||||
- "--storage.path=/alertmanager"
|
||||
ports:
|
||||
- "9093:9093"
|
||||
volumes:
|
||||
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- alertmanager-data:/alertmanager
|
||||
networks:
|
||||
- edu-network
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
# ============================================================
|
||||
# Grafana - 可视化
|
||||
# ============================================================
|
||||
grafana:
|
||||
image: grafana/grafana:11.2.2
|
||||
container_name: edu-grafana
|
||||
profiles: ["monitoring"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
networks:
|
||||
- edu-network
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
# ============================================================
|
||||
# node-exporter - 主机指标采集
|
||||
# ============================================================
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.8.2
|
||||
container_name: edu-node-exporter
|
||||
profiles: ["monitoring"]
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--path.rootfs=/host"
|
||||
ports:
|
||||
- "9100:9100"
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/host:ro
|
||||
networks:
|
||||
- edu-network
|
||||
|
||||
# ============================================================
|
||||
# blackbox-exporter - 黑盒探测(HTTP / TCP / ICMP)
|
||||
# ============================================================
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.25.0
|
||||
container_name: edu-blackbox-exporter
|
||||
profiles: ["monitoring"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9115:9115"
|
||||
volumes:
|
||||
- ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
|
||||
networks:
|
||||
- edu-network
|
||||
394
infra/grafana/dashboards/microservices-overview.json
Normal file
394
infra/grafana/dashboards/microservices-overview.json
Normal file
@@ -0,0 +1,394 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Edu 平台微服务总览 - QPS / 错误率 / P99 延迟 / 熔断器状态",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": ["mean", "max"],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(http_requests_total[1m])) by (service)",
|
||||
"legendFormat": "{{service}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "请求 QPS(按服务)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "line"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.05
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": ["mean", "max"],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
|
||||
"legendFormat": "{{service}} 5xx",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "错误率(5xx 占比)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "line"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": ["mean", "max"],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))",
|
||||
"legendFormat": "{{service}} P99",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "P99 延迟",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"options": {
|
||||
"0": {
|
||||
"color": "green",
|
||||
"index": 0,
|
||||
"text": "CLOSED"
|
||||
},
|
||||
"1": {
|
||||
"color": "red",
|
||||
"index": 1,
|
||||
"text": "OPEN"
|
||||
}
|
||||
},
|
||||
"type": "value"
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "max(circuit_breaker_state{state=\"open\"}) by (service)",
|
||||
"legendFormat": "{{service}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "熔断器状态(0=CLOSED, 1=OPEN)",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": ["edu", "microservices", "overview"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Edu 微服务总览",
|
||||
"uid": "edu-microservices-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
15
infra/grafana/provisioning/dashboards/dashboards.yml
Normal file
15
infra/grafana/provisioning/dashboards/dashboards.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
# Grafana 仪表盘 provisioning
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: edu-dashboards
|
||||
orgId: 1
|
||||
folder: "Edu"
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
# 从该目录加载所有仪表盘 JSON
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
15
infra/grafana/provisioning/datasources/prometheus.yml
Normal file
15
infra/grafana/provisioning/datasources/prometheus.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
# Grafana 数据源 provisioning
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: "15s"
|
||||
httpMethod: POST
|
||||
manageAlerts: false
|
||||
55
infra/k8s/README.md
Normal file
55
infra/k8s/README.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# K8s 部署说明(骨架)
|
||||
|
||||
> **说明**:本目录为 K8s 部署**骨架**,仅提供最小可用 manifest,**生产环境请使用 Helm Chart**。
|
||||
|
||||
## 适用范围
|
||||
|
||||
- 开发 / Staging 环境快速拉起
|
||||
- 验证服务在 K8s 上的基本可用性
|
||||
- 作为后续 Helm 化的过渡参考
|
||||
|
||||
## Manifest 类别
|
||||
|
||||
| 文件 | 类别 | 用途 |
|
||||
|------|------|------|
|
||||
| `namespace.yaml` | namespace | 划分 4 个命名空间:system / services / monitoring / ingress |
|
||||
| `configmap-*.yaml` | configmap | 非敏感配置(服务端口、特性开关等) |
|
||||
| `secret-*.yaml` | secret | 敏感配置(密钥、连接串),生产用 External Secrets |
|
||||
| `*-deployment.yaml` | deployment | 服务无状态部署,含探针 / 资源 / 副本数 |
|
||||
| `*-service.yaml` | service | ClusterIP 服务发现 |
|
||||
| `ingress.yaml` | ingress | 对外入口,TLS 终结 + 路由 |
|
||||
| `*-hpa.yaml` | hpa | 水平自动扩缩容 |
|
||||
|
||||
## 命名空间规划
|
||||
|
||||
| Namespace | 用途 |
|
||||
|-----------|------|
|
||||
| `edu-system` | 系统组件(数据库代理、配置等) |
|
||||
| `edu-services` | 业务微服务(api-gateway / iam / core-edu / content / msg) |
|
||||
| `edu-monitoring` | 监控栈(Prometheus / Grafana / Alertmanager) |
|
||||
| `edu-ingress` | 入口控制器(NGINX Ingress / cert-manager) |
|
||||
|
||||
## 部署顺序
|
||||
|
||||
```bash
|
||||
kubectl apply -f namespace.yaml
|
||||
kubectl apply -f configmap-*.yaml
|
||||
kubectl apply -f secret-*.yaml # 生产请改用 External Secrets
|
||||
kubectl apply -f *-deployment.yaml
|
||||
kubectl apply -f *-service.yaml
|
||||
kubectl apply -f ingress.yaml
|
||||
kubectl apply -f *-hpa.yaml
|
||||
```
|
||||
|
||||
## Helm 化路线(后续)
|
||||
|
||||
1. 将骨架迁移为 Helm Chart(`charts/edu-platform/`)
|
||||
2. 每个微服务一个子 Chart,统一通过 umbrella chart 编排
|
||||
3. 环境差异通过 values-<env>.yaml 管理
|
||||
4. 敏感配置接入 External Secrets Operator(对接 Vault / KMS)
|
||||
5. CI/CD 通过 ArgoCD / Flux 做 GitOps 部署
|
||||
|
||||
## 当前文件
|
||||
|
||||
- `namespace.yaml`:4 个命名空间定义
|
||||
- `api-gateway-deployment.yaml`:api-gateway Deployment + Service 骨架
|
||||
125
infra/k8s/api-gateway-deployment.yaml
Normal file
125
infra/k8s/api-gateway-deployment.yaml
Normal file
@@ -0,0 +1,125 @@
|
||||
# api-gateway Deployment + Service 骨架
|
||||
# 生产环境请通过 Helm Chart 管理,此处仅作骨架参考
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: api-gateway
|
||||
namespace: edu-services
|
||||
labels:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: gateway
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: gateway
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
containers:
|
||||
- name: api-gateway
|
||||
image: edu/api-gateway:latest # 生产请固定 tag,避免 latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
# 存活探针:失败触发重启
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
# 就绪探针:失败从 Service Endpoints 摘除
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
resources:
|
||||
requests:
|
||||
cpu: "250m"
|
||||
memory: "256Mi"
|
||||
limits:
|
||||
cpu: "1000m"
|
||||
memory: "1Gi"
|
||||
env:
|
||||
# 从 ConfigMap 引用非敏感配置
|
||||
- name: NODE_ENV
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: api-gateway-config
|
||||
key: NODE_ENV
|
||||
- name: LOG_LEVEL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: api-gateway-config
|
||||
key: LOG_LEVEL
|
||||
- name: MYSQL_HOST
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: api-gateway-config
|
||||
key: MYSQL_HOST
|
||||
# 从 Secret 引用敏感配置
|
||||
- name: MYSQL_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: api-gateway-secret
|
||||
key: MYSQL_PASSWORD
|
||||
- name: JWT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: api-gateway-secret
|
||||
key: JWT_SECRET
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: api-gateway-secret
|
||||
key: REDIS_PASSWORD
|
||||
# 生产建议挂载 /tmp 并设置 readOnlyRootFilesystem: true
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
---
|
||||
# api-gateway Service(ClusterIP)
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api-gateway
|
||||
namespace: edu-services
|
||||
labels:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: gateway
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
33
infra/k8s/namespace.yaml
Normal file
33
infra/k8s/namespace.yaml
Normal file
@@ -0,0 +1,33 @@
|
||||
# K8s 命名空间定义 - Edu 平台
|
||||
# 划分 4 个命名空间:system / services / monitoring / ingress
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: edu-system
|
||||
labels:
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: edu-services
|
||||
labels:
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: services
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: edu-monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: monitoring
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: edu-ingress
|
||||
labels:
|
||||
app.kubernetes.io/part-of: edu-platform
|
||||
app.kubernetes.io/component: ingress
|
||||
122
infra/prometheus/rules.yml
Normal file
122
infra/prometheus/rules.yml
Normal file
@@ -0,0 +1,122 @@
|
||||
# Prometheus 告警规则 - Edu 平台
|
||||
# 分组:服务可用性 / 性能 / 资源
|
||||
groups:
|
||||
# ============================================================
|
||||
# 服务可用性
|
||||
# ============================================================
|
||||
- name: service-availability
|
||||
rules:
|
||||
# 服务宕机
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
category: availability
|
||||
annotations:
|
||||
summary: "服务宕机 {{ $labels.instance }}"
|
||||
description: "目标 {{ $labels.job }} / {{ $labels.instance }} 已离线超过 1 分钟。"
|
||||
|
||||
# 熔断器打开
|
||||
- alert: CircuitBreakerOpen
|
||||
expr: circuit_breaker_state{state="open"} == 1
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
category: availability
|
||||
annotations:
|
||||
summary: "熔断器打开 {{ $labels.service }}"
|
||||
description: "服务 {{ $labels.service }} 熔断器处于 open 状态,可能正在拒绝请求。"
|
||||
|
||||
# ============================================================
|
||||
# 性能
|
||||
# ============================================================
|
||||
- name: performance
|
||||
rules:
|
||||
# 高错误率(5xx 占比 > 5%)
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
(sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
|
||||
/ sum(rate(http_requests_total[5m])) by (service))
|
||||
> 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: performance
|
||||
annotations:
|
||||
summary: "错误率过高 {{ $labels.service }}"
|
||||
description: "服务 {{ $labels.service }} 5xx 错误率超过 5%,持续 5 分钟。"
|
||||
|
||||
# P99 延迟过高
|
||||
- alert: HighLatencyP99
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))
|
||||
> 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: performance
|
||||
annotations:
|
||||
summary: "P99 延迟过高 {{ $labels.service }}"
|
||||
description: "服务 {{ $labels.service }} P99 延迟超过 1 秒,持续 5 分钟。"
|
||||
|
||||
# Kafka 消费者积压
|
||||
- alert: KafkaConsumerLag
|
||||
expr: kafka_consumergroup_lag > 1000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
category: performance
|
||||
annotations:
|
||||
summary: "Kafka 消费者积压 {{ $labels.consumergroup }}"
|
||||
description: "消费组 {{ $labels.consumergroup }} 在 topic {{ $labels.topic }} 上积压超过 1000 条,持续 10 分钟。"
|
||||
|
||||
# ============================================================
|
||||
# 资源
|
||||
# ============================================================
|
||||
- name: resources
|
||||
rules:
|
||||
# MySQL 连接数过高
|
||||
- alert: MySQLConnectionsHigh
|
||||
expr: |
|
||||
(mysql_global_status_threads_connected
|
||||
/ mysql_global_variables_max_connections)
|
||||
> 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: resource
|
||||
annotations:
|
||||
summary: "MySQL 连接数过高 {{ $labels.instance }}"
|
||||
description: "MySQL 实例 {{ $labels.instance }} 连接数超过最大连接数的 80%,持续 5 分钟。"
|
||||
|
||||
# 磁盘空间不足
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(100
|
||||
- (node_filesystem_avail_bytes
|
||||
/ node_filesystem_size_bytes * 100))
|
||||
> 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
category: resource
|
||||
annotations:
|
||||
summary: "磁盘空间不足 {{ $labels.instance }}"
|
||||
description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 磁盘使用率超过 85%,持续 10 分钟。"
|
||||
|
||||
# 内存使用过高
|
||||
- alert: MemoryHigh
|
||||
expr: |
|
||||
(1
|
||||
- (node_memory_MemAvailable_bytes
|
||||
/ node_memory_MemTotal_bytes))
|
||||
> 0.9
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
category: resource
|
||||
annotations:
|
||||
summary: "内存使用过高 {{ $labels.instance }}"
|
||||
description: "节点 {{ $labels.instance }} 内存使用率超过 90%,持续 5 分钟。"
|
||||
50
infra/security/secrets.example.env
Normal file
50
infra/security/secrets.example.env
Normal file
@@ -0,0 +1,50 @@
|
||||
# Edu 平台密钥示例环境变量文件
|
||||
# ============================================================
|
||||
# 警告:本文件仅作示例,禁止包含真实密钥。
|
||||
# 生产环境请通过 K8s Secret / External Secrets / Vault 注入。
|
||||
# 复制为 .env 后用真实值替换所有 <placeholder>。
|
||||
|
||||
# ---------- MySQL ----------
|
||||
# 用途:MySQL root 用户密码,用于初始化与备份/恢复
|
||||
# 最小长度:32 字符
|
||||
# 复杂度:含大小写字母 + 数字 + 特殊符号
|
||||
MYSQL_ROOT_PASSWORD=<replace-with-32-char-strong-password>
|
||||
|
||||
# ---------- JWT ----------
|
||||
# 用途:JWT Access Token 签名密钥(HS256)
|
||||
# 最小长度:64 字符(建议使用 openssl rand -base64 48 生成)
|
||||
# 注意:旋转后所有已签发的 Access Token 立即失效
|
||||
JWT_SECRET=<replace-with-64-char-jwt-signing-secret>
|
||||
|
||||
# 用途:JWT Refresh Token 签名密钥(HS256)
|
||||
# 最小长度:64 字符
|
||||
# 注意:与 JWT_SECRET 必须不同;旋转后所有用户需重新登录
|
||||
JWT_REFRESH_SECRET=<replace-with-64-char-refresh-signing-secret>
|
||||
|
||||
# ---------- Kafka ----------
|
||||
# 用途:Kafka SASL/PLAIN 认证密码
|
||||
# 最小长度:24 字符
|
||||
KAFKA_SASL_PASSWORD=<replace-with-24-char-kafka-password>
|
||||
|
||||
# ---------- Elasticsearch ----------
|
||||
# 用途:Elasticsearch 内置 elastic 用户密码
|
||||
# 最小长度:24 字符
|
||||
ES_PASSWORD=<replace-with-24-char-es-password>
|
||||
|
||||
# ---------- Neo4j ----------
|
||||
# 用途:Neo4j 数据库管理员密码
|
||||
# 最小长度:24 字符
|
||||
NEO4J_PASSWORD=<replace-with-24-char-neo4j-password>
|
||||
|
||||
# ---------- Redis ----------
|
||||
# 用途:Redis ACL 默认用户密码
|
||||
# 最小长度:24 字符
|
||||
# 注意:生产环境建议启用 ACL,按用户分配最小权限
|
||||
REDIS_PASSWORD=<replace-with-24-char-redis-password>
|
||||
|
||||
# ---------- 应用层加密 ----------
|
||||
# 用途:应用层字段级加密密钥(AES-256-GCM)
|
||||
# 最小长度:32 字节(base64 编码后约 44 字符)
|
||||
# 生成:openssl rand -base64 32
|
||||
# 注意:旋转前需先解密所有已加密字段,旋转后重新加密
|
||||
ENCRYPTION_KEY=<replace-with-base64-32-byte-aes-key>
|
||||
64
infra/security/waf-rules.conf
Normal file
64
infra/security/waf-rules.conf
Normal file
@@ -0,0 +1,64 @@
|
||||
# ModSecurity 兼容 WAF 规则骨架 - Edu 平台
|
||||
# ============================================================
|
||||
# 说明:本文件为 ModSecurity SecRule 语法骨架,
|
||||
# 生产部署前请配合 OWASP CRS 使用并完成规则调优。
|
||||
# 参考:https://github.com/SpiderLabs/ModSecurity
|
||||
# https://github.com/coreruleset/coreruleset
|
||||
|
||||
# ---------- 基础配置 ----------
|
||||
# 启用规则引擎
|
||||
SecRuleEngine On
|
||||
|
||||
# 请求体大小限制(10MB)
|
||||
SecRequestBodyLimit 10485760
|
||||
|
||||
# 默认动作:记录日志 + 拒绝(生产前调整)
|
||||
SecDefaultAction "phase:2,log,deny,status:403"
|
||||
|
||||
# ============================================================
|
||||
# 规则 1:SQL 注入检测
|
||||
# 检测常见 SQL 注入关键字与元字符
|
||||
# ============================================================
|
||||
SecRule ARGS|ARGS_NAMES|REQUEST_COOKIES|REQUEST_COOKIES_NAMES|REQUEST_BODY|XML:/* \
|
||||
"(?i)(union\s+select|select\s+.*\s+from|insert\s+into|update\s+.*\s+set|delete\s+from|drop\s+table|alter\s+table|create\s+table|exec\s*\(|;\s*drop\s|--\s|/\*.*\*/|or\s+1\s*=\s*1|and\s+1\s*=\s*1|'\s*or\s*'|sleep\s*\(|benchmark\s*\(|load_file\s*\(|into\s+outfile)" \
|
||||
"id:1001,phase:2,log,deny,status:403,msg:'SQL Injection attempt',tag:'attack-sqli',severity:CRITICAL"
|
||||
|
||||
# ============================================================
|
||||
# 规则 2:XSS 检测
|
||||
# 检测常见 XSS payload 与事件处理器
|
||||
# ============================================================
|
||||
SecRule ARGS|ARGS_NAMES|REQUEST_COOKIES|REQUEST_COOKIES_NAMES|REQUEST_BODY|XML:/* \
|
||||
"(?i)(<script|</script>|javascript:|onerror\s*=|onload\s*=|onclick\s*=|onmouseover\s*=|onfocus\s*=|onblur\s*=|<iframe|<img[^>]+src\s*=|<svg[^>]+onload|document\.cookie|window\.location|eval\s*\(|alert\s*\(|prompt\s*\(|String\.fromCharCode)" \
|
||||
"id:1002,phase:2,log,deny,status:403,msg:'XSS attempt',tag:'attack-xss',severity:CRITICAL"
|
||||
|
||||
# ============================================================
|
||||
# 规则 3:路径遍历检测
|
||||
# 检测 ../ 与编码变体,防止读取敏感文件
|
||||
# ============================================================
|
||||
SecRule ARGS|ARGS_NAMES|REQUEST_FILENAME|REQUEST_URI|REQUEST_BODY \
|
||||
"(?i)(\.\./|\.\.\\|\.\.%2f|\.\.%5c|%2e%2e%2f|%2e%2e%5c|/etc/passwd|/etc/shadow|/etc/hosts|/proc/self/environ|c:\\windows\\win\.ini|c:\\boot\.ini|\.\./\.\./\.\./)" \
|
||||
"id:1003,phase:2,log,deny,status:403,msg:'Path Traversal attempt',tag:'attack-lfi',severity:CRITICAL"
|
||||
|
||||
# ============================================================
|
||||
# 规则 4:User-Agent 黑名单
|
||||
# 拦截已知恶意扫描器与爬虫
|
||||
# ============================================================
|
||||
SecRule REQUEST_HEADERS:User-Agent \
|
||||
"(?i)(sqlmap|nikto|nmap|nessus|acunetix|wpscan|hydra|metasploit|burpcollaborator|masscan|zgrab|dirbuster|gobuster|fuzzer|webinspect|appscan|jaeles|xrkmd)" \
|
||||
"id:1004,phase:1,log,deny,status:403,msg:'Malicious User-Agent blocked',tag:'attack-scanner',severity:WARNING"
|
||||
|
||||
# ============================================================
|
||||
# 规则 5:远程文件包含(RFI)检测
|
||||
# 检测通过 URL 参数引入远程文件
|
||||
# ============================================================
|
||||
SecRule ARGS|ARGS_NAMES \
|
||||
"(?i)(^(http|https|ftp|php|data)://|(http|https|ftp|php|data)://.*\?(php|http|https|ftp|data)://)" \
|
||||
"id:1005,phase:2,log,deny,status:403,msg:'Remote File Inclusion attempt',tag:'attack-rfi',severity:CRITICAL"
|
||||
|
||||
# ============================================================
|
||||
# 规则 6:命令注入检测
|
||||
# 检测 shell 元字符与命令执行关键字
|
||||
# ============================================================
|
||||
SecRule ARGS|ARGS_NAMES|REQUEST_BODY|REQUEST_COOKIES \
|
||||
"(?i)(;\s*(ls|cat|id|whoami|uname|pwd|wget|curl|nc|bash|sh|python|perl|ruby)\s|\$\(|`|&&\s*\w+\s|\|\|\s*\w+\s|\|\s*(ls|cat|id|whoami|uname|pwd|wget|curl|nc|bash|sh|python|perl|ruby)\s|/bin/(ba)?sh|/bin/cat|/usr/bin/(wget|curl|python|perl))" \
|
||||
"id:1006,phase:2,log,deny,status:403,msg:'Command Injection attempt',tag:'attack-rce',severity:CRITICAL"
|
||||
Reference in New Issue
Block a user