P6 生产硬化阶段交付物(46 文件):
## 1. API Gateway 中间件链(services/api-gateway/internal/middleware/)
- circuit-breaker.go: gobreaker v2 熔断器(5s 窗口/50% 错误率/30s OPEN→HALF_OPEN)
- ratelimit.go: 令牌桶限流(sync.Map + cleanup goroutine,默认 100rps/20 burst)
- cors.go: CORS 中间件(CORS_ORIGINS 环境变量)
- recovery.go: panic 恢复 + uuid request_id
- security.go: 安全头 + 请求体 10MB 限制
- requestid.go: 请求 ID 注入
- health/health.go: /healthz + /readyz 健康检查
- main.go: 重写注册全部中间件链(Recovery→RequestID→CORS→Security→BodyLimit→RateLimit→CircuitBreaker→Auth)
## 2. 基础设施硬化(infra/)
- backup/backup-mysql.sh: MySQL 全量备份(mysqldump+gzip,按服务独立)
- backup/restore-mysql.sh: 恢复脚本
- backup/backup-cron.sh: cron 调度入口(5 服务批量备份)
- alertmanager/alertmanager.yml: 告警路由(webhook + 邮件示例)
- prometheus/rules.yml: 8 条告警规则(服务可用性/性能/资源 3 组)
- grafana/dashboards/microservices-overview.json: 4 panel 仪表盘
- grafana/provisioning/: 数据源和仪表盘 provisioning
- k8s/namespace.yaml: 4 命名空间(edu-system/services/monitoring/ingress)
- k8s/api-gateway-deployment.yaml: Deployment + Service 骨架
- chaos/experiments.yaml: 3 个 Litmus 混沌实验(pod-kill/network-latency/disk-fill)
- docker-compose.monitoring.yml: 监控栈 profile
- security/secrets.example.env: 8 项密钥占位符
- security/waf-rules.conf: ModSecurity WAF 规则骨架
## 3. 业务服务健康检查 + 优雅停机(5 个 NestJS 服务)
- services/{iam,core-edu,content,msg,classes}/src/shared/health/: /healthz + /readyz
- services/{iam,core-edu,content,msg,classes}/src/shared/lifecycle/: OnModuleInit + OnApplicationShutdown
## 4. Python 服务健康检查
- services/{ai,data-ana}/src/health/health.py: FastAPI APIRouter
## 5. 运维文档
- docs/architecture/runbooks/p6-hardening.md: P6 总览 Runbook(9 章节)
- docs/architecture/runbooks/incident-response.md: 事件响应手册(5 章节)
- docs/architecture/004-p6-addendum.md: 004 架构补记 P6 章节
- docs/troubleshooting/known-issues-p6-addendum.md: 15 条 P6 场景→技术映射
## 验收信号
- RPO ≤ 15min(MySQL 备份 + binlog PITR)
- RTO ≤ 30min(K8s 滚动更新 + DNS 切换)
- P99 ≤ 500ms(熔断 + 限流 + 缓存)
- 熔断器错误率 > 50% 触发 OPEN
- 限流 100rps/20 burst
- 备份保留 7 天
- 混沌实验每月 1 次
95 lines
2.2 KiB
Go
95 lines
2.2 KiB
Go
package middleware
|
||
|
||
import (
|
||
"net/http"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
)
|
||
|
||
// bucket 是单个客户端的令牌桶
|
||
type bucket struct {
|
||
mu sync.Mutex
|
||
tokens float64
|
||
lastTime time.Time
|
||
}
|
||
|
||
// rateLimiter 持有所有客户端 IP 的令牌桶
|
||
type rateLimiter struct {
|
||
rps float64
|
||
burst int
|
||
buckets sync.Map // map[string]*bucket
|
||
}
|
||
|
||
// RateLimit 返回基于令牌桶的限流中间件(内存版,不依赖 Redis)。
|
||
// 每个客户端 IP 独立桶,按 rps(每秒令牌数)补充,最大容量为 burst。
|
||
// 超限返回 429 + JSON {"error":"rate_limited","retry_after":60}。
|
||
// 内部启动 cleanup goroutine,每 5 分钟清理 10 分钟未访问的桶。
|
||
func RateLimit(rps float64, burst int) gin.HandlerFunc {
|
||
rl := &rateLimiter{rps: rps, burst: burst}
|
||
// 启动清理 goroutine
|
||
go rl.cleanup(5*time.Minute, 10*time.Minute)
|
||
|
||
return func(c *gin.Context) {
|
||
ip := c.ClientIP()
|
||
b := rl.getBucket(ip)
|
||
|
||
b.mu.Lock()
|
||
now := time.Now()
|
||
elapsed := now.Sub(b.lastTime).Seconds()
|
||
b.lastTime = now
|
||
// 按经过时间补充令牌
|
||
b.tokens += elapsed * rl.rps
|
||
if b.tokens > float64(rl.burst) {
|
||
b.tokens = float64(rl.burst)
|
||
}
|
||
// 令牌不足:拒绝
|
||
if b.tokens < 1 {
|
||
b.mu.Unlock()
|
||
c.Header("Retry-After", "60")
|
||
c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{
|
||
"error": "rate_limited",
|
||
"retry_after": 60,
|
||
})
|
||
return
|
||
}
|
||
b.tokens--
|
||
b.mu.Unlock()
|
||
|
||
c.Next()
|
||
}
|
||
}
|
||
|
||
// getBucket 获取指定 IP 的令牌桶,不存在则新建(初始满桶)
|
||
func (rl *rateLimiter) getBucket(ip string) *bucket {
|
||
if v, ok := rl.buckets.Load(ip); ok {
|
||
return v.(*bucket)
|
||
}
|
||
b := &bucket{
|
||
tokens: float64(rl.burst),
|
||
lastTime: time.Now(),
|
||
}
|
||
actual, _ := rl.buckets.LoadOrStore(ip, b)
|
||
return actual.(*bucket)
|
||
}
|
||
|
||
// cleanup 周期性清理长时间未访问的桶,避免内存无限增长
|
||
func (rl *rateLimiter) cleanup(interval, maxIdle time.Duration) {
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
now := time.Now()
|
||
rl.buckets.Range(func(key, value any) bool {
|
||
b := value.(*bucket)
|
||
b.mu.Lock()
|
||
idle := now.Sub(b.lastTime)
|
||
b.mu.Unlock()
|
||
if idle > maxIdle {
|
||
rl.buckets.Delete(key)
|
||
}
|
||
return true
|
||
})
|
||
}
|
||
}
|