# Prometheus 告警规则 - Edu 平台 # 分组:服务可用性 / 性能 / 资源 groups: # ============================================================ # 服务可用性 # ============================================================ - name: service-availability rules: # 服务宕机 - alert: ServiceDown expr: up == 0 for: 1m labels: severity: critical category: availability annotations: summary: "服务宕机 {{ $labels.instance }}" description: "目标 {{ $labels.job }} / {{ $labels.instance }} 已离线超过 1 分钟。" # 熔断器打开 - alert: CircuitBreakerOpen expr: circuit_breaker_state{state="open"} == 1 for: 30s labels: severity: critical category: availability annotations: summary: "熔断器打开 {{ $labels.service }}" description: "服务 {{ $labels.service }} 熔断器处于 open 状态,可能正在拒绝请求。" # ============================================================ # 性能 # ============================================================ - name: performance rules: # 高错误率(5xx 占比 > 5%) - alert: HighErrorRate expr: | (sum(rate(http_requests_total{status=~"5.."}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)) > 0.05 for: 5m labels: severity: warning category: performance annotations: summary: "错误率过高 {{ $labels.service }}" description: "服务 {{ $labels.service }} 5xx 错误率超过 5%,持续 5 分钟。" # P99 延迟过高 - alert: HighLatencyP99 expr: | histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)) > 1 for: 5m labels: severity: warning category: performance annotations: summary: "P99 延迟过高 {{ $labels.service }}" description: "服务 {{ $labels.service }} P99 延迟超过 1 秒,持续 5 分钟。" # Kafka 消费者积压 - alert: KafkaConsumerLag expr: kafka_consumergroup_lag > 1000 for: 10m labels: severity: warning category: performance annotations: summary: "Kafka 消费者积压 {{ $labels.consumergroup }}" description: "消费组 {{ $labels.consumergroup }} 在 topic {{ $labels.topic }} 上积压超过 1000 条,持续 10 分钟。" # ============================================================ # 资源 # ============================================================ - name: resources rules: # MySQL 连接数过高 - alert: MySQLConnectionsHigh expr: | (mysql_global_status_threads_connected / mysql_global_variables_max_connections) > 0.8 for: 5m labels: severity: warning category: resource annotations: summary: "MySQL 连接数过高 {{ $labels.instance }}" description: "MySQL 实例 {{ $labels.instance }} 连接数超过最大连接数的 80%,持续 5 分钟。" # 磁盘空间不足 - alert: DiskSpaceLow expr: | (100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100)) > 85 for: 10m labels: severity: warning category: resource annotations: summary: "磁盘空间不足 {{ $labels.instance }}" description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 磁盘使用率超过 85%,持续 10 分钟。" # 内存使用过高 - alert: MemoryHigh expr: | (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9 for: 5m labels: severity: critical category: resource annotations: summary: "内存使用过高 {{ $labels.instance }}" description: "节点 {{ $labels.instance }} 内存使用率超过 90%,持续 5 分钟。" # ============================================================ # parent-bff SLO 告警规则(P6.3) # SLO 目标:P95 延迟 < 200ms / 错误率 < 0.1% / 可用性 > 99.9% # 对齐 02-architecture-design.md §10.4 # 指标来源:services/parent-bff/src/shared/observability/metrics.ts # ============================================================ - name: parent-bff-slo rules: # parent-bff GraphQL P95 延迟 > 200ms - alert: ParentBffHighLatencyP95 expr: | histogram_quantile(0.95, sum(rate(parent_bff_graphql_duration_seconds_bucket[5m])) by (le, operation)) > 0.2 for: 5m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff P95 延迟过高 {{ $labels.operation }}" description: "parent-bff GraphQL 操作 {{ $labels.operation }} 的 P95 延迟超过 200ms SLO 目标,持续 5 分钟。当前值: {{ $value }}s" # parent-bff GraphQL 错误率 > 0.1% - alert: ParentBffHighErrorRate expr: | (sum(rate(parent_bff_graphql_requests_total{status="error"}[5m])) by (operation) / sum(rate(parent_bff_graphql_requests_total[5m])) by (operation)) > 0.001 for: 5m labels: severity: critical category: slo service: parent-bff annotations: summary: "parent-bff 错误率过高 {{ $labels.operation }}" description: "parent-bff GraphQL 操作 {{ $labels.operation }} 的错误率超过 0.1% SLO 目标,持续 5 分钟。当前值: {{ $value | humanizePercentage }}" # parent-bff 可用性 < 99.9%(5 分钟窗口内成功请求比例) - alert: ParentBffLowAvailability expr: | (1 - (sum(rate(parent_bff_graphql_requests_total{status="error"}[5m])) / sum(rate(parent_bff_graphql_requests_total[5m])))) < 0.999 for: 5m labels: severity: critical category: slo service: parent-bff annotations: summary: "parent-bff 可用性低于 99.9%" description: "parent-bff 整体可用性低于 99.9% SLO 目标,持续 5 分钟。当前可用性: {{ $value | humanizePercentage }}" # parent-bff 熔断器打开 - alert: ParentBffCircuitBreakerOpen expr: parent_bff_circuit_state{state="open"} == 1 for: 30s labels: severity: critical category: slo service: parent-bff annotations: summary: "parent-bff 熔断器打开 {{ $labels.service }}" description: "parent-bff 下游服务 {{ $labels.service }} 的熔断器处于 open 状态,相关请求将被快速失败。" # parent-bff 下游错误率 > 5% - alert: ParentBffDownstreamHighErrorRate expr: | (sum(rate(parent_bff_downstream_errors_total[5m])) by (service) / sum(rate(parent_bff_downstream_calls_total[5m])) by (service)) > 0.05 for: 5m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff 下游 {{ $labels.service }} 错误率高" description: "parent-bff 调用下游服务 {{ $labels.service }} 的错误率超过 5%,持续 5 分钟。当前值: {{ $value | humanizePercentage }}" # parent-bff 缓存命中率低 - alert: ParentBffLowCacheHitRate expr: | (sum(rate(parent_bff_cache_hits_total[5m])) / (sum(rate(parent_bff_cache_hits_total[5m])) + sum(rate(parent_bff_cache_misses_total[5m])))) < 0.5 for: 10m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff 缓存命中率低" description: "parent-bff 缓存命中率低于 50%,持续 10 分钟。当前命中率: {{ $value | humanizePercentage }}" # parent-bff Kafka 消费积压 - alert: ParentBffKafkaLag expr: | sum by (topic) ( rate(parent_bff_event_consumed_total[5m]) ) < 1 and sum by (topic) ( increase(parent_bff_event_consumed_total[1h]) ) > 0 for: 15m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff Kafka 消费停滞 {{ $labels.topic }}" description: "parent-bff 在 topic {{ $labels.topic }} 上的消费速率近 15 分钟为 0,但近 1 小时内有消费记录,可能存在消费停滞。" # parent-bff push-gateway 推送失败率 > 5% - alert: ParentBffPushFailureRate expr: | (sum(rate(parent_bff_event_pushed_total{push_status="failure"}[5m])) / sum(rate(parent_bff_event_pushed_total[5m]))) > 0.05 for: 5m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff push-gateway 推送失败率高" description: "parent-bff 调用 push-gateway 推送的失败率超过 5%,持续 5 分钟。当前值: {{ $value | humanizePercentage }}" # parent-bff ChildGuard 越权拦截突增 - alert: ParentBffChildGuardBlocksSpike expr: | rate(parent_bff_child_guard_blocks_total[5m]) > 0.1 for: 5m labels: severity: warning category: slo service: parent-bff annotations: summary: "parent-bff ChildGuard 拦截次数突增" description: "parent-bff ChildGuard 越权拦截速率 > 0.1 次/秒,持续 5 分钟。可能存在恶意探测或前端 bug。当前速率: {{ $value }}/s"