2229309a1e
add complete parent-bff implementation including: - GraphQL endpoint with depth/cost validation - ChildGuard越权校验 with redis cache and singleflight - parallel orchestration with partial failure fallback - three-level cache fallback strategy (Redis + LRU + downstream) - Kafka consumer for cache invalidation and notification push - opossum circuit breaker for downstream services - Prometheus metrics and SLO alerts - Helm chart for k8s deployment with multi-environment configs - Grafana dashboard for observability - complete unit and integration tests
266 lines
9.9 KiB
YAML
266 lines
9.9 KiB
YAML
# Prometheus 告警规则 - Edu 平台
|
||
# 分组：服务可用性 / 性能 / 资源
|
||
groups:
|
||
# ============================================================
|
||
# 服务可用性
|
||
# ============================================================
|
||
- name: service-availability
|
||
rules:
|
||
# 服务宕机
|
||
- alert: ServiceDown
|
||
expr: up == 0
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
category: availability
|
||
annotations:
|
||
summary: "服务宕机 {{ $labels.instance }}"
|
||
description: "目标 {{ $labels.job }} / {{ $labels.instance }} 已离线超过 1 分钟。"
|
||
|
||
# 熔断器打开
|
||
- alert: CircuitBreakerOpen
|
||
expr: circuit_breaker_state{state="open"} == 1
|
||
for: 30s
|
||
labels:
|
||
severity: critical
|
||
category: availability
|
||
annotations:
|
||
summary: "熔断器打开 {{ $labels.service }}"
|
||
description: "服务 {{ $labels.service }} 熔断器处于 open 状态，可能正在拒绝请求。"
|
||
|
||
# ============================================================
|
||
# 性能
|
||
# ============================================================
|
||
- name: performance
|
||
rules:
|
||
# 高错误率（5xx 占比 > 5%）
|
||
- alert: HighErrorRate
|
||
expr: |
|
||
(sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
|
||
/ sum(rate(http_requests_total[5m])) by (service))
|
||
> 0.05
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: performance
|
||
annotations:
|
||
summary: "错误率过高 {{ $labels.service }}"
|
||
description: "服务 {{ $labels.service }} 5xx 错误率超过 5%，持续 5 分钟。"
|
||
|
||
# P99 延迟过高
|
||
- alert: HighLatencyP99
|
||
expr: |
|
||
histogram_quantile(0.99,
|
||
sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))
|
||
> 1
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: performance
|
||
annotations:
|
||
summary: "P99 延迟过高 {{ $labels.service }}"
|
||
description: "服务 {{ $labels.service }} P99 延迟超过 1 秒，持续 5 分钟。"
|
||
|
||
# Kafka 消费者积压
|
||
- alert: KafkaConsumerLag
|
||
expr: kafka_consumergroup_lag > 1000
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
category: performance
|
||
annotations:
|
||
summary: "Kafka 消费者积压 {{ $labels.consumergroup }}"
|
||
description: "消费组 {{ $labels.consumergroup }} 在 topic {{ $labels.topic }} 上积压超过 1000 条，持续 10 分钟。"
|
||
|
||
# ============================================================
|
||
# 资源
|
||
# ============================================================
|
||
- name: resources
|
||
rules:
|
||
# MySQL 连接数过高
|
||
- alert: MySQLConnectionsHigh
|
||
expr: |
|
||
(mysql_global_status_threads_connected
|
||
/ mysql_global_variables_max_connections)
|
||
> 0.8
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: resource
|
||
annotations:
|
||
summary: "MySQL 连接数过高 {{ $labels.instance }}"
|
||
description: "MySQL 实例 {{ $labels.instance }} 连接数超过最大连接数的 80%，持续 5 分钟。"
|
||
|
||
# 磁盘空间不足
|
||
- alert: DiskSpaceLow
|
||
expr: |
|
||
(100
|
||
- (node_filesystem_avail_bytes
|
||
/ node_filesystem_size_bytes * 100))
|
||
> 85
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
category: resource
|
||
annotations:
|
||
summary: "磁盘空间不足 {{ $labels.instance }}"
|
||
description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 磁盘使用率超过 85%，持续 10 分钟。"
|
||
|
||
# 内存使用过高
|
||
- alert: MemoryHigh
|
||
expr: |
|
||
(1
|
||
- (node_memory_MemAvailable_bytes
|
||
/ node_memory_MemTotal_bytes))
|
||
> 0.9
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
category: resource
|
||
annotations:
|
||
summary: "内存使用过高 {{ $labels.instance }}"
|
||
description: "节点 {{ $labels.instance }} 内存使用率超过 90%，持续 5 分钟。"
|
||
|
||
# ============================================================
|
||
# parent-bff SLO 告警规则（P6.3）
|
||
# SLO 目标：P95 延迟 < 200ms / 错误率 < 0.1% / 可用性 > 99.9%
|
||
# 对齐 02-architecture-design.md §10.4
|
||
# 指标来源：services/parent-bff/src/shared/observability/metrics.ts
|
||
# ============================================================
|
||
- name: parent-bff-slo
|
||
rules:
|
||
# parent-bff GraphQL P95 延迟 > 200ms
|
||
- alert: ParentBffHighLatencyP95
|
||
expr: |
|
||
histogram_quantile(0.95,
|
||
sum(rate(parent_bff_graphql_duration_seconds_bucket[5m])) by (le, operation))
|
||
> 0.2
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff P95 延迟过高 {{ $labels.operation }}"
|
||
description: "parent-bff GraphQL 操作 {{ $labels.operation }} 的 P95 延迟超过 200ms SLO 目标，持续 5 分钟。当前值: {{ $value }}s"
|
||
|
||
# parent-bff GraphQL 错误率 > 0.1%
|
||
- alert: ParentBffHighErrorRate
|
||
expr: |
|
||
(sum(rate(parent_bff_graphql_requests_total{status="error"}[5m])) by (operation)
|
||
/ sum(rate(parent_bff_graphql_requests_total[5m])) by (operation))
|
||
> 0.001
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff 错误率过高 {{ $labels.operation }}"
|
||
description: "parent-bff GraphQL 操作 {{ $labels.operation }} 的错误率超过 0.1% SLO 目标，持续 5 分钟。当前值: {{ $value | humanizePercentage }}"
|
||
|
||
# parent-bff 可用性 < 99.9%（5 分钟窗口内成功请求比例）
|
||
- alert: ParentBffLowAvailability
|
||
expr: |
|
||
(1
|
||
- (sum(rate(parent_bff_graphql_requests_total{status="error"}[5m]))
|
||
/ sum(rate(parent_bff_graphql_requests_total[5m]))))
|
||
< 0.999
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff 可用性低于 99.9%"
|
||
description: "parent-bff 整体可用性低于 99.9% SLO 目标，持续 5 分钟。当前可用性: {{ $value | humanizePercentage }}"
|
||
|
||
# parent-bff 熔断器打开
|
||
- alert: ParentBffCircuitBreakerOpen
|
||
expr: parent_bff_circuit_state{state="open"} == 1
|
||
for: 30s
|
||
labels:
|
||
severity: critical
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff 熔断器打开 {{ $labels.service }}"
|
||
description: "parent-bff 下游服务 {{ $labels.service }} 的熔断器处于 open 状态，相关请求将被快速失败。"
|
||
|
||
# parent-bff 下游错误率 > 5%
|
||
- alert: ParentBffDownstreamHighErrorRate
|
||
expr: |
|
||
(sum(rate(parent_bff_downstream_errors_total[5m])) by (service)
|
||
/ sum(rate(parent_bff_downstream_calls_total[5m])) by (service))
|
||
> 0.05
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff 下游 {{ $labels.service }} 错误率高"
|
||
description: "parent-bff 调用下游服务 {{ $labels.service }} 的错误率超过 5%，持续 5 分钟。当前值: {{ $value | humanizePercentage }}"
|
||
|
||
# parent-bff 缓存命中率低
|
||
- alert: ParentBffLowCacheHitRate
|
||
expr: |
|
||
(sum(rate(parent_bff_cache_hits_total[5m]))
|
||
/ (sum(rate(parent_bff_cache_hits_total[5m])) + sum(rate(parent_bff_cache_misses_total[5m]))))
|
||
< 0.5
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff 缓存命中率低"
|
||
description: "parent-bff 缓存命中率低于 50%，持续 10 分钟。当前命中率: {{ $value | humanizePercentage }}"
|
||
|
||
# parent-bff Kafka 消费积压
|
||
- alert: ParentBffKafkaLag
|
||
expr: |
|
||
sum by (topic) (
|
||
rate(parent_bff_event_consumed_total[5m])
|
||
) < 1
|
||
and
|
||
sum by (topic) (
|
||
increase(parent_bff_event_consumed_total[1h])
|
||
) > 0
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff Kafka 消费停滞 {{ $labels.topic }}"
|
||
description: "parent-bff 在 topic {{ $labels.topic }} 上的消费速率近 15 分钟为 0，但近 1 小时内有消费记录，可能存在消费停滞。"
|
||
|
||
# parent-bff push-gateway 推送失败率 > 5%
|
||
- alert: ParentBffPushFailureRate
|
||
expr: |
|
||
(sum(rate(parent_bff_event_pushed_total{push_status="failure"}[5m]))
|
||
/ sum(rate(parent_bff_event_pushed_total[5m])))
|
||
> 0.05
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff push-gateway 推送失败率高"
|
||
description: "parent-bff 调用 push-gateway 推送的失败率超过 5%，持续 5 分钟。当前值: {{ $value | humanizePercentage }}"
|
||
|
||
# parent-bff ChildGuard 越权拦截突增
|
||
- alert: ParentBffChildGuardBlocksSpike
|
||
expr: |
|
||
rate(parent_bff_child_guard_blocks_total[5m]) > 0.1
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
category: slo
|
||
service: parent-bff
|
||
annotations:
|
||
summary: "parent-bff ChildGuard 拦截次数突增"
|
||
description: "parent-bff ChildGuard 越权拦截速率 > 0.1 次/秒，持续 5 分钟。可能存在恶意探测或前端 bug。当前速率: {{ $value }}/s"
|