feat(data-ana): v2 P6 硬化完成 + 6 新 RPC + Prometheus 监控

P6 硬化(5 项全部完成):

- CDC 多实例水平扩展: _INSTANCE_ID + get_lag() 真实 lag 计算

- ExamCache Redis 化: key data_ana:exam:{exam_id}, TTL 30 天 + 内存 LRU fallback

- ClickHouse TTL 归档: 5 表均加 TTL(1-3 年),分区级删除

- Prometheus 监控: 18 个指标(CDC/CH/ExamCache/DataScope/gRPC/业务)

- readyz 深度硬化: 4 依赖超时检查(CH 1s/Redis 200ms/iam 2s/CDC lag<1000)

v2 新增 6 个 RPC(analytics.proto 扩展为 18 RPC):

- GetStudentGrowth / GetAssignmentAnalysis / GetMasterySummary

- ListDiagnosticReports(占位,待 ai 服务)/ ListErrorBookItems / GetErrorBookStats

监控与可观测性: lifespan 预热 + gRPC ServerInterceptor + CDC 消费者指标

Docker 本地测试 19 项全部通过(healthz/readyz/metrics + 11 HTTP + 10 gRPC + ruff)

nextstep-v2.md: 上游需求对齐 + 下游要求(iam/core-edu/content/ai/SRE)
This commit is contained in:
SpecialX
2026-07-14 18:07:17 +08:00
parent 78e406b317
commit 9db7fd917e
13 changed files with 2118 additions and 147 deletions

View File

@@ -34,7 +34,7 @@ from datetime import UTC, datetime
from typing import Any
import structlog
from fastapi import APIRouter, Depends, FastAPI, Query
from fastapi import APIRouter, Depends, FastAPI, Query, Response
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
@@ -134,23 +134,38 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
if grpc_server_obj is None:
logger.warning("grpc_server_not_started_http_only")
# 2. 启动 CDC 消费者后台任务
# 2. 预热客户端(避免首次 readyz 探针因惰性初始化超时)
if settings.clickhouse_host:
ch_ok = await clickhouse_repository.ping()
logger.info("clickhouse_warmup", ok=ch_ok)
if settings.redis_url:
redis_ok = await redis_client.ping()
logger.info("redis_warmup", ok=redis_ok)
# 2.1 初始化监控指标初始值(确保 /metrics 端点暴露自定义指标)
from . import metrics as _metrics
_metrics.clickhouse_connection_status.set(1 if settings.clickhouse_host else 0)
_metrics.redis_connection_status.set(1 if settings.redis_url else 0)
_metrics.iam_grpc_connection_status.set(1 if settings.iam_grpc_endpoint else 0)
# 3. 启动 CDC 消费者后台任务
await cdc_consumer.start_consumer()
yield
logger.info("data_ana_service_stopping")
# 3. 停止 CDC 消费者
# 4. 停止 CDC 消费者
await cdc_consumer.stop_consumer()
# 4. 停止 gRPC server
# 5. 停止 gRPC server
await grpc_server.stop_grpc_server()
# 5. 关闭 Kafka producer
# 6. 关闭 Kafka producer
await kafka_producer.close_producer()
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
# 7. 关闭 Redis / iam gRPC / ClickHouse 客户端
await redis_client.close_client()
await iam_client.close_grpc()
await clickhouse_repository.close_client()
@@ -196,36 +211,78 @@ async def healthz() -> dict[str, str]:
@app.get("/readyz")
async def readyz() -> dict[str, Any]:
"""就绪检查(readiness检查 4 依赖.
async def readyz(response: Response) -> dict[str, Any]:
"""就绪检查(P6 深度硬化版,检查 4 依赖 + 超时控制 + lag 阈值.
依赖检查:
1. clickhouse已配置且可达未配置算降级就绪
2. cdc_consumerrunning / disabled
3. redis已配置且可达未配置算降级就绪
4. iam_grpc已配置且可达未配置算降级就绪
依赖检查(带超时)
1. clickhouse1s 超时,已配置且可达(未配置算降级就绪)
2. cdc_consumerrunning / disabled + lag < 1000
3. redis200ms 超时,已配置且可达(未配置算降级就绪)
4. iam_grpc2s 超时,已配置且可达(未配置算降级就绪)
返回 ready=true 的条件:
- ClickHouse 已配置且可达,或未配置(降级就绪)
- 不要求所有依赖都健康(降级模式下仍可服务骨架数据
- ClickHouse 已配置且可达1s 内),或未配置(降级就绪)
- CDC consumer lag < readyz_cdc_lag_threshold1000
不满足时返回 HTTP 503K8s 摘流量.
"""
# 1. ClickHouse
ch_ok = await clickhouse_repository.ping()
import asyncio
# 1. ClickHouse1s 超时)
try:
ch_ok = await asyncio.wait_for(
clickhouse_repository.ping(),
timeout=settings.readyz_clickhouse_timeout_s,
)
except TimeoutError:
ch_ok = False
ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
# 2. CDC 消费者
cdc_status = (
"running"
if cdc_consumer.is_running()
else ("disabled" if not settings.kafka_brokers else "failed")
)
# 2. CDC 消费者 + lag 检查
cdc_running = cdc_consumer.is_running()
cdc_lag = 0
if cdc_running:
try:
cdc_lag = await asyncio.wait_for(
cdc_consumer.get_lag(),
timeout=2.0,
)
except TimeoutError:
cdc_lag = -1 # 查询超时标记
cdc_lag_ok = cdc_lag >= 0 and cdc_lag < settings.readyz_cdc_lag_threshold
if not settings.kafka_brokers:
cdc_status = "disabled"
cdc_lag_ok = True # 未配置 Kafka 时不阻塞 ready
elif cdc_running and cdc_lag_ok:
cdc_status = f"running(lag={cdc_lag})"
elif cdc_running:
cdc_status = f"running(lag={cdc_lag},exceeded)"
else:
cdc_status = "failed"
# 3. Redis
redis_ok = await redis_client.ping() if settings.redis_url else None
# 3. Redis200ms 超时)
if settings.redis_url:
try:
redis_ok = await asyncio.wait_for(
redis_client.ping(),
timeout=settings.readyz_redis_timeout_s,
)
except TimeoutError:
redis_ok = False
else:
redis_ok = None
redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
# 4. iam gRPC
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
# 4. iam gRPC2s 超时iam_client.ping 已内置 1s 超时)
if settings.iam_grpc_endpoint:
try:
iam_ok = await asyncio.wait_for(
iam_client.ping(),
timeout=settings.readyz_iam_grpc_timeout_s,
)
except TimeoutError:
iam_ok = False
else:
iam_ok = None
iam_status = (
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
)
@@ -233,10 +290,15 @@ async def readyz() -> dict[str, Any]:
# 5. gRPC server
grpc_status = "running" if grpc_server.is_running() else "stopped"
# 就绪判定ClickHouse 可达或未配置(降级就绪)
ready = ch_ok or not settings.clickhouse_host
# 就绪判定P6 硬化):
# - ClickHouse 可达或未配置(降级就绪)
# - CDC consumer lag 未超阈值
ready = (ch_ok or not settings.clickhouse_host) and cdc_lag_ok
degraded = not ch_ok or not redis_ok or not iam_ok
if not ready:
response.status_code = 503 # K8s 摘流量
return {
"status": "ok" if ready else "not_ready",
"service": "data-ana",
@@ -250,6 +312,13 @@ async def readyz() -> dict[str, Any]:
"grpc_server": grpc_status,
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
},
"thresholds": {
"clickhouse_timeout_s": settings.readyz_clickhouse_timeout_s,
"redis_timeout_s": settings.readyz_redis_timeout_s,
"iam_grpc_timeout_s": settings.readyz_iam_grpc_timeout_s,
"cdc_lag_threshold": settings.readyz_cdc_lag_threshold,
"cdc_lag_current": cdc_lag,
},
"timestamp": datetime.now(UTC).isoformat(),
}