feat(data-ana): v2 P6 硬化完成 + 6 新 RPC + Prometheus 监控
P6 硬化(5 项全部完成):
- CDC 多实例水平扩展: _INSTANCE_ID + get_lag() 真实 lag 计算
- ExamCache Redis 化: key data_ana:exam:{exam_id}, TTL 30 天 + 内存 LRU fallback
- ClickHouse TTL 归档: 5 表均加 TTL(1-3 年),分区级删除
- Prometheus 监控: 18 个指标(CDC/CH/ExamCache/DataScope/gRPC/业务)
- readyz 深度硬化: 4 依赖超时检查(CH 1s/Redis 200ms/iam 2s/CDC lag<1000)
v2 新增 6 个 RPC(analytics.proto 扩展为 18 RPC):
- GetStudentGrowth / GetAssignmentAnalysis / GetMasterySummary
- ListDiagnosticReports(占位,待 ai 服务)/ ListErrorBookItems / GetErrorBookStats
监控与可观测性: lifespan 预热 + gRPC ServerInterceptor + CDC 消费者指标
Docker 本地测试 19 项全部通过(healthz/readyz/metrics + 11 HTTP + 10 gRPC + ruff)
nextstep-v2.md: 上游需求对齐 + 下游要求(iam/core-edu/content/ai/SRE)
This commit is contained in:
@@ -34,7 +34,7 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query, Response
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
@@ -134,23 +134,38 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
if grpc_server_obj is None:
|
||||
logger.warning("grpc_server_not_started_http_only")
|
||||
|
||||
# 2. 启动 CDC 消费者后台任务
|
||||
# 2. 预热客户端(避免首次 readyz 探针因惰性初始化超时)
|
||||
if settings.clickhouse_host:
|
||||
ch_ok = await clickhouse_repository.ping()
|
||||
logger.info("clickhouse_warmup", ok=ch_ok)
|
||||
if settings.redis_url:
|
||||
redis_ok = await redis_client.ping()
|
||||
logger.info("redis_warmup", ok=redis_ok)
|
||||
|
||||
# 2.1 初始化监控指标初始值(确保 /metrics 端点暴露自定义指标)
|
||||
from . import metrics as _metrics
|
||||
|
||||
_metrics.clickhouse_connection_status.set(1 if settings.clickhouse_host else 0)
|
||||
_metrics.redis_connection_status.set(1 if settings.redis_url else 0)
|
||||
_metrics.iam_grpc_connection_status.set(1 if settings.iam_grpc_endpoint else 0)
|
||||
|
||||
# 3. 启动 CDC 消费者后台任务
|
||||
await cdc_consumer.start_consumer()
|
||||
|
||||
yield
|
||||
|
||||
logger.info("data_ana_service_stopping")
|
||||
|
||||
# 3. 停止 CDC 消费者
|
||||
# 4. 停止 CDC 消费者
|
||||
await cdc_consumer.stop_consumer()
|
||||
|
||||
# 4. 停止 gRPC server
|
||||
# 5. 停止 gRPC server
|
||||
await grpc_server.stop_grpc_server()
|
||||
|
||||
# 5. 关闭 Kafka producer
|
||||
# 6. 关闭 Kafka producer
|
||||
await kafka_producer.close_producer()
|
||||
|
||||
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
# 7. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
await redis_client.close_client()
|
||||
await iam_client.close_grpc()
|
||||
await clickhouse_repository.close_client()
|
||||
@@ -196,36 +211,78 @@ async def healthz() -> dict[str, str]:
|
||||
|
||||
|
||||
@app.get("/readyz")
|
||||
async def readyz() -> dict[str, Any]:
|
||||
"""就绪检查(readiness,检查 4 依赖).
|
||||
async def readyz(response: Response) -> dict[str, Any]:
|
||||
"""就绪检查(P6 深度硬化版,检查 4 依赖 + 超时控制 + lag 阈值).
|
||||
|
||||
依赖检查:
|
||||
1. clickhouse:已配置且可达(未配置算降级就绪)
|
||||
2. cdc_consumer:running / disabled
|
||||
3. redis:已配置且可达(未配置算降级就绪)
|
||||
4. iam_grpc:已配置且可达(未配置算降级就绪)
|
||||
依赖检查(带超时):
|
||||
1. clickhouse:1s 超时,已配置且可达(未配置算降级就绪)
|
||||
2. cdc_consumer:running / disabled + lag < 1000
|
||||
3. redis:200ms 超时,已配置且可达(未配置算降级就绪)
|
||||
4. iam_grpc:2s 超时,已配置且可达(未配置算降级就绪)
|
||||
|
||||
返回 ready=true 的条件:
|
||||
- ClickHouse 已配置且可达,或未配置(降级就绪)
|
||||
- 不要求所有依赖都健康(降级模式下仍可服务骨架数据)
|
||||
- ClickHouse 已配置且可达(1s 内),或未配置(降级就绪)
|
||||
- CDC consumer lag < readyz_cdc_lag_threshold(1000)
|
||||
不满足时返回 HTTP 503,K8s 摘流量.
|
||||
"""
|
||||
# 1. ClickHouse
|
||||
ch_ok = await clickhouse_repository.ping()
|
||||
import asyncio
|
||||
|
||||
# 1. ClickHouse(1s 超时)
|
||||
try:
|
||||
ch_ok = await asyncio.wait_for(
|
||||
clickhouse_repository.ping(),
|
||||
timeout=settings.readyz_clickhouse_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
ch_ok = False
|
||||
ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
|
||||
|
||||
# 2. CDC 消费者
|
||||
cdc_status = (
|
||||
"running"
|
||||
if cdc_consumer.is_running()
|
||||
else ("disabled" if not settings.kafka_brokers else "failed")
|
||||
)
|
||||
# 2. CDC 消费者 + lag 检查
|
||||
cdc_running = cdc_consumer.is_running()
|
||||
cdc_lag = 0
|
||||
if cdc_running:
|
||||
try:
|
||||
cdc_lag = await asyncio.wait_for(
|
||||
cdc_consumer.get_lag(),
|
||||
timeout=2.0,
|
||||
)
|
||||
except TimeoutError:
|
||||
cdc_lag = -1 # 查询超时标记
|
||||
cdc_lag_ok = cdc_lag >= 0 and cdc_lag < settings.readyz_cdc_lag_threshold
|
||||
if not settings.kafka_brokers:
|
||||
cdc_status = "disabled"
|
||||
cdc_lag_ok = True # 未配置 Kafka 时不阻塞 ready
|
||||
elif cdc_running and cdc_lag_ok:
|
||||
cdc_status = f"running(lag={cdc_lag})"
|
||||
elif cdc_running:
|
||||
cdc_status = f"running(lag={cdc_lag},exceeded)"
|
||||
else:
|
||||
cdc_status = "failed"
|
||||
|
||||
# 3. Redis
|
||||
redis_ok = await redis_client.ping() if settings.redis_url else None
|
||||
# 3. Redis(200ms 超时)
|
||||
if settings.redis_url:
|
||||
try:
|
||||
redis_ok = await asyncio.wait_for(
|
||||
redis_client.ping(),
|
||||
timeout=settings.readyz_redis_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
redis_ok = False
|
||||
else:
|
||||
redis_ok = None
|
||||
redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
|
||||
|
||||
# 4. iam gRPC
|
||||
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
|
||||
# 4. iam gRPC(2s 超时,iam_client.ping 已内置 1s 超时)
|
||||
if settings.iam_grpc_endpoint:
|
||||
try:
|
||||
iam_ok = await asyncio.wait_for(
|
||||
iam_client.ping(),
|
||||
timeout=settings.readyz_iam_grpc_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
iam_ok = False
|
||||
else:
|
||||
iam_ok = None
|
||||
iam_status = (
|
||||
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
|
||||
)
|
||||
@@ -233,10 +290,15 @@ async def readyz() -> dict[str, Any]:
|
||||
# 5. gRPC server
|
||||
grpc_status = "running" if grpc_server.is_running() else "stopped"
|
||||
|
||||
# 就绪判定:ClickHouse 可达或未配置(降级就绪)
|
||||
ready = ch_ok or not settings.clickhouse_host
|
||||
# 就绪判定(P6 硬化):
|
||||
# - ClickHouse 可达或未配置(降级就绪)
|
||||
# - CDC consumer lag 未超阈值
|
||||
ready = (ch_ok or not settings.clickhouse_host) and cdc_lag_ok
|
||||
degraded = not ch_ok or not redis_ok or not iam_ok
|
||||
|
||||
if not ready:
|
||||
response.status_code = 503 # K8s 摘流量
|
||||
|
||||
return {
|
||||
"status": "ok" if ready else "not_ready",
|
||||
"service": "data-ana",
|
||||
@@ -250,6 +312,13 @@ async def readyz() -> dict[str, Any]:
|
||||
"grpc_server": grpc_status,
|
||||
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
|
||||
},
|
||||
"thresholds": {
|
||||
"clickhouse_timeout_s": settings.readyz_clickhouse_timeout_s,
|
||||
"redis_timeout_s": settings.readyz_redis_timeout_s,
|
||||
"iam_grpc_timeout_s": settings.readyz_iam_grpc_timeout_s,
|
||||
"cdc_lag_threshold": settings.readyz_cdc_lag_threshold,
|
||||
"cdc_lag_current": cdc_lag,
|
||||
},
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user