feat(data-ana): 完整实现 data-ana 数据分析服务
包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等
This commit is contained in:
SpecialX
570
services/data-ana/src/data_ana/grpc_server.py
Normal file
570
services/data-ana/src/data_ana/grpc_server.py
Normal file
@@ -0,0 +1,570 @@
|
||||
"""gRPC Server（AnalyticsService 12 RPC + HealthService）.
|
||||
|
||||
对齐 02-architecture-design.md §4.2：
|
||||
- AnalyticsService 12 RPC（含 1 个 server-streaming SubscribeMasteryUpdate）
|
||||
- HealthService（grpc.health.v1，供 K8s 探针）
|
||||
- 端口 50055
|
||||
- 所有 RPC 返回结构化 message（HTTP 层包装 ActionState 信封）
|
||||
|
||||
SubscribeMasteryUpdate（P5+）：
|
||||
- server-streaming RPC
|
||||
- 客户端订阅 student_id / class_id
|
||||
- 掌握度计算完成时推送 MasteryUpdateEvent
|
||||
- 基于 asyncio.Queue 实现事件分发
|
||||
|
||||
降级策略：
|
||||
- grpcio 未安装：gRPC server 不启动（仅 HTTP 服务）
|
||||
- 下游服务不可达：返回降级响应（degraded 字段标记）
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from . import analytics_service, warning_service
|
||||
from .config import settings
|
||||
from .shared.permissions import UserContext
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# SubscribeMasteryUpdate 订阅管理（P5+）
|
||||
_subscribers: dict[str, asyncio.Queue] = {}
|
||||
_subscribers_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def _add_subscriber(key: str) -> asyncio.Queue:
|
||||
"""添加订阅者."""
|
||||
async with _subscribers_lock:
|
||||
if key not in _subscribers:
|
||||
_subscribers[key] = asyncio.Queue(maxsize=100)
|
||||
return _subscribers[key]
|
||||
|
||||
|
||||
async def _remove_subscriber(key: str) -> None:
|
||||
"""移除订阅者."""
|
||||
async with _subscribers_lock:
|
||||
_subscribers.pop(key, None)
|
||||
|
||||
|
||||
async def notify_mastery_update(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
previous_level: float,
|
||||
) -> None:
|
||||
"""通知所有订阅者掌握度更新（由 mastery_service 调用）.
|
||||
|
||||
订阅匹配规则：
|
||||
- 订阅 student_id 的客户端收到通知
|
||||
- 订阅 class_id 的客户端收到该班级所有学生的通知（需要 ExamCache 查 class_id）
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
|
||||
event_id = str(uuid.uuid4())
|
||||
calculated_at = int(time.time())
|
||||
|
||||
# 通知 student_id 订阅者
|
||||
queue = _subscribers.get(f"student:{student_id}")
|
||||
if queue is not None:
|
||||
try:
|
||||
queue.put_nowait(
|
||||
{
|
||||
"event_id": event_id,
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"mastery_level": mastery_level,
|
||||
"previous_level": previous_level,
|
||||
"calculated_at": calculated_at,
|
||||
}
|
||||
)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning("mastery_update_queue_full_dropped", student_id=student_id)
|
||||
|
||||
|
||||
# ===== AnalyticsService 实现 =====
|
||||
|
||||
|
||||
class AnalyticsServiceServicer:
|
||||
"""AnalyticsService gRPC 实现（12 RPC）."""
|
||||
|
||||
async def GetClassPerformance(self, request, context):
|
||||
"""班级成绩分析."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_class_performance(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
subject_id=request.subject_id,
|
||||
start_date=request.start_date,
|
||||
end_date=request.end_date,
|
||||
)
|
||||
return _build_class_performance_response(result)
|
||||
|
||||
async def GetStudentWeakness(self, request, context):
|
||||
"""学生薄弱知识点."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_weakness(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_student_weakness_response(result)
|
||||
|
||||
async def GetLearningTrend(self, request, context):
|
||||
"""学习趋势."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_learning_trend(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
start_date=request.start_date,
|
||||
end_date=request.end_date,
|
||||
)
|
||||
return _build_learning_trend_response(result)
|
||||
|
||||
async def GetTeacherDashboard(self, request, context):
|
||||
"""教师仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_teacher_dashboard(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
)
|
||||
return _build_teacher_dashboard_response(result)
|
||||
|
||||
async def GetStudentDashboard(self, request, context):
|
||||
"""学生仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_dashboard(
|
||||
user=user,
|
||||
)
|
||||
return _build_student_dashboard_response(result)
|
||||
|
||||
async def GetParentDashboard(self, request, context):
|
||||
"""家长仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_parent_dashboard(
|
||||
user=user,
|
||||
child_id=request.student_id,
|
||||
)
|
||||
return _build_parent_dashboard_response(result)
|
||||
|
||||
async def GetAdminDashboard(self, request, context):
|
||||
"""管理员仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_admin_dashboard(
|
||||
user=user,
|
||||
school_id=request.scope_id,
|
||||
)
|
||||
return _build_admin_dashboard_response(result)
|
||||
|
||||
async def GetWarnings(self, request, context):
|
||||
"""预警列表查询."""
|
||||
# 简化：按 class_id 查询（P6 改为多条件过滤）
|
||||
warnings = await warning_service.get_warnings(
|
||||
student_id="", # gRPC 层按 class_id 查询，P6 扩展
|
||||
warning_type="",
|
||||
)
|
||||
return _build_warning_list_response(warnings)
|
||||
|
||||
async def TriggerWarning(self, request, context):
|
||||
"""手动触发预警."""
|
||||
result = await warning_service.trigger_warning_manual(
|
||||
target_id=request.target_id,
|
||||
warning_type=request.warning_type,
|
||||
threshold=0.0,
|
||||
current_value=0.0,
|
||||
severity=request.severity,
|
||||
)
|
||||
return _build_trigger_warning_response(result)
|
||||
|
||||
async def GetMasteryDistribution(self, request, context):
|
||||
"""班级掌握度分布."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_mastery_distribution(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
subject_id=request.subject_id,
|
||||
knowledge_point_id=request.knowledge_point_id,
|
||||
)
|
||||
return _build_mastery_distribution_response(result)
|
||||
|
||||
async def GetStudentMastery(self, request, context):
|
||||
"""学生知识点掌握度明细."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_mastery(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_student_mastery_response(result)
|
||||
|
||||
async def SubscribeMasteryUpdate(self, request, context):
|
||||
"""订阅掌握度更新（server-streaming，P5+）."""
|
||||
# 确定订阅 key
|
||||
if request.student_id:
|
||||
sub_key = f"student:{request.student_id}"
|
||||
elif request.class_id:
|
||||
sub_key = f"class:{request.class_id}"
|
||||
else:
|
||||
sub_key = "global"
|
||||
|
||||
queue = await _add_subscriber(sub_key)
|
||||
logger.info(
|
||||
"mastery_subscription_added",
|
||||
sub_key=sub_key,
|
||||
peer=context.peer() if hasattr(context, "peer") else "unknown",
|
||||
)
|
||||
|
||||
try:
|
||||
while context.is_active():
|
||||
try:
|
||||
event = await asyncio.wait_for(queue.get(), timeout=30.0)
|
||||
yield _build_mastery_update_event(event)
|
||||
except TimeoutError:
|
||||
# 心跳：30s 无事件时发一个空 keepalive（客户端应容忍）
|
||||
continue
|
||||
except asyncio.CancelledError:
|
||||
logger.info("mastery_subscription_cancelled", sub_key=sub_key)
|
||||
raise
|
||||
finally:
|
||||
await _remove_subscriber(sub_key)
|
||||
logger.info("mastery_subscription_removed", sub_key=sub_key)
|
||||
|
||||
|
||||
# ===== 辅助函数 =====
|
||||
|
||||
|
||||
def _extract_user(context: Any) -> UserContext:
|
||||
"""从 gRPC metadata 提取用户上下文."""
|
||||
user_id = ""
|
||||
roles: list[str] = []
|
||||
try:
|
||||
metadata = (
|
||||
dict(context.invocation_metadata()) if hasattr(context, "invocation_metadata") else {}
|
||||
)
|
||||
user_id = metadata.get("x-user-id", "")
|
||||
roles_str = metadata.get("x-user-roles", "")
|
||||
roles = [r.strip() for r in roles_str.split(",") if r.strip()] if roles_str else []
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return UserContext(user_id=user_id, roles=roles)
|
||||
|
||||
|
||||
def _build_class_performance_response(data: dict) -> Any:
|
||||
"""构建 ClassPerformance proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.ClassPerformance(
|
||||
class_id=data.get("classId", ""),
|
||||
average_score=data.get("averageScore", 0.0),
|
||||
pass_rate=data.get("passRate", 0.0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_student_weakness_response(data: dict) -> Any:
|
||||
"""构建 StudentWeakness proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
return analytics_pb2.StudentWeakness(
|
||||
student_id=data.get("studentId", ""),
|
||||
weak_points=weak_points,
|
||||
)
|
||||
|
||||
|
||||
def _build_learning_trend_response(data: dict) -> Any:
|
||||
"""构建 LearningTrend proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
points = [
|
||||
analytics_pb2.TrendPoint(
|
||||
date=p.get("date", 0),
|
||||
score=p.get("score", 0.0),
|
||||
)
|
||||
for p in data.get("points", [])
|
||||
]
|
||||
return analytics_pb2.LearningTrend(
|
||||
student_id=data.get("studentId", ""),
|
||||
points=points,
|
||||
)
|
||||
|
||||
|
||||
def _build_teacher_dashboard_response(data: dict) -> Any:
|
||||
"""构建 TeacherDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
classes = [
|
||||
analytics_pb2.ClassSummary(
|
||||
class_id=c.get("class_id", ""),
|
||||
class_name=c.get("class_name", ""),
|
||||
student_count=c.get("total_students", 0),
|
||||
average_score=c.get("average_score", 0.0),
|
||||
)
|
||||
for c in data.get("classes", [])
|
||||
]
|
||||
return analytics_pb2.TeacherDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
total_classes=data.get("totalClasses", 0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
class_avg_score=data.get("classAvgScore", 0.0),
|
||||
pending_homework_count=data.get("pendingHomeworkCount", 0),
|
||||
classes=classes,
|
||||
)
|
||||
|
||||
|
||||
def _build_student_dashboard_response(data: dict) -> Any:
|
||||
"""构建 StudentDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
trends = [
|
||||
analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0))
|
||||
for p in data.get("trend", [])
|
||||
]
|
||||
return analytics_pb2.StudentDashboard(
|
||||
user_id=data.get("studentId", data.get("userId", "")),
|
||||
avg_score=data.get("averageScore", 0.0),
|
||||
class_rank=0,
|
||||
total_students=0,
|
||||
weak_points=weak_points,
|
||||
recent_trends=trends,
|
||||
)
|
||||
|
||||
|
||||
def _build_parent_dashboard_response(data: dict) -> Any:
|
||||
"""构建 ParentDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
return analytics_pb2.ParentDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
student_id=data.get("childId", ""),
|
||||
child_avg_score=data.get("averageScore", 0.0),
|
||||
child_class_rank=0,
|
||||
total_class_students=0,
|
||||
child_weak_points=weak_points,
|
||||
)
|
||||
|
||||
|
||||
def _build_admin_dashboard_response(data: dict) -> Any:
|
||||
"""构建 AdminDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
ai_usage_data = data.get("aiUsage", {})
|
||||
by_provider = [
|
||||
analytics_pb2.AIUsageByProvider(
|
||||
provider=p.get("provider", ""),
|
||||
request_count=p.get("requestCount", 0),
|
||||
total_tokens=p.get("totalTokens", 0),
|
||||
cost_cents=p.get("costCents", 0),
|
||||
)
|
||||
for p in ai_usage_data.get("byProvider", [])
|
||||
]
|
||||
ai_usage = analytics_pb2.AIUsageSummary(
|
||||
total_requests=ai_usage_data.get("totalRequests", 0),
|
||||
total_tokens=ai_usage_data.get("totalTokens", 0),
|
||||
total_cost_cents=ai_usage_data.get("totalCostCents", 0),
|
||||
by_provider=by_provider,
|
||||
)
|
||||
return analytics_pb2.AdminDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
total_teachers=0,
|
||||
total_students=data.get("totalStudents", 0),
|
||||
total_classes=data.get("totalClasses", 0),
|
||||
school_avg_score=data.get("classAvgScore", 0.0),
|
||||
ai_usage=ai_usage,
|
||||
)
|
||||
|
||||
|
||||
def _build_warning_list_response(warnings: list[dict]) -> Any:
|
||||
"""构建 WarningList proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
warning_infos = [
|
||||
analytics_pb2.WarningInfo(
|
||||
warning_id=w.get("warning_id", ""),
|
||||
warning_type=w.get("warning_type", ""),
|
||||
target_id=w.get("target_id", ""),
|
||||
target_name=w.get("target_name", ""),
|
||||
threshold=w.get("threshold", 0.0),
|
||||
current_value=w.get("current_value", 0.0),
|
||||
severity=w.get("severity", "WARN"),
|
||||
occurred_at=int(w.get("occurred_at", 0)),
|
||||
)
|
||||
for w in warnings
|
||||
]
|
||||
return analytics_pb2.WarningList(
|
||||
warnings=warning_infos,
|
||||
total=len(warning_infos),
|
||||
)
|
||||
|
||||
|
||||
def _build_trigger_warning_response(data: dict) -> Any:
|
||||
"""构建 TriggerWarningResponse proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.TriggerWarningResponse(
|
||||
warning_id=data.get("warning_id", ""),
|
||||
triggered=data.get("triggered", False),
|
||||
)
|
||||
|
||||
|
||||
def _build_mastery_distribution_response(data: dict) -> Any:
|
||||
"""构建 MasteryDistribution proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.MasteryDistribution(
|
||||
class_id=data.get("classId", ""),
|
||||
mastered_count=data.get("masteredCount", 0),
|
||||
progressing_count=data.get("progressingCount", 0),
|
||||
weak_count=data.get("weakCount", 0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_student_mastery_response(data: dict) -> Any:
|
||||
"""构建 StudentMastery proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
knowledge_points = [
|
||||
analytics_pb2.KnowledgePointMastery(
|
||||
knowledge_point_id=kp.get("knowledge_point_id", ""),
|
||||
title=kp.get("title", ""),
|
||||
subject_id=kp.get("subject_id", ""),
|
||||
mastery_level=kp.get("mastery_level", 0.0),
|
||||
mastery_label=kp.get("mastery_label", "weak"),
|
||||
calculated_at=kp.get("calculated_at", 0),
|
||||
)
|
||||
for kp in data.get("knowledgePoints", [])
|
||||
]
|
||||
return analytics_pb2.StudentMastery(
|
||||
student_id=data.get("studentId", ""),
|
||||
knowledge_points=knowledge_points,
|
||||
overall_mastery=data.get("overallMastery", 0.0),
|
||||
)
|
||||
|
||||
|
||||
def _build_mastery_update_event(event: dict) -> Any:
|
||||
"""构建 MasteryUpdateEvent proto 响应（streaming）."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.MasteryUpdateEvent(
|
||||
event_id=event.get("event_id", ""),
|
||||
student_id=event.get("student_id", ""),
|
||||
knowledge_point_id=event.get("knowledge_point_id", ""),
|
||||
mastery_level=event.get("mastery_level", 0.0),
|
||||
previous_level=event.get("previous_level", 0.0),
|
||||
calculated_at=event.get("calculated_at", 0),
|
||||
)
|
||||
|
||||
|
||||
# ===== gRPC Server 管理 =====
|
||||
|
||||
_server: Any | None = None
|
||||
|
||||
|
||||
async def start_grpc_server() -> Any | None:
|
||||
"""启动 gRPC server :50055（lifespan 调用）.
|
||||
|
||||
降级策略：
|
||||
- grpcio 未安装：返回 None，仅 HTTP 服务
|
||||
- 启动失败：仅记录错误，不阻塞 FastAPI 主流程
|
||||
"""
|
||||
global _server
|
||||
|
||||
try:
|
||||
import grpc # type: ignore[import-not-found]
|
||||
from generated_proto import analytics_pb2_grpc # type: ignore[import-not-found]
|
||||
except ImportError as exc:
|
||||
logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc))
|
||||
return None
|
||||
|
||||
_server = grpc.aio.server()
|
||||
|
||||
# 注册 AnalyticsService
|
||||
servicer = AnalyticsServiceServicer()
|
||||
analytics_pb2_grpc.add_AnalyticsServiceServicer_to_server(servicer, _server)
|
||||
|
||||
# 注册 HealthService
|
||||
try:
|
||||
from grpc_health.v1 import ( # type: ignore[import-not-found]
|
||||
health,
|
||||
health_pb2,
|
||||
health_pb2_grpc,
|
||||
)
|
||||
|
||||
health_servicer = health.aio.HealthServicer()
|
||||
await health_servicer.set("analytics", health_pb2.HealthCheckResponse.SERVING)
|
||||
await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING) # overall
|
||||
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, _server)
|
||||
except ImportError:
|
||||
logger.warning("grpc_health_not_installed_skip_health_service")
|
||||
|
||||
# 绑定端口
|
||||
bind_address = f"[::]:{settings.grpc_port}"
|
||||
_server.add_insecure_port(bind_address)
|
||||
|
||||
try:
|
||||
await _server.start()
|
||||
logger.info(
|
||||
"grpc_server_started",
|
||||
port=settings.grpc_port,
|
||||
rpc_count=12,
|
||||
)
|
||||
return _server
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("grpc_server_start_failed", error=str(exc), port=settings.grpc_port)
|
||||
_server = None
|
||||
return None
|
||||
|
||||
|
||||
async def stop_grpc_server() -> None:
|
||||
"""停止 gRPC server（lifespan 退出时调用）."""
|
||||
global _server
|
||||
if _server is not None:
|
||||
try:
|
||||
await _server.stop(grace=5)
|
||||
logger.info("grpc_server_stopped")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("grpc_server_stop_failed", error=str(exc))
|
||||
finally:
|
||||
_server = None
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
"""gRPC server 是否运行中."""
|
||||
return _server is not None
|
||||
Reference in New Issue
Block a user