feat(data-ana): 完整实现 data-ana 数据分析服务
包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等
This commit is contained in:
SpecialX
383
services/data-ana/src/data_ana/warning_service.py
Normal file
383
services/data-ana/src/data_ana/warning_service.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""预警服务（5 类预警阈值评估 + 去重位图 + 事件发布）.
|
||||
|
||||
对齐 02-architecture-design.md §11 预警体系：
|
||||
- LOW_MASTERY：掌握度 < 0.4
|
||||
- CRITICAL_LOW：掌握度 < 0.2
|
||||
- SCORE_DROP：成绩环比下降 ≥ 20%
|
||||
- ABSENT_FREQUENT：缺勤 ≥ 3 次/周
|
||||
- TREND_DECLINE：连续 3 次成绩下降
|
||||
|
||||
去重策略：
|
||||
Redis bitmap key = data_ana:warning:{target_id}:{type}:{date}
|
||||
TTL = 25h（跨天容忍，避免边界重复触发）
|
||||
|
||||
事件发布：
|
||||
warning.triggered 复用 edu.insight.mastery.updated topic（总裁裁决 §2.11）
|
||||
Outbox 豁免（总裁裁决 §2.10）
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .config import settings
|
||||
from .repository import clickhouse_repository, kafka_producer, redis_client
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
# 预警类型常量
|
||||
class WarningType:
|
||||
"""预警类型."""
|
||||
|
||||
LOW_MASTERY = "LOW_MASTERY"
|
||||
CRITICAL_LOW = "CRITICAL_LOW"
|
||||
SCORE_DROP = "SCORE_DROP"
|
||||
ABSENT_FREQUENT = "ABSENT_FREQUENT"
|
||||
TREND_DECLINE = "TREND_DECLINE"
|
||||
|
||||
|
||||
class Severity:
|
||||
"""预警严重程度."""
|
||||
|
||||
INFO = "INFO"
|
||||
WARN = "WARN"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
# 预警类型 → 严重程度映射
|
||||
_SEVERITY_MAP: dict[str, str] = {
|
||||
WarningType.LOW_MASTERY: Severity.WARN,
|
||||
WarningType.CRITICAL_LOW: Severity.CRITICAL,
|
||||
WarningType.SCORE_DROP: Severity.WARN,
|
||||
WarningType.ABSENT_FREQUENT: Severity.WARN,
|
||||
WarningType.TREND_DECLINE: Severity.WARN,
|
||||
}
|
||||
|
||||
|
||||
def _warning_dedup_key(target_id: str, warning_type: str) -> str:
|
||||
"""预警去重位图 key（25h TTL，避免边界重复触发）."""
|
||||
today = datetime.now(UTC).strftime("%Y%m%d")
|
||||
return f"data_ana:warning:{target_id}:{warning_type}:{today}"
|
||||
|
||||
|
||||
async def _check_and_dedup(target_id: str, warning_type: str) -> bool:
|
||||
"""检查去重（首次返回 True，已触发过返回 False）."""
|
||||
key = _warning_dedup_key(target_id, warning_type)
|
||||
is_first = await redis_client.setnx_dedup(key, ttl_s=25 * 3600)
|
||||
if not is_first:
|
||||
logger.info(
|
||||
"warning_dedup_skipped",
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
)
|
||||
return is_first
|
||||
|
||||
|
||||
async def _trigger_warning(
|
||||
target_id: str,
|
||||
warning_type: str,
|
||||
threshold: float,
|
||||
current_value: float,
|
||||
student_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""触发预警（去重 + 事件发布）."""
|
||||
severity = _SEVERITY_MAP.get(warning_type, Severity.WARN)
|
||||
|
||||
# 1. 去重检查
|
||||
should_trigger = await _check_and_dedup(target_id, warning_type)
|
||||
if not should_trigger:
|
||||
return {
|
||||
"target_id": target_id,
|
||||
"warning_type": warning_type,
|
||||
"triggered": False,
|
||||
"deduplicated": True,
|
||||
"threshold": threshold,
|
||||
"current_value": current_value,
|
||||
}
|
||||
|
||||
# 2. 发布 warning.triggered 事件（Outbox 豁免，复用 mastery topic）
|
||||
published = await kafka_producer.publish_warning_triggered(
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
severity=severity,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
"warning_triggered",
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
severity=severity,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
published=published,
|
||||
)
|
||||
return {
|
||||
"target_id": target_id,
|
||||
"warning_type": warning_type,
|
||||
"severity": severity,
|
||||
"triggered": True,
|
||||
"deduplicated": False,
|
||||
"threshold": threshold,
|
||||
"current_value": current_value,
|
||||
"event_published": published,
|
||||
}
|
||||
|
||||
|
||||
async def evaluate_mastery_warning(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""评估掌握度预警（LOW_MASTERY / CRITICAL_LOW）.
|
||||
|
||||
- mastery < 0.2 → CRITICAL_LOW（最严重，先评估）
|
||||
- 0.2 ≤ mastery < 0.4 → LOW_MASTERY
|
||||
"""
|
||||
triggered: list[dict[str, Any]] = []
|
||||
|
||||
if mastery_level < settings.warning_critical_mastery_threshold:
|
||||
result = await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.CRITICAL_LOW,
|
||||
threshold=settings.warning_critical_mastery_threshold,
|
||||
current_value=mastery_level,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
if result["triggered"]:
|
||||
triggered.append(result)
|
||||
return triggered # CRITICAL_LOW 已触发，不再触发 LOW_MASTERY
|
||||
|
||||
if mastery_level < settings.warning_low_mastery_threshold:
|
||||
result = await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.LOW_MASTERY,
|
||||
threshold=settings.warning_low_mastery_threshold,
|
||||
current_value=mastery_level,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
if result["triggered"]:
|
||||
triggered.append(result)
|
||||
|
||||
return triggered
|
||||
|
||||
|
||||
async def evaluate_score_drop(
|
||||
student_id: str,
|
||||
current_score: float,
|
||||
previous_score: float,
|
||||
exam_id: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估成绩下降预警（SCORE_DROP）.
|
||||
|
||||
成绩环比下降 ≥ warning_score_drop_percent（默认 20%）触发.
|
||||
"""
|
||||
if previous_score <= 0:
|
||||
return None
|
||||
|
||||
drop_ratio = (previous_score - current_score) / previous_score
|
||||
if drop_ratio < settings.warning_score_drop_percent:
|
||||
return None
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.SCORE_DROP,
|
||||
threshold=settings.warning_score_drop_percent,
|
||||
current_value=round(drop_ratio, 4),
|
||||
student_id=student_id,
|
||||
metadata={
|
||||
"exam_id": exam_id,
|
||||
"previous_score": str(previous_score),
|
||||
"current_score": str(current_score),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_absent_frequent(
|
||||
student_id: str,
|
||||
absent_count_week: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估缺勤频繁预警（ABSENT_FREQUENT）.
|
||||
|
||||
一周内缺勤 ≥ warning_absent_per_week（默认 3 次）触发.
|
||||
"""
|
||||
if absent_count_week < settings.warning_absent_per_week:
|
||||
return None
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.ABSENT_FREQUENT,
|
||||
threshold=float(settings.warning_absent_per_week),
|
||||
current_value=float(absent_count_week),
|
||||
student_id=student_id,
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_trend_decline(
|
||||
student_id: str,
|
||||
recent_scores: list[float],
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估趋势下降预警（TREND_DECLINE）.
|
||||
|
||||
连续 3 次成绩下降触发（recent_scores 按时间正序，最早在前）.
|
||||
"""
|
||||
if len(recent_scores) < 3:
|
||||
return None
|
||||
|
||||
# 检查最近 3 次是否连续下降
|
||||
last_three = recent_scores[-3:]
|
||||
is_declining = all(last_three[i] > last_three[i + 1] for i in range(len(last_three) - 1))
|
||||
if not is_declining:
|
||||
return None
|
||||
|
||||
drop = last_three[-2] - last_three[-1]
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.TREND_DECLINE,
|
||||
threshold=0.0, # 下降幅度阈值（0 表示只要下降就触发）
|
||||
current_value=round(drop, 4),
|
||||
student_id=student_id,
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_all_warnings(
|
||||
student_id: str,
|
||||
knowledge_point_id: str = "",
|
||||
mastery_level: float | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""综合评估学生所有预警类型（GetWarnings RPC 调用）.
|
||||
|
||||
参数：
|
||||
- student_id：学生 ID
|
||||
- knowledge_point_id：知识点 ID（可选，指定则只评估该知识点）
|
||||
- mastery_level：预计算的掌握度（可选，未提供则查询）
|
||||
|
||||
返回：触发的预警列表
|
||||
"""
|
||||
triggered: list[dict[str, Any]] = []
|
||||
|
||||
# 1. 掌握度预警
|
||||
if mastery_level is None:
|
||||
snapshot = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=student_id,
|
||||
)
|
||||
if snapshot is not None:
|
||||
knowledge_points = snapshot.get("knowledgePoints", [])
|
||||
if knowledge_point_id:
|
||||
# 仅评估指定知识点
|
||||
knowledge_points = [
|
||||
kp
|
||||
for kp in knowledge_points
|
||||
if kp.get("knowledge_point_id") == knowledge_point_id
|
||||
]
|
||||
for kp in knowledge_points:
|
||||
level = kp.get("mastery_level", 0.0)
|
||||
warnings = await evaluate_mastery_warning(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=kp.get("knowledge_point_id", ""),
|
||||
mastery_level=level,
|
||||
)
|
||||
triggered.extend(warnings)
|
||||
else:
|
||||
warnings = await evaluate_mastery_warning(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
mastery_level=mastery_level,
|
||||
)
|
||||
triggered.extend(warnings)
|
||||
|
||||
# 2. 缺勤预警
|
||||
attendance = await clickhouse_repository.query_attendance(student_id=student_id)
|
||||
if attendance is not None:
|
||||
# 简化：统计总缺勤数（P6 改为按周统计）
|
||||
absent_count = attendance.get("absentCount", 0)
|
||||
if absent_count >= settings.warning_absent_per_week:
|
||||
warning = await evaluate_absent_frequent(
|
||||
student_id=student_id,
|
||||
absent_count_week=absent_count,
|
||||
)
|
||||
if warning is not None and warning.get("triggered"):
|
||||
triggered.append(warning)
|
||||
|
||||
# 3. 成绩下降预警（需要查询最近两次成绩）
|
||||
trend = await clickhouse_repository.query_learning_trend(student_id=student_id)
|
||||
if trend is not None:
|
||||
points = trend.get("points", [])
|
||||
if len(points) >= 2:
|
||||
current = points[-1].get("score", 0.0)
|
||||
previous = points[-2].get("score", 0.0)
|
||||
drop_warning = await evaluate_score_drop(
|
||||
student_id=student_id,
|
||||
current_score=current,
|
||||
previous_score=previous,
|
||||
)
|
||||
if drop_warning is not None and drop_warning.get("triggered"):
|
||||
triggered.append(drop_warning)
|
||||
|
||||
# 4. 趋势下降预警
|
||||
scores = [p.get("score", 0.0) for p in points]
|
||||
trend_warning = await evaluate_trend_decline(
|
||||
student_id=student_id,
|
||||
recent_scores=scores,
|
||||
)
|
||||
if trend_warning is not None and trend_warning.get("triggered"):
|
||||
triggered.append(trend_warning)
|
||||
|
||||
return triggered
|
||||
|
||||
|
||||
async def get_warnings(
|
||||
student_id: str,
|
||||
warning_type: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""查询预警列表（GetWarnings RPC 实现）.
|
||||
|
||||
参数：
|
||||
- student_id：学生 ID
|
||||
- warning_type：预警类型过滤（空表示全部）
|
||||
|
||||
返回：预警列表（每条含 target_id / warning_type / severity / threshold /
|
||||
current_value / triggered_at）
|
||||
"""
|
||||
# 当前实现：实时评估返回触发结果
|
||||
# P6 演进：将预警历史持久化到 ClickHouse 供查询
|
||||
triggered = await evaluate_all_warnings(student_id=student_id)
|
||||
if warning_type:
|
||||
triggered = [w for w in triggered if w.get("warning_type") == warning_type]
|
||||
return triggered
|
||||
|
||||
|
||||
async def trigger_warning_manual(
|
||||
target_id: str,
|
||||
warning_type: str,
|
||||
threshold: float,
|
||||
current_value: float,
|
||||
student_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
severity: str = "",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""手动触发预警（TriggerWarning RPC 实现，管理员/API 用）."""
|
||||
# 覆盖默认 severity
|
||||
if severity:
|
||||
_SEVERITY_MAP[warning_type] = severity
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
Reference in New Issue
Block a user