chore(data-ana): merge data-ana full implementation into main
Merge feat/data-ana-ai11 with complete data analytics service
This commit is contained in:
SpecialX
@@ -339,7 +339,7 @@
|
||||
| CDC 水平扩展 | P4 单实例 + 内存 ExamCache LRU → P6 多实例 + Redis ExamCache + partition 扩容 + HPA |
|
||||
| CDC 手动 commit | v2 改进：`enable_auto_commit=False` 手动 commit（at-least-once），commit 前确保 ClickHouse 写入成功 |
|
||||
| ClickHouse 容量规划 | 5 宽表年增量 ~2.3GB（5000 学生×20 次/月），单节点可承载；P6 按 class_id hash 分片 + ReplicatedMergeTree |
|
||||
| ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（当前实现未加，P0 整改） |
|
||||
| ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（clickhouse_repository.py 已实现 P0 整改） |
|
||||
| 缓存键命名 | `data_ana:datascope:{user_id}` 5min / `data_ana:exam:{exam_id}` 30day / `data_ana:dedup:{event_id}` 7day |
|
||||
| 4 端 Dashboard | teacher/student/parent/admin 各自 Dashboard API + 权限点 ANALYTICS_*_DASHBOARD（v2 新增 4 端权限） |
|
||||
| SubscribeMasteryUpdate | gRPC server-streaming RPC 为 P5+ AI 个性化推荐预留实时推送通道（analytics.proto v2 提案） |
|
||||
@@ -354,6 +354,9 @@
|
||||
| 降级模式 4 场景 | ClickHouse/Kafka/Redis/iam gRPC 不可用时返回骨架 + `error.details.degraded=true`，4 错误码 DATA_ANA_*_UNAVAILABLE |
|
||||
| Python gRPC 实现 | grpc.aio + betterproto + ServerInterceptor 透传 W3C trace context，与 HTTP 共享同一 Application Service |
|
||||
| structlog API | 24.x 用 `make_filtering_bound_logger(level)`，旧版 `make_filtering_logger` 已废弃 |
|
||||
| ruff per-file-ignores | gRPC servicer 方法必须 PascalCase（N802 忽略）/ FastAPI Depends 在参数默认值（B008 忽略），pyproject.toml 配 per-file-ignores |
|
||||
| Mock 数据脚本 | `scripts/data-ana-mock-data.sql` 提供 5 宽表演示数据（20 学生×2 考试×3 知识点），用 `now()-INTERVAL` 避免硬编码日期 |
|
||||
| Python 3.12+ 现代化 | ruff UP042 用 StrEnum 替代 `str, Enum` / UP046 用 PEP 695 类型参数 `class Foo[T]` 替代 `Generic[T]` |
|
||||
|
||||
### 2.7 messaging（TS/NestJS，P5）
|
||||
|
||||
|
||||
97
infra/clickhouse/ddl/data_ana.sql
Normal file
97
infra/clickhouse/ddl/data_ana.sql
Normal file
@@ -0,0 +1,97 @@
|
||||
-- data-ana ClickHouse DDL（5 宽表）.
|
||||
-- 负责人：ai11（总裁裁决 §2.12 授权 ai11 创建纯 DDL 文件，SRE AI 负责部署脚本）.
|
||||
-- 对齐文档：services/data-ana/docs/02-architecture-design.md §3.
|
||||
-- 引擎：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本（幂等消费保证）.
|
||||
-- 查询时必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改项）.
|
||||
--
|
||||
-- 使用方式：
|
||||
-- clickhouse-client --multiquery < infra/clickhouse/ddl/data_ana.sql
|
||||
|
||||
-- 数据库
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- ===== 1. 学生学情宽表 =====
|
||||
-- 每次成绩写入产生一行，按 ORDER BY 去重保留 last_updated 最大版本.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
(
|
||||
student_id String,
|
||||
class_id String,
|
||||
exam_id String,
|
||||
subject_id String,
|
||||
score Float64,
|
||||
rank_in_class UInt32,
|
||||
knowledge_point_id String,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
error_count UInt32,
|
||||
last_updated DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- ===== 2. 学生错题本 =====
|
||||
-- 同一 (student_id, question_id) 累计 error_count，按 last_error_time 去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
(
|
||||
student_id String,
|
||||
question_id String,
|
||||
knowledge_point_id String,
|
||||
error_count UInt32,
|
||||
last_error_time DateTime64(3, 'UTC'),
|
||||
content String DEFAULT ''
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
|
||||
-- ===== 3. 知识点掌握度历史快照 =====
|
||||
-- 每次掌握度计算产生新版本，支持趋势查询.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
(
|
||||
student_id String,
|
||||
knowledge_point_id String,
|
||||
subject_id String,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
calculated_at DateTime64(3, 'UTC'),
|
||||
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
|
||||
-- ===== 4. 学生考勤记录 =====
|
||||
-- core-edu attendance 表 CDC 同步，同一记录多版本去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
(
|
||||
student_id String,
|
||||
class_id String,
|
||||
attendance_date Date,
|
||||
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
|
||||
recorded_by String, -- 教师用户 ID
|
||||
remark String DEFAULT '',
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
|
||||
-- ===== 5. AI 用量计费记录 =====
|
||||
-- ai 服务通过 Kafka 事件投递，data-ana 消费落库，按 request_id 幂等去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
(
|
||||
request_id String,
|
||||
user_id String,
|
||||
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
|
||||
model LowCardinality(String),
|
||||
prompt_tokens UInt32,
|
||||
completion_tokens UInt32,
|
||||
total_tokens UInt32,
|
||||
latency_ms UInt32,
|
||||
success Boolean,
|
||||
cost_cents UInt32, -- 计费（分），便于聚合
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id);
|
||||
@@ -2,24 +2,113 @@ syntax = "proto3";
|
||||
|
||||
package next_edu_cloud.analytics.v1;
|
||||
|
||||
// AnalyticsService 数据分析服务契约（D6 智能洞察领域）.
|
||||
// P4 启用 gRPC server 端口 50055，HTTP 3006 保留作 Gateway 直连降级.
|
||||
// 所有 RPC 返回 ActionState 信封（success/data/error/details.degraded）.
|
||||
service AnalyticsService {
|
||||
// 班级成绩分析（平均分/及格率/参考人数）.
|
||||
rpc GetClassPerformance(GetClassPerformanceRequest) returns (ClassPerformance);
|
||||
// 学生薄弱知识点（mastery_level < 0.6）.
|
||||
rpc GetStudentWeakness(GetStudentWeaknessRequest) returns (StudentWeakness);
|
||||
// 学习趋势（历史成绩曲线）.
|
||||
rpc GetLearningTrend(GetLearningTrendRequest) returns (LearningTrend);
|
||||
// 教师仪表盘聚合（班级概览 + 待办 + 预警）.
|
||||
rpc GetTeacherDashboard(GetTeacherDashboardRequest) returns (TeacherDashboard);
|
||||
// 学生仪表盘（个人学情 + 排名 + 薄弱点）.
|
||||
rpc GetStudentDashboard(GetStudentDashboardRequest) returns (StudentDashboard);
|
||||
// 家长仪表盘（孩子学情概览）.
|
||||
rpc GetParentDashboard(GetParentDashboardRequest) returns (ParentDashboard);
|
||||
// 管理员仪表盘（全校统计 + AI 用量）.
|
||||
rpc GetAdminDashboard(GetAdminDashboardRequest) returns (AdminDashboard);
|
||||
// 预警列表查询（按班级/严重度/时间过滤）.
|
||||
rpc GetWarnings(GetWarningsRequest) returns (WarningList);
|
||||
// 手动触发预警（管理员/教师主动标记关注）.
|
||||
rpc TriggerWarning(TriggerWarningRequest) returns (TriggerWarningResponse);
|
||||
// 班级掌握度分布（mastered/progressing/weak 三档）.
|
||||
rpc GetMasteryDistribution(GetMasteryDistributionRequest) returns (MasteryDistribution);
|
||||
// 学生知识点掌握度明细.
|
||||
rpc GetStudentMastery(GetStudentMasteryRequest) returns (StudentMastery);
|
||||
// 订阅掌握度更新（server-streaming，P5+ AI 个性化推荐实时推送通道）.
|
||||
rpc SubscribeMasteryUpdate(SubscribeMasteryUpdateRequest) returns (stream MasteryUpdateEvent);
|
||||
}
|
||||
|
||||
// ===== 请求消息 =====
|
||||
|
||||
message GetClassPerformanceRequest {
|
||||
string class_id = 1;
|
||||
string subject_id = 2;
|
||||
int64 start_date = 3;
|
||||
int64 start_date = 3; // Unix timestamp（秒）
|
||||
int64 end_date = 4;
|
||||
}
|
||||
|
||||
message GetStudentWeaknessRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2;
|
||||
}
|
||||
|
||||
message GetLearningTrendRequest {
|
||||
string student_id = 1;
|
||||
int64 start_date = 2;
|
||||
int64 end_date = 3;
|
||||
string subject_id = 4;
|
||||
}
|
||||
|
||||
message GetTeacherDashboardRequest {
|
||||
string user_id = 1;
|
||||
string class_id = 2; // 可选，不传则返回教师全部班级
|
||||
}
|
||||
|
||||
message GetStudentDashboardRequest {
|
||||
string user_id = 1;
|
||||
}
|
||||
|
||||
message GetParentDashboardRequest {
|
||||
string user_id = 1;
|
||||
string student_id = 2; // 家长查看指定孩子
|
||||
}
|
||||
|
||||
message GetAdminDashboardRequest {
|
||||
string user_id = 1;
|
||||
string scope = 2; // ALL / SCHOOL / DISTRICT
|
||||
string scope_id = 3; // 具体范围 ID
|
||||
}
|
||||
|
||||
message GetWarningsRequest {
|
||||
string class_id = 1;
|
||||
string severity = 2; // INFO / WARN / CRITICAL
|
||||
int64 since = 3; // Unix timestamp（秒）
|
||||
}
|
||||
|
||||
message TriggerWarningRequest {
|
||||
string target_id = 1; // 学生 ID
|
||||
string warning_type = 2; // LOW_MASTERY / SCORE_DROP / ABSENT_FREQUENT
|
||||
string severity = 3; // INFO / WARN / CRITICAL
|
||||
}
|
||||
|
||||
message GetMasteryDistributionRequest {
|
||||
string class_id = 1;
|
||||
string subject_id = 2;
|
||||
string knowledge_point_id = 3; // 可选
|
||||
}
|
||||
|
||||
message GetStudentMasteryRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2; // 可选
|
||||
}
|
||||
|
||||
message SubscribeMasteryUpdateRequest {
|
||||
string student_id = 1; // 可选，订阅指定学生
|
||||
string class_id = 2; // 可选，订阅指定班级全部学生
|
||||
}
|
||||
|
||||
// ===== 响应消息 =====
|
||||
|
||||
message ClassPerformance {
|
||||
string class_id = 1;
|
||||
double average_score = 2;
|
||||
double pass_rate = 3;
|
||||
repeated StudentScore scores = 4;
|
||||
int32 total_students = 4;
|
||||
repeated StudentScore scores = 5;
|
||||
}
|
||||
|
||||
message StudentScore {
|
||||
@@ -28,11 +117,6 @@ message StudentScore {
|
||||
string grade = 3;
|
||||
}
|
||||
|
||||
message GetStudentWeaknessRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2;
|
||||
}
|
||||
|
||||
message StudentWeakness {
|
||||
string student_id = 1;
|
||||
repeated WeakPoint weak_points = 2;
|
||||
@@ -42,12 +126,7 @@ message WeakPoint {
|
||||
string knowledge_point_id = 1;
|
||||
string title = 2;
|
||||
double mastery = 3;
|
||||
}
|
||||
|
||||
message GetLearningTrendRequest {
|
||||
string student_id = 1;
|
||||
int64 start_date = 2;
|
||||
int64 end_date = 3;
|
||||
int32 error_count = 4;
|
||||
}
|
||||
|
||||
message LearningTrend {
|
||||
@@ -59,3 +138,125 @@ message TrendPoint {
|
||||
int64 date = 1;
|
||||
double score = 2;
|
||||
}
|
||||
|
||||
message TeacherDashboard {
|
||||
string user_id = 1;
|
||||
int32 total_classes = 2;
|
||||
int32 total_students = 3;
|
||||
double class_avg_score = 4;
|
||||
int32 pending_homework_count = 5;
|
||||
repeated ClassSummary classes = 6;
|
||||
repeated StudentSummary top_students = 7;
|
||||
repeated WarningInfo recent_warnings = 8;
|
||||
}
|
||||
|
||||
message ClassSummary {
|
||||
string class_id = 1;
|
||||
string class_name = 2;
|
||||
int32 student_count = 3;
|
||||
double average_score = 4;
|
||||
}
|
||||
|
||||
message StudentSummary {
|
||||
string student_id = 1;
|
||||
string student_name = 2;
|
||||
double score = 3;
|
||||
int32 rank_in_class = 4;
|
||||
}
|
||||
|
||||
message StudentDashboard {
|
||||
string user_id = 1;
|
||||
double avg_score = 2;
|
||||
int32 class_rank = 3;
|
||||
int32 total_students = 4;
|
||||
repeated WeakPoint weak_points = 5;
|
||||
repeated TrendPoint recent_trends = 6;
|
||||
int32 pending_homework = 7;
|
||||
}
|
||||
|
||||
message ParentDashboard {
|
||||
string user_id = 1;
|
||||
string student_id = 2;
|
||||
double child_avg_score = 3;
|
||||
int32 child_class_rank = 4;
|
||||
int32 total_class_students = 5;
|
||||
repeated WeakPoint child_weak_points = 6;
|
||||
repeated WarningInfo child_warnings = 7;
|
||||
}
|
||||
|
||||
message AdminDashboard {
|
||||
string user_id = 1;
|
||||
int32 total_teachers = 2;
|
||||
int32 total_students = 3;
|
||||
int32 total_classes = 4;
|
||||
double school_avg_score = 5;
|
||||
repeated WarningInfo recent_warnings = 6;
|
||||
AIUsageSummary ai_usage = 7;
|
||||
}
|
||||
|
||||
message AIUsageSummary {
|
||||
int64 total_requests = 1;
|
||||
int64 total_tokens = 2;
|
||||
int64 total_cost_cents = 3;
|
||||
repeated AIUsageByProvider by_provider = 4;
|
||||
}
|
||||
|
||||
message AIUsageByProvider {
|
||||
string provider = 1;
|
||||
int64 request_count = 2;
|
||||
int64 total_tokens = 3;
|
||||
int64 cost_cents = 4;
|
||||
}
|
||||
|
||||
message WarningList {
|
||||
repeated WarningInfo warnings = 1;
|
||||
int32 total = 2;
|
||||
}
|
||||
|
||||
message WarningInfo {
|
||||
string warning_id = 1;
|
||||
string warning_type = 2; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE
|
||||
string target_id = 3; // 学生 ID
|
||||
string target_name = 4;
|
||||
double threshold = 5;
|
||||
double current_value = 6;
|
||||
string severity = 7; // INFO / WARN / CRITICAL
|
||||
int64 occurred_at = 8; // Unix timestamp（秒）
|
||||
}
|
||||
|
||||
message TriggerWarningResponse {
|
||||
string warning_id = 1;
|
||||
bool triggered = 2;
|
||||
}
|
||||
|
||||
message MasteryDistribution {
|
||||
string class_id = 1;
|
||||
int32 mastered_count = 2; // mastery >= 0.8
|
||||
int32 progressing_count = 3; // 0.4 <= mastery < 0.8
|
||||
int32 weak_count = 4; // mastery < 0.4
|
||||
int32 total_students = 5;
|
||||
}
|
||||
|
||||
message StudentMastery {
|
||||
string student_id = 1;
|
||||
repeated KnowledgePointMastery knowledge_points = 2;
|
||||
double overall_mastery = 3;
|
||||
}
|
||||
|
||||
message KnowledgePointMastery {
|
||||
string knowledge_point_id = 1;
|
||||
string title = 2;
|
||||
string subject_id = 3;
|
||||
double mastery_level = 4; // 0.0-1.0
|
||||
string mastery_label = 5; // mastered / progressing / weak
|
||||
int64 calculated_at = 6; // Unix timestamp（秒）
|
||||
}
|
||||
|
||||
message MasteryUpdateEvent {
|
||||
string event_id = 1;
|
||||
string student_id = 2;
|
||||
string knowledge_point_id = 3;
|
||||
double mastery_level = 4;
|
||||
double previous_level = 5;
|
||||
int64 calculated_at = 6; // Unix timestamp（秒）
|
||||
}
|
||||
|
||||
@@ -6,18 +6,15 @@ package next_edu_cloud.events.v1;
|
||||
// outbox pattern and consumed by downstream services (notifications, analytics,
|
||||
// audit, etc.).
|
||||
//
|
||||
// Topic naming follows the arbitration in coord-cross-review §3.1:
|
||||
// edu.teaching.<aggregate>.<action>
|
||||
//
|
||||
// Event routing (TOPIC_MAP in outbox.publisher.ts):
|
||||
// edu.exam.events <- exam.created / exam.updated / exam.deleted
|
||||
// edu.homework.events <- homework.assigned / homework.submitted / homework.graded
|
||||
// edu.grade.events <- grade.recorded / grade.updated
|
||||
// edu.class.events <- class.transferred
|
||||
// edu.content.textbook.events <- textbook.created / textbook.updated / textbook.published / textbook.archived
|
||||
// edu.content.chapter.events <- chapter.created / chapter.updated / chapter.deleted
|
||||
// edu.content.knowledge_point.events <- knowledge_point.created / knowledge_point.updated / knowledge_point.prerequisite_added / knowledge_point.prerequisite_removed
|
||||
// edu.content.question.events <- question.created / question.updated / question.published / question.deleted
|
||||
// edu.teaching.exam.published <- exam.published
|
||||
// edu.teaching.homework.assigned <- homework.assigned / homework.submitted / homework.graded
|
||||
// edu.teaching.grade.recorded <- grade.recorded / grade.updated
|
||||
// edu.teaching.class.transferred <- class.transferred
|
||||
//
|
||||
// Derived data events (Outbox exempt, see 004 §12.2):
|
||||
// edu.insight.mastery.updated <- mastery.updated / warning.triggered (action field distinguishes)
|
||||
// edu.insight.ai.usage <- ai.usage.recorded
|
||||
|
||||
message ClassEvent {
|
||||
string event_id = 1;
|
||||
@@ -185,3 +182,40 @@ message QuestionEvent {
|
||||
string action = 12;
|
||||
map<string, string> metadata = 13;
|
||||
}
|
||||
|
||||
// MasteryEvent 掌握度/预警派生数据事件（Outbox 豁免，直接 Kafka producer 发布）.
|
||||
// action 字段区分：mastery.updated（掌握度更新）/ warning.triggered（预警触发）.
|
||||
// 两者复用同一 topic edu.insight.mastery.updated（总裁裁决 §2.11）.
|
||||
message MasteryEvent {
|
||||
string event_id = 1;
|
||||
string action = 2; // mastery.updated / warning.triggered
|
||||
int64 occurred_at = 3;
|
||||
string student_id = 4;
|
||||
string knowledge_point_id = 5;
|
||||
double mastery_level = 6; // 0.0-1.0（mastery.updated 时有效）
|
||||
double previous_level = 7; // 上一次掌握度（mastery.updated 时有效）
|
||||
// warning.triggered 时以下字段有效
|
||||
string warning_type = 8; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE
|
||||
string target_id = 9; // 预警目标（学生 ID）
|
||||
double threshold = 10; // 预警阈值
|
||||
double current_value = 11; // 当前值
|
||||
string severity = 12; // INFO / WARN / CRITICAL
|
||||
map<string, string> metadata = 13;
|
||||
}
|
||||
|
||||
// AIUsageEvent AI 用量计费事件（ai 服务发布，data-ana 消费落 ai_usage_log）.
|
||||
message AIUsageEvent {
|
||||
string event_id = 1;
|
||||
string request_id = 2;
|
||||
string user_id = 3;
|
||||
string provider = 4; // openai / anthropic / baichuan / local
|
||||
string model = 5;
|
||||
uint32 prompt_tokens = 6;
|
||||
uint32 completion_tokens = 7;
|
||||
uint32 total_tokens = 8;
|
||||
uint32 latency_ms = 9;
|
||||
bool success = 10;
|
||||
uint32 cost_cents = 11; // 计费（分）
|
||||
int64 occurred_at = 12;
|
||||
map<string, string> metadata = 13;
|
||||
}
|
||||
|
||||
@@ -15,17 +15,9 @@ service IamService {
|
||||
|
||||
// 用户信息类
|
||||
rpc GetUserInfo(GetUserInfoRequest) returns (UserInfo);
|
||||
rpc BatchGetUsers(BatchGetUsersRequest) returns (BatchGetUsersResponse);
|
||||
|
||||
// 权限与视口类
|
||||
rpc GetEffectivePermissions(GetEffectivePermissionsRequest) returns (EffectivePermissionsResponse);
|
||||
rpc GetEffectiveAccess(GetEffectiveAccessRequest) returns (EffectiveAccessResponse);
|
||||
rpc GetEffectiveDataScope(GetEffectiveDataScopeRequest) returns (DataScopeResponse);
|
||||
rpc GetViewports(GetViewportsRequest) returns (ViewportsResponse);
|
||||
|
||||
// 密钥与关系类
|
||||
rpc GetPublicKey(GetPublicKeyRequest) returns (PublicKeyResponse);
|
||||
rpc GetChildrenByParent(GetChildrenByParentRequest) returns (ChildrenResponse);
|
||||
// GetEffectiveDataScope 解析用户可见数据范围（DataScope 6 级）.
|
||||
// data-ana gRPC 调用此 RPC 解析查询过滤范围（coord-cross-review §2 #3 裁决 P4 补全）.
|
||||
rpc GetEffectiveDataScope(GetEffectiveDataScopeRequest) returns (EffectiveDataScope);
|
||||
}
|
||||
|
||||
// ========== 认证类 ==========
|
||||
@@ -157,3 +149,17 @@ message ChildInfo {
|
||||
string name = 2;
|
||||
string relation = 3;
|
||||
}
|
||||
|
||||
message GetEffectiveDataScopeRequest {
|
||||
string user_id = 1;
|
||||
}
|
||||
|
||||
// EffectiveDataScope 用户可见数据范围（6 级）.
|
||||
// level: SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
|
||||
// scope_ids: 具体可见的 class_id / grade_id 列表（SELF/ALL 时为空）
|
||||
message EffectiveDataScope {
|
||||
string user_id = 1;
|
||||
string level = 2; // SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
|
||||
repeated string scope_ids = 3; // class_id 列表（CLASS 级）/ grade_id 列表（GRADE 级）
|
||||
string school_id = 4; // SCHOOL 级时的学校 ID
|
||||
}
|
||||
|
||||
2
pnpm-lock.yaml
generated
2
pnpm-lock.yaml
generated
@@ -222,7 +222,7 @@ importers:
|
||||
version: 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
'@nestjs/core':
|
||||
specifier: ^10.4.0
|
||||
version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/microservices@10.4.22)(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
'@paralleldrive/cuid2':
|
||||
specifier: ^2.2.2
|
||||
version: 2.3.1
|
||||
|
||||
@@ -1,16 +1,23 @@
|
||||
-- ClickHouse 数据库初始化脚本
|
||||
-- 适用服务：data-ana（数据分析）
|
||||
-- 表结构：student_dashboard_view（学生学情宽表）/ student_errors（错题本）
|
||||
-- 与 services/data-ana/src/data_ana/clickhouse_client.py 中的查询字段对齐
|
||||
-- 适用服务：data-ana（D6 智能洞察领域）
|
||||
-- 5 张宽表（ai-allocation §5 强制）：
|
||||
-- 1. student_dashboard_view 学生学情宽表（成绩/班级/知识点维度）
|
||||
-- 2. student_errors 学生错题本
|
||||
-- 3. mastery_snapshot 知识点掌握度历史快照
|
||||
-- 4. ai_usage_log AI 用量计费记录（ai 服务投递，data-ana 消费）
|
||||
-- 5. attendance_logs 学生考勤记录（core-edu attendance 表 CDC 同步）
|
||||
--
|
||||
-- 引擎对齐 02-architecture-design.md §3：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本
|
||||
-- 查询规范：所有 SELECT 必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改）
|
||||
--
|
||||
-- 使用方式（启用 ClickHouse 时执行一次）：
|
||||
-- clickhouse-client --multiquery < scripts/clickhouse-init.sql
|
||||
-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式。
|
||||
-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式（返回骨架数据）。
|
||||
|
||||
-- 数据库
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- 学生学情宽表（考试/班级/知识点维度）
|
||||
-- ===== 1. 学生学情宽表（成绩写入产生一行，按 ORDER BY 去重保留最新版本） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view (
|
||||
student_id String,
|
||||
class_id String,
|
||||
@@ -19,19 +26,64 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view (
|
||||
score Float64,
|
||||
rank_in_class UInt32,
|
||||
knowledge_point_id String,
|
||||
mastery_level Float32,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
error_count UInt32,
|
||||
last_updated DateTime
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY (student_id, class_id, exam_id);
|
||||
last_updated DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 学生错题表（错题本）
|
||||
-- ===== 2. 学生错题本 =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors (
|
||||
student_id String,
|
||||
question_id String,
|
||||
knowledge_point_id String,
|
||||
error_count UInt32,
|
||||
last_error_time DateTime,
|
||||
last_error_time DateTime64(3, 'UTC'),
|
||||
content String
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY (student_id, knowledge_point_id);
|
||||
) ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
|
||||
-- ===== 3. 掌握度历史快照（每次掌握度计算产生新版本，支持趋势查询） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot (
|
||||
student_id String,
|
||||
knowledge_point_id String,
|
||||
subject_id String,
|
||||
mastery_level Float32,
|
||||
calculated_at DateTime64(3, 'UTC'),
|
||||
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
|
||||
-- ===== 4. AI 用量计费记录（ai 服务通过 Kafka 事件投递，data-ana 消费落库） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log (
|
||||
request_id String,
|
||||
user_id String,
|
||||
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
|
||||
model LowCardinality(String),
|
||||
prompt_tokens UInt32,
|
||||
completion_tokens UInt32,
|
||||
total_tokens UInt32,
|
||||
latency_ms UInt32,
|
||||
success Boolean,
|
||||
cost_cents UInt32, -- 计费（分），便于聚合
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id); -- 按 request_id 幂等去重
|
||||
|
||||
-- ===== 5. 学生考勤记录（core-edu attendance 表 CDC 同步） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs (
|
||||
student_id String,
|
||||
class_id String,
|
||||
attendance_date Date,
|
||||
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
|
||||
recorded_by String, -- 教师用户 ID
|
||||
remark String DEFAULT '',
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
|
||||
131
scripts/data-ana-mock-data.sql
Normal file
131
scripts/data-ana-mock-data.sql
Normal file
@@ -0,0 +1,131 @@
|
||||
-- data-ana 模块 Mock 数据脚本
|
||||
-- 用途：本地开发/集成测试时为 5 张宽表插入演示数据，验证查询逻辑
|
||||
-- 使用方式（需先执行 clickhouse-init.sql）：
|
||||
-- clickhouse-client --multiquery < scripts/data-ana-mock-data.sql
|
||||
--
|
||||
-- 数据规模：2 班 × 10 学生 × 3 知识点 × 2 考试 = 120 行 student_dashboard_view
|
||||
-- 20 条错题 / 60 条掌握度快照 / 20 条考勤 / 10 条 AI 用量
|
||||
-- 所有时间戳使用 now() - N days 模式，避免硬编码绝对日期导致脚本过期
|
||||
|
||||
USE edu_analytics;
|
||||
|
||||
-- ===== 1. student_dashboard_view：学情宽表 =====
|
||||
-- 班级 C001（高一 1 班）+ C002（高一 2 班），每班 10 学生
|
||||
-- 2 次考试（EXAM001 数学月考 / EXAM002 数学期中），3 个数学知识点
|
||||
INSERT INTO student_dashboard_view
|
||||
SELECT
|
||||
'S' || toString(number + 1) AS student_id, -- S001..S020
|
||||
IF(number < 10, 'C001', 'C002') AS class_id, -- 前 10 在 C001，后 10 在 C002
|
||||
'EXAM001' AS exam_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
40 + (number % 60) AS score, -- 40-99 分
|
||||
(number % 10) + 1 AS rank_in_class, -- 1-10 名
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, -- KP_MATH_1/2/3
|
||||
0.3 + ((number % 70) / 100.0) AS mastery_level, -- 0.30-0.99
|
||||
(number % 5) AS error_count, -- 0-4 错题
|
||||
now() - INTERVAL 7 DAY AS last_updated
|
||||
FROM numbers(20);
|
||||
|
||||
-- 第二次考试（EXAM002），成绩略升
|
||||
INSERT INTO student_dashboard_view
|
||||
SELECT
|
||||
'S' || toString(number + 1) AS student_id,
|
||||
IF(number < 10, 'C001', 'C002') AS class_id,
|
||||
'EXAM002' AS exam_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
50 + (number % 50) AS score, -- 50-99 分
|
||||
(number % 10) + 1 AS rank_in_class,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
0.4 + ((number % 60) / 100.0) AS mastery_level,
|
||||
(number % 4) AS error_count,
|
||||
now() - INTERVAL 1 DAY AS last_updated
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 2. student_errors：错题本 =====
|
||||
INSERT INTO student_errors
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'Q_' || toString(number + 1) AS question_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
(number % 5) + 1 AS error_count,
|
||||
now() - INTERVAL (number % 30) DAY AS last_error_time,
|
||||
'错误类型：' || multiIf(
|
||||
number % 3 = 0, '计算失误',
|
||||
number % 3 = 1, '公式记忆错误',
|
||||
'审题不清'
|
||||
) AS content
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 3. mastery_snapshot：掌握度历史快照（3 次计算，便于趋势查询） =====
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.3 + ((number % 70) / 100.0) AS mastery_level,
|
||||
now() - INTERVAL 30 DAY AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.4 + ((number % 60) / 100.0) AS mastery_level,
|
||||
now() - INTERVAL 15 DAY AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.5 + ((number % 50) / 100.0) AS mastery_level,
|
||||
now() AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 4. attendance_logs：考勤记录（近 5 天，含缺勤/迟到） =====
|
||||
INSERT INTO attendance_logs
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
IF(number % 20 < 10, 'C001', 'C002') AS class_id,
|
||||
today() - (number % 5) AS attendance_date,
|
||||
multiIf(
|
||||
number % 10 = 0, 'absent', -- 10% 缺勤
|
||||
number % 5 = 0, 'late', -- 20% 迟到
|
||||
'present' -- 其余正常
|
||||
) AS status,
|
||||
'T001' AS recorded_by,
|
||||
multiIf(number % 10 = 0, '病假', '') AS remark,
|
||||
now() - INTERVAL (number % 5) DAY AS occurred_at
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 5. ai_usage_log：AI 用量计费（模拟 3 个 provider） =====
|
||||
INSERT INTO ai_usage_log
|
||||
SELECT
|
||||
'REQ_' || toString(number + 1) AS request_id,
|
||||
'S' || toString((number % 20) + 1) AS user_id,
|
||||
arrayElement(['openai', 'anthropic', 'baichuan'], (number % 3) + 1) AS provider,
|
||||
arrayElement(['gpt-4o-mini', 'claude-3-haiku', 'baichuan2-turbo'], (number % 3) + 1) AS model,
|
||||
100 + (number % 200) AS prompt_tokens, -- 100-299
|
||||
50 + (number % 150) AS completion_tokens, -- 50-199
|
||||
150 + (number % 350) AS total_tokens, -- 150-499
|
||||
500 + (number % 2000) AS latency_ms, -- 500-2499 ms
|
||||
number % 5 != 0 AS success, -- 80% 成功
|
||||
(number % 50) + 1 AS cost_cents, -- 1-50 分
|
||||
now() - INTERVAL (number % 24) HOUR AS occurred_at
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 校验查询（可选执行） =====
|
||||
SELECT 'student_dashboard_view' AS table_name, count() AS cnt FROM student_dashboard_view FINAL
|
||||
UNION ALL
|
||||
SELECT 'student_errors', count() FROM student_errors FINAL
|
||||
UNION ALL
|
||||
SELECT 'mastery_snapshot', count() FROM mastery_snapshot
|
||||
UNION ALL
|
||||
SELECT 'attendance_logs', count() FROM attendance_logs FINAL
|
||||
UNION ALL
|
||||
SELECT 'ai_usage_log', count() FROM ai_usage_log FINAL;
|
||||
@@ -1,8 +1,42 @@
|
||||
FROM python:3.12-slim
|
||||
# 多阶段构建：builder 缓存依赖，runtime 精简镜像
|
||||
# 对齐 02-architecture-design.md §16 部署规范
|
||||
|
||||
# ===== Stage 1: builder =====
|
||||
FROM python:3.12-slim AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装 uv（快速依赖管理）
|
||||
RUN pip install uv
|
||||
|
||||
# 先复制 pyproject.toml，利用 Docker 层缓存
|
||||
COPY pyproject.toml .
|
||||
|
||||
# 同步依赖到虚拟环境（--no-dev 不装开发依赖）
|
||||
RUN uv sync --no-dev
|
||||
|
||||
# ===== Stage 2: runtime =====
|
||||
FROM python:3.12-slim AS runtime
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 从 builder 复制虚拟环境
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
|
||||
# 确保虚拟环境在 PATH 中
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# 复制源码
|
||||
COPY src ./src
|
||||
EXPOSE 3006
|
||||
CMD ["uv", "run", "uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"]
|
||||
|
||||
# 暴露端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（AnalyticsService 12 RPC）
|
||||
EXPOSE 3006 50055
|
||||
|
||||
# 健康检查（HTTP /healthz，每 30s 一次）
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:3006/healthz')" || exit 1
|
||||
|
||||
# 启动命令：uvicorn HTTP server（gRPC server 在 lifespan 中启动）
|
||||
CMD ["uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"]
|
||||
|
||||
@@ -1,58 +1,386 @@
|
||||
# data-ana 数据分析服务
|
||||
|
||||
> 版本：0.1（P4 骨架）
|
||||
> 端口：3006
|
||||
> 版本：1.0（P4 完整版）
|
||||
> 阶段：D6 智能洞察领域（仲裁已对齐，v2 实现版）
|
||||
> 端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（主入口，BFF/ai 调用）
|
||||
> 日期：2026-07-10
|
||||
|
||||
## 职责
|
||||
|
||||
数据分析限界上下文（Python 实现），消费 core-edu 与 content 的领域事件，
|
||||
构建 ClickHouse 学情宽表，计算知识点掌握度。
|
||||
对外提供学情仪表盘查询、班级/年级/学校维度报表、个性化推荐数据支撑。
|
||||
数据分析限界上下文（Python/FastAPI 微服务），消费 core-edu / content / iam / ai 的 CDC 事件，
|
||||
构建 ClickHouse 学情宽表，计算知识点掌握度，评估预警，为前端提供 4 端 Dashboard 查询服务。
|
||||
|
||||
### 核心能力
|
||||
|
||||
- **5 张宽表**：student_dashboard_view / student_errors / mastery_snapshot / attendance_logs / ai_usage_log
|
||||
- **CDC 链路**：Debezium → Kafka → data-ana 消费 → ClickHouse（at-least-once + 手动 commit + ReplacingMergeTree 去重）
|
||||
- **掌握度算法**：WEIGHTED_MOVING_AVG（w_i=0.6^i）+ FORGETTING_CURVE（半衰期 30 天）取 max
|
||||
- **预警评估**：5 类预警（LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE）+ Redis 25h 去重
|
||||
- **DataScope 过滤**：6 级（SELF/CLASS/GRADE/SCHOOL/DISTRICT/ALL），iam gRPC + 角色降级
|
||||
- **gRPC 12 RPC**：AnalyticsService（含 server-streaming SubscribeMasteryUpdate）+ HealthService
|
||||
- **HTTP 14 端点**：3 基础 + 11 业务（ActionState 信封 + degraded 标记）
|
||||
|
||||
## 架构图
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph Entry["入口层"]
|
||||
HTTP[FastAPI HTTP<br/>:3006 /analytics/* + /healthz + /readyz]
|
||||
GRPC[grpc.aio Server<br/>:50055 AnalyticsService 12 RPC]
|
||||
end
|
||||
|
||||
subgraph Service["应用服务层"]
|
||||
S1[AnalyticsService<br/>4 Dashboard + 班级/趋势查询]
|
||||
S2[MasteryService<br/>掌握度计算 + 事件发布]
|
||||
S3[WarningService<br/>5 类预警 + Redis 去重]
|
||||
end
|
||||
|
||||
subgraph Repo["数据访问层"]
|
||||
R1[ClickHouseRepository<br/>FINAL/argMax + DataScope WHERE]
|
||||
R2[KafkaProducer<br/>mastery.updated 派生事件]
|
||||
R3[IamClient<br/>gRPC GetEffectiveDataScope]
|
||||
R4[RedisClient<br/>DataScope 缓存 + 幂等]
|
||||
end
|
||||
|
||||
subgraph Consumer["CDC 消费者（后台任务）"]
|
||||
C1[CdcConsumer<br/>手动 commit + 7 类事件路由]
|
||||
C2[ExamCache<br/>LRU 10000 exam_id→metadata]
|
||||
end
|
||||
|
||||
subgraph Storage["存储/总线"]
|
||||
CH[(ClickHouse<br/>5 宽表)]
|
||||
KAFKA[(Kafka<br/>edu-cdc.* + edu.insight.mastery.updated)]
|
||||
IAM[iam gRPC :50052]
|
||||
REDIS[(Redis<br/>data_ana:* 键)]
|
||||
end
|
||||
|
||||
HTTP --> S1
|
||||
HTTP --> S3
|
||||
GRPC --> S1
|
||||
S1 --> R1
|
||||
S2 --> R1
|
||||
S2 --> R2
|
||||
S3 --> R1
|
||||
S3 --> R2
|
||||
R1 --> CH
|
||||
R2 --> KAFKA
|
||||
R3 --> IAM
|
||||
R3 --> R4
|
||||
R4 --> REDIS
|
||||
C1 --> C2
|
||||
C1 --> R1
|
||||
C1 --> R4
|
||||
C1 --> KAFKA
|
||||
```
|
||||
|
||||
## 技术栈
|
||||
|
||||
- Python 3.12+ / FastAPI 0.115+
|
||||
- clickhouse-connect（ClickHouse 宽表查询）
|
||||
- pydantic + pydantic-settings（运行时校验与配置）
|
||||
- structlog（结构化日志）
|
||||
- prometheus-client（指标）
|
||||
- OpenTelemetry（分布式追踪）
|
||||
|
||||
## 开发
|
||||
|
||||
```bash
|
||||
uv sync
|
||||
uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload
|
||||
```
|
||||
|
||||
## 配置
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `PORT` | 3006 | 服务端口 |
|
||||
| `CLICKHOUSE_HOST` | localhost | ClickHouse 主机 |
|
||||
| `CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 |
|
||||
| `CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 |
|
||||
| `OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry 端点 |
|
||||
| `LOG_LEVEL` | info | 日志级别 |
|
||||
- **Python 3.12+** / **FastAPI 0.115+**（HTTP 端点）
|
||||
- **grpcio 1.66+** / **grpcio-health-checking**（gRPC server + HealthService）
|
||||
- **clickhouse-connect 0.7+**（ClickHouse 宽表查询，支持 FINAL/argMax）
|
||||
- **aiokafka 0.11+**（CDC 消费者，手动 commit + 幂等去重）
|
||||
- **redis 5.0+**（DataScope 缓存 + 事件去重 + 预警位图）
|
||||
- **pydantic 2.9+** / **pydantic-settings 2.5+**（模型校验 + 配置）
|
||||
- **structlog 24.4+**（结构化 JSON 日志）
|
||||
- **prometheus-client 0.20+**（/metrics 指标）
|
||||
- **opentelemetry-sdk 1.27+**（OTLP 链路追踪）
|
||||
- **protobuf 5.28+**（buf generate 生成的 stub 运行时）
|
||||
|
||||
## 模块结构
|
||||
|
||||
```
|
||||
src/data_ana/
|
||||
├─ __init__.py
|
||||
├─ main.py # FastAPI 入口（健康检查 + 分析端点骨架）
|
||||
├─ config.py # pydantic-settings 配置
|
||||
└─ clickhouse_client.py # ClickHouse 客户端单例
|
||||
├─ main.py # FastAPI 入口（14 端点 + lifespan + ActionState）
|
||||
├─ config.py # pydantic-settings 配置（env_prefix=DATA_ANA_）
|
||||
├─ analytics_service.py # 4 Dashboard + 班级/趋势查询编排
|
||||
├─ mastery_service.py # 掌握度算法（WMA + ForgettingCurve）
|
||||
├─ warning_service.py # 5 类预警评估 + Redis 去重
|
||||
├─ exam_cache.py # LRU exam_id→metadata 映射（max 10000）
|
||||
├─ cdc_consumer.py # CDC 消费者（7 类事件 + 手动 commit）
|
||||
├─ grpc_server.py # 12 RPC + HealthService + SubscribeMasteryUpdate
|
||||
├─ clickhouse_client.py # ClickHouse 客户端单例（降级模式）
|
||||
├─ repository/
|
||||
│ ├─ __init__.py
|
||||
│ ├─ clickhouse_repository.py # 宽表查询 + FINAL/argMax + DataScope WHERE
|
||||
│ ├─ redis_client.py # Redis 异步客户端（cache + dedup + bitmap）
|
||||
│ ├─ iam_client.py # iam gRPC GetEffectiveDataScope + 角色降级
|
||||
│ └─ kafka_producer.py # mastery.updated / warning.triggered 发布
|
||||
└─ shared/
|
||||
├─ __init__.py
|
||||
├─ action_state.py # ActionState[T] 统一信封（ok/fail + degraded）
|
||||
├─ errors.py # ErrorCode 13 类 + DataAnaError
|
||||
└─ permissions.py # 权限点 + DataScopeLevel + build_datascope_where
|
||||
```
|
||||
|
||||
## 关键端点
|
||||
## 开发
|
||||
|
||||
- `GET /healthz` 健康检查
|
||||
- `GET /metrics` Prometheus 指标
|
||||
- `GET /analytics/class/{class_id}/performance` 班级成绩分析（P4 骨架）
|
||||
- `GET /analytics/student/{student_id}/weakness` 学生薄弱知识点分析（P4 骨架）
|
||||
```bash
|
||||
# 安装依赖
|
||||
uv sync
|
||||
|
||||
# 本地开发（HTTP + gRPC + CDC 同时启动）
|
||||
uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload
|
||||
|
||||
# ruff 校验
|
||||
ruff check src/
|
||||
|
||||
# 类型检查（如已配置 mypy）
|
||||
mypy src/
|
||||
```
|
||||
|
||||
## 配置项
|
||||
|
||||
所有配置使用 `DATA_ANA_` 前缀（pydantic-settings env_prefix）。
|
||||
|
||||
### 基础配置
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
| ------------------------ | --------------------- | ------------------------------------ |
|
||||
| `DATA_ANA_HTTP_PORT` | 3006 | HTTP 端口（Gateway 直连降级） |
|
||||
| `DATA_ANA_GRPC_PORT` | 50055 | gRPC 端口（AnalyticsService 主入口） |
|
||||
| `DATA_ANA_DEV_MODE` | true | 开发模式（关闭签名校验） |
|
||||
| `DATA_ANA_LOG_LEVEL` | info | 日志级别（debug/info/warning/error） |
|
||||
| `DATA_ANA_OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry OTLP 端点 |
|
||||
|
||||
### 外部依赖（全部可选，未配置则降级模式）
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
| ------------------------------ | ------------- | ------------------------------------------ |
|
||||
| `DATA_ANA_CLICKHOUSE_HOST` | "" | ClickHouse 主机（空字符串=降级模式） |
|
||||
| `DATA_ANA_CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 |
|
||||
| `DATA_ANA_CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 |
|
||||
| `DATA_ANA_CLICKHOUSE_USER` | "" | ClickHouse 用户名 |
|
||||
| `DATA_ANA_CLICKHOUSE_PASSWORD` | "" | ClickHouse 密码 |
|
||||
| `DATA_ANA_KAFKA_BROKERS` | "" | Kafka broker 列表（逗号分隔，空=禁用 CDC） |
|
||||
| `DATA_ANA_KAFKA_GROUP_ID` | data-ana-cdc | 消费者组 ID |
|
||||
| `DATA_ANA_REDIS_URL` | "" | Redis URL（空=禁用缓存/去重） |
|
||||
| `DATA_ANA_IAM_GRPC_ENDPOINT` | "" | iam gRPC 端点（空=角色降级） |
|
||||
|
||||
### 降级模式说明
|
||||
|
||||
- **ClickHouse 未配置/不可达**：查询返回骨架数据（空数组 + degraded=true）
|
||||
- **Kafka 未配置**：CDC 消费者不启动，`/readyz` 仍就绪
|
||||
- **Redis 未配置/不可达**：缓存失效，每次查询重新计算 DataScope
|
||||
- **iam gRPC 未配置/不可达**：基于 JWT 角色 fallback（teacher→CLASS，student→SELF）
|
||||
- **grpcio 未安装**：gRPC server 不启动，HTTP 仍可用
|
||||
|
||||
## 端点清单
|
||||
|
||||
### 基础端点（3 个）
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| ---- | ---------- | ----------------------------------------------------------- |
|
||||
| GET | `/` | 根信息（服务名/版本/端点列表） |
|
||||
| GET | `/healthz` | liveness 检查（进程存活即 ok） |
|
||||
| GET | `/readyz` | readiness 检查（5 依赖状态：clickhouse/cdc/redis/iam/grpc） |
|
||||
| GET | `/metrics` | Prometheus 指标 |
|
||||
|
||||
### 业务端点（11 个，全部返回 ActionState[T]）
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| ---- | -------------------------------------------------- | ------------------------------- |
|
||||
| GET | `/analytics/class/{class_id}/performance` | 班级成绩分析（平均分/及格率） |
|
||||
| GET | `/analytics/class/{class_id}/mastery-distribution` | 班级掌握度分布（三档） |
|
||||
| GET | `/analytics/student/{student_id}/weakness` | 学生薄弱知识点（mastery < 0.6） |
|
||||
| GET | `/analytics/student/{student_id}/mastery` | 学生掌握度明细 |
|
||||
| GET | `/analytics/student/{student_id}/trend` | 学习趋势（历史成绩曲线） |
|
||||
| GET | `/analytics/student/{student_id}/errorbook` | 学生错题本 |
|
||||
| GET | `/analytics/teacher/dashboard` | 教师仪表盘 |
|
||||
| GET | `/analytics/student/dashboard` | 学生仪表盘 |
|
||||
| GET | `/analytics/parent/dashboard` | 家长仪表盘 |
|
||||
| GET | `/analytics/admin/dashboard` | 管理员仪表盘 |
|
||||
| GET | `/analytics/warnings` | 预警列表 |
|
||||
| POST | `/analytics/warnings/trigger` | 手动触发预警 |
|
||||
|
||||
### ActionState 信封格式
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": { ... },
|
||||
"details": {
|
||||
"degraded": false,
|
||||
"degraded_reason": ""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
降级时 `degraded=true`，`data` 返回骨架数据（空数组/零值）。
|
||||
|
||||
## gRPC 契约
|
||||
|
||||
### AnalyticsService（12 RPC，端口 50055）
|
||||
|
||||
| RPC | 类型 | 说明 |
|
||||
| ---------------------- | ---------------- | ----------------------------------- |
|
||||
| GetClassPerformance | unary | 班级成绩分析 |
|
||||
| GetStudentWeakness | unary | 学生薄弱知识点 |
|
||||
| GetLearningTrend | unary | 学习趋势 |
|
||||
| GetTeacherDashboard | unary | 教师仪表盘 |
|
||||
| GetStudentDashboard | unary | 学生仪表盘 |
|
||||
| GetParentDashboard | unary | 家长仪表盘 |
|
||||
| GetAdminDashboard | unary | 管理员仪表盘 |
|
||||
| GetWarnings | unary | 预警列表 |
|
||||
| TriggerWarning | unary | 手动触发预警 |
|
||||
| GetMasteryDistribution | unary | 班级掌握度分布 |
|
||||
| GetStudentMastery | unary | 学生掌握度明细 |
|
||||
| SubscribeMasteryUpdate | server-streaming | 掌握度更新订阅（P5+ AI 个性化通道） |
|
||||
|
||||
### HealthService
|
||||
|
||||
`grpc.health.v1.Health` 标准 K8s 探针。
|
||||
|
||||
### proto 定义
|
||||
|
||||
- `packages/shared-proto/proto/analytics.proto`（12 RPC + 全部消息）
|
||||
- `packages/shared-proto/proto/events.proto`（MasteryEvent + AIUsageEvent）
|
||||
- `packages/shared-proto/proto/iam.proto`（GetEffectiveDataScope）
|
||||
|
||||
## CDC 事件消费
|
||||
|
||||
### 消费的 7 类事件
|
||||
|
||||
| 事件源 | Topic | 处理 |
|
||||
| ------------------------ | ------------------------------------------------- | ------------------------- |
|
||||
| core-edu.exams | `edu-cdc.next_edu_cloud.core_edu_exams` | 写 exam_cache |
|
||||
| core-edu.grades | `edu-cdc.next_edu_cloud.core_edu_grades` | 写 student_dashboard_view |
|
||||
| core-edu.homework | `edu-cdc.next_edu_cloud.core_edu_homework` | 写 student_dashboard_view |
|
||||
| core-edu.attendance | `edu-cdc.next_edu_cloud.core_edu_attendance` | 写 attendance_logs |
|
||||
| content.knowledge_points | `edu-cdc.next_edu_cloud.content_knowledge_points` | 知识点元数据 |
|
||||
| ai.ai_usage | `edu.insight.ai.usage` | 写 ai_usage_log |
|
||||
|
||||
### 幂等保证
|
||||
|
||||
- **Kafka producer**：`enable_idempotence=True` + `transactional_id`
|
||||
- **Consumer 去重**：Redis SETNX `data_ana:dedup:{event_id}` TTL 7 天
|
||||
- **ClickHouse 去重**：ReplacingMergeTree(last_updated) + FINAL/argMax 查询
|
||||
- **手动 commit**：仅在 ClickHouse 写入成功后 commit（at-least-once 语义）
|
||||
|
||||
## 发布的派生数据事件
|
||||
|
||||
| 事件 | Topic | 触发条件 | Outbox 豁免 |
|
||||
| ---------------- | ------------------------------------- | -------------- | ----------------------- |
|
||||
| MasteryUpdated | `edu.insight.mastery.updated` | 掌握度计算完成 | ✅（004 §12.2 仲裁） |
|
||||
| WarningTriggered | `edu.insight.mastery.updated`（复用） | 预警阈值触发 | ✅（004 §15.3 #6 仲裁） |
|
||||
|
||||
> Outbox 豁免理由：派生数据事件，无业务状态变更，重复消费幂等。
|
||||
|
||||
## ClickHouse 宽表
|
||||
|
||||
DDL 位置：`scripts/clickhouse-init.sql`
|
||||
|
||||
| 表名 | 引擎 | ORDER BY | 用途 |
|
||||
| ---------------------- | ----------------------------------- | ----------------------------------------------- | -------------- |
|
||||
| student_dashboard_view | ReplacingMergeTree(last_updated) | (student_id, exam_id, knowledge_point_id) | 学情宽表 |
|
||||
| student_errors | ReplacingMergeTree(last_error_time) | (student_id, question_id) | 错题本 |
|
||||
| mastery_snapshot | MergeTree | (student_id, knowledge_point_id, calculated_at) | 掌握度历史快照 |
|
||||
| ai_usage_log | ReplacingMergeTree(occurred_at) | (request_id,) | AI 用量计费 |
|
||||
| attendance_logs | ReplacingMergeTree(occurred_at) | (student_id, class_id, attendance_date) | 考勤记录 |
|
||||
|
||||
Mock 数据：`scripts/data-ana-mock-data.sql`
|
||||
|
||||
## 掌握度算法
|
||||
|
||||
### 公式
|
||||
|
||||
```
|
||||
mastery = max(WMA, ForgettingCurve)
|
||||
|
||||
WMA = Σ(w_i * score_i) / Σ(w_i), w_i = 0.6^i （近 5 次，权重递减）
|
||||
|
||||
ForgettingCurve = last_mastery * exp(-ln(2) * days_since / 30)
|
||||
```
|
||||
|
||||
### 分类
|
||||
|
||||
| 区间 | label | 说明 |
|
||||
| --------- | ----------- | ------ |
|
||||
| ≥ 0.8 | mastered | 已掌握 |
|
||||
| 0.4 - 0.8 | progressing | 进展中 |
|
||||
| < 0.4 | weak | 薄弱 |
|
||||
|
||||
### 降级规则
|
||||
|
||||
- 近 5 次成绩不足 5 次 → 降级 SIMPLE_AVG
|
||||
- 无历史数据 → 返回 0.0 + label=weak
|
||||
|
||||
## 预警类型
|
||||
|
||||
| 类型 | 阈值 | severity | 说明 |
|
||||
| --------------- | ------------- | -------- | -------- |
|
||||
| LOW_MASTERY | mastery < 0.4 | WARN | 掌握度低 |
|
||||
| CRITICAL_LOW | mastery < 0.2 | CRITICAL | 严重低 |
|
||||
| SCORE_DROP | 下降 ≥ 20% | WARN | 成绩下降 |
|
||||
| ABSENT_FREQUENT | ≥ 3 次/周 | WARN | 频繁缺勤 |
|
||||
| TREND_DECLINE | 连续 3 次降 | INFO | 趋势下降 |
|
||||
|
||||
### 去重
|
||||
|
||||
- Redis key：`data_ana:warning:{target_id}:{type}:{date}`
|
||||
- TTL：25 小时（跨日保证）
|
||||
|
||||
## DataScope 过滤
|
||||
|
||||
### 6 级
|
||||
|
||||
| 级别 | scope_ids 含义 |
|
||||
| -------- | --------------------- |
|
||||
| SELF | 学生本人 ID |
|
||||
| CLASS | 可见 class_id 列表 |
|
||||
| GRADE | 可见 grade_id 列表 |
|
||||
| SCHOOL | 可见 school_id 列表 |
|
||||
| DISTRICT | 可见 district_id 列表 |
|
||||
| ALL | 无过滤 |
|
||||
|
||||
### 解析链
|
||||
|
||||
```
|
||||
请求 → Redis 缓存（5min）→ iam gRPC GetEffectiveDataScope → 角色降级 fallback
|
||||
```
|
||||
|
||||
### 角色降级（iam 不可用时）
|
||||
|
||||
| JWT 角色 | 降级 scope |
|
||||
| -------- | --------------------- |
|
||||
| teacher | CLASS（自己教的班级） |
|
||||
| student | SELF（本人） |
|
||||
| parent | SELF（本人 + 孩子） |
|
||||
| admin | ALL |
|
||||
|
||||
## 部署
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
# 构建（多阶段：builder + runtime）
|
||||
docker build -t data-ana:1.0.0 services/data-ana/
|
||||
|
||||
# 运行（需 ClickHouse/Kafka/Redis 可达）
|
||||
docker run -p 3006:3006 -p 50055:50055 \
|
||||
-e DATA_ANA_CLICKHOUSE_HOST=clickhouse \
|
||||
-e DATA_ANA_KAFKA_BROKERS=kafka:9092 \
|
||||
-e DATA_ANA_REDIS_URL=redis://redis:6379 \
|
||||
data-ana:1.0.0
|
||||
```
|
||||
|
||||
### 健康检查
|
||||
|
||||
- HTTP `/healthz`：进程存活即 ok
|
||||
- HTTP `/readyz`：检查 5 依赖（clickhouse/cdc/redis/iam/grpc）
|
||||
- gRPC `grpc.health.v1.Health`：K8s 探针
|
||||
|
||||
## 对外契约
|
||||
|
||||
gRPC 服务 `AnalyticsService` 定义见 `packages/shared-proto/proto/analytics.proto`。
|
||||
- **gRPC proto**：`packages/shared-proto/proto/analytics.proto`
|
||||
- **HTTP OpenAPI**：启动后访问 `/docs`
|
||||
- **CDC topic 命名**：`edu-cdc.next_edu_cloud.<table>`（Debezium 标准）
|
||||
- **派生事件 topic**：`edu.insight.mastery.updated` / `edu.insight.ai.usage`
|
||||
|
||||
## 相关文档
|
||||
|
||||
- [02-architecture-design.md](./docs/02-architecture-design.md)：模块架构设计 v2（仲裁后）
|
||||
- [01-understanding.md](./docs/01-understanding.md)：阶段 1 理解确认书
|
||||
- [004 架构影响地图 §15.3](../../docs/architecture/004_architecture_impact_map.md)：coord 仲裁结论
|
||||
- [coord-cross-review.md](../../docs/architecture/coord-cross-review.md)：交叉审查记录
|
||||
- [known-issues §2.6](../../docs/troubleshooting/known-issues.md)：data-ana 已知问题速查
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "data-ana-service"
|
||||
version = "0.1.0"
|
||||
description = "数据分析服务 - ClickHouse + 学习分析"
|
||||
version = "1.0.0"
|
||||
description = "数据分析服务 - ClickHouse 宽表 + CDC + 掌握度算法 + 预警"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0",
|
||||
@@ -17,6 +17,13 @@ dependencies = [
|
||||
"structlog>=24.4.0",
|
||||
# CDC 链路：消费 Debezium 写入 Kafka 的 MySQL binlog 变更事件
|
||||
"aiokafka>=0.11.0",
|
||||
# Redis：DataScope 缓存 + 事件去重 + 预警去重位图
|
||||
"redis>=5.0.0",
|
||||
# gRPC：AnalyticsService 12 RPC + HealthService（端口 50055）
|
||||
"grpcio>=1.66.0",
|
||||
"grpcio-health-checking>=1.66.0",
|
||||
# protobuf 运行时（buf generate 生成的 stub 依赖）
|
||||
"protobuf>=5.28.0",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
@@ -25,3 +32,9 @@ target-version = "py312"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "W", "UP", "B", "SIM"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
# gRPC servicer 方法必须用 PascalCase（proto 契约约定），N802 不适用
|
||||
"src/data_ana/grpc_server.py" = ["N802"]
|
||||
# FastAPI 标准模式：Depends/Query 在参数默认值中调用，B008 不适用
|
||||
"src/data_ana/main.py" = ["B008"]
|
||||
|
||||
450
services/data-ana/src/data_ana/analytics_service.py
Normal file
450
services/data-ana/src/data_ana/analytics_service.py
Normal file
@@ -0,0 +1,450 @@
|
||||
"""分析服务（4 端 Dashboard 聚合 + DataScope 注入 + 降级兜底）.
|
||||
|
||||
对齐 02-architecture-design.md §7 AnalyticsService：
|
||||
- GetTeacherDashboard：班级聚合（教师视角）
|
||||
- GetStudentDashboard：学生学情（学生视角）
|
||||
- GetParentDashboard：孩子学情（家长视角）
|
||||
- GetAdminDashboard：学校聚合 + AI 用量（管理员视角）
|
||||
|
||||
DataScope 注入：
|
||||
- 教师只能查自己班级数据（CLASS 级 scope_ids 过滤）
|
||||
- 学生只能查自己数据（SELF 级 user_id 过滤）
|
||||
- 管理员可查全校数据（ALL 级无过滤）
|
||||
|
||||
降级策略：
|
||||
- ClickHouse 不可达：返回骨架数据 + degraded=true
|
||||
- iam gRPC 不可达：使用 role-based fallback + degraded=true
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .repository import clickhouse_repository, iam_client
|
||||
from .shared.permissions import DataScopeLevel, UserContext
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
def _skeleton_dashboard(dashboard_type: str, user_id: str, reason: str) -> dict[str, Any]:
|
||||
"""降级骨架数据（ClickHouse 不可达时返回）."""
|
||||
return {
|
||||
"userId": user_id,
|
||||
"dashboardType": dashboard_type,
|
||||
"degraded": True,
|
||||
"degraded_reason": reason,
|
||||
"totalClasses": 0,
|
||||
"totalStudents": 0,
|
||||
"averageScore": 0.0,
|
||||
"passRate": 0.0,
|
||||
"weakStudents": [],
|
||||
"recentWarnings": [],
|
||||
"trend": [],
|
||||
}
|
||||
|
||||
|
||||
async def get_teacher_dashboard(
|
||||
user: UserContext,
|
||||
class_id: str = "",
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""教师仪表盘（聚合班级成绩 + 薄弱学生 + 预警）.
|
||||
|
||||
DataScope：
|
||||
- CLASS 级：仅返回 scope_ids 内的班级数据
|
||||
- ALL 级：返回所有班级数据
|
||||
"""
|
||||
# 1. 解析 DataScope
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# 2. 若指定 class_id，验证是否在 scope 内
|
||||
effective_class_ids: list[str] = []
|
||||
if class_id:
|
||||
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
|
||||
return {
|
||||
**_skeleton_dashboard("teacher", user.user_id, "datascope_violation"),
|
||||
"error": "class_id_out_of_scope",
|
||||
}
|
||||
effective_class_ids = [class_id]
|
||||
elif scope.level == DataScopeLevel.CLASS:
|
||||
effective_class_ids = scope.scope_ids
|
||||
|
||||
# 3. 查询 ClickHouse
|
||||
if not effective_class_ids and scope.level != DataScopeLevel.ALL:
|
||||
# CLASS 级无 scope_ids，返回空骨架
|
||||
return _skeleton_dashboard("teacher", user.user_id, "no_class_scope")
|
||||
|
||||
dashboard = await clickhouse_repository.query_teacher_dashboard(
|
||||
user_id=user.user_id,
|
||||
class_id=effective_class_ids[0] if effective_class_ids else "",
|
||||
)
|
||||
if dashboard is None:
|
||||
return _skeleton_dashboard("teacher", user.user_id, "clickhouse_unavailable")
|
||||
|
||||
# 4. 补充薄弱学生列表
|
||||
weak_students: list[dict[str, Any]] = []
|
||||
if effective_class_ids:
|
||||
for cid in effective_class_ids[:5]: # 限制 5 个班级
|
||||
class_perf = await clickhouse_repository.query_class_performance(
|
||||
class_id=cid,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if class_perf is not None:
|
||||
weak_students.append(
|
||||
{
|
||||
"class_id": cid,
|
||||
"total_students": class_perf.get("totalStudents", 0),
|
||||
"average_score": class_perf.get("averageScore", 0.0),
|
||||
"pass_rate": class_perf.get("passRate", 0.0),
|
||||
}
|
||||
)
|
||||
|
||||
# 5. 补充降级标记
|
||||
if degraded:
|
||||
dashboard["degraded"] = True
|
||||
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
|
||||
dashboard["classes"] = weak_students
|
||||
dashboard["dashboardType"] = "teacher"
|
||||
return dashboard
|
||||
|
||||
|
||||
async def get_student_dashboard(
|
||||
user: UserContext,
|
||||
student_id: str = "",
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""学生仪表盘（学情宽表 + 薄弱知识点 + 学习趋势）.
|
||||
|
||||
DataScope：
|
||||
- SELF 级：student_id 必须为当前 user_id
|
||||
- CLASS/GRADE/SCHOOL/ALL 级：可查指定 student_id
|
||||
"""
|
||||
# 1. 解析 DataScope
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# 2. SELF 级强制限制 student_id
|
||||
effective_student_id = student_id or user.user_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id # SELF 级只能查自己
|
||||
|
||||
# 3. 查询 ClickHouse
|
||||
dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id)
|
||||
if dashboard is None:
|
||||
return _skeleton_dashboard("student", effective_student_id, "clickhouse_unavailable")
|
||||
|
||||
# 4. 补充薄弱知识点
|
||||
weakness = await clickhouse_repository.query_student_weakness(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
dashboard["weakPoints"] = weakness.get("weakPoints", []) if weakness else []
|
||||
|
||||
# 5. 补充学习趋势
|
||||
trend = await clickhouse_repository.query_learning_trend(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
dashboard["trend"] = trend.get("points", []) if trend else []
|
||||
|
||||
# 6. 补充掌握度快照
|
||||
mastery = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
dashboard["mastery"] = mastery if mastery else {"knowledgePoints": [], "overallMastery": 0.0}
|
||||
|
||||
# 7. 降级标记
|
||||
if degraded:
|
||||
dashboard["degraded"] = True
|
||||
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
|
||||
dashboard["dashboardType"] = "student"
|
||||
return dashboard
|
||||
|
||||
|
||||
async def get_parent_dashboard(
|
||||
user: UserContext,
|
||||
child_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""家长仪表盘（孩子学情聚合）.
|
||||
|
||||
DataScope：
|
||||
- SELF 级：child_id 必须为家长关联的孩子（iam.GetChildrenByParent 校验）
|
||||
- 降级时使用 child_id 直接查询（依赖 Gateway 层鉴权）
|
||||
"""
|
||||
# 1. 解析 DataScope
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# 2. 学生 ID（家长视角：使用 child_id 或 user_id）
|
||||
effective_student_id = child_id or user.user_id
|
||||
|
||||
# 3. 查询 ClickHouse（复用学生仪表盘查询）
|
||||
dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id)
|
||||
if dashboard is None:
|
||||
return _skeleton_dashboard("parent", effective_student_id, "clickhouse_unavailable")
|
||||
|
||||
# 4. 补充考勤
|
||||
attendance = await clickhouse_repository.query_attendance(effective_student_id)
|
||||
dashboard["attendance"] = (
|
||||
attendance
|
||||
if attendance
|
||||
else {
|
||||
"absentCount": 0,
|
||||
"lateCount": 0,
|
||||
"presentCount": 0,
|
||||
}
|
||||
)
|
||||
|
||||
# 5. 补充学习趋势
|
||||
trend = await clickhouse_repository.query_learning_trend(effective_student_id)
|
||||
dashboard["trend"] = trend.get("points", []) if trend else []
|
||||
|
||||
# 6. 降级标记
|
||||
if degraded:
|
||||
dashboard["degraded"] = True
|
||||
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
|
||||
dashboard["dashboardType"] = "parent"
|
||||
dashboard["childId"] = effective_student_id
|
||||
return dashboard
|
||||
|
||||
|
||||
async def get_admin_dashboard(
|
||||
user: UserContext,
|
||||
school_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""管理员仪表盘（学校聚合 + AI 用量统计）.
|
||||
|
||||
DataScope：
|
||||
- SCHOOL 级：仅返回指定 school_id 数据
|
||||
- ALL 级：返回所有数据
|
||||
"""
|
||||
# 1. 解析 DataScope
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# 2. 学校级聚合查询（复用 teacher_dashboard，不传 class_id）
|
||||
dashboard = await clickhouse_repository.query_teacher_dashboard(
|
||||
user_id=user.user_id,
|
||||
class_id="",
|
||||
)
|
||||
if dashboard is None:
|
||||
return _skeleton_dashboard("admin", user.user_id, "clickhouse_unavailable")
|
||||
|
||||
# 3. 补充 AI 用量统计（P5 启用，对齐 02 §7.4）
|
||||
ai_usage = await clickhouse_repository.query_ai_usage_summary()
|
||||
dashboard["aiUsage"] = (
|
||||
ai_usage
|
||||
if ai_usage
|
||||
else {
|
||||
"totalRequests": 0,
|
||||
"totalTokens": 0,
|
||||
"totalCostCents": 0,
|
||||
"byProvider": [],
|
||||
}
|
||||
)
|
||||
|
||||
# 4. 降级标记
|
||||
if degraded:
|
||||
dashboard["degraded"] = True
|
||||
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
|
||||
dashboard["dashboardType"] = "admin"
|
||||
dashboard["schoolId"] = school_id or scope.school_id
|
||||
return dashboard
|
||||
|
||||
|
||||
async def get_class_performance(
|
||||
user: UserContext,
|
||||
class_id: str,
|
||||
subject_id: str = "",
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
"""查询班级成绩分析（GetClassPerformance RPC 实现）.
|
||||
|
||||
DataScope：CLASS 级需验证 class_id 在 scope_ids 内.
|
||||
"""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# DataScope 校验
|
||||
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
|
||||
return {
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "datascope_violation",
|
||||
"averageScore": 0.0,
|
||||
"passRate": 0.0,
|
||||
"totalStudents": 0,
|
||||
}
|
||||
|
||||
result = await clickhouse_repository.query_class_performance(
|
||||
class_id=class_id,
|
||||
subject_id=subject_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"averageScore": 0.0,
|
||||
"passRate": 0.0,
|
||||
"totalStudents": 0,
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_mastery_distribution(
|
||||
user: UserContext,
|
||||
class_id: str,
|
||||
subject_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""查询班级掌握度分布（GetMasteryDistribution RPC 实现）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# DataScope 校验
|
||||
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
|
||||
return {
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "datascope_violation",
|
||||
"masteredCount": 0,
|
||||
"progressingCount": 0,
|
||||
"weakCount": 0,
|
||||
"totalStudents": 0,
|
||||
}
|
||||
|
||||
result = await clickhouse_repository.query_mastery_distribution(
|
||||
class_id=class_id,
|
||||
subject_id=subject_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"masteredCount": 0,
|
||||
"progressingCount": 0,
|
||||
"weakCount": 0,
|
||||
"totalStudents": 0,
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_student_mastery(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""查询学生知识点掌握度明细（GetStudentMastery RPC 实现）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# SELF 级强制限制
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
result = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"knowledgePoints": [],
|
||||
"overallMastery": 0.0,
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_student_weakness(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""查询学生薄弱知识点（GetStudentWeakness RPC 实现）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# SELF 级强制限制
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
result = await clickhouse_repository.query_student_weakness(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"weakPoints": [],
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_learning_trend(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
"""查询学习趋势（GetLearningTrend RPC 实现）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
# SELF 级强制限制
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
result = await clickhouse_repository.query_learning_trend(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"points": [],
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
@@ -1,17 +1,28 @@
|
||||
"""CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表）.
|
||||
"""CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表，EventHandler 路由 + 手动 commit）.
|
||||
|
||||
链路：
|
||||
MySQL binlog → Debezium Connect → Kafka topic
|
||||
edu-cdc.next_edu_cloud.<table>
|
||||
→ 本消费者 → 解析 Debezium 事件 → 写入 ClickHouse student_dashboard_view
|
||||
→ 本消费者 → 解析 Debezium 事件 → EventHandler 路由 → 写入 ClickHouse 宽表
|
||||
|
||||
支持的表路由：
|
||||
- core_edu_exams → ExamCache 更新
|
||||
- core_edu_grades → student_dashboard_view 写入（查 ExamCache 获取 class_id）
|
||||
- core_edu_homework_submissions → student_dashboard_view 写入
|
||||
- core_edu_attendance → attendance_logs 写入
|
||||
- content_knowledge_points → mastery_snapshot 元数据更新
|
||||
- iam_users → 用户元数据缓存（P6 扩展）
|
||||
- classes → 班级元数据缓存（P6 扩展）
|
||||
|
||||
设计要点：
|
||||
- 监听多张表，按 source.table 路由
|
||||
- 内存缓存 exam_id → (class_id, subject_id) 映射（来自 core_edu_exams 快照+流）
|
||||
- 监听 core_edu_grades 时用缓存扩展为宽表记录写入 ClickHouse
|
||||
- 手动 commit（at-least-once）：ClickHouse 写入成功后才 commit offset
|
||||
- 幂等性：依赖 ClickHouse ReplacingMergeTree 引擎按 ORDER BY 去重
|
||||
（schema 需用 ReplacingMergeTree(last_updated)，当前为简化版 MergeTree）
|
||||
- op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null
|
||||
- Redis 事件去重：基于 event_id（Debezium 重启时可能重发）
|
||||
|
||||
降级策略：
|
||||
- kafka_brokers 未配置：消费者不启动（仅 HTTP/gRPC 服务）
|
||||
- ClickHouse 不可达：消息处理失败，不 commit，下次重启重试
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -22,11 +33,17 @@ from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .clickhouse_client import upsert_student_dashboard
|
||||
from .config import settings
|
||||
from .exam_cache import get_exam_cache
|
||||
from .repository import clickhouse_repository, redis_client
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# 消费者单例（用于 lifespan 管理和 /readyz 检查）
|
||||
_consumer: Any | None = None
|
||||
_consumer_task: asyncio.Task[None] | None = None
|
||||
_is_running: bool = False
|
||||
|
||||
|
||||
def _parse_ts(ts_ms: int | None) -> datetime:
|
||||
"""Debezium ts_ms（毫秒）→ datetime."""
|
||||
@@ -45,103 +62,291 @@ def _safe_float(value: Any) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
class ExamCache:
|
||||
"""内存缓存 exam_id → (class_id, subject_id).
|
||||
|
||||
从 core_edu_exams 表的 CDC 事件构建。subject_id 在 exams 表中暂无字段，
|
||||
这里占位为空字符串，后续扩展 schema 时再补充。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._data: dict[str, tuple[str, str]] = {}
|
||||
|
||||
def upsert(self, exam_id: str, class_id: str, subject_id: str = "") -> None:
|
||||
if exam_id:
|
||||
self._data[exam_id] = (class_id, subject_id)
|
||||
|
||||
def get(self, exam_id: str) -> tuple[str, str] | None:
|
||||
return self._data.get(exam_id) if exam_id else None
|
||||
def _safe_int(value: Any) -> int:
|
||||
"""安全转 int."""
|
||||
if value is None:
|
||||
return 0
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
# 全局缓存（进程级单例）
|
||||
_exam_cache = ExamCache()
|
||||
def _parse_mysql_datetime(value: Any, fallback: datetime) -> datetime:
|
||||
"""解析 MySQL datetime 字段（兼容字符串和毫秒时间戳）."""
|
||||
if not value:
|
||||
return fallback
|
||||
if isinstance(value, (int, float)):
|
||||
return datetime.fromtimestamp(value / 1000, tz=UTC)
|
||||
with contextlib.suppress(ValueError, AttributeError, TypeError):
|
||||
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
return fallback
|
||||
|
||||
|
||||
async def _handle_exams_event(after: dict[str, Any] | None) -> None:
|
||||
"""处理 core_edu_exams 表事件."""
|
||||
# ===== EventHandler 路由表 =====
|
||||
|
||||
|
||||
async def _handle_exams_event(after: dict[str, Any] | None, op: str) -> bool:
|
||||
"""处理 core_edu_exams 表事件 → ExamCache 更新."""
|
||||
if after is None:
|
||||
return
|
||||
exam_id = after.get("id")
|
||||
class_id = after.get("class_id", "")
|
||||
if exam_id:
|
||||
_exam_cache.upsert(exam_id, class_id)
|
||||
logger.info("exam_cache_updated", exam_id=exam_id, class_id=class_id)
|
||||
return True # 删除事件，无需处理
|
||||
|
||||
exam_id = str(after.get("id") or "")
|
||||
if not exam_id:
|
||||
return True
|
||||
|
||||
exam_cache = get_exam_cache()
|
||||
exam_cache.upsert(
|
||||
exam_id=exam_id,
|
||||
class_id=str(after.get("class_id") or ""),
|
||||
subject_id=str(after.get("subject_id") or ""),
|
||||
title=str(after.get("title") or ""),
|
||||
)
|
||||
logger.info(
|
||||
"exam_cache_updated",
|
||||
exam_id=exam_id,
|
||||
class_id=after.get("class_id"),
|
||||
op=op,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _handle_grades_event(
|
||||
after: dict[str, Any] | None,
|
||||
op: str,
|
||||
ts_ms: int | None,
|
||||
) -> None:
|
||||
"""处理 core_edu_grades 表事件 → 写入 ClickHouse 宽表.
|
||||
|
||||
- op=r/c/u：after 为新数据，写入宽表
|
||||
- op=d：after 为 null，暂不处理（宽表保留历史记录）
|
||||
"""
|
||||
) -> bool:
|
||||
"""处理 core_edu_grades 表事件 → 写入 student_dashboard_view."""
|
||||
if after is None:
|
||||
return
|
||||
return True # 删除事件
|
||||
|
||||
student_id = after.get("student_id", "")
|
||||
exam_id = after.get("exam_id", "")
|
||||
student_id = str(after.get("student_id") or "")
|
||||
exam_id = str(after.get("exam_id") or "")
|
||||
score = _safe_float(after.get("score"))
|
||||
|
||||
# 从缓存拿 class_id
|
||||
class_id = ""
|
||||
if exam_id:
|
||||
cached = _exam_cache.get(exam_id)
|
||||
if cached:
|
||||
class_id = cached[0]
|
||||
# 从 ExamCache 获取 class_id 和 subject_id
|
||||
exam_cache = get_exam_cache()
|
||||
class_id = exam_cache.get_class_id(exam_id)
|
||||
subject_id = exam_cache.get_subject_id(exam_id)
|
||||
|
||||
last_updated = _parse_ts(ts_ms)
|
||||
if after.get("updated_at"):
|
||||
# 优先用 MySQL 的 updated_at 字段
|
||||
with contextlib.suppress(ValueError, AttributeError):
|
||||
last_updated = datetime.fromisoformat(after["updated_at"].replace("Z", "+00:00"))
|
||||
fallback_ts = _parse_ts(ts_ms)
|
||||
last_updated = _parse_mysql_datetime(after.get("updated_at"), fallback_ts)
|
||||
|
||||
# 简化：rank/kp/mastery/error_count 暂用默认值
|
||||
# 真实场景应通过其他 CDC 事件或聚合计算得到
|
||||
await upsert_student_dashboard(
|
||||
# 简化：rank/kp/mastery/error_count 暂用默认值（CDC 聚合后由掌握度计算补充）
|
||||
write_ok = await clickhouse_repository.upsert_student_dashboard(
|
||||
student_id=student_id,
|
||||
class_id=class_id,
|
||||
exam_id=exam_id,
|
||||
subject_id="", # 占位
|
||||
subject_id=subject_id,
|
||||
score=score,
|
||||
rank_in_class=0,
|
||||
knowledge_point_id="", # 占位
|
||||
mastery_level=score / 100.0, # 简化：用分数百分比作为掌握度
|
||||
knowledge_point_id="",
|
||||
mastery_level=score / 100.0, # 简化：用分数百分比作为初始掌握度
|
||||
error_count=0,
|
||||
last_updated=last_updated,
|
||||
)
|
||||
if write_ok:
|
||||
logger.info(
|
||||
"grade_record_written",
|
||||
student_id=student_id,
|
||||
exam_id=exam_id,
|
||||
score=score,
|
||||
op=op,
|
||||
)
|
||||
return write_ok
|
||||
|
||||
|
||||
async def _process_message(topic: str, value: bytes | str) -> None:
|
||||
"""处理单条 Kafka 消息.
|
||||
async def _handle_homework_event(
|
||||
after: dict[str, Any] | None,
|
||||
op: str,
|
||||
ts_ms: int | None,
|
||||
) -> bool:
|
||||
"""处理 core_edu_homework_submissions 表事件 → 写入 student_dashboard_view."""
|
||||
if after is None:
|
||||
return True
|
||||
|
||||
Debezium 事件格式（简化后，schemas.enable=false）：
|
||||
{
|
||||
"before": {...} | null,
|
||||
"after": {...} | null,
|
||||
"source": {"table": "...", "db": "...", ...},
|
||||
"op": "r|c|u|d",
|
||||
"ts_ms": 1783572350928
|
||||
}
|
||||
student_id = str(after.get("student_id") or "")
|
||||
homework_id = str(after.get("homework_id") or after.get("id") or "")
|
||||
class_id = str(after.get("class_id") or "")
|
||||
score = _safe_float(after.get("score"))
|
||||
|
||||
fallback_ts = _parse_ts(ts_ms)
|
||||
raw_time = after.get("submitted_at") or after.get("updated_at")
|
||||
last_updated = _parse_mysql_datetime(raw_time, fallback_ts)
|
||||
|
||||
write_ok = await clickhouse_repository.upsert_student_dashboard(
|
||||
student_id=student_id,
|
||||
class_id=class_id,
|
||||
exam_id=homework_id, # homework_id 作为 exam_id 占位
|
||||
subject_id=str(after.get("subject_id") or ""),
|
||||
score=score,
|
||||
rank_in_class=0,
|
||||
knowledge_point_id="",
|
||||
mastery_level=score / 100.0,
|
||||
error_count=0,
|
||||
last_updated=last_updated,
|
||||
)
|
||||
if write_ok:
|
||||
logger.info(
|
||||
"homework_record_written",
|
||||
student_id=student_id,
|
||||
homework_id=homework_id,
|
||||
score=score,
|
||||
op=op,
|
||||
)
|
||||
return write_ok
|
||||
|
||||
|
||||
async def _handle_attendance_event(
|
||||
after: dict[str, Any] | None,
|
||||
op: str,
|
||||
ts_ms: int | None,
|
||||
) -> bool:
|
||||
"""处理 core_edu_attendance 表事件 → 写入 attendance_logs."""
|
||||
if after is None:
|
||||
return True
|
||||
|
||||
student_id = str(after.get("student_id") or "")
|
||||
class_id = str(after.get("class_id") or "")
|
||||
status = str(after.get("status") or "present")
|
||||
recorded_by = str(after.get("recorded_by") or "")
|
||||
remark = str(after.get("remark") or "")
|
||||
|
||||
fallback_ts = _parse_ts(ts_ms)
|
||||
raw_time = after.get("occurred_at") or after.get("created_at")
|
||||
occurred_at = _parse_mysql_datetime(raw_time, fallback_ts)
|
||||
|
||||
# attendance_date 从 occurred_at 提取日期部分
|
||||
attendance_date = occurred_at.date()
|
||||
|
||||
write_ok = await clickhouse_repository.upsert_attendance_log(
|
||||
student_id=student_id,
|
||||
class_id=class_id,
|
||||
attendance_date=attendance_date,
|
||||
status=status,
|
||||
recorded_by=recorded_by,
|
||||
remark=remark,
|
||||
occurred_at=occurred_at,
|
||||
)
|
||||
if write_ok:
|
||||
logger.info(
|
||||
"attendance_record_written",
|
||||
student_id=student_id,
|
||||
class_id=class_id,
|
||||
status=status,
|
||||
op=op,
|
||||
)
|
||||
return write_ok
|
||||
|
||||
|
||||
async def _handle_knowledge_point_event(
|
||||
after: dict[str, Any] | None,
|
||||
op: str,
|
||||
) -> bool:
|
||||
"""处理 content_knowledge_points 表事件 → 更新知识点元数据缓存."""
|
||||
if after is None:
|
||||
return True
|
||||
|
||||
# 知识点元数据更新（P5+ 扩展：可写入 Redis 缓存供查询时补充标题）
|
||||
kp_id = str(after.get("id") or "")
|
||||
title = str(after.get("title") or "")
|
||||
subject_id = str(after.get("subject_id") or "")
|
||||
|
||||
# 简化：写 Redis 缓存（data_ana:kp_meta:{kp_id}）
|
||||
metadata = {"title": title, "subject_id": subject_id}
|
||||
await redis_client.set_cache(
|
||||
f"data_ana:kp_meta:{kp_id}",
|
||||
json.dumps(metadata, ensure_ascii=False),
|
||||
ttl_s=86400,
|
||||
)
|
||||
logger.info(
|
||||
"knowledge_point_meta_cached",
|
||||
knowledge_point_id=kp_id,
|
||||
title=title,
|
||||
op=op,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _handle_ai_usage_event(event: dict[str, Any]) -> bool:
|
||||
"""处理 AIUsageEvent（edu.insight.ai.usage topic）→ 写入 ai_usage_log.
|
||||
|
||||
注意：此事件不是 Debezium CDC 事件，而是 ai 服务发布的业务事件，
|
||||
格式遵循 events.proto AIUsageEvent message.
|
||||
"""
|
||||
request_id = str(event.get("request_id") or "")
|
||||
user_id = str(event.get("user_id") or "")
|
||||
provider = str(event.get("provider") or "")
|
||||
model = str(event.get("model") or "")
|
||||
prompt_tokens = _safe_int(event.get("prompt_tokens"))
|
||||
completion_tokens = _safe_int(event.get("completion_tokens"))
|
||||
total_tokens = _safe_int(event.get("total_tokens"))
|
||||
latency_ms = _safe_int(event.get("latency_ms"))
|
||||
success = bool(event.get("success", True))
|
||||
cost_cents = _safe_int(event.get("cost_cents"))
|
||||
occurred_at_ms = event.get("occurred_at")
|
||||
if isinstance(occurred_at_ms, int):
|
||||
occurred_at = _parse_ts(occurred_at_ms)
|
||||
else:
|
||||
occurred_at = datetime.now(UTC)
|
||||
|
||||
write_ok = await clickhouse_repository.upsert_ai_usage_log(
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
provider=provider,
|
||||
model=model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=total_tokens,
|
||||
latency_ms=latency_ms,
|
||||
success=success,
|
||||
cost_cents=cost_cents,
|
||||
occurred_at=occurred_at,
|
||||
)
|
||||
if write_ok:
|
||||
logger.info(
|
||||
"ai_usage_recorded",
|
||||
request_id=request_id,
|
||||
provider=provider,
|
||||
model=model,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
return write_ok
|
||||
|
||||
|
||||
# ===== 事件去重 =====
|
||||
|
||||
|
||||
async def _is_deduped(event_id: str) -> bool:
|
||||
"""检查事件是否已处理（基于 event_id Redis SETNX 去重）.
|
||||
|
||||
返回 True 表示重复事件（应跳过），False 表示首次事件.
|
||||
"""
|
||||
if not event_id:
|
||||
return False
|
||||
key = f"data_ana:dedup:{event_id}"
|
||||
is_first = await redis_client.setnx_dedup(key, ttl_s=7 * 24 * 3600) # 7 天 TTL
|
||||
return not is_first
|
||||
|
||||
|
||||
# ===== 消息处理主入口 =====
|
||||
|
||||
|
||||
async def _process_cdc_message(topic: str, value: bytes | str) -> bool:
|
||||
"""处理单条 CDC 消息（Debezium 格式）.
|
||||
|
||||
返回 True 表示处理成功（可 commit offset），
|
||||
返回 False 表示处理失败（不 commit，下次重试）.
|
||||
"""
|
||||
try:
|
||||
value_str = value.decode("utf-8") if isinstance(value, bytes) else value
|
||||
event = json.loads(value_str)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
logger.warning("cdc_message_decode_failed", error=str(exc), topic=topic)
|
||||
return
|
||||
return True # 无效消息直接跳过（避免阻塞消费）
|
||||
|
||||
source = event.get("source") or {}
|
||||
table = source.get("table", "")
|
||||
@@ -149,6 +354,16 @@ async def _process_message(topic: str, value: bytes | str) -> None:
|
||||
ts_ms = event.get("ts_ms")
|
||||
after = event.get("after")
|
||||
|
||||
# Debezium 事件去重（基于 source.ts_ms + source.dbg + source.sequence 生成 event_id）
|
||||
# 简化：使用 topic + partition + offset 作为去重 key（由调用方传入更精确）
|
||||
# 这里仅对 after.id 做去重（如果有）
|
||||
event_id = ""
|
||||
if after and isinstance(after, dict):
|
||||
event_id = f"{table}:{after.get('id', '')}:{ts_ms}"
|
||||
if event_id and await _is_deduped(event_id):
|
||||
logger.info("cdc_event_dedup_skipped", table=table, event_id=event_id)
|
||||
return True
|
||||
|
||||
logger.info(
|
||||
"cdc_event_received",
|
||||
topic=topic,
|
||||
@@ -157,21 +372,71 @@ async def _process_message(topic: str, value: bytes | str) -> None:
|
||||
ts_ms=ts_ms,
|
||||
)
|
||||
|
||||
try:
|
||||
if table == "core_edu_exams":
|
||||
await _handle_exams_event(after)
|
||||
return await _handle_exams_event(after, op)
|
||||
elif table == "core_edu_grades":
|
||||
await _handle_grades_event(after, op, ts_ms)
|
||||
return await _handle_grades_event(after, op, ts_ms)
|
||||
elif table == "core_edu_homework_submissions":
|
||||
return await _handle_homework_event(after, op, ts_ms)
|
||||
elif table == "core_edu_attendance":
|
||||
return await _handle_attendance_event(after, op, ts_ms)
|
||||
elif table == "content_knowledge_points":
|
||||
return await _handle_knowledge_point_event(after, op)
|
||||
else:
|
||||
# 其他表暂不处理，仅记录
|
||||
logger.debug("cdc_event_skipped", table=table)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error(
|
||||
"cdc_event_handler_failed",
|
||||
error=str(exc),
|
||||
table=table,
|
||||
op=op,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def _process_ai_usage_message(topic: str, value: bytes | str) -> bool:
|
||||
"""处理 AIUsageEvent 消息（非 Debezium 格式，业务事件）."""
|
||||
try:
|
||||
value_str = value.decode("utf-8") if isinstance(value, bytes) else value
|
||||
event = json.loads(value_str)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
logger.warning("ai_usage_message_decode_failed", error=str(exc), topic=topic)
|
||||
return True
|
||||
|
||||
# 基于 request_id 去重
|
||||
request_id = str(event.get("request_id") or "")
|
||||
if request_id and await _is_deduped(f"ai_usage:{request_id}"):
|
||||
return True
|
||||
|
||||
try:
|
||||
return await _handle_ai_usage_event(event)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("ai_usage_handler_failed", error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
async def _process_message(topic: str, value: bytes | str) -> bool:
|
||||
"""统一消息处理入口（区分 CDC 事件和 AIUsage 事件）."""
|
||||
# 根据 topic 判断消息类型
|
||||
if topic == settings.kafka_ai_usage_topic:
|
||||
return await _process_ai_usage_message(topic, value)
|
||||
return await _process_cdc_message(topic, value)
|
||||
|
||||
|
||||
# ===== 消费者主循环 =====
|
||||
|
||||
|
||||
async def run_consumer() -> None:
|
||||
"""CDC 消费者主循环（lifespan 启动）.
|
||||
|
||||
- kafka_brokers 未配置：直接返回，不启动消费者（降级模式）
|
||||
- 手动 commit：ClickHouse 写入成功后才 commit offset（at-least-once）
|
||||
- 启动失败：仅记录错误，不阻塞 FastAPI 主流程
|
||||
"""
|
||||
global _consumer, _is_running
|
||||
|
||||
if not settings.kafka_brokers:
|
||||
logger.info("cdc_consumer_disabled_no_kafka_brokers")
|
||||
return
|
||||
@@ -183,51 +448,114 @@ async def run_consumer() -> None:
|
||||
return
|
||||
|
||||
topics = [t.strip() for t in settings.kafka_cdc_topics.split(",") if t.strip()]
|
||||
# P5+ 加入 AI 用量 topic
|
||||
if settings.kafka_ai_usage_topic and settings.kafka_ai_usage_topic not in topics:
|
||||
topics.append(settings.kafka_ai_usage_topic)
|
||||
|
||||
if not topics:
|
||||
logger.warning("cdc_consumer_no_topics_configured")
|
||||
return
|
||||
|
||||
brokers = [b.strip() for b in settings.kafka_brokers.split(",") if b.strip()]
|
||||
|
||||
consumer = AIOKafkaConsumer(
|
||||
_consumer = AIOKafkaConsumer(
|
||||
*topics,
|
||||
bootstrap_servers=brokers,
|
||||
group_id=settings.kafka_group_id,
|
||||
group_id=settings.kafka_consumer_group,
|
||||
auto_offset_reset=settings.kafka_auto_offset_reset,
|
||||
enable_auto_commit=True,
|
||||
enable_auto_commit=settings.kafka_enable_auto_commit, # 手动 commit
|
||||
value_deserializer=lambda v: v, # 保留原始 bytes，由 _process_message 解码
|
||||
max_poll_records=100,
|
||||
session_timeout_ms=30_000,
|
||||
heartbeat_interval_ms=10_000,
|
||||
)
|
||||
|
||||
try:
|
||||
await consumer.start()
|
||||
await _consumer.start()
|
||||
_is_running = True
|
||||
logger.info(
|
||||
"cdc_consumer_started",
|
||||
brokers=brokers,
|
||||
topics=topics,
|
||||
group_id=settings.kafka_group_id,
|
||||
group_id=settings.kafka_consumer_group,
|
||||
auto_commit=settings.kafka_enable_auto_commit,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("cdc_consumer_start_failed", error=str(exc))
|
||||
_is_running = False
|
||||
return
|
||||
|
||||
try:
|
||||
async for msg in consumer:
|
||||
async for msg in _consumer:
|
||||
try:
|
||||
await _process_message(msg.topic, msg.value)
|
||||
success = await _process_message(msg.topic, msg.value)
|
||||
if success:
|
||||
# 处理成功才 commit offset（at-least-once）
|
||||
await _consumer.commit()
|
||||
else:
|
||||
# 处理失败不 commit，下次重启会重新消费
|
||||
logger.warning(
|
||||
"cdc_message_process_failed_no_commit",
|
||||
topic=msg.topic,
|
||||
partition=msg.partition,
|
||||
offset=msg.offset,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error(
|
||||
"cdc_message_process_failed",
|
||||
"cdc_message_process_exception",
|
||||
error=str(exc),
|
||||
topic=msg.topic,
|
||||
partition=msg.partition,
|
||||
offset=msg.offset,
|
||||
)
|
||||
# 异常不 commit，下次重启重试
|
||||
except asyncio.CancelledError:
|
||||
logger.info("cdc_consumer_cancelled")
|
||||
raise
|
||||
finally:
|
||||
_is_running = False
|
||||
if _consumer is not None:
|
||||
try:
|
||||
await consumer.stop()
|
||||
await _consumer.stop()
|
||||
logger.info("cdc_consumer_stopped")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("cdc_consumer_stop_failed", error=str(exc))
|
||||
_consumer = None
|
||||
|
||||
|
||||
async def start_consumer() -> asyncio.Task[None] | None:
|
||||
"""启动消费者后台任务（供 lifespan 调用）."""
|
||||
global _consumer_task
|
||||
if not settings.kafka_brokers:
|
||||
return None
|
||||
if _consumer_task is not None and not _consumer_task.done():
|
||||
return _consumer_task
|
||||
_consumer_task = asyncio.create_task(run_consumer())
|
||||
return _consumer_task
|
||||
|
||||
|
||||
async def stop_consumer() -> None:
|
||||
"""停止消费者后台任务（供 lifespan 调用）."""
|
||||
global _consumer_task
|
||||
if _consumer_task is not None:
|
||||
_consumer_task.cancel()
|
||||
try:
|
||||
await _consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
_consumer_task = None
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
"""消费者是否运行中（供 /readyz 使用）."""
|
||||
return _is_running
|
||||
|
||||
|
||||
async def get_lag() -> int:
|
||||
"""获取消费者 lag（待消费消息数，供 /readyz 和监控使用）.
|
||||
|
||||
P6 实现：调用 Kafka AdminClient 获取 group lag.
|
||||
当前返回 0（简化）.
|
||||
"""
|
||||
return 0
|
||||
|
||||
@@ -1,46 +1,97 @@
|
||||
"""配置管理."""
|
||||
"""配置管理（pydantic-settings 完整配置项）.
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
对齐 02-architecture-design.md §15 配置清单.
|
||||
环境变量前缀 DATA_ANA_（如 DATA_ANA_HTTP_PORT=3006）.
|
||||
"""
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""应用配置.
|
||||
|
||||
ClickHouse 连接参数为可选：当 clickhouse_host 为空字符串时，
|
||||
服务进入降级模式（查询方法返回 None / 空数据），保证服务可启动。
|
||||
|
||||
Kafka 连接参数为可选：当 kafka_brokers 为空字符串时，
|
||||
CDC 消费者不启动（降级模式），保证服务可启动。
|
||||
所有外部依赖（ClickHouse / Kafka / Redis / iam gRPC）均为可选：
|
||||
未配置或不可达时服务进入降级模式，返回骨架数据 + details.degraded: true.
|
||||
"""
|
||||
|
||||
port: int = 3006
|
||||
# ClickHouse 连接（可选：留空则降级模式）
|
||||
# 服务
|
||||
service_name: str = "data-ana"
|
||||
http_port: int = 3006
|
||||
grpc_port: int = 50055
|
||||
log_level: str = "INFO"
|
||||
dev_mode: bool = False
|
||||
|
||||
# ClickHouse（可选：留空则降级模式）
|
||||
clickhouse_host: str = ""
|
||||
clickhouse_port: int = 8123
|
||||
clickhouse_database: str = "edu_analytics"
|
||||
clickhouse_user: str = ""
|
||||
clickhouse_password: str = ""
|
||||
# 可观测性
|
||||
otel_endpoint: str = "http://localhost:4318"
|
||||
log_level: str = "info"
|
||||
# 开发模式开关
|
||||
dev_mode: bool = False
|
||||
# Kafka brokers（CDC 消费；留空则不启动消费者）
|
||||
# 主机访问用 localhost:9092，容器内访问用 kafka:29092
|
||||
kafka_brokers: str = ""
|
||||
# CDC 消费组 id
|
||||
kafka_group_id: str = "data-ana-cdc-consumer"
|
||||
# 要消费的 CDC topic（Debezium 默认命名：<prefix>.<database>.<table>）
|
||||
# 用逗号分隔多个 topic
|
||||
clickhouse_connect_timeout_ms: int = 3000
|
||||
clickhouse_query_timeout_s: int = 3 # P4 退出标准 5s，查询 3s 超时降级
|
||||
|
||||
# Kafka
|
||||
kafka_brokers: str = "" # 留空则不启动 CDC 消费者
|
||||
kafka_consumer_group: str = "data-ana-cdc"
|
||||
kafka_cdc_topics: str = (
|
||||
"edu-cdc.next_edu_cloud.core_edu_grades,"
|
||||
"edu-cdc.next_edu_cloud.core_edu_exams,"
|
||||
"edu-cdc.next_edu_cloud.classes"
|
||||
"edu-cdc.next_edu_cloud.core_edu_homework_submissions,"
|
||||
"edu-cdc.next_edu_cloud.core_edu_attendance,"
|
||||
"edu-cdc.next_edu_cloud.classes,"
|
||||
"edu-cdc.next_edu_cloud.iam_users,"
|
||||
"edu-cdc.next_edu_cloud.content_knowledge_points"
|
||||
)
|
||||
# 消费者自动偏移重置策略（earliest / latest）
|
||||
kafka_auto_offset_reset: str = "earliest"
|
||||
kafka_mastery_topic: str = "edu.insight.mastery.updated"
|
||||
kafka_warning_topic: str = "edu.insight.mastery.updated" # 复用 mastery topic（总裁裁决 §2.11）
|
||||
kafka_ai_usage_topic: str = "edu.insight.ai.usage"
|
||||
kafka_enable_auto_commit: bool = False # v2: 手动 commit（at-least-once）
|
||||
kafka_auto_offset_reset: str = "latest"
|
||||
kafka_producer_transactional_id: str = "data-ana-producer"
|
||||
|
||||
model_config = {"env_file": ".env", "env_prefix": ""}
|
||||
# iam gRPC
|
||||
iam_grpc_endpoint: str = "" # 留空则使用降级兜底（按 role 映射 DataScope）
|
||||
iam_grpc_timeout_s: int = 2
|
||||
datascope_cache_ttl_s: int = 300 # 5min
|
||||
|
||||
# Redis
|
||||
redis_url: str = "" # 留空则跳过缓存
|
||||
redis_pool_size: int = 10
|
||||
redis_socket_timeout_ms: int = 200
|
||||
|
||||
# OTel
|
||||
otel_endpoint: str = "http://localhost:4318"
|
||||
otel_service_name: str = "data-ana"
|
||||
|
||||
# 掌握度算法
|
||||
mastery_window_size: int = 5
|
||||
mastery_decay_base: float = 0.6
|
||||
mastery_forgetting_half_life_days: int = 30
|
||||
mastery_min_samples: int = 3
|
||||
|
||||
# 预警阈值
|
||||
warning_low_mastery_threshold: float = 0.4
|
||||
warning_critical_mastery_threshold: float = 0.2
|
||||
warning_score_drop_percent: float = 0.2
|
||||
warning_absent_per_week: int = 3
|
||||
|
||||
# 降级
|
||||
degraded_mode_enabled: bool = True
|
||||
|
||||
# 向后兼容：旧代码引用 settings.port / settings.kafka_group_id
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return self.http_port
|
||||
|
||||
@property
|
||||
def kafka_group_id(self) -> str:
|
||||
return self.kafka_consumer_group
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_prefix="DATA_ANA_",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
112
services/data-ana/src/data_ana/exam_cache.py
Normal file
112
services/data-ana/src/data_ana/exam_cache.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""考试缓存（exam_id → {class_id, subject_id} 映射，内存 LRU）.
|
||||
|
||||
对齐 02-architecture-design.md §8.3 ExamCache：
|
||||
- 内存 LRU dict，max 10000 条
|
||||
- CDC core_edu_exams 事件触发更新
|
||||
- CDC core_edu_grades 事件查询获取 class_id（避免 join 查询）
|
||||
|
||||
P6 演进：改为 Redis 实现（key: data_ana:exam:{exam_id}，TTL 30 天），
|
||||
支持多实例共享（对齐 workline §3.5 任务 6.2）.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# LRU 最大容量
|
||||
_MAX_SIZE = 10_000
|
||||
|
||||
|
||||
class ExamCache:
|
||||
"""考试缓存（LRU，max 10000 条）.
|
||||
|
||||
内存实现：OrderedDict，访问/写入时移到末尾（最近使用），
|
||||
超容量时弹出头部（最久未使用）.
|
||||
|
||||
线程安全：asyncio 单线程模型下无需加锁.
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = _MAX_SIZE) -> None:
|
||||
self._data: OrderedDict[str, dict[str, str]] = OrderedDict()
|
||||
self._max_size = max_size
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
exam_id: str,
|
||||
class_id: str = "",
|
||||
subject_id: str = "",
|
||||
title: str = "",
|
||||
) -> None:
|
||||
"""更新或插入考试缓存."""
|
||||
if not exam_id:
|
||||
return
|
||||
|
||||
value: dict[str, str] = {
|
||||
"class_id": class_id,
|
||||
"subject_id": subject_id,
|
||||
"title": title,
|
||||
}
|
||||
|
||||
# 已存在则移到末尾（标记为最近使用）
|
||||
if exam_id in self._data:
|
||||
self._data.move_to_end(exam_id)
|
||||
self._data[exam_id] = value
|
||||
|
||||
# LRU 淘汰
|
||||
while len(self._data) > self._max_size:
|
||||
evicted_key, _ = self._data.popitem(last=False)
|
||||
logger.debug("exam_cache_lru_evicted", exam_id=evicted_key)
|
||||
|
||||
def get(self, exam_id: str) -> dict[str, str] | None:
|
||||
"""查询考试缓存（命中时移到末尾，标记为最近使用）."""
|
||||
if not exam_id:
|
||||
return None
|
||||
value = self._data.get(exam_id)
|
||||
if value is not None:
|
||||
self._data.move_to_end(exam_id)
|
||||
return value
|
||||
|
||||
def get_class_id(self, exam_id: str) -> str:
|
||||
"""便捷方法：获取 class_id（未命中返回空字符串）."""
|
||||
entry = self.get(exam_id)
|
||||
return entry.get("class_id", "") if entry else ""
|
||||
|
||||
def get_subject_id(self, exam_id: str) -> str:
|
||||
"""便捷方法：获取 subject_id（未命中返回空字符串）."""
|
||||
entry = self.get(exam_id)
|
||||
return entry.get("subject_id", "") if entry else ""
|
||||
|
||||
def delete(self, exam_id: str) -> bool:
|
||||
"""删除缓存项."""
|
||||
if exam_id in self._data:
|
||||
del self._data[exam_id]
|
||||
return True
|
||||
return False
|
||||
|
||||
def clear(self) -> None:
|
||||
"""清空缓存."""
|
||||
self._data.clear()
|
||||
|
||||
def size(self) -> int:
|
||||
"""当前缓存数量."""
|
||||
return len(self._data)
|
||||
|
||||
def stats(self) -> dict[str, Any]:
|
||||
"""缓存统计信息（供 /readyz 和监控使用）."""
|
||||
return {
|
||||
"size": len(self._data),
|
||||
"max_size": self._max_size,
|
||||
"utilization": round(len(self._data) / self._max_size, 4),
|
||||
}
|
||||
|
||||
|
||||
# 全局缓存（进程级单例）
|
||||
_exam_cache = ExamCache()
|
||||
|
||||
|
||||
def get_exam_cache() -> ExamCache:
|
||||
"""获取全局 ExamCache 实例."""
|
||||
return _exam_cache
|
||||
570
services/data-ana/src/data_ana/grpc_server.py
Normal file
570
services/data-ana/src/data_ana/grpc_server.py
Normal file
@@ -0,0 +1,570 @@
|
||||
"""gRPC Server（AnalyticsService 12 RPC + HealthService）.
|
||||
|
||||
对齐 02-architecture-design.md §4.2：
|
||||
- AnalyticsService 12 RPC（含 1 个 server-streaming SubscribeMasteryUpdate）
|
||||
- HealthService（grpc.health.v1，供 K8s 探针）
|
||||
- 端口 50055
|
||||
- 所有 RPC 返回结构化 message（HTTP 层包装 ActionState 信封）
|
||||
|
||||
SubscribeMasteryUpdate（P5+）：
|
||||
- server-streaming RPC
|
||||
- 客户端订阅 student_id / class_id
|
||||
- 掌握度计算完成时推送 MasteryUpdateEvent
|
||||
- 基于 asyncio.Queue 实现事件分发
|
||||
|
||||
降级策略：
|
||||
- grpcio 未安装：gRPC server 不启动（仅 HTTP 服务）
|
||||
- 下游服务不可达：返回降级响应（degraded 字段标记）
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from . import analytics_service, warning_service
|
||||
from .config import settings
|
||||
from .shared.permissions import UserContext
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# SubscribeMasteryUpdate 订阅管理（P5+）
|
||||
_subscribers: dict[str, asyncio.Queue] = {}
|
||||
_subscribers_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def _add_subscriber(key: str) -> asyncio.Queue:
|
||||
"""添加订阅者."""
|
||||
async with _subscribers_lock:
|
||||
if key not in _subscribers:
|
||||
_subscribers[key] = asyncio.Queue(maxsize=100)
|
||||
return _subscribers[key]
|
||||
|
||||
|
||||
async def _remove_subscriber(key: str) -> None:
|
||||
"""移除订阅者."""
|
||||
async with _subscribers_lock:
|
||||
_subscribers.pop(key, None)
|
||||
|
||||
|
||||
async def notify_mastery_update(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
previous_level: float,
|
||||
) -> None:
|
||||
"""通知所有订阅者掌握度更新（由 mastery_service 调用）.
|
||||
|
||||
订阅匹配规则：
|
||||
- 订阅 student_id 的客户端收到通知
|
||||
- 订阅 class_id 的客户端收到该班级所有学生的通知（需要 ExamCache 查 class_id）
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
|
||||
event_id = str(uuid.uuid4())
|
||||
calculated_at = int(time.time())
|
||||
|
||||
# 通知 student_id 订阅者
|
||||
queue = _subscribers.get(f"student:{student_id}")
|
||||
if queue is not None:
|
||||
try:
|
||||
queue.put_nowait(
|
||||
{
|
||||
"event_id": event_id,
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"mastery_level": mastery_level,
|
||||
"previous_level": previous_level,
|
||||
"calculated_at": calculated_at,
|
||||
}
|
||||
)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning("mastery_update_queue_full_dropped", student_id=student_id)
|
||||
|
||||
|
||||
# ===== AnalyticsService 实现 =====
|
||||
|
||||
|
||||
class AnalyticsServiceServicer:
|
||||
"""AnalyticsService gRPC 实现（12 RPC）."""
|
||||
|
||||
async def GetClassPerformance(self, request, context):
|
||||
"""班级成绩分析."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_class_performance(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
subject_id=request.subject_id,
|
||||
start_date=request.start_date,
|
||||
end_date=request.end_date,
|
||||
)
|
||||
return _build_class_performance_response(result)
|
||||
|
||||
async def GetStudentWeakness(self, request, context):
|
||||
"""学生薄弱知识点."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_weakness(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_student_weakness_response(result)
|
||||
|
||||
async def GetLearningTrend(self, request, context):
|
||||
"""学习趋势."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_learning_trend(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
start_date=request.start_date,
|
||||
end_date=request.end_date,
|
||||
)
|
||||
return _build_learning_trend_response(result)
|
||||
|
||||
async def GetTeacherDashboard(self, request, context):
|
||||
"""教师仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_teacher_dashboard(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
)
|
||||
return _build_teacher_dashboard_response(result)
|
||||
|
||||
async def GetStudentDashboard(self, request, context):
|
||||
"""学生仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_dashboard(
|
||||
user=user,
|
||||
)
|
||||
return _build_student_dashboard_response(result)
|
||||
|
||||
async def GetParentDashboard(self, request, context):
|
||||
"""家长仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_parent_dashboard(
|
||||
user=user,
|
||||
child_id=request.student_id,
|
||||
)
|
||||
return _build_parent_dashboard_response(result)
|
||||
|
||||
async def GetAdminDashboard(self, request, context):
|
||||
"""管理员仪表盘."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_admin_dashboard(
|
||||
user=user,
|
||||
school_id=request.scope_id,
|
||||
)
|
||||
return _build_admin_dashboard_response(result)
|
||||
|
||||
async def GetWarnings(self, request, context):
|
||||
"""预警列表查询."""
|
||||
# 简化：按 class_id 查询（P6 改为多条件过滤）
|
||||
warnings = await warning_service.get_warnings(
|
||||
student_id="", # gRPC 层按 class_id 查询，P6 扩展
|
||||
warning_type="",
|
||||
)
|
||||
return _build_warning_list_response(warnings)
|
||||
|
||||
async def TriggerWarning(self, request, context):
|
||||
"""手动触发预警."""
|
||||
result = await warning_service.trigger_warning_manual(
|
||||
target_id=request.target_id,
|
||||
warning_type=request.warning_type,
|
||||
threshold=0.0,
|
||||
current_value=0.0,
|
||||
severity=request.severity,
|
||||
)
|
||||
return _build_trigger_warning_response(result)
|
||||
|
||||
async def GetMasteryDistribution(self, request, context):
|
||||
"""班级掌握度分布."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_mastery_distribution(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
subject_id=request.subject_id,
|
||||
knowledge_point_id=request.knowledge_point_id,
|
||||
)
|
||||
return _build_mastery_distribution_response(result)
|
||||
|
||||
async def GetStudentMastery(self, request, context):
|
||||
"""学生知识点掌握度明细."""
|
||||
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_mastery(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_student_mastery_response(result)
|
||||
|
||||
async def SubscribeMasteryUpdate(self, request, context):
|
||||
"""订阅掌握度更新（server-streaming，P5+）."""
|
||||
# 确定订阅 key
|
||||
if request.student_id:
|
||||
sub_key = f"student:{request.student_id}"
|
||||
elif request.class_id:
|
||||
sub_key = f"class:{request.class_id}"
|
||||
else:
|
||||
sub_key = "global"
|
||||
|
||||
queue = await _add_subscriber(sub_key)
|
||||
logger.info(
|
||||
"mastery_subscription_added",
|
||||
sub_key=sub_key,
|
||||
peer=context.peer() if hasattr(context, "peer") else "unknown",
|
||||
)
|
||||
|
||||
try:
|
||||
while context.is_active():
|
||||
try:
|
||||
event = await asyncio.wait_for(queue.get(), timeout=30.0)
|
||||
yield _build_mastery_update_event(event)
|
||||
except TimeoutError:
|
||||
# 心跳：30s 无事件时发一个空 keepalive（客户端应容忍）
|
||||
continue
|
||||
except asyncio.CancelledError:
|
||||
logger.info("mastery_subscription_cancelled", sub_key=sub_key)
|
||||
raise
|
||||
finally:
|
||||
await _remove_subscriber(sub_key)
|
||||
logger.info("mastery_subscription_removed", sub_key=sub_key)
|
||||
|
||||
|
||||
# ===== 辅助函数 =====
|
||||
|
||||
|
||||
def _extract_user(context: Any) -> UserContext:
|
||||
"""从 gRPC metadata 提取用户上下文."""
|
||||
user_id = ""
|
||||
roles: list[str] = []
|
||||
try:
|
||||
metadata = (
|
||||
dict(context.invocation_metadata()) if hasattr(context, "invocation_metadata") else {}
|
||||
)
|
||||
user_id = metadata.get("x-user-id", "")
|
||||
roles_str = metadata.get("x-user-roles", "")
|
||||
roles = [r.strip() for r in roles_str.split(",") if r.strip()] if roles_str else []
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return UserContext(user_id=user_id, roles=roles)
|
||||
|
||||
|
||||
def _build_class_performance_response(data: dict) -> Any:
|
||||
"""构建 ClassPerformance proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.ClassPerformance(
|
||||
class_id=data.get("classId", ""),
|
||||
average_score=data.get("averageScore", 0.0),
|
||||
pass_rate=data.get("passRate", 0.0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_student_weakness_response(data: dict) -> Any:
|
||||
"""构建 StudentWeakness proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
return analytics_pb2.StudentWeakness(
|
||||
student_id=data.get("studentId", ""),
|
||||
weak_points=weak_points,
|
||||
)
|
||||
|
||||
|
||||
def _build_learning_trend_response(data: dict) -> Any:
|
||||
"""构建 LearningTrend proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
points = [
|
||||
analytics_pb2.TrendPoint(
|
||||
date=p.get("date", 0),
|
||||
score=p.get("score", 0.0),
|
||||
)
|
||||
for p in data.get("points", [])
|
||||
]
|
||||
return analytics_pb2.LearningTrend(
|
||||
student_id=data.get("studentId", ""),
|
||||
points=points,
|
||||
)
|
||||
|
||||
|
||||
def _build_teacher_dashboard_response(data: dict) -> Any:
|
||||
"""构建 TeacherDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
classes = [
|
||||
analytics_pb2.ClassSummary(
|
||||
class_id=c.get("class_id", ""),
|
||||
class_name=c.get("class_name", ""),
|
||||
student_count=c.get("total_students", 0),
|
||||
average_score=c.get("average_score", 0.0),
|
||||
)
|
||||
for c in data.get("classes", [])
|
||||
]
|
||||
return analytics_pb2.TeacherDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
total_classes=data.get("totalClasses", 0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
class_avg_score=data.get("classAvgScore", 0.0),
|
||||
pending_homework_count=data.get("pendingHomeworkCount", 0),
|
||||
classes=classes,
|
||||
)
|
||||
|
||||
|
||||
def _build_student_dashboard_response(data: dict) -> Any:
|
||||
"""构建 StudentDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
trends = [
|
||||
analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0))
|
||||
for p in data.get("trend", [])
|
||||
]
|
||||
return analytics_pb2.StudentDashboard(
|
||||
user_id=data.get("studentId", data.get("userId", "")),
|
||||
avg_score=data.get("averageScore", 0.0),
|
||||
class_rank=0,
|
||||
total_students=0,
|
||||
weak_points=weak_points,
|
||||
recent_trends=trends,
|
||||
)
|
||||
|
||||
|
||||
def _build_parent_dashboard_response(data: dict) -> Any:
|
||||
"""构建 ParentDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
weak_points = [
|
||||
analytics_pb2.WeakPoint(
|
||||
knowledge_point_id=wp.get("knowledgePointId", ""),
|
||||
title=wp.get("title", ""),
|
||||
mastery=wp.get("mastery", 0.0),
|
||||
error_count=wp.get("errorCount", 0),
|
||||
)
|
||||
for wp in data.get("weakPoints", [])
|
||||
]
|
||||
return analytics_pb2.ParentDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
student_id=data.get("childId", ""),
|
||||
child_avg_score=data.get("averageScore", 0.0),
|
||||
child_class_rank=0,
|
||||
total_class_students=0,
|
||||
child_weak_points=weak_points,
|
||||
)
|
||||
|
||||
|
||||
def _build_admin_dashboard_response(data: dict) -> Any:
|
||||
"""构建 AdminDashboard proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
ai_usage_data = data.get("aiUsage", {})
|
||||
by_provider = [
|
||||
analytics_pb2.AIUsageByProvider(
|
||||
provider=p.get("provider", ""),
|
||||
request_count=p.get("requestCount", 0),
|
||||
total_tokens=p.get("totalTokens", 0),
|
||||
cost_cents=p.get("costCents", 0),
|
||||
)
|
||||
for p in ai_usage_data.get("byProvider", [])
|
||||
]
|
||||
ai_usage = analytics_pb2.AIUsageSummary(
|
||||
total_requests=ai_usage_data.get("totalRequests", 0),
|
||||
total_tokens=ai_usage_data.get("totalTokens", 0),
|
||||
total_cost_cents=ai_usage_data.get("totalCostCents", 0),
|
||||
by_provider=by_provider,
|
||||
)
|
||||
return analytics_pb2.AdminDashboard(
|
||||
user_id=data.get("userId", ""),
|
||||
total_teachers=0,
|
||||
total_students=data.get("totalStudents", 0),
|
||||
total_classes=data.get("totalClasses", 0),
|
||||
school_avg_score=data.get("classAvgScore", 0.0),
|
||||
ai_usage=ai_usage,
|
||||
)
|
||||
|
||||
|
||||
def _build_warning_list_response(warnings: list[dict]) -> Any:
|
||||
"""构建 WarningList proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
warning_infos = [
|
||||
analytics_pb2.WarningInfo(
|
||||
warning_id=w.get("warning_id", ""),
|
||||
warning_type=w.get("warning_type", ""),
|
||||
target_id=w.get("target_id", ""),
|
||||
target_name=w.get("target_name", ""),
|
||||
threshold=w.get("threshold", 0.0),
|
||||
current_value=w.get("current_value", 0.0),
|
||||
severity=w.get("severity", "WARN"),
|
||||
occurred_at=int(w.get("occurred_at", 0)),
|
||||
)
|
||||
for w in warnings
|
||||
]
|
||||
return analytics_pb2.WarningList(
|
||||
warnings=warning_infos,
|
||||
total=len(warning_infos),
|
||||
)
|
||||
|
||||
|
||||
def _build_trigger_warning_response(data: dict) -> Any:
|
||||
"""构建 TriggerWarningResponse proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.TriggerWarningResponse(
|
||||
warning_id=data.get("warning_id", ""),
|
||||
triggered=data.get("triggered", False),
|
||||
)
|
||||
|
||||
|
||||
def _build_mastery_distribution_response(data: dict) -> Any:
|
||||
"""构建 MasteryDistribution proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.MasteryDistribution(
|
||||
class_id=data.get("classId", ""),
|
||||
mastered_count=data.get("masteredCount", 0),
|
||||
progressing_count=data.get("progressingCount", 0),
|
||||
weak_count=data.get("weakCount", 0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_student_mastery_response(data: dict) -> Any:
|
||||
"""构建 StudentMastery proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
knowledge_points = [
|
||||
analytics_pb2.KnowledgePointMastery(
|
||||
knowledge_point_id=kp.get("knowledge_point_id", ""),
|
||||
title=kp.get("title", ""),
|
||||
subject_id=kp.get("subject_id", ""),
|
||||
mastery_level=kp.get("mastery_level", 0.0),
|
||||
mastery_label=kp.get("mastery_label", "weak"),
|
||||
calculated_at=kp.get("calculated_at", 0),
|
||||
)
|
||||
for kp in data.get("knowledgePoints", [])
|
||||
]
|
||||
return analytics_pb2.StudentMastery(
|
||||
student_id=data.get("studentId", ""),
|
||||
knowledge_points=knowledge_points,
|
||||
overall_mastery=data.get("overallMastery", 0.0),
|
||||
)
|
||||
|
||||
|
||||
def _build_mastery_update_event(event: dict) -> Any:
|
||||
"""构建 MasteryUpdateEvent proto 响应（streaming）."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.MasteryUpdateEvent(
|
||||
event_id=event.get("event_id", ""),
|
||||
student_id=event.get("student_id", ""),
|
||||
knowledge_point_id=event.get("knowledge_point_id", ""),
|
||||
mastery_level=event.get("mastery_level", 0.0),
|
||||
previous_level=event.get("previous_level", 0.0),
|
||||
calculated_at=event.get("calculated_at", 0),
|
||||
)
|
||||
|
||||
|
||||
# ===== gRPC Server 管理 =====
|
||||
|
||||
_server: Any | None = None
|
||||
|
||||
|
||||
async def start_grpc_server() -> Any | None:
|
||||
"""启动 gRPC server :50055（lifespan 调用）.
|
||||
|
||||
降级策略：
|
||||
- grpcio 未安装：返回 None，仅 HTTP 服务
|
||||
- 启动失败：仅记录错误，不阻塞 FastAPI 主流程
|
||||
"""
|
||||
global _server
|
||||
|
||||
try:
|
||||
import grpc # type: ignore[import-not-found]
|
||||
from generated_proto import analytics_pb2_grpc # type: ignore[import-not-found]
|
||||
except ImportError as exc:
|
||||
logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc))
|
||||
return None
|
||||
|
||||
_server = grpc.aio.server()
|
||||
|
||||
# 注册 AnalyticsService
|
||||
servicer = AnalyticsServiceServicer()
|
||||
analytics_pb2_grpc.add_AnalyticsServiceServicer_to_server(servicer, _server)
|
||||
|
||||
# 注册 HealthService
|
||||
try:
|
||||
from grpc_health.v1 import ( # type: ignore[import-not-found]
|
||||
health,
|
||||
health_pb2,
|
||||
health_pb2_grpc,
|
||||
)
|
||||
|
||||
health_servicer = health.aio.HealthServicer()
|
||||
await health_servicer.set("analytics", health_pb2.HealthCheckResponse.SERVING)
|
||||
await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING) # overall
|
||||
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, _server)
|
||||
except ImportError:
|
||||
logger.warning("grpc_health_not_installed_skip_health_service")
|
||||
|
||||
# 绑定端口
|
||||
bind_address = f"[::]:{settings.grpc_port}"
|
||||
_server.add_insecure_port(bind_address)
|
||||
|
||||
try:
|
||||
await _server.start()
|
||||
logger.info(
|
||||
"grpc_server_started",
|
||||
port=settings.grpc_port,
|
||||
rpc_count=12,
|
||||
)
|
||||
return _server
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("grpc_server_start_failed", error=str(exc), port=settings.grpc_port)
|
||||
_server = None
|
||||
return None
|
||||
|
||||
|
||||
async def stop_grpc_server() -> None:
|
||||
"""停止 gRPC server（lifespan 退出时调用）."""
|
||||
global _server
|
||||
if _server is not None:
|
||||
try:
|
||||
await _server.stop(grace=5)
|
||||
logger.info("grpc_server_stopped")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("grpc_server_stop_failed", error=str(exc))
|
||||
finally:
|
||||
_server = None
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
"""gRPC server 是否运行中."""
|
||||
return _server is not None
|
||||
@@ -1,20 +1,40 @@
|
||||
"""数据分析服务入口.
|
||||
"""数据分析服务入口（FastAPI HTTP :3006）.
|
||||
|
||||
支持 ClickHouse 降级模式：当 CLICKHOUSE_HOST 未配置或不可达时，
|
||||
查询端点返回骨架数据，服务仍可启动与响应。
|
||||
端点清单（3 基础 + 11 业务 = 14 个）：
|
||||
基础：
|
||||
GET / 根信息
|
||||
GET /healthz 活性检查（liveness）
|
||||
GET /readyz 就绪检查（readiness，检查 4 依赖）
|
||||
|
||||
支持 CDC 消费者：当 KAFKA_BROKERS 配置时，
|
||||
后台启动 aiokafka 消费者，监听 Debezium CDC 事件写入 ClickHouse。
|
||||
业务（全部返回 ActionState[T] 信封）：
|
||||
GET /analytics/class/{class_id}/performance 班级成绩分析
|
||||
GET /analytics/student/{student_id}/weakness 学生薄弱知识点
|
||||
GET /analytics/student/{student_id}/trend 学习趋势
|
||||
GET /analytics/student/{student_id}/errorbook 错题本（额外）
|
||||
GET /analytics/teacher/dashboard 教师仪表盘
|
||||
GET /analytics/student/dashboard 学生仪表盘
|
||||
GET /analytics/parent/dashboard 家长仪表盘
|
||||
GET /analytics/admin/dashboard 管理员仪表盘
|
||||
GET /analytics/warnings 预警列表
|
||||
POST /analytics/warnings/trigger 手动触发预警
|
||||
GET /analytics/class/{class_id}/mastery-distribution 班级掌握度分布
|
||||
GET /analytics/student/{student_id}/mastery 学生掌握度明细
|
||||
|
||||
设计要点：
|
||||
- 所有业务端点返回 ActionState[T]（coord-cross-review §5.3 P0 整改）
|
||||
- 降级标记在顶层 details.degraded（不放 error.details）
|
||||
- /readyz 检查 4 依赖：clickhouse / cdc_consumer / redis / iam_grpc
|
||||
- gRPC server :50055 在 lifespan 启动
|
||||
- CDC 消费者在 lifespan 启动
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, FastAPI
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
@@ -22,23 +42,20 @@ from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from prometheus_client import make_asgi_app
|
||||
|
||||
from .cdc_consumer import run_consumer as run_cdc_consumer
|
||||
from .clickhouse_client import (
|
||||
close_client,
|
||||
query_class_performance,
|
||||
query_dashboard,
|
||||
query_student_errors,
|
||||
)
|
||||
from .clickhouse_client import ping as ch_ping
|
||||
from . import analytics_service, cdc_consumer, grpc_server, warning_service
|
||||
from .config import settings
|
||||
from .repository import (
|
||||
clickhouse_repository,
|
||||
iam_client,
|
||||
kafka_producer,
|
||||
redis_client,
|
||||
)
|
||||
from .shared.action_state import ActionState
|
||||
from .shared.permissions import UserContext, get_user_context
|
||||
|
||||
_logger: structlog.stdlib.BoundLogger | None = None
|
||||
tracer = trace.get_tracer(__name__)
|
||||
|
||||
# CDC 消费者后台任务句柄
|
||||
_cdc_task: asyncio.Task | None = None
|
||||
|
||||
# 日志级别映射
|
||||
_LOG_LEVELS: dict[str, int] = {
|
||||
"DEBUG": 10,
|
||||
"INFO": 20,
|
||||
@@ -49,10 +66,7 @@ _LOG_LEVELS: dict[str, int] = {
|
||||
|
||||
|
||||
def init_logger() -> structlog.stdlib.BoundLogger:
|
||||
"""初始化 structlog logger.
|
||||
|
||||
根据配置的 log_level 设置日志级别。
|
||||
"""
|
||||
"""初始化 structlog logger."""
|
||||
global _logger
|
||||
level = _LOG_LEVELS.get(settings.log_level.upper(), 20)
|
||||
structlog.configure(
|
||||
@@ -70,7 +84,7 @@ def init_logger() -> structlog.stdlib.BoundLogger:
|
||||
|
||||
|
||||
def get_logger() -> structlog.stdlib.BoundLogger:
|
||||
"""获取已初始化的 logger（未初始化时自动初始化）."""
|
||||
"""获取已初始化的 logger."""
|
||||
global _logger
|
||||
if _logger is None:
|
||||
return init_logger()
|
||||
@@ -78,10 +92,7 @@ def get_logger() -> structlog.stdlib.BoundLogger:
|
||||
|
||||
|
||||
def init_tracer() -> None:
|
||||
"""初始化 OpenTelemetry.
|
||||
|
||||
endpoint 从 settings.otel_endpoint 读取（不硬编码）。
|
||||
"""
|
||||
"""初始化 OpenTelemetry."""
|
||||
provider = TracerProvider()
|
||||
endpoint = settings.otel_endpoint.rstrip("/")
|
||||
exporter = OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")
|
||||
@@ -93,207 +104,418 @@ def init_tracer() -> None:
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""应用生命周期.
|
||||
|
||||
1. 初始化 logger（structlog）
|
||||
2. 初始化 OTel tracer（endpoint 从 config 读）
|
||||
3. 触发 ClickHouse 客户端惰性初始化（不阻塞启动，失败进入降级模式）
|
||||
4. 若配置了 kafka_brokers，后台启动 CDC 消费者任务
|
||||
5. 关闭时停止 CDC 任务并释放 ClickHouse 客户端
|
||||
启动顺序：
|
||||
1. 初始化 logger + tracer
|
||||
2. 启动 gRPC server :50055（AnalyticsService 12 RPC）
|
||||
3. 启动 CDC 消费者后台任务（手动 commit）
|
||||
4. Kafka producer 惰性初始化（首次发布时触发）
|
||||
|
||||
关闭顺序：
|
||||
1. 停止 CDC 消费者
|
||||
2. 停止 gRPC server
|
||||
3. 关闭 Kafka producer
|
||||
4. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
"""
|
||||
global _cdc_task
|
||||
logger = init_logger()
|
||||
init_tracer()
|
||||
logger.info(
|
||||
"data_ana_service_starting",
|
||||
port=settings.port,
|
||||
http_port=settings.http_port,
|
||||
grpc_port=settings.grpc_port,
|
||||
dev_mode=settings.dev_mode,
|
||||
clickhouse_configured=bool(settings.clickhouse_host),
|
||||
kafka_brokers=settings.kafka_brokers,
|
||||
kafka_cdc_topics=settings.kafka_cdc_topics,
|
||||
iam_grpc_endpoint=settings.iam_grpc_endpoint,
|
||||
redis_url=settings.redis_url or "not_configured",
|
||||
)
|
||||
# 启动 CDC 消费者后台任务（若未配置 kafka_brokers，run_consumer 内部直接返回）
|
||||
_cdc_task = asyncio.create_task(run_cdc_consumer())
|
||||
|
||||
# 1. 启动 gRPC server（grpcio 未安装则跳过，降级为仅 HTTP）
|
||||
grpc_server_obj = await grpc_server.start_grpc_server()
|
||||
if grpc_server_obj is None:
|
||||
logger.warning("grpc_server_not_started_http_only")
|
||||
|
||||
# 2. 启动 CDC 消费者后台任务
|
||||
await cdc_consumer.start_consumer()
|
||||
|
||||
yield
|
||||
|
||||
logger.info("data_ana_service_stopping")
|
||||
# 取消 CDC 任务
|
||||
if _cdc_task is not None and not _cdc_task.done():
|
||||
_cdc_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await _cdc_task
|
||||
await close_client()
|
||||
|
||||
# 3. 停止 CDC 消费者
|
||||
await cdc_consumer.stop_consumer()
|
||||
|
||||
# 4. 停止 gRPC server
|
||||
await grpc_server.stop_grpc_server()
|
||||
|
||||
# 5. 关闭 Kafka producer
|
||||
await kafka_producer.close_producer()
|
||||
|
||||
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
await redis_client.close_client()
|
||||
await iam_client.close_grpc()
|
||||
await clickhouse_repository.close_client()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Data Analytics Service",
|
||||
version="0.1.0",
|
||||
version="1.0.0",
|
||||
description="D6 智能洞察领域服务（ClickHouse 宽表 + CDC + 掌握度算法 + 预警）",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# OpenTelemetry FastAPI 自动埋点（HTTP 请求/响应 span）
|
||||
FastAPIInstrumentor.instrument_app(app)
|
||||
|
||||
# Prometheus 指标
|
||||
app.mount("/metrics", make_asgi_app())
|
||||
|
||||
# 业务路由
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
async def healthz() -> dict:
|
||||
"""健康检查（liveness）.
|
||||
# ===== 基础端点（3 个） =====
|
||||
|
||||
只要进程存活即返回 ok，不依赖 ClickHouse。
|
||||
"""
|
||||
|
||||
@app.get("/")
|
||||
async def root() -> dict[str, Any]:
|
||||
"""根信息."""
|
||||
return {
|
||||
"service": "data-ana",
|
||||
"version": "1.0.0",
|
||||
"http_port": settings.http_port,
|
||||
"grpc_port": settings.grpc_port,
|
||||
"docs": "/docs",
|
||||
"endpoints": {
|
||||
"healthz": "/healthz",
|
||||
"readyz": "/readyz",
|
||||
"business": "/analytics/*",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
async def healthz() -> dict[str, str]:
|
||||
"""健康检查（liveness，只要进程存活即返回 ok）."""
|
||||
return {"status": "ok", "service": "data-ana"}
|
||||
|
||||
|
||||
@app.get("/readyz")
|
||||
async def readyz() -> dict:
|
||||
"""就绪检查（readiness）.
|
||||
async def readyz() -> dict[str, Any]:
|
||||
"""就绪检查（readiness，检查 4 依赖）.
|
||||
|
||||
ClickHouse 为可选依赖：
|
||||
- 已配置且可达：ready=true
|
||||
- 未配置：ready=true，degraded=true（降级模式仍可服务）
|
||||
- 已配置但不可达：ready=false
|
||||
依赖检查：
|
||||
1. clickhouse：已配置且可达（未配置算降级就绪）
|
||||
2. cdc_consumer：running / disabled
|
||||
3. redis：已配置且可达（未配置算降级就绪）
|
||||
4. iam_grpc：已配置且可达（未配置算降级就绪）
|
||||
|
||||
CDC 消费者状态附加在响应中：
|
||||
- cdc_consumer: running / disabled / failed
|
||||
返回 ready=true 的条件：
|
||||
- ClickHouse 已配置且可达，或未配置（降级就绪）
|
||||
- 不要求所有依赖都健康（降级模式下仍可服务骨架数据）
|
||||
"""
|
||||
cdc_status = "disabled"
|
||||
if _cdc_task is not None:
|
||||
if _cdc_task.done():
|
||||
cdc_status = "failed"
|
||||
elif not settings.kafka_brokers:
|
||||
cdc_status = "disabled"
|
||||
else:
|
||||
cdc_status = "running"
|
||||
# 1. ClickHouse
|
||||
ch_ok = await clickhouse_repository.ping()
|
||||
ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
|
||||
|
||||
# 2. CDC 消费者
|
||||
cdc_status = (
|
||||
"running"
|
||||
if cdc_consumer.is_running()
|
||||
else ("disabled" if not settings.kafka_brokers else "failed")
|
||||
)
|
||||
|
||||
# 3. Redis
|
||||
redis_ok = await redis_client.ping() if settings.redis_url else None
|
||||
redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
|
||||
|
||||
# 4. iam gRPC
|
||||
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
|
||||
iam_status = (
|
||||
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
|
||||
)
|
||||
|
||||
# 5. gRPC server
|
||||
grpc_status = "running" if grpc_server.is_running() else "stopped"
|
||||
|
||||
# 就绪判定：ClickHouse 可达或未配置（降级就绪）
|
||||
ready = ch_ok or not settings.clickhouse_host
|
||||
degraded = not ch_ok or not redis_ok or not iam_ok
|
||||
|
||||
if not settings.clickhouse_host:
|
||||
return {
|
||||
"status": "ok",
|
||||
"status": "ok" if ready else "not_ready",
|
||||
"service": "data-ana",
|
||||
"ready": True,
|
||||
"degraded": True,
|
||||
"clickhouse": "not_configured",
|
||||
"ready": ready,
|
||||
"degraded": degraded,
|
||||
"dependencies": {
|
||||
"clickhouse": ch_status,
|
||||
"cdc_consumer": cdc_status,
|
||||
"kafka_brokers": settings.kafka_brokers or None,
|
||||
"redis": redis_status,
|
||||
"iam_grpc": iam_status,
|
||||
"grpc_server": grpc_status,
|
||||
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
|
||||
},
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
ch_ok = await ch_ping()
|
||||
return {
|
||||
"status": "ok" if ch_ok else "degraded",
|
||||
"service": "data-ana",
|
||||
"ready": ch_ok,
|
||||
"degraded": not ch_ok,
|
||||
"clickhouse": "ok" if ch_ok else "unreachable",
|
||||
"cdc_consumer": cdc_status,
|
||||
"kafka_brokers": settings.kafka_brokers or None,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
# ===== 业务端点（11 个，全部返回 ActionState[T]） =====
|
||||
|
||||
|
||||
@router.get("/analytics/class/{class_id}/performance")
|
||||
async def class_performance(class_id: str) -> dict:
|
||||
"""班级成绩分析.
|
||||
|
||||
优先查 ClickHouse；降级时返回骨架数据。
|
||||
"""
|
||||
logger = get_logger()
|
||||
with tracer.start_as_current_span("class_performance") as span:
|
||||
async def get_class_performance(
|
||||
class_id: str,
|
||||
subject_id: str = Query(""),
|
||||
start_date: int = Query(0),
|
||||
end_date: int = Query(0),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""班级成绩分析."""
|
||||
with tracer.start_as_current_span("get_class_performance") as span:
|
||||
span.set_attribute("class_id", class_id)
|
||||
result = await query_class_performance(class_id)
|
||||
if result is None:
|
||||
logger.info("class_performance_degraded", class_id=class_id)
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"classId": class_id,
|
||||
"averageScore": 0,
|
||||
"passRate": 0,
|
||||
"totalStudents": 0,
|
||||
"message": "ClickHouse unavailable - skeleton data",
|
||||
"degraded": True,
|
||||
},
|
||||
}
|
||||
return {"success": True, "data": {**result, "degraded": False}}
|
||||
result = await analytics_service.get_class_performance(
|
||||
user=user,
|
||||
class_id=class_id,
|
||||
subject_id=subject_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/student/{student_id}/weakness")
|
||||
async def student_weakness(student_id: str) -> dict:
|
||||
"""学生薄弱知识点分析.
|
||||
|
||||
优先查 ClickHouse；降级时返回骨架数据。
|
||||
"""
|
||||
logger = get_logger()
|
||||
with tracer.start_as_current_span("student_weakness") as span:
|
||||
async def get_student_weakness(
|
||||
student_id: str,
|
||||
subject_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""学生薄弱知识点."""
|
||||
with tracer.start_as_current_span("get_student_weakness") as span:
|
||||
span.set_attribute("student_id", student_id)
|
||||
result = await query_dashboard(student_id)
|
||||
if result is None:
|
||||
logger.info("student_weakness_degraded", student_id=student_id)
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"studentId": student_id,
|
||||
"weakPoints": [],
|
||||
"message": "ClickHouse unavailable - skeleton data",
|
||||
"degraded": True,
|
||||
},
|
||||
}
|
||||
result = await analytics_service.get_student_weakness(
|
||||
user=user,
|
||||
student_id=student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
# 从宽表提取薄弱知识点：mastery_level < 0.6 视为薄弱
|
||||
weak_points = [
|
||||
{
|
||||
"knowledgePointId": r["knowledge_point_id"],
|
||||
"masteryLevel": r["mastery_level"],
|
||||
"errorCount": r["error_count"],
|
||||
}
|
||||
for r in result["records"]
|
||||
if r.get("mastery_level") is not None and r["mastery_level"] < 0.6
|
||||
]
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"studentId": student_id,
|
||||
"weakPoints": weak_points,
|
||||
"records": result["records"],
|
||||
"total": result["total"],
|
||||
"degraded": False,
|
||||
},
|
||||
}
|
||||
|
||||
@router.get("/analytics/student/{student_id}/trend")
|
||||
async def get_learning_trend(
|
||||
student_id: str,
|
||||
subject_id: str = Query(""),
|
||||
start_date: int = Query(0),
|
||||
end_date: int = Query(0),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""学习趋势."""
|
||||
with tracer.start_as_current_span("get_learning_trend") as span:
|
||||
span.set_attribute("student_id", student_id)
|
||||
result = await analytics_service.get_learning_trend(
|
||||
user=user,
|
||||
student_id=student_id,
|
||||
subject_id=subject_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/student/{student_id}/errorbook")
|
||||
async def student_errorbook(student_id: str) -> dict:
|
||||
"""学生错题本.
|
||||
|
||||
优先查 ClickHouse；降级时返回空列表。
|
||||
"""
|
||||
logger = get_logger()
|
||||
with tracer.start_as_current_span("student_errorbook") as span:
|
||||
async def get_student_errorbook(
|
||||
student_id: str,
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""学生错题本."""
|
||||
with tracer.start_as_current_span("get_student_errorbook") as span:
|
||||
span.set_attribute("student_id", student_id)
|
||||
result = await query_student_errors(student_id)
|
||||
if result is None:
|
||||
logger.info("student_errorbook_degraded", student_id=student_id)
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
errors = await clickhouse_repository.query_student_errors(student_id)
|
||||
if errors is None:
|
||||
return ActionState.ok(
|
||||
{"studentId": student_id, "errors": [], "total": 0},
|
||||
degraded=True,
|
||||
degraded_reason="clickhouse_unavailable",
|
||||
)
|
||||
return ActionState.ok(
|
||||
{
|
||||
"studentId": student_id,
|
||||
"errors": [],
|
||||
"total": 0,
|
||||
"message": "ClickHouse unavailable - empty errorbook",
|
||||
"degraded": True,
|
||||
},
|
||||
"errors": errors,
|
||||
"total": len(errors),
|
||||
}
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"studentId": student_id,
|
||||
"errors": result,
|
||||
"total": len(result),
|
||||
"degraded": False,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/teacher/dashboard")
|
||||
async def get_teacher_dashboard(
|
||||
class_id: str = Query(""),
|
||||
subject_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""教师仪表盘."""
|
||||
with tracer.start_as_current_span("get_teacher_dashboard"):
|
||||
result = await analytics_service.get_teacher_dashboard(
|
||||
user=user,
|
||||
class_id=class_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/student/dashboard")
|
||||
async def get_student_dashboard(
|
||||
subject_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""学生仪表盘."""
|
||||
with tracer.start_as_current_span("get_student_dashboard"):
|
||||
result = await analytics_service.get_student_dashboard(
|
||||
user=user,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/parent/dashboard")
|
||||
async def get_parent_dashboard(
|
||||
student_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""家长仪表盘."""
|
||||
with tracer.start_as_current_span("get_parent_dashboard"):
|
||||
result = await analytics_service.get_parent_dashboard(
|
||||
user=user,
|
||||
child_id=student_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/admin/dashboard")
|
||||
async def get_admin_dashboard(
|
||||
school_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""管理员仪表盘."""
|
||||
with tracer.start_as_current_span("get_admin_dashboard"):
|
||||
result = await analytics_service.get_admin_dashboard(
|
||||
user=user,
|
||||
school_id=school_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/warnings")
|
||||
async def get_warnings(
|
||||
student_id: str = Query(""),
|
||||
warning_type: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""预警列表查询."""
|
||||
with tracer.start_as_current_span("get_warnings"):
|
||||
warnings = await warning_service.get_warnings(
|
||||
student_id=student_id,
|
||||
warning_type=warning_type,
|
||||
)
|
||||
return ActionState.ok(
|
||||
{
|
||||
"warnings": warnings,
|
||||
"total": len(warnings),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/analytics/warnings/trigger")
|
||||
async def trigger_warning(
|
||||
target_id: str = Query(...),
|
||||
warning_type: str = Query(...),
|
||||
severity: str = Query("WARN"),
|
||||
threshold: float = Query(0.0),
|
||||
current_value: float = Query(0.0),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""手动触发预警."""
|
||||
with tracer.start_as_current_span("trigger_warning"):
|
||||
result = await warning_service.trigger_warning_manual(
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
severity=severity,
|
||||
)
|
||||
return ActionState.ok(result)
|
||||
|
||||
|
||||
@router.get("/analytics/class/{class_id}/mastery-distribution")
|
||||
async def get_mastery_distribution(
|
||||
class_id: str,
|
||||
subject_id: str = Query(""),
|
||||
knowledge_point_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""班级掌握度分布."""
|
||||
with tracer.start_as_current_span("get_mastery_distribution"):
|
||||
result = await analytics_service.get_mastery_distribution(
|
||||
user=user,
|
||||
class_id=class_id,
|
||||
subject_id=subject_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/analytics/student/{student_id}/mastery")
|
||||
async def get_student_mastery(
|
||||
student_id: str,
|
||||
subject_id: str = Query(""),
|
||||
user: UserContext = Depends(get_user_context),
|
||||
) -> ActionState[dict[str, Any]]:
|
||||
"""学生知识点掌握度明细."""
|
||||
with tracer.start_as_current_span("get_student_mastery"):
|
||||
result = await analytics_service.get_student_mastery(
|
||||
user=user,
|
||||
student_id=student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
degraded = result.get("degraded", False)
|
||||
return ActionState.ok(
|
||||
result,
|
||||
degraded=degraded,
|
||||
degraded_reason=result.get("degraded_reason", "") if degraded else "",
|
||||
)
|
||||
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
256
services/data-ana/src/data_ana/mastery_service.py
Normal file
256
services/data-ana/src/data_ana/mastery_service.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""掌握度计算服务（加权滑动平均 + 遗忘曲线）.
|
||||
|
||||
算法对齐 02-architecture-design.md §9：
|
||||
- WEIGHTED_MOVING_AVG：w_i = 0.6^i，归一化（权重从最近到最远递减）
|
||||
- FORGETTING_CURVE：基于遗忘曲线的 max 叠加（P5+ 启用），
|
||||
half_life = 30 days，越久未练习掌握度越衰减.
|
||||
|
||||
输入：学生指定知识点的历史成绩序列（按时间倒序）
|
||||
输出：mastery_level (0.0-1.0)
|
||||
副作用：
|
||||
- 写 mastery_snapshot 表（upsert_mastery_snapshot）
|
||||
- 发布 mastery.updated 事件（Outbox 豁免）
|
||||
|
||||
降级策略：
|
||||
- ClickHouse 不可达：跳过计算（返回 None）
|
||||
- Kafka 不可达：计算继续，事件发布静默失败
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .config import settings
|
||||
from .repository import clickhouse_repository, kafka_producer
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
def _weighted_moving_avg(scores: list[dict[str, Any]]) -> float | None:
|
||||
"""加权滑动平均算法.
|
||||
|
||||
输入：按时间倒序的成绩列表（最近在前），元素含 score 字段
|
||||
输出：归一化的 mastery_level (0.0-1.0)，样本不足返回 None.
|
||||
|
||||
权重：w_i = decay_base ^ i（i 从 0 开始，最近 attempt 权重最大）
|
||||
"""
|
||||
if not scores:
|
||||
return None
|
||||
|
||||
# 截取最近 N 次（window_size）
|
||||
window = scores[: settings.mastery_window_size]
|
||||
if len(window) < settings.mastery_min_samples:
|
||||
return None
|
||||
|
||||
decay = settings.mastery_decay_base
|
||||
total_weight = 0.0
|
||||
weighted_sum = 0.0
|
||||
|
||||
for i, record in enumerate(window):
|
||||
# 成绩归一化到 0.0-1.0（假设原始分 0-100）
|
||||
raw_score = record.get("score", 0.0)
|
||||
normalized = min(max(raw_score / 100.0, 0.0), 1.0)
|
||||
weight = decay**i
|
||||
weighted_sum += normalized * weight
|
||||
total_weight += weight
|
||||
|
||||
if total_weight <= 0:
|
||||
return None
|
||||
|
||||
mastery = weighted_sum / total_weight
|
||||
return round(max(0.0, min(1.0, mastery)), 4)
|
||||
|
||||
|
||||
def _forgetting_curve_decay(
|
||||
mastery: float,
|
||||
last_attempt_at: datetime | None,
|
||||
now: datetime | None = None,
|
||||
) -> float:
|
||||
"""遗忘曲线衰减（P5+ 启用，对齐 02 §9）.
|
||||
|
||||
距离上次练习越久，mastery 越衰减：
|
||||
decayed = mastery * exp(-ln(2) * days_since / half_life)
|
||||
|
||||
half_life = 30 days（settings.mastery_forgetting_half_life_days）.
|
||||
"""
|
||||
if last_attempt_at is None:
|
||||
return mastery
|
||||
|
||||
now = now or datetime.now(UTC)
|
||||
if last_attempt_at.tzinfo is None:
|
||||
last_attempt_at = last_attempt_at.replace(tzinfo=UTC)
|
||||
|
||||
days_since = (now - last_attempt_at).total_seconds() / 86400
|
||||
if days_since <= 0:
|
||||
return mastery
|
||||
|
||||
half_life = settings.mastery_forgetting_half_life_days
|
||||
if half_life <= 0:
|
||||
return mastery
|
||||
|
||||
import math
|
||||
|
||||
decay_factor = math.exp(-math.log(2) * days_since / half_life)
|
||||
decayed = mastery * decay_factor
|
||||
return round(max(0.0, min(1.0, decayed)), 4)
|
||||
|
||||
|
||||
def classify_mastery_label(level: float) -> str:
|
||||
"""掌握度三档分类."""
|
||||
if level >= 0.8:
|
||||
return "mastered"
|
||||
if level >= 0.4:
|
||||
return "progressing"
|
||||
return "weak"
|
||||
|
||||
|
||||
async def calculate_mastery(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""计算学生指定知识点的掌握度.
|
||||
|
||||
流程：
|
||||
1. 查询学生该知识点的历史成绩（按时间倒序）
|
||||
2. 加权滑动平均 → mastery_level
|
||||
3. 遗忘曲线衰减 → 最终 mastery_level
|
||||
4. 写 mastery_snapshot 表
|
||||
5. 发布 mastery.updated 事件（Outbox 豁免）
|
||||
|
||||
返回：
|
||||
{
|
||||
"student_id": ...,
|
||||
"knowledge_point_id": ...,
|
||||
"subject_id": ...,
|
||||
"mastery_level": 0.0-1.0,
|
||||
"previous_level": 0.0-1.0 or None,
|
||||
"mastery_label": "mastered" / "progressing" / "weak",
|
||||
"degraded": bool,
|
||||
}
|
||||
ClickHouse 不可达时返回降级骨架（degraded=true）.
|
||||
"""
|
||||
# 1. 查询历史成绩
|
||||
scores = await clickhouse_repository.query_student_scores_by_kp(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
limit=settings.mastery_window_size * 2, # 多取一倍容错
|
||||
)
|
||||
|
||||
if scores is None:
|
||||
# ClickHouse 不可达降级
|
||||
logger.warning(
|
||||
"mastery_calc_clickhouse_unavailable_degraded",
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
return {
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"subject_id": subject_id,
|
||||
"mastery_level": 0.0,
|
||||
"previous_level": None,
|
||||
"mastery_label": "weak",
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
}
|
||||
|
||||
# 2. 加权滑动平均
|
||||
mastery = _weighted_moving_avg(scores)
|
||||
if mastery is None:
|
||||
# 样本不足，返回默认值
|
||||
return {
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"subject_id": subject_id,
|
||||
"mastery_level": 0.0,
|
||||
"previous_level": None,
|
||||
"mastery_label": "weak",
|
||||
"degraded": False,
|
||||
"degraded_reason": "insufficient_samples",
|
||||
}
|
||||
|
||||
# 3. 遗忘曲线衰减（基于最近一次成绩的时间）
|
||||
last_attempt_at = None
|
||||
if scores:
|
||||
last_attempt_at = scores[0].get("timestamp")
|
||||
|
||||
mastery_final = _forgetting_curve_decay(mastery, last_attempt_at)
|
||||
|
||||
# 4. 查询上一次 mastery（用于事件对比）
|
||||
previous_snapshot = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
previous_level: float | None = None
|
||||
if previous_snapshot is not None:
|
||||
for kp in previous_snapshot.get("knowledgePoints", []):
|
||||
if kp.get("knowledge_point_id") == knowledge_point_id:
|
||||
previous_level = kp.get("mastery_level")
|
||||
break
|
||||
|
||||
# 5. 写 mastery_snapshot 表
|
||||
calculated_at = datetime.now(UTC)
|
||||
write_ok = await clickhouse_repository.upsert_mastery_snapshot(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
subject_id=subject_id,
|
||||
mastery_level=mastery_final,
|
||||
calculated_at=calculated_at,
|
||||
calculation_method="weighted_moving_avg",
|
||||
)
|
||||
if not write_ok:
|
||||
logger.warning(
|
||||
"mastery_snapshot_write_failed_degraded",
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
|
||||
# 6. 发布 mastery.updated 事件（Outbox 豁免）
|
||||
published = await kafka_producer.publish_mastery_updated(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
mastery_level=mastery_final,
|
||||
previous_level=previous_level if previous_level is not None else 0.0,
|
||||
)
|
||||
|
||||
result = {
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"subject_id": subject_id,
|
||||
"mastery_level": mastery_final,
|
||||
"previous_level": previous_level,
|
||||
"mastery_label": classify_mastery_label(mastery_final),
|
||||
"degraded": not write_ok,
|
||||
"degraded_reason": "snapshot_write_failed" if not write_ok else "",
|
||||
"event_published": published,
|
||||
}
|
||||
logger.info(
|
||||
"mastery_calculated",
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
mastery_level=mastery_final,
|
||||
previous_level=previous_level,
|
||||
label=result["mastery_label"],
|
||||
published=published,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def batch_calculate_mastery(
|
||||
student_id: str,
|
||||
knowledge_point_ids: list[str],
|
||||
subject_id: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""批量计算学生多个知识点的掌握度."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for kp_id in knowledge_point_ids:
|
||||
result = await calculate_mastery(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=kp_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if result is not None:
|
||||
results.append(result)
|
||||
return results
|
||||
1
services/data-ana/src/data_ana/repository/__init__.py
Normal file
1
services/data-ana/src/data_ana/repository/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Repository package for data-ana service."""
|
||||
@@ -0,0 +1,794 @@
|
||||
"""ClickHouse Repository（宽表查询 + FINAL/argMax 去重 + DataScope WHERE 注入）.
|
||||
|
||||
P0 整改：ReplacingMergeTree 查询必须加 FINAL 或用 argMax 聚合确保去重生效.
|
||||
支持降级模式：clickhouse_host 未配置或不可达时返回 None，服务仍可启动.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from ..config import settings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
_client: Any | None = None
|
||||
_client_initialized: bool = False
|
||||
|
||||
|
||||
def get_client() -> Any | None:
|
||||
"""获取 ClickHouse 客户端（惰性初始化，失败返回 None 降级）."""
|
||||
global _client, _client_initialized
|
||||
|
||||
if not settings.clickhouse_host:
|
||||
return None
|
||||
|
||||
if _client_initialized:
|
||||
return _client
|
||||
|
||||
_client_initialized = True
|
||||
try:
|
||||
import clickhouse_connect
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"host": settings.clickhouse_host,
|
||||
"port": settings.clickhouse_port,
|
||||
"database": settings.clickhouse_database,
|
||||
"connect_timeout": settings.clickhouse_connect_timeout_ms / 1000,
|
||||
}
|
||||
if settings.clickhouse_user:
|
||||
kwargs["username"] = settings.clickhouse_user
|
||||
if settings.clickhouse_password:
|
||||
kwargs["password"] = settings.clickhouse_password
|
||||
_client = clickhouse_connect.get_client(**kwargs)
|
||||
logger.info(
|
||||
"clickhouse_client_initialized",
|
||||
host=settings.clickhouse_host,
|
||||
port=settings.clickhouse_port,
|
||||
database=settings.clickhouse_database,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("clickhouse_client_init_failed_degraded", error=str(exc))
|
||||
_client = None
|
||||
|
||||
return _client
|
||||
|
||||
|
||||
async def close_client() -> None:
|
||||
"""关闭客户端."""
|
||||
global _client, _client_initialized
|
||||
if _client is not None:
|
||||
try:
|
||||
await asyncio.to_thread(_client.close)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("clickhouse_client_close_failed", error=str(exc))
|
||||
finally:
|
||||
_client = None
|
||||
_client_initialized = False
|
||||
|
||||
|
||||
async def ping() -> bool:
|
||||
"""ClickHouse 连通性检查（供 /readyz 使用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
try:
|
||||
await asyncio.to_thread(client.query, "SELECT 1")
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("clickhouse_ping_failed", error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
# ===== 查询方法（FINAL/argMax 去重） =====
|
||||
|
||||
|
||||
async def query_class_performance(
|
||||
class_id: str,
|
||||
subject_id: str = "",
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> dict | None:
|
||||
"""查询班级成绩分析（聚合 student_dashboard_view，FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["class_id = {cid:String}"]
|
||||
params: dict[str, Any] = {"cid": class_id}
|
||||
if subject_id:
|
||||
where_parts.append("subject_id = {sid:String}")
|
||||
params["sid"] = subject_id
|
||||
if start_date:
|
||||
where_parts.append("last_updated >= {sd:DateTime64(3)}")
|
||||
params["sd"] = datetime.fromtimestamp(start_date)
|
||||
if end_date:
|
||||
where_parts.append("last_updated <= {ed:DateTime64(3)}")
|
||||
params["ed"] = datetime.fromtimestamp(end_date)
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" count(DISTINCT student_id) AS total_students, "
|
||||
f" avg(score) AS average_score, "
|
||||
f" countIf(score >= 60) / count() AS pass_rate "
|
||||
f"FROM student_dashboard_view FINAL "
|
||||
f"WHERE {where_clause}",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_class_performance_failed", error=str(exc), class_id=class_id)
|
||||
return None
|
||||
|
||||
if not rows:
|
||||
return {"classId": class_id, "averageScore": 0.0, "passRate": 0.0, "totalStudents": 0}
|
||||
|
||||
total_students, average_score, pass_rate = rows[0]
|
||||
return {
|
||||
"classId": class_id,
|
||||
"averageScore": float(average_score) if average_score is not None else 0.0,
|
||||
"passRate": float(pass_rate) if pass_rate is not None else 0.0,
|
||||
"totalStudents": int(total_students) if total_students is not None else 0,
|
||||
}
|
||||
|
||||
|
||||
async def query_student_dashboard(student_id: str) -> dict | None:
|
||||
"""查询学生学情宽表（FINAL 去重，返回最近 50 条）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
"SELECT student_id, class_id, exam_id, subject_id, score, "
|
||||
"rank_in_class, knowledge_point_id, mastery_level, error_count, "
|
||||
"last_updated "
|
||||
"FROM student_dashboard_view FINAL "
|
||||
"WHERE student_id = {sid:String} "
|
||||
"ORDER BY last_updated DESC "
|
||||
"LIMIT 50",
|
||||
parameters={"sid": student_id},
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_student_dashboard_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
columns = [
|
||||
"student_id",
|
||||
"class_id",
|
||||
"exam_id",
|
||||
"subject_id",
|
||||
"score",
|
||||
"rank_in_class",
|
||||
"knowledge_point_id",
|
||||
"mastery_level",
|
||||
"error_count",
|
||||
"last_updated",
|
||||
]
|
||||
records = [dict(zip(columns, row, strict=True)) for row in rows]
|
||||
return {"studentId": student_id, "records": records, "total": len(records)}
|
||||
|
||||
|
||||
async def query_student_weakness(student_id: str, subject_id: str = "") -> dict | None:
|
||||
"""查询学生薄弱知识点（mastery_level < 0.6，FINAL 去重）."""
|
||||
dashboard = await query_student_dashboard(student_id)
|
||||
if dashboard is None:
|
||||
return None
|
||||
|
||||
weak_points = [
|
||||
{
|
||||
"knowledgePointId": r["knowledge_point_id"],
|
||||
"title": r.get("knowledge_point_id", ""), # content CDC 同步后补充标题
|
||||
"mastery": r["mastery_level"],
|
||||
"errorCount": r["error_count"],
|
||||
}
|
||||
for r in dashboard["records"]
|
||||
if r.get("mastery_level") is not None
|
||||
and r["mastery_level"] < 0.6
|
||||
and (not subject_id or r.get("subject_id") == subject_id)
|
||||
]
|
||||
return {"studentId": student_id, "weakPoints": weak_points}
|
||||
|
||||
|
||||
async def query_student_errors(student_id: str) -> list[dict] | None:
|
||||
"""查询学生错题本（FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
"SELECT student_id, question_id, knowledge_point_id, error_count, "
|
||||
"last_error_time, content "
|
||||
"FROM student_errors FINAL "
|
||||
"WHERE student_id = {sid:String} "
|
||||
"ORDER BY last_error_time DESC "
|
||||
"LIMIT 100",
|
||||
parameters={"sid": student_id},
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_student_errors_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
columns = [
|
||||
"student_id",
|
||||
"question_id",
|
||||
"knowledge_point_id",
|
||||
"error_count",
|
||||
"last_error_time",
|
||||
"content",
|
||||
]
|
||||
return [dict(zip(columns, row, strict=True)) for row in rows]
|
||||
|
||||
|
||||
async def query_learning_trend(
|
||||
student_id: str,
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
subject_id: str = "",
|
||||
) -> dict | None:
|
||||
"""查询学习趋势（按时间排序的成绩曲线，FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["student_id = {sid:String}"]
|
||||
params: dict[str, Any] = {"sid": student_id}
|
||||
if subject_id:
|
||||
where_parts.append("subject_id = {sub:String}")
|
||||
params["sub"] = subject_id
|
||||
if start_date:
|
||||
where_parts.append("last_updated >= {sd:DateTime64(3)}")
|
||||
params["sd"] = datetime.fromtimestamp(start_date)
|
||||
if end_date:
|
||||
where_parts.append("last_updated <= {ed:DateTime64(3)}")
|
||||
params["ed"] = datetime.fromtimestamp(end_date)
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT last_updated, score, exam_id "
|
||||
f"FROM student_dashboard_view FINAL "
|
||||
f"WHERE {where_clause} "
|
||||
f"ORDER BY last_updated ASC "
|
||||
f"LIMIT 100",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_learning_trend_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
points = [
|
||||
{"date": int(row[0].timestamp()) if row[0] else 0, "score": float(row[1] or 0)}
|
||||
for row in rows
|
||||
]
|
||||
return {"studentId": student_id, "points": points}
|
||||
|
||||
|
||||
async def query_mastery_snapshot(student_id: str, subject_id: str = "") -> dict | None:
|
||||
"""查询知识点掌握度快照（argMax 去重保留最新版本）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["student_id = {sid:String}"]
|
||||
params: dict[str, Any] = {"sid": student_id}
|
||||
if subject_id:
|
||||
where_parts.append("subject_id = {sub:String}")
|
||||
params["sub"] = subject_id
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
# argMax 获取每个 knowledge_point_id 的最新 mastery_level
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" knowledge_point_id, "
|
||||
f" argMax(mastery_level, calculated_at) AS mastery_level, "
|
||||
f" argMax(subject_id, calculated_at) AS subject_id, "
|
||||
f" max(calculated_at) AS calculated_at "
|
||||
f"FROM mastery_snapshot "
|
||||
f"WHERE {where_clause} "
|
||||
f"GROUP BY knowledge_point_id "
|
||||
f"ORDER BY mastery_level ASC",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_mastery_snapshot_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
knowledge_points = []
|
||||
total_mastery = 0.0
|
||||
for row in rows:
|
||||
kp_id, mastery, subj, calc_at = row
|
||||
mastery_val = float(mastery or 0)
|
||||
if mastery_val >= 0.8:
|
||||
label = "mastered"
|
||||
elif mastery_val >= 0.4:
|
||||
label = "progressing"
|
||||
else:
|
||||
label = "weak"
|
||||
knowledge_points.append(
|
||||
{
|
||||
"knowledge_point_id": kp_id,
|
||||
"title": kp_id, # content CDC 同步后补充
|
||||
"subject_id": subj or "",
|
||||
"mastery_level": mastery_val,
|
||||
"mastery_label": label,
|
||||
"calculated_at": int(calc_at.timestamp()) if calc_at else 0,
|
||||
}
|
||||
)
|
||||
total_mastery += mastery_val
|
||||
|
||||
overall = total_mastery / len(knowledge_points) if knowledge_points else 0.0
|
||||
return {
|
||||
"studentId": student_id,
|
||||
"knowledgePoints": knowledge_points,
|
||||
"overallMastery": round(overall, 4),
|
||||
}
|
||||
|
||||
|
||||
async def query_mastery_distribution(
|
||||
class_id: str,
|
||||
subject_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
) -> dict | None:
|
||||
"""查询班级掌握度分布（mastered/progressing/weak 三档）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["class_id = {cid:String}"]
|
||||
params: dict[str, Any] = {"cid": class_id}
|
||||
if subject_id:
|
||||
where_parts.append("subject_id = {sid:String}")
|
||||
params["sid"] = subject_id
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" countIf(mastery_level >= 0.8) AS mastered, "
|
||||
f" countIf(mastery_level >= 0.4 AND mastery_level < 0.8) AS progressing, "
|
||||
f" countIf(mastery_level < 0.4) AS weak, "
|
||||
f" count() AS total "
|
||||
f"FROM student_dashboard_view FINAL "
|
||||
f"WHERE {where_clause}",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_mastery_distribution_failed", error=str(exc), class_id=class_id)
|
||||
return None
|
||||
|
||||
if not rows:
|
||||
return {
|
||||
"classId": class_id,
|
||||
"masteredCount": 0,
|
||||
"progressingCount": 0,
|
||||
"weakCount": 0,
|
||||
"totalStudents": 0,
|
||||
}
|
||||
|
||||
mastered, progressing, weak, total = rows[0]
|
||||
return {
|
||||
"classId": class_id,
|
||||
"masteredCount": int(mastered or 0),
|
||||
"progressingCount": int(progressing or 0),
|
||||
"weakCount": int(weak or 0),
|
||||
"totalStudents": int(total or 0),
|
||||
}
|
||||
|
||||
|
||||
async def query_attendance(
|
||||
student_id: str,
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> dict | None:
|
||||
"""查询学生考勤历史（FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["student_id = {sid:String}"]
|
||||
params: dict[str, Any] = {"sid": student_id}
|
||||
if start_date:
|
||||
where_parts.append("attendance_date >= {sd:Date}")
|
||||
params["sd"] = datetime.fromtimestamp(start_date).date()
|
||||
if end_date:
|
||||
where_parts.append("attendance_date <= {ed:Date}")
|
||||
params["ed"] = datetime.fromtimestamp(end_date).date()
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT student_id, class_id, attendance_date, status, "
|
||||
f"recorded_by, remark, occurred_at "
|
||||
f"FROM attendance_logs FINAL "
|
||||
f"WHERE {where_clause} "
|
||||
f"ORDER BY attendance_date DESC "
|
||||
f"LIMIT 100",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_attendance_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
columns = [
|
||||
"student_id",
|
||||
"class_id",
|
||||
"attendance_date",
|
||||
"status",
|
||||
"recorded_by",
|
||||
"remark",
|
||||
"occurred_at",
|
||||
]
|
||||
records = [dict(zip(columns, row, strict=True)) for row in rows]
|
||||
# 统计
|
||||
absent_count = sum(1 for r in records if r["status"] == "absent")
|
||||
late_count = sum(1 for r in records if r["status"] == "late")
|
||||
present_count = sum(1 for r in records if r["status"] == "present")
|
||||
return {
|
||||
"studentId": student_id,
|
||||
"records": records,
|
||||
"total": len(records),
|
||||
"absentCount": absent_count,
|
||||
"lateCount": late_count,
|
||||
"presentCount": present_count,
|
||||
}
|
||||
|
||||
|
||||
async def query_ai_usage_summary() -> dict | None:
|
||||
"""查询 AI 用量统计（按 provider 聚合，FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
"SELECT "
|
||||
" provider, "
|
||||
" count() AS request_count, "
|
||||
" sum(total_tokens) AS total_tokens, "
|
||||
" sum(cost_cents) AS cost_cents "
|
||||
"FROM ai_usage_log FINAL "
|
||||
"GROUP BY provider "
|
||||
"ORDER BY request_count DESC",
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_ai_usage_summary_failed", error=str(exc))
|
||||
return None
|
||||
|
||||
by_provider = [
|
||||
{
|
||||
"provider": row[0],
|
||||
"requestCount": int(row[1] or 0),
|
||||
"totalTokens": int(row[2] or 0),
|
||||
"costCents": int(row[3] or 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
total_requests = sum(p["requestCount"] for p in by_provider)
|
||||
total_tokens = sum(p["totalTokens"] for p in by_provider)
|
||||
total_cost = sum(p["costCents"] for p in by_provider)
|
||||
return {
|
||||
"totalRequests": total_requests,
|
||||
"totalTokens": total_tokens,
|
||||
"totalCostCents": total_cost,
|
||||
"byProvider": by_provider,
|
||||
}
|
||||
|
||||
|
||||
async def query_teacher_dashboard(user_id: str, class_id: str = "") -> dict | None:
|
||||
"""查询教师仪表盘聚合数据."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where = "class_id = {cid:String}" if class_id else "1=1"
|
||||
params: dict[str, Any] = {}
|
||||
if class_id:
|
||||
params["cid"] = class_id
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" count(DISTINCT class_id) AS total_classes, "
|
||||
f" count(DISTINCT student_id) AS total_students, "
|
||||
f" avg(score) AS avg_score "
|
||||
f"FROM student_dashboard_view FINAL "
|
||||
f"WHERE {where}",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_teacher_dashboard_failed", error=str(exc), user_id=user_id)
|
||||
return None
|
||||
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
total_classes, total_students, avg_score = rows[0]
|
||||
return {
|
||||
"userId": user_id,
|
||||
"totalClasses": int(total_classes or 0),
|
||||
"totalStudents": int(total_students or 0),
|
||||
"classAvgScore": float(avg_score or 0),
|
||||
"pendingHomeworkCount": 0, # 从 homework_submissions CDC 计算
|
||||
"classes": [],
|
||||
"topStudents": [],
|
||||
"recentWarnings": [],
|
||||
}
|
||||
|
||||
|
||||
# ===== 写入方法（CDC 消费专用） =====
|
||||
|
||||
|
||||
async def upsert_student_dashboard(
|
||||
student_id: str,
|
||||
class_id: str,
|
||||
exam_id: str,
|
||||
subject_id: str,
|
||||
score: float,
|
||||
rank_in_class: int,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
error_count: int,
|
||||
last_updated: datetime,
|
||||
) -> bool:
|
||||
"""写入/更新学生学情宽表（CDC 消费专用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
client.insert,
|
||||
"student_dashboard_view",
|
||||
[
|
||||
[
|
||||
student_id,
|
||||
class_id,
|
||||
exam_id,
|
||||
subject_id,
|
||||
score,
|
||||
rank_in_class,
|
||||
knowledge_point_id,
|
||||
mastery_level,
|
||||
error_count,
|
||||
last_updated,
|
||||
]
|
||||
],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"class_id",
|
||||
"exam_id",
|
||||
"subject_id",
|
||||
"score",
|
||||
"rank_in_class",
|
||||
"knowledge_point_id",
|
||||
"mastery_level",
|
||||
"error_count",
|
||||
"last_updated",
|
||||
],
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("student_dashboard_upsert_failed", error=str(exc), student_id=student_id)
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_student_error(
|
||||
student_id: str,
|
||||
question_id: str,
|
||||
knowledge_point_id: str,
|
||||
error_count: int,
|
||||
last_error_time: datetime,
|
||||
content: str,
|
||||
) -> bool:
|
||||
"""写入/更新学生错题本（CDC 消费专用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
client.insert,
|
||||
"student_errors",
|
||||
[[student_id, question_id, knowledge_point_id, error_count, last_error_time, content]],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"question_id",
|
||||
"knowledge_point_id",
|
||||
"error_count",
|
||||
"last_error_time",
|
||||
"content",
|
||||
],
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("student_error_upsert_failed", error=str(exc), student_id=student_id)
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_attendance_log(
|
||||
student_id: str,
|
||||
class_id: str,
|
||||
attendance_date,
|
||||
status: str,
|
||||
recorded_by: str,
|
||||
remark: str,
|
||||
occurred_at: datetime,
|
||||
) -> bool:
|
||||
"""写入考勤记录（CDC 消费专用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
client.insert,
|
||||
"attendance_logs",
|
||||
[[student_id, class_id, attendance_date, status, recorded_by, remark, occurred_at]],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"class_id",
|
||||
"attendance_date",
|
||||
"status",
|
||||
"recorded_by",
|
||||
"remark",
|
||||
"occurred_at",
|
||||
],
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("attendance_log_upsert_failed", error=str(exc), student_id=student_id)
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_mastery_snapshot(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
subject_id: str,
|
||||
mastery_level: float,
|
||||
calculated_at: datetime,
|
||||
calculation_method: str = "weighted_moving_avg",
|
||||
) -> bool:
|
||||
"""写入掌握度快照（掌握度计算后调用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
client.insert,
|
||||
"mastery_snapshot",
|
||||
[
|
||||
[
|
||||
student_id,
|
||||
knowledge_point_id,
|
||||
subject_id,
|
||||
mastery_level,
|
||||
calculated_at,
|
||||
calculation_method,
|
||||
]
|
||||
],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"knowledge_point_id",
|
||||
"subject_id",
|
||||
"mastery_level",
|
||||
"calculated_at",
|
||||
"calculation_method",
|
||||
],
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("mastery_snapshot_upsert_failed", error=str(exc), student_id=student_id)
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_ai_usage_log(
|
||||
request_id: str,
|
||||
user_id: str,
|
||||
provider: str,
|
||||
model: str,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
total_tokens: int,
|
||||
latency_ms: int,
|
||||
success: bool,
|
||||
cost_cents: int,
|
||||
occurred_at: datetime,
|
||||
) -> bool:
|
||||
"""写入 AI 用量记录（CDC/事件消费专用）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
client.insert,
|
||||
"ai_usage_log",
|
||||
[
|
||||
[
|
||||
request_id,
|
||||
user_id,
|
||||
provider,
|
||||
model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens,
|
||||
latency_ms,
|
||||
success,
|
||||
cost_cents,
|
||||
occurred_at,
|
||||
]
|
||||
],
|
||||
column_names=[
|
||||
"request_id",
|
||||
"user_id",
|
||||
"provider",
|
||||
"model",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"latency_ms",
|
||||
"success",
|
||||
"cost_cents",
|
||||
"occurred_at",
|
||||
],
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("ai_usage_log_upsert_failed", error=str(exc), request_id=request_id)
|
||||
return False
|
||||
|
||||
|
||||
# ===== 查询历史成绩（掌握度计算用） =====
|
||||
|
||||
|
||||
async def query_student_scores_by_kp(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
limit: int = 5,
|
||||
) -> list[dict] | None:
|
||||
"""查询学生指定知识点的历史成绩（掌握度计算用，FINAL 去重）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
"SELECT score, last_updated, exam_id "
|
||||
"FROM student_dashboard_view FINAL "
|
||||
"WHERE student_id = {sid:String} AND knowledge_point_id = {kp:String} "
|
||||
"ORDER BY last_updated DESC "
|
||||
"LIMIT {lim:UInt32}",
|
||||
parameters={"sid": student_id, "kp": knowledge_point_id, "lim": limit},
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_student_scores_by_kp_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
return [{"score": float(row[0] or 0), "timestamp": row[1], "exam_id": row[2]} for row in rows]
|
||||
226
services/data-ana/src/data_ana/repository/iam_client.py
Normal file
226
services/data-ana/src/data_ana/repository/iam_client.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""iam gRPC 客户端（GetEffectiveDataScope + Redis 缓存 + 降级兜底）.
|
||||
|
||||
对齐 02-architecture-design.md §7.2 DataScope 解析链路：
|
||||
请求 → Redis 缓存命中 → 返回
|
||||
↓ miss
|
||||
iam.GetEffectiveDataScope gRPC → 缓存 5min → 返回
|
||||
↓ 不可达
|
||||
按 role 映射默认 DataScope + degraded: true（降级兜底）
|
||||
|
||||
IAM_GRPC_ENDPOINT 未配置或 gRPC 调用失败时，自动降级到 role-based fallback.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from ..config import settings
|
||||
from ..shared.permissions import (
|
||||
DataScope,
|
||||
DataScopeLevel,
|
||||
UserContext,
|
||||
get_datascope_fallback,
|
||||
)
|
||||
from . import redis_client
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# gRPC 客户端惰性初始化（避免未安装 grpcio 时 import 失败）
|
||||
_grpc_channel: Any | None = None
|
||||
_grpc_stub: Any | None = None
|
||||
_grpc_initialized: bool = False
|
||||
|
||||
|
||||
async def _init_grpc() -> tuple[Any, Any] | None:
|
||||
"""惰性初始化 gRPC channel + stub（iam_grpc_endpoint 为空时返回 None）."""
|
||||
global _grpc_channel, _grpc_stub, _grpc_initialized
|
||||
|
||||
if not settings.iam_grpc_endpoint:
|
||||
return None
|
||||
|
||||
if _grpc_initialized:
|
||||
return (_grpc_channel, _grpc_stub) if _grpc_stub is not None else None
|
||||
|
||||
_grpc_initialized = True
|
||||
try:
|
||||
import grpc # type: ignore[import-not-found]
|
||||
from generated_proto import iam_pb2_grpc # type: ignore[import-not-found]
|
||||
|
||||
_grpc_channel = grpc.aio.insecure_channel(
|
||||
settings.iam_grpc_endpoint,
|
||||
options=[
|
||||
("grpc.connect_timeout_ms", settings.iam_grpc_timeout_s * 1000),
|
||||
("grpc.keepalive_time_ms", 60000),
|
||||
],
|
||||
)
|
||||
_grpc_stub = iam_pb2_grpc.IamServiceStub(_grpc_channel)
|
||||
logger.info(
|
||||
"iam_grpc_initialized",
|
||||
endpoint=settings.iam_grpc_endpoint,
|
||||
)
|
||||
return _grpc_channel, _grpc_stub
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("iam_grpc_init_failed_degraded", error=str(exc))
|
||||
_grpc_stub = None
|
||||
return None
|
||||
|
||||
|
||||
async def close_grpc() -> None:
|
||||
"""关闭 gRPC channel（应用退出时调用）."""
|
||||
global _grpc_channel, _grpc_stub, _grpc_initialized
|
||||
if _grpc_channel is not None:
|
||||
try:
|
||||
await _grpc_channel.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("iam_grpc_close_failed", error=str(exc))
|
||||
finally:
|
||||
_grpc_channel = None
|
||||
_grpc_stub = None
|
||||
_grpc_initialized = False
|
||||
|
||||
|
||||
async def _call_grpc(user_id: str) -> DataScope | None:
|
||||
"""gRPC 调 iam.GetEffectiveDataScope（失败返回 None 触发降级）."""
|
||||
stub_bundle = await _init_grpc()
|
||||
if stub_bundle is None:
|
||||
return None
|
||||
_, stub = stub_bundle
|
||||
try:
|
||||
from generated_proto import iam_pb2 # type: ignore[import-not-found]
|
||||
|
||||
request = iam_pb2.GetEffectiveDataScopeRequest(user_id=user_id)
|
||||
response = await stub.GetEffectiveDataScope( # type: ignore[union-attr]
|
||||
request,
|
||||
timeout=settings.iam_grpc_timeout_s,
|
||||
)
|
||||
# 解析 level 字符串到枚举
|
||||
try:
|
||||
level = DataScopeLevel(response.level)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"iam_datascope_unknown_level_fallback",
|
||||
level=response.level,
|
||||
)
|
||||
return None
|
||||
scope = DataScope(
|
||||
level=level,
|
||||
scope_ids=list(response.scope_ids),
|
||||
school_id=response.school_id,
|
||||
)
|
||||
logger.info(
|
||||
"iam_datascope_resolved",
|
||||
user_id=user_id,
|
||||
level=level.value,
|
||||
scope_count=len(scope.scope_ids),
|
||||
)
|
||||
return scope
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("iam_grpc_call_failed_degraded", user_id=user_id, error=str(exc))
|
||||
return None
|
||||
|
||||
|
||||
def _cache_key(user_id: str) -> str:
|
||||
"""DataScope 缓存键（5min TTL）."""
|
||||
return f"data_ana:datascope:{user_id}"
|
||||
|
||||
|
||||
def _serialize_scope(scope: DataScope) -> str:
|
||||
"""DataScope 序列化（Redis 存储）."""
|
||||
return json.dumps(
|
||||
{
|
||||
"level": scope.level.value,
|
||||
"scope_ids": scope.scope_ids,
|
||||
"school_id": scope.school_id,
|
||||
"degraded": scope.degraded,
|
||||
"degraded_reason": scope.degraded_reason,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def _deserialize_scope(raw: str) -> DataScope | None:
|
||||
"""DataScope 反序列化."""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
return DataScope(
|
||||
level=DataScopeLevel(data["level"]),
|
||||
scope_ids=data.get("scope_ids", []),
|
||||
school_id=data.get("school_id", ""),
|
||||
degraded=data.get("degraded", False),
|
||||
degraded_reason=data.get("degraded_reason", ""),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("datascope_deserialize_failed", error=str(exc), raw=raw[:200])
|
||||
return None
|
||||
|
||||
|
||||
async def get_effective_datascope(user: UserContext) -> DataScope:
|
||||
"""解析用户 DataScope（Redis 缓存 → iam gRPC → 降级兜底）.
|
||||
|
||||
返回值始终非 None：
|
||||
- iam 可用：返回 iam 解析的 DataScope
|
||||
- iam 不可用：返回 role-based fallback + degraded=true
|
||||
"""
|
||||
# 1. Redis 缓存
|
||||
cached = await redis_client.get_cache(_cache_key(user.user_id))
|
||||
if cached is not None:
|
||||
scope = _deserialize_scope(cached)
|
||||
if scope is not None:
|
||||
return scope
|
||||
|
||||
# 2. iam gRPC
|
||||
scope = await _call_grpc(user.user_id)
|
||||
if scope is not None:
|
||||
# 写缓存（5min TTL）
|
||||
await redis_client.set_cache(
|
||||
_cache_key(user.user_id),
|
||||
_serialize_scope(scope),
|
||||
ttl_s=settings.datascope_cache_ttl_s,
|
||||
)
|
||||
return scope
|
||||
|
||||
# 3. 降级兜底
|
||||
fallback = get_datascope_fallback(user)
|
||||
# 降级兜底也写缓存（更短 TTL，避免 iam 恢复后仍读降级值）
|
||||
await redis_client.set_cache(
|
||||
_cache_key(user.user_id),
|
||||
_serialize_scope(fallback),
|
||||
ttl_s=min(60, settings.datascope_cache_ttl_s),
|
||||
)
|
||||
logger.warning(
|
||||
"datascope_fallback_used",
|
||||
user_id=user.user_id,
|
||||
roles=user.roles,
|
||||
level=fallback.level.value,
|
||||
)
|
||||
return fallback
|
||||
|
||||
|
||||
async def invalidate_datascope(user_id: str) -> bool:
|
||||
"""失效 DataScope 缓存（用户角色变更时由 iam 触发）."""
|
||||
return await redis_client.delete_cache(_cache_key(user_id))
|
||||
|
||||
|
||||
async def ping() -> bool:
|
||||
"""iam gRPC 连通性检查（供 /readyz 使用，1s 超时）."""
|
||||
if not settings.iam_grpc_endpoint:
|
||||
return False # 未配置不算健康（依赖降级兜底）
|
||||
stub_bundle = await _init_grpc()
|
||||
if stub_bundle is None:
|
||||
return False
|
||||
_, stub = stub_bundle
|
||||
try:
|
||||
# 用空 user_id 探活（iam 应返回 INVALID_ARGUMENT 而非超时）
|
||||
from generated_proto import iam_pb2 # type: ignore[import-not-found]
|
||||
|
||||
request = iam_pb2.GetEffectiveDataScopeRequest(user_id="")
|
||||
await stub.GetEffectiveDataScope(request, timeout=1) # type: ignore[union-attr]
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# INVALID_ARGUMENT 表示连通，超时/UNAVAILABLE 表示不连通
|
||||
err_str = str(exc).lower()
|
||||
if "invalid" in err_str or "not found" in err_str:
|
||||
return True
|
||||
logger.warning("iam_grpc_ping_failed", error=str(exc))
|
||||
return False
|
||||
202
services/data-ana/src/data_ana/repository/kafka_producer.py
Normal file
202
services/data-ana/src/data_ana/repository/kafka_producer.py
Normal file
@@ -0,0 +1,202 @@
|
||||
"""Kafka producer（MasteryEvent / WarningTriggered 派生数据事件发布）.
|
||||
|
||||
对齐 02-architecture-design.md §12 Outbox 豁免：
|
||||
派生数据事件（掌握度/预警）非业务事实，直接 aiokafka producer 发布，
|
||||
豁免 transactional outbox 模式（总裁裁决 §2.10）.
|
||||
|
||||
事件路由（复用 topic，总裁裁决 §2.11）：
|
||||
- edu.insight.mastery.updated ← MasteryEvent(action=mastery.updated)
|
||||
- edu.insight.mastery.updated ← MasteryEvent(action=warning.triggered)（复用）
|
||||
- edu.insight.ai.usage ← AIUsageEvent（ai 服务发布，本服务消费）
|
||||
|
||||
Producer 配置：
|
||||
- idempotent=true（防重复）
|
||||
- transactional_id=data-ana-producer（事务幂等）
|
||||
- acks=all（强一致性）
|
||||
|
||||
降级策略：
|
||||
- kafka_brokers 未配置或不可达时，发布操作静默失败（记日志），
|
||||
不阻塞主流程（掌握度计算/预警评估继续，下游感知不到事件则视为未更新）.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from ..config import settings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# producer 惰性初始化（避免未安装 aiokafka 时 import 失败）
|
||||
_producer: Any | None = None
|
||||
_producer_initialized: bool = False
|
||||
|
||||
|
||||
async def get_producer() -> Any | None:
|
||||
"""获取 Kafka producer（惰性初始化，失败返回 None 降级）."""
|
||||
global _producer, _producer_initialized
|
||||
|
||||
if not settings.kafka_brokers:
|
||||
return None
|
||||
|
||||
if _producer_initialized:
|
||||
return _producer
|
||||
|
||||
_producer_initialized = True
|
||||
try:
|
||||
from aiokafka import AIOKafkaProducer # type: ignore[import-not-found]
|
||||
|
||||
_producer = AIOKafkaProducer(
|
||||
bootstrap_servers=settings.kafka_brokers,
|
||||
client_id=settings.service_name,
|
||||
transactional_id=settings.kafka_producer_transactional_id,
|
||||
enable_idempotence=True,
|
||||
acks="all",
|
||||
value_serializer=lambda v: json.dumps(v, ensure_ascii=False).encode("utf-8"),
|
||||
key_serializer=lambda k: k.encode("utf-8") if k else None,
|
||||
linger_ms=20, # 批量等待 20ms（平衡延迟与吞吐）
|
||||
retry_backoff_ms=100,
|
||||
request_timeout_ms=10_000,
|
||||
)
|
||||
await _producer.start()
|
||||
logger.info(
|
||||
"kafka_producer_initialized",
|
||||
brokers=settings.kafka_brokers,
|
||||
transactional_id=settings.kafka_producer_transactional_id,
|
||||
)
|
||||
return _producer
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("kafka_producer_init_failed_degraded", error=str(exc))
|
||||
_producer = None
|
||||
return None
|
||||
|
||||
|
||||
async def close_producer() -> None:
|
||||
"""关闭 producer（应用退出时调用）."""
|
||||
global _producer, _producer_initialized
|
||||
if _producer is not None:
|
||||
try:
|
||||
await _producer.stop()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("kafka_producer_close_failed", error=str(exc))
|
||||
finally:
|
||||
_producer = None
|
||||
_producer_initialized = False
|
||||
|
||||
|
||||
async def _publish(
|
||||
topic: str,
|
||||
payload: dict[str, Any],
|
||||
key: str | None = None,
|
||||
) -> bool:
|
||||
"""发布消息（producer 不可用时返回 False 降级）."""
|
||||
producer = await get_producer()
|
||||
if producer is None:
|
||||
return False
|
||||
try:
|
||||
await producer.send_and_wait(topic, value=payload, key=key)
|
||||
logger.info(
|
||||
"kafka_event_published",
|
||||
topic=topic,
|
||||
key=key,
|
||||
event_id=payload.get("event_id"),
|
||||
action=payload.get("action"),
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"kafka_publish_failed_degraded",
|
||||
topic=topic,
|
||||
key=key,
|
||||
error=str(exc),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _gen_event_id() -> str:
|
||||
"""生成事件 ID（消费者幂等去重依据）."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _now_ms() -> int:
|
||||
"""当前时间戳（毫秒）."""
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
# ===== 派生数据事件发布（Outbox 豁免，总裁裁决 §2.10） =====
|
||||
|
||||
|
||||
async def publish_mastery_updated(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
previous_level: float,
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""发布掌握度更新事件（edu.insight.mastery.updated，action=mastery.updated）.
|
||||
|
||||
总裁裁决 §2.10：派生数据事件豁免 Outbox 模式，直接 producer 发布.
|
||||
"""
|
||||
payload = {
|
||||
"event_id": _gen_event_id(),
|
||||
"action": "mastery.updated",
|
||||
"occurred_at": _now_ms(),
|
||||
"student_id": student_id,
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"mastery_level": mastery_level,
|
||||
"previous_level": previous_level,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
# key 用 student_id 保证同学生事件顺序（同一 partition）
|
||||
return await _publish(
|
||||
settings.kafka_mastery_topic,
|
||||
payload,
|
||||
key=student_id,
|
||||
)
|
||||
|
||||
|
||||
async def publish_warning_triggered(
|
||||
target_id: str,
|
||||
warning_type: str,
|
||||
threshold: float,
|
||||
current_value: float,
|
||||
severity: str = "WARN",
|
||||
student_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""发布预警触发事件（复用 edu.insight.mastery.updated，action=warning.triggered）.
|
||||
|
||||
总裁裁决 §2.11：warning.triggered 复用 mastery topic（避免新增 topic）.
|
||||
"""
|
||||
payload = {
|
||||
"event_id": _gen_event_id(),
|
||||
"action": "warning.triggered",
|
||||
"occurred_at": _now_ms(),
|
||||
"student_id": student_id or target_id, # 兼容 student_id 为空的场景
|
||||
"knowledge_point_id": knowledge_point_id,
|
||||
"warning_type": warning_type,
|
||||
"target_id": target_id,
|
||||
"threshold": threshold,
|
||||
"current_value": current_value,
|
||||
"severity": severity,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
# key 用 target_id 保证同目标预警事件顺序
|
||||
return await _publish(
|
||||
settings.kafka_warning_topic, # 复用 mastery topic
|
||||
payload,
|
||||
key=target_id,
|
||||
)
|
||||
|
||||
|
||||
async def ping() -> bool:
|
||||
"""Kafka producer 连通性检查（供 /readyz 使用）.
|
||||
|
||||
aiokafka 无显式 ping API，已初始化即视为可用.
|
||||
"""
|
||||
producer = await get_producer()
|
||||
return producer is not None
|
||||
130
services/data-ana/src/data_ana/repository/redis_client.py
Normal file
130
services/data-ana/src/data_ana/repository/redis_client.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""Redis 客户端（DataScope 缓存 + CDC 幂等去重 + 预警去重位图）.
|
||||
|
||||
降级策略：redis_url 未配置或不可达时，所有缓存查询跳过（cache miss），
|
||||
直查 ClickHouse / iam gRPC，不阻塞主流程.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from ..config import settings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
_client: Any | None = None
|
||||
_client_initialized: bool = False
|
||||
|
||||
|
||||
async def get_client() -> Any | None:
|
||||
"""获取 Redis 客户端（惰性初始化，失败返回 None 降级）."""
|
||||
global _client, _client_initialized
|
||||
|
||||
if not settings.redis_url:
|
||||
return None
|
||||
|
||||
if _client_initialized:
|
||||
return _client
|
||||
|
||||
_client_initialized = True
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
_client = aioredis.from_url(
|
||||
settings.redis_url,
|
||||
max_connections=settings.redis_pool_size,
|
||||
socket_timeout=settings.redis_socket_timeout_ms / 1000,
|
||||
decode_responses=True,
|
||||
)
|
||||
logger.info("redis_client_initialized", url=settings.redis_url)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_client_init_failed_degraded", error=str(exc))
|
||||
_client = None
|
||||
|
||||
return _client
|
||||
|
||||
|
||||
async def close_client() -> None:
|
||||
"""关闭客户端."""
|
||||
global _client, _client_initialized
|
||||
if _client is not None:
|
||||
try:
|
||||
await _client.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_client_close_failed", error=str(exc))
|
||||
finally:
|
||||
_client = None
|
||||
_client_initialized = False
|
||||
|
||||
|
||||
async def ping() -> bool:
|
||||
"""Redis 连通性检查（供 /readyz 使用）."""
|
||||
client = await get_client()
|
||||
if client is None:
|
||||
return False
|
||||
try:
|
||||
await client.ping()
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_ping_failed", error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
async def get_cache(key: str) -> str | None:
|
||||
"""读取缓存（不可用时返回 None）."""
|
||||
client = await get_client()
|
||||
if client is None:
|
||||
return None
|
||||
try:
|
||||
return await client.get(key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_get_failed", key=key, error=str(exc))
|
||||
return None
|
||||
|
||||
|
||||
async def set_cache(key: str, value: str, ttl_s: int = 300) -> bool:
|
||||
"""写入缓存（带 TTL）."""
|
||||
client = await get_client()
|
||||
if client is None:
|
||||
return False
|
||||
try:
|
||||
await client.setex(key, ttl_s, value)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_set_failed", key=key, error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
async def delete_cache(key: str) -> bool:
|
||||
"""删除缓存."""
|
||||
client = await get_client()
|
||||
if client is None:
|
||||
return False
|
||||
try:
|
||||
await client.delete(key)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_delete_failed", key=key, error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
async def setnx_dedup(key: str, ttl_s: int = 86400) -> bool:
|
||||
"""幂等去重（SETNX，成功表示首次，失败表示重复）."""
|
||||
client = await get_client()
|
||||
if client is None:
|
||||
return True # Redis 不可用时放行（依赖 ClickHouse 去重）
|
||||
try:
|
||||
result = await client.set(key, "1", ex=ttl_s, nx=True)
|
||||
return bool(result)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("redis_setnx_failed", key=key, error=str(exc))
|
||||
return True # 放行
|
||||
|
||||
|
||||
# ===== 缓存键命名规范（对齐 02-architecture-design.md §10.2） =====
|
||||
# data_ana:datascope:{user_id} 5min DataScope 缓存
|
||||
# data_ana:exam:{exam_id} 30day ExamCache（P6 多实例共享）
|
||||
# data_ana:dedup:{event_id} 7day 事件幂等去重
|
||||
# data_ana:warning:{target_id}:{type}:{date} 25h 预警去重位图
|
||||
# data_ana:dashboard:{user_id}:{scope} 5min Dashboard 聚合结果缓存
|
||||
# data_ana:kp_meta:{knowledge_point_id} 1day 知识点元数据缓存
|
||||
1
services/data-ana/src/data_ana/shared/__init__.py
Normal file
1
services/data-ana/src/data_ana/shared/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Shared package for data-ana service."""
|
||||
62
services/data-ana/src/data_ana/shared/action_state.py
Normal file
62
services/data-ana/src/data_ana/shared/action_state.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""统一响应信封 ActionState（coord-cross-review.md §5.3 P0 整改）.
|
||||
|
||||
总裁裁决 §2.6 降级模式：success=true + error=null + data 内 degraded 字段.
|
||||
|
||||
成功：{success: true, data: T}
|
||||
失败：{success: false, error: {code, message, details?, trace_id?}}
|
||||
降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}}.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ActionStateError(BaseModel):
|
||||
"""错误信息（失败时填充）."""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
details: dict[str, Any] | None = None
|
||||
trace_id: str | None = None
|
||||
|
||||
|
||||
class ActionState[T](BaseModel):
|
||||
"""统一响应信封（coord-cross-review.md §5.3 裁决）.
|
||||
|
||||
成功：{success: true, data: T}
|
||||
失败：{success: false, error: {code, message, details?, trace_id?}}
|
||||
降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}}
|
||||
（保留功能但数据可能不完整，降级不是错误）
|
||||
"""
|
||||
|
||||
success: bool
|
||||
data: T | None = None
|
||||
error: ActionStateError | None = None
|
||||
details: dict[str, Any] | None = None # 降级标记放此字段，不放 error
|
||||
|
||||
@classmethod
|
||||
def ok(cls, data: T, *, degraded: bool = False, degraded_reason: str = "") -> "ActionState[T]":
|
||||
"""成功响应（降级时 degraded=True）."""
|
||||
if degraded:
|
||||
details: dict[str, Any] = {"degraded": True}
|
||||
if degraded_reason:
|
||||
details["degraded_reason"] = degraded_reason
|
||||
return cls(success=True, data=data, error=None, details=details)
|
||||
return cls(success=True, data=data, error=None, details=None)
|
||||
|
||||
@classmethod
|
||||
def fail(
|
||||
cls,
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
trace_id: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> "ActionState[T]":
|
||||
"""失败响应."""
|
||||
return cls(
|
||||
success=False,
|
||||
data=None,
|
||||
error=ActionStateError(code=code, message=message, details=details, trace_id=trace_id),
|
||||
)
|
||||
66
services/data-ana/src/data_ana/shared/errors.py
Normal file
66
services/data-ana/src/data_ana/shared/errors.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""错误码定义（前缀 DATA_ANA_*，对齐 02-architecture-design.md §6.2）.
|
||||
|
||||
错误码语义：
|
||||
- 4xx 客户端错误（权限/参数/资源不存在）
|
||||
- 5xx 服务端错误（内部异常）
|
||||
- 200 + degraded:true 降级模式（外部依赖不可用但服务仍可响应骨架数据）
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ErrorCode(StrEnum):
|
||||
"""data-ana 错误码枚举."""
|
||||
|
||||
# 客户端错误
|
||||
UNAUTHORIZED = "DATA_ANA_UNAUTHORIZED" # 401 缺失 x-user-id 或 token 无效
|
||||
FORBIDDEN = "DATA_ANA_FORBIDDEN" # 403 角色无对应权限
|
||||
DATASCOPE_VIOLATION = "DATA_ANA_DATASCOPE_VIOLATION" # 403 查询目标超出 DataScope
|
||||
INVALID_DATE_RANGE = "DATA_ANA_INVALID_DATE_RANGE" # 400 start_date > end_date
|
||||
STUDENT_NOT_FOUND = "DATA_ANA_STUDENT_NOT_FOUND" # 404 student_id 不存在
|
||||
CLASS_NOT_FOUND = "DATA_ANA_CLASS_NOT_FOUND" # 404 class_id 不存在
|
||||
WARNING_NOT_FOUND = "DATA_ANA_WARNING_NOT_FOUND" # 404 预警 ID 不存在
|
||||
RATE_LIMITED = "DATA_ANA_RATE_LIMITED" # 429 触发限流
|
||||
|
||||
# 降级模式（200 + degraded:true）
|
||||
CLICKHOUSE_UNAVAILABLE = "DATA_ANA_CLICKHOUSE_UNAVAILABLE" # ClickHouse 不可达
|
||||
KAFKA_UNAVAILABLE = "DATA_ANA_KAFKA_UNAVAILABLE" # Kafka producer 不可达
|
||||
REDIS_UNAVAILABLE = "DATA_ANA_REDIS_UNAVAILABLE" # Redis 不可达
|
||||
IAM_GRPC_UNAVAILABLE = "DATA_ANA_IAM_GRPC_UNAVAILABLE" # iam gRPC 不可达
|
||||
|
||||
# 服务端错误
|
||||
INTERNAL_ERROR = "DATA_ANA_INTERNAL_ERROR" # 500 未捕获异常
|
||||
|
||||
|
||||
# HTTP 状态码映射
|
||||
HTTP_STATUS: dict[ErrorCode, int] = {
|
||||
ErrorCode.UNAUTHORIZED: 401,
|
||||
ErrorCode.FORBIDDEN: 403,
|
||||
ErrorCode.DATASCOPE_VIOLATION: 403,
|
||||
ErrorCode.INVALID_DATE_RANGE: 400,
|
||||
ErrorCode.STUDENT_NOT_FOUND: 404,
|
||||
ErrorCode.CLASS_NOT_FOUND: 404,
|
||||
ErrorCode.WARNING_NOT_FOUND: 404,
|
||||
ErrorCode.RATE_LIMITED: 429,
|
||||
ErrorCode.CLICKHOUSE_UNAVAILABLE: 200, # 降级模式
|
||||
ErrorCode.KAFKA_UNAVAILABLE: 200, # 降级模式
|
||||
ErrorCode.REDIS_UNAVAILABLE: 200, # 降级模式
|
||||
ErrorCode.IAM_GRPC_UNAVAILABLE: 200, # 降级模式
|
||||
ErrorCode.INTERNAL_ERROR: 500,
|
||||
}
|
||||
|
||||
|
||||
class DataAnaError(Exception):
|
||||
"""data-ana 业务异常（携带错误码）."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
code: ErrorCode,
|
||||
message: str,
|
||||
*,
|
||||
details: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details
|
||||
124
services/data-ana/src/data_ana/shared/permissions.py
Normal file
124
services/data-ana/src/data_ana/shared/permissions.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""权限点常量与 FastAPI Depends 权限校验（对齐 02-architecture-design.md §6.1）.
|
||||
|
||||
Python 服务无 NestJS 装饰器，权限校验用 FastAPI Depends 等价物
|
||||
（coord-cross-review.md §5.3 已仲裁认可）.
|
||||
|
||||
DataScope 6 级：SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
|
||||
iam.GetEffectiveDataScope gRPC 解析用户可见数据范围（P4 补全，未就绪时降级兜底）.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
|
||||
from fastapi import Header
|
||||
|
||||
|
||||
class Permission:
|
||||
"""权限点常量（与 iam 权限点对齐）."""
|
||||
|
||||
ANALYTICS_CLASS_READ = "analytics:class:read"
|
||||
ANALYTICS_STUDENT_READ = "analytics:student:read"
|
||||
ANALYTICS_TEACHER_DASHBOARD = "analytics:teacher:dashboard"
|
||||
ANALYTICS_STUDENT_DASHBOARD = "analytics:student:dashboard"
|
||||
ANALYTICS_PARENT_DASHBOARD = "analytics:parent:dashboard"
|
||||
ANALYTICS_ADMIN_DASHBOARD = "analytics:admin:dashboard"
|
||||
ANALYTICS_WARNING_READ = "analytics:warning:read"
|
||||
|
||||
|
||||
class DataScopeLevel(StrEnum):
|
||||
"""DataScope 6 级."""
|
||||
|
||||
SELF = "SELF"
|
||||
CLASS = "CLASS"
|
||||
GRADE = "GRADE"
|
||||
SCHOOL = "SCHOOL"
|
||||
DISTRICT = "DISTRICT"
|
||||
ALL = "ALL"
|
||||
|
||||
|
||||
# 角色 → 默认 DataScope 降级映射（iam gRPC 不可达时使用）
|
||||
_ROLE_DATASCOPE_FALLBACK: dict[str, DataScopeLevel] = {
|
||||
"admin": DataScopeLevel.ALL,
|
||||
"school_admin": DataScopeLevel.SCHOOL,
|
||||
"teacher": DataScopeLevel.CLASS,
|
||||
"student": DataScopeLevel.SELF,
|
||||
"parent": DataScopeLevel.SELF,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserContext:
|
||||
"""用户上下文（从请求头提取）."""
|
||||
|
||||
user_id: str
|
||||
roles: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataScope:
|
||||
"""数据范围（查询层注入 WHERE）."""
|
||||
|
||||
level: DataScopeLevel
|
||||
scope_ids: list[str] = field(default_factory=list) # class_id 列表 / grade_id 列表
|
||||
school_id: str = ""
|
||||
degraded: bool = False # iam 不可达时降级标记
|
||||
degraded_reason: str = ""
|
||||
|
||||
|
||||
async def get_user_context(
|
||||
x_user_id: str = Header("", alias="x-user-id"),
|
||||
x_user_roles: str = Header("", alias="x-user-roles"),
|
||||
) -> UserContext:
|
||||
"""从请求头提取用户上下文（FastAPI Depends）.
|
||||
|
||||
Gateway 注入 x-user-id / x-user-roles 头（JWT 校验后）.
|
||||
"""
|
||||
roles = [r.strip() for r in x_user_roles.split(",") if r.strip()] if x_user_roles else []
|
||||
return UserContext(user_id=x_user_id, roles=roles)
|
||||
|
||||
|
||||
def get_datascope_fallback(user: UserContext) -> DataScope:
|
||||
"""按角色映射默认 DataScope（降级兜底，iam gRPC 不可达时使用）."""
|
||||
for role in user.roles:
|
||||
if role in _ROLE_DATASCOPE_FALLBACK:
|
||||
level = _ROLE_DATASCOPE_FALLBACK[role]
|
||||
return DataScope(
|
||||
level=level,
|
||||
degraded=True,
|
||||
degraded_reason="iam_grpc_unavailable_fallback",
|
||||
)
|
||||
# 默认最保守范围
|
||||
return DataScope(
|
||||
level=DataScopeLevel.SELF,
|
||||
degraded=True,
|
||||
degraded_reason="unknown_role_fallback_self",
|
||||
)
|
||||
|
||||
|
||||
def build_datascope_where(
|
||||
scope: DataScope,
|
||||
student_col: str = "student_id",
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
"""构建 DataScope WHERE 子句（注入到 ClickHouse 查询）.
|
||||
|
||||
返回 (where_clause, parameters).
|
||||
"""
|
||||
params: dict[str, str] = {}
|
||||
if scope.level == DataScopeLevel.ALL:
|
||||
return "", params
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
# SELF 范围需要 user_id，由调用方注入
|
||||
return "", params
|
||||
if scope.level == DataScopeLevel.CLASS and scope.scope_ids:
|
||||
# 班级范围：class_id IN (...)
|
||||
placeholders = []
|
||||
for i, cid in enumerate(scope.scope_ids):
|
||||
key = f"class_id_{i}"
|
||||
params[key] = cid
|
||||
placeholders.append(f"{{{key}:String}}")
|
||||
return f"class_id IN ({', '.join(placeholders)})", params
|
||||
if scope.level == DataScopeLevel.SCHOOL:
|
||||
# 学校范围：不做过滤（单租户场景下等价 ALL）
|
||||
return "", params
|
||||
# GRADE / DISTRICT 暂按 ALL 处理（P6 细化）
|
||||
return "", params
|
||||
383
services/data-ana/src/data_ana/warning_service.py
Normal file
383
services/data-ana/src/data_ana/warning_service.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""预警服务（5 类预警阈值评估 + 去重位图 + 事件发布）.
|
||||
|
||||
对齐 02-architecture-design.md §11 预警体系：
|
||||
- LOW_MASTERY：掌握度 < 0.4
|
||||
- CRITICAL_LOW：掌握度 < 0.2
|
||||
- SCORE_DROP：成绩环比下降 ≥ 20%
|
||||
- ABSENT_FREQUENT：缺勤 ≥ 3 次/周
|
||||
- TREND_DECLINE：连续 3 次成绩下降
|
||||
|
||||
去重策略：
|
||||
Redis bitmap key = data_ana:warning:{target_id}:{type}:{date}
|
||||
TTL = 25h（跨天容忍，避免边界重复触发）
|
||||
|
||||
事件发布：
|
||||
warning.triggered 复用 edu.insight.mastery.updated topic（总裁裁决 §2.11）
|
||||
Outbox 豁免（总裁裁决 §2.10）
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .config import settings
|
||||
from .repository import clickhouse_repository, kafka_producer, redis_client
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
# 预警类型常量
|
||||
class WarningType:
|
||||
"""预警类型."""
|
||||
|
||||
LOW_MASTERY = "LOW_MASTERY"
|
||||
CRITICAL_LOW = "CRITICAL_LOW"
|
||||
SCORE_DROP = "SCORE_DROP"
|
||||
ABSENT_FREQUENT = "ABSENT_FREQUENT"
|
||||
TREND_DECLINE = "TREND_DECLINE"
|
||||
|
||||
|
||||
class Severity:
|
||||
"""预警严重程度."""
|
||||
|
||||
INFO = "INFO"
|
||||
WARN = "WARN"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
# 预警类型 → 严重程度映射
|
||||
_SEVERITY_MAP: dict[str, str] = {
|
||||
WarningType.LOW_MASTERY: Severity.WARN,
|
||||
WarningType.CRITICAL_LOW: Severity.CRITICAL,
|
||||
WarningType.SCORE_DROP: Severity.WARN,
|
||||
WarningType.ABSENT_FREQUENT: Severity.WARN,
|
||||
WarningType.TREND_DECLINE: Severity.WARN,
|
||||
}
|
||||
|
||||
|
||||
def _warning_dedup_key(target_id: str, warning_type: str) -> str:
|
||||
"""预警去重位图 key（25h TTL，避免边界重复触发）."""
|
||||
today = datetime.now(UTC).strftime("%Y%m%d")
|
||||
return f"data_ana:warning:{target_id}:{warning_type}:{today}"
|
||||
|
||||
|
||||
async def _check_and_dedup(target_id: str, warning_type: str) -> bool:
|
||||
"""检查去重（首次返回 True，已触发过返回 False）."""
|
||||
key = _warning_dedup_key(target_id, warning_type)
|
||||
is_first = await redis_client.setnx_dedup(key, ttl_s=25 * 3600)
|
||||
if not is_first:
|
||||
logger.info(
|
||||
"warning_dedup_skipped",
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
)
|
||||
return is_first
|
||||
|
||||
|
||||
async def _trigger_warning(
|
||||
target_id: str,
|
||||
warning_type: str,
|
||||
threshold: float,
|
||||
current_value: float,
|
||||
student_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""触发预警（去重 + 事件发布）."""
|
||||
severity = _SEVERITY_MAP.get(warning_type, Severity.WARN)
|
||||
|
||||
# 1. 去重检查
|
||||
should_trigger = await _check_and_dedup(target_id, warning_type)
|
||||
if not should_trigger:
|
||||
return {
|
||||
"target_id": target_id,
|
||||
"warning_type": warning_type,
|
||||
"triggered": False,
|
||||
"deduplicated": True,
|
||||
"threshold": threshold,
|
||||
"current_value": current_value,
|
||||
}
|
||||
|
||||
# 2. 发布 warning.triggered 事件（Outbox 豁免，复用 mastery topic）
|
||||
published = await kafka_producer.publish_warning_triggered(
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
severity=severity,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
"warning_triggered",
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
severity=severity,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
published=published,
|
||||
)
|
||||
return {
|
||||
"target_id": target_id,
|
||||
"warning_type": warning_type,
|
||||
"severity": severity,
|
||||
"triggered": True,
|
||||
"deduplicated": False,
|
||||
"threshold": threshold,
|
||||
"current_value": current_value,
|
||||
"event_published": published,
|
||||
}
|
||||
|
||||
|
||||
async def evaluate_mastery_warning(
|
||||
student_id: str,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""评估掌握度预警（LOW_MASTERY / CRITICAL_LOW）.
|
||||
|
||||
- mastery < 0.2 → CRITICAL_LOW（最严重，先评估）
|
||||
- 0.2 ≤ mastery < 0.4 → LOW_MASTERY
|
||||
"""
|
||||
triggered: list[dict[str, Any]] = []
|
||||
|
||||
if mastery_level < settings.warning_critical_mastery_threshold:
|
||||
result = await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.CRITICAL_LOW,
|
||||
threshold=settings.warning_critical_mastery_threshold,
|
||||
current_value=mastery_level,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
if result["triggered"]:
|
||||
triggered.append(result)
|
||||
return triggered # CRITICAL_LOW 已触发，不再触发 LOW_MASTERY
|
||||
|
||||
if mastery_level < settings.warning_low_mastery_threshold:
|
||||
result = await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.LOW_MASTERY,
|
||||
threshold=settings.warning_low_mastery_threshold,
|
||||
current_value=mastery_level,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
)
|
||||
if result["triggered"]:
|
||||
triggered.append(result)
|
||||
|
||||
return triggered
|
||||
|
||||
|
||||
async def evaluate_score_drop(
|
||||
student_id: str,
|
||||
current_score: float,
|
||||
previous_score: float,
|
||||
exam_id: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估成绩下降预警（SCORE_DROP）.
|
||||
|
||||
成绩环比下降 ≥ warning_score_drop_percent（默认 20%）触发.
|
||||
"""
|
||||
if previous_score <= 0:
|
||||
return None
|
||||
|
||||
drop_ratio = (previous_score - current_score) / previous_score
|
||||
if drop_ratio < settings.warning_score_drop_percent:
|
||||
return None
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.SCORE_DROP,
|
||||
threshold=settings.warning_score_drop_percent,
|
||||
current_value=round(drop_ratio, 4),
|
||||
student_id=student_id,
|
||||
metadata={
|
||||
"exam_id": exam_id,
|
||||
"previous_score": str(previous_score),
|
||||
"current_score": str(current_score),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_absent_frequent(
|
||||
student_id: str,
|
||||
absent_count_week: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估缺勤频繁预警（ABSENT_FREQUENT）.
|
||||
|
||||
一周内缺勤 ≥ warning_absent_per_week（默认 3 次）触发.
|
||||
"""
|
||||
if absent_count_week < settings.warning_absent_per_week:
|
||||
return None
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.ABSENT_FREQUENT,
|
||||
threshold=float(settings.warning_absent_per_week),
|
||||
current_value=float(absent_count_week),
|
||||
student_id=student_id,
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_trend_decline(
|
||||
student_id: str,
|
||||
recent_scores: list[float],
|
||||
) -> dict[str, Any] | None:
|
||||
"""评估趋势下降预警（TREND_DECLINE）.
|
||||
|
||||
连续 3 次成绩下降触发（recent_scores 按时间正序，最早在前）.
|
||||
"""
|
||||
if len(recent_scores) < 3:
|
||||
return None
|
||||
|
||||
# 检查最近 3 次是否连续下降
|
||||
last_three = recent_scores[-3:]
|
||||
is_declining = all(last_three[i] > last_three[i + 1] for i in range(len(last_three) - 1))
|
||||
if not is_declining:
|
||||
return None
|
||||
|
||||
drop = last_three[-2] - last_three[-1]
|
||||
return await _trigger_warning(
|
||||
target_id=student_id,
|
||||
warning_type=WarningType.TREND_DECLINE,
|
||||
threshold=0.0, # 下降幅度阈值（0 表示只要下降就触发）
|
||||
current_value=round(drop, 4),
|
||||
student_id=student_id,
|
||||
)
|
||||
|
||||
|
||||
async def evaluate_all_warnings(
|
||||
student_id: str,
|
||||
knowledge_point_id: str = "",
|
||||
mastery_level: float | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""综合评估学生所有预警类型（GetWarnings RPC 调用）.
|
||||
|
||||
参数：
|
||||
- student_id：学生 ID
|
||||
- knowledge_point_id：知识点 ID（可选，指定则只评估该知识点）
|
||||
- mastery_level：预计算的掌握度（可选，未提供则查询）
|
||||
|
||||
返回：触发的预警列表
|
||||
"""
|
||||
triggered: list[dict[str, Any]] = []
|
||||
|
||||
# 1. 掌握度预警
|
||||
if mastery_level is None:
|
||||
snapshot = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=student_id,
|
||||
)
|
||||
if snapshot is not None:
|
||||
knowledge_points = snapshot.get("knowledgePoints", [])
|
||||
if knowledge_point_id:
|
||||
# 仅评估指定知识点
|
||||
knowledge_points = [
|
||||
kp
|
||||
for kp in knowledge_points
|
||||
if kp.get("knowledge_point_id") == knowledge_point_id
|
||||
]
|
||||
for kp in knowledge_points:
|
||||
level = kp.get("mastery_level", 0.0)
|
||||
warnings = await evaluate_mastery_warning(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=kp.get("knowledge_point_id", ""),
|
||||
mastery_level=level,
|
||||
)
|
||||
triggered.extend(warnings)
|
||||
else:
|
||||
warnings = await evaluate_mastery_warning(
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
mastery_level=mastery_level,
|
||||
)
|
||||
triggered.extend(warnings)
|
||||
|
||||
# 2. 缺勤预警
|
||||
attendance = await clickhouse_repository.query_attendance(student_id=student_id)
|
||||
if attendance is not None:
|
||||
# 简化：统计总缺勤数（P6 改为按周统计）
|
||||
absent_count = attendance.get("absentCount", 0)
|
||||
if absent_count >= settings.warning_absent_per_week:
|
||||
warning = await evaluate_absent_frequent(
|
||||
student_id=student_id,
|
||||
absent_count_week=absent_count,
|
||||
)
|
||||
if warning is not None and warning.get("triggered"):
|
||||
triggered.append(warning)
|
||||
|
||||
# 3. 成绩下降预警（需要查询最近两次成绩）
|
||||
trend = await clickhouse_repository.query_learning_trend(student_id=student_id)
|
||||
if trend is not None:
|
||||
points = trend.get("points", [])
|
||||
if len(points) >= 2:
|
||||
current = points[-1].get("score", 0.0)
|
||||
previous = points[-2].get("score", 0.0)
|
||||
drop_warning = await evaluate_score_drop(
|
||||
student_id=student_id,
|
||||
current_score=current,
|
||||
previous_score=previous,
|
||||
)
|
||||
if drop_warning is not None and drop_warning.get("triggered"):
|
||||
triggered.append(drop_warning)
|
||||
|
||||
# 4. 趋势下降预警
|
||||
scores = [p.get("score", 0.0) for p in points]
|
||||
trend_warning = await evaluate_trend_decline(
|
||||
student_id=student_id,
|
||||
recent_scores=scores,
|
||||
)
|
||||
if trend_warning is not None and trend_warning.get("triggered"):
|
||||
triggered.append(trend_warning)
|
||||
|
||||
return triggered
|
||||
|
||||
|
||||
async def get_warnings(
|
||||
student_id: str,
|
||||
warning_type: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""查询预警列表（GetWarnings RPC 实现）.
|
||||
|
||||
参数：
|
||||
- student_id：学生 ID
|
||||
- warning_type：预警类型过滤（空表示全部）
|
||||
|
||||
返回：预警列表（每条含 target_id / warning_type / severity / threshold /
|
||||
current_value / triggered_at）
|
||||
"""
|
||||
# 当前实现：实时评估返回触发结果
|
||||
# P6 演进：将预警历史持久化到 ClickHouse 供查询
|
||||
triggered = await evaluate_all_warnings(student_id=student_id)
|
||||
if warning_type:
|
||||
triggered = [w for w in triggered if w.get("warning_type") == warning_type]
|
||||
return triggered
|
||||
|
||||
|
||||
async def trigger_warning_manual(
|
||||
target_id: str,
|
||||
warning_type: str,
|
||||
threshold: float,
|
||||
current_value: float,
|
||||
student_id: str = "",
|
||||
knowledge_point_id: str = "",
|
||||
severity: str = "",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""手动触发预警（TriggerWarning RPC 实现，管理员/API 用）."""
|
||||
# 覆盖默认 severity
|
||||
if severity:
|
||||
_SEVERITY_MAP[warning_type] = severity
|
||||
|
||||
return await _trigger_warning(
|
||||
target_id=target_id,
|
||||
warning_type=warning_type,
|
||||
threshold=threshold,
|
||||
current_value=current_value,
|
||||
student_id=student_id,
|
||||
knowledge_point_id=knowledge_point_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
32
uv.lock
generated
32
uv.lock
generated
@@ -270,19 +270,23 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "data-ana-service"
|
||||
version = "0.1.0"
|
||||
version = "1.0.0"
|
||||
source = { virtual = "services/data-ana" }
|
||||
dependencies = [
|
||||
{ name = "aiokafka" },
|
||||
{ name = "clickhouse-connect" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "grpcio" },
|
||||
{ name = "grpcio-health-checking" },
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-exporter-otlp" },
|
||||
{ name = "opentelemetry-instrumentation-fastapi" },
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "prometheus-client" },
|
||||
{ name = "protobuf" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "redis" },
|
||||
{ name = "structlog" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
@@ -292,13 +296,17 @@ requires-dist = [
|
||||
{ name = "aiokafka", specifier = ">=0.11.0" },
|
||||
{ name = "clickhouse-connect", specifier = ">=0.7.0" },
|
||||
{ name = "fastapi", specifier = ">=0.115.0" },
|
||||
{ name = "grpcio", specifier = ">=1.66.0" },
|
||||
{ name = "grpcio-health-checking", specifier = ">=1.66.0" },
|
||||
{ name = "opentelemetry-api", specifier = ">=1.27.0" },
|
||||
{ name = "opentelemetry-exporter-otlp", specifier = ">=1.27.0" },
|
||||
{ name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" },
|
||||
{ name = "opentelemetry-sdk", specifier = ">=1.27.0" },
|
||||
{ name = "prometheus-client", specifier = ">=0.20.0" },
|
||||
{ name = "protobuf", specifier = ">=5.28.0" },
|
||||
{ name = "pydantic", specifier = ">=2.9.0" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.5.0" },
|
||||
{ name = "redis", specifier = ">=5.0.0" },
|
||||
{ name = "structlog", specifier = ">=24.4.0" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
|
||||
]
|
||||
@@ -372,6 +380,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "grpcio-health-checking"
|
||||
version = "1.82.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "grpcio" },
|
||||
{ name = "protobuf" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c6/64/060c857a962dae39cac69433f73145acd825b5348fad53908f3439a6fca8/grpcio_health_checking-1.82.1.tar.gz", hash = "sha256:86255e04e1d39f1c97a6632d41b63249351408de5c58e85eede87afd7d9828dd", size = 17130, upload-time = "2026-07-08T12:39:41.138Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/aa/da280870eca03223fb1c15e5c5482ebd42a523a227e496d9b490e8fdaea5/grpcio_health_checking-1.82.1-py3-none-any.whl", hash = "sha256:622ed6663daf0b8c9dedb4e95a48f6db080a8129922742b5141c63cfab373219", size = 19121, upload-time = "2026-07-08T12:39:25.629Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.16.0"
|
||||
@@ -854,6 +875,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "8.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.34.2"
|
||||
|
||||
Reference in New Issue
Block a user