xiner/Edu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-ana): 完整实现 data-ana 数据分析服务

Browse Source 
包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等
This commit is contained in:
SpecialX
2026-07-10 19:09:27 +08:00
parent 033057a302
commit ca3780aa24
29 changed files with 5401 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/troubleshooting/known-issues.md
View File

@@ -312,7 +312,7 @@
| CDC 水平扩展 | P4 单实例 + 内存 ExamCache LRU → P6 多实例 + Redis ExamCache + partition 扩容 + HPA | | CDC 水平扩展 | P4 单实例 + 内存 ExamCache LRU → P6 多实例 + Redis ExamCache + partition 扩容 + HPA |
| CDC 手动 commit | v2 改进：`enable_auto_commit=False` 手动 commit（at-least-once），commit 前确保 ClickHouse 写入成功 | | CDC 手动 commit | v2 改进：`enable_auto_commit=False` 手动 commit（at-least-once），commit 前确保 ClickHouse 写入成功 |
| ClickHouse 容量规划 | 5 宽表年增量 ~2.3GB（5000 学生×20 次/月），单节点可承载；P6 按 class_id hash 分片 + ReplicatedMergeTree | | ClickHouse 容量规划 | 5 宽表年增量 ~2.3GB（5000 学生×20 次/月），单节点可承载；P6 按 class_id hash 分片 + ReplicatedMergeTree |
| ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（当前实现未加，P0 整改） | | ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（clickhouse_repository.py 已实现 P0 整改） |
| 缓存键命名 | `data_ana:datascope:{user_id}` 5min / `data_ana:exam:{exam_id}` 30day / `data_ana:dedup:{event_id}` 7day | | 缓存键命名 | `data_ana:datascope:{user_id}` 5min / `data_ana:exam:{exam_id}` 30day / `data_ana:dedup:{event_id}` 7day |
| 4 端 Dashboard | teacher/student/parent/admin 各自 Dashboard API + 权限点 ANALYTICS_*_DASHBOARD（v2 新增 4 端权限） | | 4 端 Dashboard | teacher/student/parent/admin 各自 Dashboard API + 权限点 ANALYTICS_*_DASHBOARD（v2 新增 4 端权限） |
| SubscribeMasteryUpdate | gRPC server-streaming RPC 为 P5+ AI 个性化推荐预留实时推送通道（analytics.proto v2 提案） | | SubscribeMasteryUpdate | gRPC server-streaming RPC 为 P5+ AI 个性化推荐预留实时推送通道（analytics.proto v2 提案） |
@@ -327,6 +327,9 @@
| 降级模式 4 场景 | ClickHouse/Kafka/Redis/iam gRPC 不可用时返回骨架 + `error.details.degraded=true`，4 错误码 DATA_ANA_*_UNAVAILABLE | | 降级模式 4 场景 | ClickHouse/Kafka/Redis/iam gRPC 不可用时返回骨架 + `error.details.degraded=true`，4 错误码 DATA_ANA_*_UNAVAILABLE |
| Python gRPC 实现 | grpc.aio + betterproto + ServerInterceptor 透传 W3C trace context，与 HTTP 共享同一 Application Service | | Python gRPC 实现 | grpc.aio + betterproto + ServerInterceptor 透传 W3C trace context，与 HTTP 共享同一 Application Service |
| structlog API | 24.x 用 `make_filtering_bound_logger(level)`，旧版 `make_filtering_logger` 已废弃 | | structlog API | 24.x 用 `make_filtering_bound_logger(level)`，旧版 `make_filtering_logger` 已废弃 |
| ruff per-file-ignores | gRPC servicer 方法必须 PascalCase（N802 忽略）/ FastAPI Depends 在参数默认值（B008 忽略），pyproject.toml 配 per-file-ignores |
| Mock 数据脚本 | `scripts/data-ana-mock-data.sql` 提供 5 宽表演示数据（20 学生×2 考试×3 知识点），用 `now()-INTERVAL` 避免硬编码日期 |
| Python 3.12+ 现代化 | ruff UP042 用 StrEnum 替代 `str, Enum` / UP046 用 PEP 695 类型参数 `class Foo[T]` 替代 `Generic[T]` |
### 2.7 messaging（TS/NestJS，P5） ### 2.7 messaging（TS/NestJS，P5）

97
infra/clickhouse/ddl/data_ana.sql Normal file
View File

@@ -0,0 +1,97 @@
-- data-ana ClickHouse DDL（5 宽表）.
-- 负责人：ai11（总裁裁决 §2.12 授权 ai11 创建纯 DDL 文件，SRE AI 负责部署脚本）.
-- 对齐文档：services/data-ana/docs/02-architecture-design.md §3.
-- 引擎：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本（幂等消费保证）.
-- 查询时必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改项）.
--
-- 使用方式：
-- clickhouse-client --multiquery < infra/clickhouse/ddl/data_ana.sql
-- 数据库
CREATE DATABASE IF NOT EXISTS edu_analytics;
-- ===== 1. 学生学情宽表 =====
-- 每次成绩写入产生一行，按 ORDER BY 去重保留 last_updated 最大版本.
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
(
student_id String,
class_id String,
exam_id String,
subject_id String,
score Float64,
rank_in_class UInt32,
knowledge_point_id String,
mastery_level Float32, -- 0.0-1.0
error_count UInt32,
last_updated DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(last_updated)
PARTITION BY toYYYYMM(last_updated)
ORDER BY (student_id, exam_id, knowledge_point_id)
SETTINGS index_granularity = 8192;
-- ===== 2. 学生错题本 =====
-- 同一 (student_id, question_id) 累计 error_count，按 last_error_time 去重.
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
(
student_id String,
question_id String,
knowledge_point_id String,
error_count UInt32,
last_error_time DateTime64(3, 'UTC'),
content String DEFAULT ''
)
ENGINE = ReplacingMergeTree(last_error_time)
PARTITION BY toYYYYMM(last_error_time)
ORDER BY (student_id, question_id);
-- ===== 3. 知识点掌握度历史快照 =====
-- 每次掌握度计算产生新版本，支持趋势查询.
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
(
student_id String,
knowledge_point_id String,
subject_id String,
mastery_level Float32, -- 0.0-1.0
calculated_at DateTime64(3, 'UTC'),
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(calculated_at)
ORDER BY (student_id, knowledge_point_id, calculated_at);
-- ===== 4. 学生考勤记录 =====
-- core-edu attendance 表 CDC 同步，同一记录多版本去重.
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
(
student_id String,
class_id String,
attendance_date Date,
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
recorded_by String, -- 教师用户 ID
remark String DEFAULT '',
occurred_at DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(attendance_date)
ORDER BY (student_id, class_id, attendance_date);
-- ===== 5. AI 用量计费记录 =====
-- ai 服务通过 Kafka 事件投递，data-ana 消费落库，按 request_id 幂等去重.
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
(
request_id String,
user_id String,
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
model LowCardinality(String),
prompt_tokens UInt32,
completion_tokens UInt32,
total_tokens UInt32,
latency_ms UInt32,
success Boolean,
cost_cents UInt32, -- 计费（分），便于聚合
occurred_at DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(occurred_at)
ORDER BY (request_id);

227
packages/shared-proto/proto/analytics.proto
View File

@@ -2,24 +2,113 @@ syntax = "proto3";
package next_edu_cloud.analytics.v1; package next_edu_cloud.analytics.v1;
// AnalyticsService 数据分析服务契约（D6 智能洞察领域）.
// P4 启用 gRPC server 端口 50055，HTTP 3006 保留作 Gateway 直连降级.
// 所有 RPC 返回 ActionState 信封（success/data/error/details.degraded）.
service AnalyticsService { service AnalyticsService {
// 班级成绩分析（平均分/及格率/参考人数）.
rpc GetClassPerformance(GetClassPerformanceRequest) returns (ClassPerformance); rpc GetClassPerformance(GetClassPerformanceRequest) returns (ClassPerformance);
// 学生薄弱知识点（mastery_level < 0.6）.
rpc GetStudentWeakness(GetStudentWeaknessRequest) returns (StudentWeakness); rpc GetStudentWeakness(GetStudentWeaknessRequest) returns (StudentWeakness);
// 学习趋势（历史成绩曲线）.
rpc GetLearningTrend(GetLearningTrendRequest) returns (LearningTrend); rpc GetLearningTrend(GetLearningTrendRequest) returns (LearningTrend);
// 教师仪表盘聚合（班级概览 + 待办 + 预警）.
rpc GetTeacherDashboard(GetTeacherDashboardRequest) returns (TeacherDashboard);
// 学生仪表盘（个人学情 + 排名 + 薄弱点）.
rpc GetStudentDashboard(GetStudentDashboardRequest) returns (StudentDashboard);
// 家长仪表盘（孩子学情概览）.
rpc GetParentDashboard(GetParentDashboardRequest) returns (ParentDashboard);
// 管理员仪表盘（全校统计 + AI 用量）.
rpc GetAdminDashboard(GetAdminDashboardRequest) returns (AdminDashboard);
// 预警列表查询（按班级/严重度/时间过滤）.
rpc GetWarnings(GetWarningsRequest) returns (WarningList);
// 手动触发预警（管理员/教师主动标记关注）.
rpc TriggerWarning(TriggerWarningRequest) returns (TriggerWarningResponse);
// 班级掌握度分布（mastered/progressing/weak 三档）.
rpc GetMasteryDistribution(GetMasteryDistributionRequest) returns (MasteryDistribution);
// 学生知识点掌握度明细.
rpc GetStudentMastery(GetStudentMasteryRequest) returns (StudentMastery);
// 订阅掌握度更新（server-streaming，P5+ AI 个性化推荐实时推送通道）.
rpc SubscribeMasteryUpdate(SubscribeMasteryUpdateRequest) returns (stream MasteryUpdateEvent);
} }
// ===== 请求消息 =====
message GetClassPerformanceRequest { message GetClassPerformanceRequest {
string class_id = 1; string class_id = 1;
string subject_id = 2; string subject_id = 2;
int64 start_date = 3; int64 start_date = 3; // Unix timestamp（秒）
int64 end_date = 4; int64 end_date = 4;
} }
message GetStudentWeaknessRequest {
string student_id = 1;
string subject_id = 2;
}
message GetLearningTrendRequest {
string student_id = 1;
int64 start_date = 2;
int64 end_date = 3;
string subject_id = 4;
}
message GetTeacherDashboardRequest {
string user_id = 1;
string class_id = 2; // 可选，不传则返回教师全部班级
}
message GetStudentDashboardRequest {
string user_id = 1;
}
message GetParentDashboardRequest {
string user_id = 1;
string student_id = 2; // 家长查看指定孩子
}
message GetAdminDashboardRequest {
string user_id = 1;
string scope = 2; // ALL / SCHOOL / DISTRICT
string scope_id = 3; // 具体范围 ID
}
message GetWarningsRequest {
string class_id = 1;
string severity = 2; // INFO / WARN / CRITICAL
int64 since = 3; // Unix timestamp（秒）
}
message TriggerWarningRequest {
string target_id = 1; // 学生 ID
string warning_type = 2; // LOW_MASTERY / SCORE_DROP / ABSENT_FREQUENT
string severity = 3; // INFO / WARN / CRITICAL
}
message GetMasteryDistributionRequest {
string class_id = 1;
string subject_id = 2;
string knowledge_point_id = 3; // 可选
}
message GetStudentMasteryRequest {
string student_id = 1;
string subject_id = 2; // 可选
}
message SubscribeMasteryUpdateRequest {
string student_id = 1; // 可选，订阅指定学生
string class_id = 2; // 可选，订阅指定班级全部学生
}
// ===== 响应消息 =====
message ClassPerformance { message ClassPerformance {
string class_id = 1; string class_id = 1;
double average_score = 2; double average_score = 2;
double pass_rate = 3; double pass_rate = 3;
repeated StudentScore scores = 4; int32 total_students = 4;
repeated StudentScore scores = 5;
} }
message StudentScore { message StudentScore {
@@ -28,11 +117,6 @@ message StudentScore {
string grade = 3; string grade = 3;
} }
message GetStudentWeaknessRequest {
string student_id = 1;
string subject_id = 2;
}
message StudentWeakness { message StudentWeakness {
string student_id = 1; string student_id = 1;
repeated WeakPoint weak_points = 2; repeated WeakPoint weak_points = 2;
@@ -42,12 +126,7 @@ message WeakPoint {
string knowledge_point_id = 1; string knowledge_point_id = 1;
string title = 2; string title = 2;
double mastery = 3; double mastery = 3;
} int32 error_count = 4;
message GetLearningTrendRequest {
string student_id = 1;
int64 start_date = 2;
int64 end_date = 3;
} }
message LearningTrend { message LearningTrend {
@@ -59,3 +138,125 @@ message TrendPoint {
int64 date = 1; int64 date = 1;
double score = 2; double score = 2;
} }
message TeacherDashboard {
string user_id = 1;
int32 total_classes = 2;
int32 total_students = 3;
double class_avg_score = 4;
int32 pending_homework_count = 5;
repeated ClassSummary classes = 6;
repeated StudentSummary top_students = 7;
repeated WarningInfo recent_warnings = 8;
}
message ClassSummary {
string class_id = 1;
string class_name = 2;
int32 student_count = 3;
double average_score = 4;
}
message StudentSummary {
string student_id = 1;
string student_name = 2;
double score = 3;
int32 rank_in_class = 4;
}
message StudentDashboard {
string user_id = 1;
double avg_score = 2;
int32 class_rank = 3;
int32 total_students = 4;
repeated WeakPoint weak_points = 5;
repeated TrendPoint recent_trends = 6;
int32 pending_homework = 7;
}
message ParentDashboard {
string user_id = 1;
string student_id = 2;
double child_avg_score = 3;
int32 child_class_rank = 4;
int32 total_class_students = 5;
repeated WeakPoint child_weak_points = 6;
repeated WarningInfo child_warnings = 7;
}
message AdminDashboard {
string user_id = 1;
int32 total_teachers = 2;
int32 total_students = 3;
int32 total_classes = 4;
double school_avg_score = 5;
repeated WarningInfo recent_warnings = 6;
AIUsageSummary ai_usage = 7;
}
message AIUsageSummary {
int64 total_requests = 1;
int64 total_tokens = 2;
int64 total_cost_cents = 3;
repeated AIUsageByProvider by_provider = 4;
}
message AIUsageByProvider {
string provider = 1;
int64 request_count = 2;
int64 total_tokens = 3;
int64 cost_cents = 4;
}
message WarningList {
repeated WarningInfo warnings = 1;
int32 total = 2;
}
message WarningInfo {
string warning_id = 1;
string warning_type = 2; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE
string target_id = 3; // 学生 ID
string target_name = 4;
double threshold = 5;
double current_value = 6;
string severity = 7; // INFO / WARN / CRITICAL
int64 occurred_at = 8; // Unix timestamp（秒）
}
message TriggerWarningResponse {
string warning_id = 1;
bool triggered = 2;
}
message MasteryDistribution {
string class_id = 1;
int32 mastered_count = 2; // mastery >= 0.8
int32 progressing_count = 3; // 0.4 <= mastery < 0.8
int32 weak_count = 4; // mastery < 0.4
int32 total_students = 5;
}
message StudentMastery {
string student_id = 1;
repeated KnowledgePointMastery knowledge_points = 2;
double overall_mastery = 3;
}
message KnowledgePointMastery {
string knowledge_point_id = 1;
string title = 2;
string subject_id = 3;
double mastery_level = 4; // 0.0-1.0
string mastery_label = 5; // mastered / progressing / weak
int64 calculated_at = 6; // Unix timestamp（秒）
}
message MasteryUpdateEvent {
string event_id = 1;
string student_id = 2;
string knowledge_point_id = 3;
double mastery_level = 4;
double previous_level = 5;
int64 calculated_at = 6; // Unix timestamp（秒）
}

51
packages/shared-proto/proto/events.proto
View File

@@ -4,13 +4,17 @@ package next_edu_cloud.events.v1;
// Cross-service event contracts published by CoreEdu via the transactional // Cross-service event contracts published by CoreEdu via the transactional
// outbox pattern and consumed by downstream services (notifications, analytics, // outbox pattern and consumed by downstream services (notifications, analytics,
// audit, etc.). Topics follow the convention edu.{domain}.events. // audit, etc.).
// //
// Event routing (TOPIC_MAP in outbox.publisher.ts): // Event routing (TOPIC_MAP in outbox.publisher.ts):
// edu.exam.events <- exam.created / exam.updated / exam.deleted // edu.teaching.exam.published <- exam.published
// edu.homework.events <- homework.assigned / homework.submitted / homework.graded // edu.teaching.homework.assigned <- homework.assigned / homework.submitted / homework.graded
// edu.grade.events <- grade.recorded / grade.updated // edu.teaching.grade.recorded <- grade.recorded / grade.updated
// edu.class.events <- class.transferred // edu.teaching.class.transferred <- class.transferred
//
// Derived data events (Outbox exempt, see 004 §12.2):
// edu.insight.mastery.updated <- mastery.updated / warning.triggered (action field distinguishes)
// edu.insight.ai.usage <- ai.usage.recorded
message ClassEvent { message ClassEvent {
string event_id = 1; string event_id = 1;
@@ -58,3 +62,40 @@ message GradeEvent {
string action = 8; string action = 8;
map<string, string> metadata = 9; map<string, string> metadata = 9;
} }
// MasteryEvent 掌握度/预警派生数据事件（Outbox 豁免，直接 Kafka producer 发布）.
// action 字段区分：mastery.updated（掌握度更新）/ warning.triggered（预警触发）.
// 两者复用同一 topic edu.insight.mastery.updated（总裁裁决 §2.11）.
message MasteryEvent {
string event_id = 1;
string action = 2; // mastery.updated / warning.triggered
int64 occurred_at = 3;
string student_id = 4;
string knowledge_point_id = 5;
double mastery_level = 6; // 0.0-1.0（mastery.updated 时有效）
double previous_level = 7; // 上一次掌握度（mastery.updated 时有效）
// warning.triggered 时以下字段有效
string warning_type = 8; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE
string target_id = 9; // 预警目标（学生 ID）
double threshold = 10; // 预警阈值
double current_value = 11; // 当前值
string severity = 12; // INFO / WARN / CRITICAL
map<string, string> metadata = 13;
}
// AIUsageEvent AI 用量计费事件（ai 服务发布，data-ana 消费落 ai_usage_log）.
message AIUsageEvent {
string event_id = 1;
string request_id = 2;
string user_id = 3;
string provider = 4; // openai / anthropic / baichuan / local
string model = 5;
uint32 prompt_tokens = 6;
uint32 completion_tokens = 7;
uint32 total_tokens = 8;
uint32 latency_ms = 9;
bool success = 10;
uint32 cost_cents = 11; // 计费（分）
int64 occurred_at = 12;
map<string, string> metadata = 13;
}

17
packages/shared-proto/proto/iam.proto
View File

@@ -9,6 +9,9 @@ service IamService {
rpc Login(LoginRequest) returns (AuthResponse); rpc Login(LoginRequest) returns (AuthResponse);
rpc RefreshToken(RefreshTokenRequest) returns (TokenPair); rpc RefreshToken(RefreshTokenRequest) returns (TokenPair);
rpc GetUserInfo(GetUserInfoRequest) returns (UserInfo); rpc GetUserInfo(GetUserInfoRequest) returns (UserInfo);
// GetEffectiveDataScope 解析用户可见数据范围（DataScope 6 级）.
// data-ana gRPC 调用此 RPC 解析查询过滤范围（coord-cross-review §2 #3 裁决 P4 补全）.
rpc GetEffectiveDataScope(GetEffectiveDataScopeRequest) returns (EffectiveDataScope);
} }
message RegisterRequest { message RegisterRequest {
@@ -48,3 +51,17 @@ message UserInfo {
repeated string roles = 4; repeated string roles = 4;
repeated string permissions = 5; repeated string permissions = 5;
} }
message GetEffectiveDataScopeRequest {
string user_id = 1;
}
// EffectiveDataScope 用户可见数据范围（6 级）.
// level: SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
// scope_ids: 具体可见的 class_id / grade_id 列表（SELF/ALL 时为空）
message EffectiveDataScope {
string user_id = 1;
string level = 2; // SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
repeated string scope_ids = 3; // class_id 列表（CLASS 级）/ grade_id 列表（GRADE 级）
string school_id = 4; // SCHOOL 级时的学校 ID
}

93
pnpm-lock.yaml generated
View File

@@ -82,8 +82,88 @@ importers:
specifier: ^5.6.0 specifier: ^5.6.0
version: 5.9.3 version: 5.9.3
packages/hooks:
dependencies:
'@edu/ui-components':
specifier: workspace:*
version: link:../ui-components
react:
specifier: ^18.3.0
version: 18.3.1
react-dom:
specifier: ^18.3.0
version: 18.3.1(react@18.3.1)
devDependencies:
'@types/react':
specifier: ^18.3.0
version: 18.3.31
'@types/react-dom':
specifier: ^18.3.0
version: 18.3.7(@types/react@18.3.31)
typescript:
specifier: ^5.6.0
version: 5.9.3
packages/shared-proto: {} packages/shared-proto: {}
packages/shared-ts:
dependencies:
'@nestjs/common':
specifier: ^10.4.0
version: 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2)
'@nestjs/core':
specifier: ^10.4.0
version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2)
'@paralleldrive/cuid2':
specifier: ^2.2.2
version: 2.3.1
drizzle-orm:
specifier: ^0.31.0
version: 0.31.4(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.6.1)(@types/react@18.3.31)(better-sqlite3@11.10.0)(mysql2@3.22.6(@types/node@22.20.0))(react@18.3.1)
kafkajs:
specifier: ^2.2.0
version: 2.2.4
pino:
specifier: ^9.4.0
version: 9.14.0
reflect-metadata:
specifier: ^0.2.2
version: 0.2.2
rxjs:
specifier: ^7.8.0
version: 7.8.2
devDependencies:
'@types/node':
specifier: ^22.0.0
version: 22.20.0
typescript:
specifier: ^5.6.0
version: 5.9.3
packages/ui-components:
dependencies:
'@edu/ui-tokens':
specifier: workspace:*
version: link:../ui-tokens
react:
specifier: ^18.3.0
version: 18.3.1
react-dom:
specifier: ^18.3.0
version: 18.3.1(react@18.3.1)
devDependencies:
'@types/react':
specifier: ^18.3.0
version: 18.3.31
'@types/react-dom':
specifier: ^18.3.0
version: 18.3.7(@types/react@18.3.31)
typescript:
specifier: ^5.6.0
version: 5.9.3
packages/ui-tokens: {}
scripts/arch-scan: scripts/arch-scan:
dependencies: dependencies:
better-sqlite3: better-sqlite3:
@@ -1494,6 +1574,10 @@ packages:
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
'@noble/hashes@1.8.0':
resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
engines: {node: ^14.21.3 || >=16}
'@nodelib/fs.scandir@2.1.5': '@nodelib/fs.scandir@2.1.5':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'} engines: {node: '>= 8'}
@@ -2503,6 +2587,9 @@ packages:
peerDependencies: peerDependencies:
'@opentelemetry/api': ^1.1.0 '@opentelemetry/api': ^1.1.0
'@paralleldrive/cuid2@2.3.1':
resolution: {integrity: sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==}
'@pinojs/redact@0.4.0': '@pinojs/redact@0.4.0':
resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
@@ -6642,6 +6729,8 @@ snapshots:
'@next/swc-win32-x64-msvc@14.2.33': '@next/swc-win32-x64-msvc@14.2.33':
optional: true optional: true
'@noble/hashes@1.8.0': {}
'@nodelib/fs.scandir@2.1.5': '@nodelib/fs.scandir@2.1.5':
dependencies: dependencies:
'@nodelib/fs.stat': 2.0.5 '@nodelib/fs.stat': 2.0.5
@@ -8135,6 +8224,10 @@ snapshots:
'@opentelemetry/api': 1.9.1 '@opentelemetry/api': 1.9.1
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
'@paralleldrive/cuid2@2.3.1':
dependencies:
'@noble/hashes': 1.8.0
'@pinojs/redact@0.4.0': {} '@pinojs/redact@0.4.0': {}
'@pkgjs/parseargs@0.11.0': '@pkgjs/parseargs@0.11.0':

104
scripts/clickhouse-init.sql
View File

@@ -1,37 +1,89 @@
-- ClickHouse 数据库初始化脚本 -- ClickHouse 数据库初始化脚本
-- 适用服务：data-ana（数据分析） -- 适用服务：data-ana（D6 智能洞察领域）
-- 表结构：student_dashboard_view（学生学情宽表）/ student_errors（错题本） -- 5 张宽表（ai-allocation §5 强制）：
-- 与 services/data-ana/src/data_ana/clickhouse_client.py 中的查询字段对齐 -- 1. student_dashboard_view 学生学情宽表（成绩/班级/知识点维度）
-- 2. student_errors 学生错题本
-- 3. mastery_snapshot 知识点掌握度历史快照
-- 4. ai_usage_log AI 用量计费记录（ai 服务投递，data-ana 消费）
-- 5. attendance_logs 学生考勤记录（core-edu attendance 表 CDC 同步）
--
-- 引擎对齐 02-architecture-design.md §3：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本
-- 查询规范：所有 SELECT 必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改）
-- --
-- 使用方式（启用 ClickHouse 时执行一次）： -- 使用方式（启用 ClickHouse 时执行一次）：
-- clickhouse-client --multiquery < scripts/clickhouse-init.sql -- clickhouse-client --multiquery < scripts/clickhouse-init.sql
-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式。 -- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式（返回骨架数据）。
-- 数据库 -- 数据库
CREATE DATABASE IF NOT EXISTS edu_analytics; CREATE DATABASE IF NOT EXISTS edu_analytics;
-- 学生学情宽表（考试/班级/知识点维度） -- ===== 1. 学生学情宽表（成绩写入产生一行，按 ORDER BY 去重保留最新版本） =====
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view ( CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view (
student_id String, student_id String,
class_id String, class_id String,
exam_id String, exam_id String,
subject_id String, subject_id String,
score Float64, score Float64,
rank_in_class UInt32, rank_in_class UInt32,
knowledge_point_id String, knowledge_point_id String,
mastery_level Float32, mastery_level Float32, -- 0.0-1.0
error_count UInt32, error_count UInt32,
last_updated DateTime last_updated DateTime64(3, 'UTC')
) ENGINE = MergeTree() ) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (student_id, class_id, exam_id); PARTITION BY toYYYYMM(last_updated)
ORDER BY (student_id, exam_id, knowledge_point_id)
SETTINGS index_granularity = 8192;
-- 学生错题表（错题本） -- ===== 2. 学生错题本 =====
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors ( CREATE TABLE IF NOT EXISTS edu_analytics.student_errors (
student_id String, student_id String,
question_id String, question_id String,
knowledge_point_id String, knowledge_point_id String,
error_count UInt32, error_count UInt32,
last_error_time DateTime, last_error_time DateTime64(3, 'UTC'),
content String content String
) ENGINE = MergeTree() ) ENGINE = ReplacingMergeTree(last_error_time)
ORDER BY (student_id, knowledge_point_id); PARTITION BY toYYYYMM(last_error_time)
ORDER BY (student_id, question_id);
-- ===== 3. 掌握度历史快照（每次掌握度计算产生新版本，支持趋势查询） =====
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot (
student_id String,
knowledge_point_id String,
subject_id String,
mastery_level Float32,
calculated_at DateTime64(3, 'UTC'),
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
) ENGINE = MergeTree
PARTITION BY toYYYYMM(calculated_at)
ORDER BY (student_id, knowledge_point_id, calculated_at);
-- ===== 4. AI 用量计费记录（ai 服务通过 Kafka 事件投递，data-ana 消费落库） =====
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log (
request_id String,
user_id String,
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
model LowCardinality(String),
prompt_tokens UInt32,
completion_tokens UInt32,
total_tokens UInt32,
latency_ms UInt32,
success Boolean,
cost_cents UInt32, -- 计费（分），便于聚合
occurred_at DateTime64(3, 'UTC')
) ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(occurred_at)
ORDER BY (request_id); -- 按 request_id 幂等去重
-- ===== 5. 学生考勤记录（core-edu attendance 表 CDC 同步） =====
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs (
student_id String,
class_id String,
attendance_date Date,
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
recorded_by String, -- 教师用户 ID
remark String DEFAULT '',
occurred_at DateTime64(3, 'UTC')
) ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(attendance_date)
ORDER BY (student_id, class_id, attendance_date);

131
scripts/data-ana-mock-data.sql Normal file
View File

@@ -0,0 +1,131 @@
-- data-ana 模块 Mock 数据脚本
-- 用途：本地开发/集成测试时为 5 张宽表插入演示数据，验证查询逻辑
-- 使用方式（需先执行 clickhouse-init.sql）：
-- clickhouse-client --multiquery < scripts/data-ana-mock-data.sql
--
-- 数据规模：2 班 × 10 学生 × 3 知识点 × 2 考试 = 120 行 student_dashboard_view
-- 20 条错题 / 60 条掌握度快照 / 20 条考勤 / 10 条 AI 用量
-- 所有时间戳使用 now() - N days 模式，避免硬编码绝对日期导致脚本过期
USE edu_analytics;
-- ===== 1. student_dashboard_view：学情宽表 =====
-- 班级 C001（高一 1 班）+ C002（高一 2 班），每班 10 学生
-- 2 次考试（EXAM001 数学月考 / EXAM002 数学期中），3 个数学知识点
INSERT INTO student_dashboard_view
SELECT
'S' || toString(number + 1) AS student_id, -- S001..S020
IF(number < 10, 'C001', 'C002') AS class_id, -- 前 10 在 C001，后 10 在 C002
'EXAM001' AS exam_id,
'SUBJ_MATH' AS subject_id,
40 + (number % 60) AS score, -- 40-99 分
(number % 10) + 1 AS rank_in_class, -- 1-10 名
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, -- KP_MATH_1/2/3
0.3 + ((number % 70) / 100.0) AS mastery_level, -- 0.30-0.99
(number % 5) AS error_count, -- 0-4 错题
now() - INTERVAL 7 DAY AS last_updated
FROM numbers(20);
-- 第二次考试（EXAM002），成绩略升
INSERT INTO student_dashboard_view
SELECT
'S' || toString(number + 1) AS student_id,
IF(number < 10, 'C001', 'C002') AS class_id,
'EXAM002' AS exam_id,
'SUBJ_MATH' AS subject_id,
50 + (number % 50) AS score, -- 50-99 分
(number % 10) + 1 AS rank_in_class,
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
0.4 + ((number % 60) / 100.0) AS mastery_level,
(number % 4) AS error_count,
now() - INTERVAL 1 DAY AS last_updated
FROM numbers(20);
-- ===== 2. student_errors：错题本 =====
INSERT INTO student_errors
SELECT
'S' || toString((number % 20) + 1) AS student_id,
'Q_' || toString(number + 1) AS question_id,
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
(number % 5) + 1 AS error_count,
now() - INTERVAL (number % 30) DAY AS last_error_time,
'错误类型：' || multiIf(
number % 3 = 0, '计算失误',
number % 3 = 1, '公式记忆错误',
'审题不清'
) AS content
FROM numbers(20);
-- ===== 3. mastery_snapshot：掌握度历史快照（3 次计算，便于趋势查询） =====
INSERT INTO mastery_snapshot
SELECT
'S' || toString((number % 20) + 1) AS student_id,
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
'SUBJ_MATH' AS subject_id,
0.3 + ((number % 70) / 100.0) AS mastery_level,
now() - INTERVAL 30 DAY AS calculated_at,
'weighted_moving_avg' AS calculation_method
FROM numbers(20);
INSERT INTO mastery_snapshot
SELECT
'S' || toString((number % 20) + 1) AS student_id,
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
'SUBJ_MATH' AS subject_id,
0.4 + ((number % 60) / 100.0) AS mastery_level,
now() - INTERVAL 15 DAY AS calculated_at,
'weighted_moving_avg' AS calculation_method
FROM numbers(20);
INSERT INTO mastery_snapshot
SELECT
'S' || toString((number % 20) + 1) AS student_id,
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
'SUBJ_MATH' AS subject_id,
0.5 + ((number % 50) / 100.0) AS mastery_level,
now() AS calculated_at,
'weighted_moving_avg' AS calculation_method
FROM numbers(20);
-- ===== 4. attendance_logs：考勤记录（近 5 天，含缺勤/迟到） =====
INSERT INTO attendance_logs
SELECT
'S' || toString((number % 20) + 1) AS student_id,
IF(number % 20 < 10, 'C001', 'C002') AS class_id,
today() - (number % 5) AS attendance_date,
multiIf(
number % 10 = 0, 'absent', -- 10% 缺勤
number % 5 = 0, 'late', -- 20% 迟到
'present' -- 其余正常
) AS status,
'T001' AS recorded_by,
multiIf(number % 10 = 0, '病假', '') AS remark,
now() - INTERVAL (number % 5) DAY AS occurred_at
FROM numbers(20);
-- ===== 5. ai_usage_log：AI 用量计费（模拟 3 个 provider） =====
INSERT INTO ai_usage_log
SELECT
'REQ_' || toString(number + 1) AS request_id,
'S' || toString((number % 20) + 1) AS user_id,
arrayElement(['openai', 'anthropic', 'baichuan'], (number % 3) + 1) AS provider,
arrayElement(['gpt-4o-mini', 'claude-3-haiku', 'baichuan2-turbo'], (number % 3) + 1) AS model,
100 + (number % 200) AS prompt_tokens, -- 100-299
50 + (number % 150) AS completion_tokens, -- 50-199
150 + (number % 350) AS total_tokens, -- 150-499
500 + (number % 2000) AS latency_ms, -- 500-2499 ms
number % 5 != 0 AS success, -- 80% 成功
(number % 50) + 1 AS cost_cents, -- 1-50 分
now() - INTERVAL (number % 24) HOUR AS occurred_at
FROM numbers(20);
-- ===== 校验查询（可选执行） =====
SELECT 'student_dashboard_view' AS table_name, count() AS cnt FROM student_dashboard_view FINAL
UNION ALL
SELECT 'student_errors', count() FROM student_errors FINAL
UNION ALL
SELECT 'mastery_snapshot', count() FROM mastery_snapshot
UNION ALL
SELECT 'attendance_logs', count() FROM attendance_logs FINAL
UNION ALL
SELECT 'ai_usage_log', count() FROM ai_usage_log FINAL;

40
services/data-ana/Dockerfile
View File

@@ -1,8 +1,42 @@
FROM python:3.12-slim # 多阶段构建：builder 缓存依赖，runtime 精简镜像
# 对齐 02-architecture-design.md §16 部署规范
# ===== Stage 1: builder =====
FROM python:3.12-slim AS builder
WORKDIR /app WORKDIR /app
# 安装 uv（快速依赖管理）
RUN pip install uv RUN pip install uv
# 先复制 pyproject.toml，利用 Docker 层缓存
COPY pyproject.toml . COPY pyproject.toml .
# 同步依赖到虚拟环境（--no-dev 不装开发依赖）
RUN uv sync --no-dev RUN uv sync --no-dev
# ===== Stage 2: runtime =====
FROM python:3.12-slim AS runtime
WORKDIR /app
# 从 builder 复制虚拟环境
COPY --from=builder /app/.venv /app/.venv
# 确保虚拟环境在 PATH 中
ENV PATH="/app/.venv/bin:$PATH"
ENV PYTHONUNBUFFERED=1
ENV PYTHONDONTWRITEBYTECODE=1
# 复制源码
COPY src ./src COPY src ./src
EXPOSE 3006
CMD ["uv", "run", "uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"] # 暴露端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（AnalyticsService 12 RPC）
EXPOSE 3006 50055
# 健康检查（HTTP /healthz，每 30s 一次）
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:3006/healthz')" || exit 1
# 启动命令：uvicorn HTTP server（gRPC server 在 lifespan 中启动）
CMD ["uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"]

404
services/data-ana/README.md
View File

@@ -1,58 +1,386 @@
# data-ana 数据分析服务 # data-ana 数据分析服务
> 版本：0.1（P4 骨架） > 版本：1.0（P4 完整版）
> 端口：3006 > 阶段：D6 智能洞察领域（仲裁已对齐，v2 实现版）
> 端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（主入口，BFF/ai 调用）
> 日期：2026-07-10
## 职责 ## 职责
数据分析限界上下文（Python 实现），消费 core-edu 与 content 的领域事件， 数据分析限界上下文（Python/FastAPI 微服务），消费 core-edu / content / iam / ai 的 CDC 事件，
构建 ClickHouse 学情宽表，计算知识点掌握度。 构建 ClickHouse 学情宽表，计算知识点掌握度，评估预警，为前端提供 4 端 Dashboard 查询服务。
对外提供学情仪表盘查询、班级/年级/学校维度报表、个性化推荐数据支撑。
### 核心能力
- **5 张宽表**：student_dashboard_view / student_errors / mastery_snapshot / attendance_logs / ai_usage_log
- **CDC 链路**：Debezium → Kafka → data-ana 消费 → ClickHouse（at-least-once + 手动 commit + ReplacingMergeTree 去重）
- **掌握度算法**：WEIGHTED_MOVING_AVG（w_i=0.6^i）+ FORGETTING_CURVE（半衰期 30 天）取 max
- **预警评估**：5 类预警（LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE）+ Redis 25h 去重
- **DataScope 过滤**：6 级（SELF/CLASS/GRADE/SCHOOL/DISTRICT/ALL），iam gRPC + 角色降级
- **gRPC 12 RPC**：AnalyticsService（含 server-streaming SubscribeMasteryUpdate）+ HealthService
- **HTTP 14 端点**：3 基础 + 11 业务（ActionState 信封 + degraded 标记）
## 架构图
```mermaid
flowchart TB
subgraph Entry["入口层"]
HTTP[FastAPI HTTP<br/>:3006 /analytics/* + /healthz + /readyz]
GRPC[grpc.aio Server<br/>:50055 AnalyticsService 12 RPC]
end
subgraph Service["应用服务层"]
S1[AnalyticsService<br/>4 Dashboard + 班级/趋势查询]
S2[MasteryService<br/>掌握度计算 + 事件发布]
S3[WarningService<br/>5 类预警 + Redis 去重]
end
subgraph Repo["数据访问层"]
R1[ClickHouseRepository<br/>FINAL/argMax + DataScope WHERE]
R2[KafkaProducer<br/>mastery.updated 派生事件]
R3[IamClient<br/>gRPC GetEffectiveDataScope]
R4[RedisClient<br/>DataScope 缓存 + 幂等]
end
subgraph Consumer["CDC 消费者（后台任务）"]
C1[CdcConsumer<br/>手动 commit + 7 类事件路由]
C2[ExamCache<br/>LRU 10000 exam_id→metadata]
end
subgraph Storage["存储/总线"]
CH[(ClickHouse<br/>5 宽表)]
KAFKA[(Kafka<br/>edu-cdc.* + edu.insight.mastery.updated)]
IAM[iam gRPC :50052]
REDIS[(Redis<br/>data_ana:* 键)]
end
HTTP --> S1
HTTP --> S3
GRPC --> S1
S1 --> R1
S2 --> R1
S2 --> R2
S3 --> R1
S3 --> R2
R1 --> CH
R2 --> KAFKA
R3 --> IAM
R3 --> R4
R4 --> REDIS
C1 --> C2
C1 --> R1
C1 --> R4
C1 --> KAFKA
```
## 技术栈 ## 技术栈
- Python 3.12+ / FastAPI 0.115+ - **Python 3.12+** / **FastAPI 0.115+**（HTTP 端点）
- clickhouse-connect（ClickHouse 宽表查询） - **grpcio 1.66+** / **grpcio-health-checking**（gRPC server + HealthService）
- pydantic + pydantic-settings（运行时校验与配置） - **clickhouse-connect 0.7+**（ClickHouse 宽表查询，支持 FINAL/argMax）
- structlog（结构化日志） - **aiokafka 0.11+**（CDC 消费者，手动 commit + 幂等去重）
- prometheus-client（指标） - **redis 5.0+**（DataScope 缓存 + 事件去重 + 预警位图）
- OpenTelemetry（分布式追踪） - **pydantic 2.9+** / **pydantic-settings 2.5+**（模型校验 + 配置）
- **structlog 24.4+**（结构化 JSON 日志）
## 开发 - **prometheus-client 0.20+**（/metrics 指标）
- **opentelemetry-sdk 1.27+**（OTLP 链路追踪）
```bash - **protobuf 5.28+**（buf generate 生成的 stub 运行时）
uv sync
uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload
```
## 配置
| 变量 | 默认值 | 说明 |
|------|--------|------|
| `PORT` | 3006 | 服务端口 |
| `CLICKHOUSE_HOST` | localhost | ClickHouse 主机 |
| `CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 |
| `CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 |
| `OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry 端点 |
| `LOG_LEVEL` | info | 日志级别 |
## 模块结构 ## 模块结构
``` ```
src/data_ana/ src/data_ana/
├─ __init__.py ├─ __init__.py
├─ main.py # FastAPI 入口（健康检查 + 分析端点骨架） ├─ main.py # FastAPI 入口（14 端点 + lifespan + ActionState）
├─ config.py # pydantic-settings 配置 ├─ config.py # pydantic-settings 配置（env_prefix=DATA_ANA_）
└─ clickhouse_client.py # ClickHouse 客户端单例 ├─ analytics_service.py # 4 Dashboard + 班级/趋势查询编排
├─ mastery_service.py # 掌握度算法（WMA + ForgettingCurve）
├─ warning_service.py # 5 类预警评估 + Redis 去重
├─ exam_cache.py # LRU exam_id→metadata 映射（max 10000）
├─ cdc_consumer.py # CDC 消费者（7 类事件 + 手动 commit）
├─ grpc_server.py # 12 RPC + HealthService + SubscribeMasteryUpdate
├─ clickhouse_client.py # ClickHouse 客户端单例（降级模式）
├─ repository/
│ ├─ __init__.py
│ ├─ clickhouse_repository.py # 宽表查询 + FINAL/argMax + DataScope WHERE
│ ├─ redis_client.py # Redis 异步客户端（cache + dedup + bitmap）
│ ├─ iam_client.py # iam gRPC GetEffectiveDataScope + 角色降级
│ └─ kafka_producer.py # mastery.updated / warning.triggered 发布
└─ shared/
├─ __init__.py
├─ action_state.py # ActionState[T] 统一信封（ok/fail + degraded）
├─ errors.py # ErrorCode 13 类 + DataAnaError
└─ permissions.py # 权限点 + DataScopeLevel + build_datascope_where
``` ```
## 关键端点 ## 开发
- `GET /healthz` 健康检查 ```bash
- `GET /metrics` Prometheus 指标 # 安装依赖
- `GET /analytics/class/{class_id}/performance` 班级成绩分析（P4 骨架） uv sync
- `GET /analytics/student/{student_id}/weakness` 学生薄弱知识点分析（P4 骨架）
# 本地开发（HTTP + gRPC + CDC 同时启动）
uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload
# ruff 校验
ruff check src/
# 类型检查（如已配置 mypy）
mypy src/
```
## 配置项
所有配置使用 `DATA_ANA_` 前缀（pydantic-settings env_prefix）。
### 基础配置
| 变量 | 默认值 | 说明 |
| ------------------------ | --------------------- | ------------------------------------ |
| `DATA_ANA_HTTP_PORT` | 3006 | HTTP 端口（Gateway 直连降级） |
| `DATA_ANA_GRPC_PORT` | 50055 | gRPC 端口（AnalyticsService 主入口） |
| `DATA_ANA_DEV_MODE` | true | 开发模式（关闭签名校验） |
| `DATA_ANA_LOG_LEVEL` | info | 日志级别（debug/info/warning/error） |
| `DATA_ANA_OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry OTLP 端点 |
### 外部依赖（全部可选，未配置则降级模式）
| 变量 | 默认值 | 说明 |
| ------------------------------ | ------------- | ------------------------------------------ |
| `DATA_ANA_CLICKHOUSE_HOST` | "" | ClickHouse 主机（空字符串=降级模式） |
| `DATA_ANA_CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 |
| `DATA_ANA_CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 |
| `DATA_ANA_CLICKHOUSE_USER` | "" | ClickHouse 用户名 |
| `DATA_ANA_CLICKHOUSE_PASSWORD` | "" | ClickHouse 密码 |
| `DATA_ANA_KAFKA_BROKERS` | "" | Kafka broker 列表（逗号分隔，空=禁用 CDC） |
| `DATA_ANA_KAFKA_GROUP_ID` | data-ana-cdc | 消费者组 ID |
| `DATA_ANA_REDIS_URL` | "" | Redis URL（空=禁用缓存/去重） |
| `DATA_ANA_IAM_GRPC_ENDPOINT` | "" | iam gRPC 端点（空=角色降级） |
### 降级模式说明
- **ClickHouse 未配置/不可达**：查询返回骨架数据（空数组 + degraded=true）
- **Kafka 未配置**：CDC 消费者不启动，`/readyz` 仍就绪
- **Redis 未配置/不可达**：缓存失效，每次查询重新计算 DataScope
- **iam gRPC 未配置/不可达**：基于 JWT 角色 fallback（teacher→CLASS，student→SELF）
- **grpcio 未安装**：gRPC server 不启动，HTTP 仍可用
## 端点清单
### 基础端点（3 个）
| 方法 | 路径 | 说明 |
| ---- | ---------- | ----------------------------------------------------------- |
| GET | `/` | 根信息（服务名/版本/端点列表） |
| GET | `/healthz` | liveness 检查（进程存活即 ok） |
| GET | `/readyz` | readiness 检查（5 依赖状态：clickhouse/cdc/redis/iam/grpc） |
| GET | `/metrics` | Prometheus 指标 |
### 业务端点（11 个，全部返回 ActionState[T]）
| 方法 | 路径 | 说明 |
| ---- | -------------------------------------------------- | ------------------------------- |
| GET | `/analytics/class/{class_id}/performance` | 班级成绩分析（平均分/及格率） |
| GET | `/analytics/class/{class_id}/mastery-distribution` | 班级掌握度分布（三档） |
| GET | `/analytics/student/{student_id}/weakness` | 学生薄弱知识点（mastery < 0.6） |
| GET | `/analytics/student/{student_id}/mastery` | 学生掌握度明细 |
| GET | `/analytics/student/{student_id}/trend` | 学习趋势（历史成绩曲线） |
| GET | `/analytics/student/{student_id}/errorbook` | 学生错题本 |
| GET | `/analytics/teacher/dashboard` | 教师仪表盘 |
| GET | `/analytics/student/dashboard` | 学生仪表盘 |
| GET | `/analytics/parent/dashboard` | 家长仪表盘 |
| GET | `/analytics/admin/dashboard` | 管理员仪表盘 |
| GET | `/analytics/warnings` | 预警列表 |
| POST | `/analytics/warnings/trigger` | 手动触发预警 |
### ActionState 信封格式
```json
{
"success": true,
"data": { ... },
"details": {
"degraded": false,
"degraded_reason": ""
}
}
```
降级时 `degraded=true`，`data` 返回骨架数据（空数组/零值）。
## gRPC 契约
### AnalyticsService（12 RPC，端口 50055）
| RPC | 类型 | 说明 |
| ---------------------- | ---------------- | ----------------------------------- |
| GetClassPerformance | unary | 班级成绩分析 |
| GetStudentWeakness | unary | 学生薄弱知识点 |
| GetLearningTrend | unary | 学习趋势 |
| GetTeacherDashboard | unary | 教师仪表盘 |
| GetStudentDashboard | unary | 学生仪表盘 |
| GetParentDashboard | unary | 家长仪表盘 |
| GetAdminDashboard | unary | 管理员仪表盘 |
| GetWarnings | unary | 预警列表 |
| TriggerWarning | unary | 手动触发预警 |
| GetMasteryDistribution | unary | 班级掌握度分布 |
| GetStudentMastery | unary | 学生掌握度明细 |
| SubscribeMasteryUpdate | server-streaming | 掌握度更新订阅（P5+ AI 个性化通道） |
### HealthService
`grpc.health.v1.Health` 标准 K8s 探针。
### proto 定义
- `packages/shared-proto/proto/analytics.proto`（12 RPC + 全部消息）
- `packages/shared-proto/proto/events.proto`（MasteryEvent + AIUsageEvent）
- `packages/shared-proto/proto/iam.proto`（GetEffectiveDataScope）
## CDC 事件消费
### 消费的 7 类事件
| 事件源 | Topic | 处理 |
| ------------------------ | ------------------------------------------------- | ------------------------- |
| core-edu.exams | `edu-cdc.next_edu_cloud.core_edu_exams` | 写 exam_cache |
| core-edu.grades | `edu-cdc.next_edu_cloud.core_edu_grades` | 写 student_dashboard_view |
| core-edu.homework | `edu-cdc.next_edu_cloud.core_edu_homework` | 写 student_dashboard_view |
| core-edu.attendance | `edu-cdc.next_edu_cloud.core_edu_attendance` | 写 attendance_logs |
| content.knowledge_points | `edu-cdc.next_edu_cloud.content_knowledge_points` | 知识点元数据 |
| ai.ai_usage | `edu.insight.ai.usage` | 写 ai_usage_log |
### 幂等保证
- **Kafka producer**：`enable_idempotence=True` + `transactional_id`
- **Consumer 去重**：Redis SETNX `data_ana:dedup:{event_id}` TTL 7 天
- **ClickHouse 去重**：ReplacingMergeTree(last_updated) + FINAL/argMax 查询
- **手动 commit**：仅在 ClickHouse 写入成功后 commit（at-least-once 语义）
## 发布的派生数据事件
| 事件 | Topic | 触发条件 | Outbox 豁免 |
| ---------------- | ------------------------------------- | -------------- | ----------------------- |
| MasteryUpdated | `edu.insight.mastery.updated` | 掌握度计算完成 | ✅（004 §12.2 仲裁） |
| WarningTriggered | `edu.insight.mastery.updated`（复用） | 预警阈值触发 | ✅（004 §15.3 #6 仲裁） |
> Outbox 豁免理由：派生数据事件，无业务状态变更，重复消费幂等。
## ClickHouse 宽表
DDL 位置：`scripts/clickhouse-init.sql`
| 表名 | 引擎 | ORDER BY | 用途 |
| ---------------------- | ----------------------------------- | ----------------------------------------------- | -------------- |
| student_dashboard_view | ReplacingMergeTree(last_updated) | (student_id, exam_id, knowledge_point_id) | 学情宽表 |
| student_errors | ReplacingMergeTree(last_error_time) | (student_id, question_id) | 错题本 |
| mastery_snapshot | MergeTree | (student_id, knowledge_point_id, calculated_at) | 掌握度历史快照 |
| ai_usage_log | ReplacingMergeTree(occurred_at) | (request_id,) | AI 用量计费 |
| attendance_logs | ReplacingMergeTree(occurred_at) | (student_id, class_id, attendance_date) | 考勤记录 |
Mock 数据：`scripts/data-ana-mock-data.sql`
## 掌握度算法
### 公式
```
mastery = max(WMA, ForgettingCurve)
WMA = Σ(w_i * score_i) / Σ(w_i), w_i = 0.6^i （近 5 次，权重递减）
ForgettingCurve = last_mastery * exp(-ln(2) * days_since / 30)
```
### 分类
| 区间 | label | 说明 |
| --------- | ----------- | ------ |
| ≥ 0.8 | mastered | 已掌握 |
| 0.4 - 0.8 | progressing | 进展中 |
| < 0.4 | weak | 薄弱 |
### 降级规则
- 近 5 次成绩不足 5 次 → 降级 SIMPLE_AVG
- 无历史数据 → 返回 0.0 + label=weak
## 预警类型
| 类型 | 阈值 | severity | 说明 |
| --------------- | ------------- | -------- | -------- |
| LOW_MASTERY | mastery < 0.4 | WARN | 掌握度低 |
| CRITICAL_LOW | mastery < 0.2 | CRITICAL | 严重低 |
| SCORE_DROP | 下降 ≥ 20% | WARN | 成绩下降 |
| ABSENT_FREQUENT | ≥ 3 次/周 | WARN | 频繁缺勤 |
| TREND_DECLINE | 连续 3 次降 | INFO | 趋势下降 |
### 去重
- Redis key：`data_ana:warning:{target_id}:{type}:{date}`
- TTL：25 小时（跨日保证）
## DataScope 过滤
### 6 级
| 级别 | scope_ids 含义 |
| -------- | --------------------- |
| SELF | 学生本人 ID |
| CLASS | 可见 class_id 列表 |
| GRADE | 可见 grade_id 列表 |
| SCHOOL | 可见 school_id 列表 |
| DISTRICT | 可见 district_id 列表 |
| ALL | 无过滤 |
### 解析链
```
请求 → Redis 缓存（5min）→ iam gRPC GetEffectiveDataScope → 角色降级 fallback
```
### 角色降级（iam 不可用时）
| JWT 角色 | 降级 scope |
| -------- | --------------------- |
| teacher | CLASS（自己教的班级） |
| student | SELF（本人） |
| parent | SELF（本人 + 孩子） |
| admin | ALL |
## 部署
### Docker
```bash
# 构建（多阶段：builder + runtime）
docker build -t data-ana:1.0.0 services/data-ana/
# 运行（需 ClickHouse/Kafka/Redis 可达）
docker run -p 3006:3006 -p 50055:50055 \
-e DATA_ANA_CLICKHOUSE_HOST=clickhouse \
-e DATA_ANA_KAFKA_BROKERS=kafka:9092 \
-e DATA_ANA_REDIS_URL=redis://redis:6379 \
data-ana:1.0.0
```
### 健康检查
- HTTP `/healthz`：进程存活即 ok
- HTTP `/readyz`：检查 5 依赖（clickhouse/cdc/redis/iam/grpc）
- gRPC `grpc.health.v1.Health`：K8s 探针
## 对外契约 ## 对外契约
gRPC 服务 `AnalyticsService` 定义见 `packages/shared-proto/proto/analytics.proto`。 - **gRPC proto**：`packages/shared-proto/proto/analytics.proto`
- **HTTP OpenAPI**：启动后访问 `/docs`
- **CDC topic 命名**：`edu-cdc.next_edu_cloud.<table>`（Debezium 标准）
- **派生事件 topic**：`edu.insight.mastery.updated` / `edu.insight.ai.usage`
## 相关文档
- [02-architecture-design.md](./docs/02-architecture-design.md)：模块架构设计 v2（仲裁后）
- [01-understanding.md](./docs/01-understanding.md)：阶段 1 理解确认书
- [004 架构影响地图 §15.3](../../docs/architecture/004_architecture_impact_map.md)：coord 仲裁结论
- [coord-cross-review.md](../../docs/architecture/coord-cross-review.md)：交叉审查记录
- [known-issues §2.6](../../docs/troubleshooting/known-issues.md)：data-ana 已知问题速查

17
services/data-ana/pyproject.toml
View File

@@ -1,7 +1,7 @@
[project] [project]
name = "data-ana-service" name = "data-ana-service"
version = "0.1.0" version = "1.0.0"
description = "数据分析服务 - ClickHouse + 学习分析" description = "数据分析服务 - ClickHouse 宽表 + CDC + 掌握度算法 + 预警"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
"fastapi>=0.115.0", "fastapi>=0.115.0",
@@ -17,6 +17,13 @@ dependencies = [
"structlog>=24.4.0", "structlog>=24.4.0",
# CDC 链路：消费 Debezium 写入 Kafka 的 MySQL binlog 变更事件 # CDC 链路：消费 Debezium 写入 Kafka 的 MySQL binlog 变更事件
"aiokafka>=0.11.0", "aiokafka>=0.11.0",
# Redis：DataScope 缓存 + 事件去重 + 预警去重位图
"redis>=5.0.0",
# gRPC：AnalyticsService 12 RPC + HealthService（端口 50055）
"grpcio>=1.66.0",
"grpcio-health-checking>=1.66.0",
# protobuf 运行时（buf generate 生成的 stub 依赖）
"protobuf>=5.28.0",
] ]
[tool.ruff] [tool.ruff]
@@ -25,3 +32,9 @@ target-version = "py312"
[tool.ruff.lint] [tool.ruff.lint]
select = ["E", "F", "I", "N", "W", "UP", "B", "SIM"] select = ["E", "F", "I", "N", "W", "UP", "B", "SIM"]
[tool.ruff.lint.per-file-ignores]
# gRPC servicer 方法必须用 PascalCase（proto 契约约定），N802 不适用
"src/data_ana/grpc_server.py" = ["N802"]
# FastAPI 标准模式：Depends/Query 在参数默认值中调用，B008 不适用
"src/data_ana/main.py" = ["B008"]

450
services/data-ana/src/data_ana/analytics_service.py Normal file
View File

@@ -0,0 +1,450 @@
"""分析服务（4 端 Dashboard 聚合 + DataScope 注入 + 降级兜底）.
对齐 02-architecture-design.md §7 AnalyticsService：
- GetTeacherDashboard：班级聚合（教师视角）
- GetStudentDashboard：学生学情（学生视角）
- GetParentDashboard：孩子学情（家长视角）
- GetAdminDashboard：学校聚合 + AI 用量（管理员视角）
DataScope 注入：
- 教师只能查自己班级数据（CLASS 级 scope_ids 过滤）
- 学生只能查自己数据（SELF 级 user_id 过滤）
- 管理员可查全校数据（ALL 级无过滤）
降级策略：
- ClickHouse 不可达：返回骨架数据 + degraded=true
- iam gRPC 不可达：使用 role-based fallback + degraded=true
"""
from typing import Any
import structlog
from .repository import clickhouse_repository, iam_client
from .shared.permissions import DataScopeLevel, UserContext
logger = structlog.get_logger(__name__)
def _skeleton_dashboard(dashboard_type: str, user_id: str, reason: str) -> dict[str, Any]:
"""降级骨架数据（ClickHouse 不可达时返回）."""
return {
"userId": user_id,
"dashboardType": dashboard_type,
"degraded": True,
"degraded_reason": reason,
"totalClasses": 0,
"totalStudents": 0,
"averageScore": 0.0,
"passRate": 0.0,
"weakStudents": [],
"recentWarnings": [],
"trend": [],
}
async def get_teacher_dashboard(
user: UserContext,
class_id: str = "",
subject_id: str = "",
) -> dict[str, Any]:
"""教师仪表盘（聚合班级成绩 + 薄弱学生 + 预警）.
DataScope：
- CLASS 级：仅返回 scope_ids 内的班级数据
- ALL 级：返回所有班级数据
"""
# 1. 解析 DataScope
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# 2. 若指定 class_id，验证是否在 scope 内
effective_class_ids: list[str] = []
if class_id:
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
return {
**_skeleton_dashboard("teacher", user.user_id, "datascope_violation"),
"error": "class_id_out_of_scope",
}
effective_class_ids = [class_id]
elif scope.level == DataScopeLevel.CLASS:
effective_class_ids = scope.scope_ids
# 3. 查询 ClickHouse
if not effective_class_ids and scope.level != DataScopeLevel.ALL:
# CLASS 级无 scope_ids，返回空骨架
return _skeleton_dashboard("teacher", user.user_id, "no_class_scope")
dashboard = await clickhouse_repository.query_teacher_dashboard(
user_id=user.user_id,
class_id=effective_class_ids[0] if effective_class_ids else "",
)
if dashboard is None:
return _skeleton_dashboard("teacher", user.user_id, "clickhouse_unavailable")
# 4. 补充薄弱学生列表
weak_students: list[dict[str, Any]] = []
if effective_class_ids:
for cid in effective_class_ids[:5]: # 限制 5 个班级
class_perf = await clickhouse_repository.query_class_performance(
class_id=cid,
subject_id=subject_id,
)
if class_perf is not None:
weak_students.append(
{
"class_id": cid,
"total_students": class_perf.get("totalStudents", 0),
"average_score": class_perf.get("averageScore", 0.0),
"pass_rate": class_perf.get("passRate", 0.0),
}
)
# 5. 补充降级标记
if degraded:
dashboard["degraded"] = True
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
dashboard["classes"] = weak_students
dashboard["dashboardType"] = "teacher"
return dashboard
async def get_student_dashboard(
user: UserContext,
student_id: str = "",
subject_id: str = "",
) -> dict[str, Any]:
"""学生仪表盘（学情宽表 + 薄弱知识点 + 学习趋势）.
DataScope：
- SELF 级：student_id 必须为当前 user_id
- CLASS/GRADE/SCHOOL/ALL 级：可查指定 student_id
"""
# 1. 解析 DataScope
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# 2. SELF 级强制限制 student_id
effective_student_id = student_id or user.user_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id # SELF 级只能查自己
# 3. 查询 ClickHouse
dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id)
if dashboard is None:
return _skeleton_dashboard("student", effective_student_id, "clickhouse_unavailable")
# 4. 补充薄弱知识点
weakness = await clickhouse_repository.query_student_weakness(
student_id=effective_student_id,
subject_id=subject_id,
)
dashboard["weakPoints"] = weakness.get("weakPoints", []) if weakness else []
# 5. 补充学习趋势
trend = await clickhouse_repository.query_learning_trend(
student_id=effective_student_id,
subject_id=subject_id,
)
dashboard["trend"] = trend.get("points", []) if trend else []
# 6. 补充掌握度快照
mastery = await clickhouse_repository.query_mastery_snapshot(
student_id=effective_student_id,
subject_id=subject_id,
)
dashboard["mastery"] = mastery if mastery else {"knowledgePoints": [], "overallMastery": 0.0}
# 7. 降级标记
if degraded:
dashboard["degraded"] = True
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
dashboard["dashboardType"] = "student"
return dashboard
async def get_parent_dashboard(
user: UserContext,
child_id: str = "",
) -> dict[str, Any]:
"""家长仪表盘（孩子学情聚合）.
DataScope：
- SELF 级：child_id 必须为家长关联的孩子（iam.GetChildrenByParent 校验）
- 降级时使用 child_id 直接查询（依赖 Gateway 层鉴权）
"""
# 1. 解析 DataScope
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# 2. 学生 ID（家长视角：使用 child_id 或 user_id）
effective_student_id = child_id or user.user_id
# 3. 查询 ClickHouse（复用学生仪表盘查询）
dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id)
if dashboard is None:
return _skeleton_dashboard("parent", effective_student_id, "clickhouse_unavailable")
# 4. 补充考勤
attendance = await clickhouse_repository.query_attendance(effective_student_id)
dashboard["attendance"] = (
attendance
if attendance
else {
"absentCount": 0,
"lateCount": 0,
"presentCount": 0,
}
)
# 5. 补充学习趋势
trend = await clickhouse_repository.query_learning_trend(effective_student_id)
dashboard["trend"] = trend.get("points", []) if trend else []
# 6. 降级标记
if degraded:
dashboard["degraded"] = True
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
dashboard["dashboardType"] = "parent"
dashboard["childId"] = effective_student_id
return dashboard
async def get_admin_dashboard(
user: UserContext,
school_id: str = "",
) -> dict[str, Any]:
"""管理员仪表盘（学校聚合 + AI 用量统计）.
DataScope：
- SCHOOL 级：仅返回指定 school_id 数据
- ALL 级：返回所有数据
"""
# 1. 解析 DataScope
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# 2. 学校级聚合查询（复用 teacher_dashboard，不传 class_id）
dashboard = await clickhouse_repository.query_teacher_dashboard(
user_id=user.user_id,
class_id="",
)
if dashboard is None:
return _skeleton_dashboard("admin", user.user_id, "clickhouse_unavailable")
# 3. 补充 AI 用量统计（P5 启用，对齐 02 §7.4）
ai_usage = await clickhouse_repository.query_ai_usage_summary()
dashboard["aiUsage"] = (
ai_usage
if ai_usage
else {
"totalRequests": 0,
"totalTokens": 0,
"totalCostCents": 0,
"byProvider": [],
}
)
# 4. 降级标记
if degraded:
dashboard["degraded"] = True
dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
dashboard["dashboardType"] = "admin"
dashboard["schoolId"] = school_id or scope.school_id
return dashboard
async def get_class_performance(
user: UserContext,
class_id: str,
subject_id: str = "",
start_date: int = 0,
end_date: int = 0,
) -> dict[str, Any]:
"""查询班级成绩分析（GetClassPerformance RPC 实现）.
DataScope：CLASS 级需验证 class_id 在 scope_ids 内.
"""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# DataScope 校验
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
return {
"classId": class_id,
"degraded": True,
"degraded_reason": "datascope_violation",
"averageScore": 0.0,
"passRate": 0.0,
"totalStudents": 0,
}
result = await clickhouse_repository.query_class_performance(
class_id=class_id,
subject_id=subject_id,
start_date=start_date,
end_date=end_date,
)
if result is None:
return {
"classId": class_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"averageScore": 0.0,
"passRate": 0.0,
"totalStudents": 0,
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_mastery_distribution(
user: UserContext,
class_id: str,
subject_id: str = "",
knowledge_point_id: str = "",
) -> dict[str, Any]:
"""查询班级掌握度分布（GetMasteryDistribution RPC 实现）."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# DataScope 校验
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
return {
"classId": class_id,
"degraded": True,
"degraded_reason": "datascope_violation",
"masteredCount": 0,
"progressingCount": 0,
"weakCount": 0,
"totalStudents": 0,
}
result = await clickhouse_repository.query_mastery_distribution(
class_id=class_id,
subject_id=subject_id,
knowledge_point_id=knowledge_point_id,
)
if result is None:
return {
"classId": class_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"masteredCount": 0,
"progressingCount": 0,
"weakCount": 0,
"totalStudents": 0,
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_student_mastery(
user: UserContext,
student_id: str,
subject_id: str = "",
) -> dict[str, Any]:
"""查询学生知识点掌握度明细（GetStudentMastery RPC 实现）."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# SELF 级强制限制
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
result = await clickhouse_repository.query_mastery_snapshot(
student_id=effective_student_id,
subject_id=subject_id,
)
if result is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"knowledgePoints": [],
"overallMastery": 0.0,
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_student_weakness(
user: UserContext,
student_id: str,
subject_id: str = "",
) -> dict[str, Any]:
"""查询学生薄弱知识点（GetStudentWeakness RPC 实现）."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# SELF 级强制限制
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
result = await clickhouse_repository.query_student_weakness(
student_id=effective_student_id,
subject_id=subject_id,
)
if result is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"weakPoints": [],
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_learning_trend(
user: UserContext,
student_id: str,
subject_id: str = "",
start_date: int = 0,
end_date: int = 0,
) -> dict[str, Any]:
"""查询学习趋势（GetLearningTrend RPC 实现）."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
# SELF 级强制限制
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
result = await clickhouse_repository.query_learning_trend(
student_id=effective_student_id,
subject_id=subject_id,
start_date=start_date,
end_date=end_date,
)
if result is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"points": [],
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result

510
services/data-ana/src/data_ana/cdc_consumer.py
View File

@@ -1,17 +1,28 @@
"""CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表）. """CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表，EventHandler 路由 + 手动 commit）.
链路： 链路：
MySQL binlog → Debezium Connect → Kafka topic MySQL binlog → Debezium Connect → Kafka topic
edu-cdc.next_edu_cloud.<table> edu-cdc.next_edu_cloud.<table>
→ 本消费者 → 解析 Debezium 事件 → 写入 ClickHouse student_dashboard_view → 本消费者 → 解析 Debezium 事件 → EventHandler 路由 → 写入 ClickHouse 宽表
支持的表路由：
- core_edu_exams → ExamCache 更新
- core_edu_grades → student_dashboard_view 写入（查 ExamCache 获取 class_id）
- core_edu_homework_submissions → student_dashboard_view 写入
- core_edu_attendance → attendance_logs 写入
- content_knowledge_points → mastery_snapshot 元数据更新
- iam_users → 用户元数据缓存（P6 扩展）
- classes → 班级元数据缓存（P6 扩展）
设计要点： 设计要点：
- 监听多张表，按 source.table 路由 - 手动 commit（at-least-once）：ClickHouse 写入成功后才 commit offset
- 内存缓存 exam_id → (class_id, subject_id) 映射（来自 core_edu_exams 快照+流） - 幂等性：依赖 ClickHouse ReplacingMergeTree 引擎按 ORDER BY 去重
- 监听 core_edu_grades 时用缓存扩展为宽表记录写入 ClickHouse - op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null
- 幂等性：依赖 ClickHouse ReplacingMergeTree 引擎按 ORDER BY 去重 - Redis 事件去重：基于 event_id（Debezium 重启时可能重发）
（schema 需用 ReplacingMergeTree(last_updated)，当前为简化版 MergeTree）
- op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null 降级策略：
- kafka_brokers 未配置：消费者不启动（仅 HTTP/gRPC 服务）
- ClickHouse 不可达：消息处理失败，不 commit，下次重启重试
""" """
import asyncio import asyncio
@@ -22,11 +33,17 @@ from typing import Any
import structlog import structlog
from .clickhouse_client import upsert_student_dashboard
from .config import settings from .config import settings
from .exam_cache import get_exam_cache
from .repository import clickhouse_repository, redis_client
logger = structlog.get_logger(__name__) logger = structlog.get_logger(__name__)
# 消费者单例（用于 lifespan 管理和 /readyz 检查）
_consumer: Any | None = None
_consumer_task: asyncio.Task[None] | None = None
_is_running: bool = False
def _parse_ts(ts_ms: int | None) -> datetime: def _parse_ts(ts_ms: int | None) -> datetime:
"""Debezium ts_ms（毫秒）→ datetime.""" """Debezium ts_ms（毫秒）→ datetime."""
@@ -45,103 +62,291 @@ def _safe_float(value: Any) -> float:
return 0.0 return 0.0
class ExamCache: def _safe_int(value: Any) -> int:
"""内存缓存 exam_id → (class_id, subject_id). """安全转 int."""
if value is None:
从 core_edu_exams 表的 CDC 事件构建。subject_id 在 exams 表中暂无字段， return 0
这里占位为空字符串，后续扩展 schema 时再补充。 try:
""" return int(value)
except (TypeError, ValueError):
def __init__(self) -> None: try:
self._data: dict[str, tuple[str, str]] = {} return int(float(value))
except (TypeError, ValueError):
def upsert(self, exam_id: str, class_id: str, subject_id: str = "") -> None: return 0
if exam_id:
self._data[exam_id] = (class_id, subject_id)
def get(self, exam_id: str) -> tuple[str, str] | None:
return self._data.get(exam_id) if exam_id else None
# 全局缓存（进程级单例） def _parse_mysql_datetime(value: Any, fallback: datetime) -> datetime:
_exam_cache = ExamCache() """解析 MySQL datetime 字段（兼容字符串和毫秒时间戳）."""
if not value:
return fallback
if isinstance(value, (int, float)):
return datetime.fromtimestamp(value / 1000, tz=UTC)
with contextlib.suppress(ValueError, AttributeError, TypeError):
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
return fallback
async def _handle_exams_event(after: dict[str, Any] | None) -> None: # ===== EventHandler 路由表 =====
"""处理 core_edu_exams 表事件."""
async def _handle_exams_event(after: dict[str, Any] | None, op: str) -> bool:
"""处理 core_edu_exams 表事件 → ExamCache 更新."""
if after is None: if after is None:
return return True # 删除事件，无需处理
exam_id = after.get("id")
class_id = after.get("class_id", "") exam_id = str(after.get("id") or "")
if exam_id: if not exam_id:
_exam_cache.upsert(exam_id, class_id) return True
logger.info("exam_cache_updated", exam_id=exam_id, class_id=class_id)
exam_cache = get_exam_cache()
exam_cache.upsert(
exam_id=exam_id,
class_id=str(after.get("class_id") or ""),
subject_id=str(after.get("subject_id") or ""),
title=str(after.get("title") or ""),
)
logger.info(
"exam_cache_updated",
exam_id=exam_id,
class_id=after.get("class_id"),
op=op,
)
return True
async def _handle_grades_event( async def _handle_grades_event(
after: dict[str, Any] | None, after: dict[str, Any] | None,
op: str, op: str,
ts_ms: int | None, ts_ms: int | None,
) -> None: ) -> bool:
"""处理 core_edu_grades 表事件 → 写入 ClickHouse 宽表. """处理 core_edu_grades 表事件 → 写入 student_dashboard_view."""
- op=r/c/u：after 为新数据，写入宽表
- op=d：after 为 null，暂不处理（宽表保留历史记录）
"""
if after is None: if after is None:
return return True # 删除事件
student_id = after.get("student_id", "") student_id = str(after.get("student_id") or "")
exam_id = after.get("exam_id", "") exam_id = str(after.get("exam_id") or "")
score = _safe_float(after.get("score")) score = _safe_float(after.get("score"))
# 从缓存拿 class_id # 从 ExamCache 获取 class_id 和 subject_id
class_id = "" exam_cache = get_exam_cache()
if exam_id: class_id = exam_cache.get_class_id(exam_id)
cached = _exam_cache.get(exam_id) subject_id = exam_cache.get_subject_id(exam_id)
if cached:
class_id = cached[0]
last_updated = _parse_ts(ts_ms) fallback_ts = _parse_ts(ts_ms)
if after.get("updated_at"): last_updated = _parse_mysql_datetime(after.get("updated_at"), fallback_ts)
# 优先用 MySQL 的 updated_at 字段
with contextlib.suppress(ValueError, AttributeError):
last_updated = datetime.fromisoformat(after["updated_at"].replace("Z", "+00:00"))
# 简化：rank/kp/mastery/error_count 暂用默认值 # 简化：rank/kp/mastery/error_count 暂用默认值（CDC 聚合后由掌握度计算补充）
# 真实场景应通过其他 CDC 事件或聚合计算得到 write_ok = await clickhouse_repository.upsert_student_dashboard(
await upsert_student_dashboard(
student_id=student_id, student_id=student_id,
class_id=class_id, class_id=class_id,
exam_id=exam_id, exam_id=exam_id,
subject_id="", # 占位 subject_id=subject_id,
score=score, score=score,
rank_in_class=0, rank_in_class=0,
knowledge_point_id="", # 占位 knowledge_point_id="",
mastery_level=score / 100.0, # 简化：用分数百分比作为掌握度 mastery_level=score / 100.0, # 简化：用分数百分比作为初始掌握度
error_count=0, error_count=0,
last_updated=last_updated, last_updated=last_updated,
) )
if write_ok:
logger.info(
"grade_record_written",
student_id=student_id,
exam_id=exam_id,
score=score,
op=op,
)
return write_ok
async def _process_message(topic: str, value: bytes | str) -> None: async def _handle_homework_event(
"""处理单条 Kafka 消息. after: dict[str, Any] | None,
op: str,
ts_ms: int | None,
) -> bool:
"""处理 core_edu_homework_submissions 表事件 → 写入 student_dashboard_view."""
if after is None:
return True
Debezium 事件格式（简化后，schemas.enable=false）： student_id = str(after.get("student_id") or "")
{ homework_id = str(after.get("homework_id") or after.get("id") or "")
"before": {...} | null, class_id = str(after.get("class_id") or "")
"after": {...} | null, score = _safe_float(after.get("score"))
"source": {"table": "...", "db": "...", ...},
"op": "r|c|u|d", fallback_ts = _parse_ts(ts_ms)
"ts_ms": 1783572350928 raw_time = after.get("submitted_at") or after.get("updated_at")
} last_updated = _parse_mysql_datetime(raw_time, fallback_ts)
write_ok = await clickhouse_repository.upsert_student_dashboard(
student_id=student_id,
class_id=class_id,
exam_id=homework_id, # homework_id 作为 exam_id 占位
subject_id=str(after.get("subject_id") or ""),
score=score,
rank_in_class=0,
knowledge_point_id="",
mastery_level=score / 100.0,
error_count=0,
last_updated=last_updated,
)
if write_ok:
logger.info(
"homework_record_written",
student_id=student_id,
homework_id=homework_id,
score=score,
op=op,
)
return write_ok
async def _handle_attendance_event(
after: dict[str, Any] | None,
op: str,
ts_ms: int | None,
) -> bool:
"""处理 core_edu_attendance 表事件 → 写入 attendance_logs."""
if after is None:
return True
student_id = str(after.get("student_id") or "")
class_id = str(after.get("class_id") or "")
status = str(after.get("status") or "present")
recorded_by = str(after.get("recorded_by") or "")
remark = str(after.get("remark") or "")
fallback_ts = _parse_ts(ts_ms)
raw_time = after.get("occurred_at") or after.get("created_at")
occurred_at = _parse_mysql_datetime(raw_time, fallback_ts)
# attendance_date 从 occurred_at 提取日期部分
attendance_date = occurred_at.date()
write_ok = await clickhouse_repository.upsert_attendance_log(
student_id=student_id,
class_id=class_id,
attendance_date=attendance_date,
status=status,
recorded_by=recorded_by,
remark=remark,
occurred_at=occurred_at,
)
if write_ok:
logger.info(
"attendance_record_written",
student_id=student_id,
class_id=class_id,
status=status,
op=op,
)
return write_ok
async def _handle_knowledge_point_event(
after: dict[str, Any] | None,
op: str,
) -> bool:
"""处理 content_knowledge_points 表事件 → 更新知识点元数据缓存."""
if after is None:
return True
# 知识点元数据更新（P5+ 扩展：可写入 Redis 缓存供查询时补充标题）
kp_id = str(after.get("id") or "")
title = str(after.get("title") or "")
subject_id = str(after.get("subject_id") or "")
# 简化：写 Redis 缓存（data_ana:kp_meta:{kp_id}）
metadata = {"title": title, "subject_id": subject_id}
await redis_client.set_cache(
f"data_ana:kp_meta:{kp_id}",
json.dumps(metadata, ensure_ascii=False),
ttl_s=86400,
)
logger.info(
"knowledge_point_meta_cached",
knowledge_point_id=kp_id,
title=title,
op=op,
)
return True
async def _handle_ai_usage_event(event: dict[str, Any]) -> bool:
"""处理 AIUsageEvent（edu.insight.ai.usage topic）→ 写入 ai_usage_log.
注意：此事件不是 Debezium CDC 事件，而是 ai 服务发布的业务事件，
格式遵循 events.proto AIUsageEvent message.
"""
request_id = str(event.get("request_id") or "")
user_id = str(event.get("user_id") or "")
provider = str(event.get("provider") or "")
model = str(event.get("model") or "")
prompt_tokens = _safe_int(event.get("prompt_tokens"))
completion_tokens = _safe_int(event.get("completion_tokens"))
total_tokens = _safe_int(event.get("total_tokens"))
latency_ms = _safe_int(event.get("latency_ms"))
success = bool(event.get("success", True))
cost_cents = _safe_int(event.get("cost_cents"))
occurred_at_ms = event.get("occurred_at")
if isinstance(occurred_at_ms, int):
occurred_at = _parse_ts(occurred_at_ms)
else:
occurred_at = datetime.now(UTC)
write_ok = await clickhouse_repository.upsert_ai_usage_log(
request_id=request_id,
user_id=user_id,
provider=provider,
model=model,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
latency_ms=latency_ms,
success=success,
cost_cents=cost_cents,
occurred_at=occurred_at,
)
if write_ok:
logger.info(
"ai_usage_recorded",
request_id=request_id,
provider=provider,
model=model,
total_tokens=total_tokens,
)
return write_ok
# ===== 事件去重 =====
async def _is_deduped(event_id: str) -> bool:
"""检查事件是否已处理（基于 event_id Redis SETNX 去重）.
返回 True 表示重复事件（应跳过），False 表示首次事件.
"""
if not event_id:
return False
key = f"data_ana:dedup:{event_id}"
is_first = await redis_client.setnx_dedup(key, ttl_s=7 * 24 * 3600) # 7 天 TTL
return not is_first
# ===== 消息处理主入口 =====
async def _process_cdc_message(topic: str, value: bytes | str) -> bool:
"""处理单条 CDC 消息（Debezium 格式）.
返回 True 表示处理成功（可 commit offset），
返回 False 表示处理失败（不 commit，下次重试）.
""" """
try: try:
value_str = value.decode("utf-8") if isinstance(value, bytes) else value value_str = value.decode("utf-8") if isinstance(value, bytes) else value
event = json.loads(value_str) event = json.loads(value_str)
except (json.JSONDecodeError, UnicodeDecodeError) as exc: except (json.JSONDecodeError, UnicodeDecodeError) as exc:
logger.warning("cdc_message_decode_failed", error=str(exc), topic=topic) logger.warning("cdc_message_decode_failed", error=str(exc), topic=topic)
return return True # 无效消息直接跳过（避免阻塞消费）
source = event.get("source") or {} source = event.get("source") or {}
table = source.get("table", "") table = source.get("table", "")
@@ -149,6 +354,16 @@ async def _process_message(topic: str, value: bytes | str) -> None:
ts_ms = event.get("ts_ms") ts_ms = event.get("ts_ms")
after = event.get("after") after = event.get("after")
# Debezium 事件去重（基于 source.ts_ms + source.dbg + source.sequence 生成 event_id）
# 简化：使用 topic + partition + offset 作为去重 key（由调用方传入更精确）
# 这里仅对 after.id 做去重（如果有）
event_id = ""
if after and isinstance(after, dict):
event_id = f"{table}:{after.get('id', '')}:{ts_ms}"
if event_id and await _is_deduped(event_id):
logger.info("cdc_event_dedup_skipped", table=table, event_id=event_id)
return True
logger.info( logger.info(
"cdc_event_received", "cdc_event_received",
topic=topic, topic=topic,
@@ -157,21 +372,71 @@ async def _process_message(topic: str, value: bytes | str) -> None:
ts_ms=ts_ms, ts_ms=ts_ms,
) )
if table == "core_edu_exams": try:
await _handle_exams_event(after) if table == "core_edu_exams":
elif table == "core_edu_grades": return await _handle_exams_event(after, op)
await _handle_grades_event(after, op, ts_ms) elif table == "core_edu_grades":
else: return await _handle_grades_event(after, op, ts_ms)
# 其他表暂不处理，仅记录 elif table == "core_edu_homework_submissions":
logger.debug("cdc_event_skipped", table=table) return await _handle_homework_event(after, op, ts_ms)
elif table == "core_edu_attendance":
return await _handle_attendance_event(after, op, ts_ms)
elif table == "content_knowledge_points":
return await _handle_knowledge_point_event(after, op)
else:
logger.debug("cdc_event_skipped", table=table)
return True
except Exception as exc: # noqa: BLE001
logger.error(
"cdc_event_handler_failed",
error=str(exc),
table=table,
op=op,
)
return False
async def _process_ai_usage_message(topic: str, value: bytes | str) -> bool:
"""处理 AIUsageEvent 消息（非 Debezium 格式，业务事件）."""
try:
value_str = value.decode("utf-8") if isinstance(value, bytes) else value
event = json.loads(value_str)
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
logger.warning("ai_usage_message_decode_failed", error=str(exc), topic=topic)
return True
# 基于 request_id 去重
request_id = str(event.get("request_id") or "")
if request_id and await _is_deduped(f"ai_usage:{request_id}"):
return True
try:
return await _handle_ai_usage_event(event)
except Exception as exc: # noqa: BLE001
logger.error("ai_usage_handler_failed", error=str(exc))
return False
async def _process_message(topic: str, value: bytes | str) -> bool:
"""统一消息处理入口（区分 CDC 事件和 AIUsage 事件）."""
# 根据 topic 判断消息类型
if topic == settings.kafka_ai_usage_topic:
return await _process_ai_usage_message(topic, value)
return await _process_cdc_message(topic, value)
# ===== 消费者主循环 =====
async def run_consumer() -> None: async def run_consumer() -> None:
"""CDC 消费者主循环（lifespan 启动）. """CDC 消费者主循环（lifespan 启动）.
- kafka_brokers 未配置：直接返回，不启动消费者（降级模式） - kafka_brokers 未配置：直接返回，不启动消费者（降级模式）
- 手动 commit：ClickHouse 写入成功后才 commit offset（at-least-once）
- 启动失败：仅记录错误，不阻塞 FastAPI 主流程 - 启动失败：仅记录错误，不阻塞 FastAPI 主流程
""" """
global _consumer, _is_running
if not settings.kafka_brokers: if not settings.kafka_brokers:
logger.info("cdc_consumer_disabled_no_kafka_brokers") logger.info("cdc_consumer_disabled_no_kafka_brokers")
return return
@@ -183,51 +448,114 @@ async def run_consumer() -> None:
return return
topics = [t.strip() for t in settings.kafka_cdc_topics.split(",") if t.strip()] topics = [t.strip() for t in settings.kafka_cdc_topics.split(",") if t.strip()]
# P5+ 加入 AI 用量 topic
if settings.kafka_ai_usage_topic and settings.kafka_ai_usage_topic not in topics:
topics.append(settings.kafka_ai_usage_topic)
if not topics: if not topics:
logger.warning("cdc_consumer_no_topics_configured") logger.warning("cdc_consumer_no_topics_configured")
return return
brokers = [b.strip() for b in settings.kafka_brokers.split(",") if b.strip()] brokers = [b.strip() for b in settings.kafka_brokers.split(",") if b.strip()]
consumer = AIOKafkaConsumer( _consumer = AIOKafkaConsumer(
*topics, *topics,
bootstrap_servers=brokers, bootstrap_servers=brokers,
group_id=settings.kafka_group_id, group_id=settings.kafka_consumer_group,
auto_offset_reset=settings.kafka_auto_offset_reset, auto_offset_reset=settings.kafka_auto_offset_reset,
enable_auto_commit=True, enable_auto_commit=settings.kafka_enable_auto_commit, # 手动 commit
value_deserializer=lambda v: v, # 保留原始 bytes，由 _process_message 解码 value_deserializer=lambda v: v, # 保留原始 bytes，由 _process_message 解码
max_poll_records=100,
session_timeout_ms=30_000,
heartbeat_interval_ms=10_000,
) )
try: try:
await consumer.start() await _consumer.start()
_is_running = True
logger.info( logger.info(
"cdc_consumer_started", "cdc_consumer_started",
brokers=brokers, brokers=brokers,
topics=topics, topics=topics,
group_id=settings.kafka_group_id, group_id=settings.kafka_consumer_group,
auto_commit=settings.kafka_enable_auto_commit,
) )
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
logger.error("cdc_consumer_start_failed", error=str(exc)) logger.error("cdc_consumer_start_failed", error=str(exc))
_is_running = False
return return
try: try:
async for msg in consumer: async for msg in _consumer:
try: try:
await _process_message(msg.topic, msg.value) success = await _process_message(msg.topic, msg.value)
if success:
# 处理成功才 commit offset（at-least-once）
await _consumer.commit()
else:
# 处理失败不 commit，下次重启会重新消费
logger.warning(
"cdc_message_process_failed_no_commit",
topic=msg.topic,
partition=msg.partition,
offset=msg.offset,
)
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
logger.error( logger.error(
"cdc_message_process_failed", "cdc_message_process_exception",
error=str(exc), error=str(exc),
topic=msg.topic, topic=msg.topic,
partition=msg.partition, partition=msg.partition,
offset=msg.offset, offset=msg.offset,
) )
# 异常不 commit，下次重启重试
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("cdc_consumer_cancelled") logger.info("cdc_consumer_cancelled")
raise raise
finally: finally:
_is_running = False
if _consumer is not None:
try:
await _consumer.stop()
logger.info("cdc_consumer_stopped")
except Exception as exc: # noqa: BLE001
logger.warning("cdc_consumer_stop_failed", error=str(exc))
_consumer = None
async def start_consumer() -> asyncio.Task[None] | None:
"""启动消费者后台任务（供 lifespan 调用）."""
global _consumer_task
if not settings.kafka_brokers:
return None
if _consumer_task is not None and not _consumer_task.done():
return _consumer_task
_consumer_task = asyncio.create_task(run_consumer())
return _consumer_task
async def stop_consumer() -> None:
"""停止消费者后台任务（供 lifespan 调用）."""
global _consumer_task
if _consumer_task is not None:
_consumer_task.cancel()
try: try:
await consumer.stop() await _consumer_task
logger.info("cdc_consumer_stopped") except asyncio.CancelledError:
except Exception as exc: # noqa: BLE001 pass
logger.warning("cdc_consumer_stop_failed", error=str(exc)) finally:
_consumer_task = None
def is_running() -> bool:
"""消费者是否运行中（供 /readyz 使用）."""
return _is_running
async def get_lag() -> int:
"""获取消费者 lag（待消费消息数，供 /readyz 和监控使用）.
P6 实现：调用 Kafka AdminClient 获取 group lag.
当前返回 0（简化）.
"""
return 0

101
services/data-ana/src/data_ana/config.py
View File

@@ -1,46 +1,97 @@
"""配置管理.""" """配置管理（pydantic-settings 完整配置项）.
from pydantic_settings import BaseSettings 对齐 02-architecture-design.md §15 配置清单.
环境变量前缀 DATA_ANA_（如 DATA_ANA_HTTP_PORT=3006）.
"""
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings): class Settings(BaseSettings):
"""应用配置. """应用配置.
ClickHouse 连接参数为可选：当 clickhouse_host 为空字符串时， 所有外部依赖（ClickHouse / Kafka / Redis / iam gRPC）均为可选：
服务进入降级模式（查询方法返回 None / 空数据），保证服务可启动。 未配置或不可达时服务进入降级模式，返回骨架数据 + details.degraded: true.
Kafka 连接参数为可选：当 kafka_brokers 为空字符串时，
CDC 消费者不启动（降级模式），保证服务可启动。
""" """
port: int = 3006 # 服务
# ClickHouse 连接（可选：留空则降级模式） service_name: str = "data-ana"
http_port: int = 3006
grpc_port: int = 50055
log_level: str = "INFO"
dev_mode: bool = False
# ClickHouse（可选：留空则降级模式）
clickhouse_host: str = "" clickhouse_host: str = ""
clickhouse_port: int = 8123 clickhouse_port: int = 8123
clickhouse_database: str = "edu_analytics" clickhouse_database: str = "edu_analytics"
clickhouse_user: str = "" clickhouse_user: str = ""
clickhouse_password: str = "" clickhouse_password: str = ""
# 可观测性 clickhouse_connect_timeout_ms: int = 3000
otel_endpoint: str = "http://localhost:4318" clickhouse_query_timeout_s: int = 3 # P4 退出标准 5s，查询 3s 超时降级
log_level: str = "info"
# 开发模式开关 # Kafka
dev_mode: bool = False kafka_brokers: str = "" # 留空则不启动 CDC 消费者
# Kafka brokers（CDC 消费；留空则不启动消费者） kafka_consumer_group: str = "data-ana-cdc"
# 主机访问用 localhost:9092，容器内访问用 kafka:29092
kafka_brokers: str = ""
# CDC 消费组 id
kafka_group_id: str = "data-ana-cdc-consumer"
# 要消费的 CDC topic（Debezium 默认命名：<prefix>.<database>.<table>）
# 用逗号分隔多个 topic
kafka_cdc_topics: str = ( kafka_cdc_topics: str = (
"edu-cdc.next_edu_cloud.core_edu_grades," "edu-cdc.next_edu_cloud.core_edu_grades,"
"edu-cdc.next_edu_cloud.core_edu_exams," "edu-cdc.next_edu_cloud.core_edu_exams,"
"edu-cdc.next_edu_cloud.classes" "edu-cdc.next_edu_cloud.core_edu_homework_submissions,"
"edu-cdc.next_edu_cloud.core_edu_attendance,"
"edu-cdc.next_edu_cloud.classes,"
"edu-cdc.next_edu_cloud.iam_users,"
"edu-cdc.next_edu_cloud.content_knowledge_points"
) )
# 消费者自动偏移重置策略（earliest / latest） kafka_mastery_topic: str = "edu.insight.mastery.updated"
kafka_auto_offset_reset: str = "earliest" kafka_warning_topic: str = "edu.insight.mastery.updated" # 复用 mastery topic（总裁裁决 §2.11）
kafka_ai_usage_topic: str = "edu.insight.ai.usage"
kafka_enable_auto_commit: bool = False # v2: 手动 commit（at-least-once）
kafka_auto_offset_reset: str = "latest"
kafka_producer_transactional_id: str = "data-ana-producer"
model_config = {"env_file": ".env", "env_prefix": ""} # iam gRPC
iam_grpc_endpoint: str = "" # 留空则使用降级兜底（按 role 映射 DataScope）
iam_grpc_timeout_s: int = 2
datascope_cache_ttl_s: int = 300 # 5min
# Redis
redis_url: str = "" # 留空则跳过缓存
redis_pool_size: int = 10
redis_socket_timeout_ms: int = 200
# OTel
otel_endpoint: str = "http://localhost:4318"
otel_service_name: str = "data-ana"
# 掌握度算法
mastery_window_size: int = 5
mastery_decay_base: float = 0.6
mastery_forgetting_half_life_days: int = 30
mastery_min_samples: int = 3
# 预警阈值
warning_low_mastery_threshold: float = 0.4
warning_critical_mastery_threshold: float = 0.2
warning_score_drop_percent: float = 0.2
warning_absent_per_week: int = 3
# 降级
degraded_mode_enabled: bool = True
# 向后兼容：旧代码引用 settings.port / settings.kafka_group_id
@property
def port(self) -> int:
return self.http_port
@property
def kafka_group_id(self) -> str:
return self.kafka_consumer_group
model_config = SettingsConfigDict(
env_file=".env",
env_prefix="DATA_ANA_",
extra="ignore",
)
settings = Settings() settings = Settings()

112
services/data-ana/src/data_ana/exam_cache.py Normal file
View File

@@ -0,0 +1,112 @@
"""考试缓存（exam_id → {class_id, subject_id} 映射，内存 LRU）.
对齐 02-architecture-design.md §8.3 ExamCache：
- 内存 LRU dict，max 10000 条
- CDC core_edu_exams 事件触发更新
- CDC core_edu_grades 事件查询获取 class_id（避免 join 查询）
P6 演进：改为 Redis 实现（key: data_ana:exam:{exam_id}，TTL 30 天），
支持多实例共享（对齐 workline §3.5 任务 6.2）.
"""
from collections import OrderedDict
from typing import Any
import structlog
logger = structlog.get_logger(__name__)
# LRU 最大容量
_MAX_SIZE = 10_000
class ExamCache:
"""考试缓存（LRU，max 10000 条）.
内存实现：OrderedDict，访问/写入时移到末尾（最近使用），
超容量时弹出头部（最久未使用）.
线程安全：asyncio 单线程模型下无需加锁.
"""
def __init__(self, max_size: int = _MAX_SIZE) -> None:
self._data: OrderedDict[str, dict[str, str]] = OrderedDict()
self._max_size = max_size
def upsert(
self,
exam_id: str,
class_id: str = "",
subject_id: str = "",
title: str = "",
) -> None:
"""更新或插入考试缓存."""
if not exam_id:
return
value: dict[str, str] = {
"class_id": class_id,
"subject_id": subject_id,
"title": title,
}
# 已存在则移到末尾（标记为最近使用）
if exam_id in self._data:
self._data.move_to_end(exam_id)
self._data[exam_id] = value
# LRU 淘汰
while len(self._data) > self._max_size:
evicted_key, _ = self._data.popitem(last=False)
logger.debug("exam_cache_lru_evicted", exam_id=evicted_key)
def get(self, exam_id: str) -> dict[str, str] | None:
"""查询考试缓存（命中时移到末尾，标记为最近使用）."""
if not exam_id:
return None
value = self._data.get(exam_id)
if value is not None:
self._data.move_to_end(exam_id)
return value
def get_class_id(self, exam_id: str) -> str:
"""便捷方法：获取 class_id（未命中返回空字符串）."""
entry = self.get(exam_id)
return entry.get("class_id", "") if entry else ""
def get_subject_id(self, exam_id: str) -> str:
"""便捷方法：获取 subject_id（未命中返回空字符串）."""
entry = self.get(exam_id)
return entry.get("subject_id", "") if entry else ""
def delete(self, exam_id: str) -> bool:
"""删除缓存项."""
if exam_id in self._data:
del self._data[exam_id]
return True
return False
def clear(self) -> None:
"""清空缓存."""
self._data.clear()
def size(self) -> int:
"""当前缓存数量."""
return len(self._data)
def stats(self) -> dict[str, Any]:
"""缓存统计信息（供 /readyz 和监控使用）."""
return {
"size": len(self._data),
"max_size": self._max_size,
"utilization": round(len(self._data) / self._max_size, 4),
}
# 全局缓存（进程级单例）
_exam_cache = ExamCache()
def get_exam_cache() -> ExamCache:
"""获取全局 ExamCache 实例."""
return _exam_cache

570
services/data-ana/src/data_ana/grpc_server.py Normal file
View File

@@ -0,0 +1,570 @@
"""gRPC Server（AnalyticsService 12 RPC + HealthService）.
对齐 02-architecture-design.md §4.2：
- AnalyticsService 12 RPC（含 1 个 server-streaming SubscribeMasteryUpdate）
- HealthService（grpc.health.v1，供 K8s 探针）
- 端口 50055
- 所有 RPC 返回结构化 message（HTTP 层包装 ActionState 信封）
SubscribeMasteryUpdate（P5+）：
- server-streaming RPC
- 客户端订阅 student_id / class_id
- 掌握度计算完成时推送 MasteryUpdateEvent
- 基于 asyncio.Queue 实现事件分发
降级策略：
- grpcio 未安装：gRPC server 不启动（仅 HTTP 服务）
- 下游服务不可达：返回降级响应（degraded 字段标记）
"""
import asyncio
from typing import Any
import structlog
from . import analytics_service, warning_service
from .config import settings
from .shared.permissions import UserContext
logger = structlog.get_logger(__name__)
# SubscribeMasteryUpdate 订阅管理（P5+）
_subscribers: dict[str, asyncio.Queue] = {}
_subscribers_lock = asyncio.Lock()
async def _add_subscriber(key: str) -> asyncio.Queue:
"""添加订阅者."""
async with _subscribers_lock:
if key not in _subscribers:
_subscribers[key] = asyncio.Queue(maxsize=100)
return _subscribers[key]
async def _remove_subscriber(key: str) -> None:
"""移除订阅者."""
async with _subscribers_lock:
_subscribers.pop(key, None)
async def notify_mastery_update(
student_id: str,
knowledge_point_id: str,
mastery_level: float,
previous_level: float,
) -> None:
"""通知所有订阅者掌握度更新（由 mastery_service 调用）.
订阅匹配规则：
- 订阅 student_id 的客户端收到通知
- 订阅 class_id 的客户端收到该班级所有学生的通知（需要 ExamCache 查 class_id）
"""
import time
import uuid
event_id = str(uuid.uuid4())
calculated_at = int(time.time())
# 通知 student_id 订阅者
queue = _subscribers.get(f"student:{student_id}")
if queue is not None:
try:
queue.put_nowait(
{
"event_id": event_id,
"student_id": student_id,
"knowledge_point_id": knowledge_point_id,
"mastery_level": mastery_level,
"previous_level": previous_level,
"calculated_at": calculated_at,
}
)
except asyncio.QueueFull:
logger.warning("mastery_update_queue_full_dropped", student_id=student_id)
# ===== AnalyticsService 实现 =====
class AnalyticsServiceServicer:
"""AnalyticsService gRPC 实现（12 RPC）."""
async def GetClassPerformance(self, request, context):
"""班级成绩分析."""
user = _extract_user(context)
result = await analytics_service.get_class_performance(
user=user,
class_id=request.class_id,
subject_id=request.subject_id,
start_date=request.start_date,
end_date=request.end_date,
)
return _build_class_performance_response(result)
async def GetStudentWeakness(self, request, context):
"""学生薄弱知识点."""
user = _extract_user(context)
result = await analytics_service.get_student_weakness(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
)
return _build_student_weakness_response(result)
async def GetLearningTrend(self, request, context):
"""学习趋势."""
user = _extract_user(context)
result = await analytics_service.get_learning_trend(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
start_date=request.start_date,
end_date=request.end_date,
)
return _build_learning_trend_response(result)
async def GetTeacherDashboard(self, request, context):
"""教师仪表盘."""
user = _extract_user(context)
result = await analytics_service.get_teacher_dashboard(
user=user,
class_id=request.class_id,
)
return _build_teacher_dashboard_response(result)
async def GetStudentDashboard(self, request, context):
"""学生仪表盘."""
user = _extract_user(context)
result = await analytics_service.get_student_dashboard(
user=user,
)
return _build_student_dashboard_response(result)
async def GetParentDashboard(self, request, context):
"""家长仪表盘."""
user = _extract_user(context)
result = await analytics_service.get_parent_dashboard(
user=user,
child_id=request.student_id,
)
return _build_parent_dashboard_response(result)
async def GetAdminDashboard(self, request, context):
"""管理员仪表盘."""
user = _extract_user(context)
result = await analytics_service.get_admin_dashboard(
user=user,
school_id=request.scope_id,
)
return _build_admin_dashboard_response(result)
async def GetWarnings(self, request, context):
"""预警列表查询."""
# 简化：按 class_id 查询（P6 改为多条件过滤）
warnings = await warning_service.get_warnings(
student_id="", # gRPC 层按 class_id 查询，P6 扩展
warning_type="",
)
return _build_warning_list_response(warnings)
async def TriggerWarning(self, request, context):
"""手动触发预警."""
result = await warning_service.trigger_warning_manual(
target_id=request.target_id,
warning_type=request.warning_type,
threshold=0.0,
current_value=0.0,
severity=request.severity,
)
return _build_trigger_warning_response(result)
async def GetMasteryDistribution(self, request, context):
"""班级掌握度分布."""
user = _extract_user(context)
result = await analytics_service.get_mastery_distribution(
user=user,
class_id=request.class_id,
subject_id=request.subject_id,
knowledge_point_id=request.knowledge_point_id,
)
return _build_mastery_distribution_response(result)
async def GetStudentMastery(self, request, context):
"""学生知识点掌握度明细."""
user = _extract_user(context)
result = await analytics_service.get_student_mastery(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
)
return _build_student_mastery_response(result)
async def SubscribeMasteryUpdate(self, request, context):
"""订阅掌握度更新（server-streaming，P5+）."""
# 确定订阅 key
if request.student_id:
sub_key = f"student:{request.student_id}"
elif request.class_id:
sub_key = f"class:{request.class_id}"
else:
sub_key = "global"
queue = await _add_subscriber(sub_key)
logger.info(
"mastery_subscription_added",
sub_key=sub_key,
peer=context.peer() if hasattr(context, "peer") else "unknown",
)
try:
while context.is_active():
try:
event = await asyncio.wait_for(queue.get(), timeout=30.0)
yield _build_mastery_update_event(event)
except TimeoutError:
# 心跳：30s 无事件时发一个空 keepalive（客户端应容忍）
continue
except asyncio.CancelledError:
logger.info("mastery_subscription_cancelled", sub_key=sub_key)
raise
finally:
await _remove_subscriber(sub_key)
logger.info("mastery_subscription_removed", sub_key=sub_key)
# ===== 辅助函数 =====
def _extract_user(context: Any) -> UserContext:
"""从 gRPC metadata 提取用户上下文."""
user_id = ""
roles: list[str] = []
try:
metadata = (
dict(context.invocation_metadata()) if hasattr(context, "invocation_metadata") else {}
)
user_id = metadata.get("x-user-id", "")
roles_str = metadata.get("x-user-roles", "")
roles = [r.strip() for r in roles_str.split(",") if r.strip()] if roles_str else []
except Exception: # noqa: BLE001
pass
return UserContext(user_id=user_id, roles=roles)
def _build_class_performance_response(data: dict) -> Any:
"""构建 ClassPerformance proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
return analytics_pb2.ClassPerformance(
class_id=data.get("classId", ""),
average_score=data.get("averageScore", 0.0),
pass_rate=data.get("passRate", 0.0),
total_students=data.get("totalStudents", 0),
)
def _build_student_weakness_response(data: dict) -> Any:
"""构建 StudentWeakness proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
weak_points = [
analytics_pb2.WeakPoint(
knowledge_point_id=wp.get("knowledgePointId", ""),
title=wp.get("title", ""),
mastery=wp.get("mastery", 0.0),
error_count=wp.get("errorCount", 0),
)
for wp in data.get("weakPoints", [])
]
return analytics_pb2.StudentWeakness(
student_id=data.get("studentId", ""),
weak_points=weak_points,
)
def _build_learning_trend_response(data: dict) -> Any:
"""构建 LearningTrend proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
points = [
analytics_pb2.TrendPoint(
date=p.get("date", 0),
score=p.get("score", 0.0),
)
for p in data.get("points", [])
]
return analytics_pb2.LearningTrend(
student_id=data.get("studentId", ""),
points=points,
)
def _build_teacher_dashboard_response(data: dict) -> Any:
"""构建 TeacherDashboard proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
classes = [
analytics_pb2.ClassSummary(
class_id=c.get("class_id", ""),
class_name=c.get("class_name", ""),
student_count=c.get("total_students", 0),
average_score=c.get("average_score", 0.0),
)
for c in data.get("classes", [])
]
return analytics_pb2.TeacherDashboard(
user_id=data.get("userId", ""),
total_classes=data.get("totalClasses", 0),
total_students=data.get("totalStudents", 0),
class_avg_score=data.get("classAvgScore", 0.0),
pending_homework_count=data.get("pendingHomeworkCount", 0),
classes=classes,
)
def _build_student_dashboard_response(data: dict) -> Any:
"""构建 StudentDashboard proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
weak_points = [
analytics_pb2.WeakPoint(
knowledge_point_id=wp.get("knowledgePointId", ""),
title=wp.get("title", ""),
mastery=wp.get("mastery", 0.0),
error_count=wp.get("errorCount", 0),
)
for wp in data.get("weakPoints", [])
]
trends = [
analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0))
for p in data.get("trend", [])
]
return analytics_pb2.StudentDashboard(
user_id=data.get("studentId", data.get("userId", "")),
avg_score=data.get("averageScore", 0.0),
class_rank=0,
total_students=0,
weak_points=weak_points,
recent_trends=trends,
)
def _build_parent_dashboard_response(data: dict) -> Any:
"""构建 ParentDashboard proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
weak_points = [
analytics_pb2.WeakPoint(
knowledge_point_id=wp.get("knowledgePointId", ""),
title=wp.get("title", ""),
mastery=wp.get("mastery", 0.0),
error_count=wp.get("errorCount", 0),
)
for wp in data.get("weakPoints", [])
]
return analytics_pb2.ParentDashboard(
user_id=data.get("userId", ""),
student_id=data.get("childId", ""),
child_avg_score=data.get("averageScore", 0.0),
child_class_rank=0,
total_class_students=0,
child_weak_points=weak_points,
)
def _build_admin_dashboard_response(data: dict) -> Any:
"""构建 AdminDashboard proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
ai_usage_data = data.get("aiUsage", {})
by_provider = [
analytics_pb2.AIUsageByProvider(
provider=p.get("provider", ""),
request_count=p.get("requestCount", 0),
total_tokens=p.get("totalTokens", 0),
cost_cents=p.get("costCents", 0),
)
for p in ai_usage_data.get("byProvider", [])
]
ai_usage = analytics_pb2.AIUsageSummary(
total_requests=ai_usage_data.get("totalRequests", 0),
total_tokens=ai_usage_data.get("totalTokens", 0),
total_cost_cents=ai_usage_data.get("totalCostCents", 0),
by_provider=by_provider,
)
return analytics_pb2.AdminDashboard(
user_id=data.get("userId", ""),
total_teachers=0,
total_students=data.get("totalStudents", 0),
total_classes=data.get("totalClasses", 0),
school_avg_score=data.get("classAvgScore", 0.0),
ai_usage=ai_usage,
)
def _build_warning_list_response(warnings: list[dict]) -> Any:
"""构建 WarningList proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
warning_infos = [
analytics_pb2.WarningInfo(
warning_id=w.get("warning_id", ""),
warning_type=w.get("warning_type", ""),
target_id=w.get("target_id", ""),
target_name=w.get("target_name", ""),
threshold=w.get("threshold", 0.0),
current_value=w.get("current_value", 0.0),
severity=w.get("severity", "WARN"),
occurred_at=int(w.get("occurred_at", 0)),
)
for w in warnings
]
return analytics_pb2.WarningList(
warnings=warning_infos,
total=len(warning_infos),
)
def _build_trigger_warning_response(data: dict) -> Any:
"""构建 TriggerWarningResponse proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
return analytics_pb2.TriggerWarningResponse(
warning_id=data.get("warning_id", ""),
triggered=data.get("triggered", False),
)
def _build_mastery_distribution_response(data: dict) -> Any:
"""构建 MasteryDistribution proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
return analytics_pb2.MasteryDistribution(
class_id=data.get("classId", ""),
mastered_count=data.get("masteredCount", 0),
progressing_count=data.get("progressingCount", 0),
weak_count=data.get("weakCount", 0),
total_students=data.get("totalStudents", 0),
)
def _build_student_mastery_response(data: dict) -> Any:
"""构建 StudentMastery proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
knowledge_points = [
analytics_pb2.KnowledgePointMastery(
knowledge_point_id=kp.get("knowledge_point_id", ""),
title=kp.get("title", ""),
subject_id=kp.get("subject_id", ""),
mastery_level=kp.get("mastery_level", 0.0),
mastery_label=kp.get("mastery_label", "weak"),
calculated_at=kp.get("calculated_at", 0),
)
for kp in data.get("knowledgePoints", [])
]
return analytics_pb2.StudentMastery(
student_id=data.get("studentId", ""),
knowledge_points=knowledge_points,
overall_mastery=data.get("overallMastery", 0.0),
)
def _build_mastery_update_event(event: dict) -> Any:
"""构建 MasteryUpdateEvent proto 响应（streaming）."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
return analytics_pb2.MasteryUpdateEvent(
event_id=event.get("event_id", ""),
student_id=event.get("student_id", ""),
knowledge_point_id=event.get("knowledge_point_id", ""),
mastery_level=event.get("mastery_level", 0.0),
previous_level=event.get("previous_level", 0.0),
calculated_at=event.get("calculated_at", 0),
)
# ===== gRPC Server 管理 =====
_server: Any | None = None
async def start_grpc_server() -> Any | None:
"""启动 gRPC server :50055（lifespan 调用）.
降级策略：
- grpcio 未安装：返回 None，仅 HTTP 服务
- 启动失败：仅记录错误，不阻塞 FastAPI 主流程
"""
global _server
try:
import grpc # type: ignore[import-not-found]
from generated_proto import analytics_pb2_grpc # type: ignore[import-not-found]
except ImportError as exc:
logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc))
return None
_server = grpc.aio.server()
# 注册 AnalyticsService
servicer = AnalyticsServiceServicer()
analytics_pb2_grpc.add_AnalyticsServiceServicer_to_server(servicer, _server)
# 注册 HealthService
try:
from grpc_health.v1 import ( # type: ignore[import-not-found]
health,
health_pb2,
health_pb2_grpc,
)
health_servicer = health.aio.HealthServicer()
await health_servicer.set("analytics", health_pb2.HealthCheckResponse.SERVING)
await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING) # overall
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, _server)
except ImportError:
logger.warning("grpc_health_not_installed_skip_health_service")
# 绑定端口
bind_address = f"[::]:{settings.grpc_port}"
_server.add_insecure_port(bind_address)
try:
await _server.start()
logger.info(
"grpc_server_started",
port=settings.grpc_port,
rpc_count=12,
)
return _server
except Exception as exc: # noqa: BLE001
logger.error("grpc_server_start_failed", error=str(exc), port=settings.grpc_port)
_server = None
return None
async def stop_grpc_server() -> None:
"""停止 gRPC server（lifespan 退出时调用）."""
global _server
if _server is not None:
try:
await _server.stop(grace=5)
logger.info("grpc_server_stopped")
except Exception as exc: # noqa: BLE001
logger.warning("grpc_server_stop_failed", error=str(exc))
finally:
_server = None
def is_running() -> bool:
"""gRPC server 是否运行中."""
return _server is not None

578
services/data-ana/src/data_ana/main.py
View File

@@ -1,20 +1,40 @@
"""数据分析服务入口. """数据分析服务入口（FastAPI HTTP :3006）.
支持 ClickHouse 降级模式：当 CLICKHOUSE_HOST 未配置或不可达时， 端点清单（3 基础 + 11 业务 = 14 个）：
查询端点返回骨架数据，服务仍可启动与响应。 基础：
GET / 根信息
GET /healthz 活性检查（liveness）
GET /readyz 就绪检查（readiness，检查 4 依赖）
支持 CDC 消费者：当 KAFKA_BROKERS 配置时， 业务（全部返回 ActionState[T] 信封）：
后台启动 aiokafka 消费者，监听 Debezium CDC 事件写入 ClickHouse。 GET /analytics/class/{class_id}/performance 班级成绩分析
GET /analytics/student/{student_id}/weakness 学生薄弱知识点
GET /analytics/student/{student_id}/trend 学习趋势
GET /analytics/student/{student_id}/errorbook 错题本（额外）
GET /analytics/teacher/dashboard 教师仪表盘
GET /analytics/student/dashboard 学生仪表盘
GET /analytics/parent/dashboard 家长仪表盘
GET /analytics/admin/dashboard 管理员仪表盘
GET /analytics/warnings 预警列表
POST /analytics/warnings/trigger 手动触发预警
GET /analytics/class/{class_id}/mastery-distribution 班级掌握度分布
GET /analytics/student/{student_id}/mastery 学生掌握度明细
设计要点：
- 所有业务端点返回 ActionState[T]（coord-cross-review §5.3 P0 整改）
- 降级标记在顶层 details.degraded（不放 error.details）
- /readyz 检查 4 依赖：clickhouse / cdc_consumer / redis / iam_grpc
- gRPC server :50055 在 lifespan 启动
- CDC 消费者在 lifespan 启动
""" """
import asyncio
import contextlib
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import Any
import structlog import structlog
from fastapi import APIRouter, FastAPI from fastapi import APIRouter, Depends, FastAPI, Query
from opentelemetry import trace from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
@@ -22,23 +42,20 @@ from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.trace.export import BatchSpanProcessor
from prometheus_client import make_asgi_app from prometheus_client import make_asgi_app
from .cdc_consumer import run_consumer as run_cdc_consumer from . import analytics_service, cdc_consumer, grpc_server, warning_service
from .clickhouse_client import (
close_client,
query_class_performance,
query_dashboard,
query_student_errors,
)
from .clickhouse_client import ping as ch_ping
from .config import settings from .config import settings
from .repository import (
clickhouse_repository,
iam_client,
kafka_producer,
redis_client,
)
from .shared.action_state import ActionState
from .shared.permissions import UserContext, get_user_context
_logger: structlog.stdlib.BoundLogger | None = None _logger: structlog.stdlib.BoundLogger | None = None
tracer = trace.get_tracer(__name__) tracer = trace.get_tracer(__name__)
# CDC 消费者后台任务句柄
_cdc_task: asyncio.Task | None = None
# 日志级别映射
_LOG_LEVELS: dict[str, int] = { _LOG_LEVELS: dict[str, int] = {
"DEBUG": 10, "DEBUG": 10,
"INFO": 20, "INFO": 20,
@@ -49,10 +66,7 @@ _LOG_LEVELS: dict[str, int] = {
def init_logger() -> structlog.stdlib.BoundLogger: def init_logger() -> structlog.stdlib.BoundLogger:
"""初始化 structlog logger. """初始化 structlog logger."""
根据配置的 log_level 设置日志级别。
"""
global _logger global _logger
level = _LOG_LEVELS.get(settings.log_level.upper(), 20) level = _LOG_LEVELS.get(settings.log_level.upper(), 20)
structlog.configure( structlog.configure(
@@ -70,7 +84,7 @@ def init_logger() -> structlog.stdlib.BoundLogger:
def get_logger() -> structlog.stdlib.BoundLogger: def get_logger() -> structlog.stdlib.BoundLogger:
"""获取已初始化的 logger（未初始化时自动初始化）.""" """获取已初始化的 logger."""
global _logger global _logger
if _logger is None: if _logger is None:
return init_logger() return init_logger()
@@ -78,10 +92,7 @@ def get_logger() -> structlog.stdlib.BoundLogger:
def init_tracer() -> None: def init_tracer() -> None:
"""初始化 OpenTelemetry. """初始化 OpenTelemetry."""
endpoint 从 settings.otel_endpoint 读取（不硬编码）。
"""
provider = TracerProvider() provider = TracerProvider()
endpoint = settings.otel_endpoint.rstrip("/") endpoint = settings.otel_endpoint.rstrip("/")
exporter = OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces") exporter = OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")
@@ -93,207 +104,418 @@ def init_tracer() -> None:
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""应用生命周期. """应用生命周期.
1. 初始化 logger（structlog） 启动顺序：
2. 初始化 OTel tracer（endpoint 从 config 读） 1. 初始化 logger + tracer
3. 触发 ClickHouse 客户端惰性初始化（不阻塞启动，失败进入降级模式） 2. 启动 gRPC server :50055（AnalyticsService 12 RPC）
4. 若配置了 kafka_brokers，后台启动 CDC 消费者任务 3. 启动 CDC 消费者后台任务（手动 commit）
5. 关闭时停止 CDC 任务并释放 ClickHouse 客户端 4. Kafka producer 惰性初始化（首次发布时触发）
关闭顺序：
1. 停止 CDC 消费者
2. 停止 gRPC server
3. 关闭 Kafka producer
4. 关闭 Redis / iam gRPC / ClickHouse 客户端
""" """
global _cdc_task
logger = init_logger() logger = init_logger()
init_tracer() init_tracer()
logger.info( logger.info(
"data_ana_service_starting", "data_ana_service_starting",
port=settings.port, http_port=settings.http_port,
grpc_port=settings.grpc_port,
dev_mode=settings.dev_mode, dev_mode=settings.dev_mode,
clickhouse_configured=bool(settings.clickhouse_host), clickhouse_configured=bool(settings.clickhouse_host),
kafka_brokers=settings.kafka_brokers, kafka_brokers=settings.kafka_brokers,
kafka_cdc_topics=settings.kafka_cdc_topics, iam_grpc_endpoint=settings.iam_grpc_endpoint,
redis_url=settings.redis_url or "not_configured",
) )
# 启动 CDC 消费者后台任务（若未配置 kafka_brokers，run_consumer 内部直接返回）
_cdc_task = asyncio.create_task(run_cdc_consumer()) # 1. 启动 gRPC server（grpcio 未安装则跳过，降级为仅 HTTP）
grpc_server_obj = await grpc_server.start_grpc_server()
if grpc_server_obj is None:
logger.warning("grpc_server_not_started_http_only")
# 2. 启动 CDC 消费者后台任务
await cdc_consumer.start_consumer()
yield yield
logger.info("data_ana_service_stopping") logger.info("data_ana_service_stopping")
# 取消 CDC 任务
if _cdc_task is not None and not _cdc_task.done(): # 3. 停止 CDC 消费者
_cdc_task.cancel() await cdc_consumer.stop_consumer()
with contextlib.suppress(asyncio.CancelledError):
await _cdc_task # 4. 停止 gRPC server
await close_client() await grpc_server.stop_grpc_server()
# 5. 关闭 Kafka producer
await kafka_producer.close_producer()
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
await redis_client.close_client()
await iam_client.close_grpc()
await clickhouse_repository.close_client()
app = FastAPI( app = FastAPI(
title="Data Analytics Service", title="Data Analytics Service",
version="0.1.0", version="1.0.0",
description="D6 智能洞察领域服务（ClickHouse 宽表 + CDC + 掌握度算法 + 预警）",
lifespan=lifespan, lifespan=lifespan,
) )
# OpenTelemetry FastAPI 自动埋点（HTTP 请求/响应 span）
FastAPIInstrumentor.instrument_app(app) FastAPIInstrumentor.instrument_app(app)
# Prometheus 指标
app.mount("/metrics", make_asgi_app()) app.mount("/metrics", make_asgi_app())
# 业务路由
router = APIRouter() router = APIRouter()
@app.get("/healthz") # ===== 基础端点（3 个） =====
async def healthz() -> dict:
"""健康检查（liveness）.
只要进程存活即返回 ok，不依赖 ClickHouse。
""" @app.get("/")
async def root() -> dict[str, Any]:
"""根信息."""
return {
"service": "data-ana",
"version": "1.0.0",
"http_port": settings.http_port,
"grpc_port": settings.grpc_port,
"docs": "/docs",
"endpoints": {
"healthz": "/healthz",
"readyz": "/readyz",
"business": "/analytics/*",
},
}
@app.get("/healthz")
async def healthz() -> dict[str, str]:
"""健康检查（liveness，只要进程存活即返回 ok）."""
return {"status": "ok", "service": "data-ana"} return {"status": "ok", "service": "data-ana"}
@app.get("/readyz") @app.get("/readyz")
async def readyz() -> dict: async def readyz() -> dict[str, Any]:
"""就绪检查（readiness）. """就绪检查（readiness，检查 4 依赖）.
ClickHouse 为可选依赖： 依赖检查：
- 已配置且可达：ready=true 1. clickhouse：已配置且可达（未配置算降级就绪）
- 未配置：ready=true，degraded=true（降级模式仍可服务） 2. cdc_consumer：running / disabled
- 已配置但不可达：ready=false 3. redis：已配置且可达（未配置算降级就绪）
4. iam_grpc：已配置且可达（未配置算降级就绪）
CDC 消费者状态附加在响应中： 返回 ready=true 的条件：
- cdc_consumer: running / disabled / failed - ClickHouse 已配置且可达，或未配置（降级就绪）
- 不要求所有依赖都健康（降级模式下仍可服务骨架数据）
""" """
cdc_status = "disabled" # 1. ClickHouse
if _cdc_task is not None: ch_ok = await clickhouse_repository.ping()
if _cdc_task.done(): ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
cdc_status = "failed"
elif not settings.kafka_brokers:
cdc_status = "disabled"
else:
cdc_status = "running"
if not settings.clickhouse_host: # 2. CDC 消费者
return { cdc_status = (
"status": "ok", "running"
"service": "data-ana", if cdc_consumer.is_running()
"ready": True, else ("disabled" if not settings.kafka_brokers else "failed")
"degraded": True, )
"clickhouse": "not_configured",
"cdc_consumer": cdc_status, # 3. Redis
"kafka_brokers": settings.kafka_brokers or None, redis_ok = await redis_client.ping() if settings.redis_url else None
"timestamp": datetime.now(UTC).isoformat(), redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
}
# 4. iam gRPC
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
iam_status = (
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
)
# 5. gRPC server
grpc_status = "running" if grpc_server.is_running() else "stopped"
# 就绪判定：ClickHouse 可达或未配置（降级就绪）
ready = ch_ok or not settings.clickhouse_host
degraded = not ch_ok or not redis_ok or not iam_ok
ch_ok = await ch_ping()
return { return {
"status": "ok" if ch_ok else "degraded", "status": "ok" if ready else "not_ready",
"service": "data-ana", "service": "data-ana",
"ready": ch_ok, "ready": ready,
"degraded": not ch_ok, "degraded": degraded,
"clickhouse": "ok" if ch_ok else "unreachable", "dependencies": {
"cdc_consumer": cdc_status, "clickhouse": ch_status,
"kafka_brokers": settings.kafka_brokers or None, "cdc_consumer": cdc_status,
"redis": redis_status,
"iam_grpc": iam_status,
"grpc_server": grpc_status,
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
},
"timestamp": datetime.now(UTC).isoformat(), "timestamp": datetime.now(UTC).isoformat(),
} }
@router.get("/analytics/class/{class_id}/performance") # ===== 业务端点（11 个，全部返回 ActionState[T]） =====
async def class_performance(class_id: str) -> dict:
"""班级成绩分析.
优先查 ClickHouse；降级时返回骨架数据。
""" @router.get("/analytics/class/{class_id}/performance")
logger = get_logger() async def get_class_performance(
with tracer.start_as_current_span("class_performance") as span: class_id: str,
subject_id: str = Query(""),
start_date: int = Query(0),
end_date: int = Query(0),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""班级成绩分析."""
with tracer.start_as_current_span("get_class_performance") as span:
span.set_attribute("class_id", class_id) span.set_attribute("class_id", class_id)
result = await query_class_performance(class_id) result = await analytics_service.get_class_performance(
if result is None: user=user,
logger.info("class_performance_degraded", class_id=class_id) class_id=class_id,
return { subject_id=subject_id,
"success": True, start_date=start_date,
"data": { end_date=end_date,
"classId": class_id, )
"averageScore": 0, degraded = result.get("degraded", False)
"passRate": 0, return ActionState.ok(
"totalStudents": 0, result,
"message": "ClickHouse unavailable - skeleton data", degraded=degraded,
"degraded": True, degraded_reason=result.get("degraded_reason", "") if degraded else "",
}, )
}
return {"success": True, "data": {**result, "degraded": False}}
@router.get("/analytics/student/{student_id}/weakness") @router.get("/analytics/student/{student_id}/weakness")
async def student_weakness(student_id: str) -> dict: async def get_student_weakness(
"""学生薄弱知识点分析. student_id: str,
subject_id: str = Query(""),
优先查 ClickHouse；降级时返回骨架数据。 user: UserContext = Depends(get_user_context),
""" ) -> ActionState[dict[str, Any]]:
logger = get_logger() """学生薄弱知识点."""
with tracer.start_as_current_span("student_weakness") as span: with tracer.start_as_current_span("get_student_weakness") as span:
span.set_attribute("student_id", student_id) span.set_attribute("student_id", student_id)
result = await query_dashboard(student_id) result = await analytics_service.get_student_weakness(
if result is None: user=user,
logger.info("student_weakness_degraded", student_id=student_id) student_id=student_id,
return { subject_id=subject_id,
"success": True, )
"data": { degraded = result.get("degraded", False)
"studentId": student_id, return ActionState.ok(
"weakPoints": [], result,
"message": "ClickHouse unavailable - skeleton data", degraded=degraded,
"degraded": True, degraded_reason=result.get("degraded_reason", "") if degraded else "",
}, )
}
# 从宽表提取薄弱知识点：mastery_level < 0.6 视为薄弱
weak_points = [ @router.get("/analytics/student/{student_id}/trend")
{ async def get_learning_trend(
"knowledgePointId": r["knowledge_point_id"], student_id: str,
"masteryLevel": r["mastery_level"], subject_id: str = Query(""),
"errorCount": r["error_count"], start_date: int = Query(0),
} end_date: int = Query(0),
for r in result["records"] user: UserContext = Depends(get_user_context),
if r.get("mastery_level") is not None and r["mastery_level"] < 0.6 ) -> ActionState[dict[str, Any]]:
] """学习趋势."""
return { with tracer.start_as_current_span("get_learning_trend") as span:
"success": True, span.set_attribute("student_id", student_id)
"data": { result = await analytics_service.get_learning_trend(
"studentId": student_id, user=user,
"weakPoints": weak_points, student_id=student_id,
"records": result["records"], subject_id=subject_id,
"total": result["total"], start_date=start_date,
"degraded": False, end_date=end_date,
}, )
} degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/student/{student_id}/errorbook") @router.get("/analytics/student/{student_id}/errorbook")
async def student_errorbook(student_id: str) -> dict: async def get_student_errorbook(
"""学生错题本. student_id: str,
user: UserContext = Depends(get_user_context),
优先查 ClickHouse；降级时返回空列表。 ) -> ActionState[dict[str, Any]]:
""" """学生错题本."""
logger = get_logger() with tracer.start_as_current_span("get_student_errorbook") as span:
with tracer.start_as_current_span("student_errorbook") as span:
span.set_attribute("student_id", student_id) span.set_attribute("student_id", student_id)
result = await query_student_errors(student_id) errors = await clickhouse_repository.query_student_errors(student_id)
if result is None: if errors is None:
logger.info("student_errorbook_degraded", student_id=student_id) return ActionState.ok(
return { {"studentId": student_id, "errors": [], "total": 0},
"success": True, degraded=True,
"data": { degraded_reason="clickhouse_unavailable",
"studentId": student_id, )
"errors": [], return ActionState.ok(
"total": 0, {
"message": "ClickHouse unavailable - empty errorbook",
"degraded": True,
},
}
return {
"success": True,
"data": {
"studentId": student_id, "studentId": student_id,
"errors": result, "errors": errors,
"total": len(result), "total": len(errors),
"degraded": False, }
}, )
}
@router.get("/analytics/teacher/dashboard")
async def get_teacher_dashboard(
class_id: str = Query(""),
subject_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""教师仪表盘."""
with tracer.start_as_current_span("get_teacher_dashboard"):
result = await analytics_service.get_teacher_dashboard(
user=user,
class_id=class_id,
subject_id=subject_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/student/dashboard")
async def get_student_dashboard(
subject_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""学生仪表盘."""
with tracer.start_as_current_span("get_student_dashboard"):
result = await analytics_service.get_student_dashboard(
user=user,
subject_id=subject_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/parent/dashboard")
async def get_parent_dashboard(
student_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""家长仪表盘."""
with tracer.start_as_current_span("get_parent_dashboard"):
result = await analytics_service.get_parent_dashboard(
user=user,
child_id=student_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/admin/dashboard")
async def get_admin_dashboard(
school_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""管理员仪表盘."""
with tracer.start_as_current_span("get_admin_dashboard"):
result = await analytics_service.get_admin_dashboard(
user=user,
school_id=school_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/warnings")
async def get_warnings(
student_id: str = Query(""),
warning_type: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""预警列表查询."""
with tracer.start_as_current_span("get_warnings"):
warnings = await warning_service.get_warnings(
student_id=student_id,
warning_type=warning_type,
)
return ActionState.ok(
{
"warnings": warnings,
"total": len(warnings),
}
)
@router.post("/analytics/warnings/trigger")
async def trigger_warning(
target_id: str = Query(...),
warning_type: str = Query(...),
severity: str = Query("WARN"),
threshold: float = Query(0.0),
current_value: float = Query(0.0),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""手动触发预警."""
with tracer.start_as_current_span("trigger_warning"):
result = await warning_service.trigger_warning_manual(
target_id=target_id,
warning_type=warning_type,
threshold=threshold,
current_value=current_value,
severity=severity,
)
return ActionState.ok(result)
@router.get("/analytics/class/{class_id}/mastery-distribution")
async def get_mastery_distribution(
class_id: str,
subject_id: str = Query(""),
knowledge_point_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""班级掌握度分布."""
with tracer.start_as_current_span("get_mastery_distribution"):
result = await analytics_service.get_mastery_distribution(
user=user,
class_id=class_id,
subject_id=subject_id,
knowledge_point_id=knowledge_point_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
@router.get("/analytics/student/{student_id}/mastery")
async def get_student_mastery(
student_id: str,
subject_id: str = Query(""),
user: UserContext = Depends(get_user_context),
) -> ActionState[dict[str, Any]]:
"""学生知识点掌握度明细."""
with tracer.start_as_current_span("get_student_mastery"):
result = await analytics_service.get_student_mastery(
user=user,
student_id=student_id,
subject_id=subject_id,
)
degraded = result.get("degraded", False)
return ActionState.ok(
result,
degraded=degraded,
degraded_reason=result.get("degraded_reason", "") if degraded else "",
)
app.include_router(router) app.include_router(router)

256
services/data-ana/src/data_ana/mastery_service.py Normal file
View File

@@ -0,0 +1,256 @@
"""掌握度计算服务（加权滑动平均 + 遗忘曲线）.
算法对齐 02-architecture-design.md §9：
- WEIGHTED_MOVING_AVG：w_i = 0.6^i，归一化（权重从最近到最远递减）
- FORGETTING_CURVE：基于遗忘曲线的 max 叠加（P5+ 启用），
half_life = 30 days，越久未练习掌握度越衰减.
输入：学生指定知识点的历史成绩序列（按时间倒序）
输出：mastery_level (0.0-1.0)
副作用：
- 写 mastery_snapshot 表（upsert_mastery_snapshot）
- 发布 mastery.updated 事件（Outbox 豁免）
降级策略：
- ClickHouse 不可达：跳过计算（返回 None）
- Kafka 不可达：计算继续，事件发布静默失败
"""
from datetime import UTC, datetime
from typing import Any
import structlog
from .config import settings
from .repository import clickhouse_repository, kafka_producer
logger = structlog.get_logger(__name__)
def _weighted_moving_avg(scores: list[dict[str, Any]]) -> float | None:
"""加权滑动平均算法.
输入：按时间倒序的成绩列表（最近在前），元素含 score 字段
输出：归一化的 mastery_level (0.0-1.0)，样本不足返回 None.
权重：w_i = decay_base ^ i（i 从 0 开始，最近 attempt 权重最大）
"""
if not scores:
return None
# 截取最近 N 次（window_size）
window = scores[: settings.mastery_window_size]
if len(window) < settings.mastery_min_samples:
return None
decay = settings.mastery_decay_base
total_weight = 0.0
weighted_sum = 0.0
for i, record in enumerate(window):
# 成绩归一化到 0.0-1.0（假设原始分 0-100）
raw_score = record.get("score", 0.0)
normalized = min(max(raw_score / 100.0, 0.0), 1.0)
weight = decay**i
weighted_sum += normalized * weight
total_weight += weight
if total_weight <= 0:
return None
mastery = weighted_sum / total_weight
return round(max(0.0, min(1.0, mastery)), 4)
def _forgetting_curve_decay(
mastery: float,
last_attempt_at: datetime | None,
now: datetime | None = None,
) -> float:
"""遗忘曲线衰减（P5+ 启用，对齐 02 §9）.
距离上次练习越久，mastery 越衰减：
decayed = mastery * exp(-ln(2) * days_since / half_life)
half_life = 30 days（settings.mastery_forgetting_half_life_days）.
"""
if last_attempt_at is None:
return mastery
now = now or datetime.now(UTC)
if last_attempt_at.tzinfo is None:
last_attempt_at = last_attempt_at.replace(tzinfo=UTC)
days_since = (now - last_attempt_at).total_seconds() / 86400
if days_since <= 0:
return mastery
half_life = settings.mastery_forgetting_half_life_days
if half_life <= 0:
return mastery
import math
decay_factor = math.exp(-math.log(2) * days_since / half_life)
decayed = mastery * decay_factor
return round(max(0.0, min(1.0, decayed)), 4)
def classify_mastery_label(level: float) -> str:
"""掌握度三档分类."""
if level >= 0.8:
return "mastered"
if level >= 0.4:
return "progressing"
return "weak"
async def calculate_mastery(
student_id: str,
knowledge_point_id: str,
subject_id: str = "",
) -> dict[str, Any] | None:
"""计算学生指定知识点的掌握度.
流程：
1. 查询学生该知识点的历史成绩（按时间倒序）
2. 加权滑动平均 → mastery_level
3. 遗忘曲线衰减 → 最终 mastery_level
4. 写 mastery_snapshot 表
5. 发布 mastery.updated 事件（Outbox 豁免）
返回：
{
"student_id": ...,
"knowledge_point_id": ...,
"subject_id": ...,
"mastery_level": 0.0-1.0,
"previous_level": 0.0-1.0 or None,
"mastery_label": "mastered" / "progressing" / "weak",
"degraded": bool,
}
ClickHouse 不可达时返回降级骨架（degraded=true）.
"""
# 1. 查询历史成绩
scores = await clickhouse_repository.query_student_scores_by_kp(
student_id=student_id,
knowledge_point_id=knowledge_point_id,
limit=settings.mastery_window_size * 2, # 多取一倍容错
)
if scores is None:
# ClickHouse 不可达降级
logger.warning(
"mastery_calc_clickhouse_unavailable_degraded",
student_id=student_id,
knowledge_point_id=knowledge_point_id,
)
return {
"student_id": student_id,
"knowledge_point_id": knowledge_point_id,
"subject_id": subject_id,
"mastery_level": 0.0,
"previous_level": None,
"mastery_label": "weak",
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
}
# 2. 加权滑动平均
mastery = _weighted_moving_avg(scores)
if mastery is None:
# 样本不足，返回默认值
return {
"student_id": student_id,
"knowledge_point_id": knowledge_point_id,
"subject_id": subject_id,
"mastery_level": 0.0,
"previous_level": None,
"mastery_label": "weak",
"degraded": False,
"degraded_reason": "insufficient_samples",
}
# 3. 遗忘曲线衰减（基于最近一次成绩的时间）
last_attempt_at = None
if scores:
last_attempt_at = scores[0].get("timestamp")
mastery_final = _forgetting_curve_decay(mastery, last_attempt_at)
# 4. 查询上一次 mastery（用于事件对比）
previous_snapshot = await clickhouse_repository.query_mastery_snapshot(
student_id=student_id,
subject_id=subject_id,
)
previous_level: float | None = None
if previous_snapshot is not None:
for kp in previous_snapshot.get("knowledgePoints", []):
if kp.get("knowledge_point_id") == knowledge_point_id:
previous_level = kp.get("mastery_level")
break
# 5. 写 mastery_snapshot 表
calculated_at = datetime.now(UTC)
write_ok = await clickhouse_repository.upsert_mastery_snapshot(
student_id=student_id,
knowledge_point_id=knowledge_point_id,
subject_id=subject_id,
mastery_level=mastery_final,
calculated_at=calculated_at,
calculation_method="weighted_moving_avg",
)
if not write_ok:
logger.warning(
"mastery_snapshot_write_failed_degraded",
student_id=student_id,
knowledge_point_id=knowledge_point_id,
)
# 6. 发布 mastery.updated 事件（Outbox 豁免）
published = await kafka_producer.publish_mastery_updated(
student_id=student_id,
knowledge_point_id=knowledge_point_id,
mastery_level=mastery_final,
previous_level=previous_level if previous_level is not None else 0.0,
)
result = {
"student_id": student_id,
"knowledge_point_id": knowledge_point_id,
"subject_id": subject_id,
"mastery_level": mastery_final,
"previous_level": previous_level,
"mastery_label": classify_mastery_label(mastery_final),
"degraded": not write_ok,
"degraded_reason": "snapshot_write_failed" if not write_ok else "",
"event_published": published,
}
logger.info(
"mastery_calculated",
student_id=student_id,
knowledge_point_id=knowledge_point_id,
mastery_level=mastery_final,
previous_level=previous_level,
label=result["mastery_label"],
published=published,
)
return result
async def batch_calculate_mastery(
student_id: str,
knowledge_point_ids: list[str],
subject_id: str = "",
) -> list[dict[str, Any]]:
"""批量计算学生多个知识点的掌握度."""
results: list[dict[str, Any]] = []
for kp_id in knowledge_point_ids:
result = await calculate_mastery(
student_id=student_id,
knowledge_point_id=kp_id,
subject_id=subject_id,
)
if result is not None:
results.append(result)
return results

1
services/data-ana/src/data_ana/repository/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Repository package for data-ana service."""

794
services/data-ana/src/data_ana/repository/clickhouse_repository.py Normal file
View File

@@ -0,0 +1,794 @@
"""ClickHouse Repository（宽表查询 + FINAL/argMax 去重 + DataScope WHERE 注入）.
P0 整改：ReplacingMergeTree 查询必须加 FINAL 或用 argMax 聚合确保去重生效.
支持降级模式：clickhouse_host 未配置或不可达时返回 None，服务仍可启动.
"""
import asyncio
from datetime import datetime
from typing import Any
import structlog
from ..config import settings
logger = structlog.get_logger(__name__)
_client: Any | None = None
_client_initialized: bool = False
def get_client() -> Any | None:
"""获取 ClickHouse 客户端（惰性初始化，失败返回 None 降级）."""
global _client, _client_initialized
if not settings.clickhouse_host:
return None
if _client_initialized:
return _client
_client_initialized = True
try:
import clickhouse_connect
kwargs: dict[str, Any] = {
"host": settings.clickhouse_host,
"port": settings.clickhouse_port,
"database": settings.clickhouse_database,
"connect_timeout": settings.clickhouse_connect_timeout_ms / 1000,
}
if settings.clickhouse_user:
kwargs["username"] = settings.clickhouse_user
if settings.clickhouse_password:
kwargs["password"] = settings.clickhouse_password
_client = clickhouse_connect.get_client(**kwargs)
logger.info(
"clickhouse_client_initialized",
host=settings.clickhouse_host,
port=settings.clickhouse_port,
database=settings.clickhouse_database,
)
except Exception as exc: # noqa: BLE001
logger.warning("clickhouse_client_init_failed_degraded", error=str(exc))
_client = None
return _client
async def close_client() -> None:
"""关闭客户端."""
global _client, _client_initialized
if _client is not None:
try:
await asyncio.to_thread(_client.close)
except Exception as exc: # noqa: BLE001
logger.warning("clickhouse_client_close_failed", error=str(exc))
finally:
_client = None
_client_initialized = False
async def ping() -> bool:
"""ClickHouse 连通性检查（供 /readyz 使用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(client.query, "SELECT 1")
return True
except Exception as exc: # noqa: BLE001
logger.warning("clickhouse_ping_failed", error=str(exc))
return False
# ===== 查询方法（FINAL/argMax 去重） =====
async def query_class_performance(
class_id: str,
subject_id: str = "",
start_date: int = 0,
end_date: int = 0,
) -> dict | None:
"""查询班级成绩分析（聚合 student_dashboard_view，FINAL 去重）."""
client = get_client()
if client is None:
return None
where_parts = ["class_id = {cid:String}"]
params: dict[str, Any] = {"cid": class_id}
if subject_id:
where_parts.append("subject_id = {sid:String}")
params["sid"] = subject_id
if start_date:
where_parts.append("last_updated >= {sd:DateTime64(3)}")
params["sd"] = datetime.fromtimestamp(start_date)
if end_date:
where_parts.append("last_updated <= {ed:DateTime64(3)}")
params["ed"] = datetime.fromtimestamp(end_date)
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" count(DISTINCT student_id) AS total_students, "
f" avg(score) AS average_score, "
f" countIf(score >= 60) / count() AS pass_rate "
f"FROM student_dashboard_view FINAL "
f"WHERE {where_clause}",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_class_performance_failed", error=str(exc), class_id=class_id)
return None
if not rows:
return {"classId": class_id, "averageScore": 0.0, "passRate": 0.0, "totalStudents": 0}
total_students, average_score, pass_rate = rows[0]
return {
"classId": class_id,
"averageScore": float(average_score) if average_score is not None else 0.0,
"passRate": float(pass_rate) if pass_rate is not None else 0.0,
"totalStudents": int(total_students) if total_students is not None else 0,
}
async def query_student_dashboard(student_id: str) -> dict | None:
"""查询学生学情宽表（FINAL 去重，返回最近 50 条）."""
client = get_client()
if client is None:
return None
try:
result = await asyncio.to_thread(
client.query,
"SELECT student_id, class_id, exam_id, subject_id, score, "
"rank_in_class, knowledge_point_id, mastery_level, error_count, "
"last_updated "
"FROM student_dashboard_view FINAL "
"WHERE student_id = {sid:String} "
"ORDER BY last_updated DESC "
"LIMIT 50",
parameters={"sid": student_id},
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_student_dashboard_failed", error=str(exc), student_id=student_id)
return None
columns = [
"student_id",
"class_id",
"exam_id",
"subject_id",
"score",
"rank_in_class",
"knowledge_point_id",
"mastery_level",
"error_count",
"last_updated",
]
records = [dict(zip(columns, row, strict=True)) for row in rows]
return {"studentId": student_id, "records": records, "total": len(records)}
async def query_student_weakness(student_id: str, subject_id: str = "") -> dict | None:
"""查询学生薄弱知识点（mastery_level < 0.6，FINAL 去重）."""
dashboard = await query_student_dashboard(student_id)
if dashboard is None:
return None
weak_points = [
{
"knowledgePointId": r["knowledge_point_id"],
"title": r.get("knowledge_point_id", ""), # content CDC 同步后补充标题
"mastery": r["mastery_level"],
"errorCount": r["error_count"],
}
for r in dashboard["records"]
if r.get("mastery_level") is not None
and r["mastery_level"] < 0.6
and (not subject_id or r.get("subject_id") == subject_id)
]
return {"studentId": student_id, "weakPoints": weak_points}
async def query_student_errors(student_id: str) -> list[dict] | None:
"""查询学生错题本（FINAL 去重）."""
client = get_client()
if client is None:
return None
try:
result = await asyncio.to_thread(
client.query,
"SELECT student_id, question_id, knowledge_point_id, error_count, "
"last_error_time, content "
"FROM student_errors FINAL "
"WHERE student_id = {sid:String} "
"ORDER BY last_error_time DESC "
"LIMIT 100",
parameters={"sid": student_id},
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_student_errors_failed", error=str(exc), student_id=student_id)
return None
columns = [
"student_id",
"question_id",
"knowledge_point_id",
"error_count",
"last_error_time",
"content",
]
return [dict(zip(columns, row, strict=True)) for row in rows]
async def query_learning_trend(
student_id: str,
start_date: int = 0,
end_date: int = 0,
subject_id: str = "",
) -> dict | None:
"""查询学习趋势（按时间排序的成绩曲线，FINAL 去重）."""
client = get_client()
if client is None:
return None
where_parts = ["student_id = {sid:String}"]
params: dict[str, Any] = {"sid": student_id}
if subject_id:
where_parts.append("subject_id = {sub:String}")
params["sub"] = subject_id
if start_date:
where_parts.append("last_updated >= {sd:DateTime64(3)}")
params["sd"] = datetime.fromtimestamp(start_date)
if end_date:
where_parts.append("last_updated <= {ed:DateTime64(3)}")
params["ed"] = datetime.fromtimestamp(end_date)
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT last_updated, score, exam_id "
f"FROM student_dashboard_view FINAL "
f"WHERE {where_clause} "
f"ORDER BY last_updated ASC "
f"LIMIT 100",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_learning_trend_failed", error=str(exc), student_id=student_id)
return None
points = [
{"date": int(row[0].timestamp()) if row[0] else 0, "score": float(row[1] or 0)}
for row in rows
]
return {"studentId": student_id, "points": points}
async def query_mastery_snapshot(student_id: str, subject_id: str = "") -> dict | None:
"""查询知识点掌握度快照（argMax 去重保留最新版本）."""
client = get_client()
if client is None:
return None
where_parts = ["student_id = {sid:String}"]
params: dict[str, Any] = {"sid": student_id}
if subject_id:
where_parts.append("subject_id = {sub:String}")
params["sub"] = subject_id
where_clause = " AND ".join(where_parts)
try:
# argMax 获取每个 knowledge_point_id 的最新 mastery_level
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" knowledge_point_id, "
f" argMax(mastery_level, calculated_at) AS mastery_level, "
f" argMax(subject_id, calculated_at) AS subject_id, "
f" max(calculated_at) AS calculated_at "
f"FROM mastery_snapshot "
f"WHERE {where_clause} "
f"GROUP BY knowledge_point_id "
f"ORDER BY mastery_level ASC",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_mastery_snapshot_failed", error=str(exc), student_id=student_id)
return None
knowledge_points = []
total_mastery = 0.0
for row in rows:
kp_id, mastery, subj, calc_at = row
mastery_val = float(mastery or 0)
if mastery_val >= 0.8:
label = "mastered"
elif mastery_val >= 0.4:
label = "progressing"
else:
label = "weak"
knowledge_points.append(
{
"knowledge_point_id": kp_id,
"title": kp_id, # content CDC 同步后补充
"subject_id": subj or "",
"mastery_level": mastery_val,
"mastery_label": label,
"calculated_at": int(calc_at.timestamp()) if calc_at else 0,
}
)
total_mastery += mastery_val
overall = total_mastery / len(knowledge_points) if knowledge_points else 0.0
return {
"studentId": student_id,
"knowledgePoints": knowledge_points,
"overallMastery": round(overall, 4),
}
async def query_mastery_distribution(
class_id: str,
subject_id: str = "",
knowledge_point_id: str = "",
) -> dict | None:
"""查询班级掌握度分布（mastered/progressing/weak 三档）."""
client = get_client()
if client is None:
return None
where_parts = ["class_id = {cid:String}"]
params: dict[str, Any] = {"cid": class_id}
if subject_id:
where_parts.append("subject_id = {sid:String}")
params["sid"] = subject_id
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" countIf(mastery_level >= 0.8) AS mastered, "
f" countIf(mastery_level >= 0.4 AND mastery_level < 0.8) AS progressing, "
f" countIf(mastery_level < 0.4) AS weak, "
f" count() AS total "
f"FROM student_dashboard_view FINAL "
f"WHERE {where_clause}",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_mastery_distribution_failed", error=str(exc), class_id=class_id)
return None
if not rows:
return {
"classId": class_id,
"masteredCount": 0,
"progressingCount": 0,
"weakCount": 0,
"totalStudents": 0,
}
mastered, progressing, weak, total = rows[0]
return {
"classId": class_id,
"masteredCount": int(mastered or 0),
"progressingCount": int(progressing or 0),
"weakCount": int(weak or 0),
"totalStudents": int(total or 0),
}
async def query_attendance(
student_id: str,
start_date: int = 0,
end_date: int = 0,
) -> dict | None:
"""查询学生考勤历史（FINAL 去重）."""
client = get_client()
if client is None:
return None
where_parts = ["student_id = {sid:String}"]
params: dict[str, Any] = {"sid": student_id}
if start_date:
where_parts.append("attendance_date >= {sd:Date}")
params["sd"] = datetime.fromtimestamp(start_date).date()
if end_date:
where_parts.append("attendance_date <= {ed:Date}")
params["ed"] = datetime.fromtimestamp(end_date).date()
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT student_id, class_id, attendance_date, status, "
f"recorded_by, remark, occurred_at "
f"FROM attendance_logs FINAL "
f"WHERE {where_clause} "
f"ORDER BY attendance_date DESC "
f"LIMIT 100",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_attendance_failed", error=str(exc), student_id=student_id)
return None
columns = [
"student_id",
"class_id",
"attendance_date",
"status",
"recorded_by",
"remark",
"occurred_at",
]
records = [dict(zip(columns, row, strict=True)) for row in rows]
# 统计
absent_count = sum(1 for r in records if r["status"] == "absent")
late_count = sum(1 for r in records if r["status"] == "late")
present_count = sum(1 for r in records if r["status"] == "present")
return {
"studentId": student_id,
"records": records,
"total": len(records),
"absentCount": absent_count,
"lateCount": late_count,
"presentCount": present_count,
}
async def query_ai_usage_summary() -> dict | None:
"""查询 AI 用量统计（按 provider 聚合，FINAL 去重）."""
client = get_client()
if client is None:
return None
try:
result = await asyncio.to_thread(
client.query,
"SELECT "
" provider, "
" count() AS request_count, "
" sum(total_tokens) AS total_tokens, "
" sum(cost_cents) AS cost_cents "
"FROM ai_usage_log FINAL "
"GROUP BY provider "
"ORDER BY request_count DESC",
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_ai_usage_summary_failed", error=str(exc))
return None
by_provider = [
{
"provider": row[0],
"requestCount": int(row[1] or 0),
"totalTokens": int(row[2] or 0),
"costCents": int(row[3] or 0),
}
for row in rows
]
total_requests = sum(p["requestCount"] for p in by_provider)
total_tokens = sum(p["totalTokens"] for p in by_provider)
total_cost = sum(p["costCents"] for p in by_provider)
return {
"totalRequests": total_requests,
"totalTokens": total_tokens,
"totalCostCents": total_cost,
"byProvider": by_provider,
}
async def query_teacher_dashboard(user_id: str, class_id: str = "") -> dict | None:
"""查询教师仪表盘聚合数据."""
client = get_client()
if client is None:
return None
where = "class_id = {cid:String}" if class_id else "1=1"
params: dict[str, Any] = {}
if class_id:
params["cid"] = class_id
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" count(DISTINCT class_id) AS total_classes, "
f" count(DISTINCT student_id) AS total_students, "
f" avg(score) AS avg_score "
f"FROM student_dashboard_view FINAL "
f"WHERE {where}",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_teacher_dashboard_failed", error=str(exc), user_id=user_id)
return None
if not rows:
return None
total_classes, total_students, avg_score = rows[0]
return {
"userId": user_id,
"totalClasses": int(total_classes or 0),
"totalStudents": int(total_students or 0),
"classAvgScore": float(avg_score or 0),
"pendingHomeworkCount": 0, # 从 homework_submissions CDC 计算
"classes": [],
"topStudents": [],
"recentWarnings": [],
}
# ===== 写入方法（CDC 消费专用） =====
async def upsert_student_dashboard(
student_id: str,
class_id: str,
exam_id: str,
subject_id: str,
score: float,
rank_in_class: int,
knowledge_point_id: str,
mastery_level: float,
error_count: int,
last_updated: datetime,
) -> bool:
"""写入/更新学生学情宽表（CDC 消费专用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(
client.insert,
"student_dashboard_view",
[
[
student_id,
class_id,
exam_id,
subject_id,
score,
rank_in_class,
knowledge_point_id,
mastery_level,
error_count,
last_updated,
]
],
column_names=[
"student_id",
"class_id",
"exam_id",
"subject_id",
"score",
"rank_in_class",
"knowledge_point_id",
"mastery_level",
"error_count",
"last_updated",
],
)
return True
except Exception as exc: # noqa: BLE001
logger.warning("student_dashboard_upsert_failed", error=str(exc), student_id=student_id)
return False
async def upsert_student_error(
student_id: str,
question_id: str,
knowledge_point_id: str,
error_count: int,
last_error_time: datetime,
content: str,
) -> bool:
"""写入/更新学生错题本（CDC 消费专用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(
client.insert,
"student_errors",
[[student_id, question_id, knowledge_point_id, error_count, last_error_time, content]],
column_names=[
"student_id",
"question_id",
"knowledge_point_id",
"error_count",
"last_error_time",
"content",
],
)
return True
except Exception as exc: # noqa: BLE001
logger.warning("student_error_upsert_failed", error=str(exc), student_id=student_id)
return False
async def upsert_attendance_log(
student_id: str,
class_id: str,
attendance_date,
status: str,
recorded_by: str,
remark: str,
occurred_at: datetime,
) -> bool:
"""写入考勤记录（CDC 消费专用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(
client.insert,
"attendance_logs",
[[student_id, class_id, attendance_date, status, recorded_by, remark, occurred_at]],
column_names=[
"student_id",
"class_id",
"attendance_date",
"status",
"recorded_by",
"remark",
"occurred_at",
],
)
return True
except Exception as exc: # noqa: BLE001
logger.warning("attendance_log_upsert_failed", error=str(exc), student_id=student_id)
return False
async def upsert_mastery_snapshot(
student_id: str,
knowledge_point_id: str,
subject_id: str,
mastery_level: float,
calculated_at: datetime,
calculation_method: str = "weighted_moving_avg",
) -> bool:
"""写入掌握度快照（掌握度计算后调用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(
client.insert,
"mastery_snapshot",
[
[
student_id,
knowledge_point_id,
subject_id,
mastery_level,
calculated_at,
calculation_method,
]
],
column_names=[
"student_id",
"knowledge_point_id",
"subject_id",
"mastery_level",
"calculated_at",
"calculation_method",
],
)
return True
except Exception as exc: # noqa: BLE001
logger.warning("mastery_snapshot_upsert_failed", error=str(exc), student_id=student_id)
return False
async def upsert_ai_usage_log(
request_id: str,
user_id: str,
provider: str,
model: str,
prompt_tokens: int,
completion_tokens: int,
total_tokens: int,
latency_ms: int,
success: bool,
cost_cents: int,
occurred_at: datetime,
) -> bool:
"""写入 AI 用量记录（CDC/事件消费专用）."""
client = get_client()
if client is None:
return False
try:
await asyncio.to_thread(
client.insert,
"ai_usage_log",
[
[
request_id,
user_id,
provider,
model,
prompt_tokens,
completion_tokens,
total_tokens,
latency_ms,
success,
cost_cents,
occurred_at,
]
],
column_names=[
"request_id",
"user_id",
"provider",
"model",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"latency_ms",
"success",
"cost_cents",
"occurred_at",
],
)
return True
except Exception as exc: # noqa: BLE001
logger.warning("ai_usage_log_upsert_failed", error=str(exc), request_id=request_id)
return False
# ===== 查询历史成绩（掌握度计算用） =====
async def query_student_scores_by_kp(
student_id: str,
knowledge_point_id: str,
limit: int = 5,
) -> list[dict] | None:
"""查询学生指定知识点的历史成绩（掌握度计算用，FINAL 去重）."""
client = get_client()
if client is None:
return None
try:
result = await asyncio.to_thread(
client.query,
"SELECT score, last_updated, exam_id "
"FROM student_dashboard_view FINAL "
"WHERE student_id = {sid:String} AND knowledge_point_id = {kp:String} "
"ORDER BY last_updated DESC "
"LIMIT {lim:UInt32}",
parameters={"sid": student_id, "kp": knowledge_point_id, "lim": limit},
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_student_scores_by_kp_failed", error=str(exc), student_id=student_id)
return None
return [{"score": float(row[0] or 0), "timestamp": row[1], "exam_id": row[2]} for row in rows]

226
services/data-ana/src/data_ana/repository/iam_client.py Normal file
View File

@@ -0,0 +1,226 @@
"""iam gRPC 客户端（GetEffectiveDataScope + Redis 缓存 + 降级兜底）.
对齐 02-architecture-design.md §7.2 DataScope 解析链路：
请求 → Redis 缓存命中 → 返回
↓ miss
iam.GetEffectiveDataScope gRPC → 缓存 5min → 返回
↓ 不可达
按 role 映射默认 DataScope + degraded: true（降级兜底）
IAM_GRPC_ENDPOINT 未配置或 gRPC 调用失败时，自动降级到 role-based fallback.
"""
import json
from typing import Any
import structlog
from ..config import settings
from ..shared.permissions import (
DataScope,
DataScopeLevel,
UserContext,
get_datascope_fallback,
)
from . import redis_client
logger = structlog.get_logger(__name__)
# gRPC 客户端惰性初始化（避免未安装 grpcio 时 import 失败）
_grpc_channel: Any | None = None
_grpc_stub: Any | None = None
_grpc_initialized: bool = False
async def _init_grpc() -> tuple[Any, Any] | None:
"""惰性初始化 gRPC channel + stub（iam_grpc_endpoint 为空时返回 None）."""
global _grpc_channel, _grpc_stub, _grpc_initialized
if not settings.iam_grpc_endpoint:
return None
if _grpc_initialized:
return (_grpc_channel, _grpc_stub) if _grpc_stub is not None else None
_grpc_initialized = True
try:
import grpc # type: ignore[import-not-found]
from generated_proto import iam_pb2_grpc # type: ignore[import-not-found]
_grpc_channel = grpc.aio.insecure_channel(
settings.iam_grpc_endpoint,
options=[
("grpc.connect_timeout_ms", settings.iam_grpc_timeout_s * 1000),
("grpc.keepalive_time_ms", 60000),
],
)
_grpc_stub = iam_pb2_grpc.IamServiceStub(_grpc_channel)
logger.info(
"iam_grpc_initialized",
endpoint=settings.iam_grpc_endpoint,
)
return _grpc_channel, _grpc_stub
except Exception as exc: # noqa: BLE001
logger.warning("iam_grpc_init_failed_degraded", error=str(exc))
_grpc_stub = None
return None
async def close_grpc() -> None:
"""关闭 gRPC channel（应用退出时调用）."""
global _grpc_channel, _grpc_stub, _grpc_initialized
if _grpc_channel is not None:
try:
await _grpc_channel.close()
except Exception as exc: # noqa: BLE001
logger.warning("iam_grpc_close_failed", error=str(exc))
finally:
_grpc_channel = None
_grpc_stub = None
_grpc_initialized = False
async def _call_grpc(user_id: str) -> DataScope | None:
"""gRPC 调 iam.GetEffectiveDataScope（失败返回 None 触发降级）."""
stub_bundle = await _init_grpc()
if stub_bundle is None:
return None
_, stub = stub_bundle
try:
from generated_proto import iam_pb2 # type: ignore[import-not-found]
request = iam_pb2.GetEffectiveDataScopeRequest(user_id=user_id)
response = await stub.GetEffectiveDataScope( # type: ignore[union-attr]
request,
timeout=settings.iam_grpc_timeout_s,
)
# 解析 level 字符串到枚举
try:
level = DataScopeLevel(response.level)
except ValueError:
logger.warning(
"iam_datascope_unknown_level_fallback",
level=response.level,
)
return None
scope = DataScope(
level=level,
scope_ids=list(response.scope_ids),
school_id=response.school_id,
)
logger.info(
"iam_datascope_resolved",
user_id=user_id,
level=level.value,
scope_count=len(scope.scope_ids),
)
return scope
except Exception as exc: # noqa: BLE001
logger.warning("iam_grpc_call_failed_degraded", user_id=user_id, error=str(exc))
return None
def _cache_key(user_id: str) -> str:
"""DataScope 缓存键（5min TTL）."""
return f"data_ana:datascope:{user_id}"
def _serialize_scope(scope: DataScope) -> str:
"""DataScope 序列化（Redis 存储）."""
return json.dumps(
{
"level": scope.level.value,
"scope_ids": scope.scope_ids,
"school_id": scope.school_id,
"degraded": scope.degraded,
"degraded_reason": scope.degraded_reason,
},
ensure_ascii=False,
)
def _deserialize_scope(raw: str) -> DataScope | None:
"""DataScope 反序列化."""
try:
data = json.loads(raw)
return DataScope(
level=DataScopeLevel(data["level"]),
scope_ids=data.get("scope_ids", []),
school_id=data.get("school_id", ""),
degraded=data.get("degraded", False),
degraded_reason=data.get("degraded_reason", ""),
)
except Exception as exc: # noqa: BLE001
logger.warning("datascope_deserialize_failed", error=str(exc), raw=raw[:200])
return None
async def get_effective_datascope(user: UserContext) -> DataScope:
"""解析用户 DataScope（Redis 缓存 → iam gRPC → 降级兜底）.
返回值始终非 None：
- iam 可用：返回 iam 解析的 DataScope
- iam 不可用：返回 role-based fallback + degraded=true
"""
# 1. Redis 缓存
cached = await redis_client.get_cache(_cache_key(user.user_id))
if cached is not None:
scope = _deserialize_scope(cached)
if scope is not None:
return scope
# 2. iam gRPC
scope = await _call_grpc(user.user_id)
if scope is not None:
# 写缓存（5min TTL）
await redis_client.set_cache(
_cache_key(user.user_id),
_serialize_scope(scope),
ttl_s=settings.datascope_cache_ttl_s,
)
return scope
# 3. 降级兜底
fallback = get_datascope_fallback(user)
# 降级兜底也写缓存（更短 TTL，避免 iam 恢复后仍读降级值）
await redis_client.set_cache(
_cache_key(user.user_id),
_serialize_scope(fallback),
ttl_s=min(60, settings.datascope_cache_ttl_s),
)
logger.warning(
"datascope_fallback_used",
user_id=user.user_id,
roles=user.roles,
level=fallback.level.value,
)
return fallback
async def invalidate_datascope(user_id: str) -> bool:
"""失效 DataScope 缓存（用户角色变更时由 iam 触发）."""
return await redis_client.delete_cache(_cache_key(user_id))
async def ping() -> bool:
"""iam gRPC 连通性检查（供 /readyz 使用，1s 超时）."""
if not settings.iam_grpc_endpoint:
return False # 未配置不算健康（依赖降级兜底）
stub_bundle = await _init_grpc()
if stub_bundle is None:
return False
_, stub = stub_bundle
try:
# 用空 user_id 探活（iam 应返回 INVALID_ARGUMENT 而非超时）
from generated_proto import iam_pb2 # type: ignore[import-not-found]
request = iam_pb2.GetEffectiveDataScopeRequest(user_id="")
await stub.GetEffectiveDataScope(request, timeout=1) # type: ignore[union-attr]
return True
except Exception as exc: # noqa: BLE001
# INVALID_ARGUMENT 表示连通，超时/UNAVAILABLE 表示不连通
err_str = str(exc).lower()
if "invalid" in err_str or "not found" in err_str:
return True
logger.warning("iam_grpc_ping_failed", error=str(exc))
return False

202
services/data-ana/src/data_ana/repository/kafka_producer.py Normal file
View File

@@ -0,0 +1,202 @@
"""Kafka producer（MasteryEvent / WarningTriggered 派生数据事件发布）.
对齐 02-architecture-design.md §12 Outbox 豁免：
派生数据事件（掌握度/预警）非业务事实，直接 aiokafka producer 发布，
豁免 transactional outbox 模式（总裁裁决 §2.10）.
事件路由（复用 topic，总裁裁决 §2.11）：
- edu.insight.mastery.updated ← MasteryEvent(action=mastery.updated)
- edu.insight.mastery.updated ← MasteryEvent(action=warning.triggered)（复用）
- edu.insight.ai.usage ← AIUsageEvent（ai 服务发布，本服务消费）
Producer 配置：
- idempotent=true（防重复）
- transactional_id=data-ana-producer（事务幂等）
- acks=all（强一致性）
降级策略：
- kafka_brokers 未配置或不可达时，发布操作静默失败（记日志），
不阻塞主流程（掌握度计算/预警评估继续，下游感知不到事件则视为未更新）.
"""
import json
import time
import uuid
from typing import Any
import structlog
from ..config import settings
logger = structlog.get_logger(__name__)
# producer 惰性初始化（避免未安装 aiokafka 时 import 失败）
_producer: Any | None = None
_producer_initialized: bool = False
async def get_producer() -> Any | None:
"""获取 Kafka producer（惰性初始化，失败返回 None 降级）."""
global _producer, _producer_initialized
if not settings.kafka_brokers:
return None
if _producer_initialized:
return _producer
_producer_initialized = True
try:
from aiokafka import AIOKafkaProducer # type: ignore[import-not-found]
_producer = AIOKafkaProducer(
bootstrap_servers=settings.kafka_brokers,
client_id=settings.service_name,
transactional_id=settings.kafka_producer_transactional_id,
enable_idempotence=True,
acks="all",
value_serializer=lambda v: json.dumps(v, ensure_ascii=False).encode("utf-8"),
key_serializer=lambda k: k.encode("utf-8") if k else None,
linger_ms=20, # 批量等待 20ms（平衡延迟与吞吐）
retry_backoff_ms=100,
request_timeout_ms=10_000,
)
await _producer.start()
logger.info(
"kafka_producer_initialized",
brokers=settings.kafka_brokers,
transactional_id=settings.kafka_producer_transactional_id,
)
return _producer
except Exception as exc: # noqa: BLE001
logger.warning("kafka_producer_init_failed_degraded", error=str(exc))
_producer = None
return None
async def close_producer() -> None:
"""关闭 producer（应用退出时调用）."""
global _producer, _producer_initialized
if _producer is not None:
try:
await _producer.stop()
except Exception as exc: # noqa: BLE001
logger.warning("kafka_producer_close_failed", error=str(exc))
finally:
_producer = None
_producer_initialized = False
async def _publish(
topic: str,
payload: dict[str, Any],
key: str | None = None,
) -> bool:
"""发布消息（producer 不可用时返回 False 降级）."""
producer = await get_producer()
if producer is None:
return False
try:
await producer.send_and_wait(topic, value=payload, key=key)
logger.info(
"kafka_event_published",
topic=topic,
key=key,
event_id=payload.get("event_id"),
action=payload.get("action"),
)
return True
except Exception as exc: # noqa: BLE001
logger.warning(
"kafka_publish_failed_degraded",
topic=topic,
key=key,
error=str(exc),
)
return False
def _gen_event_id() -> str:
"""生成事件 ID（消费者幂等去重依据）."""
return str(uuid.uuid4())
def _now_ms() -> int:
"""当前时间戳（毫秒）."""
return int(time.time() * 1000)
# ===== 派生数据事件发布（Outbox 豁免，总裁裁决 §2.10） =====
async def publish_mastery_updated(
student_id: str,
knowledge_point_id: str,
mastery_level: float,
previous_level: float,
metadata: dict[str, str] | None = None,
) -> bool:
"""发布掌握度更新事件（edu.insight.mastery.updated，action=mastery.updated）.
总裁裁决 §2.10：派生数据事件豁免 Outbox 模式，直接 producer 发布.
"""
payload = {
"event_id": _gen_event_id(),
"action": "mastery.updated",
"occurred_at": _now_ms(),
"student_id": student_id,
"knowledge_point_id": knowledge_point_id,
"mastery_level": mastery_level,
"previous_level": previous_level,
"metadata": metadata or {},
}
# key 用 student_id 保证同学生事件顺序（同一 partition）
return await _publish(
settings.kafka_mastery_topic,
payload,
key=student_id,
)
async def publish_warning_triggered(
target_id: str,
warning_type: str,
threshold: float,
current_value: float,
severity: str = "WARN",
student_id: str = "",
knowledge_point_id: str = "",
metadata: dict[str, str] | None = None,
) -> bool:
"""发布预警触发事件（复用 edu.insight.mastery.updated，action=warning.triggered）.
总裁裁决 §2.11：warning.triggered 复用 mastery topic（避免新增 topic）.
"""
payload = {
"event_id": _gen_event_id(),
"action": "warning.triggered",
"occurred_at": _now_ms(),
"student_id": student_id or target_id, # 兼容 student_id 为空的场景
"knowledge_point_id": knowledge_point_id,
"warning_type": warning_type,
"target_id": target_id,
"threshold": threshold,
"current_value": current_value,
"severity": severity,
"metadata": metadata or {},
}
# key 用 target_id 保证同目标预警事件顺序
return await _publish(
settings.kafka_warning_topic, # 复用 mastery topic
payload,
key=target_id,
)
async def ping() -> bool:
"""Kafka producer 连通性检查（供 /readyz 使用）.
aiokafka 无显式 ping API，已初始化即视为可用.
"""
producer = await get_producer()
return producer is not None

130
services/data-ana/src/data_ana/repository/redis_client.py Normal file
View File

@@ -0,0 +1,130 @@
"""Redis 客户端（DataScope 缓存 + CDC 幂等去重 + 预警去重位图）.
降级策略：redis_url 未配置或不可达时，所有缓存查询跳过（cache miss），
直查 ClickHouse / iam gRPC，不阻塞主流程.
"""
from typing import Any
import structlog
from ..config import settings
logger = structlog.get_logger(__name__)
_client: Any | None = None
_client_initialized: bool = False
async def get_client() -> Any | None:
"""获取 Redis 客户端（惰性初始化，失败返回 None 降级）."""
global _client, _client_initialized
if not settings.redis_url:
return None
if _client_initialized:
return _client
_client_initialized = True
try:
import redis.asyncio as aioredis
_client = aioredis.from_url(
settings.redis_url,
max_connections=settings.redis_pool_size,
socket_timeout=settings.redis_socket_timeout_ms / 1000,
decode_responses=True,
)
logger.info("redis_client_initialized", url=settings.redis_url)
except Exception as exc: # noqa: BLE001
logger.warning("redis_client_init_failed_degraded", error=str(exc))
_client = None
return _client
async def close_client() -> None:
"""关闭客户端."""
global _client, _client_initialized
if _client is not None:
try:
await _client.close()
except Exception as exc: # noqa: BLE001
logger.warning("redis_client_close_failed", error=str(exc))
finally:
_client = None
_client_initialized = False
async def ping() -> bool:
"""Redis 连通性检查（供 /readyz 使用）."""
client = await get_client()
if client is None:
return False
try:
await client.ping()
return True
except Exception as exc: # noqa: BLE001
logger.warning("redis_ping_failed", error=str(exc))
return False
async def get_cache(key: str) -> str | None:
"""读取缓存（不可用时返回 None）."""
client = await get_client()
if client is None:
return None
try:
return await client.get(key)
except Exception as exc: # noqa: BLE001
logger.warning("redis_get_failed", key=key, error=str(exc))
return None
async def set_cache(key: str, value: str, ttl_s: int = 300) -> bool:
"""写入缓存（带 TTL）."""
client = await get_client()
if client is None:
return False
try:
await client.setex(key, ttl_s, value)
return True
except Exception as exc: # noqa: BLE001
logger.warning("redis_set_failed", key=key, error=str(exc))
return False
async def delete_cache(key: str) -> bool:
"""删除缓存."""
client = await get_client()
if client is None:
return False
try:
await client.delete(key)
return True
except Exception as exc: # noqa: BLE001
logger.warning("redis_delete_failed", key=key, error=str(exc))
return False
async def setnx_dedup(key: str, ttl_s: int = 86400) -> bool:
"""幂等去重（SETNX，成功表示首次，失败表示重复）."""
client = await get_client()
if client is None:
return True # Redis 不可用时放行（依赖 ClickHouse 去重）
try:
result = await client.set(key, "1", ex=ttl_s, nx=True)
return bool(result)
except Exception as exc: # noqa: BLE001
logger.warning("redis_setnx_failed", key=key, error=str(exc))
return True # 放行
# ===== 缓存键命名规范（对齐 02-architecture-design.md §10.2） =====
# data_ana:datascope:{user_id} 5min DataScope 缓存
# data_ana:exam:{exam_id} 30day ExamCache（P6 多实例共享）
# data_ana:dedup:{event_id} 7day 事件幂等去重
# data_ana:warning:{target_id}:{type}:{date} 25h 预警去重位图
# data_ana:dashboard:{user_id}:{scope} 5min Dashboard 聚合结果缓存
# data_ana:kp_meta:{knowledge_point_id} 1day 知识点元数据缓存

1
services/data-ana/src/data_ana/shared/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Shared package for data-ana service."""

62
services/data-ana/src/data_ana/shared/action_state.py Normal file
View File

@@ -0,0 +1,62 @@
"""统一响应信封 ActionState（coord-cross-review.md §5.3 P0 整改）.
总裁裁决 §2.6 降级模式：success=true + error=null + data 内 degraded 字段.
成功：{success: true, data: T}
失败：{success: false, error: {code, message, details?, trace_id?}}
降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}}.
"""
from typing import Any
from pydantic import BaseModel
class ActionStateError(BaseModel):
"""错误信息（失败时填充）."""
code: str
message: str
details: dict[str, Any] | None = None
trace_id: str | None = None
class ActionState[T](BaseModel):
"""统一响应信封（coord-cross-review.md §5.3 裁决）.
成功：{success: true, data: T}
失败：{success: false, error: {code, message, details?, trace_id?}}
降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}}
（保留功能但数据可能不完整，降级不是错误）
"""
success: bool
data: T | None = None
error: ActionStateError | None = None
details: dict[str, Any] | None = None # 降级标记放此字段，不放 error
@classmethod
def ok(cls, data: T, *, degraded: bool = False, degraded_reason: str = "") -> "ActionState[T]":
"""成功响应（降级时 degraded=True）."""
if degraded:
details: dict[str, Any] = {"degraded": True}
if degraded_reason:
details["degraded_reason"] = degraded_reason
return cls(success=True, data=data, error=None, details=details)
return cls(success=True, data=data, error=None, details=None)
@classmethod
def fail(
cls,
code: str,
message: str,
*,
trace_id: str | None = None,
details: dict[str, Any] | None = None,
) -> "ActionState[T]":
"""失败响应."""
return cls(
success=False,
data=None,
error=ActionStateError(code=code, message=message, details=details, trace_id=trace_id),
)

66
services/data-ana/src/data_ana/shared/errors.py Normal file
View File

@@ -0,0 +1,66 @@
"""错误码定义（前缀 DATA_ANA_*，对齐 02-architecture-design.md §6.2）.
错误码语义：
- 4xx 客户端错误（权限/参数/资源不存在）
- 5xx 服务端错误（内部异常）
- 200 + degraded:true 降级模式（外部依赖不可用但服务仍可响应骨架数据）
"""
from enum import StrEnum
class ErrorCode(StrEnum):
"""data-ana 错误码枚举."""
# 客户端错误
UNAUTHORIZED = "DATA_ANA_UNAUTHORIZED" # 401 缺失 x-user-id 或 token 无效
FORBIDDEN = "DATA_ANA_FORBIDDEN" # 403 角色无对应权限
DATASCOPE_VIOLATION = "DATA_ANA_DATASCOPE_VIOLATION" # 403 查询目标超出 DataScope
INVALID_DATE_RANGE = "DATA_ANA_INVALID_DATE_RANGE" # 400 start_date > end_date
STUDENT_NOT_FOUND = "DATA_ANA_STUDENT_NOT_FOUND" # 404 student_id 不存在
CLASS_NOT_FOUND = "DATA_ANA_CLASS_NOT_FOUND" # 404 class_id 不存在
WARNING_NOT_FOUND = "DATA_ANA_WARNING_NOT_FOUND" # 404 预警 ID 不存在
RATE_LIMITED = "DATA_ANA_RATE_LIMITED" # 429 触发限流
# 降级模式（200 + degraded:true）
CLICKHOUSE_UNAVAILABLE = "DATA_ANA_CLICKHOUSE_UNAVAILABLE" # ClickHouse 不可达
KAFKA_UNAVAILABLE = "DATA_ANA_KAFKA_UNAVAILABLE" # Kafka producer 不可达
REDIS_UNAVAILABLE = "DATA_ANA_REDIS_UNAVAILABLE" # Redis 不可达
IAM_GRPC_UNAVAILABLE = "DATA_ANA_IAM_GRPC_UNAVAILABLE" # iam gRPC 不可达
# 服务端错误
INTERNAL_ERROR = "DATA_ANA_INTERNAL_ERROR" # 500 未捕获异常
# HTTP 状态码映射
HTTP_STATUS: dict[ErrorCode, int] = {
ErrorCode.UNAUTHORIZED: 401,
ErrorCode.FORBIDDEN: 403,
ErrorCode.DATASCOPE_VIOLATION: 403,
ErrorCode.INVALID_DATE_RANGE: 400,
ErrorCode.STUDENT_NOT_FOUND: 404,
ErrorCode.CLASS_NOT_FOUND: 404,
ErrorCode.WARNING_NOT_FOUND: 404,
ErrorCode.RATE_LIMITED: 429,
ErrorCode.CLICKHOUSE_UNAVAILABLE: 200, # 降级模式
ErrorCode.KAFKA_UNAVAILABLE: 200, # 降级模式
ErrorCode.REDIS_UNAVAILABLE: 200, # 降级模式
ErrorCode.IAM_GRPC_UNAVAILABLE: 200, # 降级模式
ErrorCode.INTERNAL_ERROR: 500,
}
class DataAnaError(Exception):
"""data-ana 业务异常（携带错误码）."""
def __init__(
self,
code: ErrorCode,
message: str,
*,
details: dict[str, str] | None = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.details = details

124
services/data-ana/src/data_ana/shared/permissions.py Normal file
View File

@@ -0,0 +1,124 @@
"""权限点常量与 FastAPI Depends 权限校验（对齐 02-architecture-design.md §6.1）.
Python 服务无 NestJS 装饰器，权限校验用 FastAPI Depends 等价物
（coord-cross-review.md §5.3 已仲裁认可）.
DataScope 6 级：SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL
iam.GetEffectiveDataScope gRPC 解析用户可见数据范围（P4 补全，未就绪时降级兜底）.
"""
from dataclasses import dataclass, field
from enum import StrEnum
from fastapi import Header
class Permission:
"""权限点常量（与 iam 权限点对齐）."""
ANALYTICS_CLASS_READ = "analytics:class:read"
ANALYTICS_STUDENT_READ = "analytics:student:read"
ANALYTICS_TEACHER_DASHBOARD = "analytics:teacher:dashboard"
ANALYTICS_STUDENT_DASHBOARD = "analytics:student:dashboard"
ANALYTICS_PARENT_DASHBOARD = "analytics:parent:dashboard"
ANALYTICS_ADMIN_DASHBOARD = "analytics:admin:dashboard"
ANALYTICS_WARNING_READ = "analytics:warning:read"
class DataScopeLevel(StrEnum):
"""DataScope 6 级."""
SELF = "SELF"
CLASS = "CLASS"
GRADE = "GRADE"
SCHOOL = "SCHOOL"
DISTRICT = "DISTRICT"
ALL = "ALL"
# 角色 → 默认 DataScope 降级映射（iam gRPC 不可达时使用）
_ROLE_DATASCOPE_FALLBACK: dict[str, DataScopeLevel] = {
"admin": DataScopeLevel.ALL,
"school_admin": DataScopeLevel.SCHOOL,
"teacher": DataScopeLevel.CLASS,
"student": DataScopeLevel.SELF,
"parent": DataScopeLevel.SELF,
}
@dataclass
class UserContext:
"""用户上下文（从请求头提取）."""
user_id: str
roles: list[str] = field(default_factory=list)
@dataclass
class DataScope:
"""数据范围（查询层注入 WHERE）."""
level: DataScopeLevel
scope_ids: list[str] = field(default_factory=list) # class_id 列表 / grade_id 列表
school_id: str = ""
degraded: bool = False # iam 不可达时降级标记
degraded_reason: str = ""
async def get_user_context(
x_user_id: str = Header("", alias="x-user-id"),
x_user_roles: str = Header("", alias="x-user-roles"),
) -> UserContext:
"""从请求头提取用户上下文（FastAPI Depends）.
Gateway 注入 x-user-id / x-user-roles 头（JWT 校验后）.
"""
roles = [r.strip() for r in x_user_roles.split(",") if r.strip()] if x_user_roles else []
return UserContext(user_id=x_user_id, roles=roles)
def get_datascope_fallback(user: UserContext) -> DataScope:
"""按角色映射默认 DataScope（降级兜底，iam gRPC 不可达时使用）."""
for role in user.roles:
if role in _ROLE_DATASCOPE_FALLBACK:
level = _ROLE_DATASCOPE_FALLBACK[role]
return DataScope(
level=level,
degraded=True,
degraded_reason="iam_grpc_unavailable_fallback",
)
# 默认最保守范围
return DataScope(
level=DataScopeLevel.SELF,
degraded=True,
degraded_reason="unknown_role_fallback_self",
)
def build_datascope_where(
scope: DataScope,
student_col: str = "student_id",
) -> tuple[str, dict[str, str]]:
"""构建 DataScope WHERE 子句（注入到 ClickHouse 查询）.
返回 (where_clause, parameters).
"""
params: dict[str, str] = {}
if scope.level == DataScopeLevel.ALL:
return "", params
if scope.level == DataScopeLevel.SELF:
# SELF 范围需要 user_id，由调用方注入
return "", params
if scope.level == DataScopeLevel.CLASS and scope.scope_ids:
# 班级范围：class_id IN (...)
placeholders = []
for i, cid in enumerate(scope.scope_ids):
key = f"class_id_{i}"
params[key] = cid
placeholders.append(f"{{{key}:String}}")
return f"class_id IN ({', '.join(placeholders)})", params
if scope.level == DataScopeLevel.SCHOOL:
# 学校范围：不做过滤（单租户场景下等价 ALL）
return "", params
# GRADE / DISTRICT 暂按 ALL 处理（P6 细化）
return "", params

383
services/data-ana/src/data_ana/warning_service.py Normal file
View File

@@ -0,0 +1,383 @@
"""预警服务（5 类预警阈值评估 + 去重位图 + 事件发布）.
对齐 02-architecture-design.md §11 预警体系：
- LOW_MASTERY：掌握度 < 0.4
- CRITICAL_LOW：掌握度 < 0.2
- SCORE_DROP：成绩环比下降 ≥ 20%
- ABSENT_FREQUENT：缺勤 ≥ 3 次/周
- TREND_DECLINE：连续 3 次成绩下降
去重策略：
Redis bitmap key = data_ana:warning:{target_id}:{type}:{date}
TTL = 25h（跨天容忍，避免边界重复触发）
事件发布：
warning.triggered 复用 edu.insight.mastery.updated topic（总裁裁决 §2.11）
Outbox 豁免（总裁裁决 §2.10）
"""
from datetime import UTC, datetime
from typing import Any
import structlog
from .config import settings
from .repository import clickhouse_repository, kafka_producer, redis_client
logger = structlog.get_logger(__name__)
# 预警类型常量
class WarningType:
"""预警类型."""
LOW_MASTERY = "LOW_MASTERY"
CRITICAL_LOW = "CRITICAL_LOW"
SCORE_DROP = "SCORE_DROP"
ABSENT_FREQUENT = "ABSENT_FREQUENT"
TREND_DECLINE = "TREND_DECLINE"
class Severity:
"""预警严重程度."""
INFO = "INFO"
WARN = "WARN"
CRITICAL = "CRITICAL"
# 预警类型 → 严重程度映射
_SEVERITY_MAP: dict[str, str] = {
WarningType.LOW_MASTERY: Severity.WARN,
WarningType.CRITICAL_LOW: Severity.CRITICAL,
WarningType.SCORE_DROP: Severity.WARN,
WarningType.ABSENT_FREQUENT: Severity.WARN,
WarningType.TREND_DECLINE: Severity.WARN,
}
def _warning_dedup_key(target_id: str, warning_type: str) -> str:
"""预警去重位图 key（25h TTL，避免边界重复触发）."""
today = datetime.now(UTC).strftime("%Y%m%d")
return f"data_ana:warning:{target_id}:{warning_type}:{today}"
async def _check_and_dedup(target_id: str, warning_type: str) -> bool:
"""检查去重（首次返回 True，已触发过返回 False）."""
key = _warning_dedup_key(target_id, warning_type)
is_first = await redis_client.setnx_dedup(key, ttl_s=25 * 3600)
if not is_first:
logger.info(
"warning_dedup_skipped",
target_id=target_id,
warning_type=warning_type,
)
return is_first
async def _trigger_warning(
target_id: str,
warning_type: str,
threshold: float,
current_value: float,
student_id: str = "",
knowledge_point_id: str = "",
metadata: dict[str, str] | None = None,
) -> dict[str, Any]:
"""触发预警（去重 + 事件发布）."""
severity = _SEVERITY_MAP.get(warning_type, Severity.WARN)
# 1. 去重检查
should_trigger = await _check_and_dedup(target_id, warning_type)
if not should_trigger:
return {
"target_id": target_id,
"warning_type": warning_type,
"triggered": False,
"deduplicated": True,
"threshold": threshold,
"current_value": current_value,
}
# 2. 发布 warning.triggered 事件（Outbox 豁免，复用 mastery topic）
published = await kafka_producer.publish_warning_triggered(
target_id=target_id,
warning_type=warning_type,
threshold=threshold,
current_value=current_value,
severity=severity,
student_id=student_id,
knowledge_point_id=knowledge_point_id,
metadata=metadata,
)
logger.warning(
"warning_triggered",
target_id=target_id,
warning_type=warning_type,
severity=severity,
threshold=threshold,
current_value=current_value,
published=published,
)
return {
"target_id": target_id,
"warning_type": warning_type,
"severity": severity,
"triggered": True,
"deduplicated": False,
"threshold": threshold,
"current_value": current_value,
"event_published": published,
}
async def evaluate_mastery_warning(
student_id: str,
knowledge_point_id: str,
mastery_level: float,
) -> list[dict[str, Any]]:
"""评估掌握度预警（LOW_MASTERY / CRITICAL_LOW）.
- mastery < 0.2 → CRITICAL_LOW（最严重，先评估）
- 0.2 ≤ mastery < 0.4 → LOW_MASTERY
"""
triggered: list[dict[str, Any]] = []
if mastery_level < settings.warning_critical_mastery_threshold:
result = await _trigger_warning(
target_id=student_id,
warning_type=WarningType.CRITICAL_LOW,
threshold=settings.warning_critical_mastery_threshold,
current_value=mastery_level,
student_id=student_id,
knowledge_point_id=knowledge_point_id,
)
if result["triggered"]:
triggered.append(result)
return triggered # CRITICAL_LOW 已触发，不再触发 LOW_MASTERY
if mastery_level < settings.warning_low_mastery_threshold:
result = await _trigger_warning(
target_id=student_id,
warning_type=WarningType.LOW_MASTERY,
threshold=settings.warning_low_mastery_threshold,
current_value=mastery_level,
student_id=student_id,
knowledge_point_id=knowledge_point_id,
)
if result["triggered"]:
triggered.append(result)
return triggered
async def evaluate_score_drop(
student_id: str,
current_score: float,
previous_score: float,
exam_id: str = "",
) -> dict[str, Any] | None:
"""评估成绩下降预警（SCORE_DROP）.
成绩环比下降 ≥ warning_score_drop_percent（默认 20%）触发.
"""
if previous_score <= 0:
return None
drop_ratio = (previous_score - current_score) / previous_score
if drop_ratio < settings.warning_score_drop_percent:
return None
return await _trigger_warning(
target_id=student_id,
warning_type=WarningType.SCORE_DROP,
threshold=settings.warning_score_drop_percent,
current_value=round(drop_ratio, 4),
student_id=student_id,
metadata={
"exam_id": exam_id,
"previous_score": str(previous_score),
"current_score": str(current_score),
},
)
async def evaluate_absent_frequent(
student_id: str,
absent_count_week: int,
) -> dict[str, Any] | None:
"""评估缺勤频繁预警（ABSENT_FREQUENT）.
一周内缺勤 ≥ warning_absent_per_week（默认 3 次）触发.
"""
if absent_count_week < settings.warning_absent_per_week:
return None
return await _trigger_warning(
target_id=student_id,
warning_type=WarningType.ABSENT_FREQUENT,
threshold=float(settings.warning_absent_per_week),
current_value=float(absent_count_week),
student_id=student_id,
)
async def evaluate_trend_decline(
student_id: str,
recent_scores: list[float],
) -> dict[str, Any] | None:
"""评估趋势下降预警（TREND_DECLINE）.
连续 3 次成绩下降触发（recent_scores 按时间正序，最早在前）.
"""
if len(recent_scores) < 3:
return None
# 检查最近 3 次是否连续下降
last_three = recent_scores[-3:]
is_declining = all(last_three[i] > last_three[i + 1] for i in range(len(last_three) - 1))
if not is_declining:
return None
drop = last_three[-2] - last_three[-1]
return await _trigger_warning(
target_id=student_id,
warning_type=WarningType.TREND_DECLINE,
threshold=0.0, # 下降幅度阈值（0 表示只要下降就触发）
current_value=round(drop, 4),
student_id=student_id,
)
async def evaluate_all_warnings(
student_id: str,
knowledge_point_id: str = "",
mastery_level: float | None = None,
) -> list[dict[str, Any]]:
"""综合评估学生所有预警类型（GetWarnings RPC 调用）.
参数：
- student_id：学生 ID
- knowledge_point_id：知识点 ID（可选，指定则只评估该知识点）
- mastery_level：预计算的掌握度（可选，未提供则查询）
返回：触发的预警列表
"""
triggered: list[dict[str, Any]] = []
# 1. 掌握度预警
if mastery_level is None:
snapshot = await clickhouse_repository.query_mastery_snapshot(
student_id=student_id,
)
if snapshot is not None:
knowledge_points = snapshot.get("knowledgePoints", [])
if knowledge_point_id:
# 仅评估指定知识点
knowledge_points = [
kp
for kp in knowledge_points
if kp.get("knowledge_point_id") == knowledge_point_id
]
for kp in knowledge_points:
level = kp.get("mastery_level", 0.0)
warnings = await evaluate_mastery_warning(
student_id=student_id,
knowledge_point_id=kp.get("knowledge_point_id", ""),
mastery_level=level,
)
triggered.extend(warnings)
else:
warnings = await evaluate_mastery_warning(
student_id=student_id,
knowledge_point_id=knowledge_point_id,
mastery_level=mastery_level,
)
triggered.extend(warnings)
# 2. 缺勤预警
attendance = await clickhouse_repository.query_attendance(student_id=student_id)
if attendance is not None:
# 简化：统计总缺勤数（P6 改为按周统计）
absent_count = attendance.get("absentCount", 0)
if absent_count >= settings.warning_absent_per_week:
warning = await evaluate_absent_frequent(
student_id=student_id,
absent_count_week=absent_count,
)
if warning is not None and warning.get("triggered"):
triggered.append(warning)
# 3. 成绩下降预警（需要查询最近两次成绩）
trend = await clickhouse_repository.query_learning_trend(student_id=student_id)
if trend is not None:
points = trend.get("points", [])
if len(points) >= 2:
current = points[-1].get("score", 0.0)
previous = points[-2].get("score", 0.0)
drop_warning = await evaluate_score_drop(
student_id=student_id,
current_score=current,
previous_score=previous,
)
if drop_warning is not None and drop_warning.get("triggered"):
triggered.append(drop_warning)
# 4. 趋势下降预警
scores = [p.get("score", 0.0) for p in points]
trend_warning = await evaluate_trend_decline(
student_id=student_id,
recent_scores=scores,
)
if trend_warning is not None and trend_warning.get("triggered"):
triggered.append(trend_warning)
return triggered
async def get_warnings(
student_id: str,
warning_type: str = "",
) -> list[dict[str, Any]]:
"""查询预警列表（GetWarnings RPC 实现）.
参数：
- student_id：学生 ID
- warning_type：预警类型过滤（空表示全部）
返回：预警列表（每条含 target_id / warning_type / severity / threshold /
current_value / triggered_at）
"""
# 当前实现：实时评估返回触发结果
# P6 演进：将预警历史持久化到 ClickHouse 供查询
triggered = await evaluate_all_warnings(student_id=student_id)
if warning_type:
triggered = [w for w in triggered if w.get("warning_type") == warning_type]
return triggered
async def trigger_warning_manual(
target_id: str,
warning_type: str,
threshold: float,
current_value: float,
student_id: str = "",
knowledge_point_id: str = "",
severity: str = "",
metadata: dict[str, str] | None = None,
) -> dict[str, Any]:
"""手动触发预警（TriggerWarning RPC 实现，管理员/API 用）."""
# 覆盖默认 severity
if severity:
_SEVERITY_MAP[warning_type] = severity
return await _trigger_warning(
target_id=target_id,
warning_type=warning_type,
threshold=threshold,
current_value=current_value,
student_id=student_id,
knowledge_point_id=knowledge_point_id,
metadata=metadata,
)

32
uv.lock generated
View File

@@ -270,19 +270,23 @@ wheels = [
[[package]] [[package]]
name = "data-ana-service" name = "data-ana-service"
version = "0.1.0" version = "1.0.0"
source = { virtual = "services/data-ana" } source = { virtual = "services/data-ana" }
dependencies = [ dependencies = [
{ name = "aiokafka" }, { name = "aiokafka" },
{ name = "clickhouse-connect" }, { name = "clickhouse-connect" },
{ name = "fastapi" }, { name = "fastapi" },
{ name = "grpcio" },
{ name = "grpcio-health-checking" },
{ name = "opentelemetry-api" }, { name = "opentelemetry-api" },
{ name = "opentelemetry-exporter-otlp" }, { name = "opentelemetry-exporter-otlp" },
{ name = "opentelemetry-instrumentation-fastapi" }, { name = "opentelemetry-instrumentation-fastapi" },
{ name = "opentelemetry-sdk" }, { name = "opentelemetry-sdk" },
{ name = "prometheus-client" }, { name = "prometheus-client" },
{ name = "protobuf" },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pydantic-settings" }, { name = "pydantic-settings" },
{ name = "redis" },
{ name = "structlog" }, { name = "structlog" },
{ name = "uvicorn", extra = ["standard"] }, { name = "uvicorn", extra = ["standard"] },
] ]
@@ -292,13 +296,17 @@ requires-dist = [
{ name = "aiokafka", specifier = ">=0.11.0" }, { name = "aiokafka", specifier = ">=0.11.0" },
{ name = "clickhouse-connect", specifier = ">=0.7.0" }, { name = "clickhouse-connect", specifier = ">=0.7.0" },
{ name = "fastapi", specifier = ">=0.115.0" }, { name = "fastapi", specifier = ">=0.115.0" },
{ name = "grpcio", specifier = ">=1.66.0" },
{ name = "grpcio-health-checking", specifier = ">=1.66.0" },
{ name = "opentelemetry-api", specifier = ">=1.27.0" }, { name = "opentelemetry-api", specifier = ">=1.27.0" },
{ name = "opentelemetry-exporter-otlp", specifier = ">=1.27.0" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.27.0" },
{ name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" }, { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" },
{ name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "opentelemetry-sdk", specifier = ">=1.27.0" },
{ name = "prometheus-client", specifier = ">=0.20.0" }, { name = "prometheus-client", specifier = ">=0.20.0" },
{ name = "protobuf", specifier = ">=5.28.0" },
{ name = "pydantic", specifier = ">=2.9.0" }, { name = "pydantic", specifier = ">=2.9.0" },
{ name = "pydantic-settings", specifier = ">=2.5.0" }, { name = "pydantic-settings", specifier = ">=2.5.0" },
{ name = "redis", specifier = ">=5.0.0" },
{ name = "structlog", specifier = ">=24.4.0" }, { name = "structlog", specifier = ">=24.4.0" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
] ]
@@ -372,6 +380,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" },
] ]
[[package]]
name = "grpcio-health-checking"
version = "1.82.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "grpcio" },
{ name = "protobuf" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c6/64/060c857a962dae39cac69433f73145acd825b5348fad53908f3439a6fca8/grpcio_health_checking-1.82.1.tar.gz", hash = "sha256:86255e04e1d39f1c97a6632d41b63249351408de5c58e85eede87afd7d9828dd", size = 17130, upload-time = "2026-07-08T12:39:41.138Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/aa/da280870eca03223fb1c15e5c5482ebd42a523a227e496d9b490e8fdaea5/grpcio_health_checking-1.82.1-py3-none-any.whl", hash = "sha256:622ed6663daf0b8c9dedb4e95a48f6db080a8129922742b5141c63cfab373219", size = 19121, upload-time = "2026-07-08T12:39:25.629Z" },
]
[[package]] [[package]]
name = "h11" name = "h11"
version = "0.16.0" version = "0.16.0"
@@ -854,6 +875,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
] ]
[[package]]
name = "redis"
version = "8.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" },
]
[[package]] [[package]]
name = "requests" name = "requests"
version = "2.34.2" version = "2.34.2"