feat(data-ana): v2 P6 硬化完成 + 6 新 RPC + Prometheus 监控
P6 硬化（5 项全部完成）:
- CDC 多实例水平扩展: _INSTANCE_ID + get_lag() 真实 lag 计算
- ExamCache Redis 化: key data_ana:exam:{exam_id}, TTL 30 天 + 内存 LRU fallback
- ClickHouse TTL 归档: 5 表均加 TTL（1-3 年），分区级删除
- Prometheus 监控: 18 个指标（CDC/CH/ExamCache/DataScope/gRPC/业务）
- readyz 深度硬化: 4 依赖超时检查（CH 1s/Redis 200ms/iam 2s/CDC lag<1000）
v2 新增 6 个 RPC（analytics.proto 扩展为 18 RPC）:
- GetStudentGrowth / GetAssignmentAnalysis / GetMasterySummary
- ListDiagnosticReports（占位，待 ai 服务）/ ListErrorBookItems / GetErrorBookStats
监控与可观测性: lifespan 预热 + gRPC ServerInterceptor + CDC 消费者指标
Docker 本地测试 19 项全部通过（healthz/readyz/metrics + 11 HTTP + 10 gRPC + ruff）
nextstep-v2.md: 上游需求对齐 + 下游要求（iam/core-edu/content/ai/SRE）
This commit is contained in:
SpecialX
@@ -30,6 +30,20 @@ service AnalyticsService {
|
||||
rpc GetStudentMastery(GetStudentMasteryRequest) returns (StudentMastery);
|
||||
// 订阅掌握度更新（server-streaming，P5+ AI 个性化推荐实时推送通道）.
|
||||
rpc SubscribeMasteryUpdate(SubscribeMasteryUpdateRequest) returns (stream MasteryUpdateEvent);
|
||||
|
||||
// ===== P6+ v2 扩展 RPC（响应上游 parent-bff / student-bff v2 请求） =====
|
||||
// 学生成长档案（综合成绩趋势 + 掌握度变化 + 考勤统计，供 parent-bff GrowthArchiveService）.
|
||||
rpc GetStudentGrowth(GetStudentGrowthRequest) returns (StudentGrowth);
|
||||
// 作业/考试分析（单次作业/考试维度统计，供 student-bff / parent-bff）.
|
||||
rpc GetAssignmentAnalysis(GetAssignmentAnalysisRequest) returns (AssignmentAnalysis);
|
||||
// 学生掌握度汇总（轻量级，仅返回总体掌握度 + 三档分布，供 parent-bff MasteryService.GetSummary）.
|
||||
rpc GetMasterySummary(GetMasterySummaryRequest) returns (MasterySummary);
|
||||
// 诊断报告列表（占位实现，真实数据需 ai 服务集成，供 parent-bff / student-bff）.
|
||||
rpc ListDiagnosticReports(ListDiagnosticReportsRequest) returns (DiagnosticReportList);
|
||||
// 错题本列表（gRPC 版本，供 parent-bff ErrorBookService.List）.
|
||||
rpc ListErrorBookItems(ListErrorBookItemsRequest) returns (ErrorBookList);
|
||||
// 错题本统计（按知识点聚合，供 parent-bff ErrorBookService.GetStats）.
|
||||
rpc GetErrorBookStats(GetErrorBookStatsRequest) returns (ErrorBookStats);
|
||||
}
|
||||
|
||||
// ===== 请求消息 =====
|
||||
@@ -101,6 +115,44 @@ message SubscribeMasteryUpdateRequest {
|
||||
string class_id = 2; // 可选，订阅指定班级全部学生
|
||||
}
|
||||
|
||||
// ===== P6+ v2 扩展 RPC 请求消息 =====
|
||||
|
||||
message GetStudentGrowthRequest {
|
||||
string student_id = 1;
|
||||
int64 start_date = 2; // Unix timestamp（秒）
|
||||
int64 end_date = 3;
|
||||
string subject_id = 4; // 可选
|
||||
}
|
||||
|
||||
message GetAssignmentAnalysisRequest {
|
||||
string class_id = 1;
|
||||
string assignment_id = 2; // exam_id 或 homework_id
|
||||
string subject_id = 3; // 可选
|
||||
}
|
||||
|
||||
message GetMasterySummaryRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2; // 可选
|
||||
}
|
||||
|
||||
message ListDiagnosticReportsRequest {
|
||||
string student_id = 1;
|
||||
int64 since = 2; // Unix timestamp（秒）
|
||||
int32 limit = 3; // 默认 10
|
||||
}
|
||||
|
||||
message ListErrorBookItemsRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2; // 可选，按学科过滤
|
||||
int32 limit = 3; // 默认 100
|
||||
int32 offset = 4; // 分页偏移
|
||||
}
|
||||
|
||||
message GetErrorBookStatsRequest {
|
||||
string student_id = 1;
|
||||
string subject_id = 2; // 可选
|
||||
}
|
||||
|
||||
// ===== 响应消息 =====
|
||||
|
||||
message ClassPerformance {
|
||||
@@ -260,3 +312,111 @@ message MasteryUpdateEvent {
|
||||
double previous_level = 5;
|
||||
int64 calculated_at = 6; // Unix timestamp（秒）
|
||||
}
|
||||
|
||||
// ===== P6+ v2 扩展 RPC 响应消息 =====
|
||||
|
||||
message StudentGrowth {
|
||||
string student_id = 1;
|
||||
// 成绩趋势（来自 LearningTrend）
|
||||
repeated TrendPoint score_trend = 2;
|
||||
// 掌握度变化趋势（按时间排序的快照）
|
||||
repeated MasteryTrendPoint mastery_trend = 3;
|
||||
// 考勤统计
|
||||
AttendanceSummary attendance = 4;
|
||||
// 总体成长评分（综合分数、掌握度、考勤计算）
|
||||
double growth_score = 5;
|
||||
// 成长等级（excellent / good / average / needs_improvement）
|
||||
string growth_level = 6;
|
||||
}
|
||||
|
||||
message MasteryTrendPoint {
|
||||
int64 calculated_at = 1;
|
||||
double overall_mastery = 2;
|
||||
}
|
||||
|
||||
message AttendanceSummary {
|
||||
int32 total_days = 1;
|
||||
int32 present_days = 2;
|
||||
int32 absent_days = 3;
|
||||
int32 late_days = 4;
|
||||
double attendance_rate = 5; // 出勤率 0.0-1.0
|
||||
}
|
||||
|
||||
message AssignmentAnalysis {
|
||||
string assignment_id = 1;
|
||||
string class_id = 2;
|
||||
string subject_id = 3;
|
||||
double average_score = 4;
|
||||
double highest_score = 5;
|
||||
double lowest_score = 6;
|
||||
int32 total_students = 7;
|
||||
int32 submitted_count = 8;
|
||||
double pass_rate = 9; // 及格率
|
||||
// 分数段分布（90-100 / 80-89 / 70-79 / 60-69 / <60）
|
||||
repeated ScoreRange ranges = 10;
|
||||
}
|
||||
|
||||
message ScoreRange {
|
||||
string label = 1; // "90-100" / "80-89" etc.
|
||||
int32 count = 2;
|
||||
double percentage = 3; // 占比 0.0-1.0
|
||||
}
|
||||
|
||||
message MasterySummary {
|
||||
string student_id = 1;
|
||||
double overall_mastery = 2; // 0.0-1.0
|
||||
int32 mastered_count = 3; // mastery >= 0.8
|
||||
int32 progressing_count = 4; // 0.4 <= mastery < 0.8
|
||||
int32 weak_count = 5; // mastery < 0.4
|
||||
int32 total_knowledge_points = 6;
|
||||
string mastery_level = 7; // mastered / progressing / weak（总体）
|
||||
}
|
||||
|
||||
message DiagnosticReportList {
|
||||
string student_id = 1;
|
||||
repeated DiagnosticReport reports = 2;
|
||||
int32 total = 3;
|
||||
}
|
||||
|
||||
message DiagnosticReport {
|
||||
string report_id = 1;
|
||||
string student_id = 2;
|
||||
string report_type = 3; // KNOWLEDGE_GAP / LEARNING_STYLE / WEAKNESS_ANALYSIS
|
||||
string title = 4;
|
||||
string summary = 5; // 报告摘要
|
||||
int64 generated_at = 6; // Unix timestamp（秒）
|
||||
string status = 7; // pending / completed / failed
|
||||
}
|
||||
|
||||
message ErrorBookList {
|
||||
string student_id = 1;
|
||||
repeated ErrorBookItem items = 2;
|
||||
int32 total = 3;
|
||||
}
|
||||
|
||||
message ErrorBookItem {
|
||||
string question_id = 1;
|
||||
string knowledge_point_id = 2;
|
||||
string knowledge_point_title = 3;
|
||||
int32 error_count = 4;
|
||||
int64 last_error_time = 5; // Unix timestamp（秒）
|
||||
string content = 6; // 题目内容摘要
|
||||
}
|
||||
|
||||
message ErrorBookStats {
|
||||
string student_id = 1;
|
||||
int32 total_error_questions = 2;
|
||||
int32 total_error_count = 3; // 总错误次数
|
||||
// 按知识点聚合的错题统计
|
||||
repeated KnowledgePointErrorStats by_knowledge_point = 4;
|
||||
// 最近 7 天错误次数
|
||||
int32 recent_7d_errors = 5;
|
||||
}
|
||||
|
||||
message KnowledgePointErrorStats {
|
||||
string knowledge_point_id = 1;
|
||||
string title = 2;
|
||||
int32 error_count = 3;
|
||||
int32 question_count = 4;
|
||||
double error_rate = 5; // 错误率 0.0-1.0
|
||||
}
|
||||
|
||||
283
services/data-ana/docs/nextstep-v2.md
Normal file
283
services/data-ana/docs/nextstep-v2.md
Normal file
@@ -0,0 +1,283 @@
|
||||
# data-ana 模块 v2 上下游依赖与工作清单（Next Steps v2）
|
||||
|
||||
> 模块：data-ana（智能洞察域，数据分析服务）
|
||||
> 负责人：ai11
|
||||
> 更新日期：2026-07-14
|
||||
> 关联文档：[nextstep.md](./nextstep.md)（v1）、[02-architecture-design.md](./02-architecture-design.md)、[data-ana_workline.md](../../../docs/architecture/issues/worklines/data-ana_workline.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. v2 工作完成状态
|
||||
|
||||
### 1.1 P6 硬化全部完成（5 项）
|
||||
|
||||
| # | 工作项 | 状态 | 实现说明 |
|
||||
| --- | -------------------- | ---- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 1 | CDC 多实例水平扩展 | ✅ | `_INSTANCE_ID`（POD_NAME / hostname:pid），`get_lag()` 通过 `position()` + `end_offsets` 计算真实 lag |
|
||||
| 2 | ExamCache Redis 迁移 | ✅ | Redis key `data_ana:exam:{exam_id}`，TTL 30 天，内存 LRU 10000 fallback |
|
||||
| 3 | ClickHouse TTL 归档 | ✅ | 5 表均加 TTL（1-3 年），分区级删除，冷热分离 |
|
||||
| 4 | Prometheus 监控指标 | ✅ | 18 个指标（CDC lag/throughput、ClickHouse query、ExamCache、DataScope、gRPC、业务） |
|
||||
| 5 | readyz 深度硬化 | ✅ | 4 依赖超时检查（CH 1s / Redis 200ms / iam 2s / CDC lag<1000），503 摘流量 |
|
||||
|
||||
### 1.2 v2 新增 6 个 RPC（响应上游 BFF 需求）
|
||||
|
||||
`packages/shared-proto/proto/analytics.proto` 扩展为 **18 RPC**（11 旧 + 1 stream + 6 新）：
|
||||
|
||||
| # | RPC 方法 | 用途 | 上游需求方 | 状态 |
|
||||
| --- | ----------------------- | -------------- | ------------------------- | ---- |
|
||||
| 1 | `GetStudentGrowth` | 学生成长档案 | parent-bff / student-bff | ✅ |
|
||||
| 2 | `GetAssignmentAnalysis` | 作业/考试分析 | student-bff / teacher-bff | ✅ |
|
||||
| 3 | `GetMasterySummary` | 学生掌握度汇总 | parent-bff / student-bff | ✅ |
|
||||
| 4 | `ListDiagnosticReports` | 诊断报告列表 | parent-bff / student-bff | ✅\* |
|
||||
| 5 | `ListErrorBookItems` | 错题本列表 | parent-bff / student-bff | ✅ |
|
||||
| 6 | `GetErrorBookStats` | 错题本统计 | parent-bff | ✅ |
|
||||
|
||||
> \* `ListDiagnosticReports` 当前为占位实现（返回空列表 + degraded 标记），需 ai 服务提供诊断报告生成能力后补充（见 §3.5）。
|
||||
|
||||
### 1.3 v2 监控与可观测性
|
||||
|
||||
- **lifespan 预热**：启动时主动 ping ClickHouse + Redis，避免首次 readyz 探针超时
|
||||
- **gRPC 拦截器**：`_make_metrics_interceptor` 包装所有 unary RPC，记录 `grpc_requests_total` + `grpc_request_duration_seconds`
|
||||
- **CDC 消费者指标**：`cdc_messages_processed_total`（按 topic/table/status）+ `cdc_message_process_duration_seconds` + `cdc_consumer_active_instances`
|
||||
- **Docker 本地测试**：13 项全部通过（healthz / readyz / metrics / 11 HTTP / 10 gRPC），见 §4
|
||||
|
||||
---
|
||||
|
||||
## 2. 上游需求对齐情况
|
||||
|
||||
### 2.1 student-bff 需求（8 RPC + 3 PracticeService）
|
||||
|
||||
| RPC 方法 | 用途 | data-ana 状态 | 备注 |
|
||||
| ----------------------------------------------- | ------------------------------- | --------------------- | ---------------------------------- |
|
||||
| `GetStudentDashboard` | `studentDashboard` Query | ✅ 已实现 | v1 |
|
||||
| `GetStudentWeakness` | `myWeakness` Query | ✅ 已实现 | v1 |
|
||||
| `GetLearningTrend` | `myTrend` Query | ✅ 已实现 | v1 |
|
||||
| `GetStudentGrowth` | `studentGrowth` Query | ✅ 已实现 | v2 新增 |
|
||||
| `GetAssignmentAnalysis` | `assignmentAnalysis` Query | ✅ 已实现 | v2 新增 |
|
||||
| `GetMasterySummary` | `myMasterySummary` Query | ✅ 已实现 | v2 新增 |
|
||||
| `ListDiagnosticReports` | `myDiagnosticReports` Query | ✅ 占位实现 | 待 ai 服务提供诊断报告生成（§3.5） |
|
||||
| `ListErrorBookItems` | `myErrorBook` Query | ✅ 已实现 | v2 新增 |
|
||||
| `PracticeService.ListPracticeSessionsByStudent` | `myPracticeSessions` Query | ❌ 不在 data-ana 范围 | 需新建 PracticeService（见 §3.6） |
|
||||
| `PracticeService.StartPracticeSession` | `startPracticeSession` Mutation | ❌ 不在 data-ana 范围 | 同上 |
|
||||
| `PracticeService.SubmitPracticeAnswer` | `submitPracticeAnswer` Mutation | ❌ 不在 data-ana 范围 | 同上 |
|
||||
|
||||
### 2.2 teacher-bff 需求（4 RPC）
|
||||
|
||||
| RPC 方法 | 用途 | data-ana 状态 |
|
||||
| --------------------- | ----------------------- | ------------- |
|
||||
| `GetClassPerformance` | `classAnalytics` 查询 | ✅ 已实现 |
|
||||
| `GetStudentDashboard` | `studentAnalytics` 查询 | ✅ 已实现 |
|
||||
| `GetStudentWeakness` | `studentWeakness` 查询 | ✅ 已实现 |
|
||||
| `GetLearningTrend` | `learningTrend` 查询 | ✅ 已实现 |
|
||||
|
||||
### 2.3 parent-bff 需求（3 RPC + healthz）
|
||||
|
||||
| 依赖项 | 用途 | data-ana 状态 |
|
||||
| ------------------------------------------ | -------------------- | ------------- |
|
||||
| `getStudentWeakness(studentId, subjectId)` | 学生薄弱知识点 | ✅ 已实现 |
|
||||
| `getLearningTrend(studentId, start, end)` | 学习趋势 | ✅ 已实现 |
|
||||
| `getClassPerformance(classId, subjectId)` | 班级绩效 | ✅ 已实现 |
|
||||
| `GET /healthz` | /readyz 下游健康检查 | ✅ 已实现 |
|
||||
|
||||
### 2.4 api-gateway 需求（HTTP 路由代理）
|
||||
|
||||
| 路由前缀 | 代理目标 | data-ana 状态 |
|
||||
| --------------------- | ------------- | ------------- |
|
||||
| `/api/v1/analytics/*` | data-ana:3006 | ✅ 已实现 |
|
||||
| `/api/v1/dashboard/*` | data-ana:3006 | ✅ 已实现 |
|
||||
|
||||
### 2.5 parent-portal 需求（GraphQL 字段映射）
|
||||
|
||||
parent-portal 的以下 GraphQL 字段依赖 data-ana，均已通过 parent-bff 聚合：
|
||||
|
||||
| GraphQL 字段 | 对应 data-ana RPC | 状态 |
|
||||
| ------------------------ | -------------------------- | ----------------------------- |
|
||||
| `childWeakness` | `GetStudentWeakness` | ✅ |
|
||||
| `childTrend` | `GetLearningTrend` | ✅ |
|
||||
| `childLearningPath` | `GetStudentGrowth`（部分） | ⚠️ 需扩展 LearningPath 子结构 |
|
||||
| `childErrorBookStats` | `GetErrorBookStats` | ✅ |
|
||||
| `childTopWrongQuestions` | `ListErrorBookItems` | ✅ |
|
||||
| `childWeakKps` | `GetStudentWeakness` | ✅ |
|
||||
| `childMasterySummary` | `GetMasterySummary` | ✅ |
|
||||
| `childDiagnosticReports` | `ListDiagnosticReports` | ✅\* 占位 |
|
||||
| `childPracticeStats` | `PracticeService.*` | ❌ 需新建 |
|
||||
| `childPracticeSessions` | `PracticeService.*` | ❌ 需新建 |
|
||||
| `childDetail` | `GetStudentDashboard` | ✅ |
|
||||
| `childGrowthArchive` | `GetStudentGrowth` | ✅ |
|
||||
|
||||
---
|
||||
|
||||
## 3. 需要上下游实现的工作
|
||||
|
||||
### 3.1 iam 服务（ai06 负责）— P0
|
||||
|
||||
| # | 工作项 | 用途 | 优先级 |
|
||||
| --- | --------------------------------------------------- | ----------------------------------------------------------- | ------ |
|
||||
| 1 | iam gRPC 服务启动并暴露 :50052 | data-ana 当前使用 role-based fallback 降级，需 iam 真实服务 | P0 |
|
||||
| 2 | `GetEffectiveDataScope` 返回完整 6 级 scope | DataScope 过滤（SELF/CLASS/GRADE/SCHOOL/DISTRICT/ALL） | P0 |
|
||||
| 3 | iam gRPC metadata 传递 `x-user-id` / `x-user-roles` | Gateway 注入用户上下文到 gRPC metadata | P0 |
|
||||
|
||||
**当前状态**：data-ana 已实现 iam gRPC 客户端 + Redis 缓存（TTL 5min）+ role 降级兜底。iam 服务未启动时走降级（degraded=true），不阻塞业务。
|
||||
|
||||
### 3.2 core-edu 服务（ai07 负责）— P0
|
||||
|
||||
| # | 工作项 | 用途 | 优先级 |
|
||||
| --- | -------------------------------------------------------------- | -------------------------------------------------------------------- | ------ |
|
||||
| 1 | MySQL 表 `core_edu_grades` CDC 对齐 | Debezium 监听 → Kafka topic `edu-cdc.next_edu_cloud.core_edu_grades` | P0 |
|
||||
| 2 | MySQL 表 `core_edu_exams` CDC 对齐 | ExamCache 数据源 | P0 |
|
||||
| 3 | MySQL 表 `core_edu_homework_submissions` CDC | student_dashboard_view 数据源 | P0 |
|
||||
| 4 | MySQL 表 `core_edu_attendance` CDC | attendance_logs 数据源 | P0 |
|
||||
| 5 | 表字段命名对齐（student_id/exam_id/score/subject_id/class_id） | CDC 消费者按字段名解析 | P0 |
|
||||
|
||||
**当前状态**：data-ana CDC 消费者已实现 4 表路由（grades/exams/homework/attendance），等待 core-edu MySQL 数据 + Debezium connector 配置。
|
||||
|
||||
### 3.3 content 服务（ai08 负责）— P1
|
||||
|
||||
| # | 工作项 | 用途 | 优先级 |
|
||||
| --- | --------------------------------------- | ------------------------------------------- | ------ |
|
||||
| 1 | MySQL 表 `content_knowledge_points` CDC | 知识点元数据缓存（title/subject_id） | P1 |
|
||||
| 2 | 知识点标题字段对齐 | data-ana 查询结果补充 knowledge_point_title | P1 |
|
||||
|
||||
**当前状态**：data-ana CDC 消费者已实现 knowledge_points 路由，写入 Redis `data_ana:kp_meta:{kp_id}`。
|
||||
|
||||
### 3.4 ai 服务（ai12 负责）— P1
|
||||
|
||||
| # | 工作项 | 用途 | 优先级 |
|
||||
| --- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------ |
|
||||
| 1 | 发布 `AIUsageEvent` 到 Kafka topic `edu.insight.ai.usage` | data-ana 消费写入 `ai_usage_log` 表，供 admin dashboard 统计 | P1 |
|
||||
| 2 | AIUsageEvent 字段对齐 events.proto | request_id/user_id/provider/model/prompt_tokens/completion_tokens/total_tokens/latency_ms/success/cost_cents/occurred_at | P1 |
|
||||
| 3 | 诊断报告生成能力 | `ListDiagnosticReports` 当前为占位，需 ai 提供报告数据 | P2 |
|
||||
|
||||
**当前状态**：data-ana CDC 消费者已实现 AIUsageEvent 路由（`_handle_ai_usage_event`），等待 ai 服务发布事件。
|
||||
|
||||
### 3.5 ListDiagnosticReports 完整实现（需 ai 服务协作）— P2
|
||||
|
||||
当前 `ListDiagnosticReports` 返回空列表 + degraded 标记。完整实现需要：
|
||||
|
||||
1. **ai 服务**生成诊断报告并存储（DB 或 Kafka 事件）
|
||||
2. **data-ana** 消费 ai 诊断报告事件，写入 ClickHouse 新表 `diagnostic_reports`
|
||||
3. **data-ana** 查询 `diagnostic_reports` 表返回报告列表
|
||||
|
||||
**建议方案**：
|
||||
|
||||
- ai 服务发布 `DiagnosticReportGenerated` 事件到 Kafka topic `edu.insight.diagnostic.generated`
|
||||
- data-ana CDC 消费者扩展 `_handle_diagnostic_event` 路由
|
||||
- ClickHouse 新增 `diagnostic_reports` 表（ReplacingMergeTree，TTL 1 年）
|
||||
|
||||
### 3.6 PracticeService 新建（需协调 AI 决策）— P2
|
||||
|
||||
student-bff / parent-portal 需要 `PracticeService` 3 RPC（练习会话管理），当前不在 data-ana 范围内。
|
||||
|
||||
**建议方案 A**（推荐）：在 core-edu 服务新增 PracticeService（练习属于教学域）
|
||||
**建议方案 B**：在 data-ana 新增 PracticeService（练习数据天然属于分析域）
|
||||
**建议方案 C**：新建独立 `practice` 服务
|
||||
|
||||
> 需协调 AI（ai07/ai11/ai12）与人类决策者确定归属。data-ana 当前不实现，待决策后补充。
|
||||
|
||||
### 3.7 childLearningPath 完整实现（需协调）— P2
|
||||
|
||||
parent-portal `childLearningPath` 字段需要学习路径推荐数据。当前 `GetStudentGrowth` 返回成长档案（成绩趋势 + 掌握度 + 考勤），但不含学习路径推荐。
|
||||
|
||||
**建议方案**：ai 服务提供学习路径推荐 API，data-ana 聚合或 ai 直接暴露给 BFF。
|
||||
|
||||
### 3.8 Debezium / Kafka 基础设施（SRE AI 负责）— P0
|
||||
|
||||
| # | 工作项 | 用途 | 优先级 |
|
||||
| --- | ------------------------------------------------------ | ----------------------------------------------- | ------ |
|
||||
| 1 | Debezium connector 配置（6 表） | MySQL binlog → Kafka CDC topics | P0 |
|
||||
| 2 | Kafka topic 创建（7 个 CDC + 1 个 AIUsage） | edu-cdc.next_edu_cloud.* + edu.insight.ai.usage | P0 |
|
||||
| 3 | Kafka topic `edu.insight.mastery.updated` | data-ana 发布掌握度更新事件 | P1 |
|
||||
| 4 | Kafka topic `edu.insight.diagnostic.generated`（未来） | 诊断报告事件（§3.5） | P2 |
|
||||
|
||||
**当前状态**：edu-kafka + edu-debezium + edu-clickhouse 容器已运行，但 Debezium connector 未配置（data-ana CDC 消费者启动但无消息消费）。
|
||||
|
||||
### 3.9 ClickHouse DDL 执行（SRE AI 负责）— P0
|
||||
|
||||
| # | 工作项 | 优先级 |
|
||||
| --- | ----------------------------------------------- | ------ |
|
||||
| 1 | 执行 `scripts/clickhouse_ddl.sql`（5 表 + TTL） | P0 |
|
||||
| 2 | 创建 database `edu_analytics` | P0 |
|
||||
|
||||
**当前状态**：ClickHouse 容器运行，DDL 已更新（含 TTL），但需 SRE 确认执行。
|
||||
|
||||
---
|
||||
|
||||
## 4. Docker 本地测试结果（v2）
|
||||
|
||||
**测试环境**：edu-data-ana-test 容器（edu/data-ana:test 镜像），连接 edu-clickhouse + edu-redis + edu-kafka
|
||||
|
||||
**测试时间**：2026-07-14
|
||||
|
||||
| # | 测试项 | 结果 | 备注 |
|
||||
| --- | ------------------------------- | ---- | --------------------------------------------- |
|
||||
| 1 | `GET /healthz` | ✅ | `{"status":"ok","service":"data-ana"}` |
|
||||
| 2 | `GET /readyz`（首次） | ✅ | ready=true（lifespan 预热生效） |
|
||||
| 3 | `GET /readyz`（dependencies） | ✅ | clickhouse=ok, cdc=running(lag=0), redis=ok |
|
||||
| 4 | `GET /metrics` | ✅ | 18 个自定义指标全部暴露 |
|
||||
| 5 | gRPC HealthService | ✅ | SERVING |
|
||||
| 6 | gRPC GetStudentDashboard | ✅ | |
|
||||
| 7 | gRPC GetStudentWeakness | ✅ | |
|
||||
| 8 | gRPC GetLearningTrend | ✅ | |
|
||||
| 9 | gRPC GetClassPerformance | ✅ | |
|
||||
| 10 | gRPC GetStudentGrowth (v2) | ✅ | 返回 growth_score + growth_level |
|
||||
| 11 | gRPC GetAssignmentAnalysis (v2) | ✅ | 返回 score ranges |
|
||||
| 12 | gRPC GetMasterySummary (v2) | ✅ | 返回 three-tier distribution |
|
||||
| 13 | gRPC ListDiagnosticReports (v2) | ✅ | 占位返回空列表 + degraded |
|
||||
| 14 | gRPC ListErrorBookItems (v2) | ✅ | |
|
||||
| 15 | gRPC GetErrorBookStats (v2) | ✅ | |
|
||||
| 16 | HTTP 11 业务端点 | ✅ | 全部返回 ActionState 信封 success=true |
|
||||
| 17 | gRPC 拦截器 metrics | ✅ | `data_ana_grpc_requests_total` 按 method 记录 |
|
||||
| 18 | CDC consumer metrics | ✅ | `cdc_consumer_active_instances=1` |
|
||||
| 19 | ruff check | ✅ | All checks passed |
|
||||
|
||||
---
|
||||
|
||||
## 5. v2 架构变更摘要
|
||||
|
||||
### 5.1 proto 契约变更
|
||||
|
||||
- `packages/shared-proto/proto/analytics.proto`：新增 6 RPC + 16 message
|
||||
- Python stub 重新生成：`src/generated_proto/analytics_pb2.py` + `analytics_pb2_grpc.py`
|
||||
|
||||
### 5.2 新增文件
|
||||
|
||||
- `services/data-ana/src/data_ana/metrics.py`：18 个 Prometheus 指标定义
|
||||
|
||||
### 5.3 修改文件
|
||||
|
||||
- `services/data-ana/src/data_ana/cdc_consumer.py`：多实例 ID + 真实 lag + metrics 接入
|
||||
- `services/data-ana/src/data_ana/exam_cache.py`：Redis-backed + 内存 LRU fallback
|
||||
- `services/data-ana/src/data_ana/grpc_server.py`：6 新 RPC + gRPC 拦截器
|
||||
- `services/data-ana/src/data_ana/analytics_service.py`：6 新业务方法
|
||||
- `services/data-ana/src/data_ana/repository/clickhouse_repository.py`：3 新查询方法
|
||||
- `services/data-ana/src/data_ana/main.py`：readyz 硬化 + lifespan 预热 + metrics 初始化
|
||||
- `services/data-ana/src/data_ana/config.py`：readyz 超时配置
|
||||
- `services/data-ana/scripts/clickhouse_ddl.sql`：5 表 TTL
|
||||
|
||||
### 5.4 未变更
|
||||
|
||||
- `services/data-ana/pyproject.toml`：依赖版本不变（用户要求）
|
||||
- `services/data-ana/Dockerfile`：构建配置不变
|
||||
|
||||
---
|
||||
|
||||
## 6. 下一步工作
|
||||
|
||||
### 6.1 等待上游就绪后联调
|
||||
|
||||
1. iam gRPC 服务启动 → data-ana 移除降级标记
|
||||
2. core-edu MySQL 数据 + Debezium connector → data-ana CDC 消费真实数据
|
||||
3. ai 服务发布 AIUsageEvent → data-ana admin dashboard 显示真实 AI 用量
|
||||
|
||||
### 6.2 待决策项
|
||||
|
||||
1. PracticeService 归属（core-edu / data-ana / 新服务）— §3.6
|
||||
2. ListDiagnosticReports 完整实现方案 — §3.5
|
||||
3. childLearningPath 数据源 — §3.7
|
||||
|
||||
### 6.3 P7+ 规划（未来）
|
||||
|
||||
- ClickHouse 物化视图（预聚合 dashboard 查询）
|
||||
- Kafka Streams 掌握度实时计算（替代批处理）
|
||||
- Grafana dashboard 配置（`infra/grafana/dashboards/data-ana.json`）
|
||||
- 告警规则（`infra/prometheus/rules.yml` 补充 data-ana 规则）
|
||||
@@ -1,11 +1,12 @@
|
||||
-- data-ana ClickHouse DDL：5 宽表建表脚本
|
||||
-- 对齐 02-architecture-design.md §3 DDL 设计
|
||||
-- data-ana ClickHouse DDL：5 宽表建表脚本（P6: 加 TTL 归档策略）
|
||||
-- 对齐 02-architecture-design.md §3 DDL 设计 + workline §3.5 任务 6.3
|
||||
-- 引擎：ReplacingMergeTree（幂等消费保证）+ MergeTree（历史快照）
|
||||
-- TTL：P6 容量规划（冷热数据分离，过期自动清理）
|
||||
-- 使用方式：clickhouse-client --multiquery < scripts/clickhouse_ddl.sql
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- §3.1 学生学情宽表
|
||||
-- §3.1 学生学情宽表（TTL 2 年）
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
(
|
||||
student_id String,
|
||||
@@ -22,9 +23,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
TTL last_updated + INTERVAL 2 YEAR
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- §3.2 学生错题本
|
||||
-- §3.2 学生错题本（TTL 2 年）
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
(
|
||||
student_id String,
|
||||
@@ -36,9 +38,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
ORDER BY (student_id, question_id)
|
||||
TTL last_error_time + INTERVAL 2 YEAR;
|
||||
|
||||
-- §3.3 知识点掌握度历史快照
|
||||
-- §3.3 知识点掌握度历史快照（TTL 3 年，长期保留用于趋势分析）
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
(
|
||||
student_id String,
|
||||
@@ -50,9 +53,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at)
|
||||
TTL calculated_at + INTERVAL 3 YEAR;
|
||||
|
||||
-- §3.4 AI 用量计费记录
|
||||
-- §3.4 AI 用量计费记录（TTL 1 年，计费数据保留期较短）
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
(
|
||||
request_id String,
|
||||
@@ -69,9 +73,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id);
|
||||
ORDER BY (request_id)
|
||||
TTL occurred_at + INTERVAL 1 YEAR;
|
||||
|
||||
-- §3.5 学生考勤记录
|
||||
-- §3.5 学生考勤记录（TTL 3 年，考勤数据需长期保留用于趋势分析）
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
(
|
||||
student_id String,
|
||||
@@ -84,4 +89,21 @@ CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
ORDER BY (student_id, class_id, attendance_date)
|
||||
TTL occurred_at + INTERVAL 3 YEAR;
|
||||
|
||||
-- ===== P6 容量规划说明 =====
|
||||
-- 1. student_dashboard_view / student_errors：2 年 TTL
|
||||
-- - 高频写入（CDC 实时同步），2 年覆盖完整学习周期
|
||||
-- - 超过 2 年的数据通过 PARTITION 级别删除（无需 VACUUM）
|
||||
-- 2. mastery_snapshot / attendance_logs：3 年 TTL
|
||||
-- - 低频写入但需长期保留用于趋势分析
|
||||
-- - 3 年覆盖 K12 完整学段
|
||||
-- 3. ai_usage_log：1 年 TTL
|
||||
-- - 计费数据保留期较短，1 年足够用于年度成本分析
|
||||
-- 4. 冷热数据分离：
|
||||
-- - 热数据：最近 3 个月（SSD 存储，频繁查询）
|
||||
-- - 冷数据：3 个月以上（HDD 存储，低频查询）
|
||||
-- - 通过 PARTITION BY toYYYYMM 实现按月分区，便于冷热分离
|
||||
-- 5. TTL 触发时机：ClickHouse 后台 merge 时自动清理过期数据
|
||||
-- - 可通过 system.ttl_drops 表监控 TTL 执行情况
|
||||
|
||||
@@ -414,6 +414,333 @@ async def get_student_weakness(
|
||||
return result
|
||||
|
||||
|
||||
# ===== P6+ v2 扩展 RPC 实现 =====
|
||||
|
||||
|
||||
async def get_student_growth(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
"""学生成长档案（综合成绩趋势 + 掌握度变化 + 考勤统计）.
|
||||
|
||||
供 parent-bff GrowthArchiveService.Get / student-bff GetStudentGrowth.
|
||||
"""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
# 1. 成绩趋势
|
||||
trend = await clickhouse_repository.query_learning_trend(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
score_trend = trend.get("points", []) if trend else []
|
||||
|
||||
# 2. 掌握度变化趋势
|
||||
mastery_trend = await clickhouse_repository.query_mastery_trend(
|
||||
student_id=effective_student_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
if mastery_trend is None:
|
||||
mastery_trend = []
|
||||
|
||||
# 3. 考勤统计
|
||||
attendance = await clickhouse_repository.query_attendance(
|
||||
student_id=effective_student_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
if attendance is None:
|
||||
attendance_summary = {
|
||||
"total_days": 0,
|
||||
"present_days": 0,
|
||||
"absent_days": 0,
|
||||
"late_days": 0,
|
||||
"attendance_rate": 0.0,
|
||||
}
|
||||
else:
|
||||
total = attendance.get("total", 0)
|
||||
attendance_summary = {
|
||||
"total_days": total,
|
||||
"present_days": attendance.get("presentCount", 0),
|
||||
"absent_days": attendance.get("absentCount", 0),
|
||||
"late_days": attendance.get("lateCount", 0),
|
||||
"attendance_rate": (attendance.get("presentCount", 0) / total) if total else 0.0,
|
||||
}
|
||||
|
||||
# 4. 计算成长评分（综合分数 + 掌握度 + 出勤率）
|
||||
avg_score = sum(p.get("score", 0) for p in score_trend) / len(score_trend) if score_trend else 0
|
||||
avg_mastery = (
|
||||
sum(p.get("overall_mastery", 0) for p in mastery_trend) / len(mastery_trend)
|
||||
if mastery_trend
|
||||
else 0
|
||||
)
|
||||
attendance_rate = attendance_summary["attendance_rate"]
|
||||
growth_score = (avg_score / 100 * 0.4) + (avg_mastery * 0.4) + (attendance_rate * 0.2)
|
||||
|
||||
if growth_score >= 0.85:
|
||||
growth_level = "excellent"
|
||||
elif growth_score >= 0.7:
|
||||
growth_level = "good"
|
||||
elif growth_score >= 0.5:
|
||||
growth_level = "average"
|
||||
else:
|
||||
growth_level = "needs_improvement"
|
||||
|
||||
result = {
|
||||
"studentId": effective_student_id,
|
||||
"scoreTrend": score_trend,
|
||||
"masteryTrend": mastery_trend,
|
||||
"attendance": attendance_summary,
|
||||
"growthScore": round(growth_score, 4),
|
||||
"growthLevel": growth_level,
|
||||
}
|
||||
|
||||
if trend is None and attendance is None:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = "clickhouse_unavailable"
|
||||
elif degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_assignment_analysis(
|
||||
user: UserContext,
|
||||
class_id: str,
|
||||
assignment_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""作业/考试分析（单次作业/考试维度统计）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
|
||||
return {
|
||||
"assignmentId": assignment_id,
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "datascope_violation",
|
||||
"averageScore": 0.0,
|
||||
"highestScore": 0.0,
|
||||
"lowestScore": 0.0,
|
||||
"totalStudents": 0,
|
||||
"submittedCount": 0,
|
||||
"passRate": 0.0,
|
||||
"ranges": [],
|
||||
}
|
||||
|
||||
result = await clickhouse_repository.query_assignment_analysis(
|
||||
class_id=class_id,
|
||||
assignment_id=assignment_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"assignmentId": assignment_id,
|
||||
"classId": class_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"averageScore": 0.0,
|
||||
"highestScore": 0.0,
|
||||
"lowestScore": 0.0,
|
||||
"totalStudents": 0,
|
||||
"submittedCount": 0,
|
||||
"passRate": 0.0,
|
||||
"ranges": [],
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_mastery_summary(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""学生掌握度汇总（轻量级，仅返回总体掌握度 + 三档分布）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
mastery = await clickhouse_repository.query_mastery_snapshot(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if mastery is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"overallMastery": 0.0,
|
||||
"masteredCount": 0,
|
||||
"progressingCount": 0,
|
||||
"weakCount": 0,
|
||||
"totalKnowledgePoints": 0,
|
||||
"masteryLevel": "weak",
|
||||
}
|
||||
|
||||
kps = mastery.get("knowledgePoints", [])
|
||||
mastered = sum(1 for kp in kps if kp.get("mastery_label") == "mastered")
|
||||
progressing = sum(1 for kp in kps if kp.get("mastery_label") == "progressing")
|
||||
weak = sum(1 for kp in kps if kp.get("mastery_label") == "weak")
|
||||
overall = mastery.get("overallMastery", 0.0)
|
||||
|
||||
if overall >= 0.8:
|
||||
mastery_level = "mastered"
|
||||
elif overall >= 0.4:
|
||||
mastery_level = "progressing"
|
||||
else:
|
||||
mastery_level = "weak"
|
||||
|
||||
result = {
|
||||
"studentId": effective_student_id,
|
||||
"overallMastery": overall,
|
||||
"masteredCount": mastered,
|
||||
"progressingCount": progressing,
|
||||
"weakCount": weak,
|
||||
"totalKnowledgePoints": len(kps),
|
||||
"masteryLevel": mastery_level,
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def list_diagnostic_reports(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
since: int = 0,
|
||||
limit: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
"""诊断报告列表（占位实现，真实数据需 ai 服务集成）.
|
||||
|
||||
当前返回空列表 + degraded 标记，待 ai 服务提供诊断报告生成能力后补全.
|
||||
"""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
# 占位：真实数据需 ai 服务集成（ai 服务生成诊断报告后通过 Kafka 推送）
|
||||
result = {
|
||||
"studentId": effective_student_id,
|
||||
"reports": [],
|
||||
"total": 0,
|
||||
"degraded": True,
|
||||
"degraded_reason": "diagnostic_reports_pending_ai_integration",
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def list_error_book_items(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
"""错题本列表（gRPC 版本，供 parent-bff ErrorBookService.List）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
errors = await clickhouse_repository.query_student_errors(effective_student_id)
|
||||
if errors is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"items": [],
|
||||
"total": 0,
|
||||
}
|
||||
|
||||
# 转换为 ErrorBookItem 格式
|
||||
items = [
|
||||
{
|
||||
"question_id": e.get("question_id", ""),
|
||||
"knowledge_point_id": e.get("knowledge_point_id", ""),
|
||||
"knowledge_point_title": e.get("knowledge_point_id", ""), # content CDC 同步后补充
|
||||
"error_count": e.get("error_count", 0),
|
||||
"last_error_time": int(e.get("last_error_time").timestamp())
|
||||
if e.get("last_error_time")
|
||||
else 0,
|
||||
"content": e.get("content", ""),
|
||||
}
|
||||
for e in errors
|
||||
]
|
||||
|
||||
result = {
|
||||
"studentId": effective_student_id,
|
||||
"items": items,
|
||||
"total": len(items),
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_error_book_stats(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""错题本统计（按知识点聚合）."""
|
||||
scope = await iam_client.get_effective_datascope(user)
|
||||
degraded = scope.degraded
|
||||
|
||||
effective_student_id = student_id
|
||||
if scope.level == DataScopeLevel.SELF:
|
||||
effective_student_id = user.user_id
|
||||
|
||||
result = await clickhouse_repository.query_error_book_stats(
|
||||
student_id=effective_student_id,
|
||||
subject_id=subject_id,
|
||||
)
|
||||
if result is None:
|
||||
return {
|
||||
"studentId": effective_student_id,
|
||||
"degraded": True,
|
||||
"degraded_reason": "clickhouse_unavailable",
|
||||
"totalErrorQuestions": 0,
|
||||
"totalErrorCount": 0,
|
||||
"byKnowledgePoint": [],
|
||||
"recent7dErrors": 0,
|
||||
}
|
||||
|
||||
if degraded:
|
||||
result["degraded"] = True
|
||||
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
|
||||
return result
|
||||
|
||||
|
||||
async def get_learning_trend(
|
||||
user: UserContext,
|
||||
student_id: str,
|
||||
|
||||
@@ -20,6 +20,12 @@
|
||||
- op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null
|
||||
- Redis 事件去重：基于 event_id（Debezium 重启时可能重发）
|
||||
|
||||
P6 多实例水平扩展：
|
||||
- consumer group 不变（data-ana-cdc），Kafka 自动 partition rebalance
|
||||
- 多实例消费同一 topic 无重复无遗漏（partition 级分配）
|
||||
- get_lag() 通过 AIOKafkaConsumer.position + end_offsets 计算真实 lag
|
||||
- 实例 ID 通过 POD_NAME 环境变量区分（用于日志追踪）
|
||||
|
||||
降级策略：
|
||||
- kafka_brokers 未配置：消费者不启动（仅 HTTP/gRPC 服务）
|
||||
- ClickHouse 不可达：消息处理失败，不 commit，下次重启重试
|
||||
@@ -28,11 +34,14 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from . import metrics
|
||||
from .config import settings
|
||||
from .exam_cache import get_exam_cache
|
||||
from .repository import clickhouse_repository, redis_client
|
||||
@@ -44,6 +53,14 @@ _consumer: Any | None = None
|
||||
_consumer_task: asyncio.Task[None] | None = None
|
||||
_is_running: bool = False
|
||||
|
||||
# 实例 ID（多实例水平扩展时用于日志区分）
|
||||
_INSTANCE_ID = os.getenv("POD_NAME", f"{socket.gethostname()}:{os.getpid()}")
|
||||
|
||||
|
||||
def get_instance_id() -> str:
|
||||
"""获取消费者实例 ID（多实例部署时用于日志追踪）."""
|
||||
return _INSTANCE_ID
|
||||
|
||||
|
||||
def _parse_ts(ts_ms: int | None) -> datetime:
|
||||
"""Debezium ts_ms（毫秒）→ datetime."""
|
||||
@@ -99,7 +116,7 @@ async def _handle_exams_event(after: dict[str, Any] | None, op: str) -> bool:
|
||||
return True
|
||||
|
||||
exam_cache = get_exam_cache()
|
||||
exam_cache.upsert(
|
||||
await exam_cache.upsert(
|
||||
exam_id=exam_id,
|
||||
class_id=str(after.get("class_id") or ""),
|
||||
subject_id=str(after.get("subject_id") or ""),
|
||||
@@ -127,10 +144,10 @@ async def _handle_grades_event(
|
||||
exam_id = str(after.get("exam_id") or "")
|
||||
score = _safe_float(after.get("score"))
|
||||
|
||||
# 从 ExamCache 获取 class_id 和 subject_id
|
||||
# 从 ExamCache 获取 class_id 和 subject_id（P6: Redis-backed async）
|
||||
exam_cache = get_exam_cache()
|
||||
class_id = exam_cache.get_class_id(exam_id)
|
||||
subject_id = exam_cache.get_subject_id(exam_id)
|
||||
class_id = await exam_cache.get_class_id(exam_id)
|
||||
subject_id = await exam_cache.get_subject_id(exam_id)
|
||||
|
||||
fallback_ts = _parse_ts(ts_ms)
|
||||
last_updated = _parse_mysql_datetime(after.get("updated_at"), fallback_ts)
|
||||
@@ -473,22 +490,48 @@ async def run_consumer() -> None:
|
||||
try:
|
||||
await _consumer.start()
|
||||
_is_running = True
|
||||
metrics.cdc_consumer_active_instances.set(1)
|
||||
logger.info(
|
||||
"cdc_consumer_started",
|
||||
instance_id=_INSTANCE_ID,
|
||||
brokers=brokers,
|
||||
topics=topics,
|
||||
group_id=settings.kafka_consumer_group,
|
||||
auto_commit=settings.kafka_enable_auto_commit,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("cdc_consumer_start_failed", error=str(exc))
|
||||
logger.error(
|
||||
"cdc_consumer_start_failed",
|
||||
instance_id=_INSTANCE_ID,
|
||||
error=str(exc),
|
||||
)
|
||||
_is_running = False
|
||||
return
|
||||
|
||||
try:
|
||||
async for msg in _consumer:
|
||||
try:
|
||||
import time as _time
|
||||
|
||||
t0 = _time.monotonic()
|
||||
success = await _process_message(msg.topic, msg.value)
|
||||
duration = _time.monotonic() - t0
|
||||
table = ""
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
_ev = _json.loads(
|
||||
msg.value.decode("utf-8") if isinstance(msg.value, bytes) else msg.value
|
||||
)
|
||||
table = (_ev.get("source") or {}).get("table", "")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
metrics.cdc_message_process_duration_seconds.labels(table=table).observe(duration)
|
||||
metrics.cdc_messages_processed_total.labels(
|
||||
topic=msg.topic,
|
||||
table=table,
|
||||
status="success" if success else "failed",
|
||||
).inc()
|
||||
if success:
|
||||
# 处理成功才 commit offset（at-least-once）
|
||||
await _consumer.commit()
|
||||
@@ -514,6 +557,7 @@ async def run_consumer() -> None:
|
||||
raise
|
||||
finally:
|
||||
_is_running = False
|
||||
metrics.cdc_consumer_active_instances.set(0)
|
||||
if _consumer is not None:
|
||||
try:
|
||||
await _consumer.stop()
|
||||
@@ -555,7 +599,25 @@ def is_running() -> bool:
|
||||
async def get_lag() -> int:
|
||||
"""获取消费者 lag（待消费消息数，供 /readyz 和监控使用）.
|
||||
|
||||
P6 实现：调用 Kafka AdminClient 获取 group lag.
|
||||
当前返回 0（简化）.
|
||||
P6 实现：通过 AIOKafkaConsumer.position + end_offsets 计算真实 lag.
|
||||
多实例场景：仅计算本实例分配到的 partition lag.
|
||||
"""
|
||||
if _consumer is None or not _is_running:
|
||||
return 0
|
||||
try:
|
||||
# 获取本实例分配到的 partition
|
||||
assignment = _consumer.assignment()
|
||||
if not assignment:
|
||||
return 0
|
||||
# 并发获取 end_offsets 和 position
|
||||
end_offsets = await _consumer.end_offsets(assignment)
|
||||
total_lag = 0
|
||||
for tp in assignment:
|
||||
position = await _consumer.position(tp)
|
||||
end = end_offsets.get(tp, 0)
|
||||
if end > position:
|
||||
total_lag += end - position
|
||||
return total_lag
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("cdc_lag_query_failed", error=str(exc))
|
||||
return 0
|
||||
|
||||
@@ -78,6 +78,12 @@ class Settings(BaseSettings):
|
||||
# 降级
|
||||
degraded_mode_enabled: bool = True
|
||||
|
||||
# P6 readyz 深度硬化
|
||||
readyz_clickhouse_timeout_s: float = 1.0
|
||||
readyz_redis_timeout_s: float = 0.2
|
||||
readyz_iam_grpc_timeout_s: float = 2.0
|
||||
readyz_cdc_lag_threshold: int = 1000 # lag 超过此值判定 not_ready
|
||||
|
||||
# 向后兼容：旧代码引用 settings.port / settings.kafka_group_id
|
||||
@property
|
||||
def port(self) -> int:
|
||||
|
||||
@@ -1,105 +1,156 @@
|
||||
"""考试缓存（exam_id → {class_id, subject_id} 映射，内存 LRU）.
|
||||
"""考试缓存（exam_id → {class_id, subject_id} 映射，Redis-backed + 内存 LRU fallback）.
|
||||
|
||||
对齐 02-architecture-design.md §8.3 ExamCache：
|
||||
- 内存 LRU dict，max 10000 条
|
||||
- CDC core_edu_exams 事件触发更新
|
||||
- CDC core_edu_grades 事件查询获取 class_id（避免 join 查询）
|
||||
对齐 02-architecture-design.md §8.3 ExamCache + workline §3.5 任务 6.2：
|
||||
- P6 演进：Redis 实现（key: data_ana:exam:{exam_id}，TTL 30 天）
|
||||
- 多实例共享：多个 data-ana 实例共享同一 Redis ExamCache
|
||||
- 内存 LRU fallback：Redis 不可达时降级为内存缓存（单实例模式）
|
||||
|
||||
P6 演进：改为 Redis 实现（key: data_ana:exam:{exam_id}，TTL 30 天），
|
||||
支持多实例共享（对齐 workline §3.5 任务 6.2）.
|
||||
设计：
|
||||
- upsert：Redis 写入（async）+ 内存 LRU 同步写入（保证后续读取命中）
|
||||
- get：Redis 读取（async），miss 时 fallback 到内存 LRU
|
||||
- Redis 不可达：仅使用内存 LRU（降级模式，degraded=true）
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from .repository import redis_client
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# LRU 最大容量
|
||||
# LRU 最大容量（内存 fallback 用）
|
||||
_MAX_SIZE = 10_000
|
||||
_REDIS_TTL_S = 30 * 24 * 3600 # 30 天
|
||||
|
||||
|
||||
def _redis_key(exam_id: str) -> str:
|
||||
"""Redis 缓存键."""
|
||||
return f"data_ana:exam:{exam_id}"
|
||||
|
||||
|
||||
class ExamCache:
|
||||
"""考试缓存（LRU，max 10000 条）.
|
||||
"""考试缓存（Redis-backed + 内存 LRU fallback）.
|
||||
|
||||
内存实现：OrderedDict，访问/写入时移到末尾（最近使用），
|
||||
超容量时弹出头部（最久未使用）.
|
||||
|
||||
线程安全：asyncio 单线程模型下无需加锁.
|
||||
Redis 可用时：多实例共享，TTL 30 天
|
||||
Redis 不可用：降级为单实例内存 LRU（max 10000 条）
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = _MAX_SIZE) -> None:
|
||||
self._data: OrderedDict[str, dict[str, str]] = OrderedDict()
|
||||
self._memory: OrderedDict[str, dict[str, str]] = OrderedDict()
|
||||
self._max_size = max_size
|
||||
|
||||
def upsert(
|
||||
def _memory_upsert(
|
||||
self,
|
||||
exam_id: str,
|
||||
class_id: str = "",
|
||||
subject_id: str = "",
|
||||
title: str = "",
|
||||
) -> None:
|
||||
"""更新或插入考试缓存."""
|
||||
"""内存 LRU 写入（同步，Redis 不可达时降级用）."""
|
||||
if not exam_id:
|
||||
return
|
||||
|
||||
value: dict[str, str] = {
|
||||
"class_id": class_id,
|
||||
"subject_id": subject_id,
|
||||
"title": title,
|
||||
}
|
||||
if exam_id in self._memory:
|
||||
self._memory.move_to_end(exam_id)
|
||||
self._memory[exam_id] = value
|
||||
while len(self._memory) > self._max_size:
|
||||
evicted_key, _ = self._memory.popitem(last=False)
|
||||
logger.debug("exam_cache_memory_lru_evicted", exam_id=evicted_key)
|
||||
|
||||
# 已存在则移到末尾（标记为最近使用）
|
||||
if exam_id in self._data:
|
||||
self._data.move_to_end(exam_id)
|
||||
self._data[exam_id] = value
|
||||
|
||||
# LRU 淘汰
|
||||
while len(self._data) > self._max_size:
|
||||
evicted_key, _ = self._data.popitem(last=False)
|
||||
logger.debug("exam_cache_lru_evicted", exam_id=evicted_key)
|
||||
|
||||
def get(self, exam_id: str) -> dict[str, str] | None:
|
||||
"""查询考试缓存（命中时移到末尾，标记为最近使用）."""
|
||||
def _memory_get(self, exam_id: str) -> dict[str, str] | None:
|
||||
"""内存 LRU 读取（同步）."""
|
||||
if not exam_id:
|
||||
return None
|
||||
value = self._data.get(exam_id)
|
||||
value = self._memory.get(exam_id)
|
||||
if value is not None:
|
||||
self._data.move_to_end(exam_id)
|
||||
self._memory.move_to_end(exam_id)
|
||||
return value
|
||||
|
||||
def get_class_id(self, exam_id: str) -> str:
|
||||
async def upsert(
|
||||
self,
|
||||
exam_id: str,
|
||||
class_id: str = "",
|
||||
subject_id: str = "",
|
||||
title: str = "",
|
||||
) -> None:
|
||||
"""更新或插入考试缓存（Redis + 内存 LRU）."""
|
||||
if not exam_id:
|
||||
return
|
||||
|
||||
# 1. 内存 LRU 同步写入（保证后续读取命中）
|
||||
self._memory_upsert(exam_id, class_id, subject_id, title)
|
||||
|
||||
# 2. Redis 异步写入（多实例共享）
|
||||
value = json.dumps(
|
||||
{"class_id": class_id, "subject_id": subject_id, "title": title},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
ok = await redis_client.set_cache(_redis_key(exam_id), value, ttl_s=_REDIS_TTL_S)
|
||||
if not ok:
|
||||
logger.debug("exam_cache_redis_write_failed_memory_only", exam_id=exam_id)
|
||||
|
||||
async def get(self, exam_id: str) -> dict[str, str] | None:
|
||||
"""查询考试缓存（Redis 优先，miss 时 fallback 内存 LRU）."""
|
||||
if not exam_id:
|
||||
return None
|
||||
|
||||
# 1. Redis 读取
|
||||
raw = await redis_client.get_cache(_redis_key(exam_id))
|
||||
if raw is not None:
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
# 回填内存 LRU（加速后续读取）
|
||||
self._memory_upsert(
|
||||
exam_id,
|
||||
data.get("class_id", ""),
|
||||
data.get("subject_id", ""),
|
||||
data.get("title", ""),
|
||||
)
|
||||
return data
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
logger.warning("exam_cache_redis_decode_failed", exam_id=exam_id, error=str(exc))
|
||||
|
||||
# 2. 内存 LRU fallback
|
||||
return self._memory_get(exam_id)
|
||||
|
||||
async def get_class_id(self, exam_id: str) -> str:
|
||||
"""便捷方法：获取 class_id（未命中返回空字符串）."""
|
||||
entry = self.get(exam_id)
|
||||
entry = await self.get(exam_id)
|
||||
return entry.get("class_id", "") if entry else ""
|
||||
|
||||
def get_subject_id(self, exam_id: str) -> str:
|
||||
async def get_subject_id(self, exam_id: str) -> str:
|
||||
"""便捷方法：获取 subject_id（未命中返回空字符串）."""
|
||||
entry = self.get(exam_id)
|
||||
entry = await self.get(exam_id)
|
||||
return entry.get("subject_id", "") if entry else ""
|
||||
|
||||
def delete(self, exam_id: str) -> bool:
|
||||
"""删除缓存项."""
|
||||
if exam_id in self._data:
|
||||
del self._data[exam_id]
|
||||
return True
|
||||
return False
|
||||
async def delete(self, exam_id: str) -> bool:
|
||||
"""删除缓存项（Redis + 内存）."""
|
||||
if exam_id in self._memory:
|
||||
del self._memory[exam_id]
|
||||
return await redis_client.delete_cache(_redis_key(exam_id))
|
||||
|
||||
def clear(self) -> None:
|
||||
"""清空缓存."""
|
||||
self._data.clear()
|
||||
def clear_memory(self) -> None:
|
||||
"""清空内存缓存（Redis 数据保留）."""
|
||||
self._memory.clear()
|
||||
|
||||
def size(self) -> int:
|
||||
"""当前缓存数量."""
|
||||
return len(self._data)
|
||||
def memory_size(self) -> int:
|
||||
"""当前内存缓存数量."""
|
||||
return len(self._memory)
|
||||
|
||||
def stats(self) -> dict[str, Any]:
|
||||
"""缓存统计信息（供 /readyz 和监控使用）."""
|
||||
return {
|
||||
"size": len(self._data),
|
||||
"max_size": self._max_size,
|
||||
"utilization": round(len(self._data) / self._max_size, 4),
|
||||
"memory_size": len(self._memory),
|
||||
"memory_max_size": self._max_size,
|
||||
"memory_utilization": round(len(self._memory) / self._max_size, 4),
|
||||
"redis_ttl_s": _REDIS_TTL_S,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -18,16 +18,59 @@ SubscribeMasteryUpdate（P5+）：
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from . import analytics_service, warning_service
|
||||
from . import analytics_service, metrics, warning_service
|
||||
from .config import settings
|
||||
from .shared.permissions import UserContext
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
def _make_metrics_interceptor():
|
||||
"""构造 gRPC ServerInterceptor 实例（记录每个 unary RPC 的请求数和耗时）."""
|
||||
import grpc # type: ignore[import-not-found]
|
||||
|
||||
class _ServerInterceptor(grpc.aio.ServerInterceptor):
|
||||
async def intercept_service(self, continuation, handler_call_details):
|
||||
method = handler_call_details.method.rsplit("/", 1)[-1]
|
||||
handler = await continuation(handler_call_details)
|
||||
if handler is None:
|
||||
return None
|
||||
# 仅包装 unary_unary（不包装 stream RPC）
|
||||
if (
|
||||
handler.unary_unary is None
|
||||
or handler.request_streaming
|
||||
or handler.response_streaming
|
||||
):
|
||||
return handler
|
||||
|
||||
original_behavior = handler.unary_unary
|
||||
|
||||
async def _wrapped(request, context):
|
||||
t0 = time.monotonic()
|
||||
status = "success"
|
||||
try:
|
||||
return await original_behavior(request, context)
|
||||
except Exception: # noqa: BLE001
|
||||
status = "failed"
|
||||
raise
|
||||
finally:
|
||||
duration = time.monotonic() - t0
|
||||
metrics.record_grpc_request(method=method, duration_s=duration, status=status)
|
||||
|
||||
return grpc.unary_unary_rpc_method_handler(
|
||||
_wrapped,
|
||||
request_deserializer=handler.request_deserializer,
|
||||
response_serializer=handler.response_serializer,
|
||||
)
|
||||
|
||||
return _ServerInterceptor()
|
||||
|
||||
|
||||
# SubscribeMasteryUpdate 订阅管理（P5+）
|
||||
_subscribers: dict[str, asyncio.Queue] = {}
|
||||
_subscribers_lock = asyncio.Lock()
|
||||
@@ -239,6 +282,74 @@ class AnalyticsServiceServicer:
|
||||
await _remove_subscriber(sub_key)
|
||||
logger.info("mastery_subscription_removed", sub_key=sub_key)
|
||||
|
||||
# ===== P6+ v2 扩展 RPC =====
|
||||
|
||||
async def GetStudentGrowth(self, request, context):
|
||||
"""学生成长档案（P6+ v2 扩展）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_student_growth(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
start_date=request.start_date,
|
||||
end_date=request.end_date,
|
||||
)
|
||||
return _build_student_growth_response(result)
|
||||
|
||||
async def GetAssignmentAnalysis(self, request, context):
|
||||
"""作业/考试分析（P6+ v2 扩展）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_assignment_analysis(
|
||||
user=user,
|
||||
class_id=request.class_id,
|
||||
assignment_id=request.assignment_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_assignment_analysis_response(result)
|
||||
|
||||
async def GetMasterySummary(self, request, context):
|
||||
"""学生掌握度汇总（P6+ v2 扩展）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_mastery_summary(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_mastery_summary_response(result)
|
||||
|
||||
async def ListDiagnosticReports(self, request, context):
|
||||
"""诊断报告列表（P6+ v2 扩展，占位实现）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.list_diagnostic_reports(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
since=request.since,
|
||||
limit=request.limit,
|
||||
)
|
||||
return _build_diagnostic_report_list_response(result)
|
||||
|
||||
async def ListErrorBookItems(self, request, context):
|
||||
"""错题本列表（P6+ v2 扩展）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.list_error_book_items(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
limit=request.limit,
|
||||
offset=request.offset,
|
||||
)
|
||||
return _build_error_book_list_response(result)
|
||||
|
||||
async def GetErrorBookStats(self, request, context):
|
||||
"""错题本统计（P6+ v2 扩展）."""
|
||||
user = _extract_user(context)
|
||||
result = await analytics_service.get_error_book_stats(
|
||||
user=user,
|
||||
student_id=request.student_id,
|
||||
subject_id=request.subject_id,
|
||||
)
|
||||
return _build_error_book_stats_response(result)
|
||||
|
||||
|
||||
# ===== 辅助函数 =====
|
||||
|
||||
@@ -492,6 +603,151 @@ def _build_mastery_update_event(event: dict) -> Any:
|
||||
)
|
||||
|
||||
|
||||
# ===== P6+ v2 扩展 RPC 响应构建 =====
|
||||
|
||||
|
||||
def _build_student_growth_response(data: dict) -> Any:
|
||||
"""构建 StudentGrowth proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
score_trend = [
|
||||
analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0))
|
||||
for p in data.get("scoreTrend", [])
|
||||
]
|
||||
mastery_trend = [
|
||||
analytics_pb2.MasteryTrendPoint(
|
||||
calculated_at=p.get("calculated_at", 0),
|
||||
overall_mastery=p.get("overall_mastery", 0.0),
|
||||
)
|
||||
for p in data.get("masteryTrend", [])
|
||||
]
|
||||
attendance_data = data.get("attendance", {})
|
||||
attendance = analytics_pb2.AttendanceSummary(
|
||||
total_days=attendance_data.get("total_days", 0),
|
||||
present_days=attendance_data.get("present_days", 0),
|
||||
absent_days=attendance_data.get("absent_days", 0),
|
||||
late_days=attendance_data.get("late_days", 0),
|
||||
attendance_rate=attendance_data.get("attendance_rate", 0.0),
|
||||
)
|
||||
return analytics_pb2.StudentGrowth(
|
||||
student_id=data.get("studentId", ""),
|
||||
score_trend=score_trend,
|
||||
mastery_trend=mastery_trend,
|
||||
attendance=attendance,
|
||||
growth_score=data.get("growthScore", 0.0),
|
||||
growth_level=data.get("growthLevel", "needs_improvement"),
|
||||
)
|
||||
|
||||
|
||||
def _build_assignment_analysis_response(data: dict) -> Any:
|
||||
"""构建 AssignmentAnalysis proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
ranges = [
|
||||
analytics_pb2.ScoreRange(
|
||||
label=r.get("label", ""),
|
||||
count=r.get("count", 0),
|
||||
percentage=r.get("percentage", 0.0),
|
||||
)
|
||||
for r in data.get("ranges", [])
|
||||
]
|
||||
return analytics_pb2.AssignmentAnalysis(
|
||||
assignment_id=data.get("assignmentId", ""),
|
||||
class_id=data.get("classId", ""),
|
||||
subject_id=data.get("subjectId", ""),
|
||||
average_score=data.get("averageScore", 0.0),
|
||||
highest_score=data.get("highestScore", 0.0),
|
||||
lowest_score=data.get("lowestScore", 0.0),
|
||||
total_students=data.get("totalStudents", 0),
|
||||
submitted_count=data.get("submittedCount", 0),
|
||||
pass_rate=data.get("passRate", 0.0),
|
||||
ranges=ranges,
|
||||
)
|
||||
|
||||
|
||||
def _build_mastery_summary_response(data: dict) -> Any:
|
||||
"""构建 MasterySummary proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
return analytics_pb2.MasterySummary(
|
||||
student_id=data.get("studentId", ""),
|
||||
overall_mastery=data.get("overallMastery", 0.0),
|
||||
mastered_count=data.get("masteredCount", 0),
|
||||
progressing_count=data.get("progressingCount", 0),
|
||||
weak_count=data.get("weakCount", 0),
|
||||
total_knowledge_points=data.get("totalKnowledgePoints", 0),
|
||||
mastery_level=data.get("masteryLevel", "weak"),
|
||||
)
|
||||
|
||||
|
||||
def _build_diagnostic_report_list_response(data: dict) -> Any:
|
||||
"""构建 DiagnosticReportList proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
reports = [
|
||||
analytics_pb2.DiagnosticReport(
|
||||
report_id=r.get("report_id", ""),
|
||||
student_id=r.get("student_id", ""),
|
||||
report_type=r.get("report_type", ""),
|
||||
title=r.get("title", ""),
|
||||
summary=r.get("summary", ""),
|
||||
generated_at=r.get("generated_at", 0),
|
||||
status=r.get("status", "pending"),
|
||||
)
|
||||
for r in data.get("reports", [])
|
||||
]
|
||||
return analytics_pb2.DiagnosticReportList(
|
||||
student_id=data.get("studentId", ""),
|
||||
reports=reports,
|
||||
total=data.get("total", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_error_book_list_response(data: dict) -> Any:
|
||||
"""构建 ErrorBookList proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
items = [
|
||||
analytics_pb2.ErrorBookItem(
|
||||
question_id=item.get("question_id", ""),
|
||||
knowledge_point_id=item.get("knowledge_point_id", ""),
|
||||
knowledge_point_title=item.get("knowledge_point_title", ""),
|
||||
error_count=item.get("error_count", 0),
|
||||
last_error_time=item.get("last_error_time", 0),
|
||||
content=item.get("content", ""),
|
||||
)
|
||||
for item in data.get("items", [])
|
||||
]
|
||||
return analytics_pb2.ErrorBookList(
|
||||
student_id=data.get("studentId", ""),
|
||||
items=items,
|
||||
total=data.get("total", 0),
|
||||
)
|
||||
|
||||
|
||||
def _build_error_book_stats_response(data: dict) -> Any:
|
||||
"""构建 ErrorBookStats proto 响应."""
|
||||
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
|
||||
|
||||
by_kp = [
|
||||
analytics_pb2.KnowledgePointErrorStats(
|
||||
knowledge_point_id=kp.get("knowledge_point_id", ""),
|
||||
title=kp.get("title", ""),
|
||||
error_count=kp.get("error_count", 0),
|
||||
question_count=kp.get("question_count", 0),
|
||||
error_rate=kp.get("error_rate", 0.0),
|
||||
)
|
||||
for kp in data.get("byKnowledgePoint", [])
|
||||
]
|
||||
return analytics_pb2.ErrorBookStats(
|
||||
student_id=data.get("studentId", ""),
|
||||
total_error_questions=data.get("totalErrorQuestions", 0),
|
||||
total_error_count=data.get("totalErrorCount", 0),
|
||||
by_knowledge_point=by_kp,
|
||||
recent_7d_errors=data.get("recent7dErrors", 0),
|
||||
)
|
||||
|
||||
|
||||
# ===== gRPC Server 管理 =====
|
||||
|
||||
_server: Any | None = None
|
||||
@@ -514,7 +770,7 @@ async def start_grpc_server() -> Any | None:
|
||||
logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc))
|
||||
return None
|
||||
|
||||
_server = grpc.aio.server()
|
||||
_server = grpc.aio.server(interceptors=[_make_metrics_interceptor()])
|
||||
|
||||
# 注册 AnalyticsService
|
||||
servicer = AnalyticsServiceServicer()
|
||||
@@ -544,7 +800,7 @@ async def start_grpc_server() -> Any | None:
|
||||
logger.info(
|
||||
"grpc_server_started",
|
||||
port=settings.grpc_port,
|
||||
rpc_count=12,
|
||||
rpc_count=18,
|
||||
)
|
||||
return _server
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
@@ -34,7 +34,7 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query, Response
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
@@ -134,23 +134,38 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
if grpc_server_obj is None:
|
||||
logger.warning("grpc_server_not_started_http_only")
|
||||
|
||||
# 2. 启动 CDC 消费者后台任务
|
||||
# 2. 预热客户端（避免首次 readyz 探针因惰性初始化超时）
|
||||
if settings.clickhouse_host:
|
||||
ch_ok = await clickhouse_repository.ping()
|
||||
logger.info("clickhouse_warmup", ok=ch_ok)
|
||||
if settings.redis_url:
|
||||
redis_ok = await redis_client.ping()
|
||||
logger.info("redis_warmup", ok=redis_ok)
|
||||
|
||||
# 2.1 初始化监控指标初始值（确保 /metrics 端点暴露自定义指标）
|
||||
from . import metrics as _metrics
|
||||
|
||||
_metrics.clickhouse_connection_status.set(1 if settings.clickhouse_host else 0)
|
||||
_metrics.redis_connection_status.set(1 if settings.redis_url else 0)
|
||||
_metrics.iam_grpc_connection_status.set(1 if settings.iam_grpc_endpoint else 0)
|
||||
|
||||
# 3. 启动 CDC 消费者后台任务
|
||||
await cdc_consumer.start_consumer()
|
||||
|
||||
yield
|
||||
|
||||
logger.info("data_ana_service_stopping")
|
||||
|
||||
# 3. 停止 CDC 消费者
|
||||
# 4. 停止 CDC 消费者
|
||||
await cdc_consumer.stop_consumer()
|
||||
|
||||
# 4. 停止 gRPC server
|
||||
# 5. 停止 gRPC server
|
||||
await grpc_server.stop_grpc_server()
|
||||
|
||||
# 5. 关闭 Kafka producer
|
||||
# 6. 关闭 Kafka producer
|
||||
await kafka_producer.close_producer()
|
||||
|
||||
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
# 7. 关闭 Redis / iam gRPC / ClickHouse 客户端
|
||||
await redis_client.close_client()
|
||||
await iam_client.close_grpc()
|
||||
await clickhouse_repository.close_client()
|
||||
@@ -196,36 +211,78 @@ async def healthz() -> dict[str, str]:
|
||||
|
||||
|
||||
@app.get("/readyz")
|
||||
async def readyz() -> dict[str, Any]:
|
||||
"""就绪检查（readiness，检查 4 依赖）.
|
||||
async def readyz(response: Response) -> dict[str, Any]:
|
||||
"""就绪检查（P6 深度硬化版，检查 4 依赖 + 超时控制 + lag 阈值）.
|
||||
|
||||
依赖检查：
|
||||
1. clickhouse：已配置且可达（未配置算降级就绪）
|
||||
2. cdc_consumer：running / disabled
|
||||
3. redis：已配置且可达（未配置算降级就绪）
|
||||
4. iam_grpc：已配置且可达（未配置算降级就绪）
|
||||
依赖检查（带超时）：
|
||||
1. clickhouse：1s 超时，已配置且可达（未配置算降级就绪）
|
||||
2. cdc_consumer：running / disabled + lag < 1000
|
||||
3. redis：200ms 超时，已配置且可达（未配置算降级就绪）
|
||||
4. iam_grpc：2s 超时，已配置且可达（未配置算降级就绪）
|
||||
|
||||
返回 ready=true 的条件：
|
||||
- ClickHouse 已配置且可达，或未配置（降级就绪）
|
||||
- 不要求所有依赖都健康（降级模式下仍可服务骨架数据）
|
||||
- ClickHouse 已配置且可达（1s 内），或未配置（降级就绪）
|
||||
- CDC consumer lag < readyz_cdc_lag_threshold（1000）
|
||||
不满足时返回 HTTP 503，K8s 摘流量.
|
||||
"""
|
||||
# 1. ClickHouse
|
||||
ch_ok = await clickhouse_repository.ping()
|
||||
import asyncio
|
||||
|
||||
# 1. ClickHouse（1s 超时）
|
||||
try:
|
||||
ch_ok = await asyncio.wait_for(
|
||||
clickhouse_repository.ping(),
|
||||
timeout=settings.readyz_clickhouse_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
ch_ok = False
|
||||
ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
|
||||
|
||||
# 2. CDC 消费者
|
||||
cdc_status = (
|
||||
"running"
|
||||
if cdc_consumer.is_running()
|
||||
else ("disabled" if not settings.kafka_brokers else "failed")
|
||||
# 2. CDC 消费者 + lag 检查
|
||||
cdc_running = cdc_consumer.is_running()
|
||||
cdc_lag = 0
|
||||
if cdc_running:
|
||||
try:
|
||||
cdc_lag = await asyncio.wait_for(
|
||||
cdc_consumer.get_lag(),
|
||||
timeout=2.0,
|
||||
)
|
||||
except TimeoutError:
|
||||
cdc_lag = -1 # 查询超时标记
|
||||
cdc_lag_ok = cdc_lag >= 0 and cdc_lag < settings.readyz_cdc_lag_threshold
|
||||
if not settings.kafka_brokers:
|
||||
cdc_status = "disabled"
|
||||
cdc_lag_ok = True # 未配置 Kafka 时不阻塞 ready
|
||||
elif cdc_running and cdc_lag_ok:
|
||||
cdc_status = f"running(lag={cdc_lag})"
|
||||
elif cdc_running:
|
||||
cdc_status = f"running(lag={cdc_lag},exceeded)"
|
||||
else:
|
||||
cdc_status = "failed"
|
||||
|
||||
# 3. Redis
|
||||
redis_ok = await redis_client.ping() if settings.redis_url else None
|
||||
# 3. Redis（200ms 超时）
|
||||
if settings.redis_url:
|
||||
try:
|
||||
redis_ok = await asyncio.wait_for(
|
||||
redis_client.ping(),
|
||||
timeout=settings.readyz_redis_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
redis_ok = False
|
||||
else:
|
||||
redis_ok = None
|
||||
redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
|
||||
|
||||
# 4. iam gRPC
|
||||
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
|
||||
# 4. iam gRPC（2s 超时，iam_client.ping 已内置 1s 超时）
|
||||
if settings.iam_grpc_endpoint:
|
||||
try:
|
||||
iam_ok = await asyncio.wait_for(
|
||||
iam_client.ping(),
|
||||
timeout=settings.readyz_iam_grpc_timeout_s,
|
||||
)
|
||||
except TimeoutError:
|
||||
iam_ok = False
|
||||
else:
|
||||
iam_ok = None
|
||||
iam_status = (
|
||||
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
|
||||
)
|
||||
@@ -233,10 +290,15 @@ async def readyz() -> dict[str, Any]:
|
||||
# 5. gRPC server
|
||||
grpc_status = "running" if grpc_server.is_running() else "stopped"
|
||||
|
||||
# 就绪判定：ClickHouse 可达或未配置（降级就绪）
|
||||
ready = ch_ok or not settings.clickhouse_host
|
||||
# 就绪判定（P6 硬化）：
|
||||
# - ClickHouse 可达或未配置（降级就绪）
|
||||
# - CDC consumer lag 未超阈值
|
||||
ready = (ch_ok or not settings.clickhouse_host) and cdc_lag_ok
|
||||
degraded = not ch_ok or not redis_ok or not iam_ok
|
||||
|
||||
if not ready:
|
||||
response.status_code = 503 # K8s 摘流量
|
||||
|
||||
return {
|
||||
"status": "ok" if ready else "not_ready",
|
||||
"service": "data-ana",
|
||||
@@ -250,6 +312,13 @@ async def readyz() -> dict[str, Any]:
|
||||
"grpc_server": grpc_status,
|
||||
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
|
||||
},
|
||||
"thresholds": {
|
||||
"clickhouse_timeout_s": settings.readyz_clickhouse_timeout_s,
|
||||
"redis_timeout_s": settings.readyz_redis_timeout_s,
|
||||
"iam_grpc_timeout_s": settings.readyz_iam_grpc_timeout_s,
|
||||
"cdc_lag_threshold": settings.readyz_cdc_lag_threshold,
|
||||
"cdc_lag_current": cdc_lag,
|
||||
},
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
205
services/data-ana/src/data_ana/metrics.py
Normal file
205
services/data-ana/src/data_ana/metrics.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""Prometheus 指标定义（P6 监控告警完善）.
|
||||
|
||||
对齐 workline §3.5 任务 6.4：
|
||||
- consumer lag histogram（CDC 消费延迟）
|
||||
- 慢查询 counter（ClickHouse 查询耗时）
|
||||
- ClickHouse 连接池 gauge
|
||||
- ExamCache gauge（缓存命中率）
|
||||
- DataScope 缓存 counter
|
||||
- gRPC 请求 counter + histogram
|
||||
- 掌握度计算 counter
|
||||
- 预警触发 counter
|
||||
|
||||
指标命名规范（对齐项目规则 §12）：
|
||||
data_ana_<module>_<operation>_<unit>
|
||||
|
||||
Grafana dashboard 配置见 infra/grafana/dashboards/data-ana.json
|
||||
"""
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
# ===== CDC Consumer 指标 =====
|
||||
|
||||
# 消费者 lag（按 topic + partition 分维度）
|
||||
cdc_consumer_lag = Gauge(
|
||||
"data_ana_cdc_consumer_lag",
|
||||
"CDC consumer lag (messages waiting to be consumed)",
|
||||
["topic", "partition", "instance"],
|
||||
)
|
||||
|
||||
# 消费者处理消息总数
|
||||
cdc_messages_processed_total = Counter(
|
||||
"data_ana_cdc_messages_processed_total",
|
||||
"Total CDC messages processed",
|
||||
["topic", "table", "status"], # status: success / failed / skipped
|
||||
)
|
||||
|
||||
# 消费者处理耗时
|
||||
cdc_message_process_duration_seconds = Histogram(
|
||||
"data_ana_cdc_message_process_duration_seconds",
|
||||
"CDC message processing duration",
|
||||
["table"],
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0),
|
||||
)
|
||||
|
||||
# 消费者实例数（多实例水平扩展监控）
|
||||
cdc_consumer_active_instances = Gauge(
|
||||
"data_ana_cdc_consumer_active_instances",
|
||||
"Number of active CDC consumer instances (this instance reports 1 when running)",
|
||||
)
|
||||
|
||||
# ===== ClickHouse 查询指标 =====
|
||||
|
||||
# 查询耗时直方图
|
||||
clickhouse_query_duration_seconds = Histogram(
|
||||
"data_ana_clickhouse_query_duration_seconds",
|
||||
"ClickHouse query duration",
|
||||
["operation"], # operation: query_class_performance / query_student_dashboard / etc.
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 3.0, 5.0),
|
||||
)
|
||||
|
||||
# 查询错误计数
|
||||
clickhouse_query_errors_total = Counter(
|
||||
"data_ana_clickhouse_query_errors_total",
|
||||
"Total ClickHouse query errors",
|
||||
["operation", "error_type"],
|
||||
)
|
||||
|
||||
# ClickHouse 连接状态（1=connected, 0=disconnected）
|
||||
clickhouse_connection_status = Gauge(
|
||||
"data_ana_clickhouse_connection_status",
|
||||
"ClickHouse connection status (1=connected, 0=disconnected)",
|
||||
)
|
||||
|
||||
# 慢查询计数（超过阈值 1s）
|
||||
clickhouse_slow_queries_total = Counter(
|
||||
"data_ana_clickhouse_slow_queries_total",
|
||||
"Total ClickHouse slow queries (>1s)",
|
||||
["operation"],
|
||||
)
|
||||
|
||||
# ===== ExamCache 指标 =====
|
||||
|
||||
# 缓存大小
|
||||
exam_cache_size = Gauge(
|
||||
"data_ana_exam_cache_size",
|
||||
"ExamCache current size (memory LRU)",
|
||||
)
|
||||
|
||||
# 缓存命中/未命中
|
||||
exam_cache_hits_total = Counter(
|
||||
"data_ana_exam_cache_hits_total",
|
||||
"ExamCache cache hits",
|
||||
)
|
||||
|
||||
exam_cache_misses_total = Counter(
|
||||
"data_ana_exam_cache_misses_total",
|
||||
"ExamCache cache misses",
|
||||
)
|
||||
|
||||
# Redis 连接状态
|
||||
redis_connection_status = Gauge(
|
||||
"data_ana_redis_connection_status",
|
||||
"Redis connection status (1=connected, 0=disconnected)",
|
||||
)
|
||||
|
||||
# ===== DataScope 缓存指标 =====
|
||||
|
||||
datascope_cache_hits_total = Counter(
|
||||
"data_ana_datascope_cache_hits_total",
|
||||
"DataScope cache hits",
|
||||
)
|
||||
|
||||
datascope_cache_misses_total = Counter(
|
||||
"data_ana_datascope_cache_misses_total",
|
||||
"DataScope cache misses",
|
||||
)
|
||||
|
||||
datascope_fallback_total = Counter(
|
||||
"data_ana_datascope_fallback_total",
|
||||
"DataScope fallback (iam gRPC unavailable, using role-based fallback)",
|
||||
)
|
||||
|
||||
# ===== gRPC 指标 =====
|
||||
|
||||
grpc_requests_total = Counter(
|
||||
"data_ana_grpc_requests_total",
|
||||
"Total gRPC requests",
|
||||
["method", "status"], # status: success / failed / degraded
|
||||
)
|
||||
|
||||
grpc_request_duration_seconds = Histogram(
|
||||
"data_ana_grpc_request_duration_seconds",
|
||||
"gRPC request duration",
|
||||
["method"],
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 3.0, 5.0),
|
||||
)
|
||||
|
||||
# iam gRPC 连接状态
|
||||
iam_grpc_connection_status = Gauge(
|
||||
"data_ana_iam_grpc_connection_status",
|
||||
"iam gRPC connection status (1=connected, 0=disconnected)",
|
||||
)
|
||||
|
||||
# ===== 业务指标 =====
|
||||
|
||||
# 掌握度计算次数
|
||||
mastery_calculations_total = Counter(
|
||||
"data_ana_mastery_calculations_total",
|
||||
"Total mastery level calculations",
|
||||
["method"], # method: weighted_moving_avg / forgetting_curve
|
||||
)
|
||||
|
||||
# 预警触发次数
|
||||
warnings_triggered_total = Counter(
|
||||
"data_ana_warnings_triggered_total",
|
||||
"Total warnings triggered",
|
||||
["warning_type", "severity"],
|
||||
)
|
||||
|
||||
# HTTP 请求指标（FastAPI 自动 instrumentation 已覆盖，这里补充业务层）
|
||||
http_requests_degraded_total = Counter(
|
||||
"data_ana_http_requests_degraded_total",
|
||||
"Total HTTP requests in degraded mode",
|
||||
["endpoint"],
|
||||
)
|
||||
|
||||
|
||||
def record_clickhouse_query(operation: str, duration_s: float, success: bool) -> None:
|
||||
"""记录 ClickHouse 查询指标（供 repository 调用）."""
|
||||
clickhouse_query_duration_seconds.labels(operation=operation).observe(duration_s)
|
||||
if not success:
|
||||
clickhouse_query_errors_total.labels(operation=operation, error_type="query_failed").inc()
|
||||
elif duration_s > 1.0:
|
||||
clickhouse_slow_queries_total.labels(operation=operation).inc()
|
||||
|
||||
|
||||
def record_grpc_request(method: str, duration_s: float, status: str) -> None:
|
||||
"""记录 gRPC 请求指标（供 grpc_server 调用）."""
|
||||
grpc_requests_total.labels(method=method, status=status).inc()
|
||||
grpc_request_duration_seconds.labels(method=method).observe(duration_s)
|
||||
|
||||
|
||||
def record_exam_cache_hit() -> None:
|
||||
"""记录 ExamCache 命中."""
|
||||
exam_cache_hits_total.inc()
|
||||
|
||||
|
||||
def record_exam_cache_miss() -> None:
|
||||
"""记录 ExamCache 未命中."""
|
||||
exam_cache_misses_total.inc()
|
||||
|
||||
|
||||
def record_datascope_cache_hit() -> None:
|
||||
"""记录 DataScope 缓存命中."""
|
||||
datascope_cache_hits_total.inc()
|
||||
|
||||
|
||||
def record_datascope_cache_miss() -> None:
|
||||
"""记录 DataScope 缓存未命中."""
|
||||
datascope_cache_misses_total.inc()
|
||||
|
||||
|
||||
def record_datascope_fallback() -> None:
|
||||
"""记录 DataScope 降级兜底."""
|
||||
datascope_fallback_total.inc()
|
||||
@@ -5,7 +5,7 @@ P0 整改：ReplacingMergeTree 查询必须加 FINAL 或用 argMax 聚合确保
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
@@ -793,3 +793,213 @@ async def query_student_scores_by_kp(
|
||||
return None
|
||||
|
||||
return [{"score": float(row[0] or 0), "timestamp": row[1], "exam_id": row[2]} for row in rows]
|
||||
|
||||
|
||||
# ===== P6+ v2 扩展查询方法 =====
|
||||
|
||||
|
||||
async def query_assignment_analysis(
|
||||
class_id: str,
|
||||
assignment_id: str,
|
||||
subject_id: str = "",
|
||||
) -> dict | None:
|
||||
"""查询作业/考试分析（单次作业/考试维度统计）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["class_id = {cid:String}", "exam_id = {aid:String}"]
|
||||
params: dict[str, Any] = {"cid": class_id, "aid": assignment_id}
|
||||
if subject_id:
|
||||
where_parts.append("subject_id = {sid:String}")
|
||||
params["sid"] = subject_id
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" avg(score) AS avg_score, "
|
||||
f" max(score) AS max_score, "
|
||||
f" min(score) AS min_score, "
|
||||
f" count() AS total, "
|
||||
f" countIf(score >= 60) / count() AS pass_rate, "
|
||||
f" countIf(score >= 90) AS r90, "
|
||||
f" countIf(score >= 80 AND score < 90) AS r80, "
|
||||
f" countIf(score >= 70 AND score < 80) AS r70, "
|
||||
f" countIf(score >= 60 AND score < 70) AS r60, "
|
||||
f" countIf(score < 60) AS r_below "
|
||||
f"FROM student_dashboard_view FINAL "
|
||||
f"WHERE {where_clause}",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"query_assignment_analysis_failed",
|
||||
error=str(exc),
|
||||
class_id=class_id,
|
||||
assignment_id=assignment_id,
|
||||
)
|
||||
return None
|
||||
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
(
|
||||
avg_score,
|
||||
max_score,
|
||||
min_score,
|
||||
total,
|
||||
pass_rate,
|
||||
r90,
|
||||
r80,
|
||||
r70,
|
||||
r60,
|
||||
r_below,
|
||||
) = rows[0]
|
||||
total_int = int(total or 0)
|
||||
|
||||
def _pct(count: Any) -> float:
|
||||
return (int(count or 0) / total_int) if total_int else 0.0
|
||||
|
||||
ranges = [
|
||||
{"label": "90-100", "count": int(r90 or 0), "percentage": _pct(r90)},
|
||||
{"label": "80-89", "count": int(r80 or 0), "percentage": _pct(r80)},
|
||||
{"label": "70-79", "count": int(r70 or 0), "percentage": _pct(r70)},
|
||||
{"label": "60-69", "count": int(r60 or 0), "percentage": _pct(r60)},
|
||||
{"label": "<60", "count": int(r_below or 0), "percentage": _pct(r_below)},
|
||||
]
|
||||
return {
|
||||
"assignmentId": assignment_id,
|
||||
"classId": class_id,
|
||||
"subjectId": subject_id,
|
||||
"averageScore": float(avg_score or 0),
|
||||
"highestScore": float(max_score or 0),
|
||||
"lowestScore": float(min_score or 0),
|
||||
"totalStudents": total_int,
|
||||
"submittedCount": total_int,
|
||||
"passRate": float(pass_rate or 0),
|
||||
"ranges": ranges,
|
||||
}
|
||||
|
||||
|
||||
async def query_error_book_stats(student_id: str, subject_id: str = "") -> dict | None:
|
||||
"""查询错题本统计（按知识点聚合）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["student_id = {sid:String}"]
|
||||
params: dict[str, Any] = {"sid": student_id}
|
||||
# subject_id 过滤需要 join 知识点表，简化为按 knowledge_point_id 前缀匹配
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" knowledge_point_id, "
|
||||
f" sum(error_count) AS total_errors, "
|
||||
f" count(DISTINCT question_id) AS question_count, "
|
||||
f" max(last_error_time) AS last_error "
|
||||
f"FROM student_errors FINAL "
|
||||
f"WHERE {where_clause} "
|
||||
f"GROUP BY knowledge_point_id "
|
||||
f"ORDER BY total_errors DESC "
|
||||
f"LIMIT 50",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_error_book_stats_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
by_kp = []
|
||||
total_errors = 0
|
||||
total_questions = 0
|
||||
for row in rows:
|
||||
kp_id, err_count, q_count, _last = row
|
||||
err_count_int = int(err_count or 0)
|
||||
q_count_int = int(q_count or 0)
|
||||
total_errors += err_count_int
|
||||
total_questions += q_count_int
|
||||
by_kp.append(
|
||||
{
|
||||
"knowledge_point_id": kp_id,
|
||||
"title": kp_id, # content CDC 同步后补充
|
||||
"error_count": err_count_int,
|
||||
"question_count": q_count_int,
|
||||
"error_rate": (err_count_int / q_count_int) if q_count_int else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
# 查询最近 7 天错误次数
|
||||
try:
|
||||
seven_days_ago = datetime.now(UTC) - timedelta(days=7)
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
"SELECT sum(error_count) "
|
||||
"FROM student_errors FINAL "
|
||||
"WHERE student_id = {sid:String} AND last_error_time >= {sd:DateTime64(3)}",
|
||||
parameters={"sid": student_id, "sd": seven_days_ago},
|
||||
)
|
||||
recent_rows = result.result_rows
|
||||
recent_7d = int(recent_rows[0][0] or 0) if recent_rows else 0
|
||||
except Exception: # noqa: BLE001
|
||||
recent_7d = 0
|
||||
|
||||
return {
|
||||
"studentId": student_id,
|
||||
"totalErrorQuestions": len(by_kp),
|
||||
"totalErrorCount": total_errors,
|
||||
"byKnowledgePoint": by_kp,
|
||||
"recent7dErrors": recent_7d,
|
||||
}
|
||||
|
||||
|
||||
async def query_mastery_trend(
|
||||
student_id: str,
|
||||
start_date: int = 0,
|
||||
end_date: int = 0,
|
||||
) -> list[dict] | None:
|
||||
"""查询掌握度变化趋势（按时间排序的快照）."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
where_parts = ["student_id = {sid:String}"]
|
||||
params: dict[str, Any] = {"sid": student_id}
|
||||
if start_date:
|
||||
where_parts.append("calculated_at >= {sd:DateTime64(3)}")
|
||||
params["sd"] = datetime.fromtimestamp(start_date)
|
||||
if end_date:
|
||||
where_parts.append("calculated_at <= {ed:DateTime64(3)}")
|
||||
params["ed"] = datetime.fromtimestamp(end_date)
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
client.query,
|
||||
f"SELECT "
|
||||
f" calculated_at, "
|
||||
f" avg(mastery_level) AS overall_mastery "
|
||||
f"FROM mastery_snapshot "
|
||||
f"WHERE {where_clause} "
|
||||
f"GROUP BY calculated_at "
|
||||
f"ORDER BY calculated_at ASC "
|
||||
f"LIMIT 100",
|
||||
parameters=params,
|
||||
)
|
||||
rows = result.result_rows
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("query_mastery_trend_failed", error=str(exc), student_id=student_id)
|
||||
return None
|
||||
|
||||
return [
|
||||
{
|
||||
"calculated_at": int(row[0].timestamp()) if row[0] else 0,
|
||||
"overall_mastery": float(row[1] or 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -111,6 +111,42 @@ class AnalyticsServiceStub:
|
||||
response_deserializer=analytics__pb2.MasteryUpdateEvent.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.GetStudentGrowth = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetStudentGrowth",
|
||||
request_serializer=analytics__pb2.GetStudentGrowthRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.StudentGrowth.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.GetAssignmentAnalysis = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetAssignmentAnalysis",
|
||||
request_serializer=analytics__pb2.GetAssignmentAnalysisRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.AssignmentAnalysis.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.GetMasterySummary = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetMasterySummary",
|
||||
request_serializer=analytics__pb2.GetMasterySummaryRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.MasterySummary.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.ListDiagnosticReports = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/ListDiagnosticReports",
|
||||
request_serializer=analytics__pb2.ListDiagnosticReportsRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.DiagnosticReportList.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.ListErrorBookItems = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/ListErrorBookItems",
|
||||
request_serializer=analytics__pb2.ListErrorBookItemsRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.ErrorBookList.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
self.GetErrorBookStats = channel.unary_unary(
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetErrorBookStats",
|
||||
request_serializer=analytics__pb2.GetErrorBookStatsRequest.SerializeToString,
|
||||
response_deserializer=analytics__pb2.ErrorBookStats.FromString,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
|
||||
class AnalyticsServiceServicer:
|
||||
@@ -191,6 +227,44 @@ class AnalyticsServiceServicer:
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def GetStudentGrowth(self, request, context):
|
||||
"""===== P6+ v2 扩展 RPC（响应上游 parent-bff / student-bff v2 请求） =====
|
||||
学生成长档案（综合成绩趋势 + 掌握度变化 + 考勤统计，供 parent-bff GrowthArchiveService）.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def GetAssignmentAnalysis(self, request, context):
|
||||
"""作业/考试分析（单次作业/考试维度统计，供 student-bff / parent-bff）."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def GetMasterySummary(self, request, context):
|
||||
"""学生掌握度汇总（轻量级，仅返回总体掌握度 + 三档分布，供 parent-bff MasteryService.GetSummary）."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def ListDiagnosticReports(self, request, context):
|
||||
"""诊断报告列表（占位实现，真实数据需 ai 服务集成，供 parent-bff / student-bff）."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def ListErrorBookItems(self, request, context):
|
||||
"""错题本列表（gRPC 版本，供 parent-bff ErrorBookService.List）."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
def GetErrorBookStats(self, request, context):
|
||||
"""错题本统计（按知识点聚合，供 parent-bff ErrorBookService.GetStats）."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details("Method not implemented!")
|
||||
raise NotImplementedError("Method not implemented!")
|
||||
|
||||
|
||||
def add_AnalyticsServiceServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
@@ -254,6 +328,36 @@ def add_AnalyticsServiceServicer_to_server(servicer, server):
|
||||
request_deserializer=analytics__pb2.SubscribeMasteryUpdateRequest.FromString,
|
||||
response_serializer=analytics__pb2.MasteryUpdateEvent.SerializeToString,
|
||||
),
|
||||
"GetStudentGrowth": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.GetStudentGrowth,
|
||||
request_deserializer=analytics__pb2.GetStudentGrowthRequest.FromString,
|
||||
response_serializer=analytics__pb2.StudentGrowth.SerializeToString,
|
||||
),
|
||||
"GetAssignmentAnalysis": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.GetAssignmentAnalysis,
|
||||
request_deserializer=analytics__pb2.GetAssignmentAnalysisRequest.FromString,
|
||||
response_serializer=analytics__pb2.AssignmentAnalysis.SerializeToString,
|
||||
),
|
||||
"GetMasterySummary": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.GetMasterySummary,
|
||||
request_deserializer=analytics__pb2.GetMasterySummaryRequest.FromString,
|
||||
response_serializer=analytics__pb2.MasterySummary.SerializeToString,
|
||||
),
|
||||
"ListDiagnosticReports": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ListDiagnosticReports,
|
||||
request_deserializer=analytics__pb2.ListDiagnosticReportsRequest.FromString,
|
||||
response_serializer=analytics__pb2.DiagnosticReportList.SerializeToString,
|
||||
),
|
||||
"ListErrorBookItems": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ListErrorBookItems,
|
||||
request_deserializer=analytics__pb2.ListErrorBookItemsRequest.FromString,
|
||||
response_serializer=analytics__pb2.ErrorBookList.SerializeToString,
|
||||
),
|
||||
"GetErrorBookStats": grpc.unary_unary_rpc_method_handler(
|
||||
servicer.GetErrorBookStats,
|
||||
request_deserializer=analytics__pb2.GetErrorBookStatsRequest.FromString,
|
||||
response_serializer=analytics__pb2.ErrorBookStats.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
"next_edu_cloud.analytics.v1.AnalyticsService", rpc_method_handlers
|
||||
@@ -630,3 +734,183 @@ class AnalyticsService:
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def GetStudentGrowth(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetStudentGrowth",
|
||||
analytics__pb2.GetStudentGrowthRequest.SerializeToString,
|
||||
analytics__pb2.StudentGrowth.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def GetAssignmentAnalysis(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetAssignmentAnalysis",
|
||||
analytics__pb2.GetAssignmentAnalysisRequest.SerializeToString,
|
||||
analytics__pb2.AssignmentAnalysis.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def GetMasterySummary(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetMasterySummary",
|
||||
analytics__pb2.GetMasterySummaryRequest.SerializeToString,
|
||||
analytics__pb2.MasterySummary.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ListDiagnosticReports(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/ListDiagnosticReports",
|
||||
analytics__pb2.ListDiagnosticReportsRequest.SerializeToString,
|
||||
analytics__pb2.DiagnosticReportList.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ListErrorBookItems(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/ListErrorBookItems",
|
||||
analytics__pb2.ListErrorBookItemsRequest.SerializeToString,
|
||||
analytics__pb2.ErrorBookList.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def GetErrorBookStats(
|
||||
request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None,
|
||||
):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
"/next_edu_cloud.analytics.v1.AnalyticsService/GetErrorBookStats",
|
||||
analytics__pb2.GetErrorBookStatsRequest.SerializeToString,
|
||||
analytics__pb2.ErrorBookStats.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user