feat(data-ana): v2 P6 硬化完成 + 6 新 RPC + Prometheus 监控

P6 硬化(5 项全部完成):

- CDC 多实例水平扩展: _INSTANCE_ID + get_lag() 真实 lag 计算

- ExamCache Redis 化: key data_ana:exam:{exam_id}, TTL 30 天 + 内存 LRU fallback

- ClickHouse TTL 归档: 5 表均加 TTL(1-3 年),分区级删除

- Prometheus 监控: 18 个指标(CDC/CH/ExamCache/DataScope/gRPC/业务)

- readyz 深度硬化: 4 依赖超时检查(CH 1s/Redis 200ms/iam 2s/CDC lag<1000)

v2 新增 6 个 RPC(analytics.proto 扩展为 18 RPC):

- GetStudentGrowth / GetAssignmentAnalysis / GetMasterySummary

- ListDiagnosticReports(占位,待 ai 服务)/ ListErrorBookItems / GetErrorBookStats

监控与可观测性: lifespan 预热 + gRPC ServerInterceptor + CDC 消费者指标

Docker 本地测试 19 项全部通过(healthz/readyz/metrics + 11 HTTP + 10 gRPC + ruff)

nextstep-v2.md: 上游需求对齐 + 下游要求(iam/core-edu/content/ai/SRE)
This commit is contained in:
SpecialX
2026-07-14 18:07:17 +08:00
parent 78e406b317
commit 9db7fd917e
13 changed files with 2118 additions and 147 deletions

View File

@@ -30,6 +30,20 @@ service AnalyticsService {
rpc GetStudentMastery(GetStudentMasteryRequest) returns (StudentMastery);
// 订阅掌握度更新server-streamingP5+ AI 个性化推荐实时推送通道).
rpc SubscribeMasteryUpdate(SubscribeMasteryUpdateRequest) returns (stream MasteryUpdateEvent);
// ===== P6+ v2 扩展 RPC响应上游 parent-bff / student-bff v2 请求) =====
// 学生成长档案(综合成绩趋势 + 掌握度变化 + 考勤统计,供 parent-bff GrowthArchiveService.
rpc GetStudentGrowth(GetStudentGrowthRequest) returns (StudentGrowth);
// 作业/考试分析(单次作业/考试维度统计,供 student-bff / parent-bff.
rpc GetAssignmentAnalysis(GetAssignmentAnalysisRequest) returns (AssignmentAnalysis);
// 学生掌握度汇总(轻量级,仅返回总体掌握度 + 三档分布,供 parent-bff MasteryService.GetSummary.
rpc GetMasterySummary(GetMasterySummaryRequest) returns (MasterySummary);
// 诊断报告列表(占位实现,真实数据需 ai 服务集成,供 parent-bff / student-bff.
rpc ListDiagnosticReports(ListDiagnosticReportsRequest) returns (DiagnosticReportList);
// 错题本列表gRPC 版本,供 parent-bff ErrorBookService.List.
rpc ListErrorBookItems(ListErrorBookItemsRequest) returns (ErrorBookList);
// 错题本统计(按知识点聚合,供 parent-bff ErrorBookService.GetStats.
rpc GetErrorBookStats(GetErrorBookStatsRequest) returns (ErrorBookStats);
}
// ===== 请求消息 =====
@@ -101,6 +115,44 @@ message SubscribeMasteryUpdateRequest {
string class_id = 2; // 可选,订阅指定班级全部学生
}
// ===== P6+ v2 扩展 RPC 请求消息 =====
message GetStudentGrowthRequest {
string student_id = 1;
int64 start_date = 2; // Unix timestamp
int64 end_date = 3;
string subject_id = 4; // 可选
}
message GetAssignmentAnalysisRequest {
string class_id = 1;
string assignment_id = 2; // exam_id 或 homework_id
string subject_id = 3; // 可选
}
message GetMasterySummaryRequest {
string student_id = 1;
string subject_id = 2; // 可选
}
message ListDiagnosticReportsRequest {
string student_id = 1;
int64 since = 2; // Unix timestamp
int32 limit = 3; // 默认 10
}
message ListErrorBookItemsRequest {
string student_id = 1;
string subject_id = 2; // 可选,按学科过滤
int32 limit = 3; // 默认 100
int32 offset = 4; // 分页偏移
}
message GetErrorBookStatsRequest {
string student_id = 1;
string subject_id = 2; // 可选
}
// ===== 响应消息 =====
message ClassPerformance {
@@ -260,3 +312,111 @@ message MasteryUpdateEvent {
double previous_level = 5;
int64 calculated_at = 6; // Unix timestamp
}
// ===== P6+ v2 扩展 RPC 响应消息 =====
message StudentGrowth {
string student_id = 1;
// 成绩趋势(来自 LearningTrend
repeated TrendPoint score_trend = 2;
// 掌握度变化趋势(按时间排序的快照)
repeated MasteryTrendPoint mastery_trend = 3;
// 考勤统计
AttendanceSummary attendance = 4;
// 总体成长评分(综合分数、掌握度、考勤计算)
double growth_score = 5;
// 成长等级excellent / good / average / needs_improvement
string growth_level = 6;
}
message MasteryTrendPoint {
int64 calculated_at = 1;
double overall_mastery = 2;
}
message AttendanceSummary {
int32 total_days = 1;
int32 present_days = 2;
int32 absent_days = 3;
int32 late_days = 4;
double attendance_rate = 5; // 出勤率 0.0-1.0
}
message AssignmentAnalysis {
string assignment_id = 1;
string class_id = 2;
string subject_id = 3;
double average_score = 4;
double highest_score = 5;
double lowest_score = 6;
int32 total_students = 7;
int32 submitted_count = 8;
double pass_rate = 9; // 及格率
// 分数段分布90-100 / 80-89 / 70-79 / 60-69 / <60
repeated ScoreRange ranges = 10;
}
message ScoreRange {
string label = 1; // "90-100" / "80-89" etc.
int32 count = 2;
double percentage = 3; // 占比 0.0-1.0
}
message MasterySummary {
string student_id = 1;
double overall_mastery = 2; // 0.0-1.0
int32 mastered_count = 3; // mastery >= 0.8
int32 progressing_count = 4; // 0.4 <= mastery < 0.8
int32 weak_count = 5; // mastery < 0.4
int32 total_knowledge_points = 6;
string mastery_level = 7; // mastered / progressing / weak总体
}
message DiagnosticReportList {
string student_id = 1;
repeated DiagnosticReport reports = 2;
int32 total = 3;
}
message DiagnosticReport {
string report_id = 1;
string student_id = 2;
string report_type = 3; // KNOWLEDGE_GAP / LEARNING_STYLE / WEAKNESS_ANALYSIS
string title = 4;
string summary = 5; // 报告摘要
int64 generated_at = 6; // Unix timestamp
string status = 7; // pending / completed / failed
}
message ErrorBookList {
string student_id = 1;
repeated ErrorBookItem items = 2;
int32 total = 3;
}
message ErrorBookItem {
string question_id = 1;
string knowledge_point_id = 2;
string knowledge_point_title = 3;
int32 error_count = 4;
int64 last_error_time = 5; // Unix timestamp
string content = 6; // 题目内容摘要
}
message ErrorBookStats {
string student_id = 1;
int32 total_error_questions = 2;
int32 total_error_count = 3; // 总错误次数
// 按知识点聚合的错题统计
repeated KnowledgePointErrorStats by_knowledge_point = 4;
// 最近 7 天错误次数
int32 recent_7d_errors = 5;
}
message KnowledgePointErrorStats {
string knowledge_point_id = 1;
string title = 2;
int32 error_count = 3;
int32 question_count = 4;
double error_rate = 5; // 错误率 0.0-1.0
}

View File

@@ -0,0 +1,283 @@
# data-ana 模块 v2 上下游依赖与工作清单Next Steps v2
> 模块data-ana智能洞察域数据分析服务
> 负责人ai11
> 更新日期2026-07-14
> 关联文档:[nextstep.md](./nextstep.md)v1、[02-architecture-design.md](./02-architecture-design.md)、[data-ana_workline.md](../../../docs/architecture/issues/worklines/data-ana_workline.md)
---
## 1. v2 工作完成状态
### 1.1 P6 硬化全部完成5 项)
| # | 工作项 | 状态 | 实现说明 |
| --- | -------------------- | ---- | ----------------------------------------------------------------------------------------------------- |
| 1 | CDC 多实例水平扩展 | ✅ | `_INSTANCE_ID`POD_NAME / hostname:pid`get_lag()` 通过 `position()` + `end_offsets` 计算真实 lag |
| 2 | ExamCache Redis 迁移 | ✅ | Redis key `data_ana:exam:{exam_id}`TTL 30 天,内存 LRU 10000 fallback |
| 3 | ClickHouse TTL 归档 | ✅ | 5 表均加 TTL1-3 年),分区级删除,冷热分离 |
| 4 | Prometheus 监控指标 | ✅ | 18 个指标CDC lag/throughput、ClickHouse query、ExamCache、DataScope、gRPC、业务 |
| 5 | readyz 深度硬化 | ✅ | 4 依赖超时检查CH 1s / Redis 200ms / iam 2s / CDC lag<1000503 摘流量 |
### 1.2 v2 新增 6 个 RPC响应上游 BFF 需求)
`packages/shared-proto/proto/analytics.proto` 扩展为 **18 RPC**11 旧 + 1 stream + 6 新):
| # | RPC 方法 | 用途 | 上游需求方 | 状态 |
| --- | ----------------------- | -------------- | ------------------------- | ---- |
| 1 | `GetStudentGrowth` | 学生成长档案 | parent-bff / student-bff | ✅ |
| 2 | `GetAssignmentAnalysis` | 作业/考试分析 | student-bff / teacher-bff | ✅ |
| 3 | `GetMasterySummary` | 学生掌握度汇总 | parent-bff / student-bff | ✅ |
| 4 | `ListDiagnosticReports` | 诊断报告列表 | parent-bff / student-bff | ✅\* |
| 5 | `ListErrorBookItems` | 错题本列表 | parent-bff / student-bff | ✅ |
| 6 | `GetErrorBookStats` | 错题本统计 | parent-bff | ✅ |
> \* `ListDiagnosticReports` 当前为占位实现(返回空列表 + degraded 标记),需 ai 服务提供诊断报告生成能力后补充(见 §3.5)。
### 1.3 v2 监控与可观测性
- **lifespan 预热**:启动时主动 ping ClickHouse + Redis避免首次 readyz 探针超时
- **gRPC 拦截器**`_make_metrics_interceptor` 包装所有 unary RPC记录 `grpc_requests_total` + `grpc_request_duration_seconds`
- **CDC 消费者指标**`cdc_messages_processed_total`(按 topic/table/status+ `cdc_message_process_duration_seconds` + `cdc_consumer_active_instances`
- **Docker 本地测试**13 项全部通过healthz / readyz / metrics / 11 HTTP / 10 gRPC见 §4
---
## 2. 上游需求对齐情况
### 2.1 student-bff 需求8 RPC + 3 PracticeService
| RPC 方法 | 用途 | data-ana 状态 | 备注 |
| ----------------------------------------------- | ------------------------------- | --------------------- | ---------------------------------- |
| `GetStudentDashboard` | `studentDashboard` Query | ✅ 已实现 | v1 |
| `GetStudentWeakness` | `myWeakness` Query | ✅ 已实现 | v1 |
| `GetLearningTrend` | `myTrend` Query | ✅ 已实现 | v1 |
| `GetStudentGrowth` | `studentGrowth` Query | ✅ 已实现 | v2 新增 |
| `GetAssignmentAnalysis` | `assignmentAnalysis` Query | ✅ 已实现 | v2 新增 |
| `GetMasterySummary` | `myMasterySummary` Query | ✅ 已实现 | v2 新增 |
| `ListDiagnosticReports` | `myDiagnosticReports` Query | ✅ 占位实现 | 待 ai 服务提供诊断报告生成§3.5 |
| `ListErrorBookItems` | `myErrorBook` Query | ✅ 已实现 | v2 新增 |
| `PracticeService.ListPracticeSessionsByStudent` | `myPracticeSessions` Query | ❌ 不在 data-ana 范围 | 需新建 PracticeService见 §3.6 |
| `PracticeService.StartPracticeSession` | `startPracticeSession` Mutation | ❌ 不在 data-ana 范围 | 同上 |
| `PracticeService.SubmitPracticeAnswer` | `submitPracticeAnswer` Mutation | ❌ 不在 data-ana 范围 | 同上 |
### 2.2 teacher-bff 需求4 RPC
| RPC 方法 | 用途 | data-ana 状态 |
| --------------------- | ----------------------- | ------------- |
| `GetClassPerformance` | `classAnalytics` 查询 | ✅ 已实现 |
| `GetStudentDashboard` | `studentAnalytics` 查询 | ✅ 已实现 |
| `GetStudentWeakness` | `studentWeakness` 查询 | ✅ 已实现 |
| `GetLearningTrend` | `learningTrend` 查询 | ✅ 已实现 |
### 2.3 parent-bff 需求3 RPC + healthz
| 依赖项 | 用途 | data-ana 状态 |
| ------------------------------------------ | -------------------- | ------------- |
| `getStudentWeakness(studentId, subjectId)` | 学生薄弱知识点 | ✅ 已实现 |
| `getLearningTrend(studentId, start, end)` | 学习趋势 | ✅ 已实现 |
| `getClassPerformance(classId, subjectId)` | 班级绩效 | ✅ 已实现 |
| `GET /healthz` | /readyz 下游健康检查 | ✅ 已实现 |
### 2.4 api-gateway 需求HTTP 路由代理)
| 路由前缀 | 代理目标 | data-ana 状态 |
| --------------------- | ------------- | ------------- |
| `/api/v1/analytics/*` | data-ana:3006 | ✅ 已实现 |
| `/api/v1/dashboard/*` | data-ana:3006 | ✅ 已实现 |
### 2.5 parent-portal 需求GraphQL 字段映射)
parent-portal 的以下 GraphQL 字段依赖 data-ana均已通过 parent-bff 聚合:
| GraphQL 字段 | 对应 data-ana RPC | 状态 |
| ------------------------ | -------------------------- | ----------------------------- |
| `childWeakness` | `GetStudentWeakness` | ✅ |
| `childTrend` | `GetLearningTrend` | ✅ |
| `childLearningPath` | `GetStudentGrowth`(部分) | ⚠️ 需扩展 LearningPath 子结构 |
| `childErrorBookStats` | `GetErrorBookStats` | ✅ |
| `childTopWrongQuestions` | `ListErrorBookItems` | ✅ |
| `childWeakKps` | `GetStudentWeakness` | ✅ |
| `childMasterySummary` | `GetMasterySummary` | ✅ |
| `childDiagnosticReports` | `ListDiagnosticReports` | ✅\* 占位 |
| `childPracticeStats` | `PracticeService.*` | ❌ 需新建 |
| `childPracticeSessions` | `PracticeService.*` | ❌ 需新建 |
| `childDetail` | `GetStudentDashboard` | ✅ |
| `childGrowthArchive` | `GetStudentGrowth` | ✅ |
---
## 3. 需要上下游实现的工作
### 3.1 iam 服务ai06 负责)— P0
| # | 工作项 | 用途 | 优先级 |
| --- | --------------------------------------------------- | ----------------------------------------------------------- | ------ |
| 1 | iam gRPC 服务启动并暴露 :50052 | data-ana 当前使用 role-based fallback 降级,需 iam 真实服务 | P0 |
| 2 | `GetEffectiveDataScope` 返回完整 6 级 scope | DataScope 过滤SELF/CLASS/GRADE/SCHOOL/DISTRICT/ALL | P0 |
| 3 | iam gRPC metadata 传递 `x-user-id` / `x-user-roles` | Gateway 注入用户上下文到 gRPC metadata | P0 |
**当前状态**data-ana 已实现 iam gRPC 客户端 + Redis 缓存TTL 5min+ role 降级兜底。iam 服务未启动时走降级degraded=true不阻塞业务。
### 3.2 core-edu 服务ai07 负责)— P0
| # | 工作项 | 用途 | 优先级 |
| --- | -------------------------------------------------------------- | -------------------------------------------------------------------- | ------ |
| 1 | MySQL 表 `core_edu_grades` CDC 对齐 | Debezium 监听 → Kafka topic `edu-cdc.next_edu_cloud.core_edu_grades` | P0 |
| 2 | MySQL 表 `core_edu_exams` CDC 对齐 | ExamCache 数据源 | P0 |
| 3 | MySQL 表 `core_edu_homework_submissions` CDC | student_dashboard_view 数据源 | P0 |
| 4 | MySQL 表 `core_edu_attendance` CDC | attendance_logs 数据源 | P0 |
| 5 | 表字段命名对齐student_id/exam_id/score/subject_id/class_id | CDC 消费者按字段名解析 | P0 |
**当前状态**data-ana CDC 消费者已实现 4 表路由grades/exams/homework/attendance等待 core-edu MySQL 数据 + Debezium connector 配置。
### 3.3 content 服务ai08 负责)— P1
| # | 工作项 | 用途 | 优先级 |
| --- | --------------------------------------- | ------------------------------------------- | ------ |
| 1 | MySQL 表 `content_knowledge_points` CDC | 知识点元数据缓存title/subject_id | P1 |
| 2 | 知识点标题字段对齐 | data-ana 查询结果补充 knowledge_point_title | P1 |
**当前状态**data-ana CDC 消费者已实现 knowledge_points 路由,写入 Redis `data_ana:kp_meta:{kp_id}`
### 3.4 ai 服务ai12 负责)— P1
| # | 工作项 | 用途 | 优先级 |
| --- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------ |
| 1 | 发布 `AIUsageEvent` 到 Kafka topic `edu.insight.ai.usage` | data-ana 消费写入 `ai_usage_log` 表,供 admin dashboard 统计 | P1 |
| 2 | AIUsageEvent 字段对齐 events.proto | request_id/user_id/provider/model/prompt_tokens/completion_tokens/total_tokens/latency_ms/success/cost_cents/occurred_at | P1 |
| 3 | 诊断报告生成能力 | `ListDiagnosticReports` 当前为占位,需 ai 提供报告数据 | P2 |
**当前状态**data-ana CDC 消费者已实现 AIUsageEvent 路由(`_handle_ai_usage_event`),等待 ai 服务发布事件。
### 3.5 ListDiagnosticReports 完整实现(需 ai 服务协作)— P2
当前 `ListDiagnosticReports` 返回空列表 + degraded 标记。完整实现需要:
1. **ai 服务**生成诊断报告并存储DB 或 Kafka 事件)
2. **data-ana** 消费 ai 诊断报告事件,写入 ClickHouse 新表 `diagnostic_reports`
3. **data-ana** 查询 `diagnostic_reports` 表返回报告列表
**建议方案**
- ai 服务发布 `DiagnosticReportGenerated` 事件到 Kafka topic `edu.insight.diagnostic.generated`
- data-ana CDC 消费者扩展 `_handle_diagnostic_event` 路由
- ClickHouse 新增 `diagnostic_reports`ReplacingMergeTreeTTL 1 年)
### 3.6 PracticeService 新建(需协调 AI 决策)— P2
student-bff / parent-portal 需要 `PracticeService` 3 RPC练习会话管理当前不在 data-ana 范围内。
**建议方案 A**(推荐):在 core-edu 服务新增 PracticeService练习属于教学域
**建议方案 B**:在 data-ana 新增 PracticeService练习数据天然属于分析域
**建议方案 C**:新建独立 `practice` 服务
> 需协调 AIai07/ai11/ai12与人类决策者确定归属。data-ana 当前不实现,待决策后补充。
### 3.7 childLearningPath 完整实现(需协调)— P2
parent-portal `childLearningPath` 字段需要学习路径推荐数据。当前 `GetStudentGrowth` 返回成长档案(成绩趋势 + 掌握度 + 考勤),但不含学习路径推荐。
**建议方案**ai 服务提供学习路径推荐 APIdata-ana 聚合或 ai 直接暴露给 BFF。
### 3.8 Debezium / Kafka 基础设施SRE AI 负责)— P0
| # | 工作项 | 用途 | 优先级 |
| --- | ------------------------------------------------------ | ----------------------------------------------- | ------ |
| 1 | Debezium connector 配置6 表) | MySQL binlog → Kafka CDC topics | P0 |
| 2 | Kafka topic 创建7 个 CDC + 1 个 AIUsage | edu-cdc.next_edu_cloud.* + edu.insight.ai.usage | P0 |
| 3 | Kafka topic `edu.insight.mastery.updated` | data-ana 发布掌握度更新事件 | P1 |
| 4 | Kafka topic `edu.insight.diagnostic.generated`(未来) | 诊断报告事件§3.5 | P2 |
**当前状态**edu-kafka + edu-debezium + edu-clickhouse 容器已运行,但 Debezium connector 未配置data-ana CDC 消费者启动但无消息消费)。
### 3.9 ClickHouse DDL 执行SRE AI 负责)— P0
| # | 工作项 | 优先级 |
| --- | ----------------------------------------------- | ------ |
| 1 | 执行 `scripts/clickhouse_ddl.sql`5 表 + TTL | P0 |
| 2 | 创建 database `edu_analytics` | P0 |
**当前状态**ClickHouse 容器运行DDL 已更新(含 TTL但需 SRE 确认执行。
---
## 4. Docker 本地测试结果v2
**测试环境**edu-data-ana-test 容器edu/data-ana:test 镜像),连接 edu-clickhouse + edu-redis + edu-kafka
**测试时间**2026-07-14
| # | 测试项 | 结果 | 备注 |
| --- | ------------------------------- | ---- | --------------------------------------------- |
| 1 | `GET /healthz` | ✅ | `{"status":"ok","service":"data-ana"}` |
| 2 | `GET /readyz`(首次) | ✅ | ready=truelifespan 预热生效) |
| 3 | `GET /readyz`dependencies | ✅ | clickhouse=ok, cdc=running(lag=0), redis=ok |
| 4 | `GET /metrics` | ✅ | 18 个自定义指标全部暴露 |
| 5 | gRPC HealthService | ✅ | SERVING |
| 6 | gRPC GetStudentDashboard | ✅ | |
| 7 | gRPC GetStudentWeakness | ✅ | |
| 8 | gRPC GetLearningTrend | ✅ | |
| 9 | gRPC GetClassPerformance | ✅ | |
| 10 | gRPC GetStudentGrowth (v2) | ✅ | 返回 growth_score + growth_level |
| 11 | gRPC GetAssignmentAnalysis (v2) | ✅ | 返回 score ranges |
| 12 | gRPC GetMasterySummary (v2) | ✅ | 返回 three-tier distribution |
| 13 | gRPC ListDiagnosticReports (v2) | ✅ | 占位返回空列表 + degraded |
| 14 | gRPC ListErrorBookItems (v2) | ✅ | |
| 15 | gRPC GetErrorBookStats (v2) | ✅ | |
| 16 | HTTP 11 业务端点 | ✅ | 全部返回 ActionState 信封 success=true |
| 17 | gRPC 拦截器 metrics | ✅ | `data_ana_grpc_requests_total` 按 method 记录 |
| 18 | CDC consumer metrics | ✅ | `cdc_consumer_active_instances=1` |
| 19 | ruff check | ✅ | All checks passed |
---
## 5. v2 架构变更摘要
### 5.1 proto 契约变更
- `packages/shared-proto/proto/analytics.proto`:新增 6 RPC + 16 message
- Python stub 重新生成:`src/generated_proto/analytics_pb2.py` + `analytics_pb2_grpc.py`
### 5.2 新增文件
- `services/data-ana/src/data_ana/metrics.py`18 个 Prometheus 指标定义
### 5.3 修改文件
- `services/data-ana/src/data_ana/cdc_consumer.py`:多实例 ID + 真实 lag + metrics 接入
- `services/data-ana/src/data_ana/exam_cache.py`Redis-backed + 内存 LRU fallback
- `services/data-ana/src/data_ana/grpc_server.py`6 新 RPC + gRPC 拦截器
- `services/data-ana/src/data_ana/analytics_service.py`6 新业务方法
- `services/data-ana/src/data_ana/repository/clickhouse_repository.py`3 新查询方法
- `services/data-ana/src/data_ana/main.py`readyz 硬化 + lifespan 预热 + metrics 初始化
- `services/data-ana/src/data_ana/config.py`readyz 超时配置
- `services/data-ana/scripts/clickhouse_ddl.sql`5 表 TTL
### 5.4 未变更
- `services/data-ana/pyproject.toml`:依赖版本不变(用户要求)
- `services/data-ana/Dockerfile`:构建配置不变
---
## 6. 下一步工作
### 6.1 等待上游就绪后联调
1. iam gRPC 服务启动 → data-ana 移除降级标记
2. core-edu MySQL 数据 + Debezium connector → data-ana CDC 消费真实数据
3. ai 服务发布 AIUsageEvent → data-ana admin dashboard 显示真实 AI 用量
### 6.2 待决策项
1. PracticeService 归属core-edu / data-ana / 新服务)— §3.6
2. ListDiagnosticReports 完整实现方案 — §3.5
3. childLearningPath 数据源 — §3.7
### 6.3 P7+ 规划(未来)
- ClickHouse 物化视图(预聚合 dashboard 查询)
- Kafka Streams 掌握度实时计算(替代批处理)
- Grafana dashboard 配置(`infra/grafana/dashboards/data-ana.json`
- 告警规则(`infra/prometheus/rules.yml` 补充 data-ana 规则)

View File

@@ -1,11 +1,12 @@
-- data-ana ClickHouse DDL5 宽表建表脚本
-- 对齐 02-architecture-design.md §3 DDL 设计
-- data-ana ClickHouse DDL5 宽表建表脚本P6: 加 TTL 归档策略)
-- 对齐 02-architecture-design.md §3 DDL 设计 + workline §3.5 任务 6.3
-- 引擎ReplacingMergeTree幂等消费保证+ MergeTree历史快照
-- TTLP6 容量规划(冷热数据分离,过期自动清理)
-- 使用方式clickhouse-client --multiquery < scripts/clickhouse_ddl.sql
CREATE DATABASE IF NOT EXISTS edu_analytics;
-- §3.1 学生学情宽表
-- §3.1 学生学情宽表TTL 2 年)
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
(
student_id String,
@@ -22,9 +23,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
ENGINE = ReplacingMergeTree(last_updated)
PARTITION BY toYYYYMM(last_updated)
ORDER BY (student_id, exam_id, knowledge_point_id)
TTL last_updated + INTERVAL 2 YEAR
SETTINGS index_granularity = 8192;
-- §3.2 学生错题本
-- §3.2 学生错题本TTL 2 年)
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
(
student_id String,
@@ -36,9 +38,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
)
ENGINE = ReplacingMergeTree(last_error_time)
PARTITION BY toYYYYMM(last_error_time)
ORDER BY (student_id, question_id);
ORDER BY (student_id, question_id)
TTL last_error_time + INTERVAL 2 YEAR;
-- §3.3 知识点掌握度历史快照
-- §3.3 知识点掌握度历史快照TTL 3 年,长期保留用于趋势分析)
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
(
student_id String,
@@ -50,9 +53,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(calculated_at)
ORDER BY (student_id, knowledge_point_id, calculated_at);
ORDER BY (student_id, knowledge_point_id, calculated_at)
TTL calculated_at + INTERVAL 3 YEAR;
-- §3.4 AI 用量计费记录
-- §3.4 AI 用量计费记录TTL 1 年,计费数据保留期较短)
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
(
request_id String,
@@ -69,9 +73,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(occurred_at)
ORDER BY (request_id);
ORDER BY (request_id)
TTL occurred_at + INTERVAL 1 YEAR;
-- §3.5 学生考勤记录
-- §3.5 学生考勤记录TTL 3 年,考勤数据需长期保留用于趋势分析)
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
(
student_id String,
@@ -84,4 +89,21 @@ CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(attendance_date)
ORDER BY (student_id, class_id, attendance_date);
ORDER BY (student_id, class_id, attendance_date)
TTL occurred_at + INTERVAL 3 YEAR;
-- ===== P6 容量规划说明 =====
-- 1. student_dashboard_view / student_errors2 年 TTL
-- - 高频写入CDC 实时同步2 年覆盖完整学习周期
-- - 超过 2 年的数据通过 PARTITION 级别删除(无需 VACUUM
-- 2. mastery_snapshot / attendance_logs3 年 TTL
-- - 低频写入但需长期保留用于趋势分析
-- - 3 年覆盖 K12 完整学段
-- 3. ai_usage_log1 年 TTL
-- - 计费数据保留期较短1 年足够用于年度成本分析
-- 4. 冷热数据分离:
-- - 热数据:最近 3 个月SSD 存储,频繁查询)
-- - 冷数据3 个月以上HDD 存储,低频查询)
-- - 通过 PARTITION BY toYYYYMM 实现按月分区,便于冷热分离
-- 5. TTL 触发时机ClickHouse 后台 merge 时自动清理过期数据
-- - 可通过 system.ttl_drops 表监控 TTL 执行情况

View File

@@ -414,6 +414,333 @@ async def get_student_weakness(
return result
# ===== P6+ v2 扩展 RPC 实现 =====
async def get_student_growth(
user: UserContext,
student_id: str,
subject_id: str = "",
start_date: int = 0,
end_date: int = 0,
) -> dict[str, Any]:
"""学生成长档案(综合成绩趋势 + 掌握度变化 + 考勤统计).
供 parent-bff GrowthArchiveService.Get / student-bff GetStudentGrowth.
"""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
# 1. 成绩趋势
trend = await clickhouse_repository.query_learning_trend(
student_id=effective_student_id,
subject_id=subject_id,
start_date=start_date,
end_date=end_date,
)
score_trend = trend.get("points", []) if trend else []
# 2. 掌握度变化趋势
mastery_trend = await clickhouse_repository.query_mastery_trend(
student_id=effective_student_id,
start_date=start_date,
end_date=end_date,
)
if mastery_trend is None:
mastery_trend = []
# 3. 考勤统计
attendance = await clickhouse_repository.query_attendance(
student_id=effective_student_id,
start_date=start_date,
end_date=end_date,
)
if attendance is None:
attendance_summary = {
"total_days": 0,
"present_days": 0,
"absent_days": 0,
"late_days": 0,
"attendance_rate": 0.0,
}
else:
total = attendance.get("total", 0)
attendance_summary = {
"total_days": total,
"present_days": attendance.get("presentCount", 0),
"absent_days": attendance.get("absentCount", 0),
"late_days": attendance.get("lateCount", 0),
"attendance_rate": (attendance.get("presentCount", 0) / total) if total else 0.0,
}
# 4. 计算成长评分(综合分数 + 掌握度 + 出勤率)
avg_score = sum(p.get("score", 0) for p in score_trend) / len(score_trend) if score_trend else 0
avg_mastery = (
sum(p.get("overall_mastery", 0) for p in mastery_trend) / len(mastery_trend)
if mastery_trend
else 0
)
attendance_rate = attendance_summary["attendance_rate"]
growth_score = (avg_score / 100 * 0.4) + (avg_mastery * 0.4) + (attendance_rate * 0.2)
if growth_score >= 0.85:
growth_level = "excellent"
elif growth_score >= 0.7:
growth_level = "good"
elif growth_score >= 0.5:
growth_level = "average"
else:
growth_level = "needs_improvement"
result = {
"studentId": effective_student_id,
"scoreTrend": score_trend,
"masteryTrend": mastery_trend,
"attendance": attendance_summary,
"growthScore": round(growth_score, 4),
"growthLevel": growth_level,
}
if trend is None and attendance is None:
result["degraded"] = True
result["degraded_reason"] = "clickhouse_unavailable"
elif degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_assignment_analysis(
user: UserContext,
class_id: str,
assignment_id: str,
subject_id: str = "",
) -> dict[str, Any]:
"""作业/考试分析(单次作业/考试维度统计)."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids:
return {
"assignmentId": assignment_id,
"classId": class_id,
"degraded": True,
"degraded_reason": "datascope_violation",
"averageScore": 0.0,
"highestScore": 0.0,
"lowestScore": 0.0,
"totalStudents": 0,
"submittedCount": 0,
"passRate": 0.0,
"ranges": [],
}
result = await clickhouse_repository.query_assignment_analysis(
class_id=class_id,
assignment_id=assignment_id,
subject_id=subject_id,
)
if result is None:
return {
"assignmentId": assignment_id,
"classId": class_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"averageScore": 0.0,
"highestScore": 0.0,
"lowestScore": 0.0,
"totalStudents": 0,
"submittedCount": 0,
"passRate": 0.0,
"ranges": [],
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_mastery_summary(
user: UserContext,
student_id: str,
subject_id: str = "",
) -> dict[str, Any]:
"""学生掌握度汇总(轻量级,仅返回总体掌握度 + 三档分布)."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
mastery = await clickhouse_repository.query_mastery_snapshot(
student_id=effective_student_id,
subject_id=subject_id,
)
if mastery is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"overallMastery": 0.0,
"masteredCount": 0,
"progressingCount": 0,
"weakCount": 0,
"totalKnowledgePoints": 0,
"masteryLevel": "weak",
}
kps = mastery.get("knowledgePoints", [])
mastered = sum(1 for kp in kps if kp.get("mastery_label") == "mastered")
progressing = sum(1 for kp in kps if kp.get("mastery_label") == "progressing")
weak = sum(1 for kp in kps if kp.get("mastery_label") == "weak")
overall = mastery.get("overallMastery", 0.0)
if overall >= 0.8:
mastery_level = "mastered"
elif overall >= 0.4:
mastery_level = "progressing"
else:
mastery_level = "weak"
result = {
"studentId": effective_student_id,
"overallMastery": overall,
"masteredCount": mastered,
"progressingCount": progressing,
"weakCount": weak,
"totalKnowledgePoints": len(kps),
"masteryLevel": mastery_level,
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def list_diagnostic_reports(
user: UserContext,
student_id: str,
since: int = 0,
limit: int = 10,
) -> dict[str, Any]:
"""诊断报告列表(占位实现,真实数据需 ai 服务集成).
当前返回空列表 + degraded 标记,待 ai 服务提供诊断报告生成能力后补全.
"""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
# 占位:真实数据需 ai 服务集成ai 服务生成诊断报告后通过 Kafka 推送)
result = {
"studentId": effective_student_id,
"reports": [],
"total": 0,
"degraded": True,
"degraded_reason": "diagnostic_reports_pending_ai_integration",
}
if degraded:
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def list_error_book_items(
user: UserContext,
student_id: str,
subject_id: str = "",
limit: int = 100,
offset: int = 0,
) -> dict[str, Any]:
"""错题本列表gRPC 版本,供 parent-bff ErrorBookService.List."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
errors = await clickhouse_repository.query_student_errors(effective_student_id)
if errors is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"items": [],
"total": 0,
}
# 转换为 ErrorBookItem 格式
items = [
{
"question_id": e.get("question_id", ""),
"knowledge_point_id": e.get("knowledge_point_id", ""),
"knowledge_point_title": e.get("knowledge_point_id", ""), # content CDC 同步后补充
"error_count": e.get("error_count", 0),
"last_error_time": int(e.get("last_error_time").timestamp())
if e.get("last_error_time")
else 0,
"content": e.get("content", ""),
}
for e in errors
]
result = {
"studentId": effective_student_id,
"items": items,
"total": len(items),
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_error_book_stats(
user: UserContext,
student_id: str,
subject_id: str = "",
) -> dict[str, Any]:
"""错题本统计(按知识点聚合)."""
scope = await iam_client.get_effective_datascope(user)
degraded = scope.degraded
effective_student_id = student_id
if scope.level == DataScopeLevel.SELF:
effective_student_id = user.user_id
result = await clickhouse_repository.query_error_book_stats(
student_id=effective_student_id,
subject_id=subject_id,
)
if result is None:
return {
"studentId": effective_student_id,
"degraded": True,
"degraded_reason": "clickhouse_unavailable",
"totalErrorQuestions": 0,
"totalErrorCount": 0,
"byKnowledgePoint": [],
"recent7dErrors": 0,
}
if degraded:
result["degraded"] = True
result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable"
return result
async def get_learning_trend(
user: UserContext,
student_id: str,

View File

@@ -20,6 +20,12 @@
- op 类型r(快照读)、c(新增)、u(更新)、d(删除)d 时 after 为 null
- Redis 事件去重:基于 event_idDebezium 重启时可能重发)
P6 多实例水平扩展:
- consumer group 不变data-ana-cdcKafka 自动 partition rebalance
- 多实例消费同一 topic 无重复无遗漏partition 级分配)
- get_lag() 通过 AIOKafkaConsumer.position + end_offsets 计算真实 lag
- 实例 ID 通过 POD_NAME 环境变量区分(用于日志追踪)
降级策略:
- kafka_brokers 未配置:消费者不启动(仅 HTTP/gRPC 服务)
- ClickHouse 不可达:消息处理失败,不 commit下次重启重试
@@ -28,11 +34,14 @@
import asyncio
import contextlib
import json
import os
import socket
from datetime import UTC, datetime
from typing import Any
import structlog
from . import metrics
from .config import settings
from .exam_cache import get_exam_cache
from .repository import clickhouse_repository, redis_client
@@ -44,6 +53,14 @@ _consumer: Any | None = None
_consumer_task: asyncio.Task[None] | None = None
_is_running: bool = False
# 实例 ID多实例水平扩展时用于日志区分
_INSTANCE_ID = os.getenv("POD_NAME", f"{socket.gethostname()}:{os.getpid()}")
def get_instance_id() -> str:
"""获取消费者实例 ID多实例部署时用于日志追踪."""
return _INSTANCE_ID
def _parse_ts(ts_ms: int | None) -> datetime:
"""Debezium ts_ms毫秒→ datetime."""
@@ -99,7 +116,7 @@ async def _handle_exams_event(after: dict[str, Any] | None, op: str) -> bool:
return True
exam_cache = get_exam_cache()
exam_cache.upsert(
await exam_cache.upsert(
exam_id=exam_id,
class_id=str(after.get("class_id") or ""),
subject_id=str(after.get("subject_id") or ""),
@@ -127,10 +144,10 @@ async def _handle_grades_event(
exam_id = str(after.get("exam_id") or "")
score = _safe_float(after.get("score"))
# 从 ExamCache 获取 class_id 和 subject_id
# 从 ExamCache 获取 class_id 和 subject_idP6: Redis-backed async
exam_cache = get_exam_cache()
class_id = exam_cache.get_class_id(exam_id)
subject_id = exam_cache.get_subject_id(exam_id)
class_id = await exam_cache.get_class_id(exam_id)
subject_id = await exam_cache.get_subject_id(exam_id)
fallback_ts = _parse_ts(ts_ms)
last_updated = _parse_mysql_datetime(after.get("updated_at"), fallback_ts)
@@ -473,22 +490,48 @@ async def run_consumer() -> None:
try:
await _consumer.start()
_is_running = True
metrics.cdc_consumer_active_instances.set(1)
logger.info(
"cdc_consumer_started",
instance_id=_INSTANCE_ID,
brokers=brokers,
topics=topics,
group_id=settings.kafka_consumer_group,
auto_commit=settings.kafka_enable_auto_commit,
)
except Exception as exc: # noqa: BLE001
logger.error("cdc_consumer_start_failed", error=str(exc))
logger.error(
"cdc_consumer_start_failed",
instance_id=_INSTANCE_ID,
error=str(exc),
)
_is_running = False
return
try:
async for msg in _consumer:
try:
import time as _time
t0 = _time.monotonic()
success = await _process_message(msg.topic, msg.value)
duration = _time.monotonic() - t0
table = ""
try:
import json as _json
_ev = _json.loads(
msg.value.decode("utf-8") if isinstance(msg.value, bytes) else msg.value
)
table = (_ev.get("source") or {}).get("table", "")
except Exception: # noqa: BLE001
pass
metrics.cdc_message_process_duration_seconds.labels(table=table).observe(duration)
metrics.cdc_messages_processed_total.labels(
topic=msg.topic,
table=table,
status="success" if success else "failed",
).inc()
if success:
# 处理成功才 commit offsetat-least-once
await _consumer.commit()
@@ -514,6 +557,7 @@ async def run_consumer() -> None:
raise
finally:
_is_running = False
metrics.cdc_consumer_active_instances.set(0)
if _consumer is not None:
try:
await _consumer.stop()
@@ -555,7 +599,25 @@ def is_running() -> bool:
async def get_lag() -> int:
"""获取消费者 lag待消费消息数供 /readyz 和监控使用).
P6 实现:调用 Kafka AdminClient 获取 group lag.
当前返回 0简化.
P6 实现:通过 AIOKafkaConsumer.position + end_offsets 计算真实 lag.
多实例场景:仅计算本实例分配到的 partition lag.
"""
return 0
if _consumer is None or not _is_running:
return 0
try:
# 获取本实例分配到的 partition
assignment = _consumer.assignment()
if not assignment:
return 0
# 并发获取 end_offsets 和 position
end_offsets = await _consumer.end_offsets(assignment)
total_lag = 0
for tp in assignment:
position = await _consumer.position(tp)
end = end_offsets.get(tp, 0)
if end > position:
total_lag += end - position
return total_lag
except Exception as exc: # noqa: BLE001
logger.warning("cdc_lag_query_failed", error=str(exc))
return 0

View File

@@ -78,6 +78,12 @@ class Settings(BaseSettings):
# 降级
degraded_mode_enabled: bool = True
# P6 readyz 深度硬化
readyz_clickhouse_timeout_s: float = 1.0
readyz_redis_timeout_s: float = 0.2
readyz_iam_grpc_timeout_s: float = 2.0
readyz_cdc_lag_threshold: int = 1000 # lag 超过此值判定 not_ready
# 向后兼容:旧代码引用 settings.port / settings.kafka_group_id
@property
def port(self) -> int:

View File

@@ -1,105 +1,156 @@
"""考试缓存exam_id → {class_id, subject_id} 映射,内存 LRU.
"""考试缓存exam_id → {class_id, subject_id} 映射,Redis-backed + 内存 LRU fallback.
对齐 02-architecture-design.md §8.3 ExamCache
- 内存 LRU dictmax 10000 条
- CDC core_edu_exams 事件触发更新
- CDC core_edu_grades 事件查询获取 class_id避免 join 查询
对齐 02-architecture-design.md §8.3 ExamCache + workline §3.5 任务 6.2
- P6 演进Redis 实现key: data_ana:exam:{exam_id}TTL 30 天)
- 多实例共享:多个 data-ana 实例共享同一 Redis ExamCache
- 内存 LRU fallbackRedis 不可达时降级为内存缓存(单实例模式
P6 演进:改为 Redis 实现key: data_ana:exam:{exam_id}TTL 30 天),
支持多实例共享(对齐 workline §3.5 任务 6.2.
设计:
- upsertRedis 写入async+ 内存 LRU 同步写入(保证后续读取命中)
- getRedis 读取asyncmiss 时 fallback 到内存 LRU
- Redis 不可达:仅使用内存 LRU降级模式degraded=true
"""
import json
from collections import OrderedDict
from typing import Any
import structlog
from .repository import redis_client
logger = structlog.get_logger(__name__)
# LRU 最大容量
# LRU 最大容量(内存 fallback 用)
_MAX_SIZE = 10_000
_REDIS_TTL_S = 30 * 24 * 3600 # 30 天
def _redis_key(exam_id: str) -> str:
"""Redis 缓存键."""
return f"data_ana:exam:{exam_id}"
class ExamCache:
"""考试缓存(LRUmax 10000 条.
"""考试缓存(Redis-backed + 内存 LRU fallback.
内存实现OrderedDict访问/写入时移到末尾(最近使用),
超容量时弹出头部(最久未使用).
线程安全asyncio 单线程模型下无需加锁.
Redis 可用时多实例共享TTL 30 天
Redis 不可用:降级为单实例内存 LRUmax 10000 条)
"""
def __init__(self, max_size: int = _MAX_SIZE) -> None:
self._data: OrderedDict[str, dict[str, str]] = OrderedDict()
self._memory: OrderedDict[str, dict[str, str]] = OrderedDict()
self._max_size = max_size
def upsert(
def _memory_upsert(
self,
exam_id: str,
class_id: str = "",
subject_id: str = "",
title: str = "",
) -> None:
"""更新或插入考试缓存."""
"""内存 LRU 写入同步Redis 不可达时降级用)."""
if not exam_id:
return
value: dict[str, str] = {
"class_id": class_id,
"subject_id": subject_id,
"title": title,
}
if exam_id in self._memory:
self._memory.move_to_end(exam_id)
self._memory[exam_id] = value
while len(self._memory) > self._max_size:
evicted_key, _ = self._memory.popitem(last=False)
logger.debug("exam_cache_memory_lru_evicted", exam_id=evicted_key)
# 已存在则移到末尾(标记为最近使用)
if exam_id in self._data:
self._data.move_to_end(exam_id)
self._data[exam_id] = value
# LRU 淘汰
while len(self._data) > self._max_size:
evicted_key, _ = self._data.popitem(last=False)
logger.debug("exam_cache_lru_evicted", exam_id=evicted_key)
def get(self, exam_id: str) -> dict[str, str] | None:
"""查询考试缓存(命中时移到末尾,标记为最近使用)."""
def _memory_get(self, exam_id: str) -> dict[str, str] | None:
"""内存 LRU 读取(同步)."""
if not exam_id:
return None
value = self._data.get(exam_id)
value = self._memory.get(exam_id)
if value is not None:
self._data.move_to_end(exam_id)
self._memory.move_to_end(exam_id)
return value
def get_class_id(self, exam_id: str) -> str:
async def upsert(
self,
exam_id: str,
class_id: str = "",
subject_id: str = "",
title: str = "",
) -> None:
"""更新或插入考试缓存Redis + 内存 LRU."""
if not exam_id:
return
# 1. 内存 LRU 同步写入(保证后续读取命中)
self._memory_upsert(exam_id, class_id, subject_id, title)
# 2. Redis 异步写入(多实例共享)
value = json.dumps(
{"class_id": class_id, "subject_id": subject_id, "title": title},
ensure_ascii=False,
)
ok = await redis_client.set_cache(_redis_key(exam_id), value, ttl_s=_REDIS_TTL_S)
if not ok:
logger.debug("exam_cache_redis_write_failed_memory_only", exam_id=exam_id)
async def get(self, exam_id: str) -> dict[str, str] | None:
"""查询考试缓存Redis 优先miss 时 fallback 内存 LRU."""
if not exam_id:
return None
# 1. Redis 读取
raw = await redis_client.get_cache(_redis_key(exam_id))
if raw is not None:
try:
data = json.loads(raw)
# 回填内存 LRU加速后续读取
self._memory_upsert(
exam_id,
data.get("class_id", ""),
data.get("subject_id", ""),
data.get("title", ""),
)
return data
except (json.JSONDecodeError, TypeError) as exc:
logger.warning("exam_cache_redis_decode_failed", exam_id=exam_id, error=str(exc))
# 2. 内存 LRU fallback
return self._memory_get(exam_id)
async def get_class_id(self, exam_id: str) -> str:
"""便捷方法:获取 class_id未命中返回空字符串."""
entry = self.get(exam_id)
entry = await self.get(exam_id)
return entry.get("class_id", "") if entry else ""
def get_subject_id(self, exam_id: str) -> str:
async def get_subject_id(self, exam_id: str) -> str:
"""便捷方法:获取 subject_id未命中返回空字符串."""
entry = self.get(exam_id)
entry = await self.get(exam_id)
return entry.get("subject_id", "") if entry else ""
def delete(self, exam_id: str) -> bool:
"""删除缓存项."""
if exam_id in self._data:
del self._data[exam_id]
return True
return False
async def delete(self, exam_id: str) -> bool:
"""删除缓存项Redis + 内存)."""
if exam_id in self._memory:
del self._memory[exam_id]
return await redis_client.delete_cache(_redis_key(exam_id))
def clear(self) -> None:
"""清空缓存."""
self._data.clear()
def clear_memory(self) -> None:
"""清空内存缓存Redis 数据保留)."""
self._memory.clear()
def size(self) -> int:
"""当前缓存数量."""
return len(self._data)
def memory_size(self) -> int:
"""当前内存缓存数量."""
return len(self._memory)
def stats(self) -> dict[str, Any]:
"""缓存统计信息(供 /readyz 和监控使用)."""
return {
"size": len(self._data),
"max_size": self._max_size,
"utilization": round(len(self._data) / self._max_size, 4),
"memory_size": len(self._memory),
"memory_max_size": self._max_size,
"memory_utilization": round(len(self._memory) / self._max_size, 4),
"redis_ttl_s": _REDIS_TTL_S,
}

View File

@@ -18,16 +18,59 @@ SubscribeMasteryUpdateP5+
"""
import asyncio
import time
from typing import Any
import structlog
from . import analytics_service, warning_service
from . import analytics_service, metrics, warning_service
from .config import settings
from .shared.permissions import UserContext
logger = structlog.get_logger(__name__)
def _make_metrics_interceptor():
"""构造 gRPC ServerInterceptor 实例(记录每个 unary RPC 的请求数和耗时)."""
import grpc # type: ignore[import-not-found]
class _ServerInterceptor(grpc.aio.ServerInterceptor):
async def intercept_service(self, continuation, handler_call_details):
method = handler_call_details.method.rsplit("/", 1)[-1]
handler = await continuation(handler_call_details)
if handler is None:
return None
# 仅包装 unary_unary不包装 stream RPC
if (
handler.unary_unary is None
or handler.request_streaming
or handler.response_streaming
):
return handler
original_behavior = handler.unary_unary
async def _wrapped(request, context):
t0 = time.monotonic()
status = "success"
try:
return await original_behavior(request, context)
except Exception: # noqa: BLE001
status = "failed"
raise
finally:
duration = time.monotonic() - t0
metrics.record_grpc_request(method=method, duration_s=duration, status=status)
return grpc.unary_unary_rpc_method_handler(
_wrapped,
request_deserializer=handler.request_deserializer,
response_serializer=handler.response_serializer,
)
return _ServerInterceptor()
# SubscribeMasteryUpdate 订阅管理P5+
_subscribers: dict[str, asyncio.Queue] = {}
_subscribers_lock = asyncio.Lock()
@@ -239,6 +282,74 @@ class AnalyticsServiceServicer:
await _remove_subscriber(sub_key)
logger.info("mastery_subscription_removed", sub_key=sub_key)
# ===== P6+ v2 扩展 RPC =====
async def GetStudentGrowth(self, request, context):
"""学生成长档案P6+ v2 扩展)."""
user = _extract_user(context)
result = await analytics_service.get_student_growth(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
start_date=request.start_date,
end_date=request.end_date,
)
return _build_student_growth_response(result)
async def GetAssignmentAnalysis(self, request, context):
"""作业/考试分析P6+ v2 扩展)."""
user = _extract_user(context)
result = await analytics_service.get_assignment_analysis(
user=user,
class_id=request.class_id,
assignment_id=request.assignment_id,
subject_id=request.subject_id,
)
return _build_assignment_analysis_response(result)
async def GetMasterySummary(self, request, context):
"""学生掌握度汇总P6+ v2 扩展)."""
user = _extract_user(context)
result = await analytics_service.get_mastery_summary(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
)
return _build_mastery_summary_response(result)
async def ListDiagnosticReports(self, request, context):
"""诊断报告列表P6+ v2 扩展,占位实现)."""
user = _extract_user(context)
result = await analytics_service.list_diagnostic_reports(
user=user,
student_id=request.student_id,
since=request.since,
limit=request.limit,
)
return _build_diagnostic_report_list_response(result)
async def ListErrorBookItems(self, request, context):
"""错题本列表P6+ v2 扩展)."""
user = _extract_user(context)
result = await analytics_service.list_error_book_items(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
limit=request.limit,
offset=request.offset,
)
return _build_error_book_list_response(result)
async def GetErrorBookStats(self, request, context):
"""错题本统计P6+ v2 扩展)."""
user = _extract_user(context)
result = await analytics_service.get_error_book_stats(
user=user,
student_id=request.student_id,
subject_id=request.subject_id,
)
return _build_error_book_stats_response(result)
# ===== 辅助函数 =====
@@ -492,6 +603,151 @@ def _build_mastery_update_event(event: dict) -> Any:
)
# ===== P6+ v2 扩展 RPC 响应构建 =====
def _build_student_growth_response(data: dict) -> Any:
"""构建 StudentGrowth proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
score_trend = [
analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0))
for p in data.get("scoreTrend", [])
]
mastery_trend = [
analytics_pb2.MasteryTrendPoint(
calculated_at=p.get("calculated_at", 0),
overall_mastery=p.get("overall_mastery", 0.0),
)
for p in data.get("masteryTrend", [])
]
attendance_data = data.get("attendance", {})
attendance = analytics_pb2.AttendanceSummary(
total_days=attendance_data.get("total_days", 0),
present_days=attendance_data.get("present_days", 0),
absent_days=attendance_data.get("absent_days", 0),
late_days=attendance_data.get("late_days", 0),
attendance_rate=attendance_data.get("attendance_rate", 0.0),
)
return analytics_pb2.StudentGrowth(
student_id=data.get("studentId", ""),
score_trend=score_trend,
mastery_trend=mastery_trend,
attendance=attendance,
growth_score=data.get("growthScore", 0.0),
growth_level=data.get("growthLevel", "needs_improvement"),
)
def _build_assignment_analysis_response(data: dict) -> Any:
"""构建 AssignmentAnalysis proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
ranges = [
analytics_pb2.ScoreRange(
label=r.get("label", ""),
count=r.get("count", 0),
percentage=r.get("percentage", 0.0),
)
for r in data.get("ranges", [])
]
return analytics_pb2.AssignmentAnalysis(
assignment_id=data.get("assignmentId", ""),
class_id=data.get("classId", ""),
subject_id=data.get("subjectId", ""),
average_score=data.get("averageScore", 0.0),
highest_score=data.get("highestScore", 0.0),
lowest_score=data.get("lowestScore", 0.0),
total_students=data.get("totalStudents", 0),
submitted_count=data.get("submittedCount", 0),
pass_rate=data.get("passRate", 0.0),
ranges=ranges,
)
def _build_mastery_summary_response(data: dict) -> Any:
"""构建 MasterySummary proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
return analytics_pb2.MasterySummary(
student_id=data.get("studentId", ""),
overall_mastery=data.get("overallMastery", 0.0),
mastered_count=data.get("masteredCount", 0),
progressing_count=data.get("progressingCount", 0),
weak_count=data.get("weakCount", 0),
total_knowledge_points=data.get("totalKnowledgePoints", 0),
mastery_level=data.get("masteryLevel", "weak"),
)
def _build_diagnostic_report_list_response(data: dict) -> Any:
"""构建 DiagnosticReportList proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
reports = [
analytics_pb2.DiagnosticReport(
report_id=r.get("report_id", ""),
student_id=r.get("student_id", ""),
report_type=r.get("report_type", ""),
title=r.get("title", ""),
summary=r.get("summary", ""),
generated_at=r.get("generated_at", 0),
status=r.get("status", "pending"),
)
for r in data.get("reports", [])
]
return analytics_pb2.DiagnosticReportList(
student_id=data.get("studentId", ""),
reports=reports,
total=data.get("total", 0),
)
def _build_error_book_list_response(data: dict) -> Any:
"""构建 ErrorBookList proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
items = [
analytics_pb2.ErrorBookItem(
question_id=item.get("question_id", ""),
knowledge_point_id=item.get("knowledge_point_id", ""),
knowledge_point_title=item.get("knowledge_point_title", ""),
error_count=item.get("error_count", 0),
last_error_time=item.get("last_error_time", 0),
content=item.get("content", ""),
)
for item in data.get("items", [])
]
return analytics_pb2.ErrorBookList(
student_id=data.get("studentId", ""),
items=items,
total=data.get("total", 0),
)
def _build_error_book_stats_response(data: dict) -> Any:
"""构建 ErrorBookStats proto 响应."""
from generated_proto import analytics_pb2 # type: ignore[import-not-found]
by_kp = [
analytics_pb2.KnowledgePointErrorStats(
knowledge_point_id=kp.get("knowledge_point_id", ""),
title=kp.get("title", ""),
error_count=kp.get("error_count", 0),
question_count=kp.get("question_count", 0),
error_rate=kp.get("error_rate", 0.0),
)
for kp in data.get("byKnowledgePoint", [])
]
return analytics_pb2.ErrorBookStats(
student_id=data.get("studentId", ""),
total_error_questions=data.get("totalErrorQuestions", 0),
total_error_count=data.get("totalErrorCount", 0),
by_knowledge_point=by_kp,
recent_7d_errors=data.get("recent7dErrors", 0),
)
# ===== gRPC Server 管理 =====
_server: Any | None = None
@@ -514,7 +770,7 @@ async def start_grpc_server() -> Any | None:
logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc))
return None
_server = grpc.aio.server()
_server = grpc.aio.server(interceptors=[_make_metrics_interceptor()])
# 注册 AnalyticsService
servicer = AnalyticsServiceServicer()
@@ -544,7 +800,7 @@ async def start_grpc_server() -> Any | None:
logger.info(
"grpc_server_started",
port=settings.grpc_port,
rpc_count=12,
rpc_count=18,
)
return _server
except Exception as exc: # noqa: BLE001

View File

@@ -34,7 +34,7 @@ from datetime import UTC, datetime
from typing import Any
import structlog
from fastapi import APIRouter, Depends, FastAPI, Query
from fastapi import APIRouter, Depends, FastAPI, Query, Response
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
@@ -134,23 +134,38 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
if grpc_server_obj is None:
logger.warning("grpc_server_not_started_http_only")
# 2. 启动 CDC 消费者后台任务
# 2. 预热客户端(避免首次 readyz 探针因惰性初始化超时)
if settings.clickhouse_host:
ch_ok = await clickhouse_repository.ping()
logger.info("clickhouse_warmup", ok=ch_ok)
if settings.redis_url:
redis_ok = await redis_client.ping()
logger.info("redis_warmup", ok=redis_ok)
# 2.1 初始化监控指标初始值(确保 /metrics 端点暴露自定义指标)
from . import metrics as _metrics
_metrics.clickhouse_connection_status.set(1 if settings.clickhouse_host else 0)
_metrics.redis_connection_status.set(1 if settings.redis_url else 0)
_metrics.iam_grpc_connection_status.set(1 if settings.iam_grpc_endpoint else 0)
# 3. 启动 CDC 消费者后台任务
await cdc_consumer.start_consumer()
yield
logger.info("data_ana_service_stopping")
# 3. 停止 CDC 消费者
# 4. 停止 CDC 消费者
await cdc_consumer.stop_consumer()
# 4. 停止 gRPC server
# 5. 停止 gRPC server
await grpc_server.stop_grpc_server()
# 5. 关闭 Kafka producer
# 6. 关闭 Kafka producer
await kafka_producer.close_producer()
# 6. 关闭 Redis / iam gRPC / ClickHouse 客户端
# 7. 关闭 Redis / iam gRPC / ClickHouse 客户端
await redis_client.close_client()
await iam_client.close_grpc()
await clickhouse_repository.close_client()
@@ -196,36 +211,78 @@ async def healthz() -> dict[str, str]:
@app.get("/readyz")
async def readyz() -> dict[str, Any]:
"""就绪检查(readiness检查 4 依赖.
async def readyz(response: Response) -> dict[str, Any]:
"""就绪检查(P6 深度硬化版,检查 4 依赖 + 超时控制 + lag 阈值.
依赖检查:
1. clickhouse已配置且可达未配置算降级就绪
2. cdc_consumerrunning / disabled
3. redis已配置且可达未配置算降级就绪
4. iam_grpc已配置且可达未配置算降级就绪
依赖检查(带超时)
1. clickhouse1s 超时,已配置且可达(未配置算降级就绪)
2. cdc_consumerrunning / disabled + lag < 1000
3. redis200ms 超时,已配置且可达(未配置算降级就绪)
4. iam_grpc2s 超时,已配置且可达(未配置算降级就绪)
返回 ready=true 的条件:
- ClickHouse 已配置且可达,或未配置(降级就绪)
- 不要求所有依赖都健康(降级模式下仍可服务骨架数据
- ClickHouse 已配置且可达1s 内),或未配置(降级就绪)
- CDC consumer lag < readyz_cdc_lag_threshold1000
不满足时返回 HTTP 503K8s 摘流量.
"""
# 1. ClickHouse
ch_ok = await clickhouse_repository.ping()
import asyncio
# 1. ClickHouse1s 超时)
try:
ch_ok = await asyncio.wait_for(
clickhouse_repository.ping(),
timeout=settings.readyz_clickhouse_timeout_s,
)
except TimeoutError:
ch_ok = False
ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured")
# 2. CDC 消费者
cdc_status = (
"running"
if cdc_consumer.is_running()
else ("disabled" if not settings.kafka_brokers else "failed")
)
# 2. CDC 消费者 + lag 检查
cdc_running = cdc_consumer.is_running()
cdc_lag = 0
if cdc_running:
try:
cdc_lag = await asyncio.wait_for(
cdc_consumer.get_lag(),
timeout=2.0,
)
except TimeoutError:
cdc_lag = -1 # 查询超时标记
cdc_lag_ok = cdc_lag >= 0 and cdc_lag < settings.readyz_cdc_lag_threshold
if not settings.kafka_brokers:
cdc_status = "disabled"
cdc_lag_ok = True # 未配置 Kafka 时不阻塞 ready
elif cdc_running and cdc_lag_ok:
cdc_status = f"running(lag={cdc_lag})"
elif cdc_running:
cdc_status = f"running(lag={cdc_lag},exceeded)"
else:
cdc_status = "failed"
# 3. Redis
redis_ok = await redis_client.ping() if settings.redis_url else None
# 3. Redis200ms 超时)
if settings.redis_url:
try:
redis_ok = await asyncio.wait_for(
redis_client.ping(),
timeout=settings.readyz_redis_timeout_s,
)
except TimeoutError:
redis_ok = False
else:
redis_ok = None
redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured")
# 4. iam gRPC
iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None
# 4. iam gRPC2s 超时iam_client.ping 已内置 1s 超时)
if settings.iam_grpc_endpoint:
try:
iam_ok = await asyncio.wait_for(
iam_client.ping(),
timeout=settings.readyz_iam_grpc_timeout_s,
)
except TimeoutError:
iam_ok = False
else:
iam_ok = None
iam_status = (
"ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured")
)
@@ -233,10 +290,15 @@ async def readyz() -> dict[str, Any]:
# 5. gRPC server
grpc_status = "running" if grpc_server.is_running() else "stopped"
# 就绪判定ClickHouse 可达或未配置(降级就绪)
ready = ch_ok or not settings.clickhouse_host
# 就绪判定P6 硬化):
# - ClickHouse 可达或未配置(降级就绪)
# - CDC consumer lag 未超阈值
ready = (ch_ok or not settings.clickhouse_host) and cdc_lag_ok
degraded = not ch_ok or not redis_ok or not iam_ok
if not ready:
response.status_code = 503 # K8s 摘流量
return {
"status": "ok" if ready else "not_ready",
"service": "data-ana",
@@ -250,6 +312,13 @@ async def readyz() -> dict[str, Any]:
"grpc_server": grpc_status,
"kafka_producer": "ok" if settings.kafka_brokers else "not_configured",
},
"thresholds": {
"clickhouse_timeout_s": settings.readyz_clickhouse_timeout_s,
"redis_timeout_s": settings.readyz_redis_timeout_s,
"iam_grpc_timeout_s": settings.readyz_iam_grpc_timeout_s,
"cdc_lag_threshold": settings.readyz_cdc_lag_threshold,
"cdc_lag_current": cdc_lag,
},
"timestamp": datetime.now(UTC).isoformat(),
}

View File

@@ -0,0 +1,205 @@
"""Prometheus 指标定义P6 监控告警完善).
对齐 workline §3.5 任务 6.4
- consumer lag histogramCDC 消费延迟)
- 慢查询 counterClickHouse 查询耗时)
- ClickHouse 连接池 gauge
- ExamCache gauge缓存命中率
- DataScope 缓存 counter
- gRPC 请求 counter + histogram
- 掌握度计算 counter
- 预警触发 counter
指标命名规范(对齐项目规则 §12
data_ana_<module>_<operation>_<unit>
Grafana dashboard 配置见 infra/grafana/dashboards/data-ana.json
"""
from prometheus_client import Counter, Gauge, Histogram
# ===== CDC Consumer 指标 =====
# 消费者 lag按 topic + partition 分维度)
cdc_consumer_lag = Gauge(
"data_ana_cdc_consumer_lag",
"CDC consumer lag (messages waiting to be consumed)",
["topic", "partition", "instance"],
)
# 消费者处理消息总数
cdc_messages_processed_total = Counter(
"data_ana_cdc_messages_processed_total",
"Total CDC messages processed",
["topic", "table", "status"], # status: success / failed / skipped
)
# 消费者处理耗时
cdc_message_process_duration_seconds = Histogram(
"data_ana_cdc_message_process_duration_seconds",
"CDC message processing duration",
["table"],
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0),
)
# 消费者实例数(多实例水平扩展监控)
cdc_consumer_active_instances = Gauge(
"data_ana_cdc_consumer_active_instances",
"Number of active CDC consumer instances (this instance reports 1 when running)",
)
# ===== ClickHouse 查询指标 =====
# 查询耗时直方图
clickhouse_query_duration_seconds = Histogram(
"data_ana_clickhouse_query_duration_seconds",
"ClickHouse query duration",
["operation"], # operation: query_class_performance / query_student_dashboard / etc.
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 3.0, 5.0),
)
# 查询错误计数
clickhouse_query_errors_total = Counter(
"data_ana_clickhouse_query_errors_total",
"Total ClickHouse query errors",
["operation", "error_type"],
)
# ClickHouse 连接状态1=connected, 0=disconnected
clickhouse_connection_status = Gauge(
"data_ana_clickhouse_connection_status",
"ClickHouse connection status (1=connected, 0=disconnected)",
)
# 慢查询计数(超过阈值 1s
clickhouse_slow_queries_total = Counter(
"data_ana_clickhouse_slow_queries_total",
"Total ClickHouse slow queries (>1s)",
["operation"],
)
# ===== ExamCache 指标 =====
# 缓存大小
exam_cache_size = Gauge(
"data_ana_exam_cache_size",
"ExamCache current size (memory LRU)",
)
# 缓存命中/未命中
exam_cache_hits_total = Counter(
"data_ana_exam_cache_hits_total",
"ExamCache cache hits",
)
exam_cache_misses_total = Counter(
"data_ana_exam_cache_misses_total",
"ExamCache cache misses",
)
# Redis 连接状态
redis_connection_status = Gauge(
"data_ana_redis_connection_status",
"Redis connection status (1=connected, 0=disconnected)",
)
# ===== DataScope 缓存指标 =====
datascope_cache_hits_total = Counter(
"data_ana_datascope_cache_hits_total",
"DataScope cache hits",
)
datascope_cache_misses_total = Counter(
"data_ana_datascope_cache_misses_total",
"DataScope cache misses",
)
datascope_fallback_total = Counter(
"data_ana_datascope_fallback_total",
"DataScope fallback (iam gRPC unavailable, using role-based fallback)",
)
# ===== gRPC 指标 =====
grpc_requests_total = Counter(
"data_ana_grpc_requests_total",
"Total gRPC requests",
["method", "status"], # status: success / failed / degraded
)
grpc_request_duration_seconds = Histogram(
"data_ana_grpc_request_duration_seconds",
"gRPC request duration",
["method"],
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 3.0, 5.0),
)
# iam gRPC 连接状态
iam_grpc_connection_status = Gauge(
"data_ana_iam_grpc_connection_status",
"iam gRPC connection status (1=connected, 0=disconnected)",
)
# ===== 业务指标 =====
# 掌握度计算次数
mastery_calculations_total = Counter(
"data_ana_mastery_calculations_total",
"Total mastery level calculations",
["method"], # method: weighted_moving_avg / forgetting_curve
)
# 预警触发次数
warnings_triggered_total = Counter(
"data_ana_warnings_triggered_total",
"Total warnings triggered",
["warning_type", "severity"],
)
# HTTP 请求指标FastAPI 自动 instrumentation 已覆盖,这里补充业务层)
http_requests_degraded_total = Counter(
"data_ana_http_requests_degraded_total",
"Total HTTP requests in degraded mode",
["endpoint"],
)
def record_clickhouse_query(operation: str, duration_s: float, success: bool) -> None:
"""记录 ClickHouse 查询指标(供 repository 调用)."""
clickhouse_query_duration_seconds.labels(operation=operation).observe(duration_s)
if not success:
clickhouse_query_errors_total.labels(operation=operation, error_type="query_failed").inc()
elif duration_s > 1.0:
clickhouse_slow_queries_total.labels(operation=operation).inc()
def record_grpc_request(method: str, duration_s: float, status: str) -> None:
"""记录 gRPC 请求指标(供 grpc_server 调用)."""
grpc_requests_total.labels(method=method, status=status).inc()
grpc_request_duration_seconds.labels(method=method).observe(duration_s)
def record_exam_cache_hit() -> None:
"""记录 ExamCache 命中."""
exam_cache_hits_total.inc()
def record_exam_cache_miss() -> None:
"""记录 ExamCache 未命中."""
exam_cache_misses_total.inc()
def record_datascope_cache_hit() -> None:
"""记录 DataScope 缓存命中."""
datascope_cache_hits_total.inc()
def record_datascope_cache_miss() -> None:
"""记录 DataScope 缓存未命中."""
datascope_cache_misses_total.inc()
def record_datascope_fallback() -> None:
"""记录 DataScope 降级兜底."""
datascope_fallback_total.inc()

View File

@@ -5,7 +5,7 @@ P0 整改ReplacingMergeTree 查询必须加 FINAL 或用 argMax 聚合确保
"""
import asyncio
from datetime import datetime
from datetime import UTC, datetime, timedelta
from typing import Any
import structlog
@@ -793,3 +793,213 @@ async def query_student_scores_by_kp(
return None
return [{"score": float(row[0] or 0), "timestamp": row[1], "exam_id": row[2]} for row in rows]
# ===== P6+ v2 扩展查询方法 =====
async def query_assignment_analysis(
class_id: str,
assignment_id: str,
subject_id: str = "",
) -> dict | None:
"""查询作业/考试分析(单次作业/考试维度统计)."""
client = get_client()
if client is None:
return None
where_parts = ["class_id = {cid:String}", "exam_id = {aid:String}"]
params: dict[str, Any] = {"cid": class_id, "aid": assignment_id}
if subject_id:
where_parts.append("subject_id = {sid:String}")
params["sid"] = subject_id
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" avg(score) AS avg_score, "
f" max(score) AS max_score, "
f" min(score) AS min_score, "
f" count() AS total, "
f" countIf(score >= 60) / count() AS pass_rate, "
f" countIf(score >= 90) AS r90, "
f" countIf(score >= 80 AND score < 90) AS r80, "
f" countIf(score >= 70 AND score < 80) AS r70, "
f" countIf(score >= 60 AND score < 70) AS r60, "
f" countIf(score < 60) AS r_below "
f"FROM student_dashboard_view FINAL "
f"WHERE {where_clause}",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning(
"query_assignment_analysis_failed",
error=str(exc),
class_id=class_id,
assignment_id=assignment_id,
)
return None
if not rows:
return None
(
avg_score,
max_score,
min_score,
total,
pass_rate,
r90,
r80,
r70,
r60,
r_below,
) = rows[0]
total_int = int(total or 0)
def _pct(count: Any) -> float:
return (int(count or 0) / total_int) if total_int else 0.0
ranges = [
{"label": "90-100", "count": int(r90 or 0), "percentage": _pct(r90)},
{"label": "80-89", "count": int(r80 or 0), "percentage": _pct(r80)},
{"label": "70-79", "count": int(r70 or 0), "percentage": _pct(r70)},
{"label": "60-69", "count": int(r60 or 0), "percentage": _pct(r60)},
{"label": "<60", "count": int(r_below or 0), "percentage": _pct(r_below)},
]
return {
"assignmentId": assignment_id,
"classId": class_id,
"subjectId": subject_id,
"averageScore": float(avg_score or 0),
"highestScore": float(max_score or 0),
"lowestScore": float(min_score or 0),
"totalStudents": total_int,
"submittedCount": total_int,
"passRate": float(pass_rate or 0),
"ranges": ranges,
}
async def query_error_book_stats(student_id: str, subject_id: str = "") -> dict | None:
"""查询错题本统计(按知识点聚合)."""
client = get_client()
if client is None:
return None
where_parts = ["student_id = {sid:String}"]
params: dict[str, Any] = {"sid": student_id}
# subject_id 过滤需要 join 知识点表,简化为按 knowledge_point_id 前缀匹配
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" knowledge_point_id, "
f" sum(error_count) AS total_errors, "
f" count(DISTINCT question_id) AS question_count, "
f" max(last_error_time) AS last_error "
f"FROM student_errors FINAL "
f"WHERE {where_clause} "
f"GROUP BY knowledge_point_id "
f"ORDER BY total_errors DESC "
f"LIMIT 50",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_error_book_stats_failed", error=str(exc), student_id=student_id)
return None
by_kp = []
total_errors = 0
total_questions = 0
for row in rows:
kp_id, err_count, q_count, _last = row
err_count_int = int(err_count or 0)
q_count_int = int(q_count or 0)
total_errors += err_count_int
total_questions += q_count_int
by_kp.append(
{
"knowledge_point_id": kp_id,
"title": kp_id, # content CDC 同步后补充
"error_count": err_count_int,
"question_count": q_count_int,
"error_rate": (err_count_int / q_count_int) if q_count_int else 0.0,
}
)
# 查询最近 7 天错误次数
try:
seven_days_ago = datetime.now(UTC) - timedelta(days=7)
result = await asyncio.to_thread(
client.query,
"SELECT sum(error_count) "
"FROM student_errors FINAL "
"WHERE student_id = {sid:String} AND last_error_time >= {sd:DateTime64(3)}",
parameters={"sid": student_id, "sd": seven_days_ago},
)
recent_rows = result.result_rows
recent_7d = int(recent_rows[0][0] or 0) if recent_rows else 0
except Exception: # noqa: BLE001
recent_7d = 0
return {
"studentId": student_id,
"totalErrorQuestions": len(by_kp),
"totalErrorCount": total_errors,
"byKnowledgePoint": by_kp,
"recent7dErrors": recent_7d,
}
async def query_mastery_trend(
student_id: str,
start_date: int = 0,
end_date: int = 0,
) -> list[dict] | None:
"""查询掌握度变化趋势(按时间排序的快照)."""
client = get_client()
if client is None:
return None
where_parts = ["student_id = {sid:String}"]
params: dict[str, Any] = {"sid": student_id}
if start_date:
where_parts.append("calculated_at >= {sd:DateTime64(3)}")
params["sd"] = datetime.fromtimestamp(start_date)
if end_date:
where_parts.append("calculated_at <= {ed:DateTime64(3)}")
params["ed"] = datetime.fromtimestamp(end_date)
where_clause = " AND ".join(where_parts)
try:
result = await asyncio.to_thread(
client.query,
f"SELECT "
f" calculated_at, "
f" avg(mastery_level) AS overall_mastery "
f"FROM mastery_snapshot "
f"WHERE {where_clause} "
f"GROUP BY calculated_at "
f"ORDER BY calculated_at ASC "
f"LIMIT 100",
parameters=params,
)
rows = result.result_rows
except Exception as exc: # noqa: BLE001
logger.warning("query_mastery_trend_failed", error=str(exc), student_id=student_id)
return None
return [
{
"calculated_at": int(row[0].timestamp()) if row[0] else 0,
"overall_mastery": float(row[1] or 0),
}
for row in rows
]

File diff suppressed because one or more lines are too long

View File

@@ -111,6 +111,42 @@ class AnalyticsServiceStub:
response_deserializer=analytics__pb2.MasteryUpdateEvent.FromString,
_registered_method=True,
)
self.GetStudentGrowth = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/GetStudentGrowth",
request_serializer=analytics__pb2.GetStudentGrowthRequest.SerializeToString,
response_deserializer=analytics__pb2.StudentGrowth.FromString,
_registered_method=True,
)
self.GetAssignmentAnalysis = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/GetAssignmentAnalysis",
request_serializer=analytics__pb2.GetAssignmentAnalysisRequest.SerializeToString,
response_deserializer=analytics__pb2.AssignmentAnalysis.FromString,
_registered_method=True,
)
self.GetMasterySummary = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/GetMasterySummary",
request_serializer=analytics__pb2.GetMasterySummaryRequest.SerializeToString,
response_deserializer=analytics__pb2.MasterySummary.FromString,
_registered_method=True,
)
self.ListDiagnosticReports = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/ListDiagnosticReports",
request_serializer=analytics__pb2.ListDiagnosticReportsRequest.SerializeToString,
response_deserializer=analytics__pb2.DiagnosticReportList.FromString,
_registered_method=True,
)
self.ListErrorBookItems = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/ListErrorBookItems",
request_serializer=analytics__pb2.ListErrorBookItemsRequest.SerializeToString,
response_deserializer=analytics__pb2.ErrorBookList.FromString,
_registered_method=True,
)
self.GetErrorBookStats = channel.unary_unary(
"/next_edu_cloud.analytics.v1.AnalyticsService/GetErrorBookStats",
request_serializer=analytics__pb2.GetErrorBookStatsRequest.SerializeToString,
response_deserializer=analytics__pb2.ErrorBookStats.FromString,
_registered_method=True,
)
class AnalyticsServiceServicer:
@@ -191,6 +227,44 @@ class AnalyticsServiceServicer:
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def GetStudentGrowth(self, request, context):
"""===== P6+ v2 扩展 RPC响应上游 parent-bff / student-bff v2 请求) =====
学生成长档案(综合成绩趋势 + 掌握度变化 + 考勤统计,供 parent-bff GrowthArchiveService.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def GetAssignmentAnalysis(self, request, context):
"""作业/考试分析(单次作业/考试维度统计,供 student-bff / parent-bff."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def GetMasterySummary(self, request, context):
"""学生掌握度汇总(轻量级,仅返回总体掌握度 + 三档分布,供 parent-bff MasteryService.GetSummary."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def ListDiagnosticReports(self, request, context):
"""诊断报告列表(占位实现,真实数据需 ai 服务集成,供 parent-bff / student-bff."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def ListErrorBookItems(self, request, context):
"""错题本列表gRPC 版本,供 parent-bff ErrorBookService.List."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def GetErrorBookStats(self, request, context):
"""错题本统计(按知识点聚合,供 parent-bff ErrorBookService.GetStats."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details("Method not implemented!")
raise NotImplementedError("Method not implemented!")
def add_AnalyticsServiceServicer_to_server(servicer, server):
rpc_method_handlers = {
@@ -254,6 +328,36 @@ def add_AnalyticsServiceServicer_to_server(servicer, server):
request_deserializer=analytics__pb2.SubscribeMasteryUpdateRequest.FromString,
response_serializer=analytics__pb2.MasteryUpdateEvent.SerializeToString,
),
"GetStudentGrowth": grpc.unary_unary_rpc_method_handler(
servicer.GetStudentGrowth,
request_deserializer=analytics__pb2.GetStudentGrowthRequest.FromString,
response_serializer=analytics__pb2.StudentGrowth.SerializeToString,
),
"GetAssignmentAnalysis": grpc.unary_unary_rpc_method_handler(
servicer.GetAssignmentAnalysis,
request_deserializer=analytics__pb2.GetAssignmentAnalysisRequest.FromString,
response_serializer=analytics__pb2.AssignmentAnalysis.SerializeToString,
),
"GetMasterySummary": grpc.unary_unary_rpc_method_handler(
servicer.GetMasterySummary,
request_deserializer=analytics__pb2.GetMasterySummaryRequest.FromString,
response_serializer=analytics__pb2.MasterySummary.SerializeToString,
),
"ListDiagnosticReports": grpc.unary_unary_rpc_method_handler(
servicer.ListDiagnosticReports,
request_deserializer=analytics__pb2.ListDiagnosticReportsRequest.FromString,
response_serializer=analytics__pb2.DiagnosticReportList.SerializeToString,
),
"ListErrorBookItems": grpc.unary_unary_rpc_method_handler(
servicer.ListErrorBookItems,
request_deserializer=analytics__pb2.ListErrorBookItemsRequest.FromString,
response_serializer=analytics__pb2.ErrorBookList.SerializeToString,
),
"GetErrorBookStats": grpc.unary_unary_rpc_method_handler(
servicer.GetErrorBookStats,
request_deserializer=analytics__pb2.GetErrorBookStatsRequest.FromString,
response_serializer=analytics__pb2.ErrorBookStats.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
"next_edu_cloud.analytics.v1.AnalyticsService", rpc_method_handlers
@@ -630,3 +734,183 @@ class AnalyticsService:
metadata,
_registered_method=True,
)
@staticmethod
def GetStudentGrowth(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/GetStudentGrowth",
analytics__pb2.GetStudentGrowthRequest.SerializeToString,
analytics__pb2.StudentGrowth.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)
@staticmethod
def GetAssignmentAnalysis(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/GetAssignmentAnalysis",
analytics__pb2.GetAssignmentAnalysisRequest.SerializeToString,
analytics__pb2.AssignmentAnalysis.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)
@staticmethod
def GetMasterySummary(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/GetMasterySummary",
analytics__pb2.GetMasterySummaryRequest.SerializeToString,
analytics__pb2.MasterySummary.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)
@staticmethod
def ListDiagnosticReports(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/ListDiagnosticReports",
analytics__pb2.ListDiagnosticReportsRequest.SerializeToString,
analytics__pb2.DiagnosticReportList.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)
@staticmethod
def ListErrorBookItems(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/ListErrorBookItems",
analytics__pb2.ListErrorBookItemsRequest.SerializeToString,
analytics__pb2.ErrorBookList.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)
@staticmethod
def GetErrorBookStats(
request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None,
):
return grpc.experimental.unary_unary(
request,
target,
"/next_edu_cloud.analytics.v1.AnalyticsService/GetErrorBookStats",
analytics__pb2.GetErrorBookStatsRequest.SerializeToString,
analytics__pb2.ErrorBookStats.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True,
)