feat(data-ana): v2 P6 硬化完成 + 6 新 RPC + Prometheus 监控
P6 硬化(5 项全部完成):
- CDC 多实例水平扩展: _INSTANCE_ID + get_lag() 真实 lag 计算
- ExamCache Redis 化: key data_ana:exam:{exam_id}, TTL 30 天 + 内存 LRU fallback
- ClickHouse TTL 归档: 5 表均加 TTL(1-3 年),分区级删除
- Prometheus 监控: 18 个指标(CDC/CH/ExamCache/DataScope/gRPC/业务)
- readyz 深度硬化: 4 依赖超时检查(CH 1s/Redis 200ms/iam 2s/CDC lag<1000)
v2 新增 6 个 RPC(analytics.proto 扩展为 18 RPC):
- GetStudentGrowth / GetAssignmentAnalysis / GetMasterySummary
- ListDiagnosticReports(占位,待 ai 服务)/ ListErrorBookItems / GetErrorBookStats
监控与可观测性: lifespan 预热 + gRPC ServerInterceptor + CDC 消费者指标
Docker 本地测试 19 项全部通过(healthz/readyz/metrics + 11 HTTP + 10 gRPC + ruff)
nextstep-v2.md: 上游需求对齐 + 下游要求(iam/core-edu/content/ai/SRE)
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
-- data-ana ClickHouse DDL:5 宽表建表脚本
|
||||
-- 对齐 02-architecture-design.md §3 DDL 设计
|
||||
-- data-ana ClickHouse DDL:5 宽表建表脚本(P6: 加 TTL 归档策略)
|
||||
-- 对齐 02-architecture-design.md §3 DDL 设计 + workline §3.5 任务 6.3
|
||||
-- 引擎:ReplacingMergeTree(幂等消费保证)+ MergeTree(历史快照)
|
||||
-- TTL:P6 容量规划(冷热数据分离,过期自动清理)
|
||||
-- 使用方式:clickhouse-client --multiquery < scripts/clickhouse_ddl.sql
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- §3.1 学生学情宽表
|
||||
-- §3.1 学生学情宽表(TTL 2 年)
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
(
|
||||
student_id String,
|
||||
@@ -22,9 +23,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
TTL last_updated + INTERVAL 2 YEAR
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- §3.2 学生错题本
|
||||
-- §3.2 学生错题本(TTL 2 年)
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
(
|
||||
student_id String,
|
||||
@@ -36,9 +38,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
ORDER BY (student_id, question_id)
|
||||
TTL last_error_time + INTERVAL 2 YEAR;
|
||||
|
||||
-- §3.3 知识点掌握度历史快照
|
||||
-- §3.3 知识点掌握度历史快照(TTL 3 年,长期保留用于趋势分析)
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
(
|
||||
student_id String,
|
||||
@@ -50,9 +53,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at)
|
||||
TTL calculated_at + INTERVAL 3 YEAR;
|
||||
|
||||
-- §3.4 AI 用量计费记录
|
||||
-- §3.4 AI 用量计费记录(TTL 1 年,计费数据保留期较短)
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
(
|
||||
request_id String,
|
||||
@@ -69,9 +73,10 @@ CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id);
|
||||
ORDER BY (request_id)
|
||||
TTL occurred_at + INTERVAL 1 YEAR;
|
||||
|
||||
-- §3.5 学生考勤记录
|
||||
-- §3.5 学生考勤记录(TTL 3 年,考勤数据需长期保留用于趋势分析)
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
(
|
||||
student_id String,
|
||||
@@ -84,4 +89,21 @@ CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
ORDER BY (student_id, class_id, attendance_date)
|
||||
TTL occurred_at + INTERVAL 3 YEAR;
|
||||
|
||||
-- ===== P6 容量规划说明 =====
|
||||
-- 1. student_dashboard_view / student_errors:2 年 TTL
|
||||
-- - 高频写入(CDC 实时同步),2 年覆盖完整学习周期
|
||||
-- - 超过 2 年的数据通过 PARTITION 级别删除(无需 VACUUM)
|
||||
-- 2. mastery_snapshot / attendance_logs:3 年 TTL
|
||||
-- - 低频写入但需长期保留用于趋势分析
|
||||
-- - 3 年覆盖 K12 完整学段
|
||||
-- 3. ai_usage_log:1 年 TTL
|
||||
-- - 计费数据保留期较短,1 年足够用于年度成本分析
|
||||
-- 4. 冷热数据分离:
|
||||
-- - 热数据:最近 3 个月(SSD 存储,频繁查询)
|
||||
-- - 冷数据:3 个月以上(HDD 存储,低频查询)
|
||||
-- - 通过 PARTITION BY toYYYYMM 实现按月分区,便于冷热分离
|
||||
-- 5. TTL 触发时机:ClickHouse 后台 merge 时自动清理过期数据
|
||||
-- - 可通过 system.ttl_drops 表监控 TTL 执行情况
|
||||
|
||||
Reference in New Issue
Block a user