feat(data-ana): 完整实现 data-ana 数据分析服务

包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等
This commit is contained in:
SpecialX
2026-07-10 19:09:27 +08:00
parent 033057a302
commit ca3780aa24
29 changed files with 5401 additions and 383 deletions

View File

@@ -0,0 +1,97 @@
-- data-ana ClickHouse DDL5 宽表).
-- 负责人ai11总裁裁决 §2.12 授权 ai11 创建纯 DDL 文件SRE AI 负责部署脚本).
-- 对齐文档services/data-ana/docs/02-architecture-design.md §3.
-- 引擎ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本(幂等消费保证).
-- 查询时必须加 FINAL 或使用 argMax 聚合确保去重生效P0 整改项).
--
-- 使用方式:
-- clickhouse-client --multiquery < infra/clickhouse/ddl/data_ana.sql
-- 数据库
CREATE DATABASE IF NOT EXISTS edu_analytics;
-- ===== 1. 学生学情宽表 =====
-- 每次成绩写入产生一行,按 ORDER BY 去重保留 last_updated 最大版本.
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
(
student_id String,
class_id String,
exam_id String,
subject_id String,
score Float64,
rank_in_class UInt32,
knowledge_point_id String,
mastery_level Float32, -- 0.0-1.0
error_count UInt32,
last_updated DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(last_updated)
PARTITION BY toYYYYMM(last_updated)
ORDER BY (student_id, exam_id, knowledge_point_id)
SETTINGS index_granularity = 8192;
-- ===== 2. 学生错题本 =====
-- 同一 (student_id, question_id) 累计 error_count按 last_error_time 去重.
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
(
student_id String,
question_id String,
knowledge_point_id String,
error_count UInt32,
last_error_time DateTime64(3, 'UTC'),
content String DEFAULT ''
)
ENGINE = ReplacingMergeTree(last_error_time)
PARTITION BY toYYYYMM(last_error_time)
ORDER BY (student_id, question_id);
-- ===== 3. 知识点掌握度历史快照 =====
-- 每次掌握度计算产生新版本,支持趋势查询.
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
(
student_id String,
knowledge_point_id String,
subject_id String,
mastery_level Float32, -- 0.0-1.0
calculated_at DateTime64(3, 'UTC'),
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(calculated_at)
ORDER BY (student_id, knowledge_point_id, calculated_at);
-- ===== 4. 学生考勤记录 =====
-- core-edu attendance 表 CDC 同步,同一记录多版本去重.
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
(
student_id String,
class_id String,
attendance_date Date,
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
recorded_by String, -- 教师用户 ID
remark String DEFAULT '',
occurred_at DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(attendance_date)
ORDER BY (student_id, class_id, attendance_date);
-- ===== 5. AI 用量计费记录 =====
-- ai 服务通过 Kafka 事件投递data-ana 消费落库,按 request_id 幂等去重.
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
(
request_id String,
user_id String,
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
model LowCardinality(String),
prompt_tokens UInt32,
completion_tokens UInt32,
total_tokens UInt32,
latency_ms UInt32,
success Boolean,
cost_cents UInt32, -- 计费(分),便于聚合
occurred_at DateTime64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(occurred_at)
PARTITION BY toYYYYMM(occurred_at)
ORDER BY (request_id);