feat(data-ana): 完整实现 data-ana 数据分析服务
包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等
This commit is contained in:
97
infra/clickhouse/ddl/data_ana.sql
Normal file
97
infra/clickhouse/ddl/data_ana.sql
Normal file
@@ -0,0 +1,97 @@
|
||||
-- data-ana ClickHouse DDL(5 宽表).
|
||||
-- 负责人:ai11(总裁裁决 §2.12 授权 ai11 创建纯 DDL 文件,SRE AI 负责部署脚本).
|
||||
-- 对齐文档:services/data-ana/docs/02-architecture-design.md §3.
|
||||
-- 引擎:ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本(幂等消费保证).
|
||||
-- 查询时必须加 FINAL 或使用 argMax 聚合确保去重生效(P0 整改项).
|
||||
--
|
||||
-- 使用方式:
|
||||
-- clickhouse-client --multiquery < infra/clickhouse/ddl/data_ana.sql
|
||||
|
||||
-- 数据库
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- ===== 1. 学生学情宽表 =====
|
||||
-- 每次成绩写入产生一行,按 ORDER BY 去重保留 last_updated 最大版本.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view
|
||||
(
|
||||
student_id String,
|
||||
class_id String,
|
||||
exam_id String,
|
||||
subject_id String,
|
||||
score Float64,
|
||||
rank_in_class UInt32,
|
||||
knowledge_point_id String,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
error_count UInt32,
|
||||
last_updated DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- ===== 2. 学生错题本 =====
|
||||
-- 同一 (student_id, question_id) 累计 error_count,按 last_error_time 去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors
|
||||
(
|
||||
student_id String,
|
||||
question_id String,
|
||||
knowledge_point_id String,
|
||||
error_count UInt32,
|
||||
last_error_time DateTime64(3, 'UTC'),
|
||||
content String DEFAULT ''
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
|
||||
-- ===== 3. 知识点掌握度历史快照 =====
|
||||
-- 每次掌握度计算产生新版本,支持趋势查询.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot
|
||||
(
|
||||
student_id String,
|
||||
knowledge_point_id String,
|
||||
subject_id String,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
calculated_at DateTime64(3, 'UTC'),
|
||||
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
|
||||
-- ===== 4. 学生考勤记录 =====
|
||||
-- core-edu attendance 表 CDC 同步,同一记录多版本去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs
|
||||
(
|
||||
student_id String,
|
||||
class_id String,
|
||||
attendance_date Date,
|
||||
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
|
||||
recorded_by String, -- 教师用户 ID
|
||||
remark String DEFAULT '',
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
|
||||
-- ===== 5. AI 用量计费记录 =====
|
||||
-- ai 服务通过 Kafka 事件投递,data-ana 消费落库,按 request_id 幂等去重.
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log
|
||||
(
|
||||
request_id String,
|
||||
user_id String,
|
||||
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
|
||||
model LowCardinality(String),
|
||||
prompt_tokens UInt32,
|
||||
completion_tokens UInt32,
|
||||
total_tokens UInt32,
|
||||
latency_ms UInt32,
|
||||
success Boolean,
|
||||
cost_cents UInt32, -- 计费(分),便于聚合
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id);
|
||||
Reference in New Issue
Block a user