chore(data-ana): merge data-ana full implementation into main
Merge feat/data-ana-ai11 with complete data analytics service
This commit is contained in:
SpecialX
@@ -1,37 +1,89 @@
|
||||
-- ClickHouse 数据库初始化脚本
|
||||
-- 适用服务：data-ana（数据分析）
|
||||
-- 表结构：student_dashboard_view（学生学情宽表）/ student_errors（错题本）
|
||||
-- 与 services/data-ana/src/data_ana/clickhouse_client.py 中的查询字段对齐
|
||||
-- 适用服务：data-ana（D6 智能洞察领域）
|
||||
-- 5 张宽表（ai-allocation §5 强制）：
|
||||
-- 1. student_dashboard_view 学生学情宽表（成绩/班级/知识点维度）
|
||||
-- 2. student_errors 学生错题本
|
||||
-- 3. mastery_snapshot 知识点掌握度历史快照
|
||||
-- 4. ai_usage_log AI 用量计费记录（ai 服务投递，data-ana 消费）
|
||||
-- 5. attendance_logs 学生考勤记录（core-edu attendance 表 CDC 同步）
|
||||
--
|
||||
-- 引擎对齐 02-architecture-design.md §3：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本
|
||||
-- 查询规范：所有 SELECT 必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改）
|
||||
--
|
||||
-- 使用方式（启用 ClickHouse 时执行一次）：
|
||||
-- clickhouse-client --multiquery < scripts/clickhouse-init.sql
|
||||
-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式。
|
||||
-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式（返回骨架数据）。
|
||||
|
||||
-- 数据库
|
||||
CREATE DATABASE IF NOT EXISTS edu_analytics;
|
||||
|
||||
-- 学生学情宽表（考试/班级/知识点维度）
|
||||
-- ===== 1. 学生学情宽表（成绩写入产生一行，按 ORDER BY 去重保留最新版本） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view (
|
||||
student_id String,
|
||||
class_id String,
|
||||
exam_id String,
|
||||
subject_id String,
|
||||
score Float64,
|
||||
rank_in_class UInt32,
|
||||
knowledge_point_id String,
|
||||
mastery_level Float32,
|
||||
error_count UInt32,
|
||||
last_updated DateTime
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY (student_id, class_id, exam_id);
|
||||
student_id String,
|
||||
class_id String,
|
||||
exam_id String,
|
||||
subject_id String,
|
||||
score Float64,
|
||||
rank_in_class UInt32,
|
||||
knowledge_point_id String,
|
||||
mastery_level Float32, -- 0.0-1.0
|
||||
error_count UInt32,
|
||||
last_updated DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(last_updated)
|
||||
PARTITION BY toYYYYMM(last_updated)
|
||||
ORDER BY (student_id, exam_id, knowledge_point_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 学生错题表（错题本）
|
||||
-- ===== 2. 学生错题本 =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.student_errors (
|
||||
student_id String,
|
||||
question_id String,
|
||||
knowledge_point_id String,
|
||||
error_count UInt32,
|
||||
last_error_time DateTime,
|
||||
content String
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY (student_id, knowledge_point_id);
|
||||
student_id String,
|
||||
question_id String,
|
||||
knowledge_point_id String,
|
||||
error_count UInt32,
|
||||
last_error_time DateTime64(3, 'UTC'),
|
||||
content String
|
||||
) ENGINE = ReplacingMergeTree(last_error_time)
|
||||
PARTITION BY toYYYYMM(last_error_time)
|
||||
ORDER BY (student_id, question_id);
|
||||
|
||||
-- ===== 3. 掌握度历史快照（每次掌握度计算产生新版本，支持趋势查询） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot (
|
||||
student_id String,
|
||||
knowledge_point_id String,
|
||||
subject_id String,
|
||||
mastery_level Float32,
|
||||
calculated_at DateTime64(3, 'UTC'),
|
||||
calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve'
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(calculated_at)
|
||||
ORDER BY (student_id, knowledge_point_id, calculated_at);
|
||||
|
||||
-- ===== 4. AI 用量计费记录（ai 服务通过 Kafka 事件投递，data-ana 消费落库） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log (
|
||||
request_id String,
|
||||
user_id String,
|
||||
provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local'
|
||||
model LowCardinality(String),
|
||||
prompt_tokens UInt32,
|
||||
completion_tokens UInt32,
|
||||
total_tokens UInt32,
|
||||
latency_ms UInt32,
|
||||
success Boolean,
|
||||
cost_cents UInt32, -- 计费（分），便于聚合
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(occurred_at)
|
||||
ORDER BY (request_id); -- 按 request_id 幂等去重
|
||||
|
||||
-- ===== 5. 学生考勤记录（core-edu attendance 表 CDC 同步） =====
|
||||
CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs (
|
||||
student_id String,
|
||||
class_id String,
|
||||
attendance_date Date,
|
||||
status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave'
|
||||
recorded_by String, -- 教师用户 ID
|
||||
remark String DEFAULT '',
|
||||
occurred_at DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(occurred_at)
|
||||
PARTITION BY toYYYYMM(attendance_date)
|
||||
ORDER BY (student_id, class_id, attendance_date);
|
||||
|
||||
131
scripts/data-ana-mock-data.sql
Normal file
131
scripts/data-ana-mock-data.sql
Normal file
@@ -0,0 +1,131 @@
|
||||
-- data-ana 模块 Mock 数据脚本
|
||||
-- 用途：本地开发/集成测试时为 5 张宽表插入演示数据，验证查询逻辑
|
||||
-- 使用方式（需先执行 clickhouse-init.sql）：
|
||||
-- clickhouse-client --multiquery < scripts/data-ana-mock-data.sql
|
||||
--
|
||||
-- 数据规模：2 班 × 10 学生 × 3 知识点 × 2 考试 = 120 行 student_dashboard_view
|
||||
-- 20 条错题 / 60 条掌握度快照 / 20 条考勤 / 10 条 AI 用量
|
||||
-- 所有时间戳使用 now() - N days 模式，避免硬编码绝对日期导致脚本过期
|
||||
|
||||
USE edu_analytics;
|
||||
|
||||
-- ===== 1. student_dashboard_view：学情宽表 =====
|
||||
-- 班级 C001（高一 1 班）+ C002（高一 2 班），每班 10 学生
|
||||
-- 2 次考试（EXAM001 数学月考 / EXAM002 数学期中），3 个数学知识点
|
||||
INSERT INTO student_dashboard_view
|
||||
SELECT
|
||||
'S' || toString(number + 1) AS student_id, -- S001..S020
|
||||
IF(number < 10, 'C001', 'C002') AS class_id, -- 前 10 在 C001，后 10 在 C002
|
||||
'EXAM001' AS exam_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
40 + (number % 60) AS score, -- 40-99 分
|
||||
(number % 10) + 1 AS rank_in_class, -- 1-10 名
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, -- KP_MATH_1/2/3
|
||||
0.3 + ((number % 70) / 100.0) AS mastery_level, -- 0.30-0.99
|
||||
(number % 5) AS error_count, -- 0-4 错题
|
||||
now() - INTERVAL 7 DAY AS last_updated
|
||||
FROM numbers(20);
|
||||
|
||||
-- 第二次考试（EXAM002），成绩略升
|
||||
INSERT INTO student_dashboard_view
|
||||
SELECT
|
||||
'S' || toString(number + 1) AS student_id,
|
||||
IF(number < 10, 'C001', 'C002') AS class_id,
|
||||
'EXAM002' AS exam_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
50 + (number % 50) AS score, -- 50-99 分
|
||||
(number % 10) + 1 AS rank_in_class,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
0.4 + ((number % 60) / 100.0) AS mastery_level,
|
||||
(number % 4) AS error_count,
|
||||
now() - INTERVAL 1 DAY AS last_updated
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 2. student_errors：错题本 =====
|
||||
INSERT INTO student_errors
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'Q_' || toString(number + 1) AS question_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
(number % 5) + 1 AS error_count,
|
||||
now() - INTERVAL (number % 30) DAY AS last_error_time,
|
||||
'错误类型：' || multiIf(
|
||||
number % 3 = 0, '计算失误',
|
||||
number % 3 = 1, '公式记忆错误',
|
||||
'审题不清'
|
||||
) AS content
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 3. mastery_snapshot：掌握度历史快照（3 次计算，便于趋势查询） =====
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.3 + ((number % 70) / 100.0) AS mastery_level,
|
||||
now() - INTERVAL 30 DAY AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.4 + ((number % 60) / 100.0) AS mastery_level,
|
||||
now() - INTERVAL 15 DAY AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
INSERT INTO mastery_snapshot
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id,
|
||||
'SUBJ_MATH' AS subject_id,
|
||||
0.5 + ((number % 50) / 100.0) AS mastery_level,
|
||||
now() AS calculated_at,
|
||||
'weighted_moving_avg' AS calculation_method
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 4. attendance_logs：考勤记录（近 5 天，含缺勤/迟到） =====
|
||||
INSERT INTO attendance_logs
|
||||
SELECT
|
||||
'S' || toString((number % 20) + 1) AS student_id,
|
||||
IF(number % 20 < 10, 'C001', 'C002') AS class_id,
|
||||
today() - (number % 5) AS attendance_date,
|
||||
multiIf(
|
||||
number % 10 = 0, 'absent', -- 10% 缺勤
|
||||
number % 5 = 0, 'late', -- 20% 迟到
|
||||
'present' -- 其余正常
|
||||
) AS status,
|
||||
'T001' AS recorded_by,
|
||||
multiIf(number % 10 = 0, '病假', '') AS remark,
|
||||
now() - INTERVAL (number % 5) DAY AS occurred_at
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 5. ai_usage_log：AI 用量计费（模拟 3 个 provider） =====
|
||||
INSERT INTO ai_usage_log
|
||||
SELECT
|
||||
'REQ_' || toString(number + 1) AS request_id,
|
||||
'S' || toString((number % 20) + 1) AS user_id,
|
||||
arrayElement(['openai', 'anthropic', 'baichuan'], (number % 3) + 1) AS provider,
|
||||
arrayElement(['gpt-4o-mini', 'claude-3-haiku', 'baichuan2-turbo'], (number % 3) + 1) AS model,
|
||||
100 + (number % 200) AS prompt_tokens, -- 100-299
|
||||
50 + (number % 150) AS completion_tokens, -- 50-199
|
||||
150 + (number % 350) AS total_tokens, -- 150-499
|
||||
500 + (number % 2000) AS latency_ms, -- 500-2499 ms
|
||||
number % 5 != 0 AS success, -- 80% 成功
|
||||
(number % 50) + 1 AS cost_cents, -- 1-50 分
|
||||
now() - INTERVAL (number % 24) HOUR AS occurred_at
|
||||
FROM numbers(20);
|
||||
|
||||
-- ===== 校验查询（可选执行） =====
|
||||
SELECT 'student_dashboard_view' AS table_name, count() AS cnt FROM student_dashboard_view FINAL
|
||||
UNION ALL
|
||||
SELECT 'student_errors', count() FROM student_errors FINAL
|
||||
UNION ALL
|
||||
SELECT 'mastery_snapshot', count() FROM mastery_snapshot
|
||||
UNION ALL
|
||||
SELECT 'attendance_logs', count() FROM attendance_logs FINAL
|
||||
UNION ALL
|
||||
SELECT 'ai_usage_log', count() FROM ai_usage_log FINAL;
|
||||
Reference in New Issue
Block a user